aboutsummaryrefslogtreecommitdiff
path: root/ci
diff options
context:
space:
mode:
Diffstat (limited to 'ci')
-rw-r--r--ci/build.env88
-rw-r--r--ci/compile.ps123
-rw-r--r--ci/config-templates/CMNext-template.json41
-rw-r--r--ci/config-templates/Essentials.Accounts-template.json (renamed from ci/config/Essentials.Accounts.json)15
-rw-r--r--ci/config-templates/PageRouter-template.json7
-rw-r--r--ci/config-templates/SessionProvider-template.json (renamed from ci/config/SessionProvider.json)10
-rw-r--r--ci/config-templates/config-template.json169
-rw-r--r--ci/config-templates/routes.xml (renamed from ci/config/routes.xml)14
-rw-r--r--ci/config/CMNext.json44
-rw-r--r--ci/config/PageRouter.json6
-rw-r--r--ci/config/config.json183
-rw-r--r--ci/container/Dockerfile109
-rw-r--r--ci/container/Taskfile.yaml86
-rw-r--r--ci/container/docker-compose.yaml73
-rw-r--r--ci/container/run.sh25
-rw-r--r--ci/install.ps115
-rw-r--r--ci/install.taskfile.yaml22
-rw-r--r--ci/plugins.taskfile.yaml162
-rw-r--r--ci/release.taskfile.yaml132
-rw-r--r--ci/taskfile.yaml225
20 files changed, 1073 insertions, 376 deletions
diff --git a/ci/build.env b/ci/build.env
new file mode 100644
index 0000000..d0739af
--- /dev/null
+++ b/ci/build.env
@@ -0,0 +1,88 @@
+#
+# Copyright (C) Vaughn Nugent
+#
+# This file contains static variables that are substituded at build time
+# to the configuration templates for bare-metal packages as a set of default
+# variables. This is so I can use the same template files for container builds
+# and bare-metal
+#
+
+##########
+# _internal_
+##########
+
+#default compression lib for bare metal
+COMPRESSION_LIB_PATH=lib/vnlib_compress.dll
+
+##########
+# Plugins
+##########
+
+DEBUG_PLUGINS=false
+CHANNEL_INDEX_FILE=blogs/channels.json
+MAX_LOGIN_ATTEMPS=10
+
+##########
+# Storage
+##########
+
+STORAGE_CUSTOM_ASSEMBLY=
+STORAGE_TYPE=
+STORAGE_SERVER_ADDRESS=
+STORAGE_USERNAME=
+STORAGE_BUCKET=
+STORAGE_USE_SSL=true
+STORAGE_PASSWORD=
+S3_REGION=
+
+##########
+# HTTP
+##########
+
+#sets the absolute maxium upload size
+MAX_CONTENT_LENGTH=204800000
+
+HTTP_TRACE_ON=false
+HTTP_DOWNSTREAM_SERVERS=[]
+
+#default ssl dir is ssl/
+SSL_JSON={"cert": "ssl/cert.pem", "privkey":"ssl/key.pem"}
+
+#############
+# HC VAULT
+#############
+
+HC_VAULT_ADDR=
+HC_VAULT_TOKEN=
+HC_VAULT_TRUST_CERT=false
+
+##########
+# SQL
+##########
+
+#default to sqlite as that is whats packaged by default, also use the data/ directory
+SQL_LIB_PATH=VNLib.Plugins.Extensions.Sql.SQLite.dll
+SQL_CONNECTION_STRING=Data Source=data/cmnext.db;
+
+##############
+# CACHE
+##############
+
+CACHE_ASM_PATH=VNLib.Data.Caching.Providers.VNCache.dll
+REDIS_CONNECTION_STRING=
+VNCACHE_INITIAL_NODES=[]
+
+#for self contained apps, memcache needs to be true
+MEMCACHE_ONLY=true
+
+############
+# SECRETS
+############
+
+PASSWORD_PEPPER=file://secrets/password-pepper.txt
+DATABASE_PASSWORD=
+REDIS_PASSWORD=
+
+#no cache, so clear vars
+VNCACHE_CLIENT_PRIVATE_KEY=
+VNCACHE_CACHE_PUBLIC_KEY= \ No newline at end of file
diff --git a/ci/compile.ps1 b/ci/compile.ps1
new file mode 100644
index 0000000..2858d06
--- /dev/null
+++ b/ci/compile.ps1
@@ -0,0 +1,23 @@
+param([String] $inputDir, [String] $outputDir)
+
+$templateFiles = Get-ChildItem -Path $inputDir -Filter "*-template.json" -File
+
+foreach ($file in $templateFiles) {
+ $baseFilename = $file.BaseName + '.json'
+ $templateFilePath = $file.FullName
+
+ #remove the -template.json suffix
+ $outputFilePath = Join-Path -Path $outputDir -ChildPath $baseFilename.replace("-template","")
+
+ #substitute environment variables for file variables
+ Get-Content $templateFilePath | ForEach-Object {
+ if ($_ -match "\$\{((\w+))\}")
+ {
+ $_ -replace "\$\{(\w+)\}",$([Environment]::GetEnvironmentVariable($Matches[1]))
+ }
+ else
+ {
+ $_
+ }
+ } | Set-Content $outputFilePath
+} \ No newline at end of file
diff --git a/ci/config-templates/CMNext-template.json b/ci/config-templates/CMNext-template.json
new file mode 100644
index 0000000..486f68a
--- /dev/null
+++ b/ci/config-templates/CMNext-template.json
@@ -0,0 +1,41 @@
+{
+ //Enables debug logging
+ "debug": ${DEBUG_PLUGINS},
+
+ "post_endpoint": {
+ "path": "/api/blog/posts"
+ },
+
+ "channel_endpoint": {
+ "path": "/api/blog/channels"
+ },
+
+ "content_endpoint": {
+ "path": "/api/blog/content",
+ "max_content_length": ${MAX_CONTENT_LENGTH}
+ },
+
+ "blog_channels": {
+ //The index file for storing channel configuration
+ "index_file_name": "${CHANNEL_INDEX_FILE}"
+ },
+
+ "storage": {
+
+ "custom_storage_assembly": "${STORAGE_CUSTOM_ASSEMBLY}",
+
+ "type": "${STORAGE_TYPE}", //s3 | ftp
+
+ //storage config
+ "server_address": "${STORAGE_SERVER_ADDRESS}",
+ "access_key": "${STORAGE_USERNAME}",
+ "bucket": "${STORAGE_BUCKET}",
+ "use_ssl": ${STORAGE_USE_SSL},
+ "Region": "${S3_REGION}"
+ },
+
+ "secrets": {
+ //Set the vault path to the s3 secret
+ "storage_secret": "${STORAGE_SECRET}"
+ }
+} \ No newline at end of file
diff --git a/ci/config/Essentials.Accounts.json b/ci/config-templates/Essentials.Accounts-template.json
index b28caf4..4ca914c 100644
--- a/ci/config/Essentials.Accounts.json
+++ b/ci/config-templates/Essentials.Accounts-template.json
@@ -1,12 +1,12 @@
{
- "debug": false,
+ "debug": ${DEBUG_PLUGINS},
//endpoints
"login_endpoint": {
"path": "/api/account/login",
- "max_login_attempts": 10, //10 failed attempts in 10 minutes
- "failed_attempt_timeout_sec": 600 //10 minutes
+ "max_login_attempts": ${MAX_LOGIN_ATTEMPS}, //10 failed attempts in 10 minutes
+ "failed_attempt_timeout_sec": 600 //10 minutes
},
"keepalive_endpoint": {
@@ -63,11 +63,14 @@
//Path/domain for all security cookies
"cookie_domain": "",
"cookie_path": "/",
- "status_cookie_name": "li", //front-end cookie name must match to detect login status
+ "status_cookie_name": "cmnli", //front-end cookie name must match to detect login status
"otp_header_name": "X-Web-Token", //Front-end header name must match
"otp_time_diff_sec": 30,
"otp_key_size": 64,
- "pubkey_cookie_name": "client-id",
- "pubkey_signing_key_size": 32
+ "pubkey_cookie_name": "cmn-id",
+ "pubkey_signing_key_size": 32,
+ "strict_origin": false,
+ "strict_path": true, //Can be enabled if front-end is running on the same server
+ //"allowed_origins": [""]
}
} \ No newline at end of file
diff --git a/ci/config-templates/PageRouter-template.json b/ci/config-templates/PageRouter-template.json
new file mode 100644
index 0000000..86a51f8
--- /dev/null
+++ b/ci/config-templates/PageRouter-template.json
@@ -0,0 +1,7 @@
+{
+ "debug": ${DEBUG_PLUGINS},
+ "store": {
+ //All builds require the routes.xml file in the config directory even after variable substitution
+ "route_file": "config/routes.xml"
+ }
+} \ No newline at end of file
diff --git a/ci/config/SessionProvider.json b/ci/config-templates/SessionProvider-template.json
index a578b0d..de3bce1 100644
--- a/ci/config/SessionProvider.json
+++ b/ci/config-templates/SessionProvider-template.json
@@ -1,6 +1,6 @@
{
- "debug": false,
+ "debug": ${DEBUG_PLUGINS},
//Provider assemblies to load
"provider_assemblies": [ "VNLib.Plugins.Sessions.VNCache.dll" ],
@@ -10,12 +10,16 @@
//Cache system key prefix
"cache_prefix": "websessions",
//The session cookie name
- "cookie_name": "VNSession",
+ "cookie_name": "cmnext-ses",
//Size in bytes for generated session ids
"cookie_size": 40,
//time (in seconds) a session is valid for
"valid_for_sec": 3600,
//The maxium number of connections waiting for the cache server responses
- "max_waiting_connections": 100
+ "max_waiting_connections": 100,
+ //Enforce strict cross-origin session checks
+ "strict_cors": true,
+ ///Enforces strict TLS to help prevent tls downgrades based on stored session variables (privacy note: this can be leaked through brute-forced if session id is stolen)
+ "strict_tls_protocol": true
}
} \ No newline at end of file
diff --git a/ci/config-templates/config-template.json b/ci/config-templates/config-template.json
new file mode 100644
index 0000000..bd67020
--- /dev/null
+++ b/ci/config-templates/config-template.json
@@ -0,0 +1,169 @@
+{
+
+ //Host application config, config is loaded as a read-only DOM that is available
+ //to the host and loaded child plugins, all elements are available to plugins via the 'HostConfig' property
+
+ "http": {
+ //The defaut HTTP version to being requests with (does not support http/2 yet)
+ "default_version": "HTTP/1.1",
+ //The maxium size (in bytes) of response messges that will be compressed
+ "compression_limit": 512000000,
+ //Minium response size (in bytes) to compress
+ "compression_minimum": 2048,
+ //The size of the buffer to use when parsing multipart/form data uploads
+ "multipart_max_buf_size": 8192,
+ //The maxium ammount of data (in bytes) allows for mulitpart/form data file uploads
+ "multipart_max_size": 80240,
+ //Absolute maximum size (in bytes) of the request entity body (exludes headers)
+ "max_entity_size": ${MAX_CONTENT_LENGTH},
+ //Keepalive ms for HTTP1.1 keepalive connections
+ "keepalive_ms": 1000000,
+ //The buffer size to use when parsing headers (also the maxium request header size allowed)
+ "header_buf_size": 8128,
+ //The maxium number of headers allowed in an HTTP request message
+ "max_request_header_count": 50,
+ //The maxium number of allowed network connections, before 503s will be issued automatically and connections closed
+ "max_connections": 5000,
+ //The size in bytes of the buffer to use when writing response messages
+ "response_buf_size": 65536,
+ //time (in ms) to wait for a response from an active connection in recv mode, before dropping it
+ "recv_timeout_ms": 5000,
+ //Time in ms to wait for the client to accept transport data before terminating the connection
+ "send_timeout_ms": 60000,
+ //The size (in bytes) of the buffer used to store all response header data
+ "response_header_buf_size": 16384,
+ //Max number of file uploads allowed per request
+ "max_uploads_per_request": 10
+ },
+
+ //Compression is installed in the container at lib/ directory along with the native library supporting gzip and brotli
+ "compression_lib": "lib/vnlib.net.compression/VNLib.Net.Compression.dll",
+
+ //Setup the native lib
+ "vnlib.net.compression": {
+ "lib_path": "${COMPRESSION_LIB_PATH}",
+ "level": 1
+ },
+
+ //Maxium ammount of time a request is allowed to be processed (includes loading or waiting for sessions) before operations will be cancelled and a 503 returned
+ "max_execution_time_ms": 20000,
+
+ //Collection of objects to define hosts+interfaces to build server listeners from
+ "virtual_hosts": [
+ {
+
+ "trace": ${HTTP_TRACE_ON},
+
+ //The interface to bind to, you may not mix TLS and non-TLS connections on the same interface
+ "interface": {
+ "address": "0.0.0.0",
+ "port": 8080
+ },
+
+ //Collection of "trusted" servers to allow proxy header support from
+ "downstream_servers": ${HTTP_DOWNSTREAM_SERVERS},
+
+ //The hostname to listen for, "*" as wildcard, and "[system]" as the default hostname for the current machine
+ "hostname": "*",
+ "path": "dist/",
+
+ //A list of file extensions to deny access to, if a resource is requested and has one of the following extensions, a 404 is returned
+ "deny_extensions": [ ".ts", ".json", ".htaccess", ".php" ],
+ //The default file extensions to append to a resource that does not have a file extension
+ "default_files": [ "index.html" ],
+
+ //A list of error file objects, files are loaded into memory (and watched for changes) and returned when the specified error code occurs
+ "error_files": [],
+
+ //The default
+ "cache_default_sec": 864000,
+
+ "ssl": ${SSL_JSON},
+ }
+ ],
+
+
+ //Defines the directory where plugin's are to be loaded from
+ "plugins": {
+ //Hot-reload creates collectable assemblies that allow full re-load support in the host application, should only be used for development purposes!
+ "hot_reload": false,
+ "path": "plugins/",
+ "config_dir": "config/",
+ "assets": "plugins/assets/"
+ },
+
+ "sys_log": {
+ "path": "data/logs/sys-log.txt",
+ "flush_sec": 5,
+ "retained_files": 31,
+ "file_size_limit": 10485760,
+ "interval": "infinite"
+ },
+
+ "app_log": {
+ "path": "data/logs/app-log.txt",
+ "flush_sec": 5,
+ "retained_files": 31,
+ "file_size_limit": 10485760,
+ "interval": "infinite"
+ },
+
+ //HASHICORP VAULT
+ "hashicorp_vault": {
+ "url": "${HC_VAULT_ADDR}",
+ "token": "${HC_VAULT_TOKEN}",
+ "trust_certificate": ${HC_VAULT_TRUST_CERT},
+ },
+
+ //SQL CONFIG
+ "sql": {
+ "provider": "${SQL_LIB_PATH}",
+ "connection_string": "${SQL_CONNECTION_STRING}"
+ },
+
+ //VNCACHE global config
+ //Enable vncache as the providers above rely on the object caching server
+ "cache": {
+
+ "assembly_name": "${CACHE_ASM_PATH}",
+ "url": "${REDIS_CONNECTION_STRING}",
+
+ //Max size (in bytes) of allowed data to be stored in each user's session object
+ "max_object_size": 8128,
+
+ //Request timeout
+ "request_timeout_sec": 10,
+
+ //Time delay between cluster node discovery
+ "discovery_interval_sec": 120,
+
+ //Initial nodes to discover from
+ "initial_nodes": ${VNCACHE_INITIAL_NODES},
+
+ //Disable TLS
+ "use_tls": false,
+
+ //Setting this value to true will cause the cache store to load a memory-only instance, without remote backing
+ "memory_only": ${MEMCACHE_ONLY},
+
+ //enable memory cache
+ "memory_cache": {
+ "buckets": 20,
+ "bucket_size": 5000,
+ "max_age_sec": 600,
+ "refresh_interval_sec": 60,
+ "zero_all": false,
+ "max_object_size": 8128
+ }
+ },
+
+ "secrets": {
+ //Special key used by the loading library for access to the PasswordHashing library to pepper password hashes
+ "passwords": "${PASSWORD_PEPPER}",
+ "db_password": "${DATABASE_PASSWORD}",
+ "client_private_key": "${VNCACHE_CLIENT_PRIVATE_KEY}",
+ "cache_public_key": "${VNCACHE_CACHE_PUBLIC_KEY}",
+ "redis_password": "${REDIS_PASSWORD}"
+ }
+}
+
diff --git a/ci/config/routes.xml b/ci/config-templates/routes.xml
index 49d9553..85f9830 100644
--- a/ci/config/routes.xml
+++ b/ci/config-templates/routes.xml
@@ -26,19 +26,21 @@
<path>/assets/*</path>
</route>
- <!--Overwrite all other requests to the index file (catch all) using the ServeOther routine (4)-->
+ <!--Show the index file when navigating to /-->
<route routine="4" privilege="0">
-
- <!--Wildcard hostname-->
<hostname>*</hostname>
+ <path>/</path>
+ <alternate>index.html</alternate>
+ </route>
- <!--Declares that all files after / will be matched by this rule-->
+ <!--Redirect all other pages back to the app (homepage)-->
+ <route routine="2" privilege="0">
+ <hostname>*</hostname>
<path>/*</path>
-
- <!--Return to the root path, lets the file processor handle extension searching-->
<alternate>/</alternate>
</route>
+
<!--All routes that do not match will be allowed, this is only / since it does not have a matching rule-->
</routes> \ No newline at end of file
diff --git a/ci/config/CMNext.json b/ci/config/CMNext.json
deleted file mode 100644
index d11a8f9..0000000
--- a/ci/config/CMNext.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
- //Enables debug logging
- "debug": false,
-
- "post_endpoint": {
- "path": "/api/blog/posts"
- },
-
- "channel_endpoint": {
- "path": "/api/blog/channels"
- },
-
- "content_endpoint": {
- "path": "/api/blog/content",
- "max_content_length": 50000000
- },
-
- "blog_channels": {
- //The index file for storing channel configuration
- "index_file_name": "blogs/channels.json"
- },
-
- //S3 setup with vault secrets
- "disabled s3_config": {
- "server_address": "",
- "access_key": "",
- "bucket": "",
- "use_ssl": true,
- "Region": null
- },
-
- "disabled ftp_config": {
- "url": "",
- "username": "",
- //Base path within the ftp user's directory
- "base_path": ""
- },
-
- "secrets": {
- //Set the vault path to the s3 secret
- "s3_secret": "",
- "ftp_password": ""
- }
-} \ No newline at end of file
diff --git a/ci/config/PageRouter.json b/ci/config/PageRouter.json
deleted file mode 100644
index 7d24da8..0000000
--- a/ci/config/PageRouter.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "debug": false,
- "store": {
- "route_file": "config/routes.xml"
- }
-} \ No newline at end of file
diff --git a/ci/config/config.json b/ci/config/config.json
deleted file mode 100644
index b410399..0000000
--- a/ci/config/config.json
+++ /dev/null
@@ -1,183 +0,0 @@
-{
-
- //Host application config, config is loaded as a read-only DOM that is available
- //to the host and loaded child plugins, all elements are available to plugins via the 'HostConfig' property
-
- "http": {
- //The defaut HTTP version to being requests with (does not support http/2 yet)
- "default_version": "HTTP/1.1",
- //The maxium size (in bytes) of response messges that will be compressed
- "compression_limit": 2048000,
- //Minium response size (in bytes) to compress
- "compression_minimum": 2048,
- //The size of the buffer to use when parsing multipart/form data uploads
- "multipart_max_buf_size": 20480,
- //The maxium ammount of data (in bytes) allows for mulitpart/form data file uploads
- "multipart_max_size": 80240,
- //Absolute maximum size (in bytes) of the request entity body (exludes headers)
- "max_entity_size": 1024000,
- //Keepalive ms for HTTP1.1 keepalive connections
- "keepalive_ms": 1000000,
- //The buffer size to use when parsing headers (also the maxium request header size allowed)
- "header_buf_size": 8128,
- //The maxium number of headers allowed in an HTTP request message
- "max_request_header_count": 50,
- //The maxium number of allowed network connections, before 503s will be issued automatically and connections closed
- "max_connections": 5000,
- //The size in bytes of the buffer to use when writing response messages
- "response_buf_size": 65535,
- //time (in ms) to wait for a response from an active connection in recv mode, before dropping it
- "recv_timeout_ms": 5000,
- //Time in ms to wait for the client to accept transport data before terminating the connection
- "send_timeout_ms": 60000,
- //The size (in bytes) of the buffer used to store all response header data
- "response_header_buf_size": 16384,
- //Max number of file uploads allowed per request
- "max_uploads_per_request": 10
- },
-
- //Path to managed compressor library
- "compression_lib": null,
-
- //Maximum ammount of time a request is allowed to be processed (includes loading or waiting for sessions) before operations will be cancelled and a 503 returned
- "max_execution_time_ms": 20000,
-
- //Collection of objects to define hosts+interfaces to build server listeners from
- "virtual_hosts": [
- {
- //The interface to bind to, you may not mix TLS and non-TLS connections on the same interface
- "interface": {
- "address": "0.0.0.0",
- "port": 8080
- },
-
- //The directory path for files served by this endpoint
- "path": "dist",
-
- //The hostname to listen for, "*" as wildcard, and "[system]" as the default hostname for the current machine
- //"hostname": "*",
-
- //Or specify an array of hostnames instead, the hostnames array property takes priority over the single hostname property, each must be unique
- "hostnames": [ "*" ],
-
- //Collection of "trusted" servers to allow proxy header support from
- "downstream_servers": [],
-
- //Specify a list of ip addresses that are allowed to connect to the server, 403 will be returned if connections are not on this list
- //whitelist works behind a trusted downstream server that supports X-Forwared-For headers
- //"whitelist": [ "127.0.0.1" ],
-
- //A list of file extensions to deny access to, if a resource is requested and has one of the following extensions, a 404 is returned
- "deny_extensions": [ ".env", ".yaml", ".cs" ],
-
- //The default file extensions to append to a resource that does not have a file extension
- "default_files": [ "index.html" ],
-
- //Key-value headers object, some headers are special and are controlled by the vh processor
- "headers": {
- "X-Content-Type-Options": "nosniff",
- "X-Xss-Protection": "1; mode=block",
- "X-Frame-Options": "DENY",
- "Content-Security-Policy": "default-src 'self' https://cdn.ckeditor.com 'unsafe-inline'; img-src 'self' blob: data:; frame-src 'none'; object-src 'none'; referrer no-referrer-when-downgrade; upgrade-insecure-requests; block-all-mixed-content;"
- },
-
- //Enables cors support for all endpoints and header controls, if false, all endpoints that are send CORS request headers will be forbidden
- //"enable_cors": true,
-
- //Allowed cors authoriy domains
- //"cors_allowed_authority": [ "localhost:8080" ],
-
- //Define a TLS certificate (enables TLS on the interface)
- "disabled ssl": {
-
- //Cert may be pem or pfx (include private key in pfx, or include private key in a pem file)
- "cert": "/path/to/cert.pfx|pem",
-
- //A pem encoded private key, REQUIRED if using a PEM certificate, may be encrypted with a password
- "privkey": "/path/to/private_key.pem",
-
- //An optional password for the ssl private key
- "password": "plain-text-password",
-
- //requires that any client connecting to this host present a valid certificate
- "client_cert_required": false
- },
-
- //A list of error file objects, files are loaded into memory (and watched for changes) and returned when the specified error code occurs
- "error_files": [
- /*
- {
- "code": 404,
- "path": "404.html"
- },
- */
- ],
-
- //The default
- "cache_default_sec": 864000
- }
- ],
-
-
- //Defines the directory where plugin's are to be loaded from
- "plugins": {
- //Hot-reload creates collectable assemblies that allow full re-load support in the host application, should only be used for development purposes!
- "hot_reload": false,
- "reload_delay_sec": 2,
- "path": "plugins",
- "config_dir": "config"
- },
-
- "disabled sys_log": {
- //"path": "path/to/syslog/file",
- //"template": "serilog template for writing to file",
- //"flush_sec": 5,
- //"retained_files": 31,
- //"file_size_limit": 10485760,
- //"interval": "infinite"
- },
-
- "disabled app_log": {
- //"path": "path/to/applog/file",
- //"template": "serilog template for writing to file",
- //"flush_sec": 5,
- //"retained_files": 31,
- //"file_size_limit": 10485760,
- //"interval": "infinite"
- },
-
- //Sql for the users database
- "sql": {
- "db_type": "sqlite", //mysql, mssql(default), sqlite
- "source": "cmnext.db" //For sqlite only
-
- //"hostname": "example.com",
- //"username": "cmnext",
- //"catalog": "cmnext",
- //"min_pool_size": 5,
- //"ms_security": false
- },
-
- //caching should be setup globally after VNCache #78a47dd
- "cache":{
- //Load the vncache dll
- "assembly_name": "VNLib.Data.Caching.Providers.VNCache.dll",
- //Defaulting to memory only for now
- "memory_only": true,
- //enable memory cache
- "memory_cache": {
- "buckets": 20,
- "bucket_size": 5000,
- "max_age_sec": 600,
- "refresh_interval_sec": 60,
- "zero_all": false,
- "max_object_size": 8128
- }
- },
-
- //Global secrets object, used by the host and pluings for a specialized secrets
- "secrets": {
- //"db_password": ""
- "passwords": "yourbase64passwordsecret"
- }
-}
diff --git a/ci/container/Dockerfile b/ci/container/Dockerfile
new file mode 100644
index 0000000..5268448
--- /dev/null
+++ b/ci/container/Dockerfile
@@ -0,0 +1,109 @@
+#Copyright (c) Vaughn Nugent
+#Licensed under the GNU AGPL V3.0
+
+#use plain alpine latest to build native libraries in
+FROM alpine:3.19 as native-cont
+
+#install public libs and build tools
+RUN apk update && apk add --no-cache build-base cmake npm git openssl
+#most universal way to use Task is from NPM
+RUN npm install -g @go-task/cli
+
+WORKDIR /build
+
+#include local artifacts
+COPY app/ .
+
+#build internal libraries and copy the libraries to the /lib output directory
+RUN mkdir out/ ssl/
+RUN task build-libs
+
+#APP CONTAINER
+#move into a clean dotnet apline lean image
+FROM mcr.microsoft.com/dotnet/runtime:8.0.3-alpine3.19-amd64 as app-cont
+
+LABEL name="vnuge/cmnext"
+LABEL maintainer="Vaughn Nugent <vnpublic@proton.me>"
+LABEL description="A dead-simple, multi-channel cms for your blog or podcast built for static storage like S3 or FTP"
+
+#copy local artifacts again in run container
+COPY app/ /app
+
+#pull compiled libs from build container
+COPY --from=native-cont /build/out /app/lib
+#copy self signed ssl certs for first startup
+COPY --from=native-cont /build/ssl /app/ssl
+
+RUN apk update && apk add --no-cache gettext icu-libs dumb-init
+
+#workdir
+WORKDIR /app
+
+#default to 8080 for TLS on TCP
+EXPOSE 8080/tcp
+
+VOLUME /app/data \
+ /app/ssl \
+#expose an assets directory for custom assets install
+ /app/usr/assets
+
+#disable dotnet invariant culture on alpine
+ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=0
+
+#add helper/required libraries
+#ENV VNLIB_SHARED_HEAP_FILE_PATH=/app/lib/libvn_rpmalloc.so not ready yet, still need to debug
+ENV VNLIB_ARGON2_DLL_PATH=/app/lib/libargon2.so \
+ COMPRESSION_LIB_PATH=/app/lib/libvn_compress.so
+
+#set default env variables
+ENV MAX_CONTENT_LENGTH=204800000 \
+ REG_TOKEN_DURATION_MIN=360 \
+ MAX_LOGIN_ATTEMPS=10
+
+#SQL Config
+ENV SQL_LIB_PATH=VNLib.Plugins.Extensions.Sql.SQLite.dll
+ENV SQL_CONNECTION_STRING="Data Source=data/cmnext.db;"
+
+#STORAGE
+ENV STORAGE_TYPE="s3" \
+ STORAGE_CUSTOM_ASSEMBLY="" \
+ STORAGE_SERVER_ADDRESS="" \
+ STORAGE_USERNAME="" \
+ STORAGE_BUCKET="" \
+ STORAGE_USE_SSL=true \
+ S3_REGION=""
+
+#HC Vault
+ENV HC_VAULT_ADDR="" \
+ HC_VAULT_TOKEN="" \
+ HC_VAULT_TRUST_CERT=false
+
+#VNCACHE (default to memory only)
+ENV CACHE_ASM_PATH=VNLib.Data.Caching.Providers.VNCache.dll \
+ MEMCACHE_ONLY=true \
+ REDIS_CONNECTION_STRING="" \
+ VNCACHE_INITIAL_NODES=[]
+
+#SECRETS
+ENV PASSWORD_PEPPER="" \
+ DATABASE_PASSWORD="" \
+ REDIS_PASSWORD="" \
+ VNCACHE_CLIENT_PRIVATE_KEY="" \
+ VNCACHE_CACHE_PUBLIC_KEY="" \
+ STORAGE_SECRET=""
+
+
+#HTTP/PROXY Config
+ENV HTTP_DOWNSTREAM_SERVERS=[] \
+ HTTP_TRACE_ON=false
+
+#set default certificate files to the self signed ones created in the build container
+ENV SSL_JSON='{"cert": "ssl/cert.pem", "privkey":"ssl/key.pem"}'
+
+#disable plugin debugging by default
+ENV DEBUG_PLUGINS=false
+
+#run the init script within dumb-init
+ENTRYPOINT ["dumb-init", "--"]
+CMD ["ash", "./run.sh"]
+
diff --git a/ci/container/Taskfile.yaml b/ci/container/Taskfile.yaml
new file mode 100644
index 0000000..0b959ab
--- /dev/null
+++ b/ci/container/Taskfile.yaml
@@ -0,0 +1,86 @@
+# https://taskfile.dev
+
+#This taskfile must be called from the CI taskfile, as it is part of the CI
+#project and it's pipleine. This file will also be copied to the container image
+#and used to build the native libraries for the project.
+
+version: "3"
+
+vars:
+ INCLUDE_FILES: "Dockerfile, docker-compose.yaml"
+ CERT_KEY_PARAMS: "ec -pkeyopt ec_paramgen_curve:secp384r1"
+
+includes:
+ install:
+ taskfile: ../install.taskfile.yaml
+ optional: true #not needed for inside container build
+
+tasks:
+ #called from inside the container to build native libraries
+ build-libs:
+ vars:
+ OUT_DIR: "{{.USER_WORKING_DIR}}/out"
+ SSL_DIR: "{{.USER_WORKING_DIR}}/ssl"
+ HOSTNAME:
+ sh: echo $HOSTNAME
+
+ cmds:
+ #build argon2 native library
+ - cd lib/argon2/ && task && cp build/libargon2.so {{.OUT_DIR}}/libargon2.so
+ #build rpmalloc library and rewrite to a standard .dll extension
+ - cd lib/vnlib_rpmalloc/ && task && cp build/libvn_rpmalloc.so {{.OUT_DIR}}/libvn_rpmalloc.so
+ #build compression and rewrite to a standard .dll extension
+ - cd lib/vnlib_compress/ && task && cp build/libvn_compress.so {{.OUT_DIR}}/libvn_compress.so
+
+ #create a fresh self-signed cert for the container during build
+ - openssl req -new -x509 -days 365 -keyout {{.SSL_DIR}}/key.pem -out {{.SSL_DIR}}/cert.pem -newkey {{.CERT_KEY_PARAMS}} --nodes -subj "/CN={{.HOSTNAME}}"
+ - echo "WARNING Self signed certificate created during build stage, DO NOT COPY THIS IMAGE"
+
+ #called from ci pipline to build the package
+ build:
+ cmds:
+ # clean up the run.sh script to remove windows line endings in my wsl default instance
+ - cmd: wsl dos2unix ./run.sh
+ platforms: [ windows/amd64 ]
+
+ #init build image
+ - task: setup-container-image
+
+ #remove the default config file as it's not needed in the container
+ - powershell -Command "rm -Force -Recurse build/app/config/"
+
+ - task: prune-sql-runtimes
+
+ postbuild_success:
+ cmds:
+ #tar up the build directory and move it to the output bin directory
+ - cmd: cd build/ && tar -czf ../../bin/{{.PACKAGE_FILE_NAME}} .
+ #clean up all the build files after build succeeds
+ - task: clean
+
+ clean:
+ ignore_error: true
+ cmds:
+ - cmd: powershell -Command "rm -Recurse -Force ./build"
+
+ setup-container-image:
+ internal: true
+ cmds:
+ #make build directory
+ - powershell -Command "mkdir build, build/app, build/app/config-templates/, build/app/static/ -Force"
+ #copy the existing linux-x64 build to the build folder, this will be the container base
+ - powershell -Command "cp -Recurse -Force ../build/linux-x86_64/* build/app/"
+ #copy local scripts and raw config templates into the build folder
+ - powershell -Command "cp -Force run.sh, Taskfile.yaml build/app/"
+ - powershell -Command "cp -Force Dockerfile, docker-compose.yaml build/"
+ - powershell -Command "cp -Force ../config-templates/* build/app/config-templates/"
+
+ prune-sql-runtimes:
+ internal: true
+ vars:
+ SQLITE_RUNTIMES: 'build/app/plugins/assets/VNLib.Plugins.Extensions.Loading.Sql.SQLite/runtimes'
+ cmds:
+ #move the linux-musl-x64 directory out of assets before removing the rest of the runtimes and then move it back
+ - powershell -Command "mv {{.SQLITE_RUNTIMES}}/linux-musl-x64 build/linux-musl-x64"
+ - powershell -Command "rm -Recurse -Force {{.SQLITE_RUNTIMES}}" && powershell -Command "mkdir {{.SQLITE_RUNTIMES}}"
+ - powershell -Command "mv build/linux-musl-x64 {{.SQLITE_RUNTIMES}}/linux-musl-x64 " \ No newline at end of file
diff --git a/ci/container/docker-compose.yaml b/ci/container/docker-compose.yaml
new file mode 100644
index 0000000..9bd1fc1
--- /dev/null
+++ b/ci/container/docker-compose.yaml
@@ -0,0 +1,73 @@
+#Copyright (c) Vaughn Nugent
+#Licensed under the GNU AGPLv3
+
+version: '3.6'
+
+services:
+ cmnext:
+ image: vnuge/cmnext
+ container_name: cmnext
+ restart: unless-stopped
+ volumes:
+ - ./data:/app/data
+ - ./assets:/app/usr/assets:ro
+ #uncomment to use your own ssl certs, otherwise a build-time cert will be used
+ #- ./ssl/cert.pem:/app/ssl/cert.pem:ro
+ #- ./ssl/key.pem:/app/ssl/key.pem:ro
+ ports:
+ - 8080:8080
+ environment:
+ CHANNEL_INDEX_FILE: "channels.json" #required, should leave default unless you know what you are doing
+ MAX_CONTENT_LENGTH: 204800000 #200MB max upload size
+ MAX_LOGIN_ATTEMPS: "10" #max login attempts before user account is locked out
+
+ #SQL Config
+ SQL_LIB_PATH: "VNLib.Plugins.Extensions.Sql.SQLite.dll"
+ SQL_CONNECTION_STRING: "Data Source=data/cmnext.db;" #when using a password, simply leave the password field blank
+
+ #storage backend setup
+ STORAGE_TYPE: "s3" #s3 | ftp
+ STORAGE_CUSTOM_ASSEMBLY: "" #optional path to a custom storage assembly
+ STORAGE_SERVER_ADDRESS: "" #s3 or ftp server address
+ STORAGE_USERNAME: "" #s3 client id or ftp username
+ STORAGE_BUCKET: "" #s3 bucket or ftp root directory
+ STORAGE_USE_SSL: "true" #force ssl for connections
+ S3_REGION: "" #optional s3 region when using s3 storage
+
+ #HC Vault client config
+ #HC_VAULT_ADDR: ""
+ #HC_VAULT_TOKEN: ""
+ #HC_VAULT_TRUST_CERT: "false"
+
+ #VNCACHE (default to memory only)
+ CACHE_ASM_PATH: "VNLib.Data.Caching.Providers.VNCache.dll"
+ MEMCACHE_ONLY: "true"
+ REDIS_CONNECTION_STRING: ""
+ #at least one node required if MEMCACHE_ONLY is false
+ VNCACHE_INITIAL_NODES: "[]"
+
+ #SECRETS
+ # All secrets may be a raw value, read from a file,
+ # an environment variable, or a vault path
+ # file://mysecret.txt reads the secret from a file (case sensitive)
+ # env://MY_SECRET reads the secret from an environment variable (case sensitive)
+ # vault://kv/data/secret?secret=value reads the value of the mysecret key in the secret/data path
+
+ PASSWORD_PEPPER: "" #Must be a base64 encoded value, of realtivley any size
+ DATABASE_PASSWORD: "" #overrides the 'Password' field in the SQL connection string
+ REDIS_PASSWORD: "" #only required if using a password protected redis server
+ #if MEMCACHE_ONLY is false, then the following keys are required to connect to a VNCACHE cluster
+ VNCACHE_CLIENT_PRIVATE_KEY: ""
+ VNCACHE_CACHE_PUBLIC_KEY: ""
+ #REQUIRED s3 or ftp secret key
+ STORAGE_SECRET: ""
+
+ #HTTP
+ HTTP_DOWNSTREAM_SERVERS: '[]' #a comma separated list of downstream (proxy) server ip addresses
+ HTTP_TRACE_ON: "false" #enable http trace logging, requires you to set --debug to SERVER_ARGS variable below
+
+ #Very Verbose plugin logging, required --debug CLI flag, prints literally everything to the logger (it's annoying)
+ DEBUG_PLUGINS: "false"
+
+ SERVER_ARGS: ""
+
diff --git a/ci/container/run.sh b/ci/container/run.sh
new file mode 100644
index 0000000..d829509
--- /dev/null
+++ b/ci/container/run.sh
@@ -0,0 +1,25 @@
+#! /bin/sh
+
+#this script will be invoked by dumb-init in the container on statup and is located at /app
+
+echo "Generating configuration files"
+
+rm -rf config && mkdir config
+
+#move the routes xml file to the output config dir
+cp config-templates/routes.xml config/routes.xml
+
+#substitude all -template files in the config-templates dir and write them to the config dir
+for file in config-templates/*-template.json; do
+ envsubst < $file > config/$(basename $file -template.json).json
+done
+
+echo "Complete"
+
+echo "Merging your asset files"
+cp usr/assets/* plugins/assets/ -rf
+echo "Complete"
+
+#start the server
+echo "Starting the server"
+dotnet webserver/VNLib.WebServer.dll --config config/config.json --input-off $SERVER_ARGS \ No newline at end of file
diff --git a/ci/install.ps1 b/ci/install.ps1
index 4c42c18..3c2a2f4 100644
--- a/ci/install.ps1
+++ b/ci/install.ps1
@@ -1,15 +1,16 @@
-param([String] $BaseUrl, [String] $ModuleName, [String] $ProjectName, [String]$FileName)
+param([String] $BaseUrl, [String] $ModuleName, [String] $ProjectName, [String]$FileName, [String]$Version)
-#get the latest file
-Invoke-WebRequest "$BaseUrl/$ModuleName/@latest" -OutFile latest.txt
-#read the file into a variable
-$latest = Get-Content latest.txt
+#random delays to space out the downloads
+$randomDelay = Get-Random -Minimum 1000 -Maximum 2000
+Start-Sleep -Milliseconds $randomDelay
+
+$_src = "$BaseUrl/$ModuleName/$Version/$ProjectName/$FileName"
#download the latest version
-Invoke-WebRequest "$BaseUrl/$ModuleName/$latest/$ProjectName/$FileName" -OutFile $FileName
+Invoke-WebRequest "$_src" -OutFile $FileName
#download latest sha256
-Invoke-WebRequest "$BaseUrl/$ModuleName/$latest/$ProjectName/$FileName.sha256" -OutFile "$FileName.sha256"
+Invoke-WebRequest "$_src.sha256" -OutFile "$FileName.sha256"
#verify the file
$hash = (Get-FileHash $FileName -Algorithm SHA256).Hash
diff --git a/ci/install.taskfile.yaml b/ci/install.taskfile.yaml
new file mode 100644
index 0000000..587ac86
--- /dev/null
+++ b/ci/install.taskfile.yaml
@@ -0,0 +1,22 @@
+# https://taskfile.dev
+
+#Called by the vnbuild system to produce builds for my website
+#https://www.vaughnnugent.com/resources/software
+
+version: "3"
+
+tasks:
+
+ install:
+ internal: true
+ cmds:
+ #make the plugin directory
+ - cmd: powershell -Command "mkdir {{.DIR}} -Force"
+ ignore_error: true
+ - cd {{.DIR}} && powershell "{{ .PROJECT_DIR }}/install.ps1" -BaseUrl {{.BUILDS_URL}} -ModuleName {{.MODULE_NAME}} -ProjectName {{.PROJECT_NAME}} -FileName {{.FILE_NAME}} -Version {{.VERSION}}
+ - cd {{.DIR}} && tar -xzf {{.FILE_NAME}}
+ #remove the archive file
+ - cd {{.DIR}} && powershell -Command "rm {{.FILE_NAME}}"
+ #remove all pckage.json files that could cause build issues
+ - cmd: cd {{.DIR}} && powershell 'Get-ChildItem -Path . -Recurse -File -Filter "package.json" | Remove-Item -Force'
+ ignore_error: true \ No newline at end of file
diff --git a/ci/plugins.taskfile.yaml b/ci/plugins.taskfile.yaml
index d67c56e..dda3838 100644
--- a/ci/plugins.taskfile.yaml
+++ b/ci/plugins.taskfile.yaml
@@ -5,132 +5,200 @@
version: "3"
+includes:
+ install:
+ taskfile: install.taskfile.yaml
+
vars:
+ CORE_VERSION: '1b590c2517fef110564943ed8a10edd11fa758b0'
+ ESSENTIALS_VERSION: '451091e93b5feee7a5e01d3a81f5d63efa7ea8be'
+ CACHE_VERSION: '930980a1e6b5db24dd4d8beeb115e4279ea79ee2'
+ USERS_VERSION: '6d48be92c39d0046c5d827fb86af7ea1f3877e69'
+ SESSION_VERSION: 'f2dedded2a8646273c4dd13013b8334d64e02d6f'
+ EXTENSIONS_VERSION: '1350c983c371fdd6a93596c8474345f9168284e1'
tasks:
all:
+ deps:
+ - install-rpmalloc
+ - install-compressor-lib
+ - install-argon2-lib
+ - install-compression
+ - install-sqlite
+ - install-router
+ - install-accounts
+ - install-sessions
+ - install-vncache
+ - install-vncache-sessions
+ - install-users
cmds:
+
- echo "Installing and configuring plugins and UI"
- - task: install-accounts
- - task: install-sessions
- - task: install-vncache
- - task: install-vncache-sessions
- - task: install-router
- - task: install-users
- - task: install-argon2-lib
- - task: build-cmnext
-
- build-cmnext:
+ - task: build-proj
+
+ build-proj:
cmds:
- #build js libraries
- - cd ../lib/admin && npm install && npm run build
- - cd ../front-end && npm install && npm run build
+ #build front-end
+ - cd ../front-end && task ci-build
#move dist files to web root
- - powershell -Command "cp -Recurse -Force ../front-end/dist {{.USER_WORKING_DIR}}"
+ - cmd: powershell -Command "cp -Recurse -Force ../front-end/dist {{.USER_WORKING_DIR}}"
#build the plugin
- cmd: cd ../back-end/src && task build
- - powershell -Command "mkdir plugins/CMNext -Force"
+ - powershell -Command "mkdir plugins/{{.PROJECT_NAME}} -Force"
#copy the plugin output
- - powershell -Command "cp -Recurse -Force ../back-end/src/bin/release/net8.0/publish/* plugins/CMNext/"
+ - powershell -Command "cp -Recurse -Force ../back-end/src/bin/release/net8.0/publish/* plugins/{{.PROJECT_NAME}}/"
install-accounts:
cmds:
#install accounts plugin
- - task: install-plugin
+ - task: install:install
vars:
PROJECT_NAME: 'VNLib.Plugins.Essentials.Accounts'
MODULE_NAME: "Plugins.Essentials"
FILE_NAME: "release.tgz"
DIR: './plugins/Essentials.Accounts'
+ VERSION: '{{.ESSENTIALS_VERSION}}'
install-router:
cmds:
#install router plugin
- - task: install-plugin
+ - task: install:install
vars:
PROJECT_NAME: 'VNLib.Plugins.Essentials.Content.Routing'
MODULE_NAME: "Plugins.Essentials"
FILE_NAME: "release.tgz"
DIR: './plugins/PageRouter'
+ VERSION: '{{.ESSENTIALS_VERSION}}'
install-sessions:
cmds:
#install sessions plugin
- - task: install-plugin
+ - task: install:install
vars:
PROJECT_NAME: 'SessionProvider'
MODULE_NAME: "VNLib.Plugins.Sessions"
FILE_NAME: "release.tgz"
DIR: './plugins/SessionProvider'
+ VERSION: '{{.SESSION_VERSION}}'
+
+ install-vncache-sessions:
+ cmds:
+ #install vncache-web-sessions plugin
+ - task: install:install
+ vars:
+ PROJECT_NAME: 'VNLib.Plugins.Sessions.VNCache'
+ MODULE_NAME: "VNLib.Plugins.Sessions"
+ FILE_NAME: "release.tgz"
+ DIR: './plugins/assets/VNLib.Plugins.Sessions.VNCache'
+ VERSION: '{{.SESSION_VERSION}}'
install-users:
cmds:
#install users plugin
- - task: install-plugin
+ - task: install:install
vars:
PROJECT_NAME: 'VNLib.Plugins.Essentials.Users'
MODULE_NAME: "VNLib.Plugins.Essentials.Users"
FILE_NAME: "release.tgz"
DIR: './plugins/assets/VNLib.Plugins.Essentials.Users'
+ VERSION: '{{.USERS_VERSION}}'
install-vncache:
cmds:
#install vncache global cache provider plugin
- - task: install-plugin
+ - task: install:install
vars:
PROJECT_NAME: 'VNLib.Data.Caching.Providers.VNCache'
MODULE_NAME: "VNLib.Data.Caching"
FILE_NAME: "release.tgz"
DIR: './plugins/assets/VNLib.Data.Caching.Providers.VNCache'
+ VERSION: '{{.CACHE_VERSION}}'
- install-vncache-sessions:
+ install-sqlite:
cmds:
- #install vncache-web-sessions plugin
- - task: install-plugin
+ #install SQLite asset package
+ - task: install:install
vars:
- PROJECT_NAME: 'VNLib.Plugins.Sessions.VNCache'
- MODULE_NAME: "VNLib.Plugins.Sessions"
+ PROJECT_NAME: 'VNLib.Plugins.Extensions.Loading.Sql.SQLite'
+ MODULE_NAME: "VNLib.Plugins.Extensions"
FILE_NAME: "release.tgz"
- DIR: './plugins/assets/VNLib.Plugins.Sessions.VNCache'
+ DIR: './plugins/assets/VNLib.Plugins.Extensions.Loading.Sql.SQLite'
+ VERSION: '{{.EXTENSIONS_VERSION}}'
+
+ install-compression:
+ cmds:
+ #install compression plugin
+ - task: install:install
+ vars:
+ PROJECT_NAME: 'VNLib.Net.Compression'
+ MODULE_NAME: "VNLib.Core"
+ FILE_NAME: "release.tgz"
+ DIR: './lib/vnlib.net.compression'
+ VERSION: '{{.CORE_VERSION}}'
+
+ install-compressor-lib:
+ cmds:
+ #install the compressor binary for Windows
+ - task: install:install
+ vars:
+ PROJECT_NAME: 'vnlib_compress'
+ MODULE_NAME: "VNLib.Core"
+ FILE_NAME: "msvc-x64-release-vnlib_compress.tgz"
+ DIR: './lib/vnlib_compress'
+ VERSION: '{{.CORE_VERSION}}'
+
+ #install compressor plugin
+ - task: install:install
+ vars:
+ PROJECT_NAME: 'vnlib_compress'
+ MODULE_NAME: "VNLib.Core"
+ FILE_NAME: "src.tgz"
+ DIR: './lib/vnlib_compress'
+ VERSION: '{{.CORE_VERSION}}'
install-argon2-lib:
cmds:
#install the argon2 binary for Windows
- - task: install-plugin
+ - task: install:install
vars:
PROJECT_NAME: 'phc-winner-argon2'
MODULE_NAME: "VNLib.Core"
- FILE_NAME: "win-x64-release-Argon2.tgz"
+ FILE_NAME: "msvc-x64-release-Argon2.tgz"
DIR: './lib/argon2'
+ VERSION: '{{.CORE_VERSION}}'
#install the argon2 source code package for Linux and Mac
- - task: install-plugin
+ - task: install:install
vars:
PROJECT_NAME: 'phc-winner-argon2'
MODULE_NAME: "VNLib.Core"
FILE_NAME: "src.tgz"
DIR: './lib/argon2'
+ VERSION: '{{.CORE_VERSION}}'
#remove unneeded files
- - cmd: powershell -Command "rm ./lib/argon2/man -Recurse"
- ignore_error: true
- - cmd: powershell -Command "rm ./lib/argon2/latex -Recurse"
- ignore_error: true
- - cmd: powershell -Command "rm ./lib/argon2/kats -Recurse"
- ignore_error: true
- - cmd: powershell -Command "rm ./lib/argon2/argon2-specs.pdf"
- ignore_error: true
- - cmd: powershell -Command "rm ./lib/argon2/package.json"
+ - for: [ man, latex, kats, argon2-specs.pdf ]
+ cmd: powershell -Command "rm ./lib/argon2/{{.ITEM}} -Recurse"
ignore_error: true
- install-plugin:
+ install-rpmalloc:
cmds:
- #make the plugin directory
- - cmd: powershell -Command "mkdir {{.DIR}} -Force"
- ignore_error: true
- - cd {{.DIR}} && powershell "{{.USER_WORKING_DIR}}/install.ps1" -BaseUrl {{.BUILDS_URL}} -ModuleName {{.MODULE_NAME}} -ProjectName {{.PROJECT_NAME}} -FileName {{.FILE_NAME}}
- - cd {{.DIR}} && tar -xzf {{.FILE_NAME}}
- #remove the archive file
- - cd {{.DIR}} && powershell -Command "rm {{.FILE_NAME}}" \ No newline at end of file
+ #install the rpmalloc source code package for Linux and Mac
+ - task: install:install
+ vars:
+ PROJECT_NAME: 'vnlib_rpmalloc'
+ MODULE_NAME: "VNLib.Core"
+ FILE_NAME: "src.tgz"
+ DIR: './lib/vnlib_rpmalloc'
+ VERSION: '{{.CORE_VERSION}}'
+
+ #install the rpmalloc binary for Windows
+ - task: install:install
+ vars:
+ PROJECT_NAME: 'vnlib_rpmalloc'
+ MODULE_NAME: "VNLib.Core"
+ FILE_NAME: "msvc-x64-release-vnlib_rpmalloc.tgz"
+ DIR: './lib/vnlib_rpmalloc'
+ VERSION: '{{.CORE_VERSION}}'
diff --git a/ci/release.taskfile.yaml b/ci/release.taskfile.yaml
new file mode 100644
index 0000000..549ee8c
--- /dev/null
+++ b/ci/release.taskfile.yaml
@@ -0,0 +1,132 @@
+# https://taskfile.dev
+
+#Inlcuded taskfile for object cache server that is used to produce
+#ci builds for standalone caching servers
+
+version: "3"
+
+vars:
+ SSL_DIR: "ssl"
+ DATA_DIR: "data"
+ DEFAULT_EC_CURVE: "secp384r1"
+ PASS_PEPPER_SIZE: 64
+
+tasks:
+ default:
+ desc: "Runs the server in realease mode"
+ interactive: true
+ cmds:
+ - task: run
+
+ run:
+ desc: "Runs the server in release mode"
+ silent: true
+ interactive: true
+ env:
+ #libraries intentionally do not have extensions, for cross-platform compatibility, the server will load them regardless
+ VNLIB_SHARED_HEAP_FILE_PATH: lib/vnlib_rpmalloc.dll
+ VNLIB_ARGON2_DLL_PATH: lib/argon2.dll
+ cmds:
+ - cmd: dotnet webserver/VNLib.WebServer.dll --config config/config.json {{.CLI_ARGS}}
+
+ setup-apt:
+ desc: "Performs initial setup on Debian apt amd64 based machines"
+ silent: true
+ cmds:
+ - apt update
+ - apt install -y dotnet-runtime-8.0 gcc cmake curl
+ - task: setup
+ - echo "Setup complete"
+
+ setup-dnf:
+ desc: "Performs initial setup on Fedora/Redhat amd (dnf) based machines"
+ silent: true
+ cmds:
+ - dnf update
+ - dnf install -y dotnet-runtime-8.0 gcc cmake curl
+ - task: setup
+ - echo "Setup complete"
+
+ setup-apk:
+ desc: "Performs initial setup using the APK package manager for amd64 based machines"
+ silent: true
+ cmds:
+ - apk update
+ - apk add --no-cache dotnet8-runtime build-base cmake curl
+ - task: setup
+ - echo "Setup complete"
+
+ setup:
+ desc: "Performs platform agnostic setup tasks without installing tools (no sudo needed)"
+ cmds:
+ #build rpmalloc lib
+ - task: build-rpmalloc
+ - task: build-argon2
+ - task: build-compress
+
+ #setup ssl dir
+ - cmd: mkdir ssl/
+ platforms: [ linux, darwin ]
+ ignore_error: true
+ - cmd: powershell -Command "mkdir ssl/"
+ platforms: [ windows/amd64 ]
+ ignore_error: true
+
+ create-pepper:
+ desc: "Generates a new pepper for hashing passwords."
+ vars:
+ PEPPER_FILE: "secrets/password-pepper.txt"
+ cmds:
+ - cmd: mkdir secrets/
+ platforms: [ linux, darwin ]
+ ignore_error: true
+ - cmd: powershell -Command "mkdir secrets/"
+ platforms: [ windows/amd64 ]
+ ignore_error: true
+
+ - cmd: openssl rand -base64 {{.PASS_PEPPER_SIZE}} > {{.PEPPER_FILE}}
+ platforms: [ linux, darwin ]
+
+ - echo "DO NOT EVER LOSE OR UPDATE THIS FILE!"
+
+ create-cert:
+ desc: "Genereates a new self-signed TLS certificate"
+ interactive: true
+ cmds:
+ - openssl req -new -x509 -days 365 -keyout {{.SSL_DIR}}/key.pem -out {{.SSL_DIR}}/cert.pem -newkey ec -pkeyopt ec_paramgen_curve:{{.DEFAULT_EC_CURVE}} --nodes
+
+ build-rpmalloc:
+ internal: true
+ dir: 'lib/'
+ cmds:
+ #build rpmalloc library for linux/mac
+ - cmd: cd vnlib_rpmalloc/ && task && cp build/libvn_rpmalloc{{if eq OS "darwin"}}.dylib{{else}}.so{{end}} ../vnlib_rpmalloc.dll
+ platforms: [ linux, darwin ]
+
+ #for windows just copy the existing dll
+ - cmd: powershell -Command "cp vnlib_rpmalloc/vnlib_rpmalloc.dll vnlib_rpmalloc.dll"
+ platforms: [ windows/amd64 ]
+
+ build-argon2:
+ internal: true
+ dir: 'lib/'
+ cmds:
+ #build argon2 library for linux/mac
+ - cmd: cd argon2/ && task && cp build/libargon2{{if eq OS "darwin"}}.dylib{{else}}.so{{end}} ../argon2.dll
+ platforms: [ linux, darwin ]
+
+ #for windows just copy the existing dll
+ - cmd: powershell -Command "cp argon2/argon2.dll argon2.dll"
+ platforms: [ windows/amd64 ]
+
+ build-compress:
+ internal: true
+ dir: 'lib/'
+ cmds:
+ #build the native compressor library for linux/mac
+ - cmd: cd vnlib_compress/ && task && cp build/libvn_compress{{if eq OS "darwin"}}.dylib{{else}}.so{{end}} ../vnlib_compress.dll
+ platforms: [ linux, darwin ]
+
+ #windows now supports pre-compiled libs
+ - cmd: powershell cp vnlib_compress/vnlib_compress.dll vnlib_compress.dll
+ platforms: [ windows/amd64 ] \ No newline at end of file
diff --git a/ci/taskfile.yaml b/ci/taskfile.yaml
index 8370793..37c732e 100644
--- a/ci/taskfile.yaml
+++ b/ci/taskfile.yaml
@@ -7,11 +7,23 @@ version: "3"
vars:
BUILDS_URL: https://www.vaughnnugent.com/public/resources/software/builds
+ SQLITE_OUT_DIR: "plugins/assets/VNLib.Plugins.Extensions.Loading.Sql.SQLite"
+ WEBSERVER_VERSION: 'a54c9d0da36ee15b5a1b6852b843bc0a40694ce5'
includes:
+ install:
+ taskfile: install.taskfile.yaml
+
plugins:
taskfile: plugins.taskfile.yaml
+ container:
+ dir: container #always run from the container directory
+ taskfile: container/Taskfile.yaml
+ vars:
+ BUILDS_URL: '{{.BUILDS_URL}}'
+ PACKAGE_FILE_NAME: "{{ lower .PROJECT_NAME }}-alpine3.19-oci.tgz" #the name of the output package file
+
tasks:
build:
@@ -19,48 +31,87 @@ tasks:
#clean out dist dir before building
- cmd: powershell -Command "rm -Recurse -Force ./dist"
ignore_error: true
-
- - task: install-plugins
+
+ - cmd: powershell -Command "mkdir lib -Force"
+ ignore_error: true
+
+ - task: compile-config
+
+ - task: plugins:all
+ #remove runtime-shared libs before they are copied
+ - task: prune-plugin-libs
- task: install-webserver
+ #runtimes must be pruned for specific targets
+ - task: prune-runtimes
+
+ #run container build last
+ - task: container:build
+
install-webserver:
cmds:
- #setup env
- - task: create-env
- vars:
- TARGET_OS: win-x64
-
- - task: create-env
- vars:
- TARGET_OS: linux-x64
-
- - task: create-env
+ - cmd : powershell -Command "mkdir webserver -Force"
+ ignore_error: true
+
+ #clone the webserver (it's cross platform when using dotnet command so just grab the linux version)
+ - task: install:install
vars:
- TARGET_OS: osx-x64
-
+ PROJECT_NAME: 'VNLib.Webserver'
+ MODULE_NAME: "VNLib.Webserver"
+ FILE_NAME: "linux-x64-release.tgz"
+ DIR: 'webserver/'
+ VERSION: '{{.WEBSERVER_VERSION}}'
+
+ #remove the executable since its not needed
+ - cmd: cd webserver/ && powershell -Command "rm VNlib.WebServer"
+
+ - task: parallel-envs
+
+ parallel-envs:
+ deps:
+ - task: create-env
+ vars: { TARGET_OS: 'windows-x86_64' }
+ - task: create-env
+ vars: { TARGET_OS: 'linux-x86_64' }
+ - task: create-env
+ vars: { TARGET_OS: 'osx-x86_64' }
+ - task: create-env
+ vars: { TARGET_OS: 'windows-arm' }
+ - task: create-env
+ vars: { TARGET_OS: 'linux-arm' }
+ - task: create-env
+ vars: { TARGET_OS: 'osx-arm' }
postbuild_success:
cmds:
- #make bin dir
- cmd: powershell -Command "mkdir bin -Force"
- ignore_error: true
+ - task: pb-parallel
- - task: pack
- vars:
- TARGET_OS: win-x64
- - task: pack
- vars:
- TARGET_OS: linux-x64
- - task: pack
- vars:
- TARGET_OS: osx-x64
+ #cleanup unnecessary build files that clog up the pipeline
+ - for: [ build, plugins, dist, lib, webserver, config ]
+ cmd: powershell -Command "rm -Recurse '{{.ITEM}}'"
+ ignore_error: true
+ - task: container:postbuild_success
- install-plugins:
+ pb-parallel:
+ internal: true
+ deps:
+ - task: pack
+ vars: { TARGET_OS: 'windows-x86_64' }
+ - task: pack
+ vars: { TARGET_OS: 'linux-x86_64' }
+ - task: pack
+ vars: { TARGET_OS: 'osx-x86_64' }
+ - task: pack
+ vars: { TARGET_OS: 'windows-arm' }
+ - task: pack
+ vars: { TARGET_OS: 'linux-arm' }
+ - task: pack
+ vars: { TARGET_OS: 'osx-arm' }
cmds:
- #add plugins
- - task: plugins:all
-
+ - echo "Packing complete"
+
create-env:
vars:
BUILD_DIR: './build/{{.TARGET_OS}}'
@@ -69,61 +120,87 @@ tasks:
- cmd: powershell -Command "mkdir {{.BUILD_DIR}} -Force"
ignore_error: true
- #copy plugins
- - cmd: powershell -Command "cp -Recurse -Force plugins {{.BUILD_DIR}}"
-
- #copy wwwroot
- - cmd: powershell -Command "cp -Recurse -Force dist {{.BUILD_DIR}}"
-
- #copy libraries
- - cmd: powershell -Command "cp -Recurse -Force lib {{.BUILD_DIR}}"
-
- #copy config
- - cmd: powershell -Command "cp -Recurse -Force config {{.BUILD_DIR}}"
+ #copy build files for target os
+ - for: [ plugins, dist, lib, config, webserver, config ]
+ cmd: powershell -Command "cp -Recurse -Force {{.ITEM}} {{.BUILD_DIR}}"
- - task: get-webserver
- vars:
- TARGET_OS: '{{.TARGET_OS}}'
- BUILD_DIR: '{{.BUILD_DIR}}'
-
- get-webserver:
+ #copy release taskfile and rename it
+ - cmd: powershell -Command "cp -Force release.taskfile.yaml {{.BUILD_DIR}}/Taskfile.yaml"
+
+ pack:
internal: true
cmds:
- - task: install
- vars:
- PROJECT_NAME: 'VNLib.Webserver'
- MODULE_NAME: "VNLib.Webserver"
- FILE_NAME: "{{.TARGET_OS}}-release.tgz"
- BUILD_DIR: '{{.BUILD_DIR}}/webserver'
+ - cmd: powershell -Command "mkdir build/{{.TARGET_OS}}/ -Force"
+ ignore_error: true
+ - cd build/{{.TARGET_OS}} && tar -czf ../../bin/{{.TARGET_OS}}-release.tgz .
- - cmd: powershell -Command "cp -Force ./config/config.json {{.BUILD_DIR}}/config.json"
+ compile-config:
+ internal: false
+ dotenv: ['build.env'] #use the local .env file when compiling config variables
+ cmds:
+ - cmd: powershell mkdir config/ -Force
+ ignore_error: true
+ - cmd: powershell './compile.ps1' -InputDir config-templates/ -OutputDir config/
+ - cmd: powershell cp config-templates/routes.xml config/routes.xml -Force
- install:
- internal: true
- vars:
- DIR: '"{{.BUILD_DIR}}"'
+ prune-plugin-libs:
cmds:
- #make the plugin directory
- - cmd: powershell -Command "mkdir {{.DIR}} -Force"
- ignore_error: true
- - cd {{.DIR}} && powershell "{{.USER_WORKING_DIR}}/install.ps1" -BaseUrl {{.BUILDS_URL}} -ModuleName {{.MODULE_NAME}} -ProjectName {{.PROJECT_NAME}} -FileName {{.FILE_NAME}}
- - cd {{.DIR}} && tar -xzf {{.FILE_NAME}}
- #remove the tar file
- - cd {{.DIR}} && powershell -Command "rm {{.FILE_NAME}}"
+ - for: ['vnlib.utils.dll' , 'vnlib.net.http.dll', 'VNLib.Hashing.Portable.dll', 'VNLib.Plugins.Essentials.dll', 'VNLib.Plugins.dll', 'Serilog.dll', 'Serilog.Sinks.Console.dll', 'Serilog.Sinks.File.dll']
+ cmd: cd plugins && powershell 'Get-ChildItem -Path . -Recurse -File -Filter "{{.ITEM}}" | Remove-Item -Force'
+
+ prune-runtimes:
+ deps:
+ #prune sqlite runtime native libraries that Im not targeting
+ #windows
+ - task: prune-files
+ vars:
+ ITEMS: 'browser-wasm,linux-arm,linux-arm64,linux-armel,linux-mips64,linux-musl-arm,linux-musl-arm64,linux-musl-x64,linux-ppc64le,linux-s390x,linux-x64,linux-x86,maccatalyst-arm64,maccatalyst-x64,osx-arm64,osx-x64,win-arm,win-arm64'
+ DIR: 'windows-x86_64'
+
+ #windows arm
+ - task: prune-files
+ vars:
+ ITEMS: 'browser-wasm,linux-arm,linux-arm64,linux-armel,linux-mips64,linux-musl-arm,linux-musl-arm64,linux-musl-x64,linux-ppc64le,linux-s390x,linux-x64,linux-x86,maccatalyst-arm64,maccatalyst-x64,osx-arm64,osx-x64,win-x86,win-x64'
+ DIR: 'windows-arm'
+
+ #linux x64
+ - task: prune-files
+ vars:
+ ITEMS: 'browser-wasm,linux-arm,linux-arm64,linux-armel,linux-musl-arm,linux-musl-arm64,maccatalyst-arm64,maccatalyst-x64,osx-arm64,osx-x64,win-arm,win-arm64,win-x86,win-x64'
+ DIR: 'linux-x86_64'
- pack:
+ #linux arm
+ - task: prune-files
+ vars:
+ ITEMS: 'browser-wasm,linux-mips64,linux-musl-x64,linux-ppc64le,linux-s390x,linux-x64,linux-x86,maccatalyst-arm64,maccatalyst-x64,osx-arm64,osx-x64,win-arm,win-arm64,win-x86,win-x64'
+ DIR: 'linux-arm'
+
+ #osx x64
+ - task: prune-files
+ vars:
+ ITEMS: 'browser-wasm,linux-arm,linux-arm64,linux-armel,linux-mips64,linux-musl-arm,linux-musl-arm64,linux-musl-x64,linux-ppc64le,linux-s390x,linux-x64,linux-x86,maccatalyst-arm64,win-arm,win-arm64,win-x86,win-x64'
+ DIR: 'osx-x86_64'
+
+ #osx arm
+ - task: prune-files
+ vars:
+ ITEMS: 'browser-wasm,linux-arm,linux-arm64,linux-armel,linux-mips64,linux-musl-arm,linux-musl-arm64,linux-musl-x64,linux-ppc64le,linux-s390x,linux-x64,linux-x86,maccatalyst-x64,osx-x64,win-arm,win-arm64,win-x86,win-x64'
+ DIR: 'osx-arm'
+
+ cmds:
+ - echo "Pruning runtime files"
+
+ prune-files:
internal: true
cmds:
- - cmd: powershell -Command "mkdir build/{{.TARGET_OS}}/ -Force"
- ignore_error: true
- - cd build/{{.TARGET_OS}} && tar -czf ../../bin/{{.TARGET_OS}}-release.tgz .
-
+ - for: { var: ITEMS, split: ','}
+ cmd: cd build/{{.DIR}}/{{.SQLITE_OUT_DIR}}/runtimes && powershell -Command "rm {{.ITEM}} -Recurse -Force"
+ ignore_error: true
clean:
ignore_error: true
cmds:
- - cmd: powershell -Command "rm -Recurse -Force ./build"
- - cmd: powershell -Command "rm -Recurse -Force ./bin"
- - cmd: powershell -Command "rm -Recurse -Force ./dist"
- - cmd: powershell -Command "rm -Recurse -Force ./plugins"
- - cmd: powershell -Command "rm -Recurse -Force ./lib" \ No newline at end of file
+ - for: [ build/, bin/, dist/, plugins/, lib/, webserver/, config/ ]
+ cmd: powershell -Command "rm -Recurse -Force '{{.ITEM}}'"
+
+ - task: container:clean \ No newline at end of file
generated by cgit (git 2.46.0) at 2024-10-01 19:18:54 +0000