From 5dfe3dc0db51624082d20a31767c5a6aa774749d Mon Sep 17 00:00:00 2001 From: vnugent Date: Fri, 15 Sep 2023 12:09:17 -0400 Subject: ci pipeline for complete builds --- ci/config/CMNext.json | 44 ++++++++++ ci/config/Essentials.Accounts.json | 80 ++++++++++++++++++ ci/config/PageRouter.json | 6 ++ ci/config/SessionProvider.json | 42 ++++++++++ ci/config/config.json | 161 +++++++++++++++++++++++++++++++++++++ ci/config/routes.xml | 44 ++++++++++ ci/install.ps1 | 26 ++++++ ci/package.json | 10 +++ ci/plugins.taskfile.yaml | 100 +++++++++++++++++++++++ ci/taskfile.yaml | 118 +++++++++++++++++++++++++++ 10 files changed, 631 insertions(+) create mode 100644 ci/config/CMNext.json create mode 100644 ci/config/Essentials.Accounts.json create mode 100644 ci/config/PageRouter.json create mode 100644 ci/config/SessionProvider.json create mode 100644 ci/config/config.json create mode 100644 ci/config/routes.xml create mode 100644 ci/install.ps1 create mode 100644 ci/package.json create mode 100644 ci/plugins.taskfile.yaml create mode 100644 ci/taskfile.yaml (limited to 'ci') diff --git a/ci/config/CMNext.json b/ci/config/CMNext.json new file mode 100644 index 0000000..1b3c516 --- /dev/null +++ b/ci/config/CMNext.json @@ -0,0 +1,44 @@ +{ + //Enables debug logging + "debug": false, + + "post_endpoint": { + "path": "/blog/posts" + }, + + "channel_endpoint": { + "path": "/blog/channels" + }, + + "content_endpoint": { + "path": "/blog/content", + "max_content_length": 50000000 + }, + + "blog_channels": { + //The index file for storing channel configuration + "index_file_name": "blogs/channels.json" + }, + + //S3 setup with vault secrets + "disabled s3_config": { + "server_address": "", + "access_key": "", + "bucket": "", + "use_ssl": true, + "Region": null + }, + + "disabled ftp_config": { + "url": "", + "username": "", + //Base path within the ftp user's directory + "base_path": "" + }, + + "secrets": { + //Set the vault path to the s3 secret + "s3_secret": "", + "ftp_password": "" + } +} \ No newline at end of file diff --git a/ci/config/Essentials.Accounts.json b/ci/config/Essentials.Accounts.json new file mode 100644 index 0000000..cb2f9d5 --- /dev/null +++ b/ci/config/Essentials.Accounts.json @@ -0,0 +1,80 @@ +{ + "debug": false, + "setup_mode":true, + + //endpoints + + "login_endpoint": { + "path": "/account/login", + "failed_count_timeout_sec": 600, //10 minutes + "failed_count_max": 10 //10 failed attempts in 10 minutes + }, + + "keepalive_endpoint": { + "path": "/account/keepalive", + //Regen token every 15 mins along with cookies + "token_refresh_sec": 600 //15 minutes + }, + + "profile_endpoint": { + "path": "/account/profile" + }, + + "password_endpoint": { + "path": "/account/reset" + }, + + "mfa_endpoint": { + "path": "/account/mfa" + }, + + "logout_endpoint": { + "path": "/account/logout" + }, + + "pki_auth_endpoint": { + "path": "/account/pki", + "jwt_time_dif_sec": 30, + + "max_login_attempts": 10, + "failed_attempt_timeout_sec": 600, + + //Configures the PATCH method to update the user's stored key when logged in + "enable_key_update": true + }, + + //If mfa is defined, configures mfa enpoints and enables mfa logins + "mfa": { + "upgrade_expires_secs": 180, + "nonce_size": 64, + + //Defines totp specific arguments + "totp": { + "digits": 6, + "issuer": "localhost", + "period_secs": 30, + "algorithm": "sha1", + "secret_size": 32, + "window_size": 2 + } + }, + + //Defines the included account provider + "account_security": { + "login_cookie_name": "VNLogin", + "login_cookie_size": 64, + + //Path/domain for all security cookies + "cookie_domain": "", + "cookie_path": "/", + + "status_cookie_name": "li", + + "otp_header_name": "X-Web-Token", + "otp_time_diff_sec": 30, + "otp_key_size": 64, + + "pubkey_cookie_name": "client-id", + "pubkey_signing_key_size": 32 + } +} \ No newline at end of file diff --git a/ci/config/PageRouter.json b/ci/config/PageRouter.json new file mode 100644 index 0000000..420757f --- /dev/null +++ b/ci/config/PageRouter.json @@ -0,0 +1,6 @@ +{ + "debug": false, + "store": { + "route_file": "plugins/routes.xml" + } +} \ No newline at end of file diff --git a/ci/config/SessionProvider.json b/ci/config/SessionProvider.json new file mode 100644 index 0000000..1875e73 --- /dev/null +++ b/ci/config/SessionProvider.json @@ -0,0 +1,42 @@ +{ + + "debug": false, + + //Provider assemblies to load + "provider_assemblies": [ "VNLib.Plugins.Sessions.VNCache.dll" ], + + //Web session provider, valid format for VNCache and also memory sessions + "web": { + //Cache system key prefix + "cache_prefix": "websessions", + //The session cookie name + "cookie_name": "VNSession", + //Size in bytes for generated session ids + "cookie_size": 40, + //time (in seconds) a session is valid for + "valid_for_sec": 3600, + //The maxium number of connections waiting for the cache server responses + "max_waiting_connections": 100 + }, + + //Enable vncache as the providers above rely on the object caching server + "vncache": { + + //Max size (in bytes) of allowed data to be stored in each user's session object + "max_object_size": 8128, + + //Initial nodes to discover from + "initial_nodes": [], + + //Setting this value to true will cause the cache store to load a memory-only instance, without remote backing + "memory_only": true, + + //enable memory cache + "memory_cache": { + "buckets": 20, + "bucket_size": 5000, + "max_age_sec": 600, + "refresh_interval_sec": 60 + } + } +} \ No newline at end of file diff --git a/ci/config/config.json b/ci/config/config.json new file mode 100644 index 0000000..acbb0fd --- /dev/null +++ b/ci/config/config.json @@ -0,0 +1,161 @@ +{ + + //Host application config, config is loaded as a read-only DOM that is available + //to the host and loaded child plugins, all elements are available to plugins via the 'HostConfig' property + + "http": { + //The defaut HTTP version to being requests with (does not support http/2 yet) + "default_version": "HTTP/1.1", + //The maxium size (in bytes) of response messges that will be compressed + "compression_limit": 512000, + //Minium response size (in bytes) to compress + "compression_minimum": 2048, + //The size of the buffer to use when parsing multipart/form data uploads + "multipart_max_buf_size": 20480, + //The maxium ammount of data (in bytes) allows for mulitpart/form data file uploads + "multipart_max_size": 80240, + //Absolute maximum size (in bytes) of the request entity body (exludes headers) + "max_entity_size": 1024000, + //Keepalive ms for HTTP1.1 keepalive connections + "keepalive_ms": 1000000, + //The buffer size to use when parsing headers (also the maxium request header size allowed) + "header_buf_size": 8128, + //The maxium number of headers allowed in an HTTP request message + "max_request_header_count": 50, + //The maxium number of allowed network connections, before 503s will be issued automatically and connections closed + "max_connections": 5000, + //The size in bytes of the buffer to use when writing response messages + "response_buf_size": 65535, + //time (in ms) to wait for a response from an active connection in recv mode, before dropping it + "recv_timeout_ms": 5000, + //Time in ms to wait for the client to accept transport data before terminating the connection + "send_timeout_ms": 60000, + //The size (in bytes) of the buffer used to store all response header data + "response_header_buf_size": 16384 + }, + + //Path to managed compressor library + "compression_lib": null, + + //Maximum ammount of time a request is allowed to be processed (includes loading or waiting for sessions) before operations will be cancelled and a 503 returned + "max_execution_time_ms": 20000, + + //Collection of objects to define hosts+interfaces to build server listeners from + "virtual_hosts": [ + { + //The interface to bind to, you may not mix TLS and non-TLS connections on the same interface + "interface": { + "address": "0.0.0.0", + "port": 8080 + }, + + //The directory path for files served by this endpoint + "path": "dist", + + //The hostname to listen for, "*" as wildcard, and "[system]" as the default hostname for the current machine + "hostname": "*", + + //Or specify an array of hostnames instead, the hostnames array property takes priority over the single hostname property, each must be unique + //"hostnames": [ ], + + //Collection of "trusted" servers to allow proxy header support from + "downstream_servers": [], + + //Specify a list of ip addresses that are allowed to connect to the server, 403 will be returned if connections are not on this list + //whitelist works behind a trusted downstream server that supports X-Forwared-For headers + //"whitelist": [ "127.0.0.1" ], + + //A list of file extensions to deny access to, if a resource is requested and has one of the following extensions, a 404 is returned + "deny_extensions": [ ".env", ".yaml", ".cs" ], + + //The default file extensions to append to a resource that does not have a file extension + "default_files": [ "index.html" ], + + //Denys files to non-browser user-agent connections + "browser_only_files": false, + + //Key-value headers object, some headers are special and are controlled by the vh processor + "headers": { + "X-Content-Type-Options": "nosniff", + "X-Xss-Protection": "1; mode=block", + "X-Frame-Options": "DENY", + "Content-Security-Policy": "default-src 'self' https://cdn.ckeditor.com 'unsafe-inline'; frame-src 'none'; object-src 'none'; referrer no-referrer-when-downgrade; upgrade-insecure-requests; block-all-mixed-content;" + }, + + //Enables cors support for all endpoints and header controls, if false, all endpoints that are send CORS request headers will be forbidden + "enable_cors": true, + + //Allowed cors authoriy domains + "cors_allowed_authority": [ + "localhost:8080" + ], + + //Define a TLS certificate (enables TLS on the interface) + "disabled ssl": { + + //Cert may be pem or pfx (include private key in pfx, or include private key in a pem file) + "cert": "/path/to/cert.pfx|pem", + + //A pem encoded private key, REQUIRED if using a PEM certificate, may be encrypted with a password + "privkey": "/path/to/private_key.pem", + + //An optional password for the ssl private key + "password": "plain-text-password", + + //requires that any client connecting to this host present a valid certificate + "client_cert_required": false + }, + + //A list of error file objects, files are loaded into memory (and watched for changes) and returned when the specified error code occurs + "error_files": [], + + //The default + "cache_default_sec": 864000 + } + ], + + + //Defines the directory where plugin's are to be loaded from + "plugins": { + //Hot-reload creates collectable assemblies that allow full re-load support in the host application, should only be used for development purposes! + "hot_reload": false, + "reload_delay_sec": 2, + "path": "plugins" + }, + + "disabled sys_log": { + //"path": "path/to/syslog/file", + //"template": "serilog template for writing to file", + //"flush_sec": 5, + //"retained_files": 31, + //"file_size_limit": 10485760, + //"interval": "infinite" + }, + + "disabled app_log": { + //"path": "path/to/applog/file", + //"template": "serilog template for writing to file", + //"flush_sec": 5, + //"retained_files": 31, + //"file_size_limit": 10485760, + //"interval": "infinite" + }, + + //Sql for the users database + "sql": { + "db_type": "sqlite", //mysql, mssql(default), sqlite + "source": "cmnext.db" //For sqlite only + + //"hostname": "example.com", + //"username": "cmnext", + //"catalog": "cmnext", + //"min_pool_size": 5, + //"ms_security": false + }, + + //Global secrets object, used by the host and pluings for a specialized secrets + "secrets": { + //"db_password": "" + "passwords": "yourbase64passwordsecret" + } +} diff --git a/ci/config/routes.xml b/ci/config/routes.xml new file mode 100644 index 0000000..49d9553 --- /dev/null +++ b/ci/config/routes.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + * + + + /assets/* + + + + + + + * + + + /* + + + / + + + + + \ No newline at end of file diff --git a/ci/install.ps1 b/ci/install.ps1 new file mode 100644 index 0000000..4c42c18 --- /dev/null +++ b/ci/install.ps1 @@ -0,0 +1,26 @@ +param([String] $BaseUrl, [String] $ModuleName, [String] $ProjectName, [String]$FileName) + +#get the latest file +Invoke-WebRequest "$BaseUrl/$ModuleName/@latest" -OutFile latest.txt +#read the file into a variable +$latest = Get-Content latest.txt + +#download the latest version +Invoke-WebRequest "$BaseUrl/$ModuleName/$latest/$ProjectName/$FileName" -OutFile $FileName + +#download latest sha256 +Invoke-WebRequest "$BaseUrl/$ModuleName/$latest/$ProjectName/$FileName.sha256" -OutFile "$FileName.sha256" + +#verify the file +$hash = (Get-FileHash $FileName -Algorithm SHA256).Hash + +#read the sha256 file +$sha256 = Get-Content "$FileName.sha256" + +#compare the hashes +if ($hash -eq $sha256) { + Write-Host "Hashes match, file is valid" -ForegroundColor Blue +} else { + throw "Hashes do not match, file is invalid" +} + diff --git a/ci/package.json b/ci/package.json new file mode 100644 index 0000000..d899975 --- /dev/null +++ b/ci/package.json @@ -0,0 +1,10 @@ +{ + "name": "Full-Package", + "version": "0.1.0", + "type": "module", + "copyright": "Copyright \u00A9 2023 Vaughn Nugent", + "author": "Vaughn Nugent", + "description": "The project contains a ready-to-run, CMNext admin server installation for Windows, Linux, and MacOS for .NET/6.0 Runtime.", + "repository": "https://github.com/VnUgE/CMNext/tree/master/ci", + "output_dir": "bin" +} \ No newline at end of file diff --git a/ci/plugins.taskfile.yaml b/ci/plugins.taskfile.yaml new file mode 100644 index 0000000..08a3525 --- /dev/null +++ b/ci/plugins.taskfile.yaml @@ -0,0 +1,100 @@ + +version: "3" + +vars: + + +tasks: + + all: + cmds: + - echo "Installing and configuring plugins and UI" + - task: install-accounts + - task: install-sessions + - task: install-vncache + - task: install-router + - task: install-users + - task: build-cmnext + + build-cmnext: + cmds: + #build js libraries + - cd ../lib/admin && npm install && npm run build + - cd ../front-end && npm install && npm run build + #move dist files to web root + - powershell -Command "cp -Recurse -Force ../front-end/dist {{.USER_WORKING_DIR}}" + #move the plugin output + - powershell -Command "cp -Recurse -Force ../back-end/src/bin/release/net6.0/publish/* ./plugins/CMNext" + #move cmnext config file + - powershell -Command "cp -Force config/CMNext.json ./plugins/CMNext/CMNext.json" + + install-accounts: + cmds: + #install accounts plugin + - task: install-plugin + vars: + NAME: 'Essentials.Accounts' + PROJECT_NAME: 'VNLib.Plugins.Essentials.Accounts' + MODULE_NAME: "Plugins.Essentials" + FILE_NAME: "release.tgz" + DIR: './plugins/Essentials.Accounts' + + install-router: + cmds: + #install router plugin + - task: install-plugin + vars: + NAME: 'PageRouter' + PROJECT_NAME: 'VNLib.Plugins.Essentials.Content.Routing' + MODULE_NAME: "Plugins.Essentials" + FILE_NAME: "release.tgz" + DIR: './plugins/PageRouter' + + install-sessions: + cmds: + #install sessions plugin + - task: install-plugin + vars: + NAME: 'SessionProvider' + PROJECT_NAME: 'SessionProvider' + MODULE_NAME: "VNLib.Plugins.Sessions" + FILE_NAME: "release.tgz" + DIR: './plugins/SessionProvider' + + install-users: + cmds: + #install users plugin + - task: install-plugin + vars: + NAME: 'VNLib.Plugins.Essentials.Users' + PROJECT_NAME: 'VNLib.Plugins.Essentials.Users' + MODULE_NAME: "VNLib.Plugins.Essentials.Users" + FILE_NAME: "release.tgz" + DIR: './plugins/assets/VNLib.Plugins.Essentials.Users' + + #move routes xml files + - cmd: powershell -Command "cp -Force config/routes.xml ./plugins/routes.xml" + + install-vncache: + cmds: + #install sessions plugin + - task: install-plugin + vars: + NAME: 'VNLib.Plugins.Sessions.VNCache' + PROJECT_NAME: 'VNLib.Plugins.Sessions.VNCache' + MODULE_NAME: "VNLib.Plugins.Sessions" + FILE_NAME: "release.tgz" + DIR: './plugins/assets/VNLib.Plugins.Sessions.VNCache' + + install-plugin: + cmds: + #make the plugin directory + - cmd: powershell -Command "mkdir {{.DIR}} -Force" + ignore_error: true + - cd {{.DIR}} && powershell "{{.USER_WORKING_DIR}}/install.ps1" -BaseUrl {{.BUILDS_URL}} -ModuleName {{.MODULE_NAME}} -ProjectName {{.PROJECT_NAME}} -FileName {{.FILE_NAME}} + - cd {{.DIR}} && tar -xzf {{.FILE_NAME}} + #remove the archive file + - cd {{.DIR}} && powershell -Command "rm {{.FILE_NAME}}" + #copy config file from config dir + - cmd: powershell -Command "cp -Force ./config/{{.NAME}}.json {{.DIR}}/{{.NAME}}.json" + ignore_error: true \ No newline at end of file diff --git a/ci/taskfile.yaml b/ci/taskfile.yaml new file mode 100644 index 0000000..5028419 --- /dev/null +++ b/ci/taskfile.yaml @@ -0,0 +1,118 @@ + +version: "3" + +vars: + BUILDS_URL: https://www.vaughnnugent.com/public/resources/software/builds + + +includes: + plugins: + taskfile: plugins.taskfile.yaml + +tasks: + + build: + cmds: + - task: install-plugins + - task: install-webserver + + install-webserver: + cmds: + #setup env + - task: create-env + vars: + TARGET_OS: win-x64 + + - task: create-env + vars: + TARGET_OS: linux-x64 + + - task: create-env + vars: + TARGET_OS: osx-x64 + + + postbuild_success: + cmds: + #make bin dir + - cmd: powershell -Command "mkdir bin -Force" + ignore_error: true + + - task: pack + vars: + TARGET_OS: win-x64 + - task: pack + vars: + TARGET_OS: linux-x64 + - task: pack + vars: + TARGET_OS: osx-x64 + + + install-plugins: + cmds: + #add plugins + - task: plugins:all + + create-env: + vars: + BUILD_DIR: './build/{{.TARGET_OS}}' + cmds: + #create dir for env + - cmd: powershell -Command "mkdir {{.BUILD_DIR}} -Force" + ignore_error: true + + #copy plugins + - cmd: powershell -Command "cp -Recurse -Force plugins {{.BUILD_DIR}}" + + #copy wwwroot + - cmd: powershell -Command "cp -Recurse -Force dist {{.BUILD_DIR}}" + + - task: get-webserver + vars: + TARGET_OS: '{{.TARGET_OS}}' + BUILD_DIR: '{{.BUILD_DIR}}' + + + get-webserver: + internal: true + cmds: + #install accounts plugin + - task: install + vars: + NAME: 'webserver' + PROJECT_NAME: 'VNLib.Webserver' + MODULE_NAME: "VNLib.Webserver" + FILE_NAME: "{{.TARGET_OS}}-release.tgz" + BUILD_DIR: '{{.BUILD_DIR}}' + + - cmd: powershell -Command "cp -Force ./config/config.json {{.BUILD_DIR}}/config.json" + + install: + internal: true + vars: + DIR: '"{{.BUILD_DIR}}"/"{{.NAME}}"' + cmds: + #make the plugin directory + - cmd: powershell -Command "mkdir {{.DIR}} -Force" + ignore_error: true + - cd {{.DIR}} && powershell "{{.USER_WORKING_DIR}}/install.ps1" -BaseUrl {{.BUILDS_URL}} -ModuleName {{.MODULE_NAME}} -ProjectName {{.PROJECT_NAME}} -FileName {{.FILE_NAME}} + - cd {{.DIR}} && tar -xzf {{.FILE_NAME}} + #remove the tar file + - cd {{.DIR}} && powershell -Command "rm {{.FILE_NAME}}" + + pack: + internal: true + cmds: + - cmd: powershell -Command "mkdir build/{{.TARGET_OS}}/ -Force" + ignore_error: true + - cd build/{{.TARGET_OS}} && tar -czf ../../bin/{{.TARGET_OS}}-release.tgz . + + + clean: + ignore_error: true + cmds: + - cmd: powershell -Command "rm -Recurse -Force ./build" + - cmd: powershell -Command "rm -Recurse -Force ./bin" + - cmd: powershell -Command "rm -Recurse -Force ./dist" + - cmd: powershell -Command "rm -Recurse -Force ./plugins" \ No newline at end of file -- cgit