diff options
Diffstat (limited to 'ci')
| -rw-r--r-- | ci/config/SimpleBookmark.json | 13 | ||||
| -rw-r--r-- | ci/config/config.json | 34 | ||||
| -rw-r--r-- | ci/container/Dockerfile | 2 | ||||
| -rw-r--r-- | ci/container/Taskfile.yaml | 5 | ||||
| -rw-r--r-- | ci/container/config-templates/SimpleBookmark-template.json | 15 | ||||
| -rw-r--r-- | ci/container/docker-compose.yaml | 4 | ||||
| -rw-r--r-- | ci/container/run.sh | 2 | ||||
| -rw-r--r-- | ci/plugins.taskfile.yaml | 39 | ||||
| -rw-r--r-- | ci/release.taskfile.yaml | 111 | ||||
| -rw-r--r-- | ci/setup.sh | 50 | ||||
| -rw-r--r-- | ci/taskfile.yaml | 89 |
11 files changed, 248 insertions, 116 deletions
diff --git a/ci/config/SimpleBookmark.json b/ci/config/SimpleBookmark.json index 6cb1b93..64be3c1 100644 --- a/ci/config/SimpleBookmark.json +++ b/ci/config/SimpleBookmark.json @@ -14,6 +14,19 @@ } }, + //System website lookup endpoint (aka curl) + "curl": { + "path": "/api/lookup", + "exe_path": "curl", //Path to the curl executable + "extra_args": [ + "--globoff", //Disables unsafe url globbing + "--no-keepalive", //Disables keepalive, uneeded for a single lookup request + "--max-filesize", "100K", //Max file size 100K + "--max-redirs", "5", //Max redirects 5 + "--location", //Follow redirects + ] + }, + "registration": { "path": "/api/register", //Path for the registration endpoint "token_lifetime_mins": 360, //Token lifetime in minutes diff --git a/ci/config/config.json b/ci/config/config.json index e4b33e8..61293b6 100644 --- a/ci/config/config.json +++ b/ci/config/config.json @@ -41,7 +41,7 @@ //Setup the native lib "vnlib.net.compression": { - "lib_path": "lib/vnlib_compress/build/<os-dependent-lib-path>", + "lib_path": "lib/vnlib_compress.dll", "level": 1 }, @@ -95,16 +95,16 @@ //"cors_allowed_authority": [ "localhost:8080" ], //Define a TLS certificate (enables TLS on the interface) - "disabled ssl": { + "ssl": { //Cert may be pem or pfx (include private key in pfx, or include private key in a pem file) - "cert": "/path/to/cert.pfx|pem", + "cert": "ssl/cert.pem", //A pem encoded private key, REQUIRED if using a PEM certificate, may be encrypted with a password - "privkey": "/path/to/private_key.pem", + "privkey": "ssl/key.pem", //An optional password for the ssl private key - "password": "plain-text-password", + //"password": "plain-text-password", //requires that any client connecting to this host present a valid certificate "client_cert_required": false @@ -129,29 +129,29 @@ "assets": "plugins/assets/" }, - "disabled sys_log": { - //"path": "path/to/syslog/file", + "sys_log": { + "path": "data/logs/syslog.txt", //"template": "serilog template for writing to file", - //"flush_sec": 5, - //"retained_files": 31, - //"file_size_limit": 10485760, - //"interval": "infinite" + "flush_sec": 5, + "retained_files": 10, + "file_size_limit": 10485760, + "interval": "infinite" }, "disabled app_log": { - //"path": "path/to/applog/file", + "path": "data/logs/applog.txt", //"template": "serilog template for writing to file", - //"flush_sec": 5, - //"retained_files": 31, - //"file_size_limit": 10485760, - //"interval": "infinite" + "flush_sec": 5, + "retained_files": 10, + "file_size_limit": 10485760, + "interval": "infinite" }, //Sql for the users database "sql": { "debug": false, "provider": "VNLib.Plugins.Extensions.Sql.SQLite.dll", - "source": "simple-bookmark.db" //For sqlite only + "source": "data/simple-bookmark.db" //For sqlite only }, //caching should be setup globally after VNCache #78a47dd diff --git a/ci/container/Dockerfile b/ci/container/Dockerfile index 6804a6e..f5ac798 100644 --- a/ci/container/Dockerfile +++ b/ci/container/Dockerfile @@ -32,7 +32,7 @@ COPY app/ /app #pull compiled libs from build container COPY --from=native-cont /build/out /app/lib -RUN apk update && apk add --no-cache gettext icu-libs dumb-init +RUN apk update && apk add --no-cache gettext icu-libs dumb-init curl #workdir WORKDIR /app diff --git a/ci/container/Taskfile.yaml b/ci/container/Taskfile.yaml index 97548dc..557e48d 100644 --- a/ci/container/Taskfile.yaml +++ b/ci/container/Taskfile.yaml @@ -41,13 +41,12 @@ tasks: cmds: # clean up the run.sh script to remove windows line endings in my wsl default instance - cmd: wsl dos2unix ./run.sh - platform: [ win-x64 ] + platforms: [ windows/amd64 ] #init build image - task: setup-container-image #remove the default config file as it's not needed in the container - - powershell -Command "rm -Force build/app/config.json" - powershell -Command "rm -Force -Recurse build/app/config/" - task: prune-sql-runtimes @@ -84,7 +83,7 @@ tasks: #make build directory - powershell -Command "mkdir build, build/app, build/app/config-templates/, build/app/static/ -Force" #copy the existing linux-x64 build to the build folder, this will be the container base - - powershell -Command "cp -Recurse -Force ../build/linux-x64/* build/app/" + - powershell -Command "cp -Recurse -Force ../build/linux-x86_64/* build/app/" #copy local scripts and config data into the build folder - powershell -Command "cp -Force run.sh, Taskfile.yaml build/app/" - powershell -Command "cp -Force Dockerfile, docker-compose.yaml build/" diff --git a/ci/container/config-templates/SimpleBookmark-template.json b/ci/container/config-templates/SimpleBookmark-template.json index a64a10a..c2bf780 100644 --- a/ci/container/config-templates/SimpleBookmark-template.json +++ b/ci/container/config-templates/SimpleBookmark-template.json @@ -14,6 +14,21 @@ } }, + //System website lookup endpoint (aka curl) + "curl": { + "path": "/api/lookup", + "exe_path": "curl", //Path to the curl executable + "extra_args": [ + "--globoff", //Disables unsafe url globbing + "--no-keepalive", //Disables keepalive, uneeded for a single lookup request + "--max-filesize", + "100K", //Max file size 100K + "--max-redirs", + "5", //Max redirects 5 + "--location", //Follow redirects + ] + }, + "registration": { "path": "/api/register", //Path for the registration endpoint "token_lifetime_mins": ${REG_TOKEN_DURATION_MIN}, //Token lifetime in minutes diff --git a/ci/container/docker-compose.yaml b/ci/container/docker-compose.yaml index eb28055..03eb815 100644 --- a/ci/container/docker-compose.yaml +++ b/ci/container/docker-compose.yaml @@ -27,6 +27,7 @@ services: CACHE_ASM_PATH: "VNLib.Data.Caching.Providers.VNCache.dll" MEMCACHE_ONLY: "true" REDIS_CONNECTION_STRING: "" + #at least one node required if MEMCACHE_ONLY is false VNCACHE_INITIAL_NODES: "[]" #ACCOUNTS MAX_LOGIN_ATTEMPS: "10" @@ -35,6 +36,7 @@ services: PASSWORD_PEPPER: "" DATABASE_PASSWORD: "" REDIS_PASSWORD: "" + #if MEMCACHE_ONLY is false, then the following keys are required to connect to a VNCACHE cluster VNCACHE_CLIENT_PRIVATE_KEY: "" VNCACHE_CACHE_PUBLIC_KEY: "" @@ -42,5 +44,5 @@ services: HTTP_DOWNSTREAM_SERVERS: '[]' #SSL_JSON: '{"cert": "ssl/cert.pem", "privkey":"ssl/priv.pem"}' - SERVER_ARGS: "--input-off" + SERVER_ARGS: "" diff --git a/ci/container/run.sh b/ci/container/run.sh index 2c2636c..c780929 100644 --- a/ci/container/run.sh +++ b/ci/container/run.sh @@ -12,4 +12,4 @@ done cp usr/assets/* plugins/assets/ -rf #start the server -dotnet webserver/VNLib.WebServer.dll --config config/config.json $SERVER_ARGS
\ No newline at end of file +dotnet webserver/VNLib.WebServer.dll --config config/config.json --input-off $SERVER_ARGS
\ No newline at end of file diff --git a/ci/plugins.taskfile.yaml b/ci/plugins.taskfile.yaml index f39121d..cab3d53 100644 --- a/ci/plugins.taskfile.yaml +++ b/ci/plugins.taskfile.yaml @@ -15,18 +15,18 @@ tasks: all: deps: - - install-accounts - - install-router - - install-sessions - - install-vncache - - install-vncache-sessions - - install-users - - install-sqlite - - install-argon2-lib - - install-compression - - install-compressor-lib - + - install-rpmalloc + - install-compressor-lib + - install-argon2-lib + - install-compression + - install-sqlite cmds: + - task: install-accounts + - task: install-router + - task: install-sessions + - task: install-vncache + - task: install-vncache-sessions + - task: install-users - echo "Installing and configuring plugins and UI" - task: build-bookmarks @@ -156,3 +156,20 @@ tasks: cmd: powershell -Command "rm ./lib/argon2/{{.ITEM}} -Recurse" ignore_error: true + install-rpmalloc: + cmds: + #install the rpmalloc source code package for Linux and Mac + - task: install:install + vars: + PROJECT_NAME: 'vnlib_rpmalloc' + MODULE_NAME: "VNLib.Core" + FILE_NAME: "src.tgz" + DIR: './lib/vnlib_rpmalloc' + + #install the rpmalloc binary for Windows + - task: install:install + vars: + PROJECT_NAME: 'vnlib_rpmalloc' + MODULE_NAME: "VNLib.Core" + FILE_NAME: "win-x64-release-vnlib_rpmalloc.tgz" + DIR: './lib/vnlib_rpmalloc' diff --git a/ci/release.taskfile.yaml b/ci/release.taskfile.yaml new file mode 100644 index 0000000..f6fdf62 --- /dev/null +++ b/ci/release.taskfile.yaml @@ -0,0 +1,111 @@ +# https://taskfile.dev + +#Inlcuded taskfile for object cache server that is used to produce +#ci builds for standalone caching servers + +version: "3" + +vars: + SSL_DIR: "ssl" + DATA_DIR: "data" + DEFAULT_EC_CURVE: "secp384r1" + +tasks: + default: + desc: "Runs the Simple-Bookmark server" + cmds: + - task: run + + run: + desc: "Runs the Simple-Bookmark server" + silent: true + env: + #libraries intentionally do not have extensions, for cross-platform compatibility, the server will load them regardless + VNLIB_SHARED_HEAP_FILE_PATH: lib/vnlib_rpmalloc.dll + VNLIB_ARGON2_DLL_PATH: lib/argon2.dll + cmds: + - cmd: dotnet webserver/VNLib.WebServer.dll --config config/config.json {{.CLI_ARGS}} + + setup-apt: + desc: "Performs initial setup on Debian apt amd64 based machines" + silent: true + cmds: + - apt update + - apt install -y dotnet-runtime-8.0 gcc cmake curl + - task: setup + - echo "Setup complete" + + setup-dnf: + desc: "Performs initial setup on Fedora/Redhat amd (dnf) based machines" + silent: true + cmds: + - dnf update + - dnf install -y dotnet-runtime-8.0 gcc cmake curl + - task: setup + - echo "Setup complete" + + setup-apk: + desc: "Performs initial setup using the APK package manager for amd64 based machines" + silent: true + cmds: + - apk update + - apk add --no-cache dotnet8-runtime build-base cmake curl + - task: setup + - echo "Setup complete" + + setup: + desc: "Performs platform agnostic setup tasks without installing tools (no sudo needed)" + cmds: + #build rpmalloc lib + - task: build-rpmalloc + - task: build-argon2 + - task: build-compress + + #setup ssl dir + - cmd: mkdir ssl/ + platforms: [ linux, darwin ] + ignore_error: true + - cmd: powershell -Command "mkdir ssl/" + platforms: [ windows/amd64 ] + ignore_error: true + + create-cert: + desc: "Genereates a new self-signed TLS certificate" + cmds: + - openssl req -new -x509 -days 365 -keyout {{.SSL_DIR}}/key.pem -out {{.SSL_DIR}}/cert.pem -newkey ec -pkeyopt ec_paramgen_curve:{{.DEFAULT_EC_CURVE}} --nodes + + build-rpmalloc: + internal: true + dir: 'lib/' + cmds: + #build rpmalloc library for linux/mac + - cmd: cd vnlib_rpmalloc/ && task && cp build/libvn_rpmalloc{{if eq OS "darwin"}}.dylib{{else}}.so{{end}} ../vnlib_rpmalloc.dll + platforms: [ linux, darwin ] + + #for windows just copy the existing dll + - cmd: powershell -Command "cp vnlib_rpmalloc/vnlib_rpmalloc.dll vnlib_rpmalloc.dll" + platforms: [ windows/amd64 ] + + build-argon2: + internal: true + dir: 'lib/' + cmds: + #build argon2 library for linux/mac + - cmd: cd argon2/ && task && cp build/libargon2{{if eq OS "darwin"}}.dylib{{else}}.so{{end}} ../argon2.dll + platforms: [ linux, darwin ] + + #for windows just copy the existing dll + - cmd: powershell -Command "cp argon2/argon2.dll argon2.dll" + platforms: [ windows/amd64 ] + + build-compress: + internal: true + dir: 'lib/' + cmds: + - cd vnlib_compress/ && task + #build the native compressor library for linux/mac + - cmd: cd vnlib_compress/ && cp build/libvn_compress{{if eq OS "darwin"}}.dylib{{else}}.so{{end}} ../vnlib_compress.dll + platforms: [ linux, darwin ] + + - cmd: powershell -Command "cp vnlib_compress/build/Release/vnlib_compress.dll vnlib_compress.dll" + platforms: [ windows/amd64 ]
\ No newline at end of file diff --git a/ci/setup.sh b/ci/setup.sh deleted file mode 100644 index 0cc153b..0000000 --- a/ci/setup.sh +++ /dev/null @@ -1,50 +0,0 @@ -#! /bin/bash - -echo "Testing for go-task" -#test for platform tools -if ! command -v task &> /dev/null -then - echo "You must install go-task: from https://taskfile.dev/installation/" - exit 1 -fi - -echo "Testing for cmake" -#test for cmake -if ! command -v cmake &> /dev/null -then - echo "You must have cmake installed globally" - exit 1 -fi - -echo "Testing for GNUMake" -#test for make -if ! command -v make &> /dev/null -then - echo "You must have GNUMake installed globally" - exit 1 -fi - -echo "Testing for git" -#test for git -if ! command -v git &> /dev/null -then - echo "You must have git installed globally" - exit 1 -fi - -#build the argon2 native library -pushd argon2 > /dev/null -echo "Building Argon2 native library" -make -argon2_path=$(find "$(pwd)" -iname "libargon2.so.*") - -echo "Add the following environment variable" -echo VNLIB_ARGON2_DLL_PATH=$argon2_path -popd > /dev/null - -#build the vnlib_compress native library -pushd vnlib_compress > /dev/null -echo "Building vnlib_compress native library" -task -echo "Finished building vnlib_compress" -popd > /dev/null
\ No newline at end of file diff --git a/ci/taskfile.yaml b/ci/taskfile.yaml index a27b1ac..43e11a8 100644 --- a/ci/taskfile.yaml +++ b/ci/taskfile.yaml @@ -7,6 +7,7 @@ version: "3" vars: BUILDS_URL: https://www.vaughnnugent.com/public/resources/software/builds + SQLITE_OUT_DIR: "plugins/assets/VNLib.Plugins.Extensions.Loading.Sql.SQLite" includes: install: @@ -30,22 +31,33 @@ tasks: - cmd: powershell -Command "rm -Recurse -Force ./dist" ignore_error: true - #copy setup script for linux - cmd: powershell -Command "mkdir lib -Force" ignore_error: true - - cmd: wsl dos2unix ./setup.sh #convert the setup script to unix line endings for linux - platform: [ win-x64 ] - - powershell -Command "cp setup.sh lib/ -Force" - - task: install-plugins + - task: plugins:all - task: install-webserver + - task: prune-runtimes #run container build last - task: container:build install-webserver: cmds: - - for: [ win-x64, linux-x64, osx-x64 ] + - cmd : powershell -Command "mkdir webserver -Force" + ignore_error: true + + #clone the webserver (it's cross platform when using dotnet command) + - task: install:install + vars: + PROJECT_NAME: 'VNLib.Webserver' + MODULE_NAME: "VNLib.Webserver" + FILE_NAME: "linux-x64-release.tgz" + DIR: 'webserver/' + + #remove the executable since its not needed + - cmd: cd webserver/ && powershell -Command "rm VNlib.WebServer" + + - for: [ windows-x86_64, linux-x86_64, osx-x86_64, windows-arm, linux-arm, osx-arm ] task: create-env vars: TARGET_OS: '{{.ITEM}}' @@ -55,7 +67,7 @@ tasks: #make bin dir - cmd: powershell -Command "mkdir bin -Force" ignore_error: true - - for: [ win-x64, linux-x64, osx-x64 ] + - for: [ windows-x86_64, linux-x86_64, osx-x86_64, windows-arm, linux-arm, osx-arm ] task: pack vars: TARGET_OS: '{{.ITEM}}' @@ -66,11 +78,6 @@ tasks: ignore_error: true - task: container:postbuild_success - - install-plugins: - cmds: - #add plugins - - task: plugins:all build-container: cmds: @@ -84,39 +91,57 @@ tasks: - cmd: powershell -Command "mkdir {{.BUILD_DIR}} -Force" ignore_error: true - #copy build files - - for: [ plugins, dist, lib, config ] + #copy build files for target os + - for: [ plugins, dist, lib, config, webserver ] cmd: powershell -Command "cp -Recurse -Force {{.ITEM}} {{.BUILD_DIR}}" - - task: get-webserver - vars: - TARGET_OS: '{{.TARGET_OS}}' - BUILD_DIR: '{{.BUILD_DIR}}' - - get-webserver: - internal: true - cmds: - - task: install:install - vars: - PROJECT_NAME: 'VNLib.Webserver' - MODULE_NAME: "VNLib.Webserver" - FILE_NAME: "{{.TARGET_OS}}-release.tgz" - DIR: '{{.BUILD_DIR}}/webserver' - - - cmd: powershell -Command "cp -Force ./config/config.json {{.BUILD_DIR}}/config.json" + #copy release taskfile and rename it + - cmd: powershell -Command "cp -Force release.taskfile.yaml {{.BUILD_DIR}}/Taskfile.yaml" pack: internal: true cmds: - cmd: powershell -Command "mkdir build/{{.TARGET_OS}}/ -Force" ignore_error: true - - cd build/{{.TARGET_OS}} && tar -czf ../../bin/{{.TARGET_OS}}-release.tgz . + - cd build/{{.TARGET_OS}} && tar -czf ../../bin/{{.TARGET_OS}}-release.tgz . + + prune-runtimes: + cmds: + #prune sqlite runtime native libraries that Im not targeting + #windows + - for: ['browser-wasm', 'linux-arm', 'linux-arm64', 'linux-armel', 'linux-mips64', 'linux-musl-arm', 'linux-musl-arm64', 'linux-musl-x64', 'linux-ppc64le', 'linux-s390x', 'linux-x64', 'linux-x86', 'maccatalyst-arm64', 'maccatalyst-x64', 'osx-arm64', 'osx-x64', 'win-arm', 'win-arm64' ] + cmd: cd build/windows-x86_64/{{.SQLITE_OUT_DIR}}/runtimes && powershell -Command "rm {{.ITEM}} -Recurse -Force" + ignore_error: true + + #windows arm + - for: ['browser-wasm', 'linux-arm', 'linux-arm64', 'linux-armel', 'linux-mips64', 'linux-musl-arm', 'linux-musl-arm64', 'linux-musl-x64', 'linux-ppc64le', 'linux-s390x', 'linux-x64', 'linux-x86', 'maccatalyst-arm64', 'maccatalyst-x64', 'osx-arm64', 'osx-x64', 'win-x86', 'win-x64' ] + cmd: cd build/windows-arm/{{.SQLITE_OUT_DIR}}/runtimes && powershell -Command "rm {{.ITEM}} -Recurse -Force" + ignore_error: true + + #linux x64 + - for: ['browser-wasm', 'linux-arm', 'linux-arm64', 'linux-armel', 'linux-musl-arm', 'linux-musl-arm64', 'maccatalyst-arm64', 'maccatalyst-x64', 'osx-arm64', 'osx-x64', 'win-arm', 'win-arm64', 'win-x86', 'win-x64' ] + cmd: cd build/linux-x86_64/{{.SQLITE_OUT_DIR}}/runtimes && powershell -Command "rm {{.ITEM}} -Recurse -Force" + ignore_error: true + #linux arm + - for: ['browser-wasm', 'linux-mips64', 'linux-musl-x64', 'linux-ppc64le', 'linux-s390x', 'linux-x64', 'linux-x86', 'maccatalyst-arm64', 'maccatalyst-x64', 'osx-arm64', 'osx-x64', 'win-arm', 'win-arm64', 'win-x86', 'win-x64' ] + cmd: cd build/linux-arm/{{.SQLITE_OUT_DIR}}/runtimes && powershell -Command "rm {{.ITEM}} -Recurse -Force" + ignore_error: true + + #osx x64 + - for: ['browser-wasm', 'linux-arm', 'linux-arm64', 'linux-armel', 'linux-mips64', 'linux-musl-arm', 'linux-musl-arm64', 'linux-musl-x64', 'linux-ppc64le', 'linux-s390x', 'linux-x64', 'linux-x86', 'maccatalyst-arm64', 'win-arm', 'win-arm64', 'win-x86', 'win-x64' ] + cmd: cd build/osx-x86_64/{{.SQLITE_OUT_DIR}}/runtimes && powershell -Command "rm {{.ITEM}} -Recurse -Force" + ignore_error: true + + #osx arm + - for: ['browser-wasm', 'linux-arm', 'linux-arm64', 'linux-armel', 'linux-mips64', 'linux-musl-arm', 'linux-musl-arm64', 'linux-musl-x64', 'linux-ppc64le', 'linux-s390x', 'linux-x64', 'linux-x86', 'maccatalyst-x64', 'osx-x64', 'win-arm', 'win-arm64', 'win-x86', 'win-x64' ] + cmd: cd build/osx-arm/{{.SQLITE_OUT_DIR}}/runtimes && powershell -Command "rm {{.ITEM}} -Recurse -Force" + ignore_error: true clean: ignore_error: true cmds: - - for: [ ./build, ./bin, ./dist, ./plugins, ./lib ] + - for: [ build/, bin/, dist/, plugins/, lib/, webserver/ ] cmd: powershell -Command "rm -Recurse -Force '{{.ITEM}}'" - task: container:clean
\ No newline at end of file |