diff options
Diffstat (limited to 'plugins/ObjectCacheServer')
14 files changed, 848 insertions, 13 deletions
diff --git a/plugins/ObjectCacheServer/Taskfile.yaml b/plugins/ObjectCacheServer/Taskfile.yaml new file mode 100644 index 0000000..a339359 --- /dev/null +++ b/plugins/ObjectCacheServer/Taskfile.yaml @@ -0,0 +1,88 @@ +# https://taskfile.dev + +#Called by the vnbuild system to produce builds for my website +#https://www.vaughnnugent.com/resources/software + +#This taskfile is called from the root of a project that is being built +#and the purpose of this taskfile is to package up the output of a build +#from the solution file, and package it up into a tgz files for distribution + +version: '3' + +vars: + TARGET: '{{.USER_WORKING_DIR}}/bin' + RELEASE_DIR: "./bin/release/{{.TARGET_FRAMEWORK}}/publish" + SOURCE_OUT: "{{.USER_WORKING_DIR}}/bin/source" + +includes: + ci: + taskfile: server/taskfile.yaml + dir: server/ #must execute from the server directory + optional: true + +tasks: + + #called by ci to build the output + build: + cmds: + - task: ci:build + + #when build succeeds, archive the output into a tgz + postbuild_success: + dir: '{{.USER_WORKING_DIR}}' + cmds: + #pack up source code + - task: packsource + + #run post in debug mode + - task: postbuild + vars: { BUILD_MODE: debug } + + #remove uncessary files from the release dir + - powershell -Command "Get-ChildItem -Recurse '{{.RELEASE_DIR}}/' -Include *.pdb,*.xml | Remove-Item" + + #run post in release mode + - task: postbuild + vars: { BUILD_MODE: release } + + - task: ci:postbuild_success + + + postbuild_failed: + dir: '{{.USER_WORKING_DIR}}' + cmds: + - echo "postbuild failed {{.PROJECT_NAME}}" + + + postbuild: + dir: '{{.USER_WORKING_DIR}}' + internal: true + vars: + #the build output directory + BUILD_OUT: "{{.USER_WORKING_DIR}}/bin/{{.BUILD_MODE}}/{{.TARGET_FRAMEWORK}}/publish" + + cmds: + + #copy license and readme to target + - cd .. && powershell -Command "Copy-Item -Path ./build.readme.md -Destination '{{.BUILD_OUT}}/readme.md'" + + #tar outputs + - cd "{{.BUILD_OUT}}" && tar -czf "{{.TARGET}}/{{.BUILD_MODE}}.tgz" . + + packsource: + dir: '{{.USER_WORKING_DIR}}' + internal: true + cmds: + #copy source code to target + - powershell -Command "Get-ChildItem -Include *.cs,*.csproj -Recurse | Where { \$_.FullName -notlike '*\obj\*' -and \$_.FullName -notlike '*\bin\*' } | Resolve-Path -Relative | tar --files-from - -czf '{{.TARGET}}/src.tgz'" + + +#Remove the output dirs on clean + clean: + dir: '{{.USER_WORKING_DIR}}' + cmds: + - for: [ 'bin/', 'obj/' ] + cmd: powershell Remove-Item -Recurse '{{.ITEM}}' + ignore_error: true + + - task: ci:clean diff --git a/plugins/ObjectCacheServer/server/config/config.json b/plugins/ObjectCacheServer/server/config/config.json new file mode 100644 index 0000000..1f8a382 --- /dev/null +++ b/plugins/ObjectCacheServer/server/config/config.json @@ -0,0 +1,104 @@ +{ + + //Host application config, config is loaded as a read-only DOM that is available + //to the host and loaded child plugins, all elements are available to plugins via the 'HostConfig' property + + "http": { + //The defaut HTTP version to being requests with (does not support http/2 yet) + "default_version": "HTTP/1.1", + //The maxium size (in bytes) of response messges that will be compressed + "compression_limit": 10000, + //Minium response size (in bytes) to compress + "compression_minimum": 2048, + //The size of the buffer to use when parsing multipart/form data uploads + "multipart_max_buf_size": 1024, + //The maxium ammount of data (in bytes) allows for mulitpart/form data file uploads + "multipart_max_size": 0, + //Absolute maximum size (in bytes) of the request entity body (exludes headers) + "max_entity_size": 10240, + //Keepalive ms for HTTP1.1 keepalive connections + "keepalive_ms": 100000, + //The buffer size to use when parsing headers (also the maxium request header size allowed) + "header_buf_size": 8128, + //The maxium number of headers allowed in an HTTP request message + "max_request_header_count": 50, + //The maxium number of allowed network connections, before 503s will be issued automatically and connections closed + "max_connections": 5000, + //The size in bytes of the buffer to use when writing response messages + "response_buf_size": 4096, + //time (in ms) to wait for a response from an active connection in recv mode, before dropping it + "recv_timeout_ms": 5000, + //Time in ms to wait for the client to accept transport data before terminating the connection + "send_timeout_ms": 60000, + //The size (in bytes) of the buffer used to store all response header data + "response_header_buf_size": 16384, + //Max number of file uploads allowed per request + "max_uploads_per_request": 1 + }, + + //Maxium ammount of time a request is allowed to be processed (includes loading or waiting for sessions) before operations will be cancelled and a 503 returned + "max_execution_time_ms": 20000, + + "virtual_hosts": [ + { + "interface": { + "address": "0.0.0.0", + "port": 2557 + }, + + //Collection of "trusted" servers to allow proxy header support from + "downstream_servers": [], + + //The hostname to listen for, "*" as wildcard, and "[system]" as the default hostname for the current machine + "hostname": "*", + "path": "root/", //Point to some place we can read nothing from + + "deny_extensions": [ ], + "default_files": [ ], + "error_files": [], + "cache_default_sec": 864000, + + "DISABLED ssl": {} + } + ], + + + //Defines the directory where plugin's are to be loaded from + "plugins": { + //Hot-reload creates collectable assemblies that allow full re-load support in the host application, should only be used for development purposes! + "hot_reload": false, + "path": "plugins/", + "config_dir": "config/", + "assets": "plugins/assets/" + }, + + "sys_log": { + "path": "data/logs/sys-log.txt", + "flush_sec": 5, + "retained_files": 31, + "file_size_limit": 10485760, + "interval": "infinite" + }, + + "app_log": { + "path": "data/logs/app-log.txt", + "flush_sec": 5, + "retained_files": 31, + "file_size_limit": 10485760, + "interval": "infinite" + }, + + //HASHICORP VAULT + "hashicorp_vault": { + "url": "", + "token": "", + "trust_cert": false + }, + + "secrets": { + //Special key used by the loading library for access to the PasswordHashing library to pepper password hashes + "cache_private_key": "", + "client_public_key": "" + } +} + diff --git a/plugins/ObjectCacheServer/server/container/Dockerfile b/plugins/ObjectCacheServer/server/container/Dockerfile new file mode 100644 index 0000000..6c466d4 --- /dev/null +++ b/plugins/ObjectCacheServer/server/container/Dockerfile @@ -0,0 +1,82 @@ +#Copyright (c) Vaughn Nugent +#Licensed under the GNU AGPL V3.0 + +#use plain alpine latest to build native libraries in +FROM alpine:3.19 as native-cont + +#install public libs and build tools +RUN apk update && apk add build-base cmake npm +#most universal way to use Task is from NPM +RUN npm install -g @go-task/cli + +WORKDIR /build + +#include local artifacts +COPY app/ . + +#build internal libraries and copy the libraries to the /lib output directory +RUN mkdir out/ +RUN task build-libs + +#APP CONTAINER +#move into a clean dotnet apline lean image +FROM mcr.microsoft.com/dotnet/runtime:8.0.2-alpine3.19-amd64 as app-cont + +LABEL name="vnuge/vncache" +LABEL maintainer="Vaughn Nugent <vnpublic@proton.me>" +LABEL description="A simple clustered network data caching service" + +#copy local artifacts again in run container +COPY app/ /app + +#pull compiled libs from build container +COPY --from=native-cont /build/out /app/lib + +RUN apk update && apk add --no-cache gettext icu-libs dumb-init + +#workdir +WORKDIR /app + +#default to 2557 for cache port +EXPOSE 2557/tcp + +VOLUME /app/ssl +#expose an assets directory for custom assets install +VOLUME /app/usr/assets + +#disable dotnet invariant culture on alpine +ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=0 + +#add helper/required libraries +#ENV VNLIB_SHARED_HEAP_FILE_PATH=/app/lib/libvn_rpmalloc.so not ready yet, still need to debug + +#cache varables +ENV MAX_ENTRIES=10000 +ENV CACHE_BUCKETS=100 +ENV CACHE_MAX_MESSAGE=20480 +ENV MAX_CONCURRENT_CONNECTIONS=1000 + +ENV VERIFY_IP=true +ENV MAX_PEER_NODES=10 +ENV DISCOVERY_INTERVAL=360 +ENV CACHE_CONNECT_PATH="/cache" +ENV DISCOVER_PATH="/discover" +ENV KNOWN_PEERS=[] + +#HC Vault +ENV HC_VAULT_ADDR="" +ENV HC_VAULT_TOKEN="" +ENV HC_VAULT_TRUST_CERT=false + +#SECRETS +ENV CACHE_PRIV_KEY="" +ENV CLIENT_PUB_KEY="" + +#HTTP/PROXY Config +ENV HTTP_DOWNSTREAM_SERVERS=[] +ENV HTTP_MAX_CONNS=5000 + +#run the init script within dumb-init +ENTRYPOINT ["dumb-init", "--"] +CMD ["ash", "./run.sh"] + diff --git a/plugins/ObjectCacheServer/server/container/Taskfile.yaml b/plugins/ObjectCacheServer/server/container/Taskfile.yaml new file mode 100644 index 0000000..10ee86b --- /dev/null +++ b/plugins/ObjectCacheServer/server/container/Taskfile.yaml @@ -0,0 +1,80 @@ +# https://taskfile.dev + +#inlcuded by the ci main taskfile to produce containerized builds, and also +#be included by the container itself to run build tasks inside the container + +version: "3" + +vars: + INCLUDE_FILES: "Dockerfile, docker-compose.yaml" + +includes: + install: + taskfile: ../install.taskfile.yaml + optional: true #not needed for inside container build + +tasks: + #called from inside the container to build native libraries + build-libs: + vars: + OUT_DIR: "{{.USER_WORKING_DIR}}/out" + + #build stage generates the following libraries + generates: + - "{{.USER_WORKING_DIR}}/out/libvn_rpmalloc.so" + + cmds: + #build rpmalloc library + - cd lib/vnlib_rpmalloc/ && task && cp build/libvn_rpmalloc.so {{.OUT_DIR}}/libvn_rpmalloc.so + + #called from ci pipline to build the package + build: + cmds: + # clean up the run.sh script to remove windows line endings in my wsl default instance + - cmd: wsl dos2unix ./run.sh + platforms: [ windows/amd64 ] + + #init build image + - task: setup-container-image + + #remove the default config file as it's not needed in the container + - powershell -Command "rm -Force -Recurse build/app/config/" + + #install rpmalloc + - task: install-rpmalloc-lib + + postbuild_success: + cmds: + #tar up the build directory and move it to the output bin directory + - cmd: cd build/ && tar -czf '{{ .BINARY_DIR }}/{{.PACKAGE_FILE_NAME}}' . + #clean up all the build files after build succeeds + - task: clean + + clean: + ignore_error: true + cmds: + - cmd: powershell -Command "rm -Recurse -Force ./build" + + install-rpmalloc-lib: + internal: true + cmds: + #install compressor plugin + - task: install:install + vars: + PROJECT_NAME: 'vnlib_rpmalloc' + MODULE_NAME: "VNLib.Core" + FILE_NAME: "src.tgz" + DIR: './build/app/lib/vnlib_rpmalloc' + + setup-container-image: + internal: true + cmds: + #make build directory + - powershell -Command "mkdir build, build/app, build/app/config-templates/, build/app/static/ -Force" + #copy the existing linux-x64 build to the build folder, this will be the container base + - powershell -Command "cp -Recurse -Force ../build/linux-x64/* build/app/" + #copy local scripts and config data into the build folder + - powershell -Command "cp -Force run.sh, Taskfile.yaml build/app/" + - powershell -Command "cp -Force Dockerfile, docker-compose.yaml build/" + - powershell -Command "cp -Force static/* build/app/static/" + - powershell -Command "cp -Force config-templates/* build/app/config-templates/" diff --git a/plugins/ObjectCacheServer/server/container/config-templates/ObjectCacheServer-template.json b/plugins/ObjectCacheServer/server/container/config-templates/ObjectCacheServer-template.json new file mode 100644 index 0000000..765c3d7 --- /dev/null +++ b/plugins/ObjectCacheServer/server/container/config-templates/ObjectCacheServer-template.json @@ -0,0 +1,54 @@ +{ + "debug": false, + + //enables cache server cluster node data + "cluster": { + //Delay to re-discover peers + "discovery_interval_sec": ${DISCOVERY_INTERVAL}, + + //The maxium number of peers to connect to + "max_peers": ${MAX_PEER_NODES}, + + //Max ev queue depth before LRU eviction + "max_queue_depth": 10000, + + //Time between queue purge + "queue_purge_interval_sec": 360000, + + //Forces strict ip address verification on upgrades (best to leave on) + "verify_ip": ${VERIFY_IP}, + + //The cache websocket endpoint path + "connect_path": "${CACHE_CONNECT_PATH}", + + //Optional to allow nodes to discover nodes we adverties + "discovery_path": "${DISCOVER_PATH}", + + //Optionally change the well-known path (clients must know this) + "well_known_path": null, + + //The maxium number of connections to this node + "max_concurrent_connections": ${MAX_CONCURRENT_CONNECTIONS} + }, + + //Cache configuration object, FBM protocol variables + "cache": { + + //Max number of cache entires to be stored + "max_cache": ${MAX_ENTRIES}, + + //the number of cache buckets to distribute load + "buckets": ${CACHE_BUCKETS}, + + //FBM buffer config + "buffer_recv_max": ${CACHE_MAX_MESSAGE}, //Up to 100Kb transfer buffer + "buffer_recv_min": 8192, //min of 8k transfer buffer + "buffer_header_max": 2048, //2k max header buffer size + "buffer_header_min": 128, //128 byte min request header buffer size + "max_message_size": ${CACHE_MAX_MESSAGE} //Absolute maxium message size allowed, also the maxium size of cache entires + }, + + //Known peers array, must point to well-known endpoint for discovery + "known_peers": ${KNOWN_PEERS} + +}
\ No newline at end of file diff --git a/plugins/ObjectCacheServer/server/container/config-templates/config-template.json b/plugins/ObjectCacheServer/server/container/config-templates/config-template.json new file mode 100644 index 0000000..6362432 --- /dev/null +++ b/plugins/ObjectCacheServer/server/container/config-templates/config-template.json @@ -0,0 +1,105 @@ +{ + + //Host application config, config is loaded as a read-only DOM that is available + //to the host and loaded child plugins, all elements are available to plugins via the 'HostConfig' property + + "http": { + //The defaut HTTP version to being requests with (does not support http/2 yet) + "default_version": "HTTP/1.1", + //The maxium size (in bytes) of response messges that will be compressed + "compression_limit": 10000, + //Minium response size (in bytes) to compress + "compression_minimum": 2048, + //The size of the buffer to use when parsing multipart/form data uploads + "multipart_max_buf_size": 1024, + //The maxium ammount of data (in bytes) allows for mulitpart/form data file uploads + "multipart_max_size": 0, + //Absolute maximum size (in bytes) of the request entity body (exludes headers) + "max_entity_size": 10240, + //Keepalive ms for HTTP1.1 keepalive connections + "keepalive_ms": 100000, + //The buffer size to use when parsing headers (also the maxium request header size allowed) + "header_buf_size": 8128, + //The maxium number of headers allowed in an HTTP request message + "max_request_header_count": 50, + //The maxium number of allowed network connections, before 503s will be issued automatically and connections closed + "max_connections": ${HTTP_MAX_CONNS}, + //The size in bytes of the buffer to use when writing response messages + "response_buf_size": 4096, + //time (in ms) to wait for a response from an active connection in recv mode, before dropping it + "recv_timeout_ms": 5000, + //Time in ms to wait for the client to accept transport data before terminating the connection + "send_timeout_ms": 60000, + //The size (in bytes) of the buffer used to store all response header data + "response_header_buf_size": 16384, + //Max number of file uploads allowed per request + "max_uploads_per_request": 1 + }, + + //Maxium ammount of time a request is allowed to be processed (includes loading or waiting for sessions) before operations will be cancelled and a 503 returned + "max_execution_time_ms": 20000, + + "virtual_hosts": [ + { + "interface": { + "address": "0.0.0.0", + "port": 2557 + }, + + //Collection of "trusted" servers to allow proxy header support from + "downstream_servers": ${HTTP_DOWNSTREAM_SERVERS}, + + //The hostname to listen for, "*" as wildcard, and "[system]" as the default hostname for the current machine + "hostname": "*", + "path": "root/", + + "deny_extensions": [ ], + "default_files": [ ], + "error_files": [], + "cache_default_sec": 864000, + + //Disabled until well-tested + //"ssl": ${SSL_JSON} + } + ], + + + //Defines the directory where plugin's are to be loaded from + "plugins": { + //Hot-reload creates collectable assemblies that allow full re-load support in the host application, should only be used for development purposes! + "hot_reload": false, + "path": "plugins/", + "config_dir": "config/", + "assets": "plugins/assets/" + }, + + "sys_log": { + "path": "data/logs/sys-log.txt", + "flush_sec": 5, + "retained_files": 31, + "file_size_limit": 10485760, + "interval": "infinite" + }, + + "app_log": { + "path": "data/logs/app-log.txt", + "flush_sec": 5, + "retained_files": 31, + "file_size_limit": 10485760, + "interval": "infinite" + }, + + //HASHICORP VAULT + "hashicorp_vault": { + "url": "${HC_VAULT_ADDR}", + "token": "${HC_VAULT_TOKEN}", + "trust_cert": ${HC_VAULT_TRUST_CERT} + }, + + "secrets": { + //Special key used by the loading library for access to the PasswordHashing library to pepper password hashes + "cache_private_key": "${CACHE_PRIV_KEY}", + "client_public_key": "${CLIENT_PUB_KEY}" + } +} + diff --git a/plugins/ObjectCacheServer/server/container/docker-compose.yaml b/plugins/ObjectCacheServer/server/container/docker-compose.yaml new file mode 100644 index 0000000..c1b61fa --- /dev/null +++ b/plugins/ObjectCacheServer/server/container/docker-compose.yaml @@ -0,0 +1,45 @@ +#Copyright (c) Vaughn Nugent +#Licensed under the GNU AGPLv3 + +version: '3.6' + +services: + vncache: + image: vnuge/vncache + container_name: vncache + restart: unless-stopped + hostname: vncache-server + volumes: + - ./assets:/app/usr/assets:ro #optional if assets are required + - ./ssl:/app/ssl:ro #optional only if SSL is enabled (currently not a feature) + ports: + - 2557:2557 + environment: + #System memory consumption is calculated as follows: + # MAX_ENTIRES x CACHE_BUCKETS x CACHE_MAX_MESSAGE = max memory consumption + + MAX_CONCURRENT_CONNECTIONS: "1000" #max number of concurrent connections + MAX_ENTRIES: "10000" #max number of cache entries per bucket + CACHE_BUCKETS: "100" #number of cache buckets for load balancing + CACHE_MAX_MESSAGE: "20480" #20KB + VERIFY_IP: "true" #verfies the IP address of clients during negotiation (recommended) + MAX_PEER_NODES: "10" #max number of other peer nodes this node shoud connect to + DISCOVERY_INTERVAL: "360" #time (in seconds) between peer node discovery + KNOWN_PEERS: '[]' #array of known peer nodes in the cluster + + #SECRETS (must be JWK formatted keys) + CACHE_PRIV_KEY: "" #REQUIRED local private key used to identify and sign messages to clients and other nodes + CLIENT_PUB_KEY: "" #REQUIRED used to verify client messages + + #HC vault + #HC_VAULT_ADDR: "" + #HC_VAULT_TOKEN: "" + #HC_VAULT_TRUST_CERT: "false" + + #HTTP + #HTTP_DOWNSTREAM_SERVERS: '[]' + #SSL_JSON: '{"cert": "ssl/cert.pem", "privkey":"ssl/priv.pem"}' + HTTP_MAX_CONNS: "5000" + + SERVER_ARGS: "--input-off" + diff --git a/plugins/ObjectCacheServer/server/container/run.sh b/plugins/ObjectCacheServer/server/container/run.sh new file mode 100644 index 0000000..2c2636c --- /dev/null +++ b/plugins/ObjectCacheServer/server/container/run.sh @@ -0,0 +1,15 @@ +#! /bin/sh + +#this script will be invoked by dumb-init in the container on statup and is located at /app + +rm -rf config && mkdir config + +#substitude all -template files in the config-templates dir and write them to the config dir +for file in config-templates/*-template.json; do + envsubst < $file > config/$(basename $file -template.json).json +done + +cp usr/assets/* plugins/assets/ -rf + +#start the server +dotnet webserver/VNLib.WebServer.dll --config config/config.json $SERVER_ARGS
\ No newline at end of file diff --git a/plugins/ObjectCacheServer/server/install.ps1 b/plugins/ObjectCacheServer/server/install.ps1 new file mode 100644 index 0000000..4c42c18 --- /dev/null +++ b/plugins/ObjectCacheServer/server/install.ps1 @@ -0,0 +1,26 @@ +param([String] $BaseUrl, [String] $ModuleName, [String] $ProjectName, [String]$FileName) + +#get the latest file +Invoke-WebRequest "$BaseUrl/$ModuleName/@latest" -OutFile latest.txt +#read the file into a variable +$latest = Get-Content latest.txt + +#download the latest version +Invoke-WebRequest "$BaseUrl/$ModuleName/$latest/$ProjectName/$FileName" -OutFile $FileName + +#download latest sha256 +Invoke-WebRequest "$BaseUrl/$ModuleName/$latest/$ProjectName/$FileName.sha256" -OutFile "$FileName.sha256" + +#verify the file +$hash = (Get-FileHash $FileName -Algorithm SHA256).Hash + +#read the sha256 file +$sha256 = Get-Content "$FileName.sha256" + +#compare the hashes +if ($hash -eq $sha256) { + Write-Host "Hashes match, file is valid" -ForegroundColor Blue +} else { + throw "Hashes do not match, file is invalid" +} + diff --git a/plugins/ObjectCacheServer/server/install.taskfile.yaml b/plugins/ObjectCacheServer/server/install.taskfile.yaml new file mode 100644 index 0000000..37baf12 --- /dev/null +++ b/plugins/ObjectCacheServer/server/install.taskfile.yaml @@ -0,0 +1,20 @@ +# https://taskfile.dev + +#Called by the vnbuild system to produce builds for my website +#https://www.vaughnnugent.com/resources/software + +version: "3" + +tasks: + + install: + internal: true + cmds: + #make the plugin directory + - cmd: powershell -Command "mkdir {{.DIR}} -Force" + ignore_error: true + - cmd: powershell -Command "pwd" + - cd {{.DIR}} && powershell "{{.SCRIPT_DIR}}/install.ps1" -BaseUrl {{.BUILDS_URL}} -ModuleName {{.MODULE_NAME}} -ProjectName {{.PROJECT_NAME}} -FileName {{.FILE_NAME}} + - cd {{.DIR}} && tar -xzf {{.FILE_NAME}} + #remove the archive file + - cd {{.DIR}} && powershell -Command "rm {{.FILE_NAME}}"
\ No newline at end of file diff --git a/plugins/ObjectCacheServer/server/taskfile.yaml b/plugins/ObjectCacheServer/server/taskfile.yaml new file mode 100644 index 0000000..38eae79 --- /dev/null +++ b/plugins/ObjectCacheServer/server/taskfile.yaml @@ -0,0 +1,193 @@ +# https://taskfile.dev + +#Inlcuded taskfile for object cache server that is used to produce +#ci builds for standalone caching servers + +version: "3" + +vars: + BUILDS_URL: https://www.vaughnnugent.com/public/resources/software/builds + SCRIPT_DIR: '{{.TASKFILE_DIR}}' + BINARY_DIR: '{{.PROJECT_DIR}}/bin' #binary dir is not available for dotnet plugis + +includes: + install: + taskfile: install.taskfile.yaml + optional: true + + container: + dir: container #always run from the container directory + taskfile: container/Taskfile.yaml + optional: true + vars: + BUILDS_URL: '{{.BUILDS_URL}}' + PACKAGE_FILE_NAME: "vncache-alpine3.19-oci.tgz" #the name of the output package file + +tasks: +# CLIENT-SIDE TASKS + default: + desc: "Runs the VNCache server" + cmds: + - task: run + + run: + desc: "Runs the VNCache server" + silent: true + env: + #server should detect the file extension and load the correct library + VNLIB_SHARED_HEAP_FILE_PATH: lib/libvn_rpmalloc + + cmds: + - cmd: dotnet webserver/VNLib.WebServer.dll --config config/config.json --input-off --inline-scheduler {{.ARGS}} + #setup sever environment + + + setup-debian: + desc: "Performs initial setup on Debian x64 based machines" + silent: true + cmds: + - apt update + - apt install -y dotnet-runtime-8.0 gcc cmake + - task: setup + - echo "Setup complete" + + setup-fedora: + desc: "Performs initial setup on Fedora/Redhat x64 (dnf) based machines" + silent: true + cmds: + - dnf update + - dnf install -y dotnet-runtime-8.0 gcc cmake + - task: setup + - echo "Setup complete" + + setup-alpine: + desc: "Performs initial setup on Alpine x64 based machines" + silent: true + cmds: + - apk update + - apk add --no-cache dotnet8-runtime gcc cmake + - task: setup + - echo "Setup complete" + + setup: + cmds: + #build rpmalloc lib + - task: build-rpmalloc + + build-rpmalloc: + internal: true + dir: 'lib/' + vars: + RPMALLOC_DIR: 'vnlib_rpmalloc' + cmds: + #build rpmalloc library + - cmd: cd vnlib_rpmalloc/ && task + + - cmd: cp vnlib_rpmalloc/build/libvn_rpmalloc.so libvn_rpmalloc.so + platforms: [ linux ] + + - cmd: cp vnlib_rpmalloc/build/libvn_rpmalloc.dylib libvn_rpmalloc.dylib + platforms: [ darwin ] + + - cmd: powershell -Command "cp vnlib_rpmalloc/build/Release/vnlib_rpmalloc.dll libvn_rpmalloc.dll" + platforms: [ windows/amd64 ] + +# CI BUILD TASKS + build: + desc: "CI ONLY! DO NOT RUN" + cmds: + - task: install-plugins + - task: install-webserver + + #run container build last + - task: container:build + + install-webserver: + internal: true + cmds: + - for: [ win-x64, linux-x64, osx-x64, linux-arm64 ] + task: create-env + vars: + TARGET_OS: '{{.ITEM}}' + + install-plugins: + internal: true + cmds: + - cmd: powershell -Command "mkdir lib -Force" + ignore_error: true + + #copy the object-cache plugin output to the local plugins directory + - cmd: powershell -Command "cp -Recurse -Force {{.PROJECT_DIR}}/bin/Release/net8.0/publish/ plugins/{{.PROJECT_NAME}}/" + + #download rpmalloc + - task: install:install + vars: + PROJECT_NAME: 'vnlib_rpmalloc' + MODULE_NAME: "VNLib.Core" + FILE_NAME: "src.tgz" + DIR: './lib/vnlib_rpmalloc' + + postbuild_success: + desc: "CI ONLY! DO NOT RUN" + cmds: + - for: [ win-x64, linux-x64, osx-x64, linux-arm64 ] + task: pack + vars: + TARGET_OS: '{{.ITEM}}' + + #cleanup unnecessary build files that clog up the pipeline + - for: [ build, plugins, lib ] + cmd: powershell -Command "rm -Recurse '{{.ITEM}}'" + ignore_error: true + + - task: container:postbuild_success + + build-container: + internal: true + cmds: + - task: container:build + + #Creates a new webserver build environment for an operating system configuration + create-env: + internal: true + vars: + BUILD_DIR: './build/{{.TARGET_OS}}' + cmds: + #create dir for env + - cmd: powershell -Command "mkdir {{.BUILD_DIR}} -Force" + ignore_error: true + + #copy build files + - for: [ plugins, lib, config, taskfile.yaml ] + cmd: powershell -Command "cp -Recurse -Force {{.ITEM}} {{.BUILD_DIR}}" + + - task: get-webserver + vars: + TARGET_OS: '{{.TARGET_OS}}' + BUILD_DIR: '{{.BUILD_DIR}}' + + #fetches a copy of (the desired os version) VNLib.WebServer project and installs it into the build directory + get-webserver: + internal: true + cmds: + - task: install:install + vars: + PROJECT_NAME: 'VNLib.Webserver' + MODULE_NAME: "VNLib.Webserver" + FILE_NAME: "{{.TARGET_OS}}-release.tgz" + DIR: '{{.BUILD_DIR}}/webserver' + + pack: + internal: true + cmds: + - cd build/{{.TARGET_OS}} && tar -czf '{{ .BINARY_DIR }}/{{ .TARGET_OS }}-release.tgz' . + + clean: + desc: "CI ONLY! DO NOT RUN" + ignore_error: true + cmds: + - for: [ build/, bin/, plugins/, lib/] + cmd: powershell -Command "rm -Recurse -Force '{{.ITEM}}'" + + - task: container:clean + diff --git a/plugins/ObjectCacheServer/src/Clustering/PeerDiscoveryManager.cs b/plugins/ObjectCacheServer/src/Clustering/PeerDiscoveryManager.cs index b8ee9c8..b9a220d 100644 --- a/plugins/ObjectCacheServer/src/Clustering/PeerDiscoveryManager.cs +++ b/plugins/ObjectCacheServer/src/Clustering/PeerDiscoveryManager.cs @@ -40,12 +40,18 @@ namespace VNLib.Data.Caching.ObjectCache.Server.Clustering * This class is responsible for resolving and discovering peer nodes in the cluster network. */ - internal sealed class PeerDiscoveryManager(CacheNodeConfiguration config, ServerClusterConfig clusterConf, ILogProvider Log, bool IsDebug, bool HasWellKnown) + internal sealed class PeerDiscoveryManager( + CacheNodeConfiguration config, + ServerClusterConfig clusterConf, + CachePeerMonitor Monitor, + ILogProvider Log, + bool IsDebug, + bool HasWellKnown + ) : IAsyncBackgroundWork, ICachePeerAdapter { private readonly List<CacheNodeAdvertisment> _connectedPeers = []; - private readonly CachePeerMonitor Monitor = new(); private readonly VNCacheClusterManager clusterMan = new(config); async Task IAsyncBackgroundWork.DoWorkAsync(ILogProvider pluginLog, CancellationToken exitToken) @@ -81,9 +87,12 @@ namespace VNLib.Data.Caching.ObjectCache.Server.Clustering /* * On every loop we will need to resolve well-known servers incase they go down * or change. There probably should be some more advanced logic and caching here. + * + * Node may not have any well-known nodes, so we need to check for that. */ - CacheNodeAdvertisment[] wellKnown = await clusterMan.ResolveWellKnownAsync(exitToken); - wellKnownFailed = wellKnown.Length == 0; + CacheNodeAdvertisment[] wellKnown = HasWellKnown ? + await clusterMan.ResolveWellKnownAsync(exitToken) : + Array.Empty<CacheNodeAdvertisment>(); //Use the monitor to get the initial peers IEnumerable<CacheNodeAdvertisment> ads = GetMonitorAds(); diff --git a/plugins/ObjectCacheServer/src/ObjectCacheSystemState.cs b/plugins/ObjectCacheServer/src/ObjectCacheSystemState.cs index 970e832..cd5bf1b 100644 --- a/plugins/ObjectCacheServer/src/ObjectCacheSystemState.cs +++ b/plugins/ObjectCacheServer/src/ObjectCacheSystemState.cs @@ -26,6 +26,9 @@ using System; using System.Linq; using System.Net.Http; using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using System.Collections.Generic; using VNLib.Utils.Logging; using VNLib.Utils.Memory; @@ -37,8 +40,6 @@ using VNLib.Plugins.Extensions.Loading; using VNLib.Data.Caching.Extensions.Clustering; using VNLib.Data.Caching.ObjectCache.Server.Cache; using VNLib.Data.Caching.ObjectCache.Server.Clustering; -using System.Threading.Tasks; -using System.Threading; namespace VNLib.Data.Caching.ObjectCache.Server { @@ -116,6 +117,8 @@ namespace VNLib.Data.Caching.ObjectCache.Server LogMemConfiguration(); + PeerEventQueue = new(plugin, ClusterConfig); + //If the plugin is in debug mode enable heap tracking SharedCacheHeap = plugin.IsDebug() ? new TrackedHeapWrapper(MemoryUtil.InitializeNewHeapForProcess(), true) @@ -128,8 +131,6 @@ namespace VNLib.Data.Caching.ObjectCache.Server ConfigurePeerDiscovery(); ConfigureCacheListener(); - - PeerEventQueue = new(plugin, ClusterConfig); } private void ConfigurePeerDiscovery() @@ -140,14 +141,22 @@ namespace VNLib.Data.Caching.ObjectCache.Server ILogProvider discLogger = plugin.Log.CreateScope(CacheConstants.LogScopes.PeerDiscovery); - NodeConfig.WithInitialPeers(kownPeers.Select(static s => new Uri(s))) + //Allow just origin nodes to be used as known peers + IEnumerable<Uri> peerUris = kownPeers.Select(static p => + { + Uri bUri = new(p, UriKind.Absolute); + return bUri.LocalPath == "/" ? new Uri(bUri, CacheConstants.DefaultWellKnownPath) : bUri; + }); + + NodeConfig.WithInitialPeers(peerUris) .WithErrorHandler(new ErrorHandler(discLogger)); discLogger.Information("Inital peer nodes: {nodes}", kownPeers); PeerDiscovery = new PeerDiscoveryManager( NodeConfig, - ClusterConfig, + ClusterConfig, + PeerMonitor, discLogger, plugin.IsDebug(), kownPeers.Length > 0 @@ -176,10 +185,15 @@ namespace VNLib.Data.Caching.ObjectCache.Server _cacheMemManager = manager; + CacheListenerPubQueue queue = new(plugin, PeerEventQueue); + + //Must register background worker to listen for changes + _ = plugin.ObserveWork(queue, 150); + //Endpoint only allows for a single reader Listener = new( plugin.LoadMemoryCacheSystem(config, manager, MemoryConfiguration), - new CacheListenerPubQueue(plugin, PeerEventQueue), + queue, plugin.Log.CreateScope(CacheConstants.LogScopes.BlobCacheListener), new SharedHeapFBMMemoryManager(SharedCacheHeap) ); diff --git a/plugins/ObjectCacheServer/src/ServerClusterConfig.cs b/plugins/ObjectCacheServer/src/ServerClusterConfig.cs index 8f81ba6..8e098cd 100644 --- a/plugins/ObjectCacheServer/src/ServerClusterConfig.cs +++ b/plugins/ObjectCacheServer/src/ServerClusterConfig.cs @@ -45,7 +45,7 @@ namespace VNLib.Data.Caching.ObjectCache.Server public int MaxQueueDepth { get; } = (int)config.GetRequiredProperty("max_queue_depth", p => p.GetUInt32()); - public string? DiscoveryPath { get; } = config.GetValueOrDefault(CacheConfigTemplate, p => p.GetString(), null); + public string? DiscoveryPath { get; } = config.GetValueOrDefault("discovery_path", p => p.GetString(), null); public string ConnectPath { get; } = config.GetRequiredProperty("connect_path", p => p.GetString()!); @@ -63,7 +63,7 @@ namespace VNLib.Data.Caching.ObjectCache.Server /// The maxium number of concurrent client connections to allow /// before rejecting new connections /// </summary> - public uint MaxConcurrentConnections { get; } + public uint MaxConcurrentConnections { get; } = config.GetValueOrDefault("max_concurrent_connections", p => p.GetUInt32(), 100u); const string CacheConfigTemplate = @" |