diff options
Diffstat (limited to 'plugins')
10 files changed, 119 insertions, 171 deletions
diff --git a/plugins/ObjectCacheServer/server/config/config.json b/plugins/ObjectCacheServer/server/config/config.json deleted file mode 100644 index 1f8a382..0000000 --- a/plugins/ObjectCacheServer/server/config/config.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - - //Host application config, config is loaded as a read-only DOM that is available - //to the host and loaded child plugins, all elements are available to plugins via the 'HostConfig' property - - "http": { - //The defaut HTTP version to being requests with (does not support http/2 yet) - "default_version": "HTTP/1.1", - //The maxium size (in bytes) of response messges that will be compressed - "compression_limit": 10000, - //Minium response size (in bytes) to compress - "compression_minimum": 2048, - //The size of the buffer to use when parsing multipart/form data uploads - "multipart_max_buf_size": 1024, - //The maxium ammount of data (in bytes) allows for mulitpart/form data file uploads - "multipart_max_size": 0, - //Absolute maximum size (in bytes) of the request entity body (exludes headers) - "max_entity_size": 10240, - //Keepalive ms for HTTP1.1 keepalive connections - "keepalive_ms": 100000, - //The buffer size to use when parsing headers (also the maxium request header size allowed) - "header_buf_size": 8128, - //The maxium number of headers allowed in an HTTP request message - "max_request_header_count": 50, - //The maxium number of allowed network connections, before 503s will be issued automatically and connections closed - "max_connections": 5000, - //The size in bytes of the buffer to use when writing response messages - "response_buf_size": 4096, - //time (in ms) to wait for a response from an active connection in recv mode, before dropping it - "recv_timeout_ms": 5000, - //Time in ms to wait for the client to accept transport data before terminating the connection - "send_timeout_ms": 60000, - //The size (in bytes) of the buffer used to store all response header data - "response_header_buf_size": 16384, - //Max number of file uploads allowed per request - "max_uploads_per_request": 1 - }, - - //Maxium ammount of time a request is allowed to be processed (includes loading or waiting for sessions) before operations will be cancelled and a 503 returned - "max_execution_time_ms": 20000, - - "virtual_hosts": [ - { - "interface": { - "address": "0.0.0.0", - "port": 2557 - }, - - //Collection of "trusted" servers to allow proxy header support from - "downstream_servers": [], - - //The hostname to listen for, "*" as wildcard, and "[system]" as the default hostname for the current machine - "hostname": "*", - "path": "root/", //Point to some place we can read nothing from - - "deny_extensions": [ ], - "default_files": [ ], - "error_files": [], - "cache_default_sec": 864000, - - "DISABLED ssl": {} - } - ], - - - //Defines the directory where plugin's are to be loaded from - "plugins": { - //Hot-reload creates collectable assemblies that allow full re-load support in the host application, should only be used for development purposes! - "hot_reload": false, - "path": "plugins/", - "config_dir": "config/", - "assets": "plugins/assets/" - }, - - "sys_log": { - "path": "data/logs/sys-log.txt", - "flush_sec": 5, - "retained_files": 31, - "file_size_limit": 10485760, - "interval": "infinite" - }, - - "app_log": { - "path": "data/logs/app-log.txt", - "flush_sec": 5, - "retained_files": 31, - "file_size_limit": 10485760, - "interval": "infinite" - }, - - //HASHICORP VAULT - "hashicorp_vault": { - "url": "", - "token": "", - "trust_cert": false - }, - - "secrets": { - //Special key used by the loading library for access to the PasswordHashing library to pepper password hashes - "cache_private_key": "", - "client_public_key": "" - } -} - diff --git a/plugins/ObjectCacheServer/server/config/config.yaml b/plugins/ObjectCacheServer/server/config/config.yaml new file mode 100644 index 0000000..6e2093f --- /dev/null +++ b/plugins/ObjectCacheServer/server/config/config.yaml @@ -0,0 +1,69 @@ + +tcp: + keepalive_sec: 60 #How long to wait for a keepalive response before closing the connection (0 to disable tcp keepalive) + keepalive_interval_sec: 60 #How long to wait between keepalive probes + max_recv_size: 655360 #640k absolute maximum recv buffer (defaults to OS socket buffer size) + max_connections: 1000 #Per listener instance + backlog: 100 #OS socket backlog + + #sets operating system buffer sizes + tx_buffer: 65536 #OS socket send buffer size + rx_buffer: 65536 #OS socket recv buffer size + +http: + default_version: "HTTP/1.1" #Default HTTP version to use + max_connections: 5000 #Maximum number of connections to accept + max_entity_size: 10240 #Maximum size of an entity body + multipart_max_buf_size: 1024 #Maximum size of a multipart buffer + multipart_max_size: 0 #Maximum size of a multipart entity (disabled) + header_buf_size: 8128 #Size of the header buffer + max_request_header_count: 50 #Maximum number of headers in a request + response_header_buf_size: 16384 #Size of the response header buffer + max_uploads_per_request: 10 #Maximum number of uploads per request + keepalive_ms: 1000000 #How long to wait for a keepalive response before closing the connection + recv_timeout_ms: 5000 #How long to wait to receive data from a client + send_timeout_ms: 60000 #How long to wait for a client to receive data from us + + compression: + enabled: false #Enable compression (not used for caching) + assembly: "" #Path to the compression assembly + max_size: 5120000 #Maximum size of a compressed entity + min_size: 2048 #Minimum size of a compressed entity + +max_execution_time_ms: 20000 #Maximum time the server should spend processing a request + +plugins: + enabled: true #Enable plugins (required for caching) + hot_reload: false #No hot reload for production + assets: "plugins/assets/" #where shared assets are stored + config_dir: "config/" #where to load plugin config files from + paths: #Paths to search for plugin assemblies + - "plugins/" + + +#Only a single host is required for caching since object +#cache is a plugin. It just needs to be accessable + +virtual_hosts: + - hostnames: [ "*" ] #Hostnames to listen on + path: "root/" #Path to the host directory + + interface: + address: "0.0.0.0" #list on all addresses + port: 2557 + ssl: false #disable ssl (it's supported yet) + + trace: false #Enable connection tracing + cache_default_sec: 864000 #Default cache time in seconds + force_port_check: false #Enforces urls to have the same port as the server is listening on + deny_extensions: [] #File extensions to deny access to + default_file: [] #Default file to serve if none is specified + #whitelist: [] #IP addresses to whitelist + + cors: + enabled: false #disable cors module + deny_cors_connections: true #deny cors connections + allow_origins: [] + + headers: + Server: "VNLib.Data.Caching Object Cache"
\ No newline at end of file diff --git a/plugins/ObjectCacheServer/server/container/Dockerfile b/plugins/ObjectCacheServer/server/container/Dockerfile index 725b9d1..1694eeb 100644 --- a/plugins/ObjectCacheServer/server/container/Dockerfile +++ b/plugins/ObjectCacheServer/server/container/Dockerfile @@ -20,7 +20,7 @@ RUN task build-libs #APP CONTAINER #move into a clean dotnet apline lean image -FROM mcr.microsoft.com/dotnet/runtime:8.0.2-alpine3.19-amd64 as app-cont +FROM mcr.microsoft.com/dotnet/runtime:8.0.3-alpine3.19-amd64 as app-cont LABEL name="vnuge/vncache" LABEL maintainer="Vaughn Nugent <vnpublic@proton.me>" @@ -51,31 +51,30 @@ ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=0 #ENV VNLIB_SHARED_HEAP_FILE_PATH=/app/lib/libvn_rpmalloc.so not ready yet, still need to debug #cache varables -ENV MAX_ENTRIES=10000 -ENV CACHE_BUCKETS=100 -ENV CACHE_MAX_MESSAGE=20480 -ENV MAX_CONCURRENT_CONNECTIONS=1000 -ENV ENABLE_CHECKSUMS=true - -ENV VERIFY_IP=true -ENV MAX_PEER_NODES=10 -ENV DISCOVERY_INTERVAL=360 -ENV CACHE_CONNECT_PATH="/cache" -ENV DISCOVER_PATH="/discover" -ENV KNOWN_PEERS=[] +ENV MAX_ENTRIES=10000 / + CACHE_BUCKETS=100 / + CACHE_MAX_MESSAGE=20480 / + MAX_CONCURRENT_CONNECTIONS=1000 / + ENABLE_CHECKSUMS=true / + VERIFY_IP=true / + MAX_PEER_NODES=10 / + DISCOVERY_INTERVAL=360 / + CACHE_CONNECT_PATH="/cache" / + DISCOVER_PATH="/discover" / + KNOWN_PEERS=[] / #HC Vault -ENV HC_VAULT_ADDR="" -ENV HC_VAULT_TOKEN="" -ENV HC_VAULT_TRUST_CERT=false +ENV HC_VAULT_ADDR="" / + HC_VAULT_TOKEN="" / + HC_VAULT_TRUST_CERT=false #SECRETS -ENV CACHE_PRIV_KEY="" -ENV CLIENT_PUB_KEY="" +ENV CACHE_PRIV_KEY="" / + CLIENT_PUB_KEY="" #HTTP/PROXY Config -ENV HTTP_DOWNSTREAM_SERVERS=[] -ENV HTTP_MAX_CONNS=5000 +ENV HTTP_DOWNSTREAM_SERVERS=[] / + HTTP_MAX_CONNS=5000 #run the init script within dumb-init ENTRYPOINT ["dumb-init", "--"] diff --git a/plugins/ObjectCacheServer/server/taskfile.yaml b/plugins/ObjectCacheServer/server/taskfile.yaml index 83b8e70..74880b5 100644 --- a/plugins/ObjectCacheServer/server/taskfile.yaml +++ b/plugins/ObjectCacheServer/server/taskfile.yaml @@ -34,7 +34,7 @@ tasks: VNLIB_SHARED_HEAP_FILE_PATH: lib/libvn_rpmalloc cmds: - - cmd: dotnet webserver/VNLib.WebServer.dll --config config/config.json --input-off {{.CLI_ARGS}} + - cmd: dotnet webserver/VNLib.WebServer.dll --config config/config.yaml --input-off {{.CLI_ARGS}} #setup sever environment setup-apt: diff --git a/plugins/VNLib.Data.Caching.Providers.VNCache/src/FBMCacheClient.cs b/plugins/VNLib.Data.Caching.Providers.VNCache/src/FBMCacheClient.cs index 28ccdcd..af12b32 100644 --- a/plugins/VNLib.Data.Caching.Providers.VNCache/src/FBMCacheClient.cs +++ b/plugins/VNLib.Data.Caching.Providers.VNCache/src/FBMCacheClient.cs @@ -112,7 +112,7 @@ namespace VNLib.Data.Caching.Providers.VNCache private FBMCacheClient(VnCacheClientConfig config, ILogProvider? debugLog, PluginBase? plugin) : base(config) { //Validate config - (config as IOnConfigValidation).Validate(); + (config as IOnConfigValidation).OnValidate(); _config = config; diff --git a/plugins/VNLib.Data.Caching.Providers.VNCache/src/MemoryCache.cs b/plugins/VNLib.Data.Caching.Providers.VNCache/src/MemoryCache.cs index 0c1a2b5..f2af461 100644 --- a/plugins/VNLib.Data.Caching.Providers.VNCache/src/MemoryCache.cs +++ b/plugins/VNLib.Data.Caching.Providers.VNCache/src/MemoryCache.cs @@ -75,7 +75,7 @@ namespace VNLib.Data.Caching.Providers.VNCache private MemoryCache(MemoryCacheConfig config, bool isDebug, ILogProvider? log, BucketLocalManagerFactory? factory) : base(config) { //Validate config - config.Validate(); + config.OnValidate(); if (isDebug) { diff --git a/plugins/VNLib.Data.Caching.Providers.VNCache/src/MemoryCacheConfig.cs b/plugins/VNLib.Data.Caching.Providers.VNCache/src/MemoryCacheConfig.cs index 5e53af7..177ba04 100644 --- a/plugins/VNLib.Data.Caching.Providers.VNCache/src/MemoryCacheConfig.cs +++ b/plugins/VNLib.Data.Caching.Providers.VNCache/src/MemoryCacheConfig.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Data.Caching.Providers.VNCache @@ -25,6 +25,8 @@ using System; using System.Text.Json.Serialization; +using VNLib.Plugins.Extensions.Loading.Configuration; + namespace VNLib.Data.Caching.Providers.VNCache { /// <summary> @@ -45,19 +47,12 @@ namespace VNLib.Data.Caching.Providers.VNCache public uint BucketSize { get; set; } = 5000; ///<inheritdoc/> - public override void Validate() + public override void OnValidate() { - base.Validate(); - - if (TableSize == 0) - { - throw new ArgumentException("You must specify a cache bucket table size", "buckets"); - } + base.OnValidate(); - if (BucketSize == 0) - { - throw new ArgumentException("You must specify the maxium number of entires allowed in each bucket ", "bucket_size"); - } + Validate.Assert(TableSize > 0, "You must specify a number of cache buckets"); + Validate.Assert(BucketSize > 0, "You must specify a 'bucket_size'"); } } }
\ No newline at end of file diff --git a/plugins/VNLib.Data.Caching.Providers.VNCache/src/RemoteBackedMemoryCache.cs b/plugins/VNLib.Data.Caching.Providers.VNCache/src/RemoteBackedMemoryCache.cs index ddc6c4b..340d23c 100644 --- a/plugins/VNLib.Data.Caching.Providers.VNCache/src/RemoteBackedMemoryCache.cs +++ b/plugins/VNLib.Data.Caching.Providers.VNCache/src/RemoteBackedMemoryCache.cs @@ -86,7 +86,7 @@ namespace VNLib.Data.Caching.Providers.VNCache ArgumentNullException.ThrowIfNull(memCache); ArgumentNullException.ThrowIfNull(backingStore); - memCache.Validate(); + memCache.OnValidate(); /* * If no buffer factory was supplied, we can create one, but it has to be diff --git a/plugins/VNLib.Data.Caching.Providers.VNCache/src/VNCacheConfig.cs b/plugins/VNLib.Data.Caching.Providers.VNCache/src/VNCacheConfig.cs index 24008b3..1911649 100644 --- a/plugins/VNLib.Data.Caching.Providers.VNCache/src/VNCacheConfig.cs +++ b/plugins/VNLib.Data.Caching.Providers.VNCache/src/VNCacheConfig.cs @@ -26,6 +26,7 @@ using System; using System.Text.Json.Serialization; using VNLib.Plugins.Extensions.Loading; +using VNLib.Plugins.Extensions.Loading.Configuration; namespace VNLib.Data.Caching.Providers.VNCache { @@ -91,12 +92,11 @@ namespace VNLib.Data.Caching.Providers.VNCache [JsonPropertyName("max_object_size")] public virtual uint MaxBlobSize { get; set; } = 16 * 1024; - public virtual void Validate() + public virtual void OnValidate() { - if (MaxBlobSize < 16) - { - throw new ArgumentException("You must configure a maximum object size", "max_object_size"); - } + + Validate.Range2<uint>(MaxBlobSize, 16, uint.MaxValue, "You must configure a maximum object size"); + } /// <summary> diff --git a/plugins/VNLib.Data.Caching.Providers.VNCache/src/VnCacheClientConfig.cs b/plugins/VNLib.Data.Caching.Providers.VNCache/src/VnCacheClientConfig.cs index 0d6cd34..6ee410c 100644 --- a/plugins/VNLib.Data.Caching.Providers.VNCache/src/VnCacheClientConfig.cs +++ b/plugins/VNLib.Data.Caching.Providers.VNCache/src/VnCacheClientConfig.cs @@ -26,6 +26,8 @@ using System; using System.Linq; using System.Text.Json.Serialization; +using VNLib.Plugins.Extensions.Loading.Configuration; + namespace VNLib.Data.Caching.Providers.VNCache { /// <summary> @@ -100,40 +102,27 @@ namespace VNLib.Data.Caching.Providers.VNCache } ///<inheritdoc/> - public override void Validate() + public override void OnValidate() { - base.Validate(); + base.OnValidate(); - if (!DiscoveryIntervalSeconds.HasValue || DiscoveryIntervalSeconds.Value < 1) - { - throw new ArgumentException("You must specify a discovery interval period greater than 0", "retry_interval_sec"); - } + Validate.Assert(DiscoveryIntervalSeconds.HasValue, "A discovery interval is required"); + Validate.Range(DiscoveryIntervalSeconds.Value, 1, int.MaxValue); - //Allow a 0 timeout to disable timeouts, not recommended, but allowed - if (!RequestTimeoutSeconds.HasValue || RequestTimeoutSeconds.Value < 0) - { - throw new ArgumentException("You must specify a positive integer FBM message timoeut", "request_timeout_sec"); - } + Validate.Assert(RequestTimeoutSeconds.HasValue, "A request timeout is required"); + Validate.Range(RequestTimeoutSeconds.Value, 1, int.MaxValue); - //Validate initial nodes - if (InitialNodes == null || InitialNodes.Length == 0) - { - throw new ArgumentException("You must specify at least one initial peer", "initial_peers"); - } + Validate.NotNull(InitialNodes, "You must specify at least one initial cache node to connect to"); + Validate.Assert(InitialNodes.Length > 0, "You must specify at least one initial cache node to connect to"); //Validate initial nodes foreach (Uri peer in GetInitialNodeUris()) { - if (!peer.IsAbsoluteUri) - { - throw new ArgumentException("You must specify an absolute URI for each initial node", "initial_nodes"); - } - - //Verify http connection - if (peer.Scheme != Uri.UriSchemeHttp && peer.Scheme != Uri.UriSchemeHttps) - { - throw new ArgumentException("You must specify an HTTP or HTTPS URI for each initial node", "initial_nodes"); - } + Validate.Assert(peer.IsAbsoluteUri, "You must specify an absolute URI for each initial node"); + Validate.Assert( + peer.Scheme == Uri.UriSchemeHttp || peer.Scheme == Uri.UriSchemeHttps, + message: "You must specify an HTTP or HTTPS URI for each initial node" + ); } } } |