From 5ecd6b39cccdc9500540b10685605b5fcba61f69 Mon Sep 17 00:00:00 2001 From: vnugent Date: Thu, 6 Jun 2024 17:19:48 -0400 Subject: Update and expose storage config for container --- back-end/src/CMNext.sample.json | 21 ++++++-------- back-end/src/Storage/FtpStorageManager.cs | 40 +++++++++++++++------------ back-end/src/Storage/ManagedStorage.cs | 23 ++++++++-------- back-end/src/Storage/MinioClientManager.cs | 24 ++++++---------- ci/build.env | 13 +++++++++ ci/config-templates/CMNext-template.json | 41 ++++++++++++++++++++++++++++ ci/config-templates/CMNext.json | 44 ------------------------------ ci/container/Dockerfile | 16 ++++++++--- ci/container/docker-compose.yaml | 36 +++++++++++++++--------- 9 files changed, 140 insertions(+), 118 deletions(-) create mode 100644 ci/config-templates/CMNext-template.json delete mode 100644 ci/config-templates/CMNext.json diff --git a/back-end/src/CMNext.sample.json b/back-end/src/CMNext.sample.json index 1b3c516..8af8e8d 100644 --- a/back-end/src/CMNext.sample.json +++ b/back-end/src/CMNext.sample.json @@ -20,25 +20,22 @@ "index_file_name": "blogs/channels.json" }, - //S3 setup with vault secrets - "disabled s3_config": { + "storage": { + //The custom storage assembly to use + "custom_storage_assembly": null, + + "type": "s3", //s3 | ftp + + //s3 config "server_address": "", "access_key": "", "bucket": "", - "use_ssl": true, + "use_ssl": false, "Region": null }, - "disabled ftp_config": { - "url": "", - "username": "", - //Base path within the ftp user's directory - "base_path": "" - }, - "secrets": { //Set the vault path to the s3 secret - "s3_secret": "", - "ftp_password": "" + "storage_secret": "", } } \ No newline at end of file diff --git a/back-end/src/Storage/FtpStorageManager.cs b/back-end/src/Storage/FtpStorageManager.cs index d64d4ea..d775d54 100644 --- a/back-end/src/Storage/FtpStorageManager.cs +++ b/back-end/src/Storage/FtpStorageManager.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: CMNext * Package: Content.Publishing.Blog.Admin @@ -24,7 +24,6 @@ using System.IO; using System.Net; using System.Threading; using System.Threading.Tasks; -using System.Collections.Generic; using FluentFTP; using FluentFTP.Exceptions; @@ -36,40 +35,47 @@ using VNLib.Plugins.Extensions.Loading; namespace Content.Publishing.Blog.Admin.Storage { - [ConfigurationName("ftp_config")] + + [ConfigurationName("storage")] internal class FtpStorageManager : StorageBase, IDisposable { private readonly AsyncFtpClient _client; - private readonly string _username; - private readonly string? _baasePath; + private readonly S3Config _storageConf; - protected override string? BasePath => _baasePath; + protected override string? BasePath => _storageConf.BaseBucket; public FtpStorageManager(PluginBase plugin, IConfigScope config) { - string? url = config["url"].GetString(); - _username = config["username"].GetString() ?? throw new KeyNotFoundException("Missing required username in config"); - _baasePath = config["base_path"].GetString(); + _storageConf = config.Deserialze(); - Uri uri = new (url!); + Uri uri = new (_storageConf.ServerAddress!); //Init new client _client = new( uri.Host, uri.Port, //Logger in debug mode - logger:plugin.IsDebug() ? new FtpDebugLogger(plugin.Log) : null + logger: plugin.IsDebug() ? new FtpDebugLogger(plugin.Log) : null ); } public override async Task ConfigureServiceAsync(PluginBase plugin) { - using ISecretResult password = await plugin.GetSecretAsync("ftp_password"); + using ISecretResult password = await plugin.Secrets().GetSecretAsync("storage_secret"); //Init client credentials - _client.Credentials = new NetworkCredential(_username, password?.Result.ToString()); - _client.Config.EncryptionMode = FtpEncryptionMode.Auto; - _client.Config.ValidateAnyCertificate = true; + _client.Credentials = new NetworkCredential(_storageConf.ClientId, password?.Result.ToString()); + + //If the user forces ssl, then assume it's an implicit connection and force certificate checking + if(_storageConf.UseSsl == true) + { + _client.Config.EncryptionMode = FtpEncryptionMode.Implicit; + } + else + { + _client.Config.EncryptionMode = FtpEncryptionMode.Auto; + _client.Config.ValidateAnyCertificate = true; + } plugin.Log.Information("Connecting to ftp server"); @@ -115,7 +121,7 @@ namespace Content.Publishing.Blog.Admin.Storage /// public override string GetExternalFilePath(string filePath) { - return string.IsNullOrWhiteSpace(_baasePath) ? filePath : $"{_baasePath}/{filePath}"; + return string.IsNullOrWhiteSpace(BasePath) ? filePath : $"{BasePath}/{filePath}"; } public void Dispose() @@ -123,7 +129,7 @@ namespace Content.Publishing.Blog.Admin.Storage _client?.Dispose(); } - sealed record class FtpDebugLogger(ILogProvider Log) : IFtpLogger + sealed class FtpDebugLogger(ILogProvider Log) : IFtpLogger { void IFtpLogger.Log(FtpLogEntry entry) { diff --git a/back-end/src/Storage/ManagedStorage.cs b/back-end/src/Storage/ManagedStorage.cs index 66e9a4a..a28ff50 100644 --- a/back-end/src/Storage/ManagedStorage.cs +++ b/back-end/src/Storage/ManagedStorage.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: CMNext * Package: Content.Publishing.Blog.Admin @@ -30,28 +30,27 @@ using VNLib.Plugins.Extensions.Loading; namespace Content.Publishing.Blog.Admin.Storage { - [ConfigurationName("storage", Required = false)] + [ConfigurationName("storage")] internal sealed class ManagedStorage : ISimpleFilesystem { - private readonly ISimpleFilesystem _backingStorage; + private readonly ISimpleFilesystem _backingStorage; - public ManagedStorage(PluginBase plugin) : this(plugin, null) - { } - - public ManagedStorage(PluginBase plugin, IConfigScope? config) + public ManagedStorage(PluginBase plugin, IConfigScope config) { + string type = config.GetRequiredProperty("type", p => p.GetString()!); + //try to get custom storage assembly - if (config != null && config.ContainsKey("custom_storage_assembly")) + if (config.TryGetProperty("custom_storage_assembly", p => p.GetString(), out string? storageAssembly) + && !string.IsNullOrWhiteSpace(storageAssembly)) { - string storageAssembly = config.GetRequiredProperty("custom_storage_assembly", p => p.GetString()!); - _backingStorage = plugin.CreateServiceExternal(storageAssembly); + _backingStorage = plugin.CreateServiceExternal(storageAssembly!); } - else if (plugin.HasConfigForType()) + else if (string.Equals(type, "s3", StringComparison.OrdinalIgnoreCase)) { //Use minio storage _backingStorage = plugin.GetOrCreateSingleton(); } - else if (plugin.HasConfigForType()) + else if (string.Equals(type, "ftp", StringComparison.OrdinalIgnoreCase)) { //Use ftp storage _backingStorage = plugin.GetOrCreateSingleton(); diff --git a/back-end/src/Storage/MinioClientManager.cs b/back-end/src/Storage/MinioClientManager.cs index c7867b2..0c161af 100644 --- a/back-end/src/Storage/MinioClientManager.cs +++ b/back-end/src/Storage/MinioClientManager.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: CMNext * Package: Content.Publishing.Blog.Admin @@ -37,18 +37,11 @@ using static Content.Publishing.Blog.Admin.Model.PostManager; namespace Content.Publishing.Blog.Admin.Storage { - [ConfigurationName("s3_config")] - internal sealed class MinioClientManager : StorageBase + [ConfigurationName("storage")] + internal sealed class MinioClientManager(PluginBase pbase, IConfigScope s3Config) : StorageBase { - private readonly MinioClient Client; - private readonly S3Config Config; - - public MinioClientManager(PluginBase pbase, IConfigScope s3Config) - { - //Deserialize the config - Config = s3Config.Deserialze(); - Client = new(); - } + private readonly MinioClient Client = new(); + private readonly S3Config Config = s3Config.Deserialze(); /// protected override string? BasePath => Config.BaseBucket; @@ -56,12 +49,11 @@ namespace Content.Publishing.Blog.Admin.Storage /// public override async Task ConfigureServiceAsync(PluginBase plugin) { - using ISecretResult? secret = await plugin.GetSecretAsync("s3_secret"); + using ISecretResult? secret = await plugin.GetSecretAsync("storage_secret"); Client.WithEndpoint(Config.ServerAddress) - .WithCredentials(Config.ClientId, secret.Result.ToString()); - - Client.WithSSL(Config.UseSsl.HasValue && Config.UseSsl.Value); + .WithCredentials(Config.ClientId, secret.Result.ToString()) + .WithSSL(Config.UseSsl == true); //Accept optional region if (!string.IsNullOrWhiteSpace(Config.Region)) diff --git a/ci/build.env b/ci/build.env index cfdf552..0c8da94 100644 --- a/ci/build.env +++ b/ci/build.env @@ -15,6 +15,19 @@ DEBUG_PLUGINS=false CHANNEL_INDEX_FILE=blogs/channels.json MAX_LOGIN_ATTEMPS=10 +########## +# Storage +########## + +STORAGE_CUSTOM_ASSEMBLY= +STORAGE_TYPE= +STORAGE_SERVER_ADDRESS= +STORAGE_USERNAME= +STORAGE_BUCKET= +STORAGE_USE_SSL=true +STORAGE_PASSWORD= +S3_REGION= + ########## # HTTP ########## diff --git a/ci/config-templates/CMNext-template.json b/ci/config-templates/CMNext-template.json new file mode 100644 index 0000000..486f68a --- /dev/null +++ b/ci/config-templates/CMNext-template.json @@ -0,0 +1,41 @@ +{ + //Enables debug logging + "debug": ${DEBUG_PLUGINS}, + + "post_endpoint": { + "path": "/api/blog/posts" + }, + + "channel_endpoint": { + "path": "/api/blog/channels" + }, + + "content_endpoint": { + "path": "/api/blog/content", + "max_content_length": ${MAX_CONTENT_LENGTH} + }, + + "blog_channels": { + //The index file for storing channel configuration + "index_file_name": "${CHANNEL_INDEX_FILE}" + }, + + "storage": { + + "custom_storage_assembly": "${STORAGE_CUSTOM_ASSEMBLY}", + + "type": "${STORAGE_TYPE}", //s3 | ftp + + //storage config + "server_address": "${STORAGE_SERVER_ADDRESS}", + "access_key": "${STORAGE_USERNAME}", + "bucket": "${STORAGE_BUCKET}", + "use_ssl": ${STORAGE_USE_SSL}, + "Region": "${S3_REGION}" + }, + + "secrets": { + //Set the vault path to the s3 secret + "storage_secret": "${STORAGE_SECRET}" + } +} \ No newline at end of file diff --git a/ci/config-templates/CMNext.json b/ci/config-templates/CMNext.json deleted file mode 100644 index ab74578..0000000 --- a/ci/config-templates/CMNext.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - //Enables debug logging - "debug": ${DEBUG_PLUGINS}, - - "post_endpoint": { - "path": "/api/blog/posts" - }, - - "channel_endpoint": { - "path": "/api/blog/channels" - }, - - "content_endpoint": { - "path": "/api/blog/content", - "max_content_length": ${MAX_CONTENT_LENGTH} - }, - - "blog_channels": { - //The index file for storing channel configuration - "index_file_name": "${CHANNEL_INDEX_FILE}" - }, - - //S3 setup with vault secrets - "disabled s3_config": { - "server_address": "", - "access_key": "", - "bucket": "", - "use_ssl": true, - "Region": null - }, - - "disabled ftp_config": { - "url": "", - "username": "", - //Base path within the ftp user's directory - "base_path": "" - }, - - "secrets": { - //Set the vault path to the s3 secret - "s3_secret": "", - "ftp_password": "" - } -} \ No newline at end of file diff --git a/ci/container/Dockerfile b/ci/container/Dockerfile index 8e6a11e..8ae12f6 100644 --- a/ci/container/Dockerfile +++ b/ci/container/Dockerfile @@ -56,14 +56,21 @@ ENV VNLIB_ARGON2_DLL_PATH=/app/lib/libargon2.so #set default env variables ENV MAX_CONTENT_LENGTH=204800000 \ - REG_TOKEN_DURATION_MIN=360 + REG_TOKEN_DURATION_MIN=360 \ + MAX_LOGIN_ATTEMPS=10 #SQL Config ENV SQL_LIB_PATH=VNLib.Plugins.Extensions.Sql.SQLite.dll ENV SQL_CONNECTION_STRING="Data Source=data/cmnext.db;" -#ACCOUNTS -ENV MAX_LOGIN_ATTEMPS=10 +#STORAGE +ENV STORAGE_TYPE="s3" \ + STORAGE_CUSTOM_ASSEMBLY="" \ + STORAGE_SERVER_ADDRESS="" \ + STORAGE_USERNAME="" \ + STORAGE_BUCKET="" \ + STORAGE_USE_SSL=true \ + S3_REGION="" #HC Vault ENV HC_VAULT_ADDR="" \ @@ -81,7 +88,8 @@ ENV PASSWORD_PEPPER="" \ DATABASE_PASSWORD="" \ REDIS_PASSWORD="" \ VNCACHE_CLIENT_PRIVATE_KEY="" \ - VNCACHE_CACHE_PUBLIC_KEY="" + VNCACHE_CACHE_PUBLIC_KEY="" \ + STORAGE_SECRET="" #HTTP/PROXY Config diff --git a/ci/container/docker-compose.yaml b/ci/container/docker-compose.yaml index d281c2c..5a029c6 100644 --- a/ci/container/docker-compose.yaml +++ b/ci/container/docker-compose.yaml @@ -17,12 +17,22 @@ services: ports: - 8080:8080 environment: - CHANNEL_INDEX_FILE: "channels.json" - MAX_CONTENT_LENGTH: 204800000 #200MB + CHANNEL_INDEX_FILE: "channels.json" #required, should leave default unless you know what you are doing + MAX_CONTENT_LENGTH: 204800000 #200MB max upload size + MAX_LOGIN_ATTEMPS: "10" #max login attempts before user account is locked out #SQL Config SQL_LIB_PATH: "VNLib.Plugins.Extensions.Sql.SQLite.dll" - SQL_CONNECTION_STRING: "Data Source=data/cmnext.db;" + SQL_CONNECTION_STRING: "Data Source=data/cmnext.db;" #when using a password, simply leave the password field blank + + #storage backend setup + STORAGE_TYPE: "s3" #s3 | ftp + STORAGE_CUSTOM_ASSEMBLY: "" #optional path to a custom storage assembly + STORAGE_SERVER_ADDRESS: "" #s3 or ftp server address + STORAGE_USERNAME: "" #s3 client id or ftp username + STORAGE_BUCKET: "" #s3 bucket or ftp root directory + STORAGE_USE_SSL: "true" #force ssl for connections + S3_REGION: "" #optional s3 region when using s3 storage #HC Vault client config #HC_VAULT_ADDR: "" @@ -36,27 +46,27 @@ services: #at least one node required if MEMCACHE_ONLY is false VNCACHE_INITIAL_NODES: "[]" - #Accounts plugin config - MAX_LOGIN_ATTEMPS: "10" - #SECRETS - #All secrets may be a raw value, read from a file, - #an environment variable, or a vault path + # All secrets may be a raw value, read from a file, + # an environment variable, or a vault path # file://mysecret.txt reads the secret from a file (case sensitive) # env://MY_SECRET reads the secret from an environment variable (case sensitive) # vault://kv/data/secret?secret=value reads the value of the mysecret key in the secret/data path + PASSWORD_PEPPER: "" #Must be a base64 encoded value, of realtivley any size - DATABASE_PASSWORD: "" - REDIS_PASSWORD: "" + DATABASE_PASSWORD: "" #overrides the 'Password' field in the SQL connection string + REDIS_PASSWORD: "" #only required if using a password protected redis server #if MEMCACHE_ONLY is false, then the following keys are required to connect to a VNCACHE cluster VNCACHE_CLIENT_PRIVATE_KEY: "" VNCACHE_CACHE_PUBLIC_KEY: "" + #REQUIRED s3 or ftp secret key + STORAGE_SECRET: "" #HTTP - HTTP_DOWNSTREAM_SERVERS: '[]' #a comma separated list of downstream ip addresses - HTTP_TRACE_ON: "false" #enable http trace logging, requires --debug CLI flag + HTTP_DOWNSTREAM_SERVERS: '[]' #a comma separated list of downstream (proxy) server ip addresses + HTTP_TRACE_ON: "false" #enable http trace logging, requires you to set --debug to SERVER_ARGS variable below - #Very Verbose plugin logging, required --debug CLI flag, prints literally everything to the logger + #Very Verbose plugin logging, required --debug CLI flag, prints literally everything to the logger (it's annoying) DEBUG_PLUGINS: "false" SERVER_ARGS: "--setup" #remove the setup flag after you are done setting up the server -- cgit