Update and expose storage config for container

8 files changed, 111 insertions, 89 deletions

diff --git a/back-end/src/CMNext.sample.json b/back-end/src/CMNext.sample.json
index 1b3c516..8af8e8d 100644
--- a/back-end/src/CMNext.sample.json
+++ b/back-end/src/CMNext.sample.json
@@ -20,25 +20,22 @@
     "index_file_name": "blogs/channels.json"
   },
 
-  //S3 setup with vault secrets
-  "disabled s3_config": {
+  "storage": {
+    //The custom storage assembly to use
+    "custom_storage_assembly": null,
+
+    "type": "s3", //s3 | ftp
+
+    //s3 config
     "server_address": "",
     "access_key": "",
     "bucket": "",
-    "use_ssl": true,
+    "use_ssl": false,
     "Region": null
   },
 
-  "disabled ftp_config": {
-    "url": "",
-    "username": "",
-    //Base path within the ftp user's directory
-    "base_path": ""
-  },
-
   "secrets": {
     //Set the vault path to the s3 secret
-    "s3_secret": "",
-    "ftp_password": ""
+    "storage_secret": "",
   }
 }
\ No newline at end of file
diff --git a/back-end/src/Storage/FtpStorageManager.cs b/back-end/src/Storage/FtpStorageManager.cs
index d64d4ea..d775d54 100644
--- a/back-end/src/Storage/FtpStorageManager.cs
+++ b/back-end/src/Storage/FtpStorageManager.cs
@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2023 Vaughn Nugent
+* Copyright (c) 2024 Vaughn Nugent
 * 
 * Library: CMNext
 * Package: Content.Publishing.Blog.Admin
@@ -24,7 +24,6 @@ using System.IO;
 using System.Net;
 using System.Threading;
 using System.Threading.Tasks;
-using System.Collections.Generic;
 
 using FluentFTP;
 using FluentFTP.Exceptions;
@@ -36,40 +35,47 @@ using VNLib.Plugins.Extensions.Loading;
 
 namespace Content.Publishing.Blog.Admin.Storage
 {
-    [ConfigurationName("ftp_config")]
+
+    [ConfigurationName("storage")]
     internal class FtpStorageManager : StorageBase, IDisposable
     {
         private readonly AsyncFtpClient _client;
-        private readonly string _username;
-        private readonly string? _baasePath;
+        private readonly S3Config _storageConf;
 
-        protected override string? BasePath => _baasePath;
+        protected override string? BasePath => _storageConf.BaseBucket;
 
         public FtpStorageManager(PluginBase plugin, IConfigScope config)
         {
-            string? url = config["url"].GetString();
-            _username = config["username"].GetString() ?? throw new KeyNotFoundException("Missing required username in config");
-            _baasePath = config["base_path"].GetString();
+            _storageConf = config.Deserialze<S3Config>();
 
-            Uri uri = new (url!);
+            Uri uri = new (_storageConf.ServerAddress!);
 
             //Init new client
             _client = new(
                 uri.Host, 
                 uri.Port, 
                 //Logger in debug mode
-                logger:plugin.IsDebug() ? new FtpDebugLogger(plugin.Log) : null
+                logger: plugin.IsDebug() ? new FtpDebugLogger(plugin.Log) : null
             );
         }
 
         public override async Task ConfigureServiceAsync(PluginBase plugin)
         {
-            using ISecretResult password = await plugin.GetSecretAsync("ftp_password");
+            using ISecretResult password = await plugin.Secrets().GetSecretAsync("storage_secret");
 
             //Init client credentials
-            _client.Credentials = new NetworkCredential(_username, password?.Result.ToString());
-            _client.Config.EncryptionMode = FtpEncryptionMode.Auto;
-            _client.Config.ValidateAnyCertificate = true;
+            _client.Credentials = new NetworkCredential(_storageConf.ClientId, password?.Result.ToString());
+
+            //If the user forces ssl, then assume it's an implicit connection and force certificate checking
+            if(_storageConf.UseSsl == true)
+            {
+                _client.Config.EncryptionMode = FtpEncryptionMode.Implicit;
+            }
+            else
+            {
+                _client.Config.EncryptionMode = FtpEncryptionMode.Auto;
+                _client.Config.ValidateAnyCertificate = true;
+            }
 
             plugin.Log.Information("Connecting to ftp server");
 
@@ -115,7 +121,7 @@ namespace Content.Publishing.Blog.Admin.Storage
         ///<inheritdoc/>
         public override string GetExternalFilePath(string filePath)
         {
-            return string.IsNullOrWhiteSpace(_baasePath) ? filePath : $"{_baasePath}/{filePath}";
+            return string.IsNullOrWhiteSpace(BasePath) ? filePath : $"{BasePath}/{filePath}";
         }
 
         public void Dispose()
@@ -123,7 +129,7 @@ namespace Content.Publishing.Blog.Admin.Storage
             _client?.Dispose();
         }
 
-        sealed record class FtpDebugLogger(ILogProvider Log) : IFtpLogger
+        sealed class FtpDebugLogger(ILogProvider Log) : IFtpLogger
         {
             void IFtpLogger.Log(FtpLogEntry entry)
             {
diff --git a/back-end/src/Storage/ManagedStorage.cs b/back-end/src/Storage/ManagedStorage.cs
index 66e9a4a..a28ff50 100644
--- a/back-end/src/Storage/ManagedStorage.cs
+++ b/back-end/src/Storage/ManagedStorage.cs
@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2023 Vaughn Nugent
+* Copyright (c) 2024 Vaughn Nugent
 * 
 * Library: CMNext
 * Package: Content.Publishing.Blog.Admin
@@ -30,28 +30,27 @@ using VNLib.Plugins.Extensions.Loading;
 
 namespace Content.Publishing.Blog.Admin.Storage
 {
-    [ConfigurationName("storage", Required = false)]
+    [ConfigurationName("storage")]
     internal sealed class ManagedStorage : ISimpleFilesystem
     {
-        private readonly ISimpleFilesystem _backingStorage;
+        private readonly ISimpleFilesystem _backingStorage;      
 
-        public ManagedStorage(PluginBase plugin) : this(plugin, null)
-        { }
-
-        public ManagedStorage(PluginBase plugin, IConfigScope? config)
+        public ManagedStorage(PluginBase plugin, IConfigScope config)
         {
+            string type = config.GetRequiredProperty("type", p => p.GetString()!);
+
             //try to get custom storage assembly
-            if (config != null && config.ContainsKey("custom_storage_assembly"))
+            if (config.TryGetProperty("custom_storage_assembly", p => p.GetString(), out string? storageAssembly) 
+                && !string.IsNullOrWhiteSpace(storageAssembly))
             {
-                string storageAssembly = config.GetRequiredProperty("custom_storage_assembly", p => p.GetString()!);
-                _backingStorage = plugin.CreateServiceExternal<ISimpleFilesystem>(storageAssembly);
+                _backingStorage = plugin.CreateServiceExternal<ISimpleFilesystem>(storageAssembly!);
             }
-            else if (plugin.HasConfigForType<MinioClientManager>())
+            else if (string.Equals(type, "s3", StringComparison.OrdinalIgnoreCase))
             {
                 //Use minio storage
                 _backingStorage = plugin.GetOrCreateSingleton<MinioClientManager>();
             }
-            else if (plugin.HasConfigForType<FtpStorageManager>())
+            else if (string.Equals(type, "ftp", StringComparison.OrdinalIgnoreCase))
             {
                 //Use ftp storage
                 _backingStorage = plugin.GetOrCreateSingleton<FtpStorageManager>();
diff --git a/back-end/src/Storage/MinioClientManager.cs b/back-end/src/Storage/MinioClientManager.cs
index c7867b2..0c161af 100644
--- a/back-end/src/Storage/MinioClientManager.cs
+++ b/back-end/src/Storage/MinioClientManager.cs
@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2023 Vaughn Nugent
+* Copyright (c) 2024 Vaughn Nugent
 * 
 * Library: CMNext
 * Package: Content.Publishing.Blog.Admin
@@ -37,18 +37,11 @@ using static Content.Publishing.Blog.Admin.Model.PostManager;
 namespace Content.Publishing.Blog.Admin.Storage
 {
 
-    [ConfigurationName("s3_config")]
-    internal sealed class MinioClientManager : StorageBase
+    [ConfigurationName("storage")]
+    internal sealed class MinioClientManager(PluginBase pbase, IConfigScope s3Config) : StorageBase
     {
-        private readonly MinioClient Client;
-        private readonly S3Config Config;
-
-        public MinioClientManager(PluginBase pbase, IConfigScope s3Config)
-        {
-            //Deserialize the config
-            Config = s3Config.Deserialze<S3Config>();
-            Client = new();
-        }
+        private readonly MinioClient Client = new();
+        private readonly S3Config Config = s3Config.Deserialze<S3Config>();
 
         ///<inheritdoc/>
         protected override string? BasePath => Config.BaseBucket;
@@ -56,12 +49,11 @@ namespace Content.Publishing.Blog.Admin.Storage
         ///<inheritdoc/>
         public override async Task ConfigureServiceAsync(PluginBase plugin)
         {
-            using ISecretResult? secret = await plugin.GetSecretAsync("s3_secret");
+            using ISecretResult? secret = await plugin.GetSecretAsync("storage_secret");
 
             Client.WithEndpoint(Config.ServerAddress)
-                    .WithCredentials(Config.ClientId, secret.Result.ToString());
-
-            Client.WithSSL(Config.UseSsl.HasValue && Config.UseSsl.Value);
+                    .WithCredentials(Config.ClientId, secret.Result.ToString())
+                    .WithSSL(Config.UseSsl == true);
 
             //Accept optional region
             if (!string.IsNullOrWhiteSpace(Config.Region))
diff --git a/ci/build.env b/ci/build.env
index cfdf552..0c8da94 100644
--- a/ci/build.env
+++ b/ci/build.env
@@ -16,6 +16,19 @@ CHANNEL_INDEX_FILE=blogs/channels.json
 MAX_LOGIN_ATTEMPS=10
 
 ##########
+#  Storage
+##########
+
+STORAGE_CUSTOM_ASSEMBLY=
+STORAGE_TYPE=
+STORAGE_SERVER_ADDRESS=
+STORAGE_USERNAME=
+STORAGE_BUCKET=
+STORAGE_USE_SSL=true
+STORAGE_PASSWORD=
+S3_REGION=
+
+##########
 #  HTTP
 ##########
 
diff --git a/ci/config-templates/CMNext.json b/ci/config-templates/CMNext-template.json
index ab74578..486f68a 100644
--- a/ci/config-templates/CMNext.json
+++ b/ci/config-templates/CMNext-template.json
@@ -20,25 +20,22 @@
     "index_file_name": "${CHANNEL_INDEX_FILE}"
   },
 
-  //S3 setup with vault secrets
-  "disabled s3_config": {
-    "server_address": "",
-    "access_key": "",
-    "bucket": "",
-    "use_ssl": true,
-    "Region": null
-  },
+  "storage": {
+
+    "custom_storage_assembly": "${STORAGE_CUSTOM_ASSEMBLY}",
+
+    "type": "${STORAGE_TYPE}", //s3 | ftp
 
-  "disabled ftp_config": {
-    "url": "",
-    "username": "",
-    //Base path within the ftp user's directory
-    "base_path": ""
+    //storage config
+    "server_address": "${STORAGE_SERVER_ADDRESS}",
+    "access_key": "${STORAGE_USERNAME}",
+    "bucket": "${STORAGE_BUCKET}",
+    "use_ssl": ${STORAGE_USE_SSL},
+    "Region": "${S3_REGION}"
   },
 
   "secrets": {
     //Set the vault path to the s3 secret
-    "s3_secret": "",
-    "ftp_password": ""
+    "storage_secret": "${STORAGE_SECRET}"
   }
 }
\ No newline at end of file
diff --git a/ci/container/Dockerfile b/ci/container/Dockerfile
index 8e6a11e..8ae12f6 100644
--- a/ci/container/Dockerfile
+++ b/ci/container/Dockerfile
@@ -56,14 +56,21 @@ ENV VNLIB_ARGON2_DLL_PATH=/app/lib/libargon2.so
 
 #set default env variables
 ENV MAX_CONTENT_LENGTH=204800000   \
- REG_TOKEN_DURATION_MIN=360
+ REG_TOKEN_DURATION_MIN=360     \
+ MAX_LOGIN_ATTEMPS=10
 
 #SQL Config
 ENV SQL_LIB_PATH=VNLib.Plugins.Extensions.Sql.SQLite.dll
 ENV SQL_CONNECTION_STRING="Data Source=data/cmnext.db;"
 
-#ACCOUNTS
-ENV MAX_LOGIN_ATTEMPS=10
+#STORAGE 
+ENV STORAGE_TYPE="s3"           \
+ STORAGE_CUSTOM_ASSEMBLY=""     \
+ STORAGE_SERVER_ADDRESS=""      \
+ STORAGE_USERNAME=""            \
+ STORAGE_BUCKET=""              \
+ STORAGE_USE_SSL=true           \
+ S3_REGION=""
 
 #HC Vault
 ENV HC_VAULT_ADDR=""            \
@@ -81,7 +88,8 @@ ENV PASSWORD_PEPPER=""          \
  DATABASE_PASSWORD=""           \
  REDIS_PASSWORD=""              \
  VNCACHE_CLIENT_PRIVATE_KEY=""  \
- VNCACHE_CACHE_PUBLIC_KEY=""
+ VNCACHE_CACHE_PUBLIC_KEY=""    \
+ STORAGE_SECRET=""
 
 
 #HTTP/PROXY Config
diff --git a/ci/container/docker-compose.yaml b/ci/container/docker-compose.yaml
index d281c2c..5a029c6 100644
--- a/ci/container/docker-compose.yaml
+++ b/ci/container/docker-compose.yaml
@@ -17,12 +17,22 @@ services:
     ports:
       - 8080:8080
     environment:
-      CHANNEL_INDEX_FILE: "channels.json"
-      MAX_CONTENT_LENGTH: 204800000   #200MB
+      CHANNEL_INDEX_FILE: "channels.json"   #required, should leave default unless you know what you are doing
+      MAX_CONTENT_LENGTH: 204800000         #200MB max upload size
+      MAX_LOGIN_ATTEMPS: "10"               #max login attempts before user account is locked out
       
       #SQL Config
       SQL_LIB_PATH: "VNLib.Plugins.Extensions.Sql.SQLite.dll"
-      SQL_CONNECTION_STRING: "Data Source=data/cmnext.db;"
+      SQL_CONNECTION_STRING: "Data Source=data/cmnext.db;"    #when using a password, simply leave the password field blank
+
+      #storage backend setup
+      STORAGE_TYPE: "s3"              #s3 | ftp
+      STORAGE_CUSTOM_ASSEMBLY: ""     #optional path to a custom storage assembly
+      STORAGE_SERVER_ADDRESS: ""      #s3 or ftp server address
+      STORAGE_USERNAME: ""            #s3 client id or ftp username
+      STORAGE_BUCKET: ""              #s3 bucket or ftp root directory
+      STORAGE_USE_SSL: "true"         #force ssl for connections
+      S3_REGION: ""                   #optional s3 region when using s3 storage
 
       #HC Vault client config
       #HC_VAULT_ADDR: ""
@@ -36,27 +46,27 @@ services:
       #at least one node required if MEMCACHE_ONLY is false
       VNCACHE_INITIAL_NODES: "[]"
 
-      #Accounts plugin config
-      MAX_LOGIN_ATTEMPS: "10"
-
       #SECRETS
-      #All secrets may be a raw value, read from a file, 
-      #an environment variable, or a vault path
+      # All secrets may be a raw value, read from a file, 
+      # an environment variable, or a vault path
       # file://mysecret.txt     reads the secret from a file (case sensitive)
       # env://MY_SECRET         reads the secret from an environment variable (case sensitive)
       # vault://kv/data/secret?secret=value   reads the value of the mysecret key in the secret/data path
+
       PASSWORD_PEPPER: ""     #Must be a base64 encoded value, of realtivley any size
-      DATABASE_PASSWORD: ""
-      REDIS_PASSWORD: ""
+      DATABASE_PASSWORD: ""   #overrides the 'Password' field in the SQL connection string
+      REDIS_PASSWORD: ""      #only required if using a password protected redis server
       #if MEMCACHE_ONLY is false, then the following keys are required to connect to a VNCACHE cluster
       VNCACHE_CLIENT_PRIVATE_KEY: ""
       VNCACHE_CACHE_PUBLIC_KEY: ""
+      #REQUIRED s3 or ftp secret key
+      STORAGE_SECRET: ""
 
       #HTTP
-      HTTP_DOWNSTREAM_SERVERS: '[]'   #a comma separated list of downstream ip addresses
-      HTTP_TRACE_ON: "false"          #enable http trace logging, requires --debug CLI flag
+      HTTP_DOWNSTREAM_SERVERS: '[]'   #a comma separated list of downstream (proxy) server ip addresses
+      HTTP_TRACE_ON: "false"          #enable http trace logging, requires you to set --debug to SERVER_ARGS variable below
 
-      #Very Verbose plugin logging, required --debug CLI flag, prints literally everything to the logger
+      #Very Verbose plugin logging, required --debug CLI flag, prints literally everything to the logger (it's annoying)
       DEBUG_PLUGINS: "false"
 
       SERVER_ARGS: "--setup" #remove the setup flag after you are done setting up the server