diff options
Diffstat (limited to 'back-end/src/Storage')
| -rw-r--r-- | back-end/src/Storage/FtpStorageManager.cs | 136 | ||||
| -rw-r--r-- | back-end/src/Storage/IStorageFacade.cs | 71 | ||||
| -rw-r--r-- | back-end/src/Storage/ManagedStorage.cs | 96 | ||||
| -rw-r--r-- | back-end/src/Storage/MinioClientManager.cs | 135 | ||||
| -rw-r--r-- | back-end/src/Storage/StorageBase.cs | 57 |
5 files changed, 495 insertions, 0 deletions
diff --git a/back-end/src/Storage/FtpStorageManager.cs b/back-end/src/Storage/FtpStorageManager.cs new file mode 100644 index 0000000..abcf5e1 --- /dev/null +++ b/back-end/src/Storage/FtpStorageManager.cs @@ -0,0 +1,136 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: CMNext +* Package: Content.Publishing.Blog.Admin +* File: FtpStorageManager.cs +* +* CMNext is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* CMNext is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using System; +using System.IO; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using System.Collections.Generic; + +using FluentFTP; +using FluentFTP.Exceptions; + +using VNLib.Net.Http; +using VNLib.Utils.Logging; +using VNLib.Utils.Resources; +using VNLib.Plugins; +using VNLib.Plugins.Extensions.Loading; + +namespace Content.Publishing.Blog.Admin.Storage +{ + [ConfigurationName("ftp_config")] + internal class FtpStorageManager : StorageBase, IDisposable + { + private readonly AsyncFtpClient _client; + private readonly string _username; + private readonly string? _baasePath; + + protected override string? BasePath => _baasePath; + + public FtpStorageManager(PluginBase plugin, IConfigScope config) + { + string? url = config["url"].GetString(); + _username = config["username"].GetString() ?? throw new KeyNotFoundException("Missing required username in config"); + _baasePath = config["base_path"].GetString(); + + Uri uri = new (url!); + + //Init new client + _client = new( + uri.Host, + uri.Port, + //Logger in debug mode + logger:plugin.IsDebug() ? new FtpDebugLogger(plugin.Log) : null + ); + } + + public override async Task ConfigureServiceAsync(PluginBase plugin) + { + using ISecretResult password = await plugin.GetSecretAsync("ftp_password"); + + //Init client credentials + _client.Credentials = new NetworkCredential(_username, password?.Result.ToString()); + _client.Config.EncryptionMode = FtpEncryptionMode.Auto; + _client.Config.ValidateAnyCertificate = true; + + plugin.Log.Information("Connecting to ftp server"); + + await _client.AutoConnect(CancellationToken.None); + plugin.Log.Information("Successfully connected to ftp server"); + } + + + ///<inheritdoc/> + public override Task DeleteFileAsync(string filePath, CancellationToken cancellation) + { + return _client.DeleteFile(GetExternalFilePath(filePath), cancellation); + } + + ///<inheritdoc/> + public override async Task<long> ReadFileAsync(string filePath, Stream output, CancellationToken cancellation) + { + try + { + //Read the file + await _client.DownloadStream(output, GetExternalFilePath(filePath), token: cancellation); + return output.Position; + } + catch (FtpMissingObjectException) + { + //File not found + return -1; + } + } + + ///<inheritdoc/> + public override async Task SetFileAsync(string filePath, Stream data, ContentType ct, CancellationToken cancellation) + { + //Upload the file to the server + FtpStatus status = await _client.UploadStream(data, GetExternalFilePath(filePath), FtpRemoteExists.Overwrite, true, token: cancellation); + + if (status == FtpStatus.Failed) + { + throw new ResourceUpdateFailedException($"Failed to update the remote resource {filePath}"); + } + } + + ///<inheritdoc/> + public override string GetExternalFilePath(string filePath) + { + return string.IsNullOrWhiteSpace(_baasePath) ? filePath : $"{_baasePath}/{filePath}"; + } + + public void Dispose() + { + _client?.Dispose(); + } + + sealed record class FtpDebugLogger(ILogProvider Log) : IFtpLogger + { + void IFtpLogger.Log(FtpLogEntry entry) + { + Log.Debug("FTP [{lvl}] -> {cnt}", entry.Severity.ToString(), entry.Message); + } + } + + } +} diff --git a/back-end/src/Storage/IStorageFacade.cs b/back-end/src/Storage/IStorageFacade.cs new file mode 100644 index 0000000..703394b --- /dev/null +++ b/back-end/src/Storage/IStorageFacade.cs @@ -0,0 +1,71 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: CMNext +* Package: Content.Publishing.Blog.Admin +* File: IStorageFacade.cs +* +* CMNext is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* CMNext is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +using VNLib.Net.Http; + +namespace Content.Publishing.Blog.Admin.Storage +{ + /// <summary> + /// Represents an opaque storage interface that abstracts simple storage operations + /// ignorant of the underlying storage system. + /// </summary> + internal interface IStorageFacade + { + /// <summary> + /// Gets the full public file path for the given relative file path + /// </summary> + /// <param name="filePath">The relative file path of the item to get the full path for</param> + /// <returns>The full relative file path</returns> + string GetExternalFilePath(string filePath); + + /// <summary> + /// Deletes a file from the storage system asynchronously + /// </summary> + /// <param name="filePath">The path to the file to delete</param> + /// <param name="cancellation">A token to cancel the operation</param> + /// <returns>A task that represents and asynchronous work</returns> + Task DeleteFileAsync(string filePath, CancellationToken cancellation); + + /// <summary> + /// Writes a file from the stream to the given file location + /// </summary> + /// <param name="filePath">The path to the file to write to</param> + /// <param name="data">The file data to stream</param> + /// <param name="ct">The content type of the file to write</param> + /// <param name="cancellation">A token to cancel the operation</param> + /// <returns>A task that represents and asynchronous work</returns> + Task SetFileAsync(string filePath, Stream data, ContentType ct, CancellationToken cancellation); + + /// <summary> + /// Reads a file from the storage system at the given path asynchronously + /// </summary> + /// <param name="filePath">The file to read</param> + /// <param name="output">The stream to write the file output to</param> + /// <param name="cancellation">A token to cancel the operation</param> + /// <returns>The number of bytes read, -1 if the operation failed</returns> + Task<long> ReadFileAsync(string filePath, Stream output, CancellationToken cancellation); + } +} diff --git a/back-end/src/Storage/ManagedStorage.cs b/back-end/src/Storage/ManagedStorage.cs new file mode 100644 index 0000000..654695f --- /dev/null +++ b/back-end/src/Storage/ManagedStorage.cs @@ -0,0 +1,96 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: CMNext +* Package: Content.Publishing.Blog.Admin +* File: ManagedStorage.cs +* +* CMNext is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* CMNext is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +using VNLib.Net.Http; +using VNLib.Plugins; +using VNLib.Plugins.Extensions.Loading; + +namespace Content.Publishing.Blog.Admin.Storage +{ + internal sealed class ManagedStorage : IStorageFacade + { + private readonly IStorageFacade _backingStorage; + + public ManagedStorage(PluginBase plugin) + { + if (plugin.HasConfigForType<MinioClientManager>()) + { + //Use minio storage + _backingStorage = plugin.GetOrCreateSingleton<MinioClientManager>(); + } + else if (plugin.HasConfigForType<FtpStorageManager>()) + { + //Use ftp storage + _backingStorage = plugin.GetOrCreateSingleton<FtpStorageManager>(); + } + else + { + throw new ArgumentException("No storage providers were found, cannot continue!"); + } + } + + ///<inheritdoc/> + public Task DeleteFileAsync(string filePath, CancellationToken cancellation) + { + return _backingStorage.DeleteFileAsync(filePath, cancellation); + } + + ///<inheritdoc/> + public string GetExternalFilePath(string filePath) + { + return _backingStorage.GetExternalFilePath(filePath); + } + + ///<inheritdoc/> + public async Task<long> ReadFileAsync(string filePath, Stream output, CancellationToken cancellation) + { + //Read the file from backing storage + long result = await _backingStorage.ReadFileAsync(filePath, output, cancellation); + + //Try to reset the stream if allowed + if (output.CanSeek) + { + //Reset stream + output.Seek(0, SeekOrigin.Begin); + } + + return result; + } + + ///<inheritdoc/> + public Task SetFileAsync(string filePath, Stream data, ContentType ct, CancellationToken cancellation) + { + //Try to reset the stream if allowed + if (data.CanSeek) + { + //Reset stream + data.Seek(0, SeekOrigin.Begin); + } + + return _backingStorage.SetFileAsync(filePath, data, ct, cancellation); + } + } +} diff --git a/back-end/src/Storage/MinioClientManager.cs b/back-end/src/Storage/MinioClientManager.cs new file mode 100644 index 0000000..e9f7c9a --- /dev/null +++ b/back-end/src/Storage/MinioClientManager.cs @@ -0,0 +1,135 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: CMNext +* Package: Content.Publishing.Blog.Admin +* File: MinioClientManager.cs +* +* CMNext is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* CMNext is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +using Minio; +using Minio.DataModel; + +using VNLib.Net.Http; +using VNLib.Utils.Memory; +using VNLib.Utils.Extensions; +using VNLib.Plugins; +using VNLib.Plugins.Extensions.Loading; + +using static Content.Publishing.Blog.Admin.Model.PostManager; + +namespace Content.Publishing.Blog.Admin.Storage +{ + + [ConfigurationName("s3_config")] + internal sealed class MinioClientManager : StorageBase + { + private readonly MinioClient Client; + private readonly S3Config Config; + + public MinioClientManager(PluginBase pbase, IConfigScope s3Config) + { + //Deserialize the config + Config = s3Config.Deserialze<S3Config>(); + Client = new(); + } + + ///<inheritdoc/> + protected override string? BasePath => Config.BaseBucket; + + ///<inheritdoc/> + public override async Task ConfigureServiceAsync(PluginBase plugin) + { + using ISecretResult? secret = await plugin.GetSecretAsync("s3_secret"); + + Client.WithEndpoint(Config.ServerAddress) + .WithCredentials(Config.ClientId, secret.Result.ToString()); + + Client.WithSSL(Config.UseSsl.HasValue && Config.UseSsl.Value); + + //Accept optional region + if (!string.IsNullOrWhiteSpace(Config.Region)) + { + Client.WithRegion(Config.Region); + } + + //10 second timeout + Client.WithTimeout(10 * 1000); + + //Setup debug trace + if (plugin.IsDebug()) + { + Client.SetTraceOn(new ReqLogger(plugin.Log)); + } + + //Build client + Client.Build(); + } + + ///<inheritdoc/> + public override Task DeleteFileAsync(string filePath, CancellationToken cancellation) + { + RemoveObjectArgs args = new(); + args.WithBucket(Config.BaseBucket) + .WithObject(filePath); + + //Remove the object + return Client.RemoveObjectAsync(args, cancellation); + } + + ///<inheritdoc/> + public override Task SetFileAsync(string filePath, Stream data, ContentType ct, CancellationToken cancellation) + { + PutObjectArgs args = new(); + args.WithBucket(Config.BaseBucket) + .WithContentType(HttpHelpers.GetContentTypeString(ct)) + .WithObject(filePath) + .WithObjectSize(data.Length) + .WithStreamData(data); + + //Upload the object + return Client.PutObjectAsync(args, cancellation); + } + + ///<inheritdoc/> + public override async Task<long> ReadFileAsync(string filePath, Stream output, CancellationToken cancellation) + { + //Get the item + GetObjectArgs args = new(); + args.WithBucket(Config.BaseBucket) + .WithObject(filePath) + .WithCallbackStream(async (stream, cancellation) => + { + //Read the objec to memory + await stream.CopyToAsync(output, 4096, MemoryUtil.Shared, cancellation); + }); + try + { + //Get the post content file + ObjectStat stat = await Client.GetObjectAsync(args, cancellation); + } + catch (Minio.Exceptions.ObjectNotFoundException) + { + //File not found + return -1; + } + return output.Position; + } + } +} diff --git a/back-end/src/Storage/StorageBase.cs b/back-end/src/Storage/StorageBase.cs new file mode 100644 index 0000000..b33d6f1 --- /dev/null +++ b/back-end/src/Storage/StorageBase.cs @@ -0,0 +1,57 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: CMNext +* Package: Content.Publishing.Blog.Admin +* File: StorageBase.cs +* +* CMNext is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* CMNext is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +using VNLib.Net.Http; +using VNLib.Plugins; +using VNLib.Plugins.Extensions.Loading; + +namespace Content.Publishing.Blog.Admin.Storage +{ + internal abstract class StorageBase : IAsyncConfigurable, IStorageFacade + { + /// <summary> + /// The base file path within the remote file system to use for external urls + /// </summary> + protected abstract string? BasePath { get; } + + ///<inheritdoc/> + public abstract Task ConfigureServiceAsync(PluginBase plugin); + + ///<inheritdoc/> + public abstract Task DeleteFileAsync(string filePath, CancellationToken cancellation); + + ///<inheritdoc/> + public abstract Task<long> ReadFileAsync(string filePath, Stream output, CancellationToken cancellation); + + ///<inheritdoc/> + public abstract Task SetFileAsync(string filePath, Stream data, ContentType ct, CancellationToken cancellation); + + ///<inheritdoc/> + public virtual string GetExternalFilePath(string filePath) + { + return string.IsNullOrWhiteSpace(BasePath) ? filePath : $"{BasePath}/{filePath}"; + } + } +} |