From 8b5f3eebb9f8d9bd55e922a809ffa3bd52e33401 Mon Sep 17 00:00:00 2001 From: vman Date: Fri, 9 Dec 2022 13:54:16 -0500 Subject: Sql essentials classes moved, secret loading updates --- VNLib.Plugins.Extensions.Loading/AssemblyLoader.cs | 3 +- .../ConfigurationExtensions.cs | 2 +- .../Events/EventHandle.cs | 1 + .../LoadingExtensions.cs | 16 +-- VNLib.Plugins.Extensions.Loading/PrivateKey.cs | 102 +++++++++++++++++ .../RoutingExtensions.cs | 23 ++++ VNLib.Plugins.Extensions.Loading/S3Config.cs | 9 +- VNLib.Plugins.Extensions.Loading/SecretResult.cs | 61 +++++++++++ VNLib.Plugins.Extensions.Loading/VaultSecrets.cs | 122 ++++++++++++++++++++- 9 files changed, 320 insertions(+), 19 deletions(-) create mode 100644 VNLib.Plugins.Extensions.Loading/PrivateKey.cs create mode 100644 VNLib.Plugins.Extensions.Loading/SecretResult.cs (limited to 'VNLib.Plugins.Extensions.Loading') diff --git a/VNLib.Plugins.Extensions.Loading/AssemblyLoader.cs b/VNLib.Plugins.Extensions.Loading/AssemblyLoader.cs index 2dffe88..5da16ec 100644 --- a/VNLib.Plugins.Extensions.Loading/AssemblyLoader.cs +++ b/VNLib.Plugins.Extensions.Loading/AssemblyLoader.cs @@ -28,9 +28,8 @@ using System.Threading; using System.Reflection; using McMaster.NETCore.Plugins; - -using VNLib.Utils; using System.Runtime.Loader; +using VNLib.Utils.Resources; namespace VNLib.Plugins.Extensions.Loading { diff --git a/VNLib.Plugins.Extensions.Loading/ConfigurationExtensions.cs b/VNLib.Plugins.Extensions.Loading/ConfigurationExtensions.cs index 62b898c..21f2fcb 100644 --- a/VNLib.Plugins.Extensions.Loading/ConfigurationExtensions.cs +++ b/VNLib.Plugins.Extensions.Loading/ConfigurationExtensions.cs @@ -192,7 +192,7 @@ namespace VNLib.Plugins.Extensions.Loading ClientId = s3conf.GetPropString("access_key"), ServerAddress = s3conf.GetPropString("server_address"), UseSsl = s3conf.TryGetValue("use_ssl", out JsonElement el) && el.GetBoolean(), - ClientSecret = plugin.TryGetSecretAsync(S3_SECRET_KEY).Result, + ClientSecret = plugin.TryGetSecretAsync(S3_SECRET_KEY), Region = s3conf.GetPropString("region"), }; } diff --git a/VNLib.Plugins.Extensions.Loading/Events/EventHandle.cs b/VNLib.Plugins.Extensions.Loading/Events/EventHandle.cs index b57ba6f..e9f3ff0 100644 --- a/VNLib.Plugins.Extensions.Loading/Events/EventHandle.cs +++ b/VNLib.Plugins.Extensions.Loading/Events/EventHandle.cs @@ -29,6 +29,7 @@ using System.Threading.Tasks; using VNLib.Utils; using VNLib.Utils.Extensions; using VNLib.Utils.Logging; +using VNLib.Utils.Resources; namespace VNLib.Plugins.Extensions.Loading.Events { diff --git a/VNLib.Plugins.Extensions.Loading/LoadingExtensions.cs b/VNLib.Plugins.Extensions.Loading/LoadingExtensions.cs index bfe0de1..7c8caee 100644 --- a/VNLib.Plugins.Extensions.Loading/LoadingExtensions.cs +++ b/VNLib.Plugins.Extensions.Loading/LoadingExtensions.cs @@ -72,15 +72,15 @@ namespace VNLib.Plugins.Extensions.Loading { PasswordHashing Passwords; //Get the global password system secret (pepper) - string pepperEl = plugin.TryGetSecretAsync(PASSWORD_HASHING_KEY).Result ?? throw new KeyNotFoundException($"Missing required key '{PASSWORD_HASHING_KEY}' in secrets"); - - byte[] pepper = Convert.FromBase64String(pepperEl); - - //wipe the pepper string - Utils.Memory.Memory.UnsafeZeroMemory(pepperEl); + using SecretResult pepperEl = plugin.TryGetSecretAsync(PASSWORD_HASHING_KEY).Result ?? throw new KeyNotFoundException($"Missing required key '{PASSWORD_HASHING_KEY}' in secrets"); + + byte[] pepper = pepperEl.GetFromBase64(); ERRNO cb(Span buffer) { + //No longer valid peper if plugin is unloaded as its set to zero, so we need to protect it + plugin.ThrowIfUnloaded(); + pepper.CopyTo(buffer); return pepper.Length; } @@ -210,7 +210,7 @@ namespace VNLib.Plugins.Extensions.Loading Task deferred = Task.Run(asyncTask); //Add task to deferred list - plugin.DeferredTasks.Add(deferred); + plugin.ObserveTask(deferred); try { //Await the task results @@ -224,7 +224,7 @@ namespace VNLib.Plugins.Extensions.Loading finally { //Remove task when complete - plugin.DeferredTasks.Remove(deferred); + plugin.RemoveObservedTask(deferred); } } } diff --git a/VNLib.Plugins.Extensions.Loading/PrivateKey.cs b/VNLib.Plugins.Extensions.Loading/PrivateKey.cs new file mode 100644 index 0000000..336f6a4 --- /dev/null +++ b/VNLib.Plugins.Extensions.Loading/PrivateKey.cs @@ -0,0 +1,102 @@ +/* +* Copyright (c) 2022 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Plugins.Extensions.Loading +* File: PrivateKey.cs +* +* PrivateKey.cs is part of VNLib.Plugins.Extensions.Loading which is part of the larger +* VNLib collection of libraries and utilities. +* +* VNLib.Plugins.Extensions.Loading is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* VNLib.Plugins.Extensions.Loading is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with VNLib.Plugins.Extensions.Loading. If not, see http://www.gnu.org/licenses/. +*/ + +using System; +using System.Text; +using System.Security.Cryptography; + +using VNLib.Utils; +using VNLib.Utils.Memory; +using VNLib.Utils.Extensions; + +namespace VNLib.Plugins.Extensions.Loading +{ + /// + /// A container for a PKSC#8 encoed private key + /// + public sealed class PrivateKey : VnDisposeable + { + private readonly byte[] _utf8RawData; + + /// + /// Decodes the PKCS#8 encoded private key from a secret, as an EC private key + /// and recovers the ECDsa algorithm from the key + /// + /// The algoritm from the private key + /// + /// + public ECDsa GetECDsa() + { + //Alloc buffer + using IMemoryHandle buffer = Memory.SafeAlloc(_utf8RawData.Length); + //Get base64 bytes from utf8 + ERRNO count = VnEncoding.Base64UrlDecode(_utf8RawData, buffer.Span); + //Parse the private key + ECDsa alg = ECDsa.Create(); + alg.ImportPkcs8PrivateKey(buffer.Span[..(int)count], out _); + //Wipe the buffer + Memory.InitializeBlock(buffer.Span); + return alg; + } + + /// + /// Decodes the PKCS#8 encoded private key from a secret, as an RSA private key + /// + /// The algorithm from the private key + /// + /// + public RSA GetRSA() + { + //Alloc buffer + using IMemoryHandle buffer = Memory.SafeAlloc(_utf8RawData.Length); + //Get base64 bytes from utf8 + ERRNO count = VnEncoding.Base64UrlDecode(_utf8RawData, buffer.Span); + //Parse the private key + RSA alg = RSA.Create(); + alg.ImportPkcs8PrivateKey(buffer.Span[..(int)count], out _); + //Wipe the buffer + Memory.InitializeBlock(buffer.Span); + return alg; + } + + internal PrivateKey(SecretResult secret) + { + //Alloc and get utf8 + byte[] buffer = new byte[secret.Result.Length]; + int count = Encoding.UTF8.GetBytes(secret.Result, buffer); + //Verify length + if(count != buffer.Length) + { + throw new FormatException("UTF8 deocde failed"); + } + //Store + _utf8RawData = buffer; + } + + protected override void Free() + { + Memory.InitializeBlock(_utf8RawData.AsSpan()); + } + } +} diff --git a/VNLib.Plugins.Extensions.Loading/RoutingExtensions.cs b/VNLib.Plugins.Extensions.Loading/RoutingExtensions.cs index 84d858f..0c2c222 100644 --- a/VNLib.Plugins.Extensions.Loading/RoutingExtensions.cs +++ b/VNLib.Plugins.Extensions.Loading/RoutingExtensions.cs @@ -27,6 +27,7 @@ using System.Linq; using System.Text.Json; using System.Reflection; using System.Collections.Generic; +using System.Runtime.CompilerServices; using VNLib.Plugins.Extensions.Loading.Events; @@ -37,6 +38,8 @@ namespace VNLib.Plugins.Extensions.Loading.Routing /// public static class RoutingExtensions { + private static readonly ConditionalWeakTable _pluginRefs = new(); + /// /// Constructs and routes the specific endpoint type for the current plugin /// @@ -60,6 +63,10 @@ namespace VNLib.Plugins.Extensions.Loading.Routing ScheduleIntervals(plugin, endpoint, endpointType, null); //Route the endpoint plugin.Route(endpoint); + + //Store ref to plugin for endpoint + _pluginRefs.Add(endpoint, plugin); + return endpoint; } else @@ -75,6 +82,10 @@ namespace VNLib.Plugins.Extensions.Loading.Routing ScheduleIntervals(plugin, endpoint, endpointType, conf); //Route the endpoint plugin.Route(endpoint); + + //Store ref to plugin for endpoint + _pluginRefs.Add(endpoint, plugin); + return endpoint; } } @@ -99,6 +110,18 @@ namespace VNLib.Plugins.Extensions.Loading.Routing return plugin.Route(configAttr?.ConfigVarName); } + /// + /// Gets the plugin that loaded the current endpoint + /// + /// + /// The plugin that loaded the current endpoint + /// + public static PluginBase GetPlugin(this IEndpoint ep) + { + _ = _pluginRefs.TryGetValue(ep, out PluginBase? pBase); + return pBase ?? throw new InvalidOperationException("Endpoint was not dynamically routed"); + } + private static void ScheduleIntervals(PluginBase plugin, T endpointInstance, Type epType, IReadOnlyDictionary? endpointLocalConfig) where T : IEndpoint { List registered = new(); diff --git a/VNLib.Plugins.Extensions.Loading/S3Config.cs b/VNLib.Plugins.Extensions.Loading/S3Config.cs index de24522..6d4ae4d 100644 --- a/VNLib.Plugins.Extensions.Loading/S3Config.cs +++ b/VNLib.Plugins.Extensions.Loading/S3Config.cs @@ -22,7 +22,7 @@ * along with VNLib.Plugins.Extensions.Loading. If not, see http://www.gnu.org/licenses/. */ -#nullable enable +using System.Threading.Tasks; namespace VNLib.Plugins.Extensions.Loading { @@ -30,9 +30,14 @@ namespace VNLib.Plugins.Extensions.Loading { public string? ServerAddress { get; init; } public string? ClientId { get; init; } - public string? ClientSecret { get; init; } + public Task ClientSecret { get; init; } public string? BaseBucket { get; init; } public bool? UseSsl { get; init; } public string? Region { get; init; } + + public S3Config() + { + ClientSecret = Task.FromResult(null); + } } } diff --git a/VNLib.Plugins.Extensions.Loading/SecretResult.cs b/VNLib.Plugins.Extensions.Loading/SecretResult.cs new file mode 100644 index 0000000..15323f3 --- /dev/null +++ b/VNLib.Plugins.Extensions.Loading/SecretResult.cs @@ -0,0 +1,61 @@ +/* +* Copyright (c) 2022 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Plugins.Extensions.Loading +* File: SecretResult.cs +* +* SecretResult.cs is part of VNLib.Plugins.Extensions.Loading which is part of the larger +* VNLib collection of libraries and utilities. +* +* VNLib.Plugins.Extensions.Loading is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* VNLib.Plugins.Extensions.Loading is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with VNLib.Plugins.Extensions.Loading. If not, see http://www.gnu.org/licenses/. +*/ + +using System; + +using VNLib.Utils; +using VNLib.Utils.Extensions; +using VNLib.Utils.Memory; + +namespace VNLib.Plugins.Extensions.Loading +{ + /// + /// The result of a secret fetch operation + /// + public sealed class SecretResult : VnDisposeable + { + private readonly char[] _secretChars; + + /// + /// The protected raw result value + /// + public ReadOnlySpan Result => _secretChars; + + + internal SecretResult(ReadOnlySpan value) => _secretChars = value.ToArray(); + + /// + protected override void Free() + { + Memory.InitializeBlock(_secretChars.AsSpan()); + } + + internal static SecretResult ToSecret(string? result) + { + SecretResult res = new(result.AsSpan()); + Memory.UnsafeZeroMemory(result); + return res; + } + } +} diff --git a/VNLib.Plugins.Extensions.Loading/VaultSecrets.cs b/VNLib.Plugins.Extensions.Loading/VaultSecrets.cs index c429312..468600f 100644 --- a/VNLib.Plugins.Extensions.Loading/VaultSecrets.cs +++ b/VNLib.Plugins.Extensions.Loading/VaultSecrets.cs @@ -39,11 +39,15 @@ using VaultSharp.V1.AuthMethods.Token; using VaultSharp.V1.AuthMethods.AppRole; using VaultSharp.V1.SecretsEngines.PKI; +using VNLib.Utils; +using VNLib.Utils.Memory; using VNLib.Utils.Logging; using VNLib.Utils.Extensions; +using VNLib.Hashing.IdentityUtility; namespace VNLib.Plugins.Extensions.Loading { + /// /// Adds loading extensions for secure/centralized configuration secrets /// @@ -76,19 +80,19 @@ namespace VNLib.Plugins.Extensions.Loading /// The element from the configuration file with the given name, or null if the configuration or property does not exist /// /// - public static Task TryGetSecretAsync(this PluginBase plugin, string secretName) + public static Task TryGetSecretAsync(this PluginBase plugin, string secretName) { //Get the secret from the config file raw string? rawSecret = TryGetSecretInternal(plugin, secretName); if (rawSecret == null) { - return Task.FromResult(null); + return Task.FromResult(null); } //Secret is a vault path, or return the raw value if (!rawSecret.StartsWith(VAULT_URL_SCHEME, StringComparison.OrdinalIgnoreCase)) { - return Task.FromResult(rawSecret); + return Task.FromResult(new(rawSecret.AsSpan())); } return GetSecretFromVaultAsync(plugin, rawSecret); } @@ -102,7 +106,7 @@ namespace VNLib.Plugins.Extensions.Loading /// /// /// - public static Task GetSecretFromVaultAsync(this PluginBase plugin, ReadOnlySpan vaultPath) + public static Task GetSecretFromVaultAsync(this PluginBase plugin, ReadOnlySpan vaultPath) { //print the path for debug if (plugin.IsDebug()) @@ -128,7 +132,7 @@ namespace VNLib.Plugins.Extensions.Loading string mount = path[..lastSep].ToString(); string secret = path[(lastSep + 1)..].ToString(); - async Task execute() + async Task execute() { //Try load client IVaultClient? client = _vaults.GetValue(plugin, TryGetVaultLoader).Value; @@ -137,7 +141,7 @@ namespace VNLib.Plugins.Extensions.Loading //run read async Secret result = await client.V1.Secrets.KeyValue.V2.ReadSecretAsync(path:secret, mountPoint:mount); //Read the secret - return result.Data.Data[secretTableKey].ToString(); + return SecretResult.ToSecret(result.Data.Data[secretTableKey].ToString()); } return Task.Run(execute); @@ -321,5 +325,111 @@ namespace VNLib.Plugins.Extensions.Loading //init lazy return new (LoadVault, LazyThreadSafetyMode.PublicationOnly); } + + /// + /// Gets the Secret value as a byte buffer + /// + /// + /// The base64 decoded secret as a byte[] + /// + /// + public static byte[] GetFromBase64(this SecretResult secret) + { + _ = secret ?? throw new ArgumentNullException(nameof(secret)); + + //Temp buffer + using UnsafeMemoryHandle buffer = Memory.UnsafeAlloc(secret.Result.Length); + + //Get base64 + if(Convert.TryFromBase64Chars(secret.Result, buffer, out int count)) + { + //Copy to array + byte[] value = buffer.Span[..count].ToArray(); + //Clear block before returning + Memory.InitializeBlock(buffer); + + return value; + } + + throw new InternalBufferTooSmallException("internal buffer too small"); + } + + /// + /// Recovers a certificate from a PEM encoded secret + /// + /// + /// The parsed from the PEM encoded data + /// + public static X509Certificate2 GetCertificate(this SecretResult secret) + { + _ = secret ?? throw new ArgumentNullException(nameof(secret)); + return X509Certificate2.CreateFromPem(secret.Result); + } + + /// + /// Gets the secret value as a secret result + /// + /// + /// The document parsed from the secret value + public static JsonDocument GetJsonDocument(this SecretResult secret) + { + _ = secret ?? throw new ArgumentNullException(nameof(secret)); + //Alloc buffer, utf8 so 1 byte per char + using IMemoryHandle buffer = Memory.SafeAlloc(secret.Result.Length); + //Get utf8 bytes + int count = Encoding.UTF8.GetBytes(secret.Result, buffer.Span); + //Reader and parse + Utf8JsonReader reader = new(buffer.Span[..count]); + return JsonDocument.ParseValue(ref reader); + } + + /// + /// Gets a SPKI encoded public key from a secret + /// + /// + /// The parsed from the SPKI public key + /// + public static PublicKey GetPublicKey(this SecretResult secret) + { + _ = secret ?? throw new ArgumentNullException(nameof(secret)); + //Alloc buffer, base64 is larger than binary value so char len is large enough + using IMemoryHandle buffer = Memory.SafeAlloc(secret.Result.Length); + //Get base64 bytes + ERRNO count = VnEncoding.TryFromBase64Chars(secret.Result, buffer.Span); + //Parse the SPKI from base64 + return PublicKey.CreateFromSubjectPublicKeyInfo(buffer.Span[..(int)count], out _); + } + + /// + /// Gets the value of the as a + /// container + /// + /// + /// The from the secret value + /// + /// + public static PrivateKey GetPrivateKey(this SecretResult secret) + { + _ = secret ?? throw new ArgumentNullException(nameof(secret)); + return new PrivateKey(secret); + } + + /// + /// Gets a from a secret value + /// + /// + /// The from the result + /// + /// + /// + public static ReadOnlyJsonWebKey GetJsonWebKey(this SecretResult secret) + { + _ = secret ?? throw new ArgumentNullException(nameof(secret)); + //Alloc buffer, utf8 so 1 byte per char + using IMemoryHandle buffer = Memory.SafeAlloc(secret.Result.Length); + //Get utf8 bytes + int count = Encoding.UTF8.GetBytes(secret.Result, buffer.Span); + return new ReadOnlyJsonWebKey(buffer.Span[..count]); + } } } -- cgit