aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLibravatar vnugent <public@vaughnnugent.com>2024-03-09 14:52:04 -0500
committerLibravatar vnugent <public@vaughnnugent.com>2024-03-09 14:52:04 -0500
commitbdf7c1bc36dbcc9f66f5caa344602817f888c49d (patch)
treed0a041e7545712e2a0e33aca362dee93b19973b5
parent5d8614d205b7bdca56684a3cc5a08db90e3804b6 (diff)
Squashed commit of the following:
commit 7a263bf54b7967ddeb9f6b662339ec1c74546ce8 Author: vnugent <public@vaughnnugent.com> Date: Sat Mar 9 14:19:31 2024 -0500 refactor: Overhaul secret loading. Remove VaultSharp as a dep commit 766e179d110db4f955fffce55f2b0ad41c139179 Author: vnugent <public@vaughnnugent.com> Date: Wed Mar 6 21:35:35 2024 -0500 refactor: changed how service constructors are invoked, moved routing
-rw-r--r--lib/VNLib.Plugins.Extensions.Loading/src/IAsyncLazy.cs6
-rw-r--r--lib/VNLib.Plugins.Extensions.Loading/src/LoadingExtensions.cs77
-rw-r--r--lib/VNLib.Plugins.Extensions.Loading/src/Routing/RoutingExtensions.cs (renamed from lib/VNLib.Plugins.Extensions.Loading/src/RoutingExtensions.cs)0
-rw-r--r--lib/VNLib.Plugins.Extensions.Loading/src/Secrets/HCVaultClient.cs419
-rw-r--r--lib/VNLib.Plugins.Extensions.Loading/src/Secrets/HCVaultException.cs41
-rw-r--r--lib/VNLib.Plugins.Extensions.Loading/src/Secrets/IHCVaultClient.cs64
-rw-r--r--lib/VNLib.Plugins.Extensions.Loading/src/Secrets/IOnDemandSecret.cs56
-rw-r--r--lib/VNLib.Plugins.Extensions.Loading/src/Secrets/OnDemandSecret.cs240
-rw-r--r--lib/VNLib.Plugins.Extensions.Loading/src/Secrets/PluginSecretConstants.cs48
-rw-r--r--lib/VNLib.Plugins.Extensions.Loading/src/Secrets/PluginSecretStore.cs135
-rw-r--r--lib/VNLib.Plugins.Extensions.Loading/src/Secrets/SecretResult.cs45
-rw-r--r--lib/VNLib.Plugins.Extensions.Loading/src/Secrets/VaultSecrets.cs338
-rw-r--r--lib/VNLib.Plugins.Extensions.Loading/src/VNLib.Plugins.Extensions.Loading.csproj1
13 files changed, 1116 insertions, 354 deletions
diff --git a/lib/VNLib.Plugins.Extensions.Loading/src/IAsyncLazy.cs b/lib/VNLib.Plugins.Extensions.Loading/src/IAsyncLazy.cs
index 98e0ebe..482785c 100644
--- a/lib/VNLib.Plugins.Extensions.Loading/src/IAsyncLazy.cs
+++ b/lib/VNLib.Plugins.Extensions.Loading/src/IAsyncLazy.cs
@@ -1,5 +1,5 @@
/*
-* Copyright (c) 2023 Vaughn Nugent
+* Copyright (c) 2024 Vaughn Nugent
*
* Library: VNLib
* Package: VNLib.Plugins.Extensions.Loading
@@ -77,8 +77,8 @@ namespace VNLib.Plugins.Extensions.Loading
/// <returns>A new <see cref="IAsyncLazy{T}"/> that returns the transformed type</returns>
public static IAsyncLazy<TResult> Transform<T, TResult>(this IAsyncLazy<T> lazy, Func<T, TResult> handler)
{
- _ = lazy ?? throw new ArgumentNullException(nameof(lazy));
- _ = handler ?? throw new ArgumentNullException(nameof(handler));
+ ArgumentNullException.ThrowIfNull(lazy);
+ ArgumentNullException.ThrowIfNull(handler);
//Await the lazy task, then pass the result to the handler
static async Task<TResult> OnResult(IAsyncLazy<T> lazy, Func<T, TResult> cb)
diff --git a/lib/VNLib.Plugins.Extensions.Loading/src/LoadingExtensions.cs b/lib/VNLib.Plugins.Extensions.Loading/src/LoadingExtensions.cs
index 3538337..b65c5e6 100644
--- a/lib/VNLib.Plugins.Extensions.Loading/src/LoadingExtensions.cs
+++ b/lib/VNLib.Plugins.Extensions.Loading/src/LoadingExtensions.cs
@@ -376,7 +376,7 @@ namespace VNLib.Plugins.Extensions.Loading
string assemblyDllName,
SearchOption search = SearchOption.AllDirectories,
AssemblyLoadContext? defaultCtx = null
- )
+ ) where T : class
{
/*
* Get or create the library for the assembly path, but only load it once
@@ -636,42 +636,16 @@ namespace VNLib.Plugins.Extensions.Loading
}
object service;
- ConstructorInfo? constructor;
try
{
- //Determin configuration requirments
- if (ConfigurationExtensions.ConfigurationRequired(serviceType) || config != null)
- {
- if (config == null)
- {
- ConfigurationExtensions.ThrowConfigNotFoundForType(serviceType);
- }
-
- //Get the constructor for required or available config
- constructor = serviceType.GetConstructor([typeof(PluginBase), typeof(IConfigScope)]);
-
- //Make sure the constructor exists
- _ = constructor ?? throw new MissingMemberException($"No constructor found for {serviceType.Name}");
-
- //Call constructore
- service = constructor.Invoke(new object[2] { plugin, config });
- }
- else if((constructor = serviceType.GetConstructor([typeof(PluginBase)])) != null)
- {
- //Call constructor
- service = constructor.Invoke(new object[1] { plugin });
- }
- //try to get empty constructor
- else if ((constructor = serviceType.GetConstructor([])) != null)
+ //Determine configuration requirments
+ if (ConfigurationExtensions.ConfigurationRequired(serviceType) && config == null)
{
- //Invoked empty constructor
- service = constructor.Invoke(null);
- }
- else
- {
- throw new MissingMemberException($"No constructor found for {serviceType.Name}");
+ ConfigurationExtensions.ThrowConfigNotFoundForType(serviceType);
}
+
+ service = InvokeServiceConstructor(serviceType, plugin, config);
}
catch(TargetInvocationException te) when (te.InnerException != null)
{
@@ -720,6 +694,45 @@ namespace VNLib.Plugins.Extensions.Loading
return service;
}
+ /*
+ * Attempts to find the most appropriate constructor for the service type
+ * if found, then invokes it to create the service instance
+ */
+
+ private static object InvokeServiceConstructor(Type serviceSType, PluginBase plugin, IConfigScope? config)
+ {
+ ConstructorInfo? constructor;
+
+ /*
+ * First try to load a constructor with the plugin and config scope
+ */
+ if (config != null)
+ {
+ constructor = serviceSType.GetConstructor([typeof(PluginBase), typeof(IConfigScope)]);
+
+ if(constructor is not null)
+ {
+ return constructor.Invoke([plugin, config]);
+ }
+ }
+
+ //Try to get plugin only constructor
+ constructor = serviceSType.GetConstructor([typeof(PluginBase)]);
+ if (constructor is not null)
+ {
+ return constructor.Invoke([plugin]);
+ }
+
+ //Finally fall back to the empty constructor
+ constructor = serviceSType.GetConstructor([]);
+ if (constructor is not null)
+ {
+ return constructor.Invoke(null);
+ }
+
+ throw new MissingMemberException($"No constructor found for {serviceSType.Name}");
+ }
+
[DoesNotReturn]
internal static void FindAndThrowInnerException(Exception ex)
{
diff --git a/lib/VNLib.Plugins.Extensions.Loading/src/RoutingExtensions.cs b/lib/VNLib.Plugins.Extensions.Loading/src/Routing/RoutingExtensions.cs
index a1817a8..a1817a8 100644
--- a/lib/VNLib.Plugins.Extensions.Loading/src/RoutingExtensions.cs
+++ b/lib/VNLib.Plugins.Extensions.Loading/src/Routing/RoutingExtensions.cs
diff --git a/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/HCVaultClient.cs b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/HCVaultClient.cs
new file mode 100644
index 0000000..a06f490
--- /dev/null
+++ b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/HCVaultClient.cs
@@ -0,0 +1,419 @@
+/*
+* Copyright (c) 2024 Vaughn Nugent
+*
+* Library: VNLib
+* Package: VNLib.Plugins.Extensions.Loading
+* File: HCVaultClient.cs
+*
+* HCVaultClient.cs is part of VNLib.Plugins.Extensions.Loading which
+* is part of the larger VNLib collection of libraries and utilities.
+*
+* VNLib.Plugins.Extensions.Loading is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Affero General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* VNLib.Plugins.Extensions.Loading is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU Affero General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program. If not, see https://www.gnu.org/licenses/.
+*/
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Text.Json;
+using System.Net;
+using System.Net.Http;
+using System.Net.Sockets;
+using System.Net.Security;
+using System.Diagnostics;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+
+using VNLib.Utils;
+using VNLib.Utils.IO;
+using VNLib.Utils.Memory;
+using VNLib.Utils.Extensions;
+
+/*
+ * The purpose of the HCVaultClient is to provide a very simple Hashicorp Vault client
+ * interface that reads KV secrets from a vault server with minimal dependencies.
+ *
+ * Since I only need the KV store for now, I don't think there is a need for the
+ * VaultSharp package which adds at least 600kb to the final package size.
+ */
+
+namespace VNLib.Plugins.Extensions.Loading
+{
+ internal sealed class HCVaultClient : VnDisposeable, IHCVaultClient
+ {
+ const string VaultTokenHeaderName = "X-Vault-Token";
+ const long MaxErrResponseContentLength = 8192;
+ const uint DefaultBufferSize = 4096;
+
+ private static readonly TimeSpan ClientDefaultTimeout = TimeSpan.FromSeconds(30);
+
+ private readonly HttpClient _client;
+ private readonly int _kvVersion;
+ private readonly IUnmangedHeap _bufferHeap;
+
+ HCVaultClient(string serverAddress, string hcToken, int kvVersion, bool trustCert, IUnmangedHeap heap)
+ {
+#pragma warning disable CA2000 // Dispose objects before losing scope
+ HttpClientHandler handler = new()
+ {
+ AllowAutoRedirect = false,
+ UseCookies = false,
+ MaxResponseHeadersLength = 2048,
+ ClientCertificateOptions = ClientCertificateOption.Automatic,
+ AutomaticDecompression = DecompressionMethods.All,
+ PreAuthenticate = false,
+
+ //Setup a callback to trust the server certificate if the cert chain is invalid
+ ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => trustCert || errors == SslPolicyErrors.None
+ };
+
+#pragma warning restore CA2000 // Dispose objects before losing scope
+
+ _client = new HttpClient(handler, true)
+ {
+ BaseAddress = new Uri(serverAddress),
+ Timeout = ClientDefaultTimeout,
+ DefaultRequestVersion = new Version(1, 1),
+ MaxResponseContentBufferSize = 4096 //Buffer only needs to be little for vault requests
+ };
+
+
+ //Set the vault access token header, should probably clean this up later
+ _client.DefaultRequestHeaders.Add(VaultTokenHeaderName, hcToken);
+ _kvVersion = kvVersion;
+ _bufferHeap = heap;
+ }
+
+ /// <summary>
+ /// Creates a new Hashicorp vault client with the given server address, token, and KV storage version
+ /// </summary>
+ /// <param name="serverAddress">The vault server address</param>
+ /// <param name="hcToken">The vault token used to connect to the vault server</param>
+ /// <param name="kvVersion">The hc vault Key value store version (must be 1 or 2)</param>
+ /// <param name="trustCert">A value that tells the HTTP client to trust the Vault server's certificate even if it's not valid</param>
+ /// <param name="heap">Heap instance to allocate internal buffers from</param>
+ /// <returns>The new client instance</returns>
+ /// <exception cref="ArgumentException"></exception>
+ /// <exception cref="ArgumentNullException"></exception>
+ public static HCVaultClient Create(string serverAddress, string hcToken, int kvVersion, bool trustCert, IUnmangedHeap heap)
+ {
+ ArgumentException.ThrowIfNullOrEmpty(serverAddress);
+ ArgumentException.ThrowIfNullOrEmpty(hcToken);
+ ArgumentNullException.ThrowIfNull(heap);
+
+ if(kvVersion != 1 && kvVersion != 2)
+ {
+ throw new ArgumentException($"Unsupported vault KV storage version {kvVersion}, must be either 1 or 2");
+ }
+
+ return new HCVaultClient(serverAddress, hcToken, kvVersion, trustCert, heap);
+ }
+
+ ///<inheritdoc/>
+ protected override void Free()
+ {
+ _client.Dispose();
+ }
+
+ ///<inheritdoc/>
+ public async Task<ISecretResult?> ReadSecretAsync(string path, string mountPoint, string secretName)
+ {
+ string secretPath = GetSecretPathForKvVersion(_kvVersion, path, mountPoint);
+ using HttpRequestMessage ms = GetRequestMessageForPath(secretPath);
+
+ try
+ {
+ using HttpResponseMessage response = await _client.SendAsync(ms, HttpCompletionOption.ResponseHeadersRead);
+
+ //Check if an error occured in the response
+ await ProcessVaultErrorResponseAsync(response, true);
+
+ //Read the response async
+ using SecretResponse res = await ReadSecretResponse(response.Content, true);
+
+ return FromResponse(res, secretName);
+ }
+ catch(HttpRequestException he) when(he.InnerException is SocketException se)
+ {
+ throw se.SocketErrorCode switch
+ {
+ SocketError.HostNotFound => new HCVaultException("Failed to connect to Hashicorp Vault server, because it's DNS hostname could not be resolved"),
+ SocketError.ConnectionRefused => new HCVaultException("Failed to establish a TCP connection to the vault server, the server refused the connection"),
+ _ => new HCVaultException("Failed to establish a TCP connection to the vault server, see inner exception", se),
+ };
+ }
+ catch(Exception ex)
+ {
+ throw new HCVaultException("Failed to retreive secret from Hashicorp Vault server, see inner exception", ex);
+ }
+ }
+
+ ///<inheritdoc/>
+ public ISecretResult? ReadSecret(string path, string mountPoint, string secretName)
+ {
+ string secretPath = GetSecretPathForKvVersion(_kvVersion, path, mountPoint);
+ using HttpRequestMessage ms = GetRequestMessageForPath(secretPath);
+
+ try
+ {
+ //Exec the response synchronously
+ using HttpResponseMessage response = _client.Send(ms, HttpCompletionOption.ResponseHeadersRead);
+
+ /*
+ * It is safe to await the error result here because its
+ * already completed when the async flag is false
+ */
+ ValueTask errTask = ProcessVaultErrorResponseAsync(response, false);
+ Debug.Assert(errTask.IsCompleted);
+ errTask.GetAwaiter().GetResult();
+
+ //Did not throw, handle a secret response
+
+ ValueTask<SecretResponse> resTask = ReadSecretResponse(response.Content, false);
+ Debug.Assert(resTask.IsCompleted);
+
+ //Always wrap response in using to clean memory
+ using SecretResponse res = resTask.GetAwaiter().GetResult();
+
+ return FromResponse(res, secretName);
+ }
+ catch (HttpRequestException he) when (he.InnerException is SocketException se)
+ {
+ throw se.SocketErrorCode switch
+ {
+ SocketError.HostNotFound => new HCVaultException("Failed to connect to Hashicorp Vault server, because it's DNS hostname could not be resolved"),
+ SocketError.ConnectionRefused => new HCVaultException("Failed to establish a TCP connection to the vault server, the server refused the connection"),
+ _ => new HCVaultException("Failed to establish a TCP connection to the vault server, see inner exception", se),
+ };
+ }
+ catch (Exception ex)
+ {
+ throw new HCVaultException("Failed to retreive secret from Hashicorp Vault server, see inner exception", ex);
+ }
+ }
+
+ private ValueTask<SecretResponse> ReadSecretResponse(HttpContent content, bool async)
+ {
+ SecretResponse res = new(DefaultBufferSize, _bufferHeap);
+ try
+ {
+ if (async)
+ {
+ return ReadStreamAsync(content, res);
+ }
+ else
+ {
+ //Read into a memory stream
+ content.CopyTo(res.StreamData, null, default);
+ res.ResetStream();
+
+ return ValueTask.FromResult(res);
+ }
+ }
+ catch
+ {
+ res.Dispose();
+ throw;
+ }
+
+ async static ValueTask<SecretResponse> ReadStreamAsync(HttpContent content, SecretResponse response)
+ {
+ try
+ {
+ await content.CopyToAsync(response.StreamData);
+
+ response.ResetStream();
+
+ return response;
+ }
+ catch
+ {
+ response.Dispose();
+ throw;
+ }
+ }
+
+ }
+
+ private static string GetSecretPathForKvVersion(int version, string path, string mount)
+ {
+ return version switch
+ {
+ 1 => $"v1/{mount}/{path}",
+ 2 => $"v1/{mount}/data/{path}",
+ _ => throw new InvalidOperationException("Invalid KV version")
+ };
+ }
+
+ private static HttpRequestMessage GetRequestMessageForPath(string secretPath)
+ {
+ return new(HttpMethod.Get, secretPath)
+ {
+ VersionPolicy = HttpVersionPolicy.RequestVersionOrHigher,
+ };
+ }
+
+ private static SecretResult? FromResponse(SecretResponse res, string secretName)
+ {
+ using JsonDocument json = res.AsJson();
+
+ if (!json.RootElement.TryGetProperty("data", out JsonElement dataEl))
+ {
+ throw new HttpRequestException("Vault KV response did not include a top-level 'data' element");
+ }
+
+ if (!dataEl.TryGetProperty("data", out dataEl))
+ {
+ throw new HttpRequestException("Vault KV response did not include a 'data' element");
+ }
+
+ //Try to get the secret from the data element
+ if (dataEl.TryGetProperty(secretName, out JsonElement secretEl))
+ {
+ string? secValue = secretEl.GetString();
+ return secValue == null ? null : SecretResult.ToSecret(secValue);
+ }
+
+ return null;
+ }
+
+ private static ValueTask ProcessVaultErrorResponseAsync(HttpResponseMessage response, bool async)
+ {
+ if (response.IsSuccessStatusCode)
+ {
+ return default;
+ }
+
+ //Make sure the response has content
+ long? ctLen = response.Content.Headers.ContentLength;
+ if(!ctLen.HasValue || ctLen.Value == 0)
+ {
+ return ValueTask.FromException(
+ new HttpRequestException($"Failed to fetch secret from vault with error code {response.StatusCode}")
+ );
+ }
+
+ //Check for way too big response entity body
+ if(ctLen.Value > MaxErrResponseContentLength)
+ {
+ return ValueTask.FromException(
+ new HttpRequestException(
+ $"Vault error {response.StatusCode}. Response content length was too large, expected less than {MaxErrResponseContentLength} but got {ctLen.Value}"
+ ));
+ }
+
+
+ //Assert json response body
+ if (!string.Equals("application/json", response.Content.Headers.ContentType?.MediaType, StringComparison.OrdinalIgnoreCase))
+ {
+ return ValueTask.FromException(
+ new HttpRequestException("Vault response was not in JSON format")
+ );
+ }
+
+ return async ? ExceptionsFromContentAsync(response) : ExceptionsFromContent(response);
+
+ static ValueTask ExceptionFromVaultErrors(HttpStatusCode code, VaultErrorMessage? errs)
+ {
+ //If the error message is null, raise an exception
+ if (errs == null || errs.Errors == null || errs.Errors.Length == 0)
+ {
+ return ValueTask.FromException(
+ new HttpRequestException($"Failed to fetch secret from vault with error code {code}")
+ );
+ }
+
+ //Join the errors into a single string with newlines
+ IEnumerable<string> errors = errs.Errors.Select(err => $"Vault Error -> {err}");
+ string errStr = string.Join(Environment.NewLine, errors);
+
+ //Finally raise the exception with all the returned errors
+ return ValueTask.FromException(
+ new HttpRequestException($"Failed to fetch secre from vault with {code}, errors:\n {errStr}")
+ );
+ }
+
+ static async ValueTask ExceptionsFromContentAsync(HttpResponseMessage response)
+ {
+ //Read stream async and deserialize async
+ using Stream stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false);
+ VaultErrorMessage? errs = await JsonSerializer.DeserializeAsync<VaultErrorMessage>(stream);
+
+ await ExceptionFromVaultErrors(response.StatusCode, errs);
+ }
+
+ static ValueTask ExceptionsFromContent(HttpResponseMessage response)
+ {
+#pragma warning disable CA1849 // Call async methods when in an async method
+
+ //Read the error content stream and deserialize
+ using Stream stream = response.Content.ReadAsStream();
+ VaultErrorMessage? errs = JsonSerializer.Deserialize<VaultErrorMessage>(stream);
+
+#pragma warning restore CA1849 // Call async methods when in an async method
+
+ return ExceptionFromVaultErrors(response.StatusCode, errs);
+ }
+ }
+
+
+ private sealed class SecretResponse : VnDisposeable
+ {
+ /*
+ * Purpose of this class is to hold a memory stream that can read
+ * the vault response into memory, use it for some operation,
+ * then zero the memory before releasing it back to the heap
+ */
+
+ private readonly MemoryHandle<byte> _memHandle;
+
+ public VnMemoryStream StreamData { get; }
+
+ public SecretResponse(uint initSize, IUnmangedHeap heap)
+ {
+ _memHandle = heap.Alloc<byte>(initSize, false);
+ StreamData = VnMemoryStream.FromHandle(_memHandle, false, 0, false);
+ }
+
+ /// <summary>
+ /// Gets a <see cref="JsonDocument"/> from the response data
+ /// </summary>
+ /// <returns></returns>
+ public JsonDocument AsJson()
+ {
+ //read the data as a raw span then parse it as json
+ Utf8JsonReader reader = new(StreamData.AsSpan());
+ return JsonDocument.ParseValue(ref reader);
+ }
+
+ /// <summary>
+ /// Resets the stream to the beginning
+ /// </summary>
+ public void ResetStream() => StreamData.Seek(0, SeekOrigin.Begin);
+
+ protected override void Free()
+ {
+ //zero the handle before disposing
+ MemoryUtil.InitializeBlock(ref _memHandle.GetReference(), _memHandle.GetIntLength());
+ _memHandle.Dispose();
+ }
+ }
+
+ private sealed class VaultErrorMessage
+ {
+ [JsonPropertyName("errors")]
+ public string[]? Errors { get; set; }
+ }
+ }
+}
diff --git a/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/HCVaultException.cs b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/HCVaultException.cs
new file mode 100644
index 0000000..c0c8080
--- /dev/null
+++ b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/HCVaultException.cs
@@ -0,0 +1,41 @@
+/*
+* Copyright (c) 2024 Vaughn Nugent
+*
+* Library: VNLib
+* Package: VNLib.Plugins.Extensions.Loading
+* File: HCVaultException.cs
+*
+* HCVaultException.cs is part of VNLib.Plugins.Extensions.Loading which
+* is part of the larger VNLib collection of libraries and utilities.
+*
+* VNLib.Plugins.Extensions.Loading is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Affero General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* VNLib.Plugins.Extensions.Loading is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU Affero General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program. If not, see https://www.gnu.org/licenses/.
+*/
+
+using System;
+using System.Net.Http;
+
+namespace VNLib.Plugins.Extensions.Loading
+{
+ public sealed class HCVaultException : HttpRequestException
+ {
+ public HCVaultException()
+ { }
+
+ public HCVaultException(string message) : base(message)
+ { }
+
+ public HCVaultException(string message, Exception innerException) : base(message, innerException)
+ { }
+ }
+}
diff --git a/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/IHCVaultClient.cs b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/IHCVaultClient.cs
new file mode 100644
index 0000000..aab2541
--- /dev/null
+++ b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/IHCVaultClient.cs
@@ -0,0 +1,64 @@
+/*
+* Copyright (c) 2024 Vaughn Nugent
+*
+* Library: VNLib
+* Package: VNLib.Plugins.Extensions.Loading
+* File: IHCVaultClient.cs
+*
+* IHCVaultClient.cs is part of VNLib.Plugins.Extensions.Loading which is
+* part of the larger VNLib collection of libraries and utilities.
+*
+* VNLib.Plugins.Extensions.Loading is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Affero General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* VNLib.Plugins.Extensions.Loading is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU Affero General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program. If not, see https://www.gnu.org/licenses/.
+*/
+
+using System;
+using System.Net.Http;
+using System.Threading.Tasks;
+
+namespace VNLib.Plugins.Extensions.Loading
+{
+ /// <summary>
+ /// A Hashicorp Vault client for reading secrets from a vault server
+ /// </summary>
+ public interface IHCVaultClient
+ {
+ /// <summary>
+ /// Reads a single KeyValue secret from the vault server asyncrhonously and returns the result
+ /// or null if the secret does not exist
+ /// </summary>
+ /// <param name="path">The path to the item within the store</param>
+ /// <param name="mountPoint">The vault mount points</param>
+ /// <param name="secretName">The name of the secret within the property array to retrieve</param>
+ /// <returns>The secret wrapper if found, null otherwise</returns>
+ /// <exception cref="ArgumentException"></exception>
+ /// <exception cref="ArgumentNullException"></exception>
+ /// <exception cref="HCVaultException"></exception>
+ /// <exception cref="HttpRequestException"></exception>
+ Task<ISecretResult?> ReadSecretAsync(string path, string mountPoint, string secretName);
+
+ /// <summary>
+ /// Reads a single KeyValue secret from the vault server syncrhonously and returns the result
+ /// or null if the secret does not exist
+ /// </summary>
+ /// <param name="path">The path to the item within the store</param>
+ /// <param name="mountPoint">The vault mount points</param>
+ /// <param name="secretName">The name of the secret within the property array to retrieve</param>
+ /// <returns>The secret wrapper if found, null otherwise</returns>
+ /// <exception cref="ArgumentException"></exception>
+ /// <exception cref="ArgumentNullException"></exception>
+ /// <exception cref="HCVaultException"></exception>
+ /// <exception cref="HttpRequestException"></exception>
+ ISecretResult? ReadSecret(string path, string mountPoint, string secretName);
+ }
+}
diff --git a/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/IOnDemandSecret.cs b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/IOnDemandSecret.cs
new file mode 100644
index 0000000..d9f24f3
--- /dev/null
+++ b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/IOnDemandSecret.cs
@@ -0,0 +1,56 @@
+/*
+* Copyright (c) 2024 Vaughn Nugent
+*
+* Library: VNLib
+* Package: VNLib.Plugins.Extensions.Loading
+* File: IOnDemandSecret.cs
+*
+* IOnDemandSecret.cs is part of VNLib.Plugins.Extensions.Loading which is
+* part of the larger VNLib collection of libraries and utilities.
+*
+* VNLib.Plugins.Extensions.Loading is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Affero General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* VNLib.Plugins.Extensions.Loading is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU Affero General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program. If not, see https://www.gnu.org/licenses/.
+*/
+
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace VNLib.Plugins.Extensions.Loading
+{
+ /// <summary>
+ /// A secret that can be fetched from it's backing store when needed
+ /// to avoid storing sensitive information in memory long term
+ /// </summary>
+ public interface IOnDemandSecret
+ {
+ /// <summary>
+ /// The name of the secret that will be fetched on demand
+ /// </summary>
+ string SecretName { get; }
+
+ /// <summary>
+ /// Fetches the secret value from the backing store
+ /// synchronously
+ /// </summary>
+ /// <returns>The secret value if found, null otherwise</returns>
+ ISecretResult? FetchSecret();
+
+ /// <summary>
+ /// Fetches the secret value from the backing store
+ /// asynchronously
+ /// </summary>
+ /// <param name="cancellation">An optionall canceallation token to cancel the operation</param>
+ /// <returns>A task that completes with the value of the secret if it exists</returns>
+ Task<ISecretResult?> FetchSecretAsync(CancellationToken cancellation = default);
+ }
+}
diff --git a/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/OnDemandSecret.cs b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/OnDemandSecret.cs
new file mode 100644
index 0000000..6e6d560
--- /dev/null
+++ b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/OnDemandSecret.cs
@@ -0,0 +1,240 @@
+/*
+* Copyright (c) 2024 Vaughn Nugent
+*
+* Library: VNLib
+* Package: VNLib.Plugins.Extensions.Loading
+* File: OnDemandSecret.cs
+*
+* OnDemandSecret.cs is part of VNLib.Plugins.Extensions.Loading which is
+* part of the larger VNLib collection of libraries and utilities.
+*
+* VNLib.Plugins.Extensions.Loading is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Affero General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* VNLib.Plugins.Extensions.Loading is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU Affero General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program. If not, see https://www.gnu.org/licenses/.
+*/
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Text.Json;
+using System.Threading;
+using System.Diagnostics;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+
+using VNLib.Utils.Memory;
+using VNLib.Utils.Logging;
+using VNLib.Utils.Extensions;
+
+using static VNLib.Plugins.Extensions.Loading.PluginSecretConstants;
+
+namespace VNLib.Plugins.Extensions.Loading
+{
+ internal sealed class OnDemandSecret(PluginBase plugin, string secretName, IHCVaultClient? vault) : IOnDemandSecret
+ {
+ public string SecretName { get; } = secretName ?? throw new ArgumentNullException(nameof(secretName));
+
+ ///<inheritdoc/>
+ public ISecretResult? FetchSecret()
+ {
+ plugin.ThrowIfUnloaded();
+
+ //Get the secret from the config file raw
+ string? rawSecret = TryGetSecretFromConfig(secretName);
+
+ if (rawSecret == null)
+ {
+ return null;
+ }
+
+ //Secret is a vault path, or return the raw value
+ if (rawSecret.StartsWith(VAULT_URL_SCHEME, StringComparison.OrdinalIgnoreCase))
+ {
+ ValueTask<ISecretResult?> res = GetSecretFromVault(rawSecret, false);
+ Debug.Assert(res.IsCompleted);
+ return res.GetAwaiter().GetResult();
+ }
+
+ //See if the secret is an environment variable path
+ if (rawSecret.StartsWith(ENV_URL_SCHEME, StringComparison.OrdinalIgnoreCase))
+ {
+ //try to get the environment variable
+ string envVar = rawSecret[ENV_URL_SCHEME.Length..];
+ string? envVal = Environment.GetEnvironmentVariable(envVar);
+
+ return envVal == null ? null : SecretResult.ToSecret(envVal);
+ }
+
+ //See if the secret is a file path
+ if (rawSecret.StartsWith(FILE_URL_SCHEME, StringComparison.OrdinalIgnoreCase))
+ {
+ string filePath = rawSecret[FILE_URL_SCHEME.Length..];
+ byte[] fileData = File.ReadAllBytes(filePath);
+
+ return GetResultFromFileData(fileData);
+ }
+
+ //Finally, return the raw value
+ return SecretResult.ToSecret(rawSecret);
+ }
+
+ ///<inheritdoc/>
+ public Task<ISecretResult?> FetchSecretAsync(CancellationToken cancellation)
+ {
+ plugin.ThrowIfUnloaded();
+
+ //Get the secret from the config file raw
+ string? rawSecret = TryGetSecretFromConfig(secretName);
+
+ if (rawSecret == null)
+ {
+ return Task.FromResult<ISecretResult?>(null);
+ }
+
+ //Secret is a vault path, or return the raw value
+ if (rawSecret.StartsWith(VAULT_URL_SCHEME, StringComparison.OrdinalIgnoreCase))
+ {
+ //Exec vault async
+ ValueTask<ISecretResult?> res = GetSecretFromVault(rawSecret, true);
+ return res.AsTask();
+ }
+
+ //See if the secret is an environment variable path
+ if (rawSecret.StartsWith(ENV_URL_SCHEME, StringComparison.OrdinalIgnoreCase))
+ {
+ //try to get the environment variable
+ string envVar = rawSecret[ENV_URL_SCHEME.Length..];
+ string? envVal = Environment.GetEnvironmentVariable(envVar);
+ return Task.FromResult<ISecretResult?>(envVal == null ? null : SecretResult.ToSecret(envVal));
+ }
+
+ //See if the secret is a file path
+ if (rawSecret.StartsWith(FILE_URL_SCHEME, StringComparison.OrdinalIgnoreCase))
+ {
+ string filePath = rawSecret[FILE_URL_SCHEME.Length..];
+ return GetResultFromFileAsync(filePath, cancellation);
+ }
+
+ //Finally, return the raw value
+ return Task.FromResult<ISecretResult?>(SecretResult.ToSecret(rawSecret));
+
+
+ static async Task<ISecretResult?> GetResultFromFileAsync(string filePath, CancellationToken ct)
+ {
+ byte[] fileData = await File.ReadAllBytesAsync(filePath, ct);
+ return GetResultFromFileData(fileData);
+ }
+ }
+
+ /// <summary>
+ /// Gets a secret at the given vault url (in the form of "vault://[mount-name]/[secret-path]?secret=[secret_name]")
+ /// </summary>
+ /// <param name="vaultPath">The raw vault url to lookup</param>
+ /// <param name="async"></param>
+ /// <returns>The string of the object at the specified vault path</returns>
+ /// <exception cref="UriFormatException"></exception>
+ /// <exception cref="KeyNotFoundException"></exception>
+ /// <exception cref="ObjectDisposedException"></exception>
+ private ValueTask<ISecretResult?> GetSecretFromVault(ReadOnlySpan<char> vaultPath, bool async)
+ {
+ ArgumentNullException.ThrowIfNull(plugin);
+
+ //print the path for debug
+ if (plugin.IsDebug())
+ {
+ plugin.Log.Debug("Retrieving secret {s} from vault", vaultPath.ToString());
+ }
+
+ //Slice off path
+ ReadOnlySpan<char> paq = vaultPath.SliceAfterParam(VAULT_URL_SCHEME);
+ ReadOnlySpan<char> path = paq.SliceBeforeParam('?');
+ ReadOnlySpan<char> query = paq.SliceAfterParam('?');
+
+ if (paq.IsEmpty)
+ {
+ throw new UriFormatException("Vault secret location not valid/empty ");
+ }
+
+ //Get the secret
+ string secretTableKey = query.SliceAfterParam("secret=").SliceBeforeParam('&').ToString();
+ string vaultType = query.SliceBeforeParam("vault_type=").SliceBeforeParam('&').ToString();
+
+ //get mount and path
+ int lastSep = path.IndexOf('/');
+ string mount = path[..lastSep].ToString();
+ string secret = path[(lastSep + 1)..].ToString();
+
+ //Try load client
+ _ = vault ?? throw new KeyNotFoundException("Vault client not found");
+
+ if (async)
+ {
+ Task<ISecretResult?> asTask = Task.Run(() => vault.ReadSecretAsync(secret, mount, secretTableKey));
+ return new ValueTask<ISecretResult?>(asTask);
+ }
+ else
+ {
+ ISecretResult? result = vault.ReadSecret(secret, mount, secretTableKey);
+ return new ValueTask<ISecretResult?>(result);
+ }
+ }
+
+ private string? TryGetSecretFromConfig(string secretName)
+ {
+ bool local = plugin.PluginConfig.TryGetProperty(SECRETS_CONFIG_KEY, out JsonElement localEl);
+ bool host = plugin.HostConfig.TryGetProperty(SECRETS_CONFIG_KEY, out JsonElement hostEl);
+
+ //total config
+ Dictionary<string, JsonElement> conf = new(StringComparer.OrdinalIgnoreCase);
+
+ if (local && host)
+ {
+ //Load both config objects to dict
+ Dictionary<string, JsonElement> localConf = localEl.EnumerateObject().ToDictionary(x => x.Name, x => x.Value);
+ Dictionary<string, JsonElement> hostConf = hostEl.EnumerateObject().ToDictionary(x => x.Name, x => x.Value);
+
+ //Enter all host secret objects, then follow up with plugin secert elements
+ hostConf.ForEach(kv => conf[kv.Key] = kv.Value);
+ localConf.ForEach(kv => conf[kv.Key] = kv.Value);
+
+ }
+ else if (local)
+ {
+ //Store only local config
+ conf = localEl.EnumerateObject().ToDictionary(x => x.Name, x => x.Value, StringComparer.OrdinalIgnoreCase);
+ }
+ else if (host)
+ {
+ //store only host config
+ conf = hostEl.EnumerateObject().ToDictionary(x => x.Name, x => x.Value, StringComparer.OrdinalIgnoreCase);
+ }
+
+ //Get the value or default json element
+ return conf.TryGetValue(secretName, out JsonElement el) ? el.GetString() : null;
+ }
+
+ private static SecretResult GetResultFromFileData(byte[] secretFileData)
+ {
+ //recover the character data from the file data
+ int chars = Encoding.UTF8.GetCharCount(secretFileData);
+ char[] secretFileChars = new char[chars];
+ Encoding.UTF8.GetChars(secretFileData, secretFileChars);
+
+ //Clear file data buffer
+ MemoryUtil.InitializeBlock(secretFileData);
+
+ //Keep the char array as a secret
+ return SecretResult.ToSecret(secretFileChars);
+ }
+ }
+}
diff --git a/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/PluginSecretConstants.cs b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/PluginSecretConstants.cs
new file mode 100644
index 0000000..5c5a644
--- /dev/null
+++ b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/PluginSecretConstants.cs
@@ -0,0 +1,48 @@
+/*
+* Copyright (c) 2024 Vaughn Nugent
+*
+* Library: VNLib
+* Package: VNLib.Plugins.Extensions.Loading
+* File: PluginSecretConstants.cs
+*
+* PluginSecretConstants.cs is part of VNLib.Plugins.Extensions.Loading which is
+* part of the larger VNLib collection of libraries and utilities.
+*
+* VNLib.Plugins.Extensions.Loading is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Affero General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* VNLib.Plugins.Extensions.Loading is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU Affero General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program. If not, see https://www.gnu.org/licenses/.
+*/
+
+namespace VNLib.Plugins.Extensions.Loading
+{
+#pragma warning disable CA1707 // Identifiers should not contain underscores
+
+ public static class PluginSecretConstants
+ {
+ public const string VAULT_OBJECT_NAME = "hashicorp_vault";
+ public const string SECRETS_CONFIG_KEY = "secrets";
+ public const string VAULT_TOKEN_KEY = "token";
+ public const string VAULT_ROLE_KEY = "role";
+ public const string VAULT_SECRET_KEY = "secret";
+ public const string VAULT_TOKNE_ENV_NAME = "VNLIB_PLUGINS_VAULT_TOKEN";
+ public const string VAULT_KV_VERSION_KEY = "kv_version";
+
+ public const string VAULT_URL_KEY = "url";
+ public const string VAULT_TRUST_CERT_KEY = "trust_certificate";
+
+ public const string VAULT_URL_SCHEME = "vault://";
+ public const string ENV_URL_SCHEME = "env://";
+ public const string FILE_URL_SCHEME = "file://";
+ }
+
+#pragma warning restore CA1707 // Identifiers should not contain underscores
+}
diff --git a/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/PluginSecretStore.cs b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/PluginSecretStore.cs
new file mode 100644
index 0000000..6b20e30
--- /dev/null
+++ b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/PluginSecretStore.cs
@@ -0,0 +1,135 @@
+/*
+* Copyright (c) 2024 Vaughn Nugent
+*
+* Library: VNLib
+* Package: VNLib.Plugins.Extensions.Loading
+* File: PluginSecretStore.cs
+*
+* PluginSecretStore.cs is part of VNLib.Plugins.Extensions.Loading which is
+* part of the larger VNLib collection of libraries and utilities.
+*
+* VNLib.Plugins.Extensions.Loading is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Affero General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* VNLib.Plugins.Extensions.Loading is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU Affero General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program. If not, see https://www.gnu.org/licenses/.
+*/
+
+using System;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+
+using VNLib.Utils.Memory;
+
+using static VNLib.Plugins.Extensions.Loading.PluginSecretConstants;
+
+namespace VNLib.Plugins.Extensions.Loading
+{
+ /// <summary>
+ /// A secret store for a plugin that can be used to fetch secrets from plugin configuration
+ /// </summary>
+ /// <param name="plugin">The plugin instance to get secrets from</param>
+ public readonly struct PluginSecretStore(PluginBase plugin) : IEquatable<PluginSecretStore>
+ {
+ private readonly PluginBase _plugin = plugin;
+
+ /// <summary>
+ /// Gets the ambient vault client for the current plugin
+ /// if the configuration is loaded, null otherwise
+ /// </summary>
+ /// <returns>The ambient <see cref="IVaultClient"/> if loaded, null otherwise</returns>
+ /// <exception cref="KeyNotFoundException"></exception>
+ /// <exception cref="ObjectDisposedException"></exception>
+ public IHCVaultClient? GetVaultClient() => LoadingExtensions.GetOrCreateSingleton(_plugin, TryGetVaultLoader);
+
+ private static HCVaultClient? TryGetVaultLoader(PluginBase pbase)
+ {
+ //Get vault config
+ IConfigScope? conf = pbase.TryGetConfig(VAULT_OBJECT_NAME);
+
+ if (conf is null)
+ {
+ return null;
+ }
+
+ //try get server address creds from config
+ string serverAddress = conf.GetRequiredProperty(VAULT_URL_KEY, p => p.GetString()!);
+ bool trustCert = conf.TryGetValue(VAULT_TRUST_CERT_KEY, out JsonElement trustCertEl) && trustCertEl.GetBoolean();
+
+ int version = 2; //Default to version 2 now
+ string? authToken;
+
+ //Get authentication method from config
+ if (conf.TryGetValue(VAULT_TOKEN_KEY, out JsonElement tokenEl))
+ {
+ //Init token
+ authToken = tokenEl.GetString();
+ }
+ //Try to get the token as an environment variable
+ else if (Environment.GetEnvironmentVariable(VAULT_TOKNE_ENV_NAME) != null)
+ {
+ authToken = Environment.GetEnvironmentVariable(VAULT_TOKNE_ENV_NAME)!;
+ }
+ else
+ {
+ throw new KeyNotFoundException($"Failed to load the vault authentication method from {VAULT_OBJECT_NAME}");
+ }
+
+ _ = authToken ?? throw new KeyNotFoundException($"Failed to load the vault authentication method from {VAULT_OBJECT_NAME}");
+
+ //Check for vault kv version, otherwise use the default
+ if (conf.TryGetValue(VAULT_KV_VERSION_KEY, out JsonElement kvVersionEl))
+ {
+ version = kvVersionEl.GetInt32();
+ }
+
+ //create vault client, invalid or nulls will raise exceptions here
+ return HCVaultClient.Create(serverAddress, authToken, version, trustCert, MemoryUtil.Shared);
+ }
+
+ ///<inheritdoc/>
+ public Task<ISecretResult?> TryGetSecretAsync(string secretName, CancellationToken cancellation = default)
+ {
+ IOnDemandSecret secret = GetOnDemandSecret(secretName);
+ return secret.FetchSecretAsync(cancellation);
+ }
+
+ ///<inheritdoc/>
+ public ISecretResult? TryGetSecret(string secretName)
+ {
+ IOnDemandSecret secret = GetOnDemandSecret(secretName);
+ return secret.FetchSecret();
+ }
+
+ ///<inheritdoc/>
+ public IOnDemandSecret GetOnDemandSecret(string secretName)
+ {
+ ArgumentException.ThrowIfNullOrWhiteSpace(secretName);
+ return new OnDemandSecret(_plugin, secretName, GetVaultClient());
+ }
+
+ ///<inheritdoc/>
+ public override bool Equals(object? obj) => obj is PluginSecretStore store && Equals(store);
+
+ ///<inheritdoc/>
+ public static bool operator ==(PluginSecretStore left, PluginSecretStore right) => left.Equals(right);
+
+ ///<inheritdoc/>
+ public static bool operator !=(PluginSecretStore left, PluginSecretStore right) => !(left == right);
+
+ /// <inheritdoc/>
+ public bool Equals(PluginSecretStore other) => ReferenceEquals(other._plugin, _plugin);
+
+ ///<inheritdoc/>
+ public override int GetHashCode() => _plugin.GetHashCode();
+ }
+}
diff --git a/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/SecretResult.cs b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/SecretResult.cs
index 2c231b7..23f2276 100644
--- a/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/SecretResult.cs
+++ b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/SecretResult.cs
@@ -1,5 +1,5 @@
/*
-* Copyright (c) 2023 Vaughn Nugent
+* Copyright (c) 2024 Vaughn Nugent
*
* Library: VNLib
* Package: VNLib.Plugins.Extensions.Loading
@@ -25,7 +25,6 @@
using System;
using VNLib.Utils;
-using VNLib.Utils.Extensions;
using VNLib.Utils.Memory;
namespace VNLib.Plugins.Extensions.Loading
@@ -41,29 +40,41 @@ namespace VNLib.Plugins.Extensions.Loading
///<inheritdoc/>
public ReadOnlySpan<char> Result => _secretChars;
-
- internal SecretResult(ReadOnlySpan<char> value) : this(value.ToArray())
- { }
-
- private SecretResult(char[] secretChars)
- {
- _secretChars = secretChars;
- }
-
+ private SecretResult(char[] secretChars) => _secretChars = secretChars;
///<inheritdoc/>
- protected override void Free()
- {
- MemoryUtil.InitializeBlock(_secretChars);
- }
+ protected override void Free() => MemoryUtil.InitializeBlock(_secretChars);
+ /// <summary>
+ /// Copies the data from the provided string into a new secret result
+ /// then erases the original string
+ /// </summary>
+ /// <param name="result">The secret string to read</param>
+ /// <returns>The <see cref="SecretResult"/> wrapper</returns>
internal static SecretResult ToSecret(string? result)
{
- SecretResult res = new(result.AsSpan());
- MemoryUtil.UnsafeZeroMemory<char>(result);
+ if (result == null)
+ {
+ return new SecretResult([]);
+ }
+
+ //Copy string data into a new char array
+ SecretResult res = new(result.ToCharArray());
+
+ //PrivateStringManager will safely erase the original string if it is able to
+ PrivateStringManager.EraseString(result);
+
return res;
}
+ /// <summary>
+ /// Copies the data from the provided span into a new secret result
+ /// by allocating a new array internally
+ /// </summary>
+ /// <param name="secretChars">The array of characters to copy</param>
+ /// <returns>The wrapped secret</returns>
+ internal static SecretResult ToSecret(ReadOnlySpan<char> secretChars) => new(secretChars.ToArray());
+
internal static SecretResult ToSecret(char[] result) => new(result);
}
}
diff --git a/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/VaultSecrets.cs b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/VaultSecrets.cs
index 6ac1c0b..a5ba550 100644
--- a/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/VaultSecrets.cs
+++ b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/VaultSecrets.cs
@@ -1,12 +1,12 @@
/*
-* Copyright (c) 2023 Vaughn Nugent
+* Copyright (c) 2024 Vaughn Nugent
*
* Library: VNLib
* Package: VNLib.Plugins.Extensions.Loading
-* File: VaultSecrets.cs
+* File: PluginSecretLoading.cs
*
-* VaultSecrets.cs is part of VNLib.Plugins.Extensions.Loading which is part of the larger
-* VNLib collection of libraries and utilities.
+* PluginSecretLoading.cs is part of VNLib.Plugins.Extensions.Loading which
+* is part of the larger VNLib collection of libraries and utilities.
*
* VNLib.Plugins.Extensions.Loading is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@@ -23,26 +23,14 @@
*/
using System;
-using System.IO;
-using System.Linq;
using System.Text;
using System.Text.Json;
-using System.Threading;
using System.Threading.Tasks;
using System.Collections.Generic;
using System.Security.Cryptography.X509Certificates;
-using VaultSharp;
-using VaultSharp.V1.Commons;
-using VaultSharp.V1.AuthMethods;
-using VaultSharp.V1.AuthMethods.Token;
-using VaultSharp.V1.AuthMethods.AppRole;
-using VaultSharp.V1.SecretsEngines.PKI;
-
using VNLib.Utils;
using VNLib.Utils.Memory;
-using VNLib.Utils.Logging;
-using VNLib.Utils.Extensions;
using VNLib.Hashing.IdentityUtility;
namespace VNLib.Plugins.Extensions.Loading
@@ -53,19 +41,12 @@ namespace VNLib.Plugins.Extensions.Loading
/// </summary>
public static class PluginSecretLoading
{
- public const string VAULT_OBJECT_NAME = "hashicorp_vault";
- public const string SECRETS_CONFIG_KEY = "secrets";
- public const string VAULT_TOKEN_KEY = "token";
- public const string VAULT_ROLE_KEY = "role";
- public const string VAULT_SECRET_KEY = "secret";
- public const string VAULT_TOKNE_ENV_NAME = "VNLIB_PLUGINS_VAULT_TOKEN";
-
- public const string VAULT_URL_KEY = "url";
-
- public const string VAULT_URL_SCHEME = "vault://";
- public const string ENV_URL_SCHEME = "env://";
- public const string FILE_URL_SCHEME = "file://";
-
+ /// <summary>
+ /// Gets a wrapper for the secret store for the current plugin
+ /// </summary>
+ /// <param name="plugin"></param>
+ /// <returns>The secret store structure</returns>
+ public static PluginSecretStore Secrets(this PluginBase plugin) => new(plugin);
/// <summary>
/// <para>
@@ -103,291 +84,43 @@ namespace VNLib.Plugins.Extensions.Loading
/// <exception cref="ObjectDisposedException"></exception>
public static Task<ISecretResult?> TryGetSecretAsync(this PluginBase plugin, string secretName)
{
- plugin.ThrowIfUnloaded();
-
- //Get the secret from the config file raw
- string? rawSecret = TryGetSecretInternal(plugin, secretName);
-
- if (rawSecret == null)
- {
- return Task.FromResult<ISecretResult?>(null);
- }
-
- //Secret is a vault path, or return the raw value
- if (rawSecret.StartsWith(VAULT_URL_SCHEME, StringComparison.OrdinalIgnoreCase))
- {
- return GetSecretFromVaultAsync(plugin, rawSecret);
- }
-
- //See if the secret is an environment variable path
- if (rawSecret.StartsWith(ENV_URL_SCHEME, StringComparison.OrdinalIgnoreCase))
- {
- //try to get the environment variable
- string envVar = rawSecret[ENV_URL_SCHEME.Length..];
- string? envVal = Environment.GetEnvironmentVariable(envVar);
-
- return Task.FromResult<ISecretResult?>(envVal == null ? null : new SecretResult(envVal));
- }
-
- //See if the secret is a file path
- if (rawSecret.StartsWith(FILE_URL_SCHEME, StringComparison.OrdinalIgnoreCase))
- {
- string filePath = rawSecret[FILE_URL_SCHEME.Length..];
- return GetSecretFromFileAsync(filePath, plugin.UnloadToken);
- }
-
- //Finally, return the raw value
- return Task.FromResult<ISecretResult?>(new SecretResult(rawSecret.AsSpan()));
- }
-
- private static async Task<ISecretResult?> GetSecretFromFileAsync(string filePath, CancellationToken ct)
- {
- //read the file data
- byte[] secretFileData = await File.ReadAllBytesAsync(filePath, ct);
-
- //recover the character data from the file data
- int chars = Encoding.UTF8.GetCharCount(secretFileData);
- char[] secretFileChars = new char[chars];
- Encoding.UTF8.GetChars(secretFileData, secretFileChars);
-
- //Create secret from the file data
- SecretResult sr = SecretResult.ToSecret(secretFileChars);
-
- //Clear file data buffer
- MemoryUtil.InitializeBlock(secretFileData.AsSpan());
- return sr;
- }
-
- /// <summary>
- /// Gets a secret at the given vault url (in the form of "vault://[mount-name]/[secret-path]?secret=[secret_name]")
- /// </summary>
- /// <param name="plugin"></param>
- /// <param name="vaultPath">The raw vault url to lookup</param>
- /// <returns>The string of the object at the specified vault path</returns>
- /// <exception cref="UriFormatException"></exception>
- /// <exception cref="KeyNotFoundException"></exception>
- /// <exception cref="ObjectDisposedException"></exception>
- public static Task<ISecretResult?> GetSecretFromVaultAsync(this PluginBase plugin, ReadOnlySpan<char> vaultPath)
- {
- //print the path for debug
- if (plugin.IsDebug())
- {
- plugin.Log.Debug("Retrieving secret {s} from vault", vaultPath.ToString());
- }
-
- //Slice off path
- ReadOnlySpan<char> paq = vaultPath.SliceAfterParam(VAULT_URL_SCHEME);
- ReadOnlySpan<char> path = paq.SliceBeforeParam('?');
- ReadOnlySpan<char> query = paq.SliceAfterParam('?');
-
- if (paq.IsEmpty)
- {
- throw new UriFormatException("Vault secret location not valid/empty ");
- }
- //Get the secret
- string secretTableKey = query.SliceAfterParam("secret=").SliceBeforeParam('&').ToString();
- string vaultType = query.SliceBeforeParam("vault_type=").SliceBeforeParam('&').ToString();
-
- //get mount and path
- int lastSep = path.IndexOf('/');
- string mount = path[..lastSep].ToString();
- string secret = path[(lastSep + 1)..].ToString();
-
- async Task<ISecretResult?> execute()
- {
- //Try load client
- IVaultClient? client = plugin.GetVault();
-
- _ = client ?? throw new KeyNotFoundException("Vault client not found");
- //run read async
- Secret<SecretData> result = await client.V1.Secrets.KeyValue.V2.ReadSecretAsync(path:secret, mountPoint:mount);
- //Read the secret
- return SecretResult.ToSecret(result.Data.Data[secretTableKey].ToString());
- }
-
- return Task.Run(execute);
+ return plugin.Secrets().TryGetSecretAsync(secretName);
}
/// <summary>
/// <para>
- /// Gets a Certicate from the "secrets" element.
+ /// Gets a required secret from the "secrets" element.
/// </para>
/// <para>
/// Secrets elements are merged from the host config and plugin local config 'secrets' element.
/// before searching. The plugin config takes precedence over the host config.
/// </para>
/// </summary>
- /// <param name="plugin"></param>
+ /// <param name="secrets"></param>
/// <param name="secretName">The name of the secret propery to get</param>
- /// <returns>The element from the configuration file with the given name, or null if the configuration or property does not exist</returns>
+ /// <returns>The element from the configuration file with the given name, raises an exception if the secret does not exist</returns>
/// <exception cref="KeyNotFoundException"></exception>
/// <exception cref="ObjectDisposedException"></exception>
- public static Task<X509Certificate?> TryGetCertificateAsync(this PluginBase plugin, string secretName)
- {
- plugin.ThrowIfUnloaded();
- //Get the secret from the config file raw
- string? rawSecret = TryGetSecretInternal(plugin, secretName);
- if (rawSecret == null)
- {
- return Task.FromResult<X509Certificate?>(null);
- }
-
- //Secret is a vault path, or return the raw value
- if (!rawSecret.StartsWith(VAULT_URL_SCHEME, StringComparison.OrdinalIgnoreCase))
- {
- return Task.FromResult<X509Certificate?>(new (rawSecret));
- }
-
- return GetCertFromVaultAsync(plugin, rawSecret);
- }
-
- public static Task<X509Certificate?> GetCertFromVaultAsync(this PluginBase plugin, ReadOnlySpan<char> vaultPath, CertificateCredentialsRequestOptions? options = null)
+ public static async Task<ISecretResult> GetSecretAsync(this PluginSecretStore secrets, string secretName)
{
- //print the path for debug
- if (plugin.IsDebug())
- {
- plugin.Log.Debug("Retrieving certificate {s} from vault", vaultPath.ToString());
- }
-
- //Slice off path
- ReadOnlySpan<char> paq = vaultPath.SliceAfterParam(VAULT_URL_SCHEME);
- ReadOnlySpan<char> path = paq.SliceBeforeParam('?');
- ReadOnlySpan<char> query = paq.SliceAfterParam('?');
-
- if (paq.IsEmpty)
- {
- throw new UriFormatException("Vault secret location not valid/empty ");
- }
-
- //Get the secret
- string role = query.SliceAfterParam("role=").SliceBeforeParam('&').ToString();
- string vaultType = query.SliceBeforeParam("vault_type=").SliceBeforeParam('&').ToString();
- string commonName = query.SliceBeforeParam("cn=").SliceBeforeParam('&').ToString();
-
- //get mount and path
- int lastSep = path.IndexOf('/');
- string mount = path[..lastSep].ToString();
- string secret = path[(lastSep + 1)..].ToString();
-
- async Task<X509Certificate?> execute()
- {
- //Try load client
- IVaultClient? client = plugin.GetVault();
-
- _ = client ?? throw new KeyNotFoundException("Vault client not found");
-
- options ??= new()
- {
- CertificateFormat = CertificateFormat.pem,
- PrivateKeyFormat = PrivateKeyFormat.pkcs8,
- CommonName = commonName,
- };
-
- //run read async
- Secret<CertificateCredentials> result = await client.V1.Secrets.PKI.GetCredentialsAsync(pkiRoleName:secret, certificateCredentialRequestOptions:options, pkiBackendMountPoint:mount);
- //Read the secret
- byte[] pemCertData = Encoding.UTF8.GetBytes(result.Data.CertificateContent);
-
- return new (pemCertData);
- }
-
- return Task.Run(execute);
+ ISecretResult? res = await secrets.TryGetSecretAsync(secretName).ConfigureAwait(false);
+ return res ?? throw new KeyNotFoundException($"Missing required secret {secretName}");
}
/// <summary>
- /// Gets the ambient vault client for the current plugin
- /// if the configuration is loaded, null otherwise
+ /// Gets a secret at the given vault url (in the form of "vault://[mount-name]/[secret-path]?secret=[secret_name]")
/// </summary>
/// <param name="plugin"></param>
- /// <returns>The ambient <see cref="IVaultClient"/> if loaded, null otherwise</returns>
+ /// <param name="vaultPath">The raw vault url to lookup</param>
+ /// <returns>The string of the object at the specified vault path</returns>
+ /// <exception cref="UriFormatException"></exception>
/// <exception cref="KeyNotFoundException"></exception>
/// <exception cref="ObjectDisposedException"></exception>
- public static IVaultClient? GetVault(this PluginBase plugin) => LoadingExtensions.GetOrCreateSingleton(plugin, TryGetVaultLoader);
-
- private static string? TryGetSecretInternal(PluginBase plugin, string secretName)
- {
- bool local = plugin.PluginConfig.TryGetProperty(SECRETS_CONFIG_KEY, out JsonElement localEl);
- bool host = plugin.HostConfig.TryGetProperty(SECRETS_CONFIG_KEY, out JsonElement hostEl);
-
- //total config
- IReadOnlyDictionary<string, JsonElement>? conf;
-
- if (local && host)
- {
- //Load both config objects to dict
- Dictionary<string, JsonElement> localConf = localEl.EnumerateObject().ToDictionary(x => x.Name, x => x.Value);
- Dictionary<string, JsonElement> hostConf = hostEl.EnumerateObject().ToDictionary(x => x.Name, x => x.Value);
-
- //merge the two configs
- foreach(KeyValuePair<string, JsonElement> lc in localConf)
- {
- //Overwrite any host config keys, plugin conf takes priority
- hostConf[lc.Key] = lc.Value;
- }
- //set the merged config
- conf = hostConf;
- }
- else if(local)
- {
- //Store only local config
- conf = localEl.EnumerateObject().ToDictionary(x => x.Name, x => x.Value);
- }
- else if(host)
- {
- //store only host config
- conf = hostEl.EnumerateObject().ToDictionary(x => x.Name, x => x.Value);
- }
- else
- {
- conf = null;
- }
-
- //Get the value or default json element
- return conf != null && conf.TryGetValue(secretName, out JsonElement el) ? el.GetString() : null;
- }
-
- private static IVaultClient? TryGetVaultLoader(PluginBase pbase)
+ [Obsolete("Deprecated in favor of Secrets")]
+ public static Task<ISecretResult?> GetSecretFromVaultAsync(this PluginBase plugin, ReadOnlySpan<char> vaultPath)
{
- //Get vault config
- IConfigScope? conf = pbase.TryGetConfig(VAULT_OBJECT_NAME);
-
- if (conf == null)
- {
- return null;
- }
-
- //try get servre address creds from config
- string? serverAddress = conf[VAULT_URL_KEY].GetString() ?? throw new KeyNotFoundException($"Failed to load the key {VAULT_URL_KEY} from object {VAULT_OBJECT_NAME}");
-
- IAuthMethodInfo authMethod;
-
- //Get authentication method from config
- if (conf.TryGetValue(VAULT_TOKEN_KEY, out JsonElement tokenEl))
- {
- //Init token
- authMethod = new TokenAuthMethodInfo(tokenEl.GetString());
- }
- else if (conf.TryGetValue(VAULT_ROLE_KEY, out JsonElement roleEl) && conf.TryGetValue(VAULT_SECRET_KEY, out JsonElement secretEl))
- {
- authMethod = new AppRoleAuthMethodInfo(roleEl.GetString(), secretEl.GetString());
- }
- //Try to get the token as an environment variable
- else if(Environment.GetEnvironmentVariable(VAULT_TOKNE_ENV_NAME) != null)
- {
- string tokenValue = Environment.GetEnvironmentVariable(VAULT_TOKNE_ENV_NAME)!;
- authMethod = new TokenAuthMethodInfo(tokenValue);
- }
- else
- {
- throw new KeyNotFoundException($"Failed to load the vault authentication method from {VAULT_OBJECT_NAME}");
- }
-
- //Settings
- VaultClientSettings settings = new(serverAddress, authMethod);
-
- //create vault client
- return new VaultClient(settings);
- }
+ throw new NotSupportedException("This method is not supported in this context");
+ }
/// <summary>
/// Gets the Secret value as a byte buffer
@@ -398,7 +131,7 @@ namespace VNLib.Plugins.Extensions.Loading
/// <exception cref="InternalBufferTooSmallException"></exception>
public static byte[] GetFromBase64(this ISecretResult secret)
{
- _ = secret ?? throw new ArgumentNullException(nameof(secret));
+ ArgumentNullException.ThrowIfNull(secret);
//Temp buffer
using UnsafeMemoryHandle<byte> buffer = MemoryUtil.UnsafeAlloc(secret.Result.Length);
@@ -517,7 +250,7 @@ namespace VNLib.Plugins.Extensions.Loading
/// <exception cref="InternalBufferTooSmallException"></exception>
public static async Task<byte[]> ToBase64Bytes(this Task<ISecretResult> secret)
{
- _ = secret ?? throw new ArgumentNullException(nameof(secret));
+ ArgumentNullException.ThrowIfNull(secret);
using ISecretResult sec = await secret.ConfigureAwait(false);
@@ -533,8 +266,8 @@ namespace VNLib.Plugins.Extensions.Loading
/// <exception cref="ArgumentNullException"></exception>
public static async Task<ReadOnlyJsonWebKey> ToJsonWebKey(this Task<ISecretResult> secret)
{
- _ = secret ?? throw new ArgumentNullException(nameof(secret));
-
+ ArgumentNullException.ThrowIfNull(secret);
+
using ISecretResult sec = await secret.ConfigureAwait(false);
return sec?.GetJsonWebKey();
@@ -554,7 +287,7 @@ namespace VNLib.Plugins.Extensions.Loading
/// <exception cref="KeyNotFoundException"></exception>
public static async Task<ReadOnlyJsonWebKey> ToJsonWebKey(this Task<ISecretResult> secret, bool required)
{
- _ = secret ?? throw new ArgumentNullException(nameof(secret));
+ ArgumentNullException.ThrowIfNull(secret);
using ISecretResult sec = await secret.ConfigureAwait(false);
@@ -573,8 +306,8 @@ namespace VNLib.Plugins.Extensions.Loading
/// <exception cref="ArgumentNullException"></exception>
public static IAsyncLazy<TResult> ToLazy<TResult>(this Task<ISecretResult> result, Func<ISecretResult, TResult> transformer)
{
- _ = result ?? throw new ArgumentNullException(nameof(result));
- _ = transformer ?? throw new ArgumentNullException(nameof(transformer));
+ ArgumentNullException.ThrowIfNull(result);
+ ArgumentNullException.ThrowIfNull(transformer);
//standard secret transformer
static async Task<TResult> Run(Task<ISecretResult> tr, Func<ISecretResult, TResult> transformer)
@@ -597,8 +330,8 @@ namespace VNLib.Plugins.Extensions.Loading
/// <exception cref="ArgumentNullException"></exception>
public static IAsyncLazy<TResult> ToLazy<TResult>(this Task<ISecretResult> result, Func<ISecretResult, Task<TResult>> transformer)
{
- _ = result ?? throw new ArgumentNullException(nameof(result));
- _ = transformer ?? throw new ArgumentNullException(nameof(transformer));
+ ArgumentNullException.ThrowIfNull(result);
+ ArgumentNullException.ThrowIfNull(transformer);
//Transform with task transformer
static async Task<TResult> Run(Task<ISecretResult?> tr, Func<ISecretResult, Task<TResult>> transformer)
@@ -609,5 +342,8 @@ namespace VNLib.Plugins.Extensions.Loading
return Run(result, transformer).AsLazy();
}
+
+#nullable enable
+
}
}
diff --git a/lib/VNLib.Plugins.Extensions.Loading/src/VNLib.Plugins.Extensions.Loading.csproj b/lib/VNLib.Plugins.Extensions.Loading/src/VNLib.Plugins.Extensions.Loading.csproj
index 63bce5a..1dfaa30 100644
--- a/lib/VNLib.Plugins.Extensions.Loading/src/VNLib.Plugins.Extensions.Loading.csproj
+++ b/lib/VNLib.Plugins.Extensions.Loading/src/VNLib.Plugins.Extensions.Loading.csproj
@@ -46,7 +46,6 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
- <PackageReference Include="VaultSharp" Version="1.13.0.1" />
</ItemGroup>
<ItemGroup>