From 7a263bf54b7967ddeb9f6b662339ec1c74546ce8 Mon Sep 17 00:00:00 2001 From: vnugent Date: Sat, 9 Mar 2024 14:19:31 -0500 Subject: refactor: Overhaul secret loading. Remove VaultSharp as a dep --- .../src/Secrets/HCVaultClient.cs | 419 +++++++++++++++++++++ 1 file changed, 419 insertions(+) create mode 100644 lib/VNLib.Plugins.Extensions.Loading/src/Secrets/HCVaultClient.cs (limited to 'lib/VNLib.Plugins.Extensions.Loading/src/Secrets/HCVaultClient.cs') diff --git a/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/HCVaultClient.cs b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/HCVaultClient.cs new file mode 100644 index 0000000..a06f490 --- /dev/null +++ b/lib/VNLib.Plugins.Extensions.Loading/src/Secrets/HCVaultClient.cs @@ -0,0 +1,419 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Plugins.Extensions.Loading +* File: HCVaultClient.cs +* +* HCVaultClient.cs is part of VNLib.Plugins.Extensions.Loading which +* is part of the larger VNLib collection of libraries and utilities. +* +* VNLib.Plugins.Extensions.Loading is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Plugins.Extensions.Loading is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using System; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Net; +using System.Net.Http; +using System.Net.Sockets; +using System.Net.Security; +using System.Diagnostics; +using System.Threading.Tasks; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +using VNLib.Utils; +using VNLib.Utils.IO; +using VNLib.Utils.Memory; +using VNLib.Utils.Extensions; + +/* + * The purpose of the HCVaultClient is to provide a very simple Hashicorp Vault client + * interface that reads KV secrets from a vault server with minimal dependencies. + * + * Since I only need the KV store for now, I don't think there is a need for the + * VaultSharp package which adds at least 600kb to the final package size. + */ + +namespace VNLib.Plugins.Extensions.Loading +{ + internal sealed class HCVaultClient : VnDisposeable, IHCVaultClient + { + const string VaultTokenHeaderName = "X-Vault-Token"; + const long MaxErrResponseContentLength = 8192; + const uint DefaultBufferSize = 4096; + + private static readonly TimeSpan ClientDefaultTimeout = TimeSpan.FromSeconds(30); + + private readonly HttpClient _client; + private readonly int _kvVersion; + private readonly IUnmangedHeap _bufferHeap; + + HCVaultClient(string serverAddress, string hcToken, int kvVersion, bool trustCert, IUnmangedHeap heap) + { +#pragma warning disable CA2000 // Dispose objects before losing scope + HttpClientHandler handler = new() + { + AllowAutoRedirect = false, + UseCookies = false, + MaxResponseHeadersLength = 2048, + ClientCertificateOptions = ClientCertificateOption.Automatic, + AutomaticDecompression = DecompressionMethods.All, + PreAuthenticate = false, + + //Setup a callback to trust the server certificate if the cert chain is invalid + ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => trustCert || errors == SslPolicyErrors.None + }; + +#pragma warning restore CA2000 // Dispose objects before losing scope + + _client = new HttpClient(handler, true) + { + BaseAddress = new Uri(serverAddress), + Timeout = ClientDefaultTimeout, + DefaultRequestVersion = new Version(1, 1), + MaxResponseContentBufferSize = 4096 //Buffer only needs to be little for vault requests + }; + + + //Set the vault access token header, should probably clean this up later + _client.DefaultRequestHeaders.Add(VaultTokenHeaderName, hcToken); + _kvVersion = kvVersion; + _bufferHeap = heap; + } + + /// + /// Creates a new Hashicorp vault client with the given server address, token, and KV storage version + /// + /// The vault server address + /// The vault token used to connect to the vault server + /// The hc vault Key value store version (must be 1 or 2) + /// A value that tells the HTTP client to trust the Vault server's certificate even if it's not valid + /// Heap instance to allocate internal buffers from + /// The new client instance + /// + /// + public static HCVaultClient Create(string serverAddress, string hcToken, int kvVersion, bool trustCert, IUnmangedHeap heap) + { + ArgumentException.ThrowIfNullOrEmpty(serverAddress); + ArgumentException.ThrowIfNullOrEmpty(hcToken); + ArgumentNullException.ThrowIfNull(heap); + + if(kvVersion != 1 && kvVersion != 2) + { + throw new ArgumentException($"Unsupported vault KV storage version {kvVersion}, must be either 1 or 2"); + } + + return new HCVaultClient(serverAddress, hcToken, kvVersion, trustCert, heap); + } + + /// + protected override void Free() + { + _client.Dispose(); + } + + /// + public async Task ReadSecretAsync(string path, string mountPoint, string secretName) + { + string secretPath = GetSecretPathForKvVersion(_kvVersion, path, mountPoint); + using HttpRequestMessage ms = GetRequestMessageForPath(secretPath); + + try + { + using HttpResponseMessage response = await _client.SendAsync(ms, HttpCompletionOption.ResponseHeadersRead); + + //Check if an error occured in the response + await ProcessVaultErrorResponseAsync(response, true); + + //Read the response async + using SecretResponse res = await ReadSecretResponse(response.Content, true); + + return FromResponse(res, secretName); + } + catch(HttpRequestException he) when(he.InnerException is SocketException se) + { + throw se.SocketErrorCode switch + { + SocketError.HostNotFound => new HCVaultException("Failed to connect to Hashicorp Vault server, because it's DNS hostname could not be resolved"), + SocketError.ConnectionRefused => new HCVaultException("Failed to establish a TCP connection to the vault server, the server refused the connection"), + _ => new HCVaultException("Failed to establish a TCP connection to the vault server, see inner exception", se), + }; + } + catch(Exception ex) + { + throw new HCVaultException("Failed to retreive secret from Hashicorp Vault server, see inner exception", ex); + } + } + + /// + public ISecretResult? ReadSecret(string path, string mountPoint, string secretName) + { + string secretPath = GetSecretPathForKvVersion(_kvVersion, path, mountPoint); + using HttpRequestMessage ms = GetRequestMessageForPath(secretPath); + + try + { + //Exec the response synchronously + using HttpResponseMessage response = _client.Send(ms, HttpCompletionOption.ResponseHeadersRead); + + /* + * It is safe to await the error result here because its + * already completed when the async flag is false + */ + ValueTask errTask = ProcessVaultErrorResponseAsync(response, false); + Debug.Assert(errTask.IsCompleted); + errTask.GetAwaiter().GetResult(); + + //Did not throw, handle a secret response + + ValueTask resTask = ReadSecretResponse(response.Content, false); + Debug.Assert(resTask.IsCompleted); + + //Always wrap response in using to clean memory + using SecretResponse res = resTask.GetAwaiter().GetResult(); + + return FromResponse(res, secretName); + } + catch (HttpRequestException he) when (he.InnerException is SocketException se) + { + throw se.SocketErrorCode switch + { + SocketError.HostNotFound => new HCVaultException("Failed to connect to Hashicorp Vault server, because it's DNS hostname could not be resolved"), + SocketError.ConnectionRefused => new HCVaultException("Failed to establish a TCP connection to the vault server, the server refused the connection"), + _ => new HCVaultException("Failed to establish a TCP connection to the vault server, see inner exception", se), + }; + } + catch (Exception ex) + { + throw new HCVaultException("Failed to retreive secret from Hashicorp Vault server, see inner exception", ex); + } + } + + private ValueTask ReadSecretResponse(HttpContent content, bool async) + { + SecretResponse res = new(DefaultBufferSize, _bufferHeap); + try + { + if (async) + { + return ReadStreamAsync(content, res); + } + else + { + //Read into a memory stream + content.CopyTo(res.StreamData, null, default); + res.ResetStream(); + + return ValueTask.FromResult(res); + } + } + catch + { + res.Dispose(); + throw; + } + + async static ValueTask ReadStreamAsync(HttpContent content, SecretResponse response) + { + try + { + await content.CopyToAsync(response.StreamData); + + response.ResetStream(); + + return response; + } + catch + { + response.Dispose(); + throw; + } + } + + } + + private static string GetSecretPathForKvVersion(int version, string path, string mount) + { + return version switch + { + 1 => $"v1/{mount}/{path}", + 2 => $"v1/{mount}/data/{path}", + _ => throw new InvalidOperationException("Invalid KV version") + }; + } + + private static HttpRequestMessage GetRequestMessageForPath(string secretPath) + { + return new(HttpMethod.Get, secretPath) + { + VersionPolicy = HttpVersionPolicy.RequestVersionOrHigher, + }; + } + + private static SecretResult? FromResponse(SecretResponse res, string secretName) + { + using JsonDocument json = res.AsJson(); + + if (!json.RootElement.TryGetProperty("data", out JsonElement dataEl)) + { + throw new HttpRequestException("Vault KV response did not include a top-level 'data' element"); + } + + if (!dataEl.TryGetProperty("data", out dataEl)) + { + throw new HttpRequestException("Vault KV response did not include a 'data' element"); + } + + //Try to get the secret from the data element + if (dataEl.TryGetProperty(secretName, out JsonElement secretEl)) + { + string? secValue = secretEl.GetString(); + return secValue == null ? null : SecretResult.ToSecret(secValue); + } + + return null; + } + + private static ValueTask ProcessVaultErrorResponseAsync(HttpResponseMessage response, bool async) + { + if (response.IsSuccessStatusCode) + { + return default; + } + + //Make sure the response has content + long? ctLen = response.Content.Headers.ContentLength; + if(!ctLen.HasValue || ctLen.Value == 0) + { + return ValueTask.FromException( + new HttpRequestException($"Failed to fetch secret from vault with error code {response.StatusCode}") + ); + } + + //Check for way too big response entity body + if(ctLen.Value > MaxErrResponseContentLength) + { + return ValueTask.FromException( + new HttpRequestException( + $"Vault error {response.StatusCode}. Response content length was too large, expected less than {MaxErrResponseContentLength} but got {ctLen.Value}" + )); + } + + + //Assert json response body + if (!string.Equals("application/json", response.Content.Headers.ContentType?.MediaType, StringComparison.OrdinalIgnoreCase)) + { + return ValueTask.FromException( + new HttpRequestException("Vault response was not in JSON format") + ); + } + + return async ? ExceptionsFromContentAsync(response) : ExceptionsFromContent(response); + + static ValueTask ExceptionFromVaultErrors(HttpStatusCode code, VaultErrorMessage? errs) + { + //If the error message is null, raise an exception + if (errs == null || errs.Errors == null || errs.Errors.Length == 0) + { + return ValueTask.FromException( + new HttpRequestException($"Failed to fetch secret from vault with error code {code}") + ); + } + + //Join the errors into a single string with newlines + IEnumerable errors = errs.Errors.Select(err => $"Vault Error -> {err}"); + string errStr = string.Join(Environment.NewLine, errors); + + //Finally raise the exception with all the returned errors + return ValueTask.FromException( + new HttpRequestException($"Failed to fetch secre from vault with {code}, errors:\n {errStr}") + ); + } + + static async ValueTask ExceptionsFromContentAsync(HttpResponseMessage response) + { + //Read stream async and deserialize async + using Stream stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false); + VaultErrorMessage? errs = await JsonSerializer.DeserializeAsync(stream); + + await ExceptionFromVaultErrors(response.StatusCode, errs); + } + + static ValueTask ExceptionsFromContent(HttpResponseMessage response) + { +#pragma warning disable CA1849 // Call async methods when in an async method + + //Read the error content stream and deserialize + using Stream stream = response.Content.ReadAsStream(); + VaultErrorMessage? errs = JsonSerializer.Deserialize(stream); + +#pragma warning restore CA1849 // Call async methods when in an async method + + return ExceptionFromVaultErrors(response.StatusCode, errs); + } + } + + + private sealed class SecretResponse : VnDisposeable + { + /* + * Purpose of this class is to hold a memory stream that can read + * the vault response into memory, use it for some operation, + * then zero the memory before releasing it back to the heap + */ + + private readonly MemoryHandle _memHandle; + + public VnMemoryStream StreamData { get; } + + public SecretResponse(uint initSize, IUnmangedHeap heap) + { + _memHandle = heap.Alloc(initSize, false); + StreamData = VnMemoryStream.FromHandle(_memHandle, false, 0, false); + } + + /// + /// Gets a from the response data + /// + /// + public JsonDocument AsJson() + { + //read the data as a raw span then parse it as json + Utf8JsonReader reader = new(StreamData.AsSpan()); + return JsonDocument.ParseValue(ref reader); + } + + /// + /// Resets the stream to the beginning + /// + public void ResetStream() => StreamData.Seek(0, SeekOrigin.Begin); + + protected override void Free() + { + //zero the handle before disposing + MemoryUtil.InitializeBlock(ref _memHandle.GetReference(), _memHandle.GetIntLength()); + _memHandle.Dispose(); + } + } + + private sealed class VaultErrorMessage + { + [JsonPropertyName("errors")] + public string[]? Errors { get; set; } + } + } +} -- cgit