From 49c3641def5ae1b7557ed61ed7bb28bbf425ccc9 Mon Sep 17 00:00:00 2001 From: vnugent Date: Sun, 24 Mar 2024 21:26:44 -0400 Subject: Squashed commit of the following: commit a504435151efbe1d19404fa44859b15c629f6d5d Author: vnugent Date: Sun Mar 24 20:55:01 2024 -0400 chore: Updated compose and added some more logging commit c74440ff12daa03cc4b7792d0c3baad46a11a465 Author: vnugent Date: Mon Mar 18 21:57:57 2024 -0400 feat: message checksum support & dynamic serializers commit 9983582db08d3e6c456295ea96e482cbb4f31f42 Author: vnugent Date: Sun Mar 10 21:58:28 2024 -0400 source tree project location updated commit 60f09bde87b5c59ef937c62ef64b7745bc3711b5 Merge: 2f75659 e5bb0ee Author: vnugent Date: Sun Mar 10 16:50:09 2024 -0400 Merge remote-tracking branch 'origin/master' into develop commit 2f7565976472f0f056db60520bf253a776112c10 Merge: 323ff67 6b87785 Author: vnugent Date: Sun Mar 10 16:45:23 2024 -0400 merge master commit 323ff67badfc46ad638d75f059d60d9425ccb2fa Author: vnugent Date: Sun Mar 10 15:50:07 2024 -0400 ci(server): Conainerize and add vncache server packages commit 5d4192880654fd6e00e587814169415b42621327 Author: vnugent Date: Sat Mar 9 19:13:21 2024 -0500 chore: #2 Minor fixes and polish before release commit a4b3504bb891829074d1efde0433eae010862181 Author: vnugent Date: Sat Mar 9 16:30:44 2024 -0500 package updates commit 4d8cfc10382105b0acbd94df93ad3d05ff91db54 Author: vnugent Date: Wed Mar 6 21:30:58 2024 -0500 refactor: #2 Centralize server state, default discovery endpoints & more commit 016a96a80cce025a86c6cf26707738f6a2eb2658 Author: vnugent Date: Thu Feb 29 21:22:38 2024 -0500 feat: add future support for memory diagnostics, and some docs commit 456ead9bc8b0f61357bae93152ad0403c4940101 Author: vnugent Date: Tue Feb 13 14:46:35 2024 -0500 fix: #1 shared cluster index on linux & latested core updates commit a481d63f964a5d5204cac2e95141f37f9a28d573 Author: vnugent Date: Tue Jan 23 15:43:50 2024 -0500 cache extension api tweaks --- Module.Taskfile.yaml | 24 +++--- Taskfile.yaml | 13 ++-- .../src/VNLib.Data.Caching.Extensions.csproj | 8 +- .../src/BlobCacheLIstener.cs | 72 +++++++++++++++-- .../src/BlobCacheListenerConfig.cs | 56 ++++++++++++++ .../src/CacheEntry.cs | 4 +- lib/VNLib.Data.Caching/src/ClientExtensions.cs | 63 +++++++++++++-- lib/VNLib.Data.Caching/src/Constants.cs | 21 ++++- .../src/Exceptions/InvalidChecksumException.cs | 45 +++++++++++ .../src/Exceptions/InvalidStatusException.cs | 4 +- lib/VNLib.Data.Caching/src/FbmMessageChecksum.cs | 90 ++++++++++++++++++++++ .../src/VNLib.Data.Caching.csproj | 2 +- .../src/VNLib.Plugins.Extensions.VNCache.csproj | 2 +- .../ObjectCacheServer/server/container/Dockerfile | 1 + .../ObjectCacheServer-template.json | 3 +- .../server/container/docker-compose.yaml | 10 ++- plugins/ObjectCacheServer/server/taskfile.yaml | 12 +-- .../src/Cache/CacheMemoryConfiguration.cs | 3 + .../src/Endpoints/ConnectEndpoint.cs | 4 +- .../ObjectCacheServer/src/ObjectCacheServer.csproj | 4 +- .../src/ObjectCacheSystemState.cs | 20 ++++- .../src/RedisClientCacheEntry.cs | 29 ++++++- .../src/VNLib.Data.Caching.Providers.Redis.csproj | 2 +- .../src/FBMCacheClient.cs | 30 +++++--- .../src/VNCacheBase.cs | 49 +++++++++--- .../src/VNCacheConfig.cs | 9 ++- .../VNLib.Data.Caching.Providers.VNCache.csproj | 2 +- 27 files changed, 488 insertions(+), 94 deletions(-) create mode 100644 lib/VNLib.Data.Caching.ObjectCache/src/BlobCacheListenerConfig.cs create mode 100644 lib/VNLib.Data.Caching/src/Exceptions/InvalidChecksumException.cs create mode 100644 lib/VNLib.Data.Caching/src/FbmMessageChecksum.cs diff --git a/Module.Taskfile.yaml b/Module.Taskfile.yaml index e36c517..0d4cd95 100644 --- a/Module.Taskfile.yaml +++ b/Module.Taskfile.yaml @@ -16,13 +16,14 @@ vars: PACK_OUT: '{{.OUTPUT_DIR}}/{{.HEAD_SHA}}/pkg' tasks: + #called by build pipeline to sync repo update: cmds: - - git remote update - - git reset --hard + - git reset --hard #clean up any local changes + - git remote update - git pull origin {{.BRANCH_NAME}} --verify-signatures - #re-write semver after hard reset so build still works properly + #re-write semver after hard reset - dotnet-gitversion.exe /updateprojectfiles #called by build pipeline to build module @@ -34,24 +35,21 @@ tasks: - task: build_debug - task: build_release - postbuild_success: + publish: cmds: + #git archive in the module directory + - git archive --format {{.ARCHIVE_FILE_FORMAT}} --output {{.ARCHIVE_FILE_NAME}} HEAD #push packages to the sleet feed (feed path is vnbuild global) - sleet push "{{.PACK_OUT}}/debug/" --source debug --config "{{.SLEET_CONFIG_PATH}}" --force - sleet push "{{.PACK_OUT}}/release/" --source release --config "{{.SLEET_CONFIG_PATH}}" --force - #git archive in the module directory - - git archive --format {{.ARCHIVE_FILE_FORMAT}} --output {{.ARCHIVE_FILE_NAME}} HEAD - - postbuild_failed: - cmds: - - echo "postbuild failed {{.MODULE_NAME}}" - #called by build pipeline to clean module clean: cmds: - #clean solution - - dotnet clean /p:BuildInParallel=true /p:MultiProcessorCompilation=true + #clean solution + - dotnet clean /p:BuildInParallel=true /p:MultiProcessorCompilation=true + - cmd: powershell -Command "rm {{ .ARCHIVE_FILE_NAME }} --Force" + ignore_error: true #Internal tasks diff --git a/Taskfile.yaml b/Taskfile.yaml index 48aef59..0b441a3 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -12,7 +12,6 @@ version: '3' vars: TARGET: '{{.USER_WORKING_DIR}}/bin' RELEASE_DIR: "./bin/release/{{.TARGET_FRAMEWORK}}/publish" - SOURCE_OUT: "{{.USER_WORKING_DIR}}/bin/source" tasks: @@ -37,9 +36,7 @@ tasks: postbuild_failed: dir: '{{.USER_WORKING_DIR}}' - cmds: - - echo "postbuild failed {{.PROJECT_NAME}}" - + cmds: [] postbuild: dir: '{{.USER_WORKING_DIR}}' @@ -48,8 +45,7 @@ tasks: #the build output directory BUILD_OUT: "{{.USER_WORKING_DIR}}/bin/{{.BUILD_MODE}}/{{.TARGET_FRAMEWORK}}/publish" - cmds: - + cmds: #copy license and readme to target - cd .. && powershell -Command "Copy-Item -Path ./build.readme.md -Destination '{{.BUILD_OUT}}/readme.md'" @@ -67,7 +63,8 @@ tasks: #Remove the output dirs on clean clean: dir: '{{.USER_WORKING_DIR}}' + ignore_error: true cmds: - - for: ['./bin', './obj'] + - for: ['bin/', 'obj/'] cmd: powershell Remove-Item -Recurse '{{.ITEM}}' - ignore_error: true + diff --git a/lib/VNLib.Data.Caching.Extensions/src/VNLib.Data.Caching.Extensions.csproj b/lib/VNLib.Data.Caching.Extensions/src/VNLib.Data.Caching.Extensions.csproj index 99879c4..87772ab 100644 --- a/lib/VNLib.Data.Caching.Extensions/src/VNLib.Data.Caching.Extensions.csproj +++ b/lib/VNLib.Data.Caching.Extensions/src/VNLib.Data.Caching.Extensions.csproj @@ -47,10 +47,10 @@ - - - - + + + + diff --git a/lib/VNLib.Data.Caching.ObjectCache/src/BlobCacheLIstener.cs b/lib/VNLib.Data.Caching.ObjectCache/src/BlobCacheLIstener.cs index 972bf5e..7908313 100644 --- a/lib/VNLib.Data.Caching.ObjectCache/src/BlobCacheLIstener.cs +++ b/lib/VNLib.Data.Caching.ObjectCache/src/BlobCacheLIstener.cs @@ -43,7 +43,6 @@ using System.Threading; using System.Threading.Tasks; using VNLib.Utils.Logging; -using VNLib.Net.Messaging.FBM; using VNLib.Net.Messaging.FBM.Server; using static VNLib.Data.Caching.Constants; @@ -57,18 +56,22 @@ namespace VNLib.Data.Caching.ObjectCache /// /// The cache table to work from /// The event queue to publish changes to - /// Writes error and debug logging information - /// The heap to alloc FBM buffers and cache buffers from + /// The listener configuration object /// - public class BlobCacheListener(IBlobCacheTable cache, ICacheListenerEventQueue queue, ILogProvider log, IFBMMemoryManager memoryManager) + public class BlobCacheListener(IBlobCacheTable cache, BlobCacheListenerConfig config, ICacheListenerEventQueue queue) : FBMListenerBase, IDisposable { private bool disposedValue; /// - protected override ILogProvider Log { get; } = log; + protected override ILogProvider Log { get; } = config.Log; /// - protected override FBMListener Listener { get; } = new(memoryManager); + protected override FBMListener Listener { get; } = new(config.MemoryManager); + + /// + /// The configuration instance for the listener + /// + public BlobCacheListenerConfig Config { get; } = config ?? throw new ArgumentNullException(nameof(config)); /// /// A queue that stores update and delete events @@ -80,6 +83,9 @@ namespace VNLib.Data.Caching.ObjectCache /// public IBlobCacheTable Cache { get; } = cache ?? throw new ArgumentNullException(nameof(cache)); + + private readonly ILogProvider _tLog = config.LogTransactions ? config.Log : new NullLogger(); + /// protected override async Task ProcessAsync(FBMContext context, T? userState, CancellationToken exitToken) { @@ -110,6 +116,25 @@ namespace VNLib.Data.Caching.ObjectCache //Create change event for the object ChangeEvent change = new(objectId, alternateId, false); + if (config.EnableMessageChecksums) + { + switch (context.Request.IsClientChecksumValid()) + { + //0 is checksum sent, supported, but invalid + case 0: + context.CloseResponse(ResponseCodes.InvalidChecksum); + return; + + case -2: //Method not supported, set an error header but allow the request + context.Response.WriteHeader(ChecksumWarning, "Checksum method not supported"); + break; + + case 1: //1 is checksum sent and valid + case -1: //No checksum sent + break; + } + } + await AddOrUpdateAsync(context, change, exitToken); return; } @@ -199,6 +224,12 @@ namespace VNLib.Data.Caching.ObjectCache if (handle.Cache.TryGetValue(objectId, out CacheEntry data)) { + //Compute an fnv message checksum and send it to the client + if (config.EnableMessageChecksums) + { + FbmMessageChecksum.WriteFnv1aChecksum(context.Response, data.GetDataSegment()); + } + //Set the status code and write the buffered data to the response buffer context.CloseResponse(ResponseCodes.Okay); @@ -222,6 +253,8 @@ namespace VNLib.Data.Caching.ObjectCache if (found) { EnqueEvent(change); + + _tLog.Debug("Deleted cache entry {id}", change.CurrentId); } } @@ -232,6 +265,8 @@ namespace VNLib.Data.Caching.ObjectCache EnqueEvent(change); + _tLog.Debug("Cache entry {id} added or updated. New ID {nid}", change.CurrentId, change.AlternateId); + context.CloseResponse(ResponseCodes.Okay); } @@ -239,7 +274,7 @@ namespace VNLib.Data.Caching.ObjectCache { EventQueue.PublishEvent(change); } - + /// protected virtual void Dispose(bool disposing) @@ -258,5 +293,28 @@ namespace VNLib.Data.Caching.ObjectCache Dispose(disposing: true); GC.SuppressFinalize(this); } + + sealed class NullLogger : ILogProvider + { + public void Flush() + { } + + public object GetLogProvider() => null!; + + + public bool IsEnabled(LogLevel level) => false; + + public void Write(LogLevel level, string value) + { } + + public void Write(LogLevel level, Exception exception, string value = "") + { } + + public void Write(LogLevel level, string value, params object?[] args) + { } + + public void Write(LogLevel level, string value, params ValueType[] args) + { } + } } } diff --git a/lib/VNLib.Data.Caching.ObjectCache/src/BlobCacheListenerConfig.cs b/lib/VNLib.Data.Caching.ObjectCache/src/BlobCacheListenerConfig.cs new file mode 100644 index 0000000..492dfb8 --- /dev/null +++ b/lib/VNLib.Data.Caching.ObjectCache/src/BlobCacheListenerConfig.cs @@ -0,0 +1,56 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Data.Caching.ObjectCache +* File: BlobCacheListenerConfig.cs +* +* BlobCacheListenerConfig.cs is part of VNLib.Data.Caching.ObjectCache which is part of the larger +* VNLib collection of libraries and utilities. +* +* VNLib.Data.Caching.ObjectCache is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Data.Caching.ObjectCache is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using VNLib.Utils.Logging; +using VNLib.Net.Messaging.FBM; + +namespace VNLib.Data.Caching.ObjectCache +{ + /// + /// A configuration object for + /// + public sealed record class BlobCacheListenerConfig + { + /// + /// Writes error and debug logging information + /// + public ILogProvider Log { get; init; } = null!; + + /// + /// The memory manager used for the internal FBM server listener + /// + public IFBMMemoryManager MemoryManager { get; init; } = null!; + + /// + /// A flag that enables verifying and sending checksums with message + /// data in FBM header fields + /// + public bool EnableMessageChecksums { get; init; } = true; + + /// + /// A flag that enables logging of transactions (events) to the log + /// + public bool LogTransactions { get; init; } + } +} diff --git a/lib/VNLib.Data.Caching.ObjectCache/src/CacheEntry.cs b/lib/VNLib.Data.Caching.ObjectCache/src/CacheEntry.cs index 9370901..eddfc42 100644 --- a/lib/VNLib.Data.Caching.ObjectCache/src/CacheEntry.cs +++ b/lib/VNLib.Data.Caching.ObjectCache/src/CacheEntry.cs @@ -89,8 +89,8 @@ namespace VNLib.Data.Caching /// public static CacheEntry FromExistingHandle(object handle, ICacheEntryMemoryManager manager) { - _ = handle ?? throw new ArgumentNullException(nameof(handle)); - _ = manager ?? throw new ArgumentNullException(nameof(manager)); + ArgumentNullException.ThrowIfNull(handle); + ArgumentNullException.ThrowIfNull(manager); //validate handle size it at least the minimum size if (manager.GetHandleSize(handle) < DATA_SEGMENT_START) diff --git a/lib/VNLib.Data.Caching/src/ClientExtensions.cs b/lib/VNLib.Data.Caching/src/ClientExtensions.cs index e0aa744..bfc8ddc 100644 --- a/lib/VNLib.Data.Caching/src/ClientExtensions.cs +++ b/lib/VNLib.Data.Caching/src/ClientExtensions.cs @@ -37,7 +37,7 @@ using VNLib.Data.Caching.Exceptions; using static VNLib.Data.Caching.Constants; namespace VNLib.Data.Caching -{ +{ /// /// Provides caching extension methods for @@ -212,11 +212,11 @@ namespace VNLib.Data.Caching return ExecAsync(client, request, objectId, cancellationToken); } - catch + catch(Exception e) { //Return the request(clears data and reset) client.ReturnRequest(request); - throw; + return Task.FromException(e); } static async Task ExecAsync(FBMClient client, FBMRequest request, string objectId, CancellationToken cancellationToken) @@ -239,6 +239,10 @@ namespace VNLib.Data.Caching { throw new ObjectNotFoundException($"object {objectId} not found on remote server"); } + else if(status.ValueEquals(ResponseCodes.InvalidChecksum, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidChecksumException($"The server rejected the message {objectId} due to an invalid checksum"); + } //Invalid status throw new InvalidStatusException("Invalid status code recived for object upsert request", status.ToString()); @@ -256,6 +260,7 @@ namespace VNLib.Data.Caching /// Gets an object from the server if it exists /// /// + /// /// /// The id of the object to get /// A token to cancel the operation @@ -368,18 +373,35 @@ namespace VNLib.Data.Caching response.ThrowIfNotSet(); //Get the status code - FBMMessageHeader status = response.Headers.FirstOrDefault(static a => a.Header == HeaderCommand.Status); + FBMMessageHeader status = response.Headers.FirstOrDefault(static a => a.Header == HeaderCommand.Status); //Check ok status code, then its safe to deserialize - if (status.Value.Equals(ResponseCodes.Okay, StringComparison.Ordinal)) + if (status.ValueEquals(ResponseCodes.Okay, StringComparison.Ordinal)) { + //Add message integrity check + FBMMessageHeader checksumType = response.Headers.FirstOrDefault(static a => a.Header == ChecksumType); + FBMMessageHeader checksum = response.Headers.FirstOrDefault(static a => a.Header == ChecksumValue); + + if(checksumType.ValueEquals(ChecksumTypes.Fnv1a, StringComparison.OrdinalIgnoreCase)) + { + //Verify the checksum + if (!FbmMessageChecksum.VerifyFnv1aChecksum(checksum.Value, response.ResponseBody)) + { + throw new InvalidChecksumException( + $"The response data integrety check failed. The message data was corrupted for id: {checksum.GetValueString()}" + ); + } + + //Valid checksum, continue + } + //Write the object data setter(state, response.ResponseBody); return true; } //Object may not exist on the server yet - if (status.Value.Equals(ResponseCodes.NotFound, StringComparison.Ordinal)) + if (status.ValueEquals(ResponseCodes.NotFound, StringComparison.Ordinal)) { return false; } @@ -540,5 +562,34 @@ namespace VNLib.Data.Caching return new (worker, retryDelay, serverUri); } + /// + /// Determines if the the client sent a message checksum, and if so, verifies the checksum + /// if the checksum type is supported. + /// + /// + /// + /// -1 if the checksum type or value is not set, + /// -2 if the checksum type is not supported, + /// 0 if the checksum is invalid, + /// 1 if the checksum is valid + /// + public static int IsClientChecksumValid(this FBMRequestMessage message) + { + string? type = message.Headers.FirstOrDefault(static h => h.Header == ChecksumType).GetValueString(); + ReadOnlySpan value = message.Headers.FirstOrDefault(static h => h.Header == ChecksumValue).Value; + + if (type == null || value.IsEmpty) + { + return -1; + } + + if(type.Equals(ChecksumTypes.Fnv1a, StringComparison.OrdinalIgnoreCase)) + { + //Verify the checksum + return FbmMessageChecksum.VerifyFnv1aChecksum(value, message.BodyData) ? 1 : 0; + } + + return -2; + } } } diff --git a/lib/VNLib.Data.Caching/src/Constants.cs b/lib/VNLib.Data.Caching/src/Constants.cs index 3fb87e0..6c2051a 100644 --- a/lib/VNLib.Data.Caching/src/Constants.cs +++ b/lib/VNLib.Data.Caching/src/Constants.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2022 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Data.Caching @@ -22,12 +22,11 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -using System; - using VNLib.Net.Messaging.FBM; namespace VNLib.Data.Caching { + public static class Constants { /// @@ -48,9 +47,25 @@ namespace VNLib.Data.Caching public const string Okay = "ok"; public const string Error = "err"; public const string NotFound = "nf"; + public const string InvalidChecksum = "cm"; } public const HeaderCommand ObjectId = (HeaderCommand)0xAA; public const HeaderCommand NewObjectId = (HeaderCommand)0xAB; + public const HeaderCommand ChecksumType = (HeaderCommand)0xA1; + public const HeaderCommand ChecksumValue = (HeaderCommand)0xA2; + public const HeaderCommand ChecksumWarning = (HeaderCommand)0xA3; + + /// + /// Contains constants for checksum type names for FBM headers + /// + public static class ChecksumTypes + { + public const string Fnv1a = "fnv1a"; + public const string Crc32 = "crc32"; + public const string Md5 = "md5"; + public const string Sha1 = "sha1"; + public const string Sha256 = "sha256"; + } } } diff --git a/lib/VNLib.Data.Caching/src/Exceptions/InvalidChecksumException.cs b/lib/VNLib.Data.Caching/src/Exceptions/InvalidChecksumException.cs new file mode 100644 index 0000000..4e60b4b --- /dev/null +++ b/lib/VNLib.Data.Caching/src/Exceptions/InvalidChecksumException.cs @@ -0,0 +1,45 @@ +/* +* Copyright (c) 2022 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Data.Caching +* File: InvalidStatusException.cs +* +* InvalidStatusException.cs is part of VNLib.Data.Caching which is part of the larger +* VNLib collection of libraries and utilities. +* +* VNLib.Data.Caching is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Data.Caching is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using System; + +using VNLib.Net.Messaging.FBM; + +namespace VNLib.Data.Caching.Exceptions +{ + /// + /// Raised when the checksum of a cache result does not match the expected checksum + /// + public class InvalidChecksumException : InvalidResponseException + { + public InvalidChecksumException(string message) : base(message) + { } + + public InvalidChecksumException(string message, Exception innerException) : base(message, innerException) + { } + + public InvalidChecksumException() + { } + } +} diff --git a/lib/VNLib.Data.Caching/src/Exceptions/InvalidStatusException.cs b/lib/VNLib.Data.Caching/src/Exceptions/InvalidStatusException.cs index 2296774..ad880b7 100644 --- a/lib/VNLib.Data.Caching/src/Exceptions/InvalidStatusException.cs +++ b/lib/VNLib.Data.Caching/src/Exceptions/InvalidStatusException.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2022 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Data.Caching @@ -42,7 +42,7 @@ namespace VNLib.Data.Caching.Exceptions /// public InvalidStatusException(string message, string statusCode):this(message) { - this.StatusCode = statusCode; + StatusCode = statusCode; } /// diff --git a/lib/VNLib.Data.Caching/src/FbmMessageChecksum.cs b/lib/VNLib.Data.Caching/src/FbmMessageChecksum.cs new file mode 100644 index 0000000..5a9af31 --- /dev/null +++ b/lib/VNLib.Data.Caching/src/FbmMessageChecksum.cs @@ -0,0 +1,90 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Data.Caching +* File: FbmMessageChecksum.cs +* +* FbmMessageChecksum.cs is part of VNLib.Data.Caching which is part of the larger +* VNLib collection of libraries and utilities. +* +* VNLib.Data.Caching is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Data.Caching is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using System; +using System.Buffers.Binary; +using System.Diagnostics; + +using VNLib.Utils; +using VNLib.Hashing.Checksums; +using VNLib.Net.Messaging.FBM; + +using static VNLib.Data.Caching.Constants; + +namespace VNLib.Data.Caching +{ + /// + /// Utility class for verifying and writing checksums for FBM messages + /// + public static class FbmMessageChecksum + { + /// + /// Verifies the checksum of the supplied data using the FNV1a algorithm + /// + /// The checksum base32 encoded string of the checksum data + /// The data to compute the checksum on + /// True if the checksum of the data matches the supplied one + public static bool VerifyFnv1aChecksum(ReadOnlySpan checksum, ReadOnlySpan data) + { + //Convert the checksum to bytes + Span asBytes = stackalloc byte[sizeof(ulong)]; + ERRNO byteSize = VnEncoding.TryFromBase32Chars(checksum, asBytes); + + Debug.Assert(byteSize == sizeof(ulong), "Failed to convert checksum to bytes"); + + //Compute the checksum of the supplied data + ulong computed = FNV1a.Compute64(data); + + //Compare the checksums + return BinaryPrimitives.ReadUInt64BigEndian(asBytes) == computed; + } + + /// + /// Writes the FNV1a checksum of the supplied data to the message header buffer + /// + /// The FBM message to write the checksum headers to + /// The message data to compute the checksum of + /// + public static void WriteFnv1aChecksum(IFBMMessage message, ReadOnlySpan data) + { + ArgumentNullException.ThrowIfNull(message); + + //Compute the checksum of the data + ulong checksum = FNV1a.Compute64(data); + + Span asBytes = stackalloc byte[sizeof(ulong)]; + Span asChars = stackalloc char[16]; + + //get big endian bytes + BinaryPrimitives.WriteUInt64BigEndian(asBytes, checksum); + ERRNO charSize = VnEncoding.TryToBase32Chars(asBytes, asChars); + + Debug.Assert(charSize > 0, "Failed to convert checksum to base32"); + + //Write the checksum and type to the response + message.WriteHeader(ChecksumType, ChecksumTypes.Fnv1a); + message.WriteHeader(ChecksumValue, asChars[..(int)charSize]); + } + } +} diff --git a/lib/VNLib.Data.Caching/src/VNLib.Data.Caching.csproj b/lib/VNLib.Data.Caching/src/VNLib.Data.Caching.csproj index d906985..3208d12 100644 --- a/lib/VNLib.Data.Caching/src/VNLib.Data.Caching.csproj +++ b/lib/VNLib.Data.Caching/src/VNLib.Data.Caching.csproj @@ -47,7 +47,7 @@ - + diff --git a/lib/VNLib.Plugins.Extensions.VNCache/src/VNLib.Plugins.Extensions.VNCache.csproj b/lib/VNLib.Plugins.Extensions.VNCache/src/VNLib.Plugins.Extensions.VNCache.csproj index 37b292f..4d1827e 100644 --- a/lib/VNLib.Plugins.Extensions.VNCache/src/VNLib.Plugins.Extensions.VNCache.csproj +++ b/lib/VNLib.Plugins.Extensions.VNCache/src/VNLib.Plugins.Extensions.VNCache.csproj @@ -36,7 +36,7 @@ - + diff --git a/plugins/ObjectCacheServer/server/container/Dockerfile b/plugins/ObjectCacheServer/server/container/Dockerfile index 6c466d4..725b9d1 100644 --- a/plugins/ObjectCacheServer/server/container/Dockerfile +++ b/plugins/ObjectCacheServer/server/container/Dockerfile @@ -55,6 +55,7 @@ ENV MAX_ENTRIES=10000 ENV CACHE_BUCKETS=100 ENV CACHE_MAX_MESSAGE=20480 ENV MAX_CONCURRENT_CONNECTIONS=1000 +ENV ENABLE_CHECKSUMS=true ENV VERIFY_IP=true ENV MAX_PEER_NODES=10 diff --git a/plugins/ObjectCacheServer/server/container/config-templates/ObjectCacheServer-template.json b/plugins/ObjectCacheServer/server/container/config-templates/ObjectCacheServer-template.json index 765c3d7..564039a 100644 --- a/plugins/ObjectCacheServer/server/container/config-templates/ObjectCacheServer-template.json +++ b/plugins/ObjectCacheServer/server/container/config-templates/ObjectCacheServer-template.json @@ -45,7 +45,8 @@ "buffer_recv_min": 8192, //min of 8k transfer buffer "buffer_header_max": 2048, //2k max header buffer size "buffer_header_min": 128, //128 byte min request header buffer size - "max_message_size": ${CACHE_MAX_MESSAGE} //Absolute maxium message size allowed, also the maxium size of cache entires + "max_message_size": ${CACHE_MAX_MESSAGE}, //Absolute maxium message size allowed, also the maxium size of cache entires + "enable_checksums": ${ENABLE_CHECKSUMS} //Enable checksums for cache entries }, //Known peers array, must point to well-known endpoint for discovery diff --git a/plugins/ObjectCacheServer/server/container/docker-compose.yaml b/plugins/ObjectCacheServer/server/container/docker-compose.yaml index c1b61fa..5aa494e 100644 --- a/plugins/ObjectCacheServer/server/container/docker-compose.yaml +++ b/plugins/ObjectCacheServer/server/container/docker-compose.yaml @@ -10,6 +10,7 @@ services: restart: unless-stopped hostname: vncache-server volumes: + - ./data/:/app/data:rw #optional writes log files to the host (may be required in the future) - ./assets:/app/usr/assets:ro #optional if assets are required - ./ssl:/app/ssl:ro #optional only if SSL is enabled (currently not a feature) ports: @@ -18,18 +19,19 @@ services: #System memory consumption is calculated as follows: # MAX_ENTIRES x CACHE_BUCKETS x CACHE_MAX_MESSAGE = max memory consumption - MAX_CONCURRENT_CONNECTIONS: "1000" #max number of concurrent connections + MAX_CONCURRENT_CONNECTIONS: "1000" #max number of concurrent client connections MAX_ENTRIES: "10000" #max number of cache entries per bucket CACHE_BUCKETS: "100" #number of cache buckets for load balancing CACHE_MAX_MESSAGE: "20480" #20KB VERIFY_IP: "true" #verfies the IP address of clients during negotiation (recommended) MAX_PEER_NODES: "10" #max number of other peer nodes this node shoud connect to DISCOVERY_INTERVAL: "360" #time (in seconds) between peer node discovery - KNOWN_PEERS: '[]' #array of known peer nodes in the cluster + KNOWN_PEERS: '[]' #array of known peer nodes in the cluster + ENABLE_CHECKSUMS: "true" #enables checksums for messages #SECRETS (must be JWK formatted keys) - CACHE_PRIV_KEY: "" #REQUIRED local private key used to identify and sign messages to clients and other nodes - CLIENT_PUB_KEY: "" #REQUIRED used to verify client messages + CACHE_PRIV_KEY: '' #REQUIRED local private key used to identify and sign messages to clients and other nodes + CLIENT_PUB_KEY: '' #REQUIRED used to verify client messages #HC vault #HC_VAULT_ADDR: "" diff --git a/plugins/ObjectCacheServer/server/taskfile.yaml b/plugins/ObjectCacheServer/server/taskfile.yaml index 38eae79..9455451 100644 --- a/plugins/ObjectCacheServer/server/taskfile.yaml +++ b/plugins/ObjectCacheServer/server/taskfile.yaml @@ -38,12 +38,12 @@ tasks: VNLIB_SHARED_HEAP_FILE_PATH: lib/libvn_rpmalloc cmds: - - cmd: dotnet webserver/VNLib.WebServer.dll --config config/config.json --input-off --inline-scheduler {{.ARGS}} + - cmd: dotnet webserver/VNLib.WebServer.dll --config config/config.json --input-off --inline-scheduler {{.CLI_ARGS}} #setup sever environment - setup-debian: - desc: "Performs initial setup on Debian x64 based machines" + setup-apt: + desc: "Performs initial setup on Debian/APT x64 based machines" silent: true cmds: - apt update @@ -51,8 +51,8 @@ tasks: - task: setup - echo "Setup complete" - setup-fedora: - desc: "Performs initial setup on Fedora/Redhat x64 (dnf) based machines" + setup-dnf: + desc: "Performs initial setup on Fedora using DNF x64 (dnf) based machines" silent: true cmds: - dnf update @@ -61,7 +61,7 @@ tasks: - echo "Setup complete" setup-alpine: - desc: "Performs initial setup on Alpine x64 based machines" + desc: "Performs initial setup on Alpine using APK x64 based machines" silent: true cmds: - apk update diff --git a/plugins/ObjectCacheServer/src/Cache/CacheMemoryConfiguration.cs b/plugins/ObjectCacheServer/src/Cache/CacheMemoryConfiguration.cs index c404cc5..0b81447 100644 --- a/plugins/ObjectCacheServer/src/Cache/CacheMemoryConfiguration.cs +++ b/plugins/ObjectCacheServer/src/Cache/CacheMemoryConfiguration.cs @@ -54,5 +54,8 @@ namespace VNLib.Data.Caching.ObjectCache.Server.Cache [JsonPropertyName("memory_lib_path")] public string? ExternLibPath { get; set; } + + [JsonPropertyName("enable_checksums")] + public bool EnableChecksums { get; set; } = true; } } diff --git a/plugins/ObjectCacheServer/src/Endpoints/ConnectEndpoint.cs b/plugins/ObjectCacheServer/src/Endpoints/ConnectEndpoint.cs index 8368d3a..42f406a 100644 --- a/plugins/ObjectCacheServer/src/Endpoints/ConnectEndpoint.cs +++ b/plugins/ObjectCacheServer/src/Endpoints/ConnectEndpoint.cs @@ -244,6 +244,8 @@ namespace VNLib.Data.Caching.ObjectCache.Server.Endpoints { WsUserState state = wss.UserState!; + Log.Debug("Client established websocket connection {sid}", wss.SocketID); + //Notify peers of new connection Peers.OnPeerConnected(state); @@ -307,7 +309,7 @@ namespace VNLib.Data.Caching.ObjectCache.Server.Endpoints //Notify monitor of disconnect Peers.OnPeerDisconnected(state); - Log.Debug("Server websocket exited"); + Log.Debug("Client websocket disconnected {sid}", wss.SocketID); } diff --git a/plugins/ObjectCacheServer/src/ObjectCacheServer.csproj b/plugins/ObjectCacheServer/src/ObjectCacheServer.csproj index c903511..009e905 100644 --- a/plugins/ObjectCacheServer/src/ObjectCacheServer.csproj +++ b/plugins/ObjectCacheServer/src/ObjectCacheServer.csproj @@ -48,8 +48,8 @@ - - + + diff --git a/plugins/ObjectCacheServer/src/ObjectCacheSystemState.cs b/plugins/ObjectCacheServer/src/ObjectCacheSystemState.cs index cd5bf1b..3b3e2c0 100644 --- a/plugins/ObjectCacheServer/src/ObjectCacheSystemState.cs +++ b/plugins/ObjectCacheServer/src/ObjectCacheSystemState.cs @@ -187,15 +187,27 @@ namespace VNLib.Data.Caching.ObjectCache.Server CacheListenerPubQueue queue = new(plugin, PeerEventQueue); - //Must register background worker to listen for changes + //Must register the queue background worker to listen for changes _ = plugin.ObserveWork(queue, 150); + BlobCacheListenerConfig conf = new() + { + Log = plugin.Log.CreateScope(CacheConstants.LogScopes.BlobCacheListener), + MemoryManager = new SharedHeapFBMMemoryManager(SharedCacheHeap), + EnableMessageChecksums = MemoryConfiguration.EnableChecksums, + LogTransactions = plugin.IsDebug() || plugin.HostArgs.HasArgument("--log-cache-events") + }; + + if (conf.LogTransactions) + { + plugin.Log.Information("Verbose cache event logging enabled"); + } + //Endpoint only allows for a single reader Listener = new( plugin.LoadMemoryCacheSystem(config, manager, MemoryConfiguration), - queue, - plugin.Log.CreateScope(CacheConstants.LogScopes.BlobCacheListener), - new SharedHeapFBMMemoryManager(SharedCacheHeap) + conf, + queue ); InternalStore = new CacheStore(Listener.Cache); diff --git a/plugins/VNLib.Data.Caching.Providers.Redis/src/RedisClientCacheEntry.cs b/plugins/VNLib.Data.Caching.Providers.Redis/src/RedisClientCacheEntry.cs index 360be58..7e91fe7 100644 --- a/plugins/VNLib.Data.Caching.Providers.Redis/src/RedisClientCacheEntry.cs +++ b/plugins/VNLib.Data.Caching.Providers.Redis/src/RedisClientCacheEntry.cs @@ -69,8 +69,6 @@ namespace VNLib.Data.Caching.Providers.Redis public RedisClientCacheEntry(PluginBase plugin, IConfigScope config) { _defaultHeap = MemoryUtil.Shared; - DefaultDeserializer = new JsonCacheObjectSerializer(256); - DefaultSerializer = new JsonCacheObjectSerializer(256); ILogProvider redisLog = plugin.Log.CreateScope("REDIS"); @@ -121,6 +119,31 @@ namespace VNLib.Data.Caching.Providers.Redis redisLog.Information("Successfully connected to Redis server"); }); } + + string? serialzerDllPath = config.GetPropString("serializer_assebly_name"); + + //See if user has specified a custom serializer assembly + if (!string.IsNullOrWhiteSpace(serialzerDllPath)) + { + //Load the custom serializer assembly and get the serializer and deserializer instances + DefaultSerializer = plugin.CreateServiceExternal(serialzerDllPath); + + //Avoid creating another instance if the deserializer is the same as the serializer + if (DefaultSerializer is ICacheObjectDeserializer cod) + { + DefaultDeserializer = cod; + } + else + { + DefaultDeserializer = plugin.CreateServiceExternal(serialzerDllPath); + } + } + else + { + //If no default serializer is set, use the default JSON serializer + DefaultDeserializer = new JsonCacheObjectSerializer(256); + DefaultSerializer = new JsonCacheObjectSerializer(256); + } } private static ConfigurationOptions GetOptionsFromConfig(IConfigScope config) @@ -311,7 +334,7 @@ namespace VNLib.Data.Caching.Providers.Redis /// public object GetUnderlyingStore() { - return _database == null ? throw new InvalidOperationException("The cache store is not available") : _database; + return _database is null ? throw new InvalidOperationException("The cache store is not available") : _database; } private sealed class AddOrUpdateBuffer: VnDisposeable, IBufferWriter diff --git a/plugins/VNLib.Data.Caching.Providers.Redis/src/VNLib.Data.Caching.Providers.Redis.csproj b/plugins/VNLib.Data.Caching.Providers.Redis/src/VNLib.Data.Caching.Providers.Redis.csproj index c6da1e6..ac5fb63 100644 --- a/plugins/VNLib.Data.Caching.Providers.Redis/src/VNLib.Data.Caching.Providers.Redis.csproj +++ b/plugins/VNLib.Data.Caching.Providers.Redis/src/VNLib.Data.Caching.Providers.Redis.csproj @@ -43,7 +43,7 @@ - + diff --git a/plugins/VNLib.Data.Caching.Providers.VNCache/src/FBMCacheClient.cs b/plugins/VNLib.Data.Caching.Providers.VNCache/src/FBMCacheClient.cs index 07fc9ee..e84a077 100644 --- a/plugins/VNLib.Data.Caching.Providers.VNCache/src/FBMCacheClient.cs +++ b/plugins/VNLib.Data.Caching.Providers.VNCache/src/FBMCacheClient.cs @@ -78,7 +78,8 @@ namespace VNLib.Data.Caching.Providers.VNCache public FBMCacheClient(PluginBase plugin, IConfigScope config) : this( config.Deserialze(), - plugin.IsDebug() ? plugin.Log : null + plugin.IsDebug() ? plugin.Log : null, + plugin ) { ILogProvider scoped = plugin.Log.CreateScope(LOG_NAME); @@ -103,7 +104,11 @@ namespace VNLib.Data.Caching.Providers.VNCache } } - public FBMCacheClient(VnCacheClientConfig config, ILogProvider? debugLog):base(config) + public FBMCacheClient(VnCacheClientConfig config, ILogProvider? debugLog) : this(config, debugLog, null) + { } + + + private FBMCacheClient(VnCacheClientConfig config, ILogProvider? debugLog, PluginBase? plugin) : base(config) { //Validate config (config as IOnConfigValidation).Validate(); @@ -115,13 +120,13 @@ namespace VNLib.Data.Caching.Providers.VNCache //Init the client with default settings FBMClientConfig conf = FBMDataCacheExtensions.GetDefaultConfig(BufferHeap, (int)config.MaxBlobSize, config.RequestTimeout, debugLog); - + FBMClientFactory clientFactory = new( - in conf, - new FBMFallbackClientWsFactory(), + in conf, + new FBMFallbackClientWsFactory(), 10 ); - + _cluster = (new CacheClientConfiguration()) .WithTls(config.UseTls) .WithInitialPeers(config.GetInitialNodeUris()) @@ -129,6 +134,9 @@ namespace VNLib.Data.Caching.Providers.VNCache //Init index _index = ClusterNodeIndex.CreateIndex(_cluster); + + //Init serializers + InitSerializers(config, plugin); } /* @@ -296,7 +304,7 @@ namespace VNLib.Data.Caching.Providers.VNCache public override Task DeleteAsync(string key, CancellationToken cancellation) { return !IsConnected - ? throw new InvalidOperationException("The underlying client is not connected to a cache node") + ? Task.FromException(new InvalidOperationException("The underlying client is not connected to a cache node")) : _client!.DeleteObjectAsync(key, cancellation); } @@ -304,7 +312,7 @@ namespace VNLib.Data.Caching.Providers.VNCache public override Task GetAsync(string key, ICacheObjectDeserializer deserializer, CancellationToken cancellation) { return !IsConnected - ? throw new InvalidOperationException("The underlying client is not connected to a cache node") + ? Task.FromException(new InvalidOperationException("The underlying client is not connected to a cache node")) : _client!.GetObjectAsync(key, deserializer, cancellation); } @@ -312,7 +320,7 @@ namespace VNLib.Data.Caching.Providers.VNCache public override Task AddOrUpdateAsync(string key, string? newKey, T value, ICacheObjectSerializer serialzer, CancellationToken cancellation) { return !IsConnected - ? throw new InvalidOperationException("The underlying client is not connected to a cache node") + ? Task.FromException(new InvalidOperationException("The underlying client is not connected to a cache node")) : _client!.AddOrUpdateObjectAsync(key, newKey, value, serialzer, cancellation); } @@ -320,7 +328,7 @@ namespace VNLib.Data.Caching.Providers.VNCache public override Task GetAsync(string key, ObjectDataSet callback, T state, CancellationToken cancellation) { return !IsConnected - ? throw new InvalidOperationException("The underlying client is not connected to a cache node") + ? Task.FromException(new InvalidOperationException("The underlying client is not connected to a cache node")) : _client!.GetObjectAsync(key, callback, state, cancellation); } @@ -328,7 +336,7 @@ namespace VNLib.Data.Caching.Providers.VNCache public override Task AddOrUpdateAsync(string key, string? newKey, ObjectDataGet callback, T state, CancellationToken cancellation) { return !IsConnected - ? throw new InvalidOperationException("The underlying client is not connected to a cache node") + ? Task.FromException(new InvalidOperationException("The underlying client is not connected to a cache node")) : _client!.AddOrUpdateObjectAsync(key, newKey, callback, state, cancellation); } diff --git a/plugins/VNLib.Data.Caching.Providers.VNCache/src/VNCacheBase.cs b/plugins/VNLib.Data.Caching.Providers.VNCache/src/VNCacheBase.cs index c337ef4..dc1ab8f 100644 --- a/plugins/VNLib.Data.Caching.Providers.VNCache/src/VNCacheBase.cs +++ b/plugins/VNLib.Data.Caching.Providers.VNCache/src/VNCacheBase.cs @@ -25,26 +25,22 @@ using System.Threading; using System.Threading.Tasks; +using VNLib.Plugins; +using VNLib.Plugins.Extensions.Loading; + namespace VNLib.Data.Caching.Providers.VNCache { - internal abstract class VNCacheBase : IGlobalCacheProvider + internal abstract class VNCacheBase(VNCacheConfig conf) : IGlobalCacheProvider { /// public abstract bool IsConnected { get; } /// - public virtual ICacheObjectDeserializer DefaultDeserializer { get; } + public virtual ICacheObjectDeserializer DefaultDeserializer => conf.CacheObjectDeserializer!; /// - public virtual ICacheObjectSerializer DefaultSerializer { get; } - - protected VNCacheBase(VNCacheConfig config) - { - //Set default serializers - DefaultDeserializer = config.CacheObjectDeserializer ?? new JsonCacheObjectSerializer(256); - DefaultSerializer = config.CacheObjectSerializer ?? new JsonCacheObjectSerializer(256); - } - + public virtual ICacheObjectSerializer DefaultSerializer => conf.CacheObjectSerializer!; + /// public abstract Task AddOrUpdateAsync(string key, string? newKey, T value, ICacheObjectSerializer serialzer, CancellationToken cancellation); @@ -62,5 +58,36 @@ namespace VNLib.Data.Caching.Providers.VNCache /// public abstract object GetUnderlyingStore(); + + /// + /// Initializes a set of cache object serializers and deserializers + /// for the configuration instance and loads external serializers if specified + /// by the user. + /// + /// The configuration instance to initialzie + /// Optional plugin for loading external serializers + protected static void InitSerializers(VNCacheConfig config, PluginBase? plugin) + { + //See if user has specified a custom serializer assembly + if (!string.IsNullOrWhiteSpace(config.SerializerDllPath)) + { + //Load the custom serializer assembly and get the serializer and deserializer instances + config.CacheObjectSerializer = plugin.CreateServiceExternal(config.SerializerDllPath); + + //Avoid creating another instance if the deserializer is the same as the serializer + if(config.CacheObjectSerializer is ICacheObjectDeserializer cod) + { + config.CacheObjectDeserializer = cod; + } + else + { + config.CacheObjectDeserializer = plugin.CreateServiceExternal(config.SerializerDllPath); + } + } + + //If no default serializer is set, use the default JSON serializer + config.CacheObjectSerializer ??= new JsonCacheObjectSerializer(256); + config.CacheObjectDeserializer ??= new JsonCacheObjectSerializer(256); + } } } \ No newline at end of file diff --git a/plugins/VNLib.Data.Caching.Providers.VNCache/src/VNCacheConfig.cs b/plugins/VNLib.Data.Caching.Providers.VNCache/src/VNCacheConfig.cs index 8311519..24008b3 100644 --- a/plugins/VNLib.Data.Caching.Providers.VNCache/src/VNCacheConfig.cs +++ b/plugins/VNLib.Data.Caching.Providers.VNCache/src/VNCacheConfig.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Data.Caching.Providers.VNCache @@ -98,6 +98,11 @@ namespace VNLib.Data.Caching.Providers.VNCache throw new ArgumentException("You must configure a maximum object size", "max_object_size"); } } - + + /// + /// Optional external cache serializer library to load + /// + [JsonPropertyName("serializer_assebly_name")] + public string? SerializerDllPath { get; set; } } } \ No newline at end of file diff --git a/plugins/VNLib.Data.Caching.Providers.VNCache/src/VNLib.Data.Caching.Providers.VNCache.csproj b/plugins/VNLib.Data.Caching.Providers.VNCache/src/VNLib.Data.Caching.Providers.VNCache.csproj index 99a5962..ded805a 100644 --- a/plugins/VNLib.Data.Caching.Providers.VNCache/src/VNLib.Data.Caching.Providers.VNCache.csproj +++ b/plugins/VNLib.Data.Caching.Providers.VNCache/src/VNLib.Data.Caching.Providers.VNCache.csproj @@ -39,7 +39,7 @@ - + -- cgit