From 49c3641def5ae1b7557ed61ed7bb28bbf425ccc9 Mon Sep 17 00:00:00 2001 From: vnugent Date: Sun, 24 Mar 2024 21:26:44 -0400 Subject: Squashed commit of the following: commit a504435151efbe1d19404fa44859b15c629f6d5d Author: vnugent Date: Sun Mar 24 20:55:01 2024 -0400 chore: Updated compose and added some more logging commit c74440ff12daa03cc4b7792d0c3baad46a11a465 Author: vnugent Date: Mon Mar 18 21:57:57 2024 -0400 feat: message checksum support & dynamic serializers commit 9983582db08d3e6c456295ea96e482cbb4f31f42 Author: vnugent Date: Sun Mar 10 21:58:28 2024 -0400 source tree project location updated commit 60f09bde87b5c59ef937c62ef64b7745bc3711b5 Merge: 2f75659 e5bb0ee Author: vnugent Date: Sun Mar 10 16:50:09 2024 -0400 Merge remote-tracking branch 'origin/master' into develop commit 2f7565976472f0f056db60520bf253a776112c10 Merge: 323ff67 6b87785 Author: vnugent Date: Sun Mar 10 16:45:23 2024 -0400 merge master commit 323ff67badfc46ad638d75f059d60d9425ccb2fa Author: vnugent Date: Sun Mar 10 15:50:07 2024 -0400 ci(server): Conainerize and add vncache server packages commit 5d4192880654fd6e00e587814169415b42621327 Author: vnugent Date: Sat Mar 9 19:13:21 2024 -0500 chore: #2 Minor fixes and polish before release commit a4b3504bb891829074d1efde0433eae010862181 Author: vnugent Date: Sat Mar 9 16:30:44 2024 -0500 package updates commit 4d8cfc10382105b0acbd94df93ad3d05ff91db54 Author: vnugent Date: Wed Mar 6 21:30:58 2024 -0500 refactor: #2 Centralize server state, default discovery endpoints & more commit 016a96a80cce025a86c6cf26707738f6a2eb2658 Author: vnugent Date: Thu Feb 29 21:22:38 2024 -0500 feat: add future support for memory diagnostics, and some docs commit 456ead9bc8b0f61357bae93152ad0403c4940101 Author: vnugent Date: Tue Feb 13 14:46:35 2024 -0500 fix: #1 shared cluster index on linux & latested core updates commit a481d63f964a5d5204cac2e95141f37f9a28d573 Author: vnugent Date: Tue Jan 23 15:43:50 2024 -0500 cache extension api tweaks --- lib/VNLib.Data.Caching/src/ClientExtensions.cs | 63 +++++++++++++-- lib/VNLib.Data.Caching/src/Constants.cs | 21 ++++- .../src/Exceptions/InvalidChecksumException.cs | 45 +++++++++++ .../src/Exceptions/InvalidStatusException.cs | 4 +- lib/VNLib.Data.Caching/src/FbmMessageChecksum.cs | 90 ++++++++++++++++++++++ .../src/VNLib.Data.Caching.csproj | 2 +- 6 files changed, 213 insertions(+), 12 deletions(-) create mode 100644 lib/VNLib.Data.Caching/src/Exceptions/InvalidChecksumException.cs create mode 100644 lib/VNLib.Data.Caching/src/FbmMessageChecksum.cs (limited to 'lib/VNLib.Data.Caching') diff --git a/lib/VNLib.Data.Caching/src/ClientExtensions.cs b/lib/VNLib.Data.Caching/src/ClientExtensions.cs index e0aa744..bfc8ddc 100644 --- a/lib/VNLib.Data.Caching/src/ClientExtensions.cs +++ b/lib/VNLib.Data.Caching/src/ClientExtensions.cs @@ -37,7 +37,7 @@ using VNLib.Data.Caching.Exceptions; using static VNLib.Data.Caching.Constants; namespace VNLib.Data.Caching -{ +{ /// /// Provides caching extension methods for @@ -212,11 +212,11 @@ namespace VNLib.Data.Caching return ExecAsync(client, request, objectId, cancellationToken); } - catch + catch(Exception e) { //Return the request(clears data and reset) client.ReturnRequest(request); - throw; + return Task.FromException(e); } static async Task ExecAsync(FBMClient client, FBMRequest request, string objectId, CancellationToken cancellationToken) @@ -239,6 +239,10 @@ namespace VNLib.Data.Caching { throw new ObjectNotFoundException($"object {objectId} not found on remote server"); } + else if(status.ValueEquals(ResponseCodes.InvalidChecksum, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidChecksumException($"The server rejected the message {objectId} due to an invalid checksum"); + } //Invalid status throw new InvalidStatusException("Invalid status code recived for object upsert request", status.ToString()); @@ -256,6 +260,7 @@ namespace VNLib.Data.Caching /// Gets an object from the server if it exists /// /// + /// /// /// The id of the object to get /// A token to cancel the operation @@ -368,18 +373,35 @@ namespace VNLib.Data.Caching response.ThrowIfNotSet(); //Get the status code - FBMMessageHeader status = response.Headers.FirstOrDefault(static a => a.Header == HeaderCommand.Status); + FBMMessageHeader status = response.Headers.FirstOrDefault(static a => a.Header == HeaderCommand.Status); //Check ok status code, then its safe to deserialize - if (status.Value.Equals(ResponseCodes.Okay, StringComparison.Ordinal)) + if (status.ValueEquals(ResponseCodes.Okay, StringComparison.Ordinal)) { + //Add message integrity check + FBMMessageHeader checksumType = response.Headers.FirstOrDefault(static a => a.Header == ChecksumType); + FBMMessageHeader checksum = response.Headers.FirstOrDefault(static a => a.Header == ChecksumValue); + + if(checksumType.ValueEquals(ChecksumTypes.Fnv1a, StringComparison.OrdinalIgnoreCase)) + { + //Verify the checksum + if (!FbmMessageChecksum.VerifyFnv1aChecksum(checksum.Value, response.ResponseBody)) + { + throw new InvalidChecksumException( + $"The response data integrety check failed. The message data was corrupted for id: {checksum.GetValueString()}" + ); + } + + //Valid checksum, continue + } + //Write the object data setter(state, response.ResponseBody); return true; } //Object may not exist on the server yet - if (status.Value.Equals(ResponseCodes.NotFound, StringComparison.Ordinal)) + if (status.ValueEquals(ResponseCodes.NotFound, StringComparison.Ordinal)) { return false; } @@ -540,5 +562,34 @@ namespace VNLib.Data.Caching return new (worker, retryDelay, serverUri); } + /// + /// Determines if the the client sent a message checksum, and if so, verifies the checksum + /// if the checksum type is supported. + /// + /// + /// + /// -1 if the checksum type or value is not set, + /// -2 if the checksum type is not supported, + /// 0 if the checksum is invalid, + /// 1 if the checksum is valid + /// + public static int IsClientChecksumValid(this FBMRequestMessage message) + { + string? type = message.Headers.FirstOrDefault(static h => h.Header == ChecksumType).GetValueString(); + ReadOnlySpan value = message.Headers.FirstOrDefault(static h => h.Header == ChecksumValue).Value; + + if (type == null || value.IsEmpty) + { + return -1; + } + + if(type.Equals(ChecksumTypes.Fnv1a, StringComparison.OrdinalIgnoreCase)) + { + //Verify the checksum + return FbmMessageChecksum.VerifyFnv1aChecksum(value, message.BodyData) ? 1 : 0; + } + + return -2; + } } } diff --git a/lib/VNLib.Data.Caching/src/Constants.cs b/lib/VNLib.Data.Caching/src/Constants.cs index 3fb87e0..6c2051a 100644 --- a/lib/VNLib.Data.Caching/src/Constants.cs +++ b/lib/VNLib.Data.Caching/src/Constants.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2022 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Data.Caching @@ -22,12 +22,11 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -using System; - using VNLib.Net.Messaging.FBM; namespace VNLib.Data.Caching { + public static class Constants { /// @@ -48,9 +47,25 @@ namespace VNLib.Data.Caching public const string Okay = "ok"; public const string Error = "err"; public const string NotFound = "nf"; + public const string InvalidChecksum = "cm"; } public const HeaderCommand ObjectId = (HeaderCommand)0xAA; public const HeaderCommand NewObjectId = (HeaderCommand)0xAB; + public const HeaderCommand ChecksumType = (HeaderCommand)0xA1; + public const HeaderCommand ChecksumValue = (HeaderCommand)0xA2; + public const HeaderCommand ChecksumWarning = (HeaderCommand)0xA3; + + /// + /// Contains constants for checksum type names for FBM headers + /// + public static class ChecksumTypes + { + public const string Fnv1a = "fnv1a"; + public const string Crc32 = "crc32"; + public const string Md5 = "md5"; + public const string Sha1 = "sha1"; + public const string Sha256 = "sha256"; + } } } diff --git a/lib/VNLib.Data.Caching/src/Exceptions/InvalidChecksumException.cs b/lib/VNLib.Data.Caching/src/Exceptions/InvalidChecksumException.cs new file mode 100644 index 0000000..4e60b4b --- /dev/null +++ b/lib/VNLib.Data.Caching/src/Exceptions/InvalidChecksumException.cs @@ -0,0 +1,45 @@ +/* +* Copyright (c) 2022 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Data.Caching +* File: InvalidStatusException.cs +* +* InvalidStatusException.cs is part of VNLib.Data.Caching which is part of the larger +* VNLib collection of libraries and utilities. +* +* VNLib.Data.Caching is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Data.Caching is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using System; + +using VNLib.Net.Messaging.FBM; + +namespace VNLib.Data.Caching.Exceptions +{ + /// + /// Raised when the checksum of a cache result does not match the expected checksum + /// + public class InvalidChecksumException : InvalidResponseException + { + public InvalidChecksumException(string message) : base(message) + { } + + public InvalidChecksumException(string message, Exception innerException) : base(message, innerException) + { } + + public InvalidChecksumException() + { } + } +} diff --git a/lib/VNLib.Data.Caching/src/Exceptions/InvalidStatusException.cs b/lib/VNLib.Data.Caching/src/Exceptions/InvalidStatusException.cs index 2296774..ad880b7 100644 --- a/lib/VNLib.Data.Caching/src/Exceptions/InvalidStatusException.cs +++ b/lib/VNLib.Data.Caching/src/Exceptions/InvalidStatusException.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2022 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Data.Caching @@ -42,7 +42,7 @@ namespace VNLib.Data.Caching.Exceptions /// public InvalidStatusException(string message, string statusCode):this(message) { - this.StatusCode = statusCode; + StatusCode = statusCode; } /// diff --git a/lib/VNLib.Data.Caching/src/FbmMessageChecksum.cs b/lib/VNLib.Data.Caching/src/FbmMessageChecksum.cs new file mode 100644 index 0000000..5a9af31 --- /dev/null +++ b/lib/VNLib.Data.Caching/src/FbmMessageChecksum.cs @@ -0,0 +1,90 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Data.Caching +* File: FbmMessageChecksum.cs +* +* FbmMessageChecksum.cs is part of VNLib.Data.Caching which is part of the larger +* VNLib collection of libraries and utilities. +* +* VNLib.Data.Caching is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Data.Caching is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using System; +using System.Buffers.Binary; +using System.Diagnostics; + +using VNLib.Utils; +using VNLib.Hashing.Checksums; +using VNLib.Net.Messaging.FBM; + +using static VNLib.Data.Caching.Constants; + +namespace VNLib.Data.Caching +{ + /// + /// Utility class for verifying and writing checksums for FBM messages + /// + public static class FbmMessageChecksum + { + /// + /// Verifies the checksum of the supplied data using the FNV1a algorithm + /// + /// The checksum base32 encoded string of the checksum data + /// The data to compute the checksum on + /// True if the checksum of the data matches the supplied one + public static bool VerifyFnv1aChecksum(ReadOnlySpan checksum, ReadOnlySpan data) + { + //Convert the checksum to bytes + Span asBytes = stackalloc byte[sizeof(ulong)]; + ERRNO byteSize = VnEncoding.TryFromBase32Chars(checksum, asBytes); + + Debug.Assert(byteSize == sizeof(ulong), "Failed to convert checksum to bytes"); + + //Compute the checksum of the supplied data + ulong computed = FNV1a.Compute64(data); + + //Compare the checksums + return BinaryPrimitives.ReadUInt64BigEndian(asBytes) == computed; + } + + /// + /// Writes the FNV1a checksum of the supplied data to the message header buffer + /// + /// The FBM message to write the checksum headers to + /// The message data to compute the checksum of + /// + public static void WriteFnv1aChecksum(IFBMMessage message, ReadOnlySpan data) + { + ArgumentNullException.ThrowIfNull(message); + + //Compute the checksum of the data + ulong checksum = FNV1a.Compute64(data); + + Span asBytes = stackalloc byte[sizeof(ulong)]; + Span asChars = stackalloc char[16]; + + //get big endian bytes + BinaryPrimitives.WriteUInt64BigEndian(asBytes, checksum); + ERRNO charSize = VnEncoding.TryToBase32Chars(asBytes, asChars); + + Debug.Assert(charSize > 0, "Failed to convert checksum to base32"); + + //Write the checksum and type to the response + message.WriteHeader(ChecksumType, ChecksumTypes.Fnv1a); + message.WriteHeader(ChecksumValue, asChars[..(int)charSize]); + } + } +} diff --git a/lib/VNLib.Data.Caching/src/VNLib.Data.Caching.csproj b/lib/VNLib.Data.Caching/src/VNLib.Data.Caching.csproj index d906985..3208d12 100644 --- a/lib/VNLib.Data.Caching/src/VNLib.Data.Caching.csproj +++ b/lib/VNLib.Data.Caching/src/VNLib.Data.Caching.csproj @@ -47,7 +47,7 @@ - + -- cgit