From 55859158fbd0bf54473a0baeb486045a025c7c5d Mon Sep 17 00:00:00 2001 From: vnugent Date: Sun, 24 Mar 2024 21:01:06 -0400 Subject: Squashed commit of the following: commit 6c1667be23597513537f8190e2f55d65eb9b7c7a Author: vnugent Date: Fri Mar 22 12:01:53 2024 -0400 refactor: Overhauled native library loading and lazy init commit ebf688f2f974295beabf7b5def7e6f6f150551d0 Author: vnugent Date: Wed Mar 20 22:16:17 2024 -0400 refactor: Update compression header files and macros + Ci build commit 9c7b564911080ccd5cbbb9851a0757b05e1e9047 Author: vnugent Date: Tue Mar 19 21:54:49 2024 -0400 refactor: JWK overhaul & add length getter to FileUpload commit 6d8c3444e09561e5957491b3cc1ae858e0abdd14 Author: vnugent Date: Mon Mar 18 16:13:20 2024 -0400 feat: Add FNV1a software checksum and basic correction tests commit 00d182088cecefc08ca80b1faee9bed3f215f40b Author: vnugent Date: Fri Mar 15 01:05:27 2024 -0400 chore: #6 Use utils filewatcher instead of built-in commit d513c10d9895c6693519ef1d459c6a5a76929436 Author: vnugent Date: Sun Mar 10 21:58:14 2024 -0400 source tree project location updated --- Module.Taskfile.yaml | 21 +- Taskfile.yaml | 8 +- lib/Hashing.Portable/src/Argon2/VnArgon2.cs | 14 +- lib/Hashing.Portable/src/Checksums/FNV1a.cs | 113 ++++++++ .../src/IdentityUtility/JsonWebKey.cs | 38 ++- .../src/IdentityUtility/ReadOnlyJsonWebKey.cs | 120 +++++---- .../src/Native/MonoCypher/MonoCypherLibrary.cs | 8 +- lib/Hashing.Portable/tests/Fnv1aTests.cs | 47 ++++ lib/Net.Compression/third-party/readme.md | 3 - lib/Net.Compression/vnlib_compress/CMakeLists.txt | 54 ++-- lib/Net.Compression/vnlib_compress/Taskfile.yaml | 47 +++- lib/Net.Compression/vnlib_compress/compression.c | 19 +- lib/Net.Compression/vnlib_compress/compression.h | 53 +++- .../vnlib_compress/feature_brotli.c | 3 +- lib/Net.Compression/vnlib_compress/feature_zlib.c | 6 +- lib/Net.Compression/vnlib_compress/util.h | 56 ++-- .../src/Core/RequestParse/Http11ParseExtensions.cs | 6 +- lib/Net.Http/src/FileUpload.cs | 7 +- lib/Plugins.Runtime/src/AssemblyWatcher.cs | 91 ++----- lib/Plugins.Runtime/src/PluginStackBuilder.cs | 6 +- lib/Plugins.Runtime/src/RuntimePluginLoader.cs | 17 +- lib/Utils.Cryptography/monocypher/CMakeLists.txt | 1 + lib/Utils.Cryptography/monocypher/argon2.c | 6 +- lib/Utils.Cryptography/monocypher/blake2b.h | 3 +- lib/Utils.Cryptography/monocypher/util.h | 71 +++-- .../monocypher/vnlib_monocypher.h | 4 +- lib/Utils/src/ArgumentList.cs | 9 +- lib/Utils/src/Memory/MemoryUtil.cs | 37 ++- lib/Utils/src/Native/NatveLibraryResolver.cs | 299 +++++++++++++++++++++ lib/Utils/src/Native/SafeLibraryHandle.cs | 260 ++++++++---------- lib/Utils/src/Resources/LazyInitializer.cs | 112 ++++++++ 31 files changed, 1069 insertions(+), 470 deletions(-) create mode 100644 lib/Hashing.Portable/src/Checksums/FNV1a.cs create mode 100644 lib/Hashing.Portable/tests/Fnv1aTests.cs delete mode 100644 lib/Net.Compression/third-party/readme.md create mode 100644 lib/Utils/src/Native/NatveLibraryResolver.cs create mode 100644 lib/Utils/src/Resources/LazyInitializer.cs diff --git a/Module.Taskfile.yaml b/Module.Taskfile.yaml index f1b9864..c604ff8 100644 --- a/Module.Taskfile.yaml +++ b/Module.Taskfile.yaml @@ -20,8 +20,8 @@ tasks: #called by build pipeline to sync repo update: cmds: - - git remote update - - git reset --hard + - git reset --hard #clean up any local changes + - git remote update - git pull origin {{.BRANCH_NAME}} --verify-signatures #re-write semver after hard reset - dotnet-gitversion.exe /updateprojectfiles @@ -35,24 +35,21 @@ tasks: - task: build_debug - task: build_release - postbuild_success: + publish: cmds: + #git archive in the module directory + - git archive --format {{.ARCHIVE_FILE_FORMAT}} --output {{.ARCHIVE_FILE_NAME}} HEAD #push packages to the sleet feed (feed path is vnbuild global) - sleet push "{{.PACK_OUT}}/debug/" --source debug --config "{{.SLEET_CONFIG_PATH}}" --force - sleet push "{{.PACK_OUT}}/release/" --source release --config "{{.SLEET_CONFIG_PATH}}" --force - #git archive in the module directory - - git archive --format {{.ARCHIVE_FILE_FORMAT}} --output {{.ARCHIVE_FILE_NAME}} HEAD - - postbuild_failed: - cmds: - - echo "postbuild failed {{.MODULE_NAME}}" - #called by build pipeline to clean module clean: cmds: - #clean solution - - dotnet clean /p:BuildInParallel=true /p:MultiProcessorCompilation=true + #clean solution + - dotnet clean /p:BuildInParallel=true /p:MultiProcessorCompilation=true + - cmd: powershell -Command "rm {{ .ARCHIVE_FILE_NAME }} --Force" + ignore_error: true #Internal tasks diff --git a/Taskfile.yaml b/Taskfile.yaml index 08f1c88..bd45655 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -12,7 +12,6 @@ version: '3' vars: TARGET: '{{.USER_WORKING_DIR}}/bin' RELEASE_DIR: "./bin/release/{{.TARGET_FRAMEWORK}}/publish" - SOURCE_OUT: "{{.USER_WORKING_DIR}}/bin/source" tasks: @@ -37,8 +36,7 @@ tasks: postbuild_failed: dir: '{{.USER_WORKING_DIR}}' - cmds: - - echo "postbuild failed {{.PROJECT_NAME}}" + cmds: [] postbuild: @@ -67,5 +65,5 @@ tasks: dir: '{{.USER_WORKING_DIR}}' ignore_error: true cmds: - - cmd: powershell Remove-Item -Recurse './bin' - - cmd: powershell Remove-Item -Recurse './obj' + - for: [ bin/, obj/ ] + cmd: powershell -Command "rm -Recurse -Force '{{.ITEM}}'" diff --git a/lib/Hashing.Portable/src/Argon2/VnArgon2.cs b/lib/Hashing.Portable/src/Argon2/VnArgon2.cs index 9d98050..b88c232 100644 --- a/lib/Hashing.Portable/src/Argon2/VnArgon2.cs +++ b/lib/Hashing.Portable/src/Argon2/VnArgon2.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Hashing.Portable @@ -24,7 +24,6 @@ using System; using System.Text; -using System.Threading; using System.Diagnostics; using System.Buffers.Text; using System.Security.Cryptography; @@ -33,6 +32,7 @@ using System.Runtime.InteropServices; using VNLib.Utils.Memory; using VNLib.Utils.Native; using VNLib.Utils.Extensions; +using VNLib.Utils.Resources; using VNLib.Hashing.Native.MonoCypher; namespace VNLib.Hashing @@ -52,12 +52,12 @@ namespace VNLib.Hashing public const string ARGON2_LIB_ENVIRONMENT_VAR_NAME = "VNLIB_ARGON2_DLL_PATH"; private static readonly Encoding LocEncoding = Encoding.Unicode; - private static readonly Lazy _heap = new (static () => MemoryUtil.InitializeNewHeapForProcess(true), LazyThreadSafetyMode.PublicationOnly); - private static readonly Lazy _nativeLibrary = new(LoadSharedLibInternal, LazyThreadSafetyMode.PublicationOnly); + private static readonly LazyInitializer _heap = new (static () => MemoryUtil.InitializeNewHeapForProcess(true)); + private static readonly LazyInitializer _nativeLibrary = new(LoadSharedLibInternal); //Private heap initialized to 10k size, and all allocated buffers will be zeroed when allocated - private static IUnmangedHeap PwHeap => _heap.Value; + private static IUnmangedHeap PwHeap => _heap.Instance; private static IArgon2Library LoadSharedLibInternal() { @@ -70,7 +70,7 @@ namespace VNLib.Hashing Trace.WriteLine("Using the native MonoCypher library for Argon2 password hashing", "VnArgon2"); //Load shared monocyphter argon2 library - return MonoCypherLibrary.Shared.Argon2CreateLibrary(_heap.Value); + return MonoCypherLibrary.Shared.Argon2CreateLibrary(_heap.Instance); } else { @@ -90,7 +90,7 @@ namespace VNLib.Hashing /// /// The shared library instance /// - public static IArgon2Library GetOrLoadSharedLib() => _nativeLibrary.Value; + public static IArgon2Library GetOrLoadSharedLib() => _nativeLibrary.Instance; /// /// Loads a native Argon2 shared library from the specified path diff --git a/lib/Hashing.Portable/src/Checksums/FNV1a.cs b/lib/Hashing.Portable/src/Checksums/FNV1a.cs new file mode 100644 index 0000000..f150638 --- /dev/null +++ b/lib/Hashing.Portable/src/Checksums/FNV1a.cs @@ -0,0 +1,113 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Hashing.Portable +* File: FNV1a.cs +* +* FNV1a.cs is part of VNLib.Hashing.Portable which is part of the larger +* VNLib collection of libraries and utilities. +* +* VNLib.Hashing.Portable is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* VNLib.Hashing.Portable is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with VNLib.Hashing.Portable. If not, see http://www.gnu.org/licenses/. +*/ + +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace VNLib.Hashing.Checksums +{ + /// + /// A managed software implementation of the FNV-1a 64-bit non cryptographic hash algorithm + /// + public static class FNV1a + { + /* + * Constants taken from the spec + * https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function + */ + const ulong FNV_PRIME = 0x100000001b3UL; + const ulong FNV_OFFSET_BASIS = 0xcbf29ce484222325UL; + + /// + /// Computes the next 64-bit FNV-1a hash value using the current hash + /// value and the next byte of data. + /// + /// The inital hash to begin the computation with + /// + /// + /// The next value of the checksum representing current and previously computed segments + /// + public static ulong Update64(ulong initalizer, ref byte data, nuint length) + { + if (Unsafe.IsNullRef(ref data)) + { + throw new ArgumentNullException(nameof(data)); + } + + ulong digest = initalizer; + + for (nuint i = 0; i < length; i++) + { + digest ^= Unsafe.AddByteOffset(ref data, i); + digest *= FNV_PRIME; + } + + return digest; + } + + /// + /// Computes the next 64-bit FNV-1a hash value using the current hash + /// value and the next byte of data. + /// + /// The initial hash to begin the computation with + /// A span structure pointing to the memory block to compute the digest of + /// The next value of the checksum representing current and previously computed segments + /// + public static ulong Update64(ulong initalizer, ReadOnlySpan data) + { + ref byte r0 = ref MemoryMarshal.GetReference(data); + return Update64(initalizer, ref r0, (nuint)data.Length); + } + + /// + /// Begins computing the FNV-1a 64-bit hash of the input data and returns the + /// initial hash value which may be updated if more data is available + /// + /// A managed pointer to the first byte of the sequence to compute + /// A platform specific integer representing the length of the input data + /// The 64bit unsigned integer representing the message sum or digest + /// + /// WARNING: This function produces a non-cryptographic hash and should not be used for + /// security or cryptographic purposes. It is intended for fast data integrity checks + /// + public static ulong Compute64(ref byte data, nuint length) => Update64(FNV_OFFSET_BASIS, ref data, length); + + /// + /// Computes the next 64-bit FNV-1a hash value using the current hash + /// value and the next byte of data. + /// + /// A span structure pointing to the memory block to compute the digest of + /// The 64bit unsigned integer representng the message sum or digest + /// + /// WARNING: This function produces a non-cryptographic hash and should not be used for + /// security or cryptographic purposes. It is intended for fast data integrity checks + /// + public static ulong Compute64(ReadOnlySpan data) + { + ref byte r0 = ref MemoryMarshal.GetReference(data); + return Compute64(ref r0, (nuint)data.Length); + } + } +} diff --git a/lib/Hashing.Portable/src/IdentityUtility/JsonWebKey.cs b/lib/Hashing.Portable/src/IdentityUtility/JsonWebKey.cs index 65f0837..a237db0 100644 --- a/lib/Hashing.Portable/src/IdentityUtility/JsonWebKey.cs +++ b/lib/Hashing.Portable/src/IdentityUtility/JsonWebKey.cs @@ -292,7 +292,7 @@ namespace VNLib.Hashing.IdentityUtility } /// - /// Gets the RSA private key algorithm from the supplied Json Web Key + /// Gets the RSA private key algorithm from the supplied Json Web Key /// /// /// The algorithm if found, or null if the element does not contain private key @@ -303,7 +303,23 @@ namespace VNLib.Hashing.IdentityUtility return rSAParameters.HasValue ? RSA.Create(rSAParameters.Value) : null; } - private static RSAParameters? GetRsaParameters(in TKey jwk, bool includePrivateKey) where TKey : IJsonWebKey + /// + /// Gets the RSA key parameters from the current Json Web Key + /// > + /// + /// A value that indicates that a private key should be parsed and included in the parameters + /// A nullable structure that contains the parsed keys, or null if required properties were empty + public static RSAParameters? GetRsaParameters(this ReadOnlyJsonWebKey jwk, bool includePrivateKey) + => GetRsaParameters(in jwk, includePrivateKey); + + /// + /// Gets the RSA key parameters from the current Json Web Key + /// + /// + /// + /// A value that indicates that a private key should be parsed and included in the parameters + /// A nullable structure that contains the parsed keys, or null if required properties were empty + public static RSAParameters? GetRsaParameters(in TKey jwk, bool includePrivateKey) where TKey : IJsonWebKey { //Get the RSA public key credentials ReadOnlySpan e = jwk.GetKeyProperty("e"); @@ -372,9 +388,23 @@ namespace VNLib.Hashing.IdentityUtility //Return new alg return ecParams.HasValue ? ECDsa.Create(ecParams.Value) : null; } - - private static ECParameters? GetECParameters(in TKey jwk, bool includePrivate) where TKey : IJsonWebKey + /// + /// Gets the EC key parameters from the current Json Web Key + /// + /// + /// A value that inidcates if private key parameters should be parsed and included + /// The parsed key parameter structure, or null if the key parameters were empty or could not be parsed + public static ECParameters? GetECParameters(this ReadOnlyJsonWebKey jwk, bool includePrivate) => GetECParameters(in jwk, includePrivate); + + /// + /// Gets the EC key parameters from the current Json Web Key + /// + /// + /// + /// A value that inidcates if private key parameters should be parsed and included + /// The parsed key parameter structure, or null if the key parameters were empty or could not be parsed + public static ECParameters? GetECParameters(in TKey jwk, bool includePrivate) where TKey : IJsonWebKey { //Get the RSA public key credentials ReadOnlySpan x = jwk.GetKeyProperty("x"); diff --git a/lib/Hashing.Portable/src/IdentityUtility/ReadOnlyJsonWebKey.cs b/lib/Hashing.Portable/src/IdentityUtility/ReadOnlyJsonWebKey.cs index 009d6bf..c97a023 100644 --- a/lib/Hashing.Portable/src/IdentityUtility/ReadOnlyJsonWebKey.cs +++ b/lib/Hashing.Portable/src/IdentityUtility/ReadOnlyJsonWebKey.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Hashing.Portable @@ -23,11 +23,12 @@ */ using System; +using System.Linq; using System.Text.Json; using System.Collections.Generic; +using System.Collections.Frozen; -using VNLib.Utils; -using VNLib.Utils.Extensions; +using VNLib.Utils.Memory; namespace VNLib.Hashing.IdentityUtility { @@ -35,24 +36,19 @@ namespace VNLib.Hashing.IdentityUtility /// A readonly Json Web Key (JWK) data structure that may be used for signing /// or verifying messages. /// - public sealed class ReadOnlyJsonWebKey : VnDisposeable, IJsonWebKey + public sealed class ReadOnlyJsonWebKey : IJsonWebKey { - private readonly JsonElement _jwk; - private readonly JsonDocument? _doc; + private readonly FrozenDictionary _properties; /// - /// Creates a new instance of from a . - /// This will call on the element and store an internal copy + /// Creates a new instance of from a dictionary of + /// JWK string properties /// - /// The to create the from - public ReadOnlyJsonWebKey(ref readonly JsonElement keyElement) + /// The frozen dictionary instance of parsed JWK properties + public ReadOnlyJsonWebKey(FrozenDictionary properties) { - _jwk = keyElement.Clone(); - //Set initial values - KeyId = _jwk.GetPropString("kid"); - KeyType = _jwk.GetPropString("kty"); - Algorithm = _jwk.GetPropString("alg"); - Use = _jwk.GetPropString("use"); + ArgumentNullException.ThrowIfNull(properties); + _properties = properties; //Create a JWT header from the values JwtHeader = new Dictionary() @@ -70,6 +66,21 @@ namespace VNLib.Hashing.IdentityUtility }; } + /// + /// Creates a new instance of from a . + /// This will call on the element and store an internal copy + /// + /// The to create the from + public ReadOnlyJsonWebKey(ref readonly JsonElement keyElement) + :this( + //Get only top-level string properties and store them in a dictionary + keyElement.EnumerateObject() + .Where(static k => k.Value.ValueKind == JsonValueKind.String) + .ToDictionary(static k => k.Name, v => v.Value.GetString(), StringComparer.OrdinalIgnoreCase) + .ToFrozenDictionary() + ) + { } + /// /// Creates a new instance of from a raw utf8 encoded json /// binary sequence @@ -77,52 +88,60 @@ namespace VNLib.Hashing.IdentityUtility /// The utf8 encoded json binary sequence /// /// - public ReadOnlyJsonWebKey(ReadOnlySpan rawValue) + public static ReadOnlyJsonWebKey FromUtf8Bytes(ReadOnlySpan rawValue) { - //Pare the raw value Utf8JsonReader reader = new (rawValue); - _doc = JsonDocument.ParseValue(ref reader); - //store element - _jwk = _doc.RootElement; - - //Set initial values - KeyId = _jwk.GetPropString("kid"); - KeyType = _jwk.GetPropString("kty"); - Algorithm = _jwk.GetPropString("alg"); - Use = _jwk.GetPropString("use"); + using JsonDocument doc = JsonDocument.ParseValue(ref reader); + JsonElement root = doc.RootElement; + return new ReadOnlyJsonWebKey(ref root); + } - //Create a JWT header from the values - JwtHeader = new Dictionary() - { - { "alg" , Algorithm }, - { "typ" , "JWT" }, - }; + /// + /// Creates a new instance of from a raw utf8 encoded json + /// memory segment + /// + /// The utf8 encoded json binary sequence + /// The readonly JWK object + /// + /// + public static ReadOnlyJsonWebKey FromUtf8Bytes(ReadOnlyMemory rawValue) + { + using JsonDocument doc = JsonDocument.Parse(rawValue); + JsonElement root = doc.RootElement; + return new ReadOnlyJsonWebKey(ref root); + } - //Configure key usage - KeyUse = (Use?.ToLower(null)) switch - { - "sig" => JwkKeyUsage.Signature, - "enc" => JwkKeyUsage.Encryption, - _ => JwkKeyUsage.None, - }; + /// + /// Creates a new instance of from a json string + /// + /// The json encoded string to recover the JWK from + /// + public static ReadOnlyJsonWebKey FromJsonString(string jsonString) + { + using JsonDocument doc = JsonDocument.Parse(jsonString); + JsonElement root = doc.RootElement; + return new ReadOnlyJsonWebKey(ref root); } /// /// The key identifier /// - public string? KeyId { get; } + public string? KeyId => _properties.GetValueOrDefault("kid"); + /// /// The key type /// - public string? KeyType { get; } + public string? KeyType => _properties.GetValueOrDefault("kty"); + /// /// The key algorithm /// - public string? Algorithm { get; } + public string? Algorithm => _properties.GetValueOrDefault("alg"); + /// /// The key "use" value /// - public string? Use { get; } + public string? Use => _properties.GetValueOrDefault("use"); /// /// Returns the JWT header that matches this key @@ -133,12 +152,17 @@ namespace VNLib.Hashing.IdentityUtility public JwkKeyUsage KeyUse { get; } /// - public string? GetKeyProperty(string propertyName) => _jwk.GetPropString(propertyName); + public string? GetKeyProperty(string propertyName) => _properties.GetValueOrDefault(propertyName); - /// - protected override void Free() + /// + /// Attemts to erase all property values from memory by securely writing over them with zeros + /// + public void EraseValues() { - _doc?.Dispose(); + foreach(string? value in _properties.Values) + { + PrivateStringManager.EraseString(value); + } } } diff --git a/lib/Hashing.Portable/src/Native/MonoCypher/MonoCypherLibrary.cs b/lib/Hashing.Portable/src/Native/MonoCypher/MonoCypherLibrary.cs index 11e524b..4c6f405 100644 --- a/lib/Hashing.Portable/src/Native/MonoCypher/MonoCypherLibrary.cs +++ b/lib/Hashing.Portable/src/Native/MonoCypher/MonoCypherLibrary.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Hashing.Portable @@ -25,10 +25,10 @@ using System; using System.Diagnostics; using System.Runtime.InteropServices; -using System.Threading; using VNLib.Utils; using VNLib.Utils.Native; +using VNLib.Utils.Resources; using VNLib.Utils.Extensions; namespace VNLib.Hashing.Native.MonoCypher @@ -50,7 +50,7 @@ namespace VNLib.Hashing.Native.MonoCypher /// true if the user enabled the default library, false otherwise public static bool CanLoadDefaultLibrary() => string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(MONOCYPHER_LIB_ENVIRONMENT_VAR_NAME)) == false; - private static readonly Lazy _defaultLib = new (LoadDefaultLibraryInternal, LazyThreadSafetyMode.PublicationOnly); + private static readonly LazyInitializer _defaultLib = new (LoadDefaultLibraryInternal); /// /// Gets the default MonoCypher library for the current process @@ -59,7 +59,7 @@ namespace VNLib.Hashing.Native.MonoCypher /// this property to ensure that the default library can be loaded /// /// - public static MonoCypherLibrary Shared => _defaultLib.Value; + public static MonoCypherLibrary Shared => _defaultLib.Instance; /// /// Loads a new instance of the MonoCypher library with environment defaults diff --git a/lib/Hashing.Portable/tests/Fnv1aTests.cs b/lib/Hashing.Portable/tests/Fnv1aTests.cs new file mode 100644 index 0000000..2c6431f --- /dev/null +++ b/lib/Hashing.Portable/tests/Fnv1aTests.cs @@ -0,0 +1,47 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +using System.Text; + +using VNLib.Hashing.Checksums; + +namespace VNLib.Hashing.Tests +{ + [TestClass()] + public class Fnv1aTests + { + const string KnownDataInputUtf81 = "Hello world, this is a test of the FNV1a algorithm"; + const string KnownData64ChecksumHex1 = "033b9d1635f1c2ad"; + + const string KnownDataInputUtf82 = "Hello world, this is another, slightly different test of the FNV1a algorithm!"; + const string KnownData64ChecksumHex2 = "a802c807e941c5d3"; + + [TestMethod()] + public void Fnv1a64Known1() + { + TestKnownData(KnownDataInputUtf81, KnownData64ChecksumHex1); + TestKnownData(KnownDataInputUtf82, KnownData64ChecksumHex2); + } + + static void TestKnownData(string input, string knownChecksumHex) + { + byte[] knownInput = Encoding.UTF8.GetBytes(input); + ulong knownChecksum = Convert.ToUInt64(knownChecksumHex, 16); + + ulong checksum = FNV1a.Compute64(knownInput); + + Assert.AreEqual(knownChecksum, checksum); + + //Split input into 2 parts + byte[] part1 = knownInput[..(knownInput.Length / 2)]; + byte[] part2 = knownInput[(knownInput.Length / 2)..]; + + //Compute checksum of part1 + ulong checksum1 = FNV1a.Compute64(part1); + ulong outputChecksum = FNV1a.Update64(checksum1, part2); + + Assert.AreNotEqual(checksum1, outputChecksum); + Assert.AreEqual(knownChecksum, outputChecksum); + } + + } +} \ No newline at end of file diff --git a/lib/Net.Compression/third-party/readme.md b/lib/Net.Compression/third-party/readme.md deleted file mode 100644 index f10eb0e..0000000 --- a/lib/Net.Compression/third-party/readme.md +++ /dev/null @@ -1,3 +0,0 @@ -# Third party - -This directory should contain the required third party libraries. Their should be a directory called `zlib` which includes the zlib source files, and a directory called `brotli` which includes the brotli source files. See the [readme](../readme.md) for more information diff --git a/lib/Net.Compression/vnlib_compress/CMakeLists.txt b/lib/Net.Compression/vnlib_compress/CMakeLists.txt index 977b9a1..f963c41 100644 --- a/lib/Net.Compression/vnlib_compress/CMakeLists.txt +++ b/lib/Net.Compression/vnlib_compress/CMakeLists.txt @@ -13,6 +13,8 @@ option(ENABLE_BROTLI "Enable brotli compression" ON) option(ENABLE_ZLIB "Enable zlib compression" ON) option(ENABLE_RPMALLOC "Enable local source code vnlib_rpmalloc memory allocator" OFF) +set(THIRD_PARTY_DIR ./third-party) + #add feature specific source files to the project if(ENABLE_BROTLI) list(APPEND VNLIB_COMPRESS_SOURCES feature_brotli.c) @@ -157,13 +159,13 @@ if(ENABLE_BROTLI) endif() #add the include directory for brotli so we can include the header files - include_directories(../third-party/brotli/c/include) + include_directories(${THIRD_PARTY_DIR}/brotli/c/include) #get common sources - file(GLOB BROTLI_SOURCES ../third-party/brotli/c/common/*.c) + file(GLOB BROTLI_SOURCES ${THIRD_PARTY_DIR}/brotli/c/common/*.c) #we need to add the brotli encoder source files to the project - file(GLOB BROTLI_ENC_SOURCES ../third-party/brotli/c/enc/*.c) + file(GLOB BROTLI_ENC_SOURCES ${THIRD_PARTY_DIR}/brotli/c/enc/*.c) #add brotli as a static library to link later add_library(lib_brotli STATIC ${BROTLI_SOURCES} ${BROTLI_ENC_SOURCES}) @@ -179,18 +181,18 @@ endif() if(ENABLE_ZLIB) #add the include directory for zlib so we can include the header files - include_directories(../third-party/zlib) + include_directories(${THIRD_PARTY_DIR}/zlib) set(ZLIB_DEFINITIONS) set(Z_C_FLAGS) #we only need to add the zlib deflate source files to the project set(ZLIB_SOURCES - ../third-party/zlib/deflate.c - ../third-party/zlib/adler32.c - ../third-party/zlib/crc32.c - ../third-party/zlib/zutil.c - ../third-party/zlib/trees.c + ${THIRD_PARTY_DIR}/zlib/deflate.c + ${THIRD_PARTY_DIR}/zlib/adler32.c + ${THIRD_PARTY_DIR}/zlib/crc32.c + ${THIRD_PARTY_DIR}/zlib/zutil.c + ${THIRD_PARTY_DIR}/zlib/trees.c ) check_type_size(off64_t OFF64_T) @@ -214,11 +216,21 @@ if(ENABLE_ZLIB) -D_CRT_NONSTDC_NO_DEPRECATE ) - #setup avx compiler support on Windows - check_c_compiler_flag(/arch:AVX HAS_AVX) - if (HAS_AVX) - list(APPEND Z_C_FLAGS /arch:AVX) - list(APPEND ZLIB_DEFINITIONS -DHAS_AVX) + + #NOTE + #During CI a pre-compiled library will be built. + #We cannot depend on users having the the same instructions as the build machine + #So some optimizations are disabled for the pre-compiled library + + if(NOT CI_PRECOMPILE) + + #setup avx compiler support on Windows + check_c_compiler_flag(/arch:AVX HAS_AVX) + if (HAS_AVX) + list(APPEND Z_C_FLAGS /arch:AVX) + list(APPEND ZLIB_DEFINITIONS -DHAS_AVX) + endif() + endif() #All x64 machines have SSE2, so we can use it as @@ -239,9 +251,9 @@ if(ENABLE_ZLIB) if(ARM_CRC) list(APPEND Z_C_FLAGS -march=armv8-a+crc) - if(EXISTS "../third-party/zlib/adler32_simd.c") + if(EXISTS "{THIRD_PARTY_DIR}/zlib/adler32_simd.c") list(APPEND ZLIB_DEFINITIONS -DADLER32_SIMD_NEON) - list(APPEND ZLIB_SOURCES ../third-party/zlib/adler32_simd.c) + list(APPEND ZLIB_SOURCES ${THIRD_PARTY_DIR}/zlib/adler32_simd.c) endif() else() @@ -254,9 +266,9 @@ if(ENABLE_ZLIB) list(APPEND Z_C_FLAGS -mssse3) #add cloudflare intrinsic optimizations, may not be present if using non-cloudflare fork - if(EXISTS "../third-party/zlib/adler32_simd.c") + if(EXISTS "${THIRD_PARTY_DIR}/zlib/adler32_simd.c") list(APPEND ZLIB_DEFINITIONS -DHAS_SSSE3 -DADLER32_SIMD_SSSE3) - list(APPEND ZLIB_SOURCES ../third-party/zlib/adler32_simd.c) + list(APPEND ZLIB_SOURCES ${THIRD_PARTY_DIR}/zlib/adler32_simd.c) endif() endif() @@ -268,9 +280,9 @@ if(ENABLE_ZLIB) list(APPEND Z_C_FLAGS -mpclmul) #add cloudflare intrinsic optimizations for PCMLONGMUL crc32, may not be present if using non-cloudflare fork - if(EXISTS "../third-party/zlib/crc32_simd.c") + if(EXISTS "${THIRD_PARTY_DIR}/zlib/crc32_simd.c") list(APPEND ZLIB_DEFINITIONS -DHAS_PCLMUL) - list(APPEND ZLIB_SOURCES ../third-party/zlib/crc32_simd.c) + list(APPEND ZLIB_SOURCES ${THIRD_PARTY_DIR}/zlib/crc32_simd.c) endif() endif() @@ -344,7 +356,7 @@ if(ENABLE_RPMALLOC) #find the rpmalloc library for unix builds find_library(VNLIB_RPMALLOC_LIB - NAMES _vnrpmalloc + NAMES libvn_rpmalloc PATHS ../../Utils.Memory/vnlib_rpmalloc/build ) diff --git a/lib/Net.Compression/vnlib_compress/Taskfile.yaml b/lib/Net.Compression/vnlib_compress/Taskfile.yaml index 28b3ff0..fd22c22 100644 --- a/lib/Net.Compression/vnlib_compress/Taskfile.yaml +++ b/lib/Net.Compression/vnlib_compress/Taskfile.yaml @@ -9,8 +9,10 @@ version: '3' vars: - THIRD_PARTY_DIR: '../third-party' + THIRD_PARTY_DIR: './third-party' PROJECT_NAME: 'vnlib_compress' + ZLIB_GIT_REPO: 'https://github.com/cloudflare/zlib.git' + BROTLI_GIT_REPO: 'https://github.com/google/brotli.git' tasks: @@ -35,26 +37,59 @@ tasks: #build for platform - cmake --build build/ --config Release - + + + #called by ci pipline to build the winx64 project + build: + cmds: + #make third-party dir before cloning libs + - cmd: powershell -Command "mkdir '{{.THIRD_PARTY_DIR}}' -Force" + platforms: [windows] + ignore_error: true + - cmd: mkdir -p '{{.THIRD_PARTY_DIR}}' + platforms: [linux, darwin] + ignore_error: true + + - task: zlib + - task: brotli + + #invoke cmake for build (notify that we are precompiling for ci pipeline and rpmalloc lib should be local) + - cmake -B./build -DCI_PRECOMPILE=ON -DENABLE_RPMALLOC=ON + + #build for platform + - cmake --build build/ --config Debug + - cmake --build build/ --config Release #when build succeeds, archive the output into a tgz postbuild_success: + vars: + #required files to include in tar + TAR_FILES: "{{.PROJECT_NAME}}.dll {{.PROJECT_NAME}}.pdb {{.PROJECT_NAME}}.lib license.txt" cmds: - cmd: powershell mkdir -Force './bin' #copy source code to target - - powershell -Command "tar --exclude build/* --exclude .vs/* --exclude bin/* -czf bin/src.tgz ." + - cmd: powershell -Command "tar --exclude build/* --exclude .vs/* --exclude bin/* --exclude third-party/* -czf bin/src.tgz ." + + #copy license file to debug and release output + - cmd: powershell -Command "cp ../LICENSE build/Debug/license.txt" + - cmd: powershell -Command "cp ../LICENSE build/Release/license.txt" + + #create static-build archives + - cd build/Debug && tar -czf ../../bin/win-x64-debug.tgz {{.TAR_FILES}} + - cd build/Release && tar -czf ../../bin/win-x64-release.tgz {{.TAR_FILES}} #Remove the output dirs on clean clean: ignore_error: true cmds: - - cmd: powershell Remove-Item -Recurse './bin' + - for: [ bin/, build/, third-party/ ] + cmd: powershell Remove-Item -Recurse '{{.ITEM}}' -Force #update or install the cloudflare fork of zlib library zlib: internal: true status: - - cd {{.THIRD_PARTY_DIR}} && git clone https://github.com/cloudflare/zlib.git + - cd {{.THIRD_PARTY_DIR}} && git clone {{.ZLIB_GIT_REPO}} cmds: - cd {{.THIRD_PARTY_DIR}}/zlib && git pull @@ -62,7 +97,7 @@ tasks: brotli: internal: true status: - - cd {{.THIRD_PARTY_DIR}} && git clone https://github.com/google/brotli.git + - cd {{.THIRD_PARTY_DIR}} && git clone {{.BROTLI_GIT_REPO}} cmds: - cd {{.THIRD_PARTY_DIR}}/brotli && git pull \ No newline at end of file diff --git a/lib/Net.Compression/vnlib_compress/compression.c b/lib/Net.Compression/vnlib_compress/compression.c index a414609..bf3ffbe 100644 --- a/lib/Net.Compression/vnlib_compress/compression.c +++ b/lib/Net.Compression/vnlib_compress/compression.c @@ -29,7 +29,10 @@ * this is a good compromise. */ +#define VNLIB_COMPRESS_EXPORTING 1 + #include "compression.h" +#include "util.h" #ifdef VNLIB_COMPRESSOR_BROTLI_ENABLED #include "feature_brotli.h" @@ -44,7 +47,7 @@ Gets the supported compressors, this is defined at compile time and is a convenience method for the user to know what compressors are supported at runtime. */ -VNLIB_EXPORT CompressorType VNLIB_CC GetSupportedCompressors(void) +VNLIB_COMPRESS_EXPORT CompressorType VNLIB_COMPRESS_CC GetSupportedCompressors(void) { /* * Supported compressors are defined at compile time @@ -69,7 +72,7 @@ VNLIB_EXPORT CompressorType VNLIB_CC GetSupportedCompressors(void) return supported; } -VNLIB_EXPORT CompressorType VNLIB_CC GetCompressorType(_In_ const void* compressor) +VNLIB_COMPRESS_EXPORT CompressorType VNLIB_COMPRESS_CC GetCompressorType(_In_ const void* compressor) { if (!compressor) { @@ -79,7 +82,7 @@ VNLIB_EXPORT CompressorType VNLIB_CC GetCompressorType(_In_ const void* compress return ((CompressorState*)compressor)->type; } -VNLIB_EXPORT CompressionLevel VNLIB_CC GetCompressorLevel(_In_ const void* compressor) +VNLIB_COMPRESS_EXPORT CompressionLevel VNLIB_COMPRESS_CC GetCompressorLevel(_In_ const void* compressor) { if (!compressor) { @@ -89,7 +92,7 @@ VNLIB_EXPORT CompressionLevel VNLIB_CC GetCompressorLevel(_In_ const void* compr return ((CompressorState*)compressor)->level; } -VNLIB_EXPORT int64_t VNLIB_CC GetCompressorBlockSize(_In_ const void* compressor) +VNLIB_COMPRESS_EXPORT int64_t VNLIB_COMPRESS_CC GetCompressorBlockSize(_In_ const void* compressor) { if (!compressor) { @@ -99,7 +102,7 @@ VNLIB_EXPORT int64_t VNLIB_CC GetCompressorBlockSize(_In_ const void* compressor return (int64_t)((CompressorState*)compressor)->blockSize; } -VNLIB_EXPORT void* VNLIB_CC AllocateCompressor(CompressorType type, CompressionLevel level) +VNLIB_COMPRESS_EXPORT void* VNLIB_COMPRESS_CC AllocateCompressor(CompressorType type, CompressionLevel level) { int result; CompressorState* state; @@ -191,7 +194,7 @@ VNLIB_EXPORT void* VNLIB_CC AllocateCompressor(CompressorType type, CompressionL } } -VNLIB_EXPORT int VNLIB_CC FreeCompressor(_In_ void* compressor) +VNLIB_COMPRESS_EXPORT int VNLIB_COMPRESS_CC FreeCompressor(_In_ void* compressor) { CompressorState* comp; int errorCode; @@ -244,7 +247,7 @@ VNLIB_EXPORT int VNLIB_CC FreeCompressor(_In_ void* compressor) return errorCode; } -VNLIB_EXPORT int64_t VNLIB_CC GetCompressedSize(_In_ const void* compressor, uint64_t inputLength, int32_t flush) +VNLIB_COMPRESS_EXPORT int64_t VNLIB_COMPRESS_CC GetCompressedSize(_In_ const void* compressor, uint64_t inputLength, int32_t flush) { CompressorState* comp; int64_t result; @@ -299,7 +302,7 @@ VNLIB_EXPORT int64_t VNLIB_CC GetCompressedSize(_In_ const void* compressor, uin * indicate failure. * @param compressor */ -VNLIB_EXPORT int VNLIB_CC CompressBlock(_In_ const void* compressor, CompressionOperation* operation) +VNLIB_COMPRESS_EXPORT int VNLIB_COMPRESS_CC CompressBlock(_In_ const void* compressor, CompressionOperation* operation) { int result; CompressorState* comp; diff --git a/lib/Net.Compression/vnlib_compress/compression.h b/lib/Net.Compression/vnlib_compress/compression.h index ae3bb8f..0524a01 100644 --- a/lib/Net.Compression/vnlib_compress/compression.h +++ b/lib/Net.Compression/vnlib_compress/compression.h @@ -38,9 +38,44 @@ #ifndef COMPRESSION_H_ #define COMPRESSION_H_ -#include "util.h" #include +#if defined(_MSC_VER) || defined(WIN32) || defined(_WIN32) + #define _IS_WINDOWS +#endif + +//Set api export calling convention (allow used to override) +#ifndef VNLIB_COMPRESS_CC + #ifdef _IS_WINDOWS + //STD for importing to other languages such as .NET + #define VNLIB_COMPRESS_CC __stdcall + #else + #define VNLIB_COMPRESS_CC + #endif +#endif // !VNLIB_CC + +#ifndef VNLIB_COMPRESS_EXPORT //Allow users to disable the export/impoty macro if using source code directly + #ifdef VNLIB_COMPRESS_EXPORTING + #ifdef _IS_WINDOWS + #define VNLIB_COMPRESS_EXPORT __declspec(dllexport) + #else + #define VNLIB_COMPRESS_EXPORT __attribute__((visibility("default"))) + #endif // IS_WINDOWS + #else + #ifdef _IS_WINDOWS + #define VNLIB_COMPRESS_EXPORT __declspec(dllimport) + #else + #define VNLIB_COMPRESS_EXPORT + #endif // IS_WINDOWS + #endif // !VNLIB_EXPORTING +#endif // !VNLIB_EXPORT + +/* +* ERRORS AND CONSTANTS +*/ +#define ERR_INVALID_PTR -1 +#define ERR_OUT_OF_MEMORY -2 + #define ERR_COMP_TYPE_NOT_SUPPORTED -9 #define ERR_COMP_LEVEL_NOT_SUPPORTED -10 #define ERR_INVALID_INPUT_DATA -11 @@ -157,7 +192,7 @@ typedef struct CompressionOperationStruct { /* * Public API functions */ -VNLIB_EXPORT CompressorType VNLIB_CC GetSupportedCompressors(void); +VNLIB_COMPRESS_EXPORT CompressorType VNLIB_COMPRESS_CC GetSupportedCompressors(void); /* * Returns the suggested block size for the underlying compressor. @@ -165,7 +200,7 @@ VNLIB_EXPORT CompressorType VNLIB_CC GetSupportedCompressors(void); * @param compressor A pointer to the desired compressor instance to query. * @return The suggested block size for the underlying compressor in bytes */ -VNLIB_EXPORT int64_t VNLIB_CC GetCompressorBlockSize(_In_ const void* compressor); +VNLIB_COMPRESS_EXPORT int64_t VNLIB_COMPRESS_CC GetCompressorBlockSize(_In_ const void* compressor); /* * Gets the compressor type of the specified compressor instance. @@ -173,7 +208,7 @@ VNLIB_EXPORT int64_t VNLIB_CC GetCompressorBlockSize(_In_ const void* compressor * @param compressor A pointer to the desired compressor instance to query. * @return The type of the specified compressor instance. */ -VNLIB_EXPORT CompressorType VNLIB_CC GetCompressorType(_In_ const void* compressor); +VNLIB_COMPRESS_EXPORT CompressorType VNLIB_COMPRESS_CC GetCompressorType(_In_ const void* compressor); /* * Gets the compression level of the specified compressor instance. @@ -181,7 +216,7 @@ VNLIB_EXPORT CompressorType VNLIB_CC GetCompressorType(_In_ const void* compress * @param compressor A pointer to the desired compressor instance to query. * @return The compression level of the specified compressor instance. */ -VNLIB_EXPORT CompressionLevel VNLIB_CC GetCompressorLevel(_In_ const void* compressor); +VNLIB_COMPRESS_EXPORT CompressionLevel VNLIB_COMPRESS_CC GetCompressorLevel(_In_ const void* compressor); /* * Allocates a new compressor instance on the native heap of the desired compressor type. @@ -191,7 +226,7 @@ VNLIB_EXPORT CompressionLevel VNLIB_CC GetCompressorLevel(_In_ const void* compr * @return A pointer to the newly allocated compressor instance. NULL if the compressor could not be allocated. */ -VNLIB_EXPORT void* VNLIB_CC AllocateCompressor(CompressorType type, CompressionLevel level); +VNLIB_COMPRESS_EXPORT void* VNLIB_COMPRESS_CC AllocateCompressor(CompressorType type, CompressionLevel level); /* * Frees a previously allocated compressor instance. @@ -199,7 +234,7 @@ VNLIB_EXPORT void* VNLIB_CC AllocateCompressor(CompressorType type, CompressionL * @param compressor A pointer to the desired compressor instance to free. * @return The underlying compressor's native return code. */ -VNLIB_EXPORT int VNLIB_CC FreeCompressor(_In_ void* compressor); +VNLIB_COMPRESS_EXPORT int VNLIB_COMPRESS_CC FreeCompressor(_In_ void* compressor); /* * Computes the maximum compressed size of the specified input data. This is not supported @@ -209,7 +244,7 @@ VNLIB_EXPORT int VNLIB_CC FreeCompressor(_In_ void* compressor); * @param inputLength The length of the input data in bytes. * @return The maximum compressed size of the specified input data in bytes. */ -VNLIB_EXPORT int64_t VNLIB_CC GetCompressedSize(_In_ const void* compressor, uint64_t inputLength, int32_t flush); +VNLIB_COMPRESS_EXPORT int64_t VNLIB_COMPRESS_CC GetCompressedSize(_In_ const void* compressor, uint64_t inputLength, int32_t flush); /* @@ -219,6 +254,6 @@ VNLIB_EXPORT int64_t VNLIB_CC GetCompressedSize(_In_ const void* compressor, uin * @param operation A pointer to the compression operation structure * @return The underlying compressor's native return code */ -VNLIB_EXPORT int VNLIB_CC CompressBlock(_In_ const void* compressor, CompressionOperation* operation); +VNLIB_COMPRESS_EXPORT int VNLIB_COMPRESS_CC CompressBlock(_In_ const void* compressor, CompressionOperation* operation); #endif /* !VNLIB_COMPRESS_MAIN_H_ */ \ No newline at end of file diff --git a/lib/Net.Compression/vnlib_compress/feature_brotli.c b/lib/Net.Compression/vnlib_compress/feature_brotli.c index 924f1af..d5ba141 100644 --- a/lib/Net.Compression/vnlib_compress/feature_brotli.c +++ b/lib/Net.Compression/vnlib_compress/feature_brotli.c @@ -19,8 +19,9 @@ * along with vnlib_compress. If not, see http://www.gnu.org/licenses/. */ -#include "feature_brotli.h" #include +#include "feature_brotli.h" +#include "util.h" #define validateCompState(state) \ if (!state) return ERR_INVALID_PTR; \ diff --git a/lib/Net.Compression/vnlib_compress/feature_zlib.c b/lib/Net.Compression/vnlib_compress/feature_zlib.c index 9993b43..a07f106 100644 --- a/lib/Net.Compression/vnlib_compress/feature_zlib.c +++ b/lib/Net.Compression/vnlib_compress/feature_zlib.c @@ -22,10 +22,12 @@ /* * Include the stub header and also the zlib header */ -#include "feature_zlib.h" -#include +#include +#include "feature_zlib.h" +#include "util.h" + #define validateCompState(state) \ if (!state) return ERR_INVALID_PTR; \ if (!state->compressor) return ERR_GZ_INVALID_STATE; \ diff --git a/lib/Net.Compression/vnlib_compress/util.h b/lib/Net.Compression/vnlib_compress/util.h index 8930da7..34659d8 100644 --- a/lib/Net.Compression/vnlib_compress/util.h +++ b/lib/Net.Compression/vnlib_compress/util.h @@ -24,50 +24,25 @@ #ifndef UTIL_H_ #define UTIL_H_ -#include - -/* -* Stub missing types and constants for GCC -*/ - /* * If a custom allocator is enabled, use the native heap api -* header and assume linking is enabled +* header and assume linking is enabled. Heap functions below +* will be enabled when heapapi.h is included. */ #ifdef VNLIB_CUSTOM_MALLOC_ENABLE -#include + #include #endif #if defined(_MSC_VER) || defined(WIN32) || defined(_WIN32) #define IS_WINDOWS #endif -//Set api export calling convention (allow used to override) -#ifndef VNLIB_CC - #ifdef IS_WINDOWS - //STD for importing to other languages such as .NET - #define VNLIB_CC __stdcall - #else - #define VNLIB_CC - #endif -#endif // !VNLIB_CC - -#ifndef VNLIB_EXPORT //Allow users to disable the export/impoty macro if using source code directly - #ifdef VNLIB_EXPORTING - #ifdef IS_WINDOWS - #define VNLIB_EXPORT __declspec(dllexport) - #else - #define VNLIB_EXPORT __attribute__((visibility("default"))) - #endif // IS_WINDOWS - #else - #ifdef IS_WINDOWS - #define VNLIB_EXPORT __declspec(dllimport) - #else - #define VNLIB_EXPORT - #endif // IS_WINDOWS - #endif // !VNLIB_EXPORTING -#endif // !VNLIB_EXPORT - +/* If not Windows, define inline */ +#ifndef IS_WINDOWS + #ifndef inline + #define inline __inline__ + #endif // !inline +#endif // !IS_WINDOWS /* If not Windows, define inline */ #ifndef IS_WINDOWS @@ -102,16 +77,14 @@ #define assert(x) {} #endif -/* -* ERRORS AND CONSTANTS -*/ -#define ERR_INVALID_PTR -1 -#define ERR_OUT_OF_MEMORY -2 #ifdef NATIVE_HEAP_API /* Defined in the NativeHeapApi */ /* * Add overrides for malloc, calloc, and free that use * the nativeheap api to allocate memory + * + * Inline fuctions are used to enforce type safety and + * api consistency. */ static inline void* vnmalloc(size_t num, size_t size) @@ -135,6 +108,11 @@ #else + /* + * Required for built-in memory api + */ + #include + /* * Stub method for malloc. All calls to vnmalloc should be freed with vnfree. */ diff --git a/lib/Net.Http/src/Core/RequestParse/Http11ParseExtensions.cs b/lib/Net.Http/src/Core/RequestParse/Http11ParseExtensions.cs index c373310..86535c3 100644 --- a/lib/Net.Http/src/Core/RequestParse/Http11ParseExtensions.cs +++ b/lib/Net.Http/src/Core/RequestParse/Http11ParseExtensions.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Net.Http @@ -176,7 +176,7 @@ namespace VNLib.Net.Http.Core /// The buffer read data from the transport with /// 0 if the request line was successfully parsed, a status code if the request could not be processed [MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.AggressiveInlining)] - public static HttpStatusCode Http1ParseHeaders(this HttpRequest Request, ref Http1ParseState parseState, ref TransportReader reader, in HttpConfig Config, Span lineBuf) + public static HttpStatusCode Http1ParseHeaders(this HttpRequest Request, ref Http1ParseState parseState, ref TransportReader reader, ref readonly HttpConfig Config, Span lineBuf) { /* * Evil mutable struct, get a local mutable reference to the request's @@ -534,7 +534,7 @@ namespace VNLib.Net.Http.Core /// The to read lines from the transport /// 0 if the request line was successfully parsed, a status code if the request could not be processed [MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.AggressiveInlining)] - public static HttpStatusCode Http1PrepareEntityBody(this HttpRequest Request, ref Http1ParseState parseState, ref TransportReader reader, in HttpConfig Config) + public static HttpStatusCode Http1PrepareEntityBody(this HttpRequest Request, ref Http1ParseState parseState, ref TransportReader reader, ref readonly HttpConfig Config) { /* * Evil mutable struct, get a local mutable reference to the request's diff --git a/lib/Net.Http/src/FileUpload.cs b/lib/Net.Http/src/FileUpload.cs index 794c623..d3104fd 100644 --- a/lib/Net.Http/src/FileUpload.cs +++ b/lib/Net.Http/src/FileUpload.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Net.Http @@ -44,6 +44,11 @@ namespace VNLib.Net.Http /// public readonly record struct FileUpload(Stream FileData, bool DisposeStream, ContentType ContentType, string? FileName) { + /// + /// Gets the length of the stream + /// + public long Length => FileData.Length; + /// /// Disposes the stream if the handle is owned /// diff --git a/lib/Plugins.Runtime/src/AssemblyWatcher.cs b/lib/Plugins.Runtime/src/AssemblyWatcher.cs index 09b49dc..69c53ed 100644 --- a/lib/Plugins.Runtime/src/AssemblyWatcher.cs +++ b/lib/Plugins.Runtime/src/AssemblyWatcher.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Plugins.Runtime @@ -22,95 +22,52 @@ * along with VNLib.Plugins.Runtime. If not, see http://www.gnu.org/licenses/. */ +using System; using System.IO; using System.Threading; -using System.Collections.Generic; using VNLib.Utils; +using VNLib.Utils.IO; using VNLib.Utils.Extensions; -namespace VNLib.Plugins.Runtime -{ - internal sealed class AssemblyWatcher : IPluginAssemblyWatcher - { - private readonly object _lock = new (); - private readonly Dictionary _watchers; - public AssemblyWatcher() - { - _watchers = new(); - } +namespace VNLib.Plugins.Runtime +{ - /// - public void StopWatching(IPluginReloadEventHandler handler) - { - lock (_lock) - { - //Find old watcher by its handler, then dispose it - if (_watchers.Remove(handler, out AsmFileWatcher? watcher)) - { - //dispose the watcher - watcher.Dispose(); - } - } - } + internal static class AssemblyWatcher + { - /// - public void WatchAssembly(IPluginReloadEventHandler handler, IPluginAssemblyLoader loader) + internal static IDisposable WatchAssembly(IPluginReloadEventHandler handler, IPluginAssemblyLoader loader) { - lock(_lock) - { - if(_watchers.Remove(handler, out AsmFileWatcher? watcher)) - { - //dispose the watcher - watcher.Dispose(); - } + ArgumentNullException.ThrowIfNull(handler); + ArgumentNullException.ThrowIfNull(loader); - //Queue up a new watcher - watcher = new(loader, handler); + DebouncedFSEventHandler dbh = new(loader, handler); + FileWatcher.Subscribe(loader.Config.AssemblyFile, dbh); - //Store watcher - _watchers.Add(handler, watcher); - } + return dbh; } - private sealed class AsmFileWatcher : VnDisposeable + internal sealed class DebouncedFSEventHandler : VnDisposeable, IFSChangeHandler { - public IPluginReloadEventHandler Handler { get; } + private readonly IPluginReloadEventHandler _handler; private readonly IPluginAssemblyLoader _loaderSource; private readonly Timer _delayTimer; - private readonly FileSystemWatcher _watcher; private bool _pause; - public AsmFileWatcher(IPluginAssemblyLoader LoaderSource, IPluginReloadEventHandler handler) + public DebouncedFSEventHandler(IPluginAssemblyLoader loader, IPluginReloadEventHandler handler) { - Handler = handler; - _loaderSource = LoaderSource; - - string dir = Path.GetDirectoryName(LoaderSource.Config.AssemblyFile)!; - - //Configure watcher to notify only when the assembly file changes - _watcher = new FileSystemWatcher(dir) - { - Filter = "*.dll", - EnableRaisingEvents = false, - IncludeSubdirectories = true, - NotifyFilter = NotifyFilters.LastWrite, - }; - - //Configure listener - _watcher.Changed += OnFileChanged; - _watcher.Created += OnFileChanged; - - _watcher.EnableRaisingEvents = true; + _handler = handler; + _loaderSource = loader; //setup delay timer to wait on the config - _delayTimer = new(OnTimeout, this, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + _delayTimer = new(OnTimeout, null, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); } - void OnFileChanged(object sender, FileSystemEventArgs e) + /// + void IFSChangeHandler.OnFileChanged(FileSystemEventArgs e) { //if were already waiting to process an event, we dont need to stage another if (_pause) @@ -130,7 +87,7 @@ namespace VNLib.Plugins.Runtime _delayTimer.Stop(); //Fire event, let exception crash app - Handler.OnPluginUnloaded(_loaderSource); + _handler.OnPluginUnloaded(_loaderSource); //Clear pause flag _pause = false; @@ -140,9 +97,7 @@ namespace VNLib.Plugins.Runtime { _delayTimer.Dispose(); - //Detach event handler and dispose watcher - _watcher.Changed -= OnFileChanged; - _watcher.Dispose(); + FileWatcher.Unsubscribe(_loaderSource.Config.AssemblyFile, this); } } } diff --git a/lib/Plugins.Runtime/src/PluginStackBuilder.cs b/lib/Plugins.Runtime/src/PluginStackBuilder.cs index eed08e2..ef3ffc9 100644 --- a/lib/Plugins.Runtime/src/PluginStackBuilder.cs +++ b/lib/Plugins.Runtime/src/PluginStackBuilder.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Plugins.Runtime @@ -83,8 +83,10 @@ namespace VNLib.Plugins.Runtime /// The current builder instance for chaining public PluginStackBuilder WithConfigurationReader(IPluginConfigReader pluginConfig) { + ArgumentNullException.ThrowIfNull(pluginConfig); + //Store binary copy - PluginConfig = pluginConfig ?? throw new ArgumentNullException(nameof(pluginConfig)); + PluginConfig = pluginConfig; return this; } diff --git a/lib/Plugins.Runtime/src/RuntimePluginLoader.cs b/lib/Plugins.Runtime/src/RuntimePluginLoader.cs index 60edf55..23bbcab 100644 --- a/lib/Plugins.Runtime/src/RuntimePluginLoader.cs +++ b/lib/Plugins.Runtime/src/RuntimePluginLoader.cs @@ -38,10 +38,9 @@ namespace VNLib.Plugins.Runtime /// public sealed class RuntimePluginLoader : VnDisposeable, IPluginReloadEventHandler { - private static readonly IPluginAssemblyWatcher Watcher = new AssemblyWatcher(); - - private readonly IPluginAssemblyLoader Loader; + private readonly IPluginAssemblyLoader Loader; private readonly ILogProvider? Log; + private readonly IDisposable? Watcher; /// /// Gets the plugin assembly loader configuration information @@ -61,13 +60,15 @@ namespace VNLib.Plugins.Runtime /// public RuntimePluginLoader(IPluginAssemblyLoader loader, ILogProvider? log) { - Log = log; - Loader = loader ?? throw new ArgumentNullException(nameof(loader)); + ArgumentNullException.ThrowIfNull(loader); + + Log = log; + Loader = loader; //Configure watcher if requested if (loader.Config.WatchForReload) { - Watcher.WatchAssembly(this, loader); + Watcher = AssemblyWatcher.WatchAssembly(this, loader); } //Init container @@ -200,10 +201,8 @@ namespace VNLib.Plugins.Runtime /// protected override void Free() { - //Stop watching for events - Watcher.StopWatching(this); - //Cleanup + Watcher?.Dispose(); Controller.Dispose(); Loader.Dispose(); } diff --git a/lib/Utils.Cryptography/monocypher/CMakeLists.txt b/lib/Utils.Cryptography/monocypher/CMakeLists.txt index 9b17ea7..8cec60b 100644 --- a/lib/Utils.Cryptography/monocypher/CMakeLists.txt +++ b/lib/Utils.Cryptography/monocypher/CMakeLists.txt @@ -39,6 +39,7 @@ message(STATUS "Build type is '${CMAKE_BUILD_TYPE}'") #if debug add_compile_definitions($<$:DEBUG>) +add_compile_definitions(VNLIB_EXPORTING) #setup flags for windows compilation diff --git a/lib/Utils.Cryptography/monocypher/argon2.c b/lib/Utils.Cryptography/monocypher/argon2.c index 9b13ce0..2606e73 100644 --- a/lib/Utils.Cryptography/monocypher/argon2.c +++ b/lib/Utils.Cryptography/monocypher/argon2.c @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * vnlib_monocypher is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published @@ -18,9 +18,11 @@ #include "argon2.h" #include +#define ARGON2_WORK_AREA_MULTIPLIER 1024 + VNLIB_EXPORT uint32_t VNLIB_CC Argon2CalcWorkAreaSize(const argon2Ctx* context) { - return context->m_cost * 1024; + return context->m_cost * ARGON2_WORK_AREA_MULTIPLIER; } /* diff --git a/lib/Utils.Cryptography/monocypher/blake2b.h b/lib/Utils.Cryptography/monocypher/blake2b.h index 4167aca..0a0b4f8 100644 --- a/lib/Utils.Cryptography/monocypher/blake2b.h +++ b/lib/Utils.Cryptography/monocypher/blake2b.h @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * vnlib_monocypher is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published @@ -17,6 +17,7 @@ #pragma once #ifndef VN_MONOCYPHER_BLAKE2_H +#define VN_MONOCYPHER_BLAKE2_H #include #include "util.h" diff --git a/lib/Utils.Cryptography/monocypher/util.h b/lib/Utils.Cryptography/monocypher/util.h index 68b3e51..7a9f638 100644 --- a/lib/Utils.Cryptography/monocypher/util.h +++ b/lib/Utils.Cryptography/monocypher/util.h @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * vnlib_monocypher is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published @@ -16,47 +16,38 @@ */ #pragma once -#ifndef VN_MONOCYPHER_UTIL_H +#ifndef VN_MONOCYPHER_UTIL_H +#define VN_MONOCYPHER_UTIL_H -#if defined(__GNUC__) - #define inline __inline__ - #define VNLIB_EXPORT __attribute__((visibility("default"))) - #define VNLIB_CC -#elif defined(_MSC_VER) - #define VNLIB_EXPORT __declspec(dllexport) - #define VNLIB_CC __cdecl -#endif /* WIN32 */ - -#ifdef USE_MEM_UTIL - - /* Include stdlib for malloc */ - #include - - /* If a custom allocator is not defined, set macros for built-in function */ - #ifndef CUSTOM_ALLOCATOR - - /* malloc and friends fallback if not defined */ - #define vnmalloc(size) malloc(size) - #define vncalloc(count, size) calloc(count, size) - #define vnrealloc(ptr, size) realloc(ptr, size) - #define vnfree(ptr) free(ptr) - - #endif /* !CUSTOM_ALLOCATOR */ - - #ifdef WIN32 - - /* required for memove on windows */ - #include - - #define _memmove(dst, src, size) memmove_s(dst, size, src, size) - #else - /* use string.h posix on non-win platforms */ - #include - - #define _memmove memmove - #endif /* WIN32 */ +#if defined(_MSC_VER) || defined(WIN32) || defined(_WIN32) + #define _P_IS_WINDOWS +#endif -#endif // USE_MEM_UTIL +//Set api export calling convention (allow used to override) +#ifndef VNLIB_CC + #ifdef _P_IS_WINDOWS + //STD for importing to other languages such as .NET + #define VNLIB_CC __stdcall + #else + #define VNLIB_CC + #endif +#endif // !NC_CC + +#ifndef VNLIB_EXPORT //Allow users to disable the export/impoty macro if using source code directly + #ifdef VNLIB_EXPORTING + #ifdef _P_IS_WINDOWS + #define VNLIB_EXPORT __declspec(dllexport) + #else + #define VNLIB_EXPORT __attribute__((visibility("default"))) + #endif // _NC_IS_WINDOWS + #else + #ifdef _P_IS_WINDOWS + #define VNLIB_EXPORT __declspec(dllimport) + #else + #define VNLIB_EXPORT + #endif // _P_IS_WINDOWS + #endif // !VNLIB_EXPORTING +#endif // !VNLIB_EXPORT #ifndef _In_ #define _In_ diff --git a/lib/Utils.Cryptography/monocypher/vnlib_monocypher.h b/lib/Utils.Cryptography/monocypher/vnlib_monocypher.h index 920def9..943431a 100644 --- a/lib/Utils.Cryptography/monocypher/vnlib_monocypher.h +++ b/lib/Utils.Cryptography/monocypher/vnlib_monocypher.h @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * vnlib_monocypher is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published @@ -16,7 +16,7 @@ */ #pragma once -#ifndef VNLIB_MONOCYPHER_H +#ifndef VNLIB_MONOCYPHER_H #define VNLIB_MONOCYPHER_H #include "util.h" diff --git a/lib/Utils/src/ArgumentList.cs b/lib/Utils/src/ArgumentList.cs index 235e62c..c02ebee 100644 --- a/lib/Utils/src/ArgumentList.cs +++ b/lib/Utils/src/ArgumentList.cs @@ -24,6 +24,7 @@ using System; using System.Linq; +using System.Collections; using System.Collections.Generic; namespace VNLib.Utils @@ -31,7 +32,7 @@ namespace VNLib.Utils /// /// Provides methods for parsing an argument list /// - public class ArgumentList : IIndexable + public class ArgumentList : IIndexable, IEnumerable { private readonly List _args; @@ -96,6 +97,10 @@ namespace VNLib.Utils return index == -1 || index + 1 >= _args.Count ? null : this[index + 1]; } - + /// + public IEnumerator GetEnumerator() => _args.GetEnumerator(); + + /// + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); } } \ No newline at end of file diff --git a/lib/Utils/src/Memory/MemoryUtil.cs b/lib/Utils/src/Memory/MemoryUtil.cs index 6efd5ba..b6eadbc 100644 --- a/lib/Utils/src/Memory/MemoryUtil.cs +++ b/lib/Utils/src/Memory/MemoryUtil.cs @@ -25,7 +25,6 @@ using System; using System.Buffers; using System.Security; -using System.Threading; using System.Diagnostics; using System.Globalization; using System.Runtime.InteropServices; @@ -33,6 +32,7 @@ using System.Runtime.CompilerServices; using System.Runtime.Intrinsics.X86; using VNLib.Utils.Memory.Diagnostics; +using VNLib.Utils.Resources; namespace VNLib.Utils.Memory { @@ -102,13 +102,13 @@ namespace VNLib.Utils.Memory /// The backing heap /// is determined by the OS type and process environment varibles. /// - public static IUnmangedHeap Shared => _sharedHeap.Value; + public static IUnmangedHeap Shared => _lazyHeap.Instance; - - private static readonly Lazy _sharedHeap = InitHeapInternal(); + + private static readonly LazyInitializer _lazyHeap = InitHeapInternal(); //Avoiding static initializer - private static Lazy InitHeapInternal() + private static LazyInitializer InitHeapInternal() { //Get env for heap diag _ = ERRNO.TryParse(Environment.GetEnvironmentVariable(SHARED_HEAP_ENABLE_DIAGNOISTICS_ENV), out ERRNO diagEnable); @@ -116,22 +116,17 @@ namespace VNLib.Utils.Memory Trace.WriteLineIf(diagEnable, "Shared heap diagnostics enabled"); Trace.WriteLineIf(globalZero, "Shared heap global zero enabled"); - - Lazy heap = new (() => InitHeapInternal(true, diagEnable, globalZero), LazyThreadSafetyMode.PublicationOnly); - - //Cleanup the heap on process exit - AppDomain.CurrentDomain.DomainUnload += DomainUnloaded; - - return heap; - } - private static void DomainUnloaded(object? sender, EventArgs e) - { - //Dispose the heap if allocated - if (_sharedHeap.IsValueCreated) + return new(() => { - _sharedHeap.Value.Dispose(); - } + //Init shared heap instance + IUnmangedHeap heap = InitHeapInternal(true, diagEnable, globalZero); + + //Register domain unload event + AppDomain.CurrentDomain.DomainUnload += (_, _) => heap.Dispose(); + + return heap; + }); } /// @@ -147,7 +142,7 @@ namespace VNLib.Utils.Memory * If heap is allocated and the heap type is a tracked heap, * get the heap's stats, otherwise return an empty handle */ - return _sharedHeap.IsValueCreated && _sharedHeap.Value is TrackedHeapWrapper h + return _lazyHeap.IsLoaded && _lazyHeap.Instance is TrackedHeapWrapper h ? h.GetCurrentStats() : default; } @@ -1610,4 +1605,4 @@ namespace VNLib.Utils.Memory public readonly void Validate(nuint count) => CheckBounds(handle, offset, count); } } -} \ No newline at end of file +} diff --git a/lib/Utils/src/Native/NatveLibraryResolver.cs b/lib/Utils/src/Native/NatveLibraryResolver.cs new file mode 100644 index 0000000..b888f42 --- /dev/null +++ b/lib/Utils/src/Native/NatveLibraryResolver.cs @@ -0,0 +1,299 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Utils +* File: NatveLibraryResolver.cs +* +* NatveLibraryResolver.cs is part of VNLib.Utils which is part of +* the larger VNLib collection of libraries and utilities. +* +* VNLib.Utils is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* VNLib.Utils is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with VNLib.Utils. If not, see http://www.gnu.org/licenses/. +*/ + +using System; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Diagnostics; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Diagnostics.CodeAnalysis; + +namespace VNLib.Utils.Native +{ + /// + /// Uses a supplied library state to resolve and load a platform native library + /// into the process memory space. + /// + /// The raw caller-supplied library path to resolve + /// A assembly loading the desired library + /// The dll loader search path requirements + internal readonly struct NatveLibraryResolver(string suppliedLibraryPath, Assembly assembly, DllImportSearchPath searchPath) + { + private readonly string _libFileName = Path.GetFileName(suppliedLibraryPath); + private readonly string? _relativeDir = Path.GetDirectoryName(suppliedLibraryPath); + + internal readonly bool HasRelativeDir => _relativeDir != null; + + internal readonly bool IsFullPath => Path.IsPathRooted(suppliedLibraryPath); + + /// + /// Resolves and attempts to load the current library into the current process. + /// + /// The if the library was successfully resolved + /// True if the library was resolved and loaded into the process, false otherwise + internal readonly bool ResolveAndLoadLibrary([NotNullWhen(true)] out SafeLibraryHandle? library) + { + //Try naive load if the path is an absolute path + if (IsFullPath && _tryLoad(suppliedLibraryPath, out library)) + { + return true; + } + + //Try to load the library from the relative directory + if (HasRelativeDir && TryLoadRelativeToWorkingDir(out library)) + { + return true; + } + + //Path is just a file name, so search for prefixes and extensions loading directly + if (TryNaiveLoadLibrary(out library)) + { + return true; + } + + //Try searching for the file in directories + return TryLoadFromSafeDirs(out library); + } + + /* + * Naive load builds file names that are platform dependent + * and probes for the library using the NativeLibrary.TryLoad rules + * to load the library. + * + * If the file has an extension, it will attempt to load the library + * directly using the file name. Otherwise, it will continue to probe + */ + private readonly bool TryNaiveLoadLibrary([NotNullWhen(true)] out SafeLibraryHandle? library) + { + foreach (string probingFileName in GetProbingFileNames(_libFileName)) + { + if (_tryLoad(probingFileName, out library)) + { + return true; + } + } + + library = null; + return false; + } + + /* + * Attempts to probe library files names that are located inside safe directories + * specified by the caller using the DllImportSearchPath enum. + */ + private readonly bool TryLoadFromSafeDirs([NotNullWhen(true)] out SafeLibraryHandle? library) + { + //Try enumerating safe directories + if (searchPath.HasFlag(DllImportSearchPath.SafeDirectories)) + { + foreach (string dir in GetSpecialDirPaths()) + { + if (TryLoadInDirectory(dir, out library)) + { + return true; + } + } + } + + //Check application directory first (including subdirectories) + if (searchPath.HasFlag(DllImportSearchPath.ApplicationDirectory)) + { + //get the current directory + if (TryLoadInDirectory(Directory.GetCurrentDirectory(), out library)) + { + return true; + } + } + + //See if search in the calling assembly directory + if (searchPath.HasFlag(DllImportSearchPath.AssemblyDirectory)) + { + //Get the calling assmblies directory + string libDir = Assembly.GetCallingAssembly().Location; + Debug.WriteLine("Native library searching for calling assembly location:{0} ", libDir); + if (TryLoadInDirectory(libDir, out library)) + { + return true; + } + } + + //Search system32 dir + if (searchPath.HasFlag(DllImportSearchPath.System32)) + { + string sys32Dir = Environment.GetFolderPath(Environment.SpecialFolder.SystemX86); + + //Get the system directory + if (TryLoadInDirectory(sys32Dir, out library)) + { + return true; + } + } + + library = null; + return false; + } + + /* + * Users may specify realtive directories to search for the library + * in the current working directory, so this function attempts to load + * the library from the relative directory if the user has specified one + */ + private readonly bool TryLoadRelativeToWorkingDir([NotNullWhen(true)] out SafeLibraryHandle? library) + { + string libDir = Directory.GetCurrentDirectory(); + return TryLoadInDirectory(Path.Combine(libDir, _relativeDir!), out library); + } + + /* + * Attempts to load libraries that are located in the specified directory + * by probing for the library file name in the directory path using + * prefixes and extensions + */ + private readonly bool TryLoadInDirectory(string baseDir, [NotNullWhen(true)] out SafeLibraryHandle? library) + { + IEnumerable fullProbingPaths = GetProbingFileNames(_libFileName).Select(p => Path.Combine(baseDir, p)); + + foreach (string probingFilePath in fullProbingPaths) + { + if (_tryLoad(probingFilePath, out library)) + { + return true; + } + } + + library = null; + return false; + } + + /* + * core load function + */ + internal readonly bool _tryLoad(string filePath, [NotNullWhen(true)] out SafeLibraryHandle? library) + { + //Attempt a naive load + if (NativeLibrary.TryLoad(filePath, assembly, searchPath, out IntPtr libHandle)) + { + library = SafeLibraryHandle.FromExisting(libHandle, true); + return true; + } + + library = null; + return false; + } + + private static readonly Environment.SpecialFolder[] SafeDirs = + [ + Environment.SpecialFolder.SystemX86, + Environment.SpecialFolder.System, + Environment.SpecialFolder.Windows, + Environment.SpecialFolder.ProgramFilesX86, + Environment.SpecialFolder.ProgramFiles + ]; + + private static IEnumerable GetSpecialDirPaths() => SafeDirs.Select(Environment.GetFolderPath); + + private static IEnumerable GetProbingFileNames(string libraryFileName) + { + //Inlcude the library file name if it has an extension + if (Path.HasExtension(libraryFileName)) + { + yield return libraryFileName; + } + + foreach (string prefix in GetNativeLibPrefixs()) + { + foreach (string libExtension in GetNativeLibExtension()) + { + yield return GetLibraryFileName(prefix, libraryFileName, libExtension); + } + } + + static string GetLibraryFileName(string prefix, string libPath, string extension) + { + //Get dir name from the lib path if it has one + + + libPath = Path.Combine(prefix, libPath); + + //If the library path already has an extension, just search for the file + if (!Path.HasExtension(libPath)) + { + //slice the lib to its file name + libPath = Path.GetFileName(libPath); + + if (extension.Length > 0) + { + libPath = Path.ChangeExtension(libPath, extension); + } + } + + return libPath; + } + + static string[] GetNativeLibPrefixs() + { + if (OperatingSystem.IsWindows()) + { + return [""]; + } + else if (OperatingSystem.IsMacOS()) + { + return ["", "lib"]; + } + else if (OperatingSystem.IsLinux()) + { + return ["", "lib_", "lib"]; + } + else + { + Debug.Fail("Unknown OS type"); + return []; + } + } + + static string[] GetNativeLibExtension() + { + if (OperatingSystem.IsWindows()) + { + return ["", ".dll"]; + } + else if (OperatingSystem.IsMacOS()) + { + return ["", ".dylib"]; + } + else if (OperatingSystem.IsLinux()) + { + return ["", ".so", ".so.1"]; + } + else + { + Debug.Fail("Unknown OS type"); + return []; + } + } + } + } +} diff --git a/lib/Utils/src/Native/SafeLibraryHandle.cs b/lib/Utils/src/Native/SafeLibraryHandle.cs index ed9dd16..5fb2283 100644 --- a/lib/Utils/src/Native/SafeLibraryHandle.cs +++ b/lib/Utils/src/Native/SafeLibraryHandle.cs @@ -23,10 +23,7 @@ */ using System; -using System.IO; -using System.Linq; using System.Reflection; -using System.Diagnostics; using System.Runtime.InteropServices; using System.Diagnostics.CodeAnalysis; @@ -42,128 +39,7 @@ namespace VNLib.Utils.Native /// public override bool IsInvalid => handle == IntPtr.Zero; - private SafeLibraryHandle(IntPtr libHandle) : base(IntPtr.Zero, true) - { - //Init handle - SetHandle(libHandle); - } - - /// - /// Finds and loads the specified native libary into the current process by its name at runtime - /// - /// The path (or name of libary) to search for - /// - /// The used to search for libaries - /// within the current filesystem - /// - /// The loaded - /// - /// - public static SafeLibraryHandle LoadLibrary(string libPath, DllImportSearchPath searchPath = DllImportSearchPath.ApplicationDirectory) - { - ArgumentNullException.ThrowIfNull(libPath); - //See if the path includes a file extension - return TryLoadLibrary(libPath, searchPath, out SafeLibraryHandle? lib) - ? lib - : throw new DllNotFoundException($"The library {libPath} or one of its dependencies could not be found"); - } - - /// - /// Attempts to load the specified native libary into the current process by its name at runtime - /// - ///The path (or name of libary) to search for - /// - /// The used to search for libaries - /// within the current filesystem - /// - /// The handle to the libary if successfully loaded - /// True if the libary was found and loaded into the current process - public static bool TryLoadLibrary(string libPath, DllImportSearchPath searchPath, [NotNullWhen(true)] out SafeLibraryHandle? lib) - { - lib = null; - //Allow full rooted paths - if (Path.IsPathRooted(libPath)) - { - //Attempt a native load - if (NativeLibrary.TryLoad(libPath, out IntPtr libHandle)) - { - lib = new(libHandle); - return true; - } - return false; - } - //Check application directory first (including subdirectories) - if ((searchPath & DllImportSearchPath.ApplicationDirectory) > 0) - { - //get the current directory - string libDir = Directory.GetCurrentDirectory(); - if (TryLoadLibraryInternal(libDir, libPath, SearchOption.TopDirectoryOnly, out lib)) - { - return true; - } - } - //See if search in the calling assembly directory - if ((searchPath & DllImportSearchPath.AssemblyDirectory) > 0) - { - //Get the calling assmblies directory - string libDir = Assembly.GetCallingAssembly().Location; - Debug.WriteLine("Native library searching for calling assembly location:{0} ", libDir); - if (TryLoadLibraryInternal(libDir, libPath, SearchOption.TopDirectoryOnly, out lib)) - { - return true; - } - } - //Search system32 dir - if ((searchPath & DllImportSearchPath.System32) > 0) - { - //Get the system directory - string libDir = Environment.GetFolderPath(Environment.SpecialFolder.SystemX86); - if (TryLoadLibraryInternal(libDir, libPath, SearchOption.TopDirectoryOnly, out lib)) - { - return true; - } - } - //Attempt a native load - { - if (NativeLibrary.TryLoad(libPath, out IntPtr libHandle)) - { - lib = new(libHandle); - return true; - } - return false; - } - } - - private static bool TryLoadLibraryInternal(string libDir, string libPath, SearchOption dirSearchOptions, [NotNullWhen(true)] out SafeLibraryHandle? libary) - { - //Try to find the libary file - string? libFile = GetLibraryFile(libDir, libPath, dirSearchOptions); - //Load libary - if (libFile != null && NativeLibrary.TryLoad(libFile, out IntPtr libHandle)) - { - libary = new SafeLibraryHandle(libHandle); - return true; - } - libary = null; - return false; - } - - private static string? GetLibraryFile(string dirPath, string libPath, SearchOption search) - { - //If the library path already has an extension, just search for the file - if (Path.HasExtension(libPath)) - { - return Directory.EnumerateFiles(dirPath, libPath, search).FirstOrDefault(); - } - else - { - //slice the lib to its file name - libPath = Path.GetFileName(libPath); - libPath = Path.ChangeExtension(libPath, OperatingSystem.IsWindows() ? ".dll" : ".so"); - //Select the first file that matches the name - return Directory.EnumerateFiles(dirPath, libPath, search).FirstOrDefault(); - } - } + private SafeLibraryHandle(IntPtr libHandle, bool ownsHandle) : base(IntPtr.Zero, ownsHandle) => SetHandle(libHandle); /// /// Loads a native function pointer from the library of the specified name and @@ -179,7 +55,7 @@ namespace VNLib.Utils.Native { //Increment handle count before obtaining a method bool success = false; - DangerousAddRef(ref success); + DangerousAddRef(ref success); ObjectDisposedException.ThrowIf(success == false, this); @@ -218,40 +94,124 @@ namespace VNLib.Utils.Native return Marshal.GetDelegateForFunctionPointer(nativeMethod); } + /// + protected override bool ReleaseHandle() + { + //Free the library and set the handle as invalid + NativeLibrary.Free(handle); + SetHandleAsInvalid(); + return true; + } + /// - /// Loads a native method from the library of the specified name and managed delegate + /// Finds and loads the specified native libary into the current process by its name at runtime. + /// This function defaults to the executing assembly /// - /// The native method delegate type - /// The name of the native method - /// A wapper handle around the native method delegate + /// The path (or name of libary) to search for + /// + /// The used to search for libaries + /// within the current filesystem + /// + /// The loaded /// - /// If the handle is closed or invalid - /// When the specified entrypoint could not be found - [Obsolete("Updated naming, use GetFunction() instead")] - public SafeMethodHandle GetMethod(string methodName) where T : Delegate => GetFunction(methodName); + /// + public static SafeLibraryHandle LoadLibrary(string libPath, DllImportSearchPath searchPath = DllImportSearchPath.ApplicationDirectory) + { + //See if the path includes a file extension + return TryLoadLibrary(libPath, searchPath, out SafeLibraryHandle? lib) + ? lib + : throw new DllNotFoundException($"The library '{libPath}' or one of its dependencies could not be found"); + } /// - /// Gets an delegate wrapper for the specified method without tracking its referrence. - /// The caller must manage the referrence count in order - /// to not leak resources or cause process corruption + /// Finds and loads the specified native libary into the current process by its name at runtime /// - /// The native method delegate type - /// The name of the native method - /// A the delegate wrapper on the native method + /// The path (or name of libary) to search for + /// + /// The used to search for libaries + /// within the current filesystem + /// + /// The assembly loading the native library + /// The loaded /// - /// If the handle is closed or invalid - /// When the specified entrypoint could not be found - [Obsolete("Updated naming, use DangerousGetFunction() instead")] - public T DangerousGetMethod(string methodName) where T : Delegate => DangerousGetFunction(methodName); + /// + public static SafeLibraryHandle LoadLibrary( + string libPath, + Assembly assembly, + DllImportSearchPath searchPath = DllImportSearchPath.ApplicationDirectory + ) + { + //See if the path includes a file extension + return TryLoadLibrary(libPath, assembly, searchPath, out SafeLibraryHandle? lib) + ? lib + : throw new DllNotFoundException($"The library '{libPath}' or one of its dependencies could not be found"); + } + /// + /// Creates a new from an existing library pointer. + /// + /// A pointer to the existing (and loaded) library + /// A value that specifies whether the wrapper owns the library handle now + /// A safe library wrapper around the existing library pointer + /// + public unsafe static SafeLibraryHandle FromExisting(nint libHandle, bool ownsHandle) + { + ArgumentNullException.ThrowIfNull(libHandle.ToPointer(), nameof(libHandle)); + return new(libHandle, ownsHandle); + } - /// - protected override bool ReleaseHandle() + /// + /// Attempts to load the specified native libary into the current process by its name at runtime. + /// This function defaults to the executing assembly + /// + ///The path (or name of libary) to search for + /// + /// The used to search for libaries + /// within the current filesystem + /// + /// The handle to the libary if successfully loaded + /// True if the libary was found and loaded into the current process + public static bool TryLoadLibrary( + string libPath, + DllImportSearchPath searchPath, + [NotNullWhen(true)] out SafeLibraryHandle? library + ) { - //Free the library and set the handle as invalid - NativeLibrary.Free(handle); - SetHandleAsInvalid(); - return true; + return TryLoadLibrary( + libPath, + Assembly.GetExecutingAssembly(), //Use the executing assembly as the default loading assembly + searchPath, + out library + ); + } + + + + /// + /// Attempts to load the specified native libary into the current process by its name at runtime + /// + ///The path (or name of libary) to search for + /// + /// The used to search for libaries + /// within the current filesystem + /// + /// The handle to the libary if successfully loaded + /// The assembly loading the native library + /// True if the libary was found and loaded into the current process + /// + public static bool TryLoadLibrary( + string libPath, + Assembly assembly, + DllImportSearchPath searchPath, + [NotNullWhen(true)] out SafeLibraryHandle? library + ) + { + ArgumentNullException.ThrowIfNull(libPath); + ArgumentNullException.ThrowIfNull(assembly); + + NatveLibraryResolver resolver = new(libPath, assembly, searchPath); + + return resolver.ResolveAndLoadLibrary(out library); } } } diff --git a/lib/Utils/src/Resources/LazyInitializer.cs b/lib/Utils/src/Resources/LazyInitializer.cs new file mode 100644 index 0000000..5de43e0 --- /dev/null +++ b/lib/Utils/src/Resources/LazyInitializer.cs @@ -0,0 +1,112 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Utils +* File: LazyInitializer.cs +* +* LazyInitializer.cs is part of VNLib.Utils which is part of the larger +* VNLib collection of libraries and utilities. +* +* VNLib.Utils is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* VNLib.Utils is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with VNLib.Utils. If not, see http://www.gnu.org/licenses/. +*/ + +using System; +using System.Threading; +using System.Diagnostics; + +namespace VNLib.Utils.Resources +{ + /// + /// A lazy initializer that creates a single instance of a type + /// and shares it across all threads. This class simply guarantees + /// that the instance is only created once and shared across all + /// threads as efficiently as possible for long running processes. + /// + /// + /// The callback function that initializes the instance + public sealed class LazyInitializer(Func initalizer) + { + private readonly object _lock = new(); + private readonly Func initalizer = initalizer ?? throw new ArgumentNullException(nameof(initalizer)); + + private T? _instance; + private bool _isLoaded; + + /// + /// A value indicating if the instance has ben loaded + /// + public bool IsLoaded => _isLoaded; + + /// + /// Gets or creates the instance only once and returns + /// the shared instance + /// + /// + /// NOTE: + /// Accessing this property may block the calling thread + /// if the instance has not yet been loaded. Only one thread + /// will create the instance, all other threads will wait + /// for the instance to be created. + /// + public T Instance + { + get + { + //See if instance is already loaded (this read is atomic in .NET) + if (_isLoaded) + { + return _instance!; + } + + /* + * Instance has not yet been loaded. Only one thread + * must load the object, all other threads must wait + * for the object to be loaded. + */ + + if (Monitor.TryEnter(_lock, 0)) + { + try + { + /* + * Lock was entered without waiting (lock was available), this will now be + * the thread that invokes the load function + */ + + _instance = initalizer(); + + //Finally set the load state + _isLoaded = true; + } + finally + { + Monitor.Exit(_lock); + } + } + else + { + //wait for lock to be released, when it is, the object should be loaded + Monitor.Enter(_lock); + Monitor.Exit(_lock); + + //object instance should now be available to non-creating threads + Debug.Assert(_isLoaded); + } + + return _instance!; + } + } + } +} \ No newline at end of file -- cgit