From a19807f7f73ffb023e4ffe93071fe91525fd2c8d Mon Sep 17 00:00:00 2001 From: vnugent Date: Sat, 7 Sep 2024 15:30:02 -0400 Subject: Squashed commit of the following: commit 97d0c461140e4badf40454471748099266c58c85 Author: vnugent Date: Sat Sep 7 15:18:31 2024 -0400 restsharp vuln update & remove github pull job for now commit f7eb445c9fcf4557f0bf1e1622673242b7da5ced Author: vnugent Date: Sat Sep 7 14:51:44 2024 -0400 update manual plugin loading api commit d3fa866898747c7b7535f2796f8046cdd9766763 Author: vnugent Date: Sat Sep 7 12:56:36 2024 -0400 package updates commit bdb62d04a7c7ea9bcea01b6314ba3306e07099f1 Author: vnugent Date: Sat Sep 7 12:50:58 2024 -0400 minor memory api internal cleanup commit d297b3a958e13a76ea61c8df588ec32ea9a40faf Author: vnugent Date: Mon Aug 26 22:21:56 2024 -0400 refactor: #7 Update compression style, platform and linking --- .onedev-buildspec.yml | 19 ---- Module.Taskfile.yaml | 2 +- .../VNLib.WebServer/src/Bootstrap/WebserverBase.cs | 4 + apps/VNLib.WebServer/src/VLogProvider.cs | 28 ++--- apps/VNLib.WebServer/src/VNLib.WebServer.csproj | 4 +- .../tests/VNLib.Hashing.PortableTests.csproj | 6 +- .../VNLib.Net.Compression/CompressionExtensions.cs | 113 +++++++++++---------- .../VNLib.Net.Compression/NativeCompressionLib.cs | 66 ++++++++---- .../VNLib.Net.CompressionTests.csproj | 6 +- lib/Net.Compression/vnlib_compress/CMakeLists.txt | 56 +++++----- lib/Net.Compression/vnlib_compress/Taskfile.yaml | 43 ++++++-- .../vnlib_compress/src/compression.c | 58 +++++------ .../vnlib_compress/src/compression.h | 83 ++++++++------- .../vnlib_compress/src/feature_brotli.c | 25 ++--- .../vnlib_compress/src/feature_brotli.h | 11 +- .../vnlib_compress/src/feature_zlib.c | 29 +++--- .../vnlib_compress/src/feature_zlib.h | 14 +-- lib/Net.Compression/vnlib_compress/src/platform.h | 90 ++++++++++++++++ lib/Net.Compression/vnlib_compress/src/util.h | 69 +++---------- .../vnlib_compress/src/vnlib_compress.vcxitems | 1 + .../src/VNLib.Net.Rest.Client.csproj | 2 +- .../src/Construction/HttpServiceStackBuilder.cs | 14 +-- lib/Utils/src/Extensions/MemoryExtensions.cs | 9 +- lib/Utils/src/Memory/MemoryHandle.cs | 9 +- lib/Utils/src/Memory/MemoryUtil.cs | 6 +- lib/Utils/src/Memory/MemoryUtilAlloc.cs | 62 +++++++++++ lib/Utils/src/Memory/UnsafeMemoryHandle.cs | 31 ++++-- lib/Utils/src/Memory/Win32PrivateHeap.cs | 61 ++++++----- lib/Utils/tests/VNLib.UtilsTests.csproj | 6 +- 29 files changed, 554 insertions(+), 373 deletions(-) create mode 100644 lib/Net.Compression/vnlib_compress/src/platform.h diff --git a/.onedev-buildspec.yml b/.onedev-buildspec.yml index 9700986..bfccb25 100644 --- a/.onedev-buildspec.yml +++ b/.onedev-buildspec.yml @@ -25,22 +25,3 @@ jobs: maxRetries: 3 retryDelay: 30 timeout: 3600 -- name: Repo Sync Pull - steps: - - !PullRepository - name: GitHub sync pull - remoteUrl: https://github.com/VnUgE/VNLib.Core.git - userName: VnUgE - passwordSecret: git-access-token - refs: refs/heads/* refs/tags/* - withLfs: false - force: false - condition: ALL_PREVIOUS_STEPS_WERE_SUCCESSFUL - triggers: - - !ScheduleTrigger - cronExpression: 0 15 10 ? * * - projects: VNLib.Core - retryCondition: never - maxRetries: 3 - retryDelay: 30 - timeout: 3600 diff --git a/Module.Taskfile.yaml b/Module.Taskfile.yaml index aa5893f..6ec8fc6 100644 --- a/Module.Taskfile.yaml +++ b/Module.Taskfile.yaml @@ -63,7 +63,7 @@ tasks: - cd lib/Utils.Memory/vnlib_mimalloc && task dev-init - cd lib/Utils.Cryptography/monocypher && task build - cd lib/Utils.Cryptography/argon2 && task build - - cd lib/Net.Compression/vnlib_compress && task build + - cd lib/Net.Compression/vnlib_compress && task build - cmd: echo "dev init complete" silent: true diff --git a/apps/VNLib.WebServer/src/Bootstrap/WebserverBase.cs b/apps/VNLib.WebServer/src/Bootstrap/WebserverBase.cs index f3832c6..67fc306 100644 --- a/apps/VNLib.WebServer/src/Bootstrap/WebserverBase.cs +++ b/apps/VNLib.WebServer/src/Bootstrap/WebserverBase.cs @@ -93,6 +93,10 @@ namespace VNLib.WebServer.Bootstrap HttpServiceStackBuilder builder = new HttpServiceStackBuilder() .LoadPluginsConcurrently(loadPluginsConcurrently) .WithBuiltInHttp(TcpConfig.ReduceBindingsForGroups, http) + .WithManualPlugins(plugins => + { + + }) .WithDomain(domain => { domain.WithServiceGroups(vh => diff --git a/apps/VNLib.WebServer/src/VLogProvider.cs b/apps/VNLib.WebServer/src/VLogProvider.cs index d20437f..67d88ef 100644 --- a/apps/VNLib.WebServer/src/VLogProvider.cs +++ b/apps/VNLib.WebServer/src/VLogProvider.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2022 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.WebServer @@ -35,14 +35,10 @@ using VNLib.Utils.Logging; namespace VNLib.WebServer { - internal sealed class VLogProvider : VnDisposeable, ILogProvider + internal sealed class VLogProvider(LoggerConfiguration config) : VnDisposeable, ILogProvider { - private readonly Logger LogCore; + private readonly Logger LogCore = config.CreateLogger(); - public VLogProvider(LoggerConfiguration config) - { - LogCore = config.CreateLogger(); - } public void Flush() { } public object GetLogProvider() => LogCore; @@ -51,22 +47,16 @@ namespace VNLib.WebServer public bool IsEnabled(LogLevel level) => LogCore.IsEnabled((LogEventLevel)level); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void Write(LogLevel level, string value) - { - LogCore.Write((LogEventLevel)level, value); - } + public void Write(LogLevel level, string value) + => LogCore.Write((LogEventLevel)level, value); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void Write(LogLevel level, Exception exception, string value = "") - { - LogCore.Write((LogEventLevel)level, exception, value); - } + public void Write(LogLevel level, Exception exception, string value = "") + => LogCore.Write((LogEventLevel)level, exception, value); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void Write(LogLevel level, string value, params object[] args) - { - LogCore.Write((LogEventLevel)level, value, args); - } + public void Write(LogLevel level, string value, params object[] args) + => LogCore.Write((LogEventLevel)level, value, args); [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Write(LogLevel level, string value, params ValueType[] args) diff --git a/apps/VNLib.WebServer/src/VNLib.WebServer.csproj b/apps/VNLib.WebServer/src/VNLib.WebServer.csproj index 37808b5..d2d9945 100644 --- a/apps/VNLib.WebServer/src/VNLib.WebServer.csproj +++ b/apps/VNLib.WebServer/src/VNLib.WebServer.csproj @@ -29,7 +29,7 @@ Copyright © 2024 Vaughn Nugent VNLib.Webserver https://www.vaughnnugent.com/resources/software/modules/vnlib.core - https://github.com/VnUgE/VNLib.Core/tree/master/app/VNLib.Webserver/ + https://github.com/VnUgE/VNLib.Core/tree/master/apps/VNLib.WebServer README.md LICENSE @@ -68,7 +68,7 @@ - + diff --git a/lib/Hashing.Portable/tests/VNLib.Hashing.PortableTests.csproj b/lib/Hashing.Portable/tests/VNLib.Hashing.PortableTests.csproj index eb95e7c..acc0c5a 100644 --- a/lib/Hashing.Portable/tests/VNLib.Hashing.PortableTests.csproj +++ b/lib/Hashing.Portable/tests/VNLib.Hashing.PortableTests.csproj @@ -14,9 +14,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/lib/Net.Compression/VNLib.Net.Compression/CompressionExtensions.cs b/lib/Net.Compression/VNLib.Net.Compression/CompressionExtensions.cs index 4509b67..0d1fc73 100644 --- a/lib/Net.Compression/VNLib.Net.Compression/CompressionExtensions.cs +++ b/lib/Net.Compression/VNLib.Net.Compression/CompressionExtensions.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Net.Compression @@ -36,12 +36,18 @@ namespace VNLib.Net.Compression /// Compresses a block using the compressor context pointer provided /// /// - /// A pointer to the compressor context + /// A pointer to the compressor context /// A buffer to write the result to /// The input block of memory to compress /// A value that indicates if a flush is requested /// The results of the compression operation - public static unsafe CompressionResult CompressBlock(this LibraryWrapper nativeLib, IntPtr comp, Memory output, ReadOnlyMemory input, bool finalBlock) + public static unsafe CompressionResult CompressBlock( + this LibraryWrapper nativeLib, + IntPtr compressorInstance, + Memory output, + ReadOnlyMemory input, + bool finalBlock + ) { /* * Since .NET only supports int32 size memory blocks @@ -51,46 +57,51 @@ namespace VNLib.Net.Compression * sizes (read/written) */ - //get pointers to the input and output buffers - using MemoryHandle inPtr = input.Pin(); - using MemoryHandle outPtr = output.Pin(); - //Create the operation struct - CompressionOperation operation; - CompressionOperation* op = &operation; + CompressionOperation operation = default; - op->flush = finalBlock ? 1 : 0; - op->bytesRead = 0; - op->bytesWritten = 0; + operation.flush = finalBlock ? 1 : 0; - //Configure the input and output buffers - op->inputBuffer = inPtr.Pointer; - op->inputSize = (uint)input.Length; + checked + { + //get pointers to the input and output buffers + using MemoryHandle inPtr = input.Pin(); + using MemoryHandle outPtr = output.Pin(); - op->outputBuffer = outPtr.Pointer; - op->outputSize = (uint)output.Length; + //Configure the input and output buffers + operation.inputBuffer = inPtr.Pointer; + operation.inputSize = (uint)input.Length; - //Call the native compress function - nativeLib!.CompressBlock(comp, &operation); + operation.outputBuffer = outPtr.Pointer; + operation.outputSize = (uint)output.Length; - //Return the number of bytes written - return new() - { - BytesRead = (int)op->bytesRead, - BytesWritten = (int)op->bytesWritten - }; + //Call the native compress function + nativeLib!.CompressBlock(compressorInstance, &operation); + + return new() + { + BytesRead = (int)operation.bytesRead, + BytesWritten = (int)operation.bytesWritten + }; + } } /// /// Compresses a block using the compressor context pointer provided /// /// - /// A pointer to the compressor context + /// A pointer to the compressor context /// A buffer to write the result to /// The input block of memory to compress /// A value that indicates if a flush is requested /// The results of the compression operation - public static unsafe CompressionResult CompressBlock(this LibraryWrapper nativeLib, IntPtr comp, Span output, ReadOnlySpan input, bool finalBlock) + public static unsafe CompressionResult CompressBlock( + this LibraryWrapper nativeLib, + IntPtr compressorInstance, + Span output, + ReadOnlySpan input, + bool finalBlock + ) { /* * Since .NET only supports int32 size memory blocks @@ -100,33 +111,31 @@ namespace VNLib.Net.Compression * sizes (read/written) */ - fixed(byte* inputPtr = &MemoryMarshal.GetReference(input), - outPtr = &MemoryMarshal.GetReference(output)) - { - //Create the operation struct - CompressionOperation operation; - CompressionOperation* op = &operation; - - op->flush = finalBlock ? 1 : 0; - op->bytesRead = 0; - op->bytesWritten = 0; - - //Configure the input and output buffers - op->inputBuffer = inputPtr; - op->inputSize = (uint)input.Length; - - op->outputBuffer = outPtr; - op->outputSize = (uint)output.Length; - - //Call the native compress function - nativeLib!.CompressBlock(comp, &operation); + //Create the operation struct + CompressionOperation operation = default; + operation.flush = finalBlock ? 1 : 0; - //Return the number of bytes written - return new() + checked + { + fixed (byte* inputPtr = &MemoryMarshal.GetReference(input), + outPtr = &MemoryMarshal.GetReference(output)) { - BytesRead = (int)op->bytesRead, - BytesWritten = (int)op->bytesWritten - }; + //Configure the input and output buffers + operation.inputBuffer = inputPtr; + operation.inputSize = (uint)input.Length; + + operation.outputBuffer = outPtr; + operation.outputSize = (uint)output.Length; + + //Call the native compress function + nativeLib!.CompressBlock(compressorInstance, &operation); + + return new() + { + BytesRead = (int)operation.bytesRead, + BytesWritten = (int)operation.bytesWritten + }; + } } } } diff --git a/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionLib.cs b/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionLib.cs index 438f777..0df8392 100644 --- a/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionLib.cs +++ b/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionLib.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Net.Compression @@ -94,16 +94,26 @@ namespace VNLib.Net.Compression public CompressionResult Compress(ReadOnlyMemory input, Memory output) { CompressorHandle.ThrowIfClosed(); - IntPtr compressor = CompressorHandle.DangerousGetHandle(); - return LibComp.CompressBlock(compressor, output, input, false); + + return LibComp.CompressBlock( + CompressorHandle.DangerousGetHandle(), + output, + input, + finalBlock: false + ); } /// public CompressionResult Compress(ReadOnlySpan input, Span output) { CompressorHandle.ThrowIfClosed(); - IntPtr compressor = CompressorHandle.DangerousGetHandle(); - return LibComp.CompressBlock(compressor, output, input, false); + + return LibComp.CompressBlock( + CompressorHandle.DangerousGetHandle(), + output, + input, + finalBlock: false + ); } /// @@ -113,8 +123,14 @@ namespace VNLib.Net.Compression public int Flush(Memory buffer) { CompressorHandle.ThrowIfClosed(); - IntPtr compressor = CompressorHandle.DangerousGetHandle(); - CompressionResult result = LibComp.CompressBlock(compressor, buffer, default, true); + + CompressionResult result = LibComp.CompressBlock( + CompressorHandle.DangerousGetHandle(), + buffer, + input: default, + finalBlock: true + ); + return result.BytesWritten; } @@ -122,8 +138,14 @@ namespace VNLib.Net.Compression public int Flush(Span buffer) { CompressorHandle.ThrowIfClosed(); - IntPtr compressor = CompressorHandle.DangerousGetHandle(); - CompressionResult result = LibComp.CompressBlock(compressor, buffer, default, true); + + CompressionResult result = LibComp.CompressBlock( + CompressorHandle.DangerousGetHandle(), + buffer, + input: default, + finalBlock: true + ); + return result.BytesWritten; } @@ -131,32 +153,42 @@ namespace VNLib.Net.Compression public uint GetBlockSize() { CompressorHandle.ThrowIfClosed(); - IntPtr compressor = CompressorHandle.DangerousGetHandle(); - return LibComp.GetBlockSize(compressor); + + return LibComp.GetBlockSize( + CompressorHandle.DangerousGetHandle() + ); } /// public uint GetCompressedSize(uint size) { CompressorHandle.ThrowIfClosed(); - IntPtr compressor = CompressorHandle.DangerousGetHandle(); - return (uint)LibComp.GetOutputSize(compressor, size, 1); + + return (uint)LibComp.GetOutputSize( + CompressorHandle.DangerousGetHandle(), + size, + flush: 1 //truthy enables flushing + ); } /// public CompressionLevel GetCompressionLevel() { CompressorHandle.ThrowIfClosed(); - IntPtr compressor = CompressorHandle.DangerousGetHandle(); - return LibComp.GetCompressorLevel(compressor); + + return LibComp.GetCompressorLevel( + CompressorHandle.DangerousGetHandle() + ); } /// public CompressionMethod GetCompressionMethod() { CompressorHandle.ThrowIfClosed(); - IntPtr compressor = CompressorHandle.DangerousGetHandle(); - return LibComp.GetCompressorType(compressor); + + return LibComp.GetCompressorType( + CompressorHandle.DangerousGetHandle() + ); } } } diff --git a/lib/Net.Compression/VNLib.Net.CompressionTests/VNLib.Net.CompressionTests.csproj b/lib/Net.Compression/VNLib.Net.CompressionTests/VNLib.Net.CompressionTests.csproj index 1698a8e..795975d 100644 --- a/lib/Net.Compression/VNLib.Net.CompressionTests/VNLib.Net.CompressionTests.csproj +++ b/lib/Net.Compression/VNLib.Net.CompressionTests/VNLib.Net.CompressionTests.csproj @@ -8,9 +8,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/lib/Net.Compression/vnlib_compress/CMakeLists.txt b/lib/Net.Compression/vnlib_compress/CMakeLists.txt index d0dbbb2..da27882 100644 --- a/lib/Net.Compression/vnlib_compress/CMakeLists.txt +++ b/lib/Net.Compression/vnlib_compress/CMakeLists.txt @@ -18,7 +18,10 @@ include(FetchContent) #the compression source file is required, all other sources will be added set(VNLIB_COMPRESS_SOURCES + src/util.h + src/platform.h src/compression.c + src/compression.h ) ############################### @@ -80,20 +83,6 @@ if(ENABLE_ZLIB) add_compile_definitions(VNLIB_COMPRESSOR_ZLIB_ENABLED) endif() -#Add support for rpmalloc memmory allocator -if(ENABLE_RPMALLOC) - - #enable greedy mode for rpmalloc - set(ENABLE_GREEDY ON) - - add_subdirectory( - ../../Utils.Memory/vnlib_rpmalloc/ - ${CMAKE_CURRENT_BINARY_DIR}/vnlib_rpmalloc/ - ) - - add_compile_definitions(VNLIB_CUSTOM_MALLOC_ENABLE) -endif() - ############################### # # CONFIGURE LIBRARY BUILD @@ -111,7 +100,7 @@ else() add_library(${_COMP_PROJ_NAME} STATIC ${VNLIB_COMPRESS_SOURCES}) endif() -target_compile_features(${_COMP_PROJ_NAME} PRIVATE c_std_90) #force compiler to use c90 standard for library +target_compile_features(${_COMP_PROJ_NAME} PRIVATE c_std_99) #force compiler to use c90 standard for library #if on unix lib will be appended, so we can adjust if(UNIX) @@ -135,15 +124,6 @@ if(ENABLE_ZLIB) target_link_libraries(${_COMP_PROJ_NAME} PRIVATE zlib) endif() -#link rpmalloc to the main project -if(ENABLE_RPMALLOC) - target_link_libraries(${_COMP_PROJ_NAME} PRIVATE vnlib_rpmalloc_static) - add_dependencies(${_COMP_PROJ_NAME} vnlib_rpmalloc_static) - - #Include the nativeheap api header - target_include_directories(${_COMP_PROJ_NAME} PRIVATE ../../Utils.Memory/vnlib_rpmalloc/) -endif() - #setup flags for windows compilation if(MSVC) target_compile_options( @@ -209,15 +189,31 @@ else() message(FATAL_ERROR "Unsupported compiler, sorry. Submit an issue for your platform and I'll work on it :)") endif() -if(NATIVE_HEAP_LIB_PATH) - - message(STATUS "Linking native heap library to the main project found at ${NATIVE_HEAP_LIB_PATH}") +if(NATIVE_HEAP_NAME) + + find_library( + _native_heap_lib + NAMES + ${NATIVE_HEAP_NAME}_static #attempt to load static library first + ${NATIVE_HEAP_NAME} + + HINTS + ${NATIVE_HEAP_SRC} + ${NATIVE_HEAP_SRC}/build + ${NATIVE_HEAP_SRC}/build/${CMAKE_BUILD_TYPE} + + NO_CACHE + REQUIRED + ) + + message(STATUS "Linking native heap library to the main project found at ${_native_heap_lib}") #Include the nativeheap api header include_directories(${NATIVE_HEAP_INCLUDES}) - + #If manual heap linking is enabled, we need to link the native heap library - target_link_libraries(${_COMP_PROJ_NAME} PRIVATE ${NATIVE_HEAP_LIB_PATH}) - target_compile_definitions(${_COMP_PROJ_NAME} PRIVATE VNLIB_CUSTOM_MALLOC_ENABLE) #configure src + target_link_libraries(${_COMP_PROJ_NAME} PRIVATE ${_native_heap_lib}) + + target_compile_definitions(${_COMP_PROJ_NAME} PRIVATE VNLIB_CUSTOM_MALLOC_ENABLE) #enable native heap memory overrides endif() \ No newline at end of file diff --git a/lib/Net.Compression/vnlib_compress/Taskfile.yaml b/lib/Net.Compression/vnlib_compress/Taskfile.yaml index 2133c0b..fec32d5 100644 --- a/lib/Net.Compression/vnlib_compress/Taskfile.yaml +++ b/lib/Net.Compression/vnlib_compress/Taskfile.yaml @@ -10,27 +10,44 @@ version: '3' vars: PROJECT_NAME: 'vnlib_compress' + RPMALLOC_SRC_DIR: '../../Utils.Memory/vnlib_rpmalloc' + BUILD_TYPE: '{{ .BUILD_TYPE | default "Release" }}' tasks: default: + desc: 'Builds the {{ .PROJECT_NAME }} library for the current platform' cmds: - cmd: echo "Building {{ .PROJECT_NAME }}" silent: true - - cmake -Bbuild/ -DCMAKE_BUILD_TYPE=Release {{ .CLI_ARGS }} + + - cmake -Bbuild/ {{ .CLI_ARGS }} + '-DCMAKE_BUILD_TYPE={{ .BUILD_TYPE }}' + - cmake --build build/ --config Release #called by ci pipline to build the winx64 project build: - cmds: + desc: 'DO NOT USE. This is an internal task' + platforms: [ windows ] + cmds: #the CI pipline may have issues reading modules if the third-party dir is not cleaned every time a build runs, only an issue after build - defer: { task: clean-third-party } - #invoke cmake for build (notify that we are precompiling for ci pipeline and rpmalloc lib should be local) - - cmake -B./build -DCI_PRECOMPILE=ON -DENABLE_RPMALLOC=ON - - #build for platform + #build the local rpmalloc library for linking + - task: build_rpmalloc + vars: { RPMALLOC_ARGS: '-DCMAKE_BUILD_TYPE={{ .BUILD_TYPE }}' } + + #configure the build with rpmalloc since we know the source must be local during CI + - cmake -B./build {{ .CLI_ARGS }} + '-DCMAKE_BUILD_TYPE={{ .BUILD_TYPE }}' + '-DCI_PRECOMPILE=ON' + '-DNATIVE_HEAP_NAME=vnlib_rpmalloc_static' + '-DNATIVE_HEAP_SRC={{ .RPMALLOC_SRC_DIR }}' + '-DNATIVE_HEAP_INCLUDES={{ .RPMALLOC_SRC_DIR }}' + + #build for platform, since Windows, build in both modes - cmake --build build/ --config debug - cmake --build build/ --config release @@ -40,6 +57,11 @@ tasks: - cmd: powershell mkdir -Force 'bin/' - task: pack_parallel + build_rpmalloc: + internal: true + cmds: + - cmd: cd {{ .RPMALLOC_SRC_DIR }} && task build -- {{ .RPMALLOC_ARGS }} + pack_parallel: internal: true vars: @@ -47,12 +69,14 @@ tasks: REL_TAR_FILES: "{{ .PROJECT_NAME }}.dll {{ .PROJECT_NAME }}.lib + platform.h compression.h license.txt" DEBUG_TAR_FILES: "{{ .PROJECT_NAME }}.dll {{ .PROJECT_NAME }}.pdb {{ .PROJECT_NAME }}.lib + platform.h compression.h license.txt" @@ -78,7 +102,9 @@ tasks: internal: true cmds: - powershell cp ../LICENSE '{{ .TARGET }}/license.txt' - - powershell cp src/compression.h '{{ .TARGET }}/compression.h' + + - for: [ platform.h, compression.h ] + cmd: powershell cp 'src/{{ .ITEM }}' '{{ .TARGET }}/{{ .ITEM }}' #packages source code for distribution pack_source: @@ -102,13 +128,14 @@ tasks: #Remove the output dirs on clean clean: + desc: 'Cleans any build artifcats and output directories' ignore_error: true cmds: - for: [ bin/, build/ ] cmd: powershell rm -Recurse '{{ .ITEM }}' -Force clean-third-party: - internal: false + internal: true ignore_error: true cmds: - cmd: powershell rm -Recurse -Force 'build/_deps/' diff --git a/lib/Net.Compression/vnlib_compress/src/compression.c b/lib/Net.Compression/vnlib_compress/src/compression.c index 0cba998..fc648d5 100644 --- a/lib/Net.Compression/vnlib_compress/src/compression.c +++ b/lib/Net.Compression/vnlib_compress/src/compression.c @@ -31,16 +31,16 @@ #define VNLIB_COMPRESS_EXPORTING 1 -#include "compression.h" #include "util.h" +#include "compression.h" #ifdef VNLIB_COMPRESSOR_BROTLI_ENABLED -#include "feature_brotli.h" + #include "feature_brotli.h" #endif /* VNLIB_COMPRESSOR_BROTLI_ENABLED */ #ifdef VNLIB_COMPRESSOR_ZLIB_ENABLED -#include "feature_zlib.h" + #include "feature_zlib.h" #endif /* VNLIB_COMPRESSOR_GZIP_ENABLED */ /* @@ -75,33 +75,32 @@ VNLIB_COMPRESS_EXPORT CompressorType VNLIB_COMPRESS_CC GetSupportedCompressors(v VNLIB_COMPRESS_EXPORT CompressorType VNLIB_COMPRESS_CC GetCompressorType(_In_ const void* compressor) { CHECK_NULL_PTR(compressor) - return ((CompressorState*)compressor)->type; + return ((_cmp_state_t*)compressor)->type; } VNLIB_COMPRESS_EXPORT CompressionLevel VNLIB_COMPRESS_CC GetCompressorLevel(_In_ const void* compressor) { CHECK_NULL_PTR(compressor) - return ((CompressorState*)compressor)->level; + return ((_cmp_state_t*)compressor)->level; } VNLIB_COMPRESS_EXPORT int64_t VNLIB_COMPRESS_CC GetCompressorBlockSize(_In_ const void* compressor) { CHECK_NULL_PTR(compressor) - return (int64_t)((CompressorState*)compressor)->blockSize; + return (int64_t)((_cmp_state_t*)compressor)->blockSize; } VNLIB_COMPRESS_EXPORT void* VNLIB_COMPRESS_CC AllocateCompressor(CompressorType type, CompressionLevel level) { - int result; - CompressorState* state; - /* Validate input arguments */ if (level < 0 || level > 9) { return (void*)ERR_COMP_LEVEL_NOT_SUPPORTED; } - state = (CompressorState*)vncalloc(1, sizeof(CompressorState)); + int result = ERR_COMP_TYPE_NOT_SUPPORTED; + + _cmp_state_t* state = (_cmp_state_t*)vncalloc(1, sizeof(_cmp_state_t)); if (!state) { @@ -110,9 +109,7 @@ VNLIB_COMPRESS_EXPORT void* VNLIB_COMPRESS_CC AllocateCompressor(CompressorType /* Configure the comp state */ state->type = type; - state->level = level; - - result = ERR_COMP_TYPE_NOT_SUPPORTED; + state->level = level; /* * Compressor types are defined at compile time @@ -184,13 +181,11 @@ VNLIB_COMPRESS_EXPORT void* VNLIB_COMPRESS_CC AllocateCompressor(CompressorType VNLIB_COMPRESS_EXPORT int VNLIB_COMPRESS_CC FreeCompressor(void* compressor) { - CompressorState* comp; - int errorCode; - CHECK_NULL_PTR(compressor) - comp = (CompressorState*)compressor; - errorCode = TRUE; + int errorCode = VNCMP_SUCCESS; + _cmp_state_t* comp = (_cmp_state_t*)compressor; + switch (comp->type) { @@ -232,21 +227,22 @@ VNLIB_COMPRESS_EXPORT int VNLIB_COMPRESS_CC FreeCompressor(void* compressor) return errorCode; } -VNLIB_COMPRESS_EXPORT int64_t VNLIB_COMPRESS_CC GetCompressedSize(_In_ const void* compressor, uint64_t inputLength, int32_t flush) +VNLIB_COMPRESS_EXPORT int64_t VNLIB_COMPRESS_CC GetCompressedSize( + _In_ const void* compressor, + uint64_t inputLength, + int32_t flush +) { - CompressorState* comp; - int64_t result; - - CHECK_NULL_PTR(compressor) + _cmp_state_t* comp = (_cmp_state_t*)compressor; + int64_t result = ERR_COMP_TYPE_NOT_SUPPORTED; + CHECK_NULL_PTR(compressor); + if (inputLength > INT64_MAX) { - return ERR_OVERFLOW; + return ERR_OUT_OF_BOUNDS; } - - comp = (CompressorState*)compressor; - result = ERR_COMP_TYPE_NOT_SUPPORTED; - + switch (comp->type) { @@ -286,10 +282,8 @@ VNLIB_COMPRESS_EXPORT int64_t VNLIB_COMPRESS_CC GetCompressedSize(_In_ const voi */ VNLIB_COMPRESS_EXPORT int VNLIB_COMPRESS_CC CompressBlock(_In_ const void* compressor, CompressionOperation* operation) { - int result; - CompressorState* comp; - - comp = (CompressorState*)compressor; + int result = ERR_INVALID_ARGUMENT; + _cmp_state_t* comp = (_cmp_state_t*)compressor; /* * Validate input arguments diff --git a/lib/Net.Compression/vnlib_compress/src/compression.h b/lib/Net.Compression/vnlib_compress/src/compression.h index 3d03145..cc0dcd5 100644 --- a/lib/Net.Compression/vnlib_compress/src/compression.h +++ b/lib/Net.Compression/vnlib_compress/src/compression.h @@ -35,18 +35,16 @@ #pragma once -#ifndef COMPRESSION_H_ -#define COMPRESSION_H_ +#ifndef _VNCMP_COMPRESSION_H_ +#define _VNCMP_COMPRESSION_H_ #include +#include "platform.h" -#if defined(_MSC_VER) || defined(WIN32) || defined(_WIN32) - #define _IS_WINDOWS -#endif /*Set api export calling convention(allow used to override)*/ #ifndef VNLIB_COMPRESS_CC - #ifdef _IS_WINDOWS + #ifdef _VNCMP_IS_WINDOWS /*STD for importing to other languages such as.NET*/ #define VNLIB_COMPRESS_CC __stdcall #else @@ -56,16 +54,16 @@ #ifndef VNLIB_COMPRESS_EXPORT /*Allow users to disable the export/impoty macro if using source code directly*/ #ifdef VNLIB_COMPRESS_EXPORTING - #ifdef _IS_WINDOWS + #ifdef _VNCMP_IS_WINDOWS #define VNLIB_COMPRESS_EXPORT __declspec(dllexport) #else #define VNLIB_COMPRESS_EXPORT __attribute__((visibility("default"))) #endif /* IS_WINDOWS */ #else - #ifdef _IS_WINDOWS + #ifdef _VNCMP_IS_WINDOWS #define VNLIB_COMPRESS_EXPORT __declspec(dllimport) #else - #define VNLIB_COMPRESS_EXPORT + #define VNLIB_COMPRESS_EXPORT extern #endif /* IS_WINDOWS */ #endif /* !VNLIB_EXPORTING */ #endif /* !VNLIB_EXPORT */ @@ -74,18 +72,22 @@ #define _In_ #endif +#define VNCMP_SUCCESS 1 + /* * ERRORS AND CONSTANTS */ -#define ERR_INVALID_PTR -1 -#define ERR_OUT_OF_MEMORY -2 - -#define ERR_COMP_TYPE_NOT_SUPPORTED -9 -#define ERR_COMP_LEVEL_NOT_SUPPORTED -10 -#define ERR_INVALID_INPUT_DATA -11 -#define ERR_INVALID_OUTPUT_DATA -12 -#define ERR_COMPRESSION_FAILED -13 -#define ERR_OVERFLOW -14 +#define ERR_INVALID_PTR -1 +#define ERR_OUT_OF_MEMORY -2 +#define ERR_OUT_OF_BOUNDS -3 +#define ERR_INVALID_ARGUMENT -4 + +#define ERR_COMP_TYPE_NOT_SUPPORTED -9 +#define ERR_COMP_LEVEL_NOT_SUPPORTED -10 +#define ERR_INVALID_INPUT_DATA -11 +#define ERR_INVALID_OUTPUT_DATA -12 +#define ERR_COMPRESSION_FAILED -13 +#define ERR_OVERFLOW -14 /* * Enumerated list of supported compression types for user selection @@ -93,11 +95,11 @@ */ typedef enum CompressorType { - COMP_TYPE_NONE = 0x00, - COMP_TYPE_GZIP = 0x01, - COMP_TYPE_DEFLATE = 0x02, - COMP_TYPE_BROTLI = 0x04, - COMP_TYPE_LZ4 = 0x08 + COMP_TYPE_NONE = 0x00, + COMP_TYPE_GZIP = 0x01, + COMP_TYPE_DEFLATE = 0x02, + COMP_TYPE_BROTLI = 0x04, + COMP_TYPE_LZ4 = 0x08 } CompressorType; @@ -111,31 +113,31 @@ typedef enum CompressionLevel The compression operation should be optimally compressed, even if the operation takes a longer time to complete. */ - COMP_LEVEL_OPTIMAL = 0, + COMP_LEVEL_OPTIMAL = 0, /* The compression operation should complete as quickly as possible, even if the resulting file is not optimally compressed. */ - COMP_LEVEL_FASTEST = 1, + COMP_LEVEL_FASTEST = 1, /* No compression should be performed on the file. */ - COMP_LEVEL_NO_COMPRESSION = 2, + COMP_LEVEL_NO_COMPRESSION = 2, /* The compression operation should create output as small as possible, even if the operation takes a longer time to complete. */ - COMP_LEVEL_SMALLEST_SIZE = 3 + COMP_LEVEL_SMALLEST_SIZE = 3 } CompressionLevel; typedef enum CompressorStatus { - COMPRESSOR_STATUS_READY = 0x00, - COMPRESSOR_STATUS_INITALIZED = 0x01, - COMPRESSOR_STATUS_NEEDS_FLUSH = 0x02 + COMPRESSOR_STATUS_READY = 0x00, + COMPRESSOR_STATUS_INITALIZED = 0x01, + COMPRESSOR_STATUS_NEEDS_FLUSH = 0x02 } CompressorStatus; -typedef struct CompressorStateStruct{ +typedef struct _vn_cmp_state_struct{ /* Pointer to the underlying compressor implementation. @@ -159,7 +161,7 @@ typedef struct CompressorStateStruct{ uint32_t blockSize; -} CompressorState; +} _cmp_state_t; /* * An extern caller generated structure passed to calls for @@ -179,10 +181,10 @@ typedef struct CompressionOperationStruct { /* * If the operation is a flush operation */ - const int32_t flush; + int32_t flush; - const uint32_t bytesInLength; - const uint32_t bytesOutLength; + uint32_t bytesInLength; + uint32_t bytesOutLength; /* * Results of the streaming operation @@ -248,7 +250,11 @@ VNLIB_COMPRESS_EXPORT int VNLIB_COMPRESS_CC FreeCompressor(void* compressor); * @param inputLength The length of the input data in bytes. * @return The maximum compressed size of the specified input data in bytes. */ -VNLIB_COMPRESS_EXPORT int64_t VNLIB_COMPRESS_CC GetCompressedSize(_In_ const void* compressor, uint64_t inputLength, int32_t flush); +VNLIB_COMPRESS_EXPORT int64_t VNLIB_COMPRESS_CC GetCompressedSize( + _In_ const void* compressor, + uint64_t inputLength, + int32_t flush +); /* @@ -258,6 +264,9 @@ VNLIB_COMPRESS_EXPORT int64_t VNLIB_COMPRESS_CC GetCompressedSize(_In_ const voi * @param operation A pointer to the compression operation structure * @return The underlying compressor's native return code */ -VNLIB_COMPRESS_EXPORT int VNLIB_COMPRESS_CC CompressBlock(_In_ const void* compressor, CompressionOperation* operation); +VNLIB_COMPRESS_EXPORT int VNLIB_COMPRESS_CC CompressBlock( + _In_ const void* compressor, + CompressionOperation* operation +); #endif /* !VNLIB_COMPRESS_MAIN_H_ */ \ No newline at end of file diff --git a/lib/Net.Compression/vnlib_compress/src/feature_brotli.c b/lib/Net.Compression/vnlib_compress/src/feature_brotli.c index 361c61a..80c0de4 100644 --- a/lib/Net.Compression/vnlib_compress/src/feature_brotli.c +++ b/lib/Net.Compression/vnlib_compress/src/feature_brotli.c @@ -21,10 +21,9 @@ #include #include "feature_brotli.h" -#include "util.h" #define validateCompState(state) \ - if (!state) return ERR_INVALID_PTR; \ + CHECK_NULL_PTR(state); \ if (!state->compressor) return ERR_BR_INVALID_STATE; \ /* @@ -32,13 +31,13 @@ */ static void* _brAllocCallback(void* opaque, size_t size) { - (void)opaque; + (void)sizeof(opaque); return vnmalloc(size, 1); } static void _brFreeCallback(void* opaque, void* address) { - (void)opaque; + (void)sizeof(opaque); /*Brotli may pass a null address to the free callback*/ if (address) @@ -48,11 +47,11 @@ static void _brFreeCallback(void* opaque, void* address) } -int BrAllocCompressor(CompressorState* state) +int BrAllocCompressor(_cmp_state_t* state) { BrotliEncoderState* comp; - assert(state != NULL); + DEBUG_ASSERT2(state != NULL, "Expected non-null compressor state argument"); /* * Never allow no compression, it is not supported by the br encoder @@ -119,12 +118,12 @@ int BrAllocCompressor(CompressorState* state) break; } - return TRUE; + return VNCMP_SUCCESS; } -void BrFreeCompressor(CompressorState* state) +void BrFreeCompressor(_cmp_state_t* state) { - assert(state != NULL); + DEBUG_ASSERT2(state != NULL, "Expected non-null state parameter"); /* * Free the compressor instance if it exists @@ -136,7 +135,7 @@ void BrFreeCompressor(CompressorState* state) } } -int BrCompressBlock(const CompressorState* state, CompressionOperation* operation) +int BrCompressBlock(_In_ const _cmp_state_t* state, CompressionOperation* operation) { BrotliEncoderOperation brOperation; BROTLI_BOOL brResult; @@ -145,6 +144,8 @@ int BrCompressBlock(const CompressorState* state, CompressionOperation* operatio const uint8_t* nextIn; uint8_t* nextOut; + DEBUG_ASSERT2(operation != NULL, "Expected non-null operation parameter"); + /* Validate inputs */ validateCompState(state) @@ -159,7 +160,7 @@ int BrCompressBlock(const CompressorState* state, CompressionOperation* operatio if (operation->bytesInLength == 0 && operation->flush < 1) { - return TRUE; + return VNCMP_SUCCESS; } /* @@ -221,7 +222,7 @@ int BrCompressBlock(const CompressorState* state, CompressionOperation* operatio } -int64_t BrGetCompressedSize(const CompressorState* state, uint64_t length, int32_t flush) +int64_t BrGetCompressedSize(_In_ const _cmp_state_t* state, uint64_t length, int32_t flush) { size_t size; diff --git a/lib/Net.Compression/vnlib_compress/src/feature_brotli.h b/lib/Net.Compression/vnlib_compress/src/feature_brotli.h index 1f2090b..6b4bbca 100644 --- a/lib/Net.Compression/vnlib_compress/src/feature_brotli.h +++ b/lib/Net.Compression/vnlib_compress/src/feature_brotli.h @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: vnlib_compress @@ -24,6 +24,7 @@ #ifndef BROTLI_STUB_H_ #define BROTLI_STUB_H_ +#include "util.h" #include "compression.h" #define ERR_BR_INVALID_STATE -24 @@ -35,12 +36,12 @@ #define BR_DEFAULT_WINDOW 22 -int BrAllocCompressor(CompressorState* state); +int BrAllocCompressor(_cmp_state_t* state); -void BrFreeCompressor(CompressorState* state); +void BrFreeCompressor(_cmp_state_t* state); -int BrCompressBlock(const CompressorState* state, CompressionOperation* operation); +int BrCompressBlock(_In_ const _cmp_state_t* state, CompressionOperation* operation); -int64_t BrGetCompressedSize(const CompressorState* state, uint64_t length, int32_t flush); +int64_t BrGetCompressedSize(_In_ const _cmp_state_t* state, uint64_t length, int32_t flush); #endif /* !BROTLI_STUB_H_ */ \ No newline at end of file diff --git a/lib/Net.Compression/vnlib_compress/src/feature_zlib.c b/lib/Net.Compression/vnlib_compress/src/feature_zlib.c index a07f106..2358cb1 100644 --- a/lib/Net.Compression/vnlib_compress/src/feature_zlib.c +++ b/lib/Net.Compression/vnlib_compress/src/feature_zlib.c @@ -26,7 +26,6 @@ #include #include "feature_zlib.h" -#include "util.h" #define validateCompState(state) \ if (!state) return ERR_INVALID_PTR; \ @@ -37,28 +36,27 @@ */ static void* _gzAllocCallback(void* opaque, uint32_t items, uint32_t size) { - (void)opaque; + (void)sizeof(opaque); return vnmalloc(items, size); } static void _gzFreeCallback(void* opaque, void* address) { - (void)opaque; + (void)sizeof(opaque); vnfree(address); } -int DeflateAllocCompressor(CompressorState* state) +int DeflateAllocCompressor(_cmp_state_t* state) { int result, compLevel; - z_stream* stream; - assert(state); + DEBUG_ASSERT2(state, "Expected non-null state parameter"); /* * Allocate the z-stream state on the heap so we can * store it in the compressor state */ - stream = (z_stream*)vncalloc(1, sizeof(z_stream)); + z_stream* stream = (z_stream*)vncalloc(1, sizeof(z_stream)); if (!stream) { @@ -146,14 +144,14 @@ int DeflateAllocCompressor(CompressorState* state) * Assign the z-stream state to the compressor state, all done! */ state->compressor = stream; - return TRUE; + return VNCMP_SUCCESS; } -int DeflateFreeCompressor(CompressorState* state) +int DeflateFreeCompressor(_cmp_state_t* state) { int result; - assert(state); + DEBUG_ASSERT2(state != NULL, "Expected non-null compressor state"); /* * Free the z-stream state, only if the compressor is initialized @@ -186,12 +184,11 @@ int DeflateFreeCompressor(CompressorState* state) return result == Z_OK || result == Z_DATA_ERROR; } - return TRUE; + return VNCMP_SUCCESS; } -int DeflateCompressBlock(const CompressorState* state, CompressionOperation* operation) +int DeflateCompressBlock(_In_ const _cmp_state_t* state, CompressionOperation* operation) { - z_stream* stream; int result; validateCompState(state) @@ -207,10 +204,10 @@ int DeflateCompressBlock(const CompressorState* state, CompressionOperation* ope if (operation->bytesInLength == 0 && operation->flush < 1) { - return TRUE; + return VNCMP_SUCCESS; } - stream = (z_stream*)state->compressor; + z_stream* stream = (z_stream*)state->compressor; /* * Overwrite the stream state with the operation parameters from @@ -264,7 +261,7 @@ int DeflateCompressBlock(const CompressorState* state, CompressionOperation* ope return result; } -int64_t DeflateGetCompressedSize(const CompressorState* state, uint64_t length, int32_t flush) +int64_t DeflateGetCompressedSize(_In_ const _cmp_state_t* state, uint64_t length, int32_t flush) { uint64_t compressedSize; diff --git a/lib/Net.Compression/vnlib_compress/src/feature_zlib.h b/lib/Net.Compression/vnlib_compress/src/feature_zlib.h index 2544d25..089c3a0 100644 --- a/lib/Net.Compression/vnlib_compress/src/feature_zlib.h +++ b/lib/Net.Compression/vnlib_compress/src/feature_zlib.h @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: vnlib_compress @@ -24,6 +24,7 @@ #ifndef ZLIB_STUB_H_ #define ZLIB_STUB_H_ +#include "util.h" #include "compression.h" #define ERR_GZ_INVALID_STATE -16 @@ -31,20 +32,19 @@ /* Allow user to define their own memory level value */ #ifndef GZ_DEFAULT_MEM_LEVEL -#define GZ_DEFAULT_MEM_LEVEL 8 + #define GZ_DEFAULT_MEM_LEVEL 8 #endif /* Specifies the window value to enable GZIP */ #define GZ_ENABLE_GZIP_WINDOW 15 + 16 #define GZ_ENABLE_RAW_DEFLATE_WINDOW -15 +int DeflateAllocCompressor(_cmp_state_t* state); -int DeflateAllocCompressor(CompressorState* state); +int DeflateFreeCompressor(_cmp_state_t* state); -int DeflateFreeCompressor(CompressorState* state); +int DeflateCompressBlock(_In_ const _cmp_state_t* state, CompressionOperation* operation); -int DeflateCompressBlock(const CompressorState* state, CompressionOperation* operation); - -int64_t DeflateGetCompressedSize(const CompressorState* state, uint64_t length, int32_t flush); +int64_t DeflateGetCompressedSize(_In_ const _cmp_state_t* state, uint64_t length, int32_t flush); #endif diff --git a/lib/Net.Compression/vnlib_compress/src/platform.h b/lib/Net.Compression/vnlib_compress/src/platform.h new file mode 100644 index 0000000..b05c8b8 --- /dev/null +++ b/lib/Net.Compression/vnlib_compress/src/platform.h @@ -0,0 +1,90 @@ +/* +* Copyright(c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: vnlib_compress +* File: platform.h +* +* vnlib_compress is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* vnlib_compress is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with vnlib_compress. If not, see http://www.gnu.org/licenses/. +*/ + + +/* +* Contains platform specific defintions +*/ + +#pragma once + +#ifndef _VNCMP_PLATFORM_H +#define _VNCMP_PLATFORM_H + +#if defined(_MSC_VER) || defined(WIN32) || defined(_WIN32) + #define _VNCMP_IS_WINDOWS +#elif defined(__linux__) || defined(__unix__) || defined(__posix__) + #define _VNCMP_IS_LINUX +#elif defined(__APPLE__) || defined(__MACH__) + #define _VNCMP_IS_MAC +#endif + +/* +* Define supported inline defintions for various compilers +* and C standards +*/ + +#if defined(_VNCMP_IS_WINDOWS) || defined(inline) || defined(__clang__) + #define _vncmp_inline inline +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 allows usage of inline keyword */ + #define _vncmp_inline inline +#elif defined(__GNUC__) || defined(__GNUG__) + #define _vncmp_inline __inline__ +#else + #define _vncmp_inline + #pragma message("Warning: No inline keyword defined for this compiler") +#endif + +/* NULL */ +#ifndef NULL + #define NULL ((void*)0) +#endif /* !NULL */ + +#ifdef DEBUG + /* Must include assert.h for assertions */ + #include + #define DEBUG_ASSERT(x) assert(x); + #define DEBUG_ASSERT2(x, message) assert(x && message); + + /* + * Compiler enabled static assertion keywords are + * only available in C11 and later. Later versions + * have macros built-in from assert.h so we can use + * the static_assert macro directly. + * + * Static assertions are only used for testing such as + * sanity checks and this library targets the c89 standard + * so static_assret very likely will not be available. + */ + #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L + #define STATIC_ASSERT(x, m) static_assert(x, m); + #elif !defined(STATIC_ASSERT) + #define STATIC_ASSERT(x, m) + #pragma message("Static assertions are not supported by this language version") + #endif + +#else + #define DEBUG_ASSERT(x) + #define DEBUG_ASSERT2(x, message) + #define STATIC_ASSERT(x, m) +#endif + +#endif // !VNCP_PLATFORM_H diff --git a/lib/Net.Compression/vnlib_compress/src/util.h b/lib/Net.Compression/vnlib_compress/src/util.h index 292e3bf..d1a1ddd 100644 --- a/lib/Net.Compression/vnlib_compress/src/util.h +++ b/lib/Net.Compression/vnlib_compress/src/util.h @@ -21,8 +21,10 @@ #pragma once -#ifndef UTIL_H_ -#define UTIL_H_ +#ifndef _VNCMP_UTIL_H_ +#define _VNCMP_UTIL_H_ + +#include "platform.h" /* * If a custom allocator is enabled, use the native heap api @@ -30,48 +32,13 @@ * will be enabled when heapapi.h is included. */ #ifdef VNLIB_CUSTOM_MALLOC_ENABLE - /* Since static linking ie snabled, heapapi must have extern symbol not dllimport */ + # /* Since static linking ie snabled, heapapi must have extern symbol not dllimport */ #define VNLIB_HEAP_API extern #include #endif -#if defined(_MSC_VER) || defined(WIN32) || defined(_WIN32) - #define IS_WINDOWS -#endif - -#if defined(IS_WINDOWS) || defined(inline) || defined(__clang__) - #define _cp_fn_inline inline -#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 allows usage of inline keyword */ - #define _cp_fn_inline inline -#elif defined(__GNUC__) || defined(__GNUG__) - #define _cp_fn_inline __inline__ -#else - #define _cp_fn_inline - #pragma message("Warning: No inline keyword defined for this compiler") -#endif - -#ifndef NULL - #define NULL ((void*)0) -#endif /* !NULL */ - -#ifndef TRUE - #define TRUE 1 -#endif /* !TRUE */ - -#ifndef FALSE - #define FALSE 0 -#endif /* !FALSE */ - -/* -* Add debug runtime assertions -*/ -#ifdef DEBUG - #include -#else - #define assert(x) {} -#endif - #define CHECK_NULL_PTR(ptr) if(!ptr) return ERR_INVALID_PTR; +#define CHECK_ARG_RANGE(x, min, max) if(x < min || x > max) return ERR_OUT_OF_BOUNDS; #ifdef NATIVE_HEAP_API /* Defined in the NativeHeapApi */ @@ -85,32 +52,22 @@ * api consistency. */ - static _cp_fn_inline void* vnmalloc(size_t num, size_t size) + static _vncmp_inline void* vnmalloc(size_t num, size_t size) { - return heapAlloc(heapGetSharedHeapHandle(), num, size, FALSE); + return heapAlloc(heapGetSharedHeapHandle(), num, size, 0); } - static _cp_fn_inline void* vncalloc(size_t num, size_t size) + static _vncmp_inline void* vncalloc(size_t num, size_t size) { - return heapAlloc(heapGetSharedHeapHandle(), num, size, TRUE); + return heapAlloc(heapGetSharedHeapHandle(), num, size, 1); } - static _cp_fn_inline void vnfree(void* ptr) + static _vncmp_inline void vnfree(void* ptr) { - #ifdef DEBUG - - ERRNO result; - result = heapFree(heapGetSharedHeapHandle(), ptr); + ERRNO result = heapFree(heapGetSharedHeapHandle(), ptr); /* track failed free results */ - assert(result > 0); - - #else - - heapFree(heapGetSharedHeapHandle(), ptr); - - #endif - + DEBUG_ASSERT(result > 0); } #else diff --git a/lib/Net.Compression/vnlib_compress/src/vnlib_compress.vcxitems b/lib/Net.Compression/vnlib_compress/src/vnlib_compress.vcxitems index 7f5eb0e..b871fde 100644 --- a/lib/Net.Compression/vnlib_compress/src/vnlib_compress.vcxitems +++ b/lib/Net.Compression/vnlib_compress/src/vnlib_compress.vcxitems @@ -21,6 +21,7 @@ + diff --git a/lib/Net.Rest.Client/src/VNLib.Net.Rest.Client.csproj b/lib/Net.Rest.Client/src/VNLib.Net.Rest.Client.csproj index 37ae341..7b11a6e 100644 --- a/lib/Net.Rest.Client/src/VNLib.Net.Rest.Client.csproj +++ b/lib/Net.Rest.Client/src/VNLib.Net.Rest.Client.csproj @@ -35,7 +35,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/lib/Plugins.Essentials.ServiceStack/src/Construction/HttpServiceStackBuilder.cs b/lib/Plugins.Essentials.ServiceStack/src/Construction/HttpServiceStackBuilder.cs index 34c68bf..5367b5c 100644 --- a/lib/Plugins.Essentials.ServiceStack/src/Construction/HttpServiceStackBuilder.cs +++ b/lib/Plugins.Essentials.ServiceStack/src/Construction/HttpServiceStackBuilder.cs @@ -54,7 +54,7 @@ namespace VNLib.Plugins.Essentials.ServiceStack.Construction private Action? _hostBuilder; private Func, IHttpServer[]>? _getServers; private Func? _getPlugins; - private IManualPlugin[]? manualPlugins; + private Action>? _addManualPlugins; private bool loadConcurrently; /// @@ -129,9 +129,9 @@ namespace VNLib.Plugins.Essentials.ServiceStack.Construction /// /// The array of plugins (or params) to add /// The current instance for chaining - public HttpServiceStackBuilder WithManualPlugins(params IManualPlugin[] plugins) + public HttpServiceStackBuilder WithManualPlugins(Action>? plugins) { - manualPlugins = plugins; + _addManualPlugins = plugins; return this; } @@ -181,8 +181,10 @@ namespace VNLib.Plugins.Essentials.ServiceStack.Construction private PluginStackInitializer GetPluginStack(ServiceDomain domain) { - //Always init manual array - manualPlugins ??= []; + List _manualPlugins = []; + + //Add manual plugins if configured + _addManualPlugins?.Invoke(_manualPlugins); //Only load plugins if the callback is configured IPluginStack? plugins = _getPlugins?.Invoke(); @@ -191,7 +193,7 @@ namespace VNLib.Plugins.Essentials.ServiceStack.Construction plugins ??= new EmptyPluginStack(); #pragma warning restore CA2000 // Dispose objects before losing scope - return new (domain.GetListener(), plugins, manualPlugins, loadConcurrently); + return new(domain.GetListener(), plugins, [.. _manualPlugins], loadConcurrently); } /* diff --git a/lib/Utils/src/Extensions/MemoryExtensions.cs b/lib/Utils/src/Extensions/MemoryExtensions.cs index f2399a2..0cee73d 100644 --- a/lib/Utils/src/Extensions/MemoryExtensions.cs +++ b/lib/Utils/src/Extensions/MemoryExtensions.cs @@ -339,7 +339,8 @@ namespace VNLib.Utils.Extensions /// The new within the block /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static SubSequence GetSubSequence(this IMemoryHandle block, nuint offset, int size) => new (block, offset, size); + public static SubSequence GetSubSequence(this IMemoryHandle block, nuint offset, int size) + => new (block, offset, size); /// /// Gets a window within the current block @@ -809,7 +810,8 @@ namespace VNLib.Utils.Extensions /// [MethodImpl(MethodImplOptions.AggressiveInlining)] [Obsolete("Functions are included directly on the type now")] - public static Span AsSpan(this in UnsafeMemoryHandle handle, int start) where T: unmanaged => handle.Span[start..]; + public static Span AsSpan(this in UnsafeMemoryHandle handle, int start) where T: unmanaged + => handle.Span[start..]; /// /// Creates a new sub-sequence over the target handle. (allows for convient sub span) @@ -822,7 +824,8 @@ namespace VNLib.Utils.Extensions /// [MethodImpl(MethodImplOptions.AggressiveInlining)] [Obsolete("Functions are included directly on the type now")] - public static Span AsSpan(this in UnsafeMemoryHandle handle, int start, int count) where T : unmanaged => handle.Span.Slice(start, count); + public static Span AsSpan(this in UnsafeMemoryHandle handle, int start, int count) where T : unmanaged + => handle.Span.Slice(start, count); /// /// Raises an if the current handle diff --git a/lib/Utils/src/Memory/MemoryHandle.cs b/lib/Utils/src/Memory/MemoryHandle.cs index fbaae95..48e8193 100644 --- a/lib/Utils/src/Memory/MemoryHandle.cs +++ b/lib/Utils/src/Memory/MemoryHandle.cs @@ -206,7 +206,8 @@ namespace VNLib.Utils.Memory /// The reference to the item at the desired offset /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public unsafe ref T GetOffsetRef(nuint offset) => ref Unsafe.AsRef(GetOffset(offset)); + public unsafe ref T GetOffsetRef(nuint offset) + => ref Unsafe.AsRef(GetOffset(offset)); /// /// @@ -256,9 +257,11 @@ namespace VNLib.Utils.Memory } /// - public override bool Equals(object? obj) => obj is MemoryHandle oHandle && Equals(oHandle); + public override bool Equals(object? obj) + => obj is MemoryHandle oHandle && Equals(oHandle); /// - public override int GetHashCode() => HashCode.Combine(base.GetHashCode(), handle.GetHashCode(), _length); + public override int GetHashCode() + => HashCode.Combine(base.GetHashCode(), handle.GetHashCode(), _length); } } \ No newline at end of file diff --git a/lib/Utils/src/Memory/MemoryUtil.cs b/lib/Utils/src/Memory/MemoryUtil.cs index 09c36d5..1d3bccb 100644 --- a/lib/Utils/src/Memory/MemoryUtil.cs +++ b/lib/Utils/src/Memory/MemoryUtil.cs @@ -159,8 +159,6 @@ namespace VNLib.Utils.Memory private static IUnmangedHeap InitHeapInternal(bool isShared, bool enableStats, bool globalZero) { - bool IsWindows = OperatingSystem.IsWindows(); - //Get environment varable string? heapDllPath = Environment.GetEnvironmentVariable(SHARED_HEAP_FILE_PATH); string? rawFlagsEnv = Environment.GetEnvironmentVariable(SHARED_HEAP_RAW_FLAGS); @@ -195,8 +193,8 @@ namespace VNLib.Utils.Memory //Attempt to load the heap heap = NativeHeap.LoadHeap(heapDllPath, DllImportSearchPath.SafeDirectories, cFlags, userFlags); } - //No user heap was specified, use fallback - else if (IsWindows) + //No user heap was specified, use fallback on windows + else if (OperatingSystem.IsWindows()) { //We can use win32 heaps diff --git a/lib/Utils/src/Memory/MemoryUtilAlloc.cs b/lib/Utils/src/Memory/MemoryUtilAlloc.cs index 742346c..fb0f371 100644 --- a/lib/Utils/src/Memory/MemoryUtilAlloc.cs +++ b/lib/Utils/src/Memory/MemoryUtilAlloc.cs @@ -483,6 +483,68 @@ namespace VNLib.Utils.Memory return SafeAlloc(elements: (int)NearestPage(elements), zero); } + /// + /// DANGEROUS: Creates a new from an existing pointer that + /// belongs to the specified heap. This handle owns the pointer and will free it back + /// to the heap when disposed. No validation is performed on the pointer, so it is + /// the responsibility of the caller to ensure the pointer is valid and points to a + /// block of memory that was allocated from the specified heap. + /// + /// The pointer to the existing block of memory + /// The element size of the block (NOT byte size) + /// The unmanaged heap this block was allocated from (and can be freed back to) + /// A flag that indicates if block should be zeroed during reallocations + /// A wrapper + public static MemoryHandle ToHandle(nint blockPtr, nuint elements, IUnmangedHeap heap, bool zero) + where T : unmanaged + { + ArgumentNullException.ThrowIfNull(heap); + + return new MemoryHandle(heap, blockPtr, elements, zero); + } + + /// + /// DANGEROUS: Creates a new from an existing pointer that + /// belongs to the specified heap. This handle owns the pointer and will free it back + /// to the heap when disposed. No validation is performed on the pointer, so it is + /// the responsibility of the caller to ensure the pointer is valid and points to a + /// block of memory that was allocated from the specified heap. + /// + /// The pointer to the existing block of memory + /// The element size of the block (NOT byte size) + /// The unmanaged heap this block was allocated from (and can be freed back to) + /// A wrapper + public static UnsafeMemoryHandle ToUnsafeHandle(nint blockPtr, int elements, IUnmangedHeap heap) + where T : unmanaged + { + ArgumentNullException.ThrowIfNull(heap); + + return new UnsafeMemoryHandle(heap, blockPtr, elements); + } + + /// + /// Determines if two memory handles are equal by comparing their + /// lengths and their base block addresses. This method is useful + /// for comparing memory handles that may have been reallocated + /// or copied. + /// + /// + /// The first handle to compare + /// The secondary handle to compare against + /// + /// True if the handle lengths are equal and the base block addresses + /// are the same, otherwise false. + /// + /// + public static bool AreHandlesEqual(IMemoryHandle handle1, IMemoryHandle handle2) + { + ArgumentNullException.ThrowIfNull(handle1); + ArgumentNullException.ThrowIfNull(handle2); + + return handle1.Length == handle2.Length + && Unsafe.AreSame(ref handle1.GetReference(), ref handle2.GetReference()); + } + #endregion } diff --git a/lib/Utils/src/Memory/UnsafeMemoryHandle.cs b/lib/Utils/src/Memory/UnsafeMemoryHandle.cs index 4041d0b..c968e7f 100644 --- a/lib/Utils/src/Memory/UnsafeMemoryHandle.cs +++ b/lib/Utils/src/Memory/UnsafeMemoryHandle.cs @@ -253,11 +253,15 @@ namespace VNLib.Utils.Memory /// /// The other handle to test /// True if the other handle points to the same block of memory as the current handle + /// + /// Memory handles are the same if - The use the same underlying allocation type, handle size is the same + /// and the address of the block is the same. + /// public readonly bool Equals(in UnsafeMemoryHandle other) { return _handleType == other._handleType - && Length == other.Length - && GetHashCode() == other.GetHashCode(); + && _length == other._length + && Unsafe.AreSame(ref GetReference(), ref other.GetReference()); } /// @@ -266,7 +270,12 @@ namespace VNLib.Utils.Memory /// /// The other handle to test /// True if the other handle points to the same block of memory as the current handle - public readonly bool Equals(UnsafeMemoryHandle other) => Equals(in other); + /// + /// Memory handles are the same if - The use the same underlying allocation type, handle size is the same + /// and the address of the block is the same. + /// + public readonly bool Equals(UnsafeMemoryHandle other) + => Equals(in other); /// /// Override for object equality operator, will cause boxing @@ -275,10 +284,14 @@ namespace VNLib.Utils.Memory /// The other object to compare /// /// True if the passed object is of type - /// and uses the structure equality operator - /// false otherwise. + /// and uses the structure equality operator false otherwise. /// - public readonly override bool Equals([NotNullWhen(true)] object? obj) => obj is UnsafeMemoryHandle other && Equals(in other); + /// + /// Memory handles are the same if - The use the same underlying allocation type, handle size is the same + /// and the address of the block is the same. + /// + public readonly override bool Equals([NotNullWhen(true)] object? obj) + => obj is UnsafeMemoryHandle other && Equals(in other); /// /// Equality overload @@ -286,7 +299,8 @@ namespace VNLib.Utils.Memory /// /// /// True if handles are equal, flase otherwise - public static bool operator ==(in UnsafeMemoryHandle left, in UnsafeMemoryHandle right) => left.Equals(in right); + public static bool operator ==(in UnsafeMemoryHandle left, in UnsafeMemoryHandle right) + => left.Equals(in right); /// /// Equality overload @@ -294,7 +308,8 @@ namespace VNLib.Utils.Memory /// /// /// True if handles are equal, flase otherwise - public static bool operator !=(in UnsafeMemoryHandle left, in UnsafeMemoryHandle right) => !left.Equals(in right); + public static bool operator !=(in UnsafeMemoryHandle left, in UnsafeMemoryHandle right) + => !left.Equals(in right); } } diff --git a/lib/Utils/src/Memory/Win32PrivateHeap.cs b/lib/Utils/src/Memory/Win32PrivateHeap.cs index d2ab201..b851dd2 100644 --- a/lib/Utils/src/Memory/Win32PrivateHeap.cs +++ b/lib/Utils/src/Memory/Win32PrivateHeap.cs @@ -31,7 +31,7 @@ using System.Runtime.CompilerServices; using VNLib.Utils.Native; using DWORD = System.Int64; -using LPVOID = System.IntPtr; +using LPVOID = nint; namespace VNLib.Utils.Memory { @@ -112,37 +112,43 @@ namespace VNLib.Utils.Memory { if (cFlags.HasFlag(HeapCreation.Shared)) { - //Clear the synchronization flag because we don't need it for a process heap + /* + * When using the process-wide heap, synchronization is enabled + * so we should clear the flag to prevent UnmanagedHeapBase from + * using CLR mutual exclusion methods + */ cFlags &= ~HeapCreation.UseSynchronization; - //Get the process heap - LPVOID handle = GetProcessHeap(); - - //The heap does not own the handle - return new Win32PrivateHeap(handle, cFlags, false); + return new Win32PrivateHeap( + GetProcessHeap(), + cFlags, + ownsHeandle: false //The heap does not own the handle because it's shared, so it cannot be freed + ); } if (cFlags.HasFlag(HeapCreation.UseSynchronization)) { /* * When the synchronization flag is set, we dont need to use - * the win32 serialization method + * the win32 serialization method because the UnmanagedHeapBase + * class will handle locking using better/faster CLR methods */ flags |= HEAP_NO_SERIALIZE; } //Call create, throw exception if the heap falled to allocate - ERRNO heapHandle = HeapCreate(flags, initialSize, maxHeapSize); + ERRNO heapHandle = (ERRNO)HeapCreate(flags, initialSize, maxHeapSize); if (!heapHandle) { throw new NativeMemoryException("Heap could not be created"); } + Trace.WriteLine($"Win32 private heap {heapHandle:x} created"); //Heap has been created so we can wrap it - return new(heapHandle, cFlags, true); + return new(heapHandle, cFlags, ownsHeandle: true); } /// @@ -152,9 +158,11 @@ namespace VNLib.Utils.Memory /// An open and valid handle to a win32 private heap /// The heap creation flags to obey /// A wrapper around the specified heap - public static Win32PrivateHeap ConsumeExisting(IntPtr win32HeapHandle, HeapCreation flags) => new (win32HeapHandle, flags, true); + public static Win32PrivateHeap ConsumeExisting(IntPtr win32HeapHandle, HeapCreation flags) + => new (win32HeapHandle, flags, ownsHeandle: true); - private Win32PrivateHeap(IntPtr heapPtr, HeapCreation flags, bool ownsHeandle) : base(flags, ownsHeandle) => handle = heapPtr; + private Win32PrivateHeap(IntPtr heapPtr, HeapCreation flags, bool ownsHeandle) : base(flags, ownsHeandle) + => handle = heapPtr; /// /// Retrieves the size of a memory block allocated from the current heap. @@ -170,14 +178,12 @@ namespace VNLib.Utils.Memory /// True if the block is valid, false otherwise public bool Validate(ref LPVOID block) { - bool result; //Lock the heap before validating lock (HeapLock) { //validate the block on the current heap - result = HeapValidate(handle, HEAP_NO_FLAGS, block); + return HeapValidate(handle, HEAP_NO_FLAGS, block); } - return result; } /// @@ -188,15 +194,12 @@ namespace VNLib.Utils.Memory /// This can be a consuming operation which will block all allocations public bool Validate() { - bool result; - //Lock the heap before validating lock (HeapLock) { //validate the entire heap - result = HeapValidate(handle, HEAP_NO_FLAGS, IntPtr.Zero); + return HeapValidate(handle, HEAP_NO_FLAGS, IntPtr.Zero); } - return result; } /// @@ -211,22 +214,28 @@ namespace VNLib.Utils.Memory [MethodImpl(MethodImplOptions.AggressiveInlining)] protected override sealed LPVOID AllocBlock(nuint elements, nuint size, bool zero) { - nuint bytes = checked(elements * size); - - return HeapAlloc(handle, zero ? HEAP_ZERO_MEMORY : HEAP_NO_FLAGS, bytes); + return HeapAlloc( + handle, + flags: zero ? HEAP_ZERO_MEMORY : HEAP_NO_FLAGS, + dwBytes: checked(elements * size) + ); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - protected override sealed bool FreeBlock(LPVOID block) => HeapFree(handle, HEAP_NO_FLAGS, block); + protected override sealed bool FreeBlock(LPVOID block) + => HeapFree(handle, HEAP_NO_FLAGS, block); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] protected override sealed LPVOID ReAllocBlock(LPVOID block, nuint elements, nuint size, bool zero) { - nuint bytes = checked(elements * size); - - return HeapReAlloc(handle, zero ? HEAP_ZERO_MEMORY : HEAP_NO_FLAGS, block, bytes); + return HeapReAlloc( + handle, + dwFlags: zero ? HEAP_ZERO_MEMORY : HEAP_NO_FLAGS, + lpMem: block, + dwBytes: checked(elements * size) + ); } } } \ No newline at end of file diff --git a/lib/Utils/tests/VNLib.UtilsTests.csproj b/lib/Utils/tests/VNLib.UtilsTests.csproj index 0a425a9..ba5a477 100644 --- a/lib/Utils/tests/VNLib.UtilsTests.csproj +++ b/lib/Utils/tests/VNLib.UtilsTests.csproj @@ -16,9 +16,9 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive -- cgit