From cdeda79bc7d358c617b05b17d24f3f3c79689379 Mon Sep 17 00:00:00 2001 From: vnugent Date: Wed, 13 Sep 2023 11:00:17 -0400 Subject: Overhaul cmake, fix some warnings, and reconfigure proper compiler options with hardware support --- .../VNLib.Net.Compression/CompressionExtensions.cs | 16 +- .../VNLib.Net.Compression/CompressionOperation.cs | 27 +-- .../VNLib.Net.Compression/CompressorManager.cs | 2 +- .../VNLib.Net.Compression/INativeCompressor.cs | 7 +- .../VNLib.Net.Compression/LibraryWrapper.cs | 40 +++-- .../VNLib.Net.Compression/NativeCompressionLib.cs | 43 +++-- .../VNLib.Net.Compression/ThrowHelper.cs | 44 ++--- .../CompressorManagerTests.cs | 11 +- lib/Net.Compression/vnlib_compress/CMakeLists.txt | 195 ++++++++++++++++++--- lib/Net.Compression/vnlib_compress/compression.c | 71 ++++---- lib/Net.Compression/vnlib_compress/compression.h | 46 +++-- .../vnlib_compress/feature_brotli.c | 46 +++-- .../vnlib_compress/feature_brotli.h | 2 +- lib/Net.Compression/vnlib_compress/feature_zlib.c | 39 +++-- lib/Net.Compression/vnlib_compress/feature_zlib.h | 2 +- lib/Net.Compression/vnlib_compress/util.h | 2 +- lib/Utils/tests/.runsettings | 2 +- 17 files changed, 390 insertions(+), 205 deletions(-) (limited to 'lib') diff --git a/lib/Net.Compression/VNLib.Net.Compression/CompressionExtensions.cs b/lib/Net.Compression/VNLib.Net.Compression/CompressionExtensions.cs index 0a4f7aa..ab451d9 100644 --- a/lib/Net.Compression/VNLib.Net.Compression/CompressionExtensions.cs +++ b/lib/Net.Compression/VNLib.Net.Compression/CompressionExtensions.cs @@ -42,6 +42,14 @@ namespace VNLib.Net.Compression /// The results of the compression operation public static unsafe CompressionResult CompressBlock(this LibraryWrapper nativeLib, IntPtr comp, Memory output, ReadOnlyMemory input, bool finalBlock) { + /* + * Since .NET only supports int32 size memory blocks + * we dont need to worry about integer overflow. + * + * Output sizes can never be larger than input + * sizes (read/written) + */ + //get pointers to the input and output buffers using MemoryHandle inPtr = input.Pin(); using MemoryHandle outPtr = output.Pin(); @@ -56,10 +64,10 @@ namespace VNLib.Net.Compression //Configure the input and output buffers op->inputBuffer = inPtr.Pointer; - op->inputSize = input.Length; + op->inputSize = (uint)input.Length; op->outputBuffer = outPtr.Pointer; - op->outputSize = output.Length; + op->outputSize = (uint)output.Length; //Call the native compress function nativeLib!.CompressBlock(comp, &operation); @@ -67,8 +75,8 @@ namespace VNLib.Net.Compression //Return the number of bytes written return new() { - BytesRead = op->bytesRead, - BytesWritten = op->bytesWritten + BytesRead = (int)op->bytesRead, + BytesWritten = (int)op->bytesWritten }; } } diff --git a/lib/Net.Compression/VNLib.Net.Compression/CompressionOperation.cs b/lib/Net.Compression/VNLib.Net.Compression/CompressionOperation.cs index 6093e01..08c03db 100644 --- a/lib/Net.Compression/VNLib.Net.Compression/CompressionOperation.cs +++ b/lib/Net.Compression/VNLib.Net.Compression/CompressionOperation.cs @@ -33,41 +33,42 @@ namespace VNLib.Net.Compression internal unsafe ref struct CompressionOperation { #region readonly - - /// - /// A value that indicates a flush operation, 0 for no flush, above 0 for flush - /// - public int flush; /// /// A pointer to the input buffer /// public void* inputBuffer; - /// - /// The size of the input buffer - /// - public int inputSize; /// /// A pointer to the output buffer /// public void* outputBuffer; + + /// + /// A value that indicates a flush operation, 0 for no flush, above 0 for flush + /// + public int flush; + + /// + /// The size of the input buffer + /// + public uint inputSize; + /// /// The size of the output buffer /// - public int outputSize; + public uint outputSize; #endregion - /// /// An output variable, the number of bytes read from the input buffer /// - public int bytesRead; + public uint bytesRead; /// /// An output variable, the number of bytes written to the output buffer /// - public int bytesWritten; + public uint bytesWritten; } } \ No newline at end of file diff --git a/lib/Net.Compression/VNLib.Net.Compression/CompressorManager.cs b/lib/Net.Compression/VNLib.Net.Compression/CompressorManager.cs index 2407ba0..a614ab8 100644 --- a/lib/Net.Compression/VNLib.Net.Compression/CompressorManager.cs +++ b/lib/Net.Compression/VNLib.Net.Compression/CompressorManager.cs @@ -120,7 +120,7 @@ namespace VNLib.Net.Compression compressor.Instance = _nativeLib!.AllocateCompressor(compMethod, _compLevel); //Return the compressor block size - return _nativeLib!.GetBlockSize(compressor.Instance); + return (int)_nativeLib!.GetBlockSize(compressor.Instance); } /// diff --git a/lib/Net.Compression/VNLib.Net.Compression/INativeCompressor.cs b/lib/Net.Compression/VNLib.Net.Compression/INativeCompressor.cs index 3455ea0..c522860 100644 --- a/lib/Net.Compression/VNLib.Net.Compression/INativeCompressor.cs +++ b/lib/Net.Compression/VNLib.Net.Compression/INativeCompressor.cs @@ -57,16 +57,14 @@ namespace VNLib.Net.Compression /// Compresses the input block and writes the compressed data to the output block /// /// The input buffer to compress - /// The number of bytes read by the compressor /// The output buffer to write compressed data to - /// The number of bytes written to the output buffer CompressionResult Compress(ReadOnlyMemory input, Memory output); /// /// Gets the compressor block size if configured /// /// The ideal input buffer size for compressing blocks, or if block size is unlimited - int GetBlockSize(); + uint GetBlockSize(); /// /// Determines the maximum number of output bytes for the given number of input bytes @@ -74,6 +72,7 @@ namespace VNLib.Net.Compression /// /// The number of bytes to get the compressed size of /// The maxium size of the compressed data - int GetCompressedSize(int size); + /// + uint GetCompressedSize(uint size); } } \ No newline at end of file diff --git a/lib/Net.Compression/VNLib.Net.Compression/LibraryWrapper.cs b/lib/Net.Compression/VNLib.Net.Compression/LibraryWrapper.cs index 2ee6018..fe8fb5a 100644 --- a/lib/Net.Compression/VNLib.Net.Compression/LibraryWrapper.cs +++ b/lib/Net.Compression/VNLib.Net.Compression/LibraryWrapper.cs @@ -28,7 +28,6 @@ using System.IO.Compression; using System.Runtime.InteropServices; using System.Runtime.CompilerServices; -using VNLib.Utils; using VNLib.Utils.Native; using VNLib.Utils.Extensions; @@ -49,7 +48,7 @@ namespace VNLib.Net.Compression [SafeMethodName("GetCompressorBlockSize")] [UnmanagedFunctionPointer(CallingConvention.Cdecl)] - delegate int GetBlockSizeDelegate(IntPtr compressor); + delegate long GetBlockSizeDelegate(IntPtr compressor); [SafeMethodName("GetCompressorType")] [UnmanagedFunctionPointer(CallingConvention.Cdecl)] @@ -69,7 +68,7 @@ namespace VNLib.Net.Compression [SafeMethodName("GetCompressedSize")] [UnmanagedFunctionPointer(CallingConvention.Cdecl)] - delegate int GetCompressedSizeDelegate(IntPtr compressor, int uncompressedSize, int flush); + delegate long GetCompressedSizeDelegate(IntPtr compressor, ulong uncompressedSize, int flush); [SafeMethodName("CompressBlock")] [UnmanagedFunctionPointer(CallingConvention.Cdecl)] @@ -85,7 +84,7 @@ namespace VNLib.Net.Compression /// and used for the lifetime of the application. /// /// - internal sealed class LibraryWrapper + internal sealed class LibraryWrapper : IDisposable { private readonly SafeLibraryHandle _lib; private MethodTable _methodTable; @@ -103,6 +102,7 @@ namespace VNLib.Net.Compression /// Loads the native library at the specified path into the current process /// /// The path to the native library to load + /// /// The native library wrapper public static LibraryWrapper LoadLibrary(string filePath, DllImportSearchPath searchType) { @@ -147,18 +147,24 @@ namespace VNLib.Net.Compression [MethodImpl(MethodImplOptions.AggressiveInlining)] public CompressionMethod GetSupportedMethods() => _methodTable.GetMethods(); + /* + * Block size is stored as a uint32 in the native library + * compressor struct + */ + /// - /// Gets the block size of the specified compressor + /// Gets the block size of the specified compressor or 0 if + /// compressor does not hint it's optimal block size /// /// A pointer to the compressor instance /// A integer value of the compressor block size /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public int GetBlockSize(IntPtr compressor) + public uint GetBlockSize(IntPtr compressor) { - int result = _methodTable.GetBlockSize(compressor); - ThrowHelper.ThrowIfError((ERRNO)result); - return result; + long result = _methodTable.GetBlockSize(compressor); + ThrowHelper.ThrowIfError(result); + return (uint)result; } /// @@ -171,7 +177,7 @@ namespace VNLib.Net.Compression public CompressionMethod GetCompressorType(IntPtr compressor) { CompressionMethod result = _methodTable.GetCompType(compressor); - ThrowHelper.ThrowIfError((int)result); + ThrowHelper.ThrowIfError((long)result); return result; } @@ -185,7 +191,7 @@ namespace VNLib.Net.Compression public CompressionLevel GetCompressorLevel(IntPtr compressor) { CompressionLevel result = _methodTable.GetCompLevel(compressor); - ThrowHelper.ThrowIfError((int)result); + ThrowHelper.ThrowIfError((long)result); return result; } @@ -201,7 +207,7 @@ namespace VNLib.Net.Compression public IntPtr AllocateCompressor(CompressionMethod type, CompressionLevel level) { IntPtr result = _methodTable.Alloc(type, level); - ThrowHelper.ThrowIfError(result); + ThrowHelper.ThrowIfError(result.ToInt64()); return result; } @@ -237,14 +243,15 @@ namespace VNLib.Net.Compression /// The size of the input block to compress /// A value that specifies a flush operation /// Returns the size of the required output buffer + /// /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public int GetOutputSize(IntPtr compressor, int inputSize, int flush) + public ulong GetOutputSize(IntPtr compressor, ulong inputSize, int flush) { - int result = _methodTable.GetOutputSize(compressor, inputSize, flush); + long result = _methodTable.GetOutputSize(compressor, inputSize, flush); ThrowHelper.ThrowIfError(result); - return result; + return (ulong)result; } /// @@ -253,6 +260,7 @@ namespace VNLib.Net.Compression /// The compressor instance used to compress data /// A pointer to the compression operation structure /// The result of the operation + /// /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -273,7 +281,7 @@ namespace VNLib.Net.Compression /// /// Manually releases the library /// - internal void ManualRelease() + public void Dispose() { _methodTable = default; _lib.Dispose(); diff --git a/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionLib.cs b/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionLib.cs index f5b9071..10bad84 100644 --- a/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionLib.cs +++ b/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionLib.cs @@ -43,7 +43,7 @@ namespace VNLib.Net.Compression private NativeCompressionLib(LibraryWrapper nativeLib) => _library = nativeLib; /// - protected override void Free() => _library.ManualRelease(); + protected override void Free() => _library.Dispose(); /// /// Loads the native compression DLL at the specified file path and search pattern @@ -59,7 +59,11 @@ namespace VNLib.Net.Compression /// /// - public CompressionMethod GetSupportedMethods() => _library.GetSupportedMethods(); + public CompressionMethod GetSupportedMethods() + { + Check(); + return _library.GetSupportedMethods(); + } /// /// @@ -77,63 +81,64 @@ namespace VNLib.Net.Compression /// public SafeHandle AllocSafeCompressorHandle(CompressionMethod method, CompressionLevel level) { + Check(); //Alloc compressor then craete a safe handle IntPtr comp = _library.AllocateCompressor(method, level); return new SafeCompressorHandle(_library, comp); } - internal sealed record class Compressor(LibraryWrapper LibComp, SafeHandle Handle) : INativeCompressor + internal sealed record class Compressor(LibraryWrapper LibComp, SafeHandle CompressorHandle) : INativeCompressor { /// public CompressionResult Compress(ReadOnlyMemory input, Memory output) { - Handle.ThrowIfClosed(); - IntPtr compressor = Handle.DangerousGetHandle(); + CompressorHandle.ThrowIfClosed(); + IntPtr compressor = CompressorHandle.DangerousGetHandle(); return LibComp.CompressBlock(compressor, output, input, false); } /// - public void Dispose() => Handle.Dispose(); + public void Dispose() => CompressorHandle.Dispose(); /// public int Flush(Memory buffer) { - Handle.ThrowIfClosed(); - IntPtr compressor = Handle.DangerousGetHandle(); + CompressorHandle.ThrowIfClosed(); + IntPtr compressor = CompressorHandle.DangerousGetHandle(); CompressionResult result = LibComp.CompressBlock(compressor, buffer, ReadOnlyMemory.Empty, true); return result.BytesWritten; } /// - public int GetBlockSize() + public uint GetBlockSize() { - Handle.ThrowIfClosed(); - IntPtr compressor = Handle.DangerousGetHandle(); + CompressorHandle.ThrowIfClosed(); + IntPtr compressor = CompressorHandle.DangerousGetHandle(); return LibComp.GetBlockSize(compressor); } /// - public int GetCompressedSize(int size) + public uint GetCompressedSize(uint size) { - Handle.ThrowIfClosed(); - IntPtr compressor = Handle.DangerousGetHandle(); - return LibComp.GetOutputSize(compressor, size, 1); + CompressorHandle.ThrowIfClosed(); + IntPtr compressor = CompressorHandle.DangerousGetHandle(); + return (uint)LibComp.GetOutputSize(compressor, size, 1); } /// public CompressionLevel GetCompressionLevel() { - Handle.ThrowIfClosed(); - IntPtr compressor = Handle.DangerousGetHandle(); + CompressorHandle.ThrowIfClosed(); + IntPtr compressor = CompressorHandle.DangerousGetHandle(); return LibComp.GetCompressorLevel(compressor); } /// public CompressionMethod GetCompressionMethod() { - Handle.ThrowIfClosed(); - IntPtr compressor = Handle.DangerousGetHandle(); + CompressorHandle.ThrowIfClosed(); + IntPtr compressor = CompressorHandle.DangerousGetHandle(); return LibComp.GetCompressorType(compressor); } } diff --git a/lib/Net.Compression/VNLib.Net.Compression/ThrowHelper.cs b/lib/Net.Compression/VNLib.Net.Compression/ThrowHelper.cs index 2793526..40ce336 100644 --- a/lib/Net.Compression/VNLib.Net.Compression/ThrowHelper.cs +++ b/lib/Net.Compression/VNLib.Net.Compression/ThrowHelper.cs @@ -24,8 +24,6 @@ using System; -using VNLib.Utils; - namespace VNLib.Net.Compression { internal static class ThrowHelper @@ -43,6 +41,8 @@ namespace VNLib.Net.Compression ErrCompLevelNotSupported = -10, ErrInvalidInput = -11, ErrInvalidOutput = -12, + ErrCompressionFailed = -13, + ErrCompOverflow = -14, ErrGzInvalidState = -16, ErrGzOverflow = -17, @@ -55,37 +55,29 @@ namespace VNLib.Net.Compression /// /// /// - public static void ThrowIfError(ERRNO result) + public static void ThrowIfError(long result) { //Check for no error - if(result > 0) + if(result >= 0) { return; } - switch ((NativeErrorType)(int)result) + throw (NativeErrorType)result switch { - case NativeErrorType.ErrInvalidPtr: - throw new NativeCompressionException("A pointer to a compressor instance was null"); - case NativeErrorType.ErrOutOfMemory: - throw new NativeCompressionException("An operation falied because the system is out of memory"); - case NativeErrorType.ErrCompTypeNotSupported: - throw new NotSupportedException("The desired compression method is not supported by the native library"); - case NativeErrorType.ErrCompLevelNotSupported: - throw new NotSupportedException("The desired compression level is not supported by the native library"); - case NativeErrorType.ErrInvalidInput: - throw new NativeCompressionException("The input buffer was null and the input size was greater than 0"); - case NativeErrorType.ErrInvalidOutput: - throw new NativeCompressionException("The output buffer was null and the output size was greater than 0"); - case NativeErrorType.ErrGzInvalidState: - throw new NativeCompressionException("A gzip operation failed because the compressor state is invalid (null compressor pointer)"); - case NativeErrorType.ErrGzOverflow: - throw new NativeCompressionException("A gzip operation failed because the output buffer is too small"); - case NativeErrorType.ErrBrInvalidState: - throw new NativeCompressionException("A brotli operation failed because the compressor state is invalid (null compressor pointer)"); - default: - break; - } + NativeErrorType.ErrInvalidPtr => new NativeCompressionException("A pointer to a compressor instance was null"), + NativeErrorType.ErrOutOfMemory => new NativeCompressionException("An operation falied because the system is out of memory"), + NativeErrorType.ErrCompTypeNotSupported => new NotSupportedException("The desired compression method is not supported by the native library"), + NativeErrorType.ErrCompLevelNotSupported => new NotSupportedException("The desired compression level is not supported by the native library"), + NativeErrorType.ErrInvalidInput => new NativeCompressionException("The input buffer was null and the input size was greater than 0"), + NativeErrorType.ErrInvalidOutput => new NativeCompressionException("The output buffer was null and the output size was greater than 0"), + NativeErrorType.ErrGzInvalidState => new NativeCompressionException("A gzip operation failed because the compressor state is invalid (null compressor pointer)"), + NativeErrorType.ErrGzOverflow => new NativeCompressionException("A gzip operation failed because the output buffer is too small"), + NativeErrorType.ErrBrInvalidState => new NativeCompressionException("A brotli operation failed because the compressor state is invalid (null compressor pointer)"), + NativeErrorType.ErrCompOverflow => new OverflowException("A call to compress block or get block size failed because the library would cause an integer overflow processing your data"), + NativeErrorType.ErrCompressionFailed => new NativeCompressionException("An operation failes because the underlying implementation would cause a memory related error. State is considered corrupted"), + _ => new NativeCompressionException($"An unknown error occurred, code: 0x{result:x}"), + }; } } } \ No newline at end of file diff --git a/lib/Net.Compression/VNLib.Net.CompressionTests/CompressorManagerTests.cs b/lib/Net.Compression/VNLib.Net.CompressionTests/CompressorManagerTests.cs index 69672c8..8d8deeb 100644 --- a/lib/Net.Compression/VNLib.Net.CompressionTests/CompressorManagerTests.cs +++ b/lib/Net.Compression/VNLib.Net.CompressionTests/CompressorManagerTests.cs @@ -3,13 +3,14 @@ using System.IO; using System.Linq; using System.Text; using System.Text.Json; +using System.Diagnostics; using System.IO.Compression; using System.Security.Cryptography; using VNLib.Utils.IO; using VNLib.Utils.Memory; -using VNLib.Net.Http; using VNLib.Utils.Extensions; +using VNLib.Net.Http; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -28,6 +29,8 @@ namespace VNLib.Net.Compression.Tests LibTestComp cp = new(lib, CompressionLevel.Fastest); + Debug.WriteLine("Loading library with compression level: Fastest"); + TestSupportedMethods(cp); //Test for supported methods @@ -150,6 +153,8 @@ namespace VNLib.Net.Compression.Tests compressor.InitCompressor(CompressionMethod.Deflate); compressor.DeinitCompressor(); } + + Debug.WriteLine($"Compressor library supports {supported}"); } private static void TestCompressorMethod(ITestCompressor compressor, CompressionMethod method) @@ -212,6 +217,8 @@ namespace VNLib.Net.Compression.Tests byte[] decompressed = DecompressData(outputStream, method); Assert.IsTrue(buffer.SequenceEqual(decompressed)); + + Debug.WriteLine($"Compressor type {method} successfully compressed valid data"); } finally { @@ -293,7 +300,7 @@ namespace VNLib.Net.Compression.Tests public int InitCompressor(CompressionMethod method) { _comp = Library.AllocCompressor(method, level); - return _comp.GetBlockSize(); + return (int)_comp.GetBlockSize(); } } } diff --git a/lib/Net.Compression/vnlib_compress/CMakeLists.txt b/lib/Net.Compression/vnlib_compress/CMakeLists.txt index bc3c1df..a7b6835 100644 --- a/lib/Net.Compression/vnlib_compress/CMakeLists.txt +++ b/lib/Net.Compression/vnlib_compress/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.0) -project(vnlib_compress) +project(vnlib_compress C) #export header files to the main project file(GLOB COMP_HEADERS *.h) @@ -8,34 +8,50 @@ file(GLOB COMP_HEADERS *.h) #Add indepednent source files to the project set(VNLIB_COMPRESS_SOURCES compression.c) +#set options for enable botli and zlib +option(ENABLE_BROTLI "Enable brotli compression" ON) +option(ENABLE_ZLIB "Enable zlib compression" ON) + #add feature specific source files to the project if(ENABLE_BROTLI) - set(VNLIB_FEATURE_BR_SOURCES feature_brotli.c) + list(APPEND VNLIB_COMPRESS_SOURCES feature_brotli.c) + #define the brotli feature macro to enable zlib support + add_definitions(-DVNLIB_COMPRESSOR_BROTLI_ENABLED) endif() if(ENABLE_ZLIB) - set(VNLIB_FEATURE_ZLIB_SOURCES feature_zlib.c) + list(APPEND VNLIB_COMPRESS_SOURCES feature_zlib.c) + #define the zlib feature macro to enable zlib support + add_definitions(-DVNLIB_COMPRESSOR_ZLIB_ENABLED) endif() #create my shared library -add_library(${CMAKE_PROJECT_NAME} SHARED ${VNLIB_COMPRESS_SOURCES} ${COMP_HEADERS} ${VNLIB_FEATURE_BR_SOURCES} ${VNLIB_FEATURE_ZLIB_SOURCES}) +add_library(${CMAKE_PROJECT_NAME} SHARED ${VNLIB_COMPRESS_SOURCES} ${COMP_HEADERS}) +#also create static library +add_library(${CMAKE_PROJECT_NAME}_static STATIC ${VNLIB_COMPRESS_SOURCES} ${COMP_HEADERS}) -#Setup the compiler options +#if on unix lib will be appended, so we can adjust +if(UNIX) + set_target_properties(${CMAKE_PROJECT_NAME} ${CMAKE_PROJECT_NAME}_static PROPERTIES OUTPUT_NAME vn_compress) +endif() -enable_language(C) + +#Setup the compiler options set(CMAKE_CXX_STANDARD 90) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS OFF) +set(DEFAULT_BUILD_TYPE "Release") #enable position independent code (for shared libraries with exports) set(CMAKE_POSITION_INDEPENDENT_CODE ON) -#strict error checking for main project -set(CMAKE_COMPILE_WARNING_AS_ERROR ON) -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +message(STATUS "Build type is '${CMAKE_BUILD_TYPE}'") -#force the compiler to use the C90/89 standard -target_compile_features(${CMAKE_PROJECT_NAME} PRIVATE c_std_90) +#include checks for zlib and brotli +include(CheckTypeSize) +include(CheckFunctionExists) +include(CheckIncludeFile) +include(CheckCCompilerFlag) +enable_testing() #setup flags for windows compilation if(MSVC) @@ -46,10 +62,7 @@ if(MSVC) /sdl /TC /GS - - #for debug configs - $<$:/Wall> - $<$:/options:strict> + $<$:/FC> $<$:/showIncludes> ) @@ -59,6 +72,12 @@ if(MSVC) ${CMAKE_PROJECT_NAME} PRIVATE + #for debug configs + $<$:/options:strict> + #disable warnings for struct padding and spectre mitigation wuen WX is enabled + $<$:/wd5045> + $<$:/wd4820> + $<$:/Wall> $<$:/WX> #warnings as errors (only for our project) $<$:/Zi> $<$:/Zo> @@ -74,8 +93,6 @@ if(MSVC) elseif(CMAKE_COMPILER_IS_GNUCC) add_compile_options( - ${CMAKE_PROJECT_NAME} - PUBLIC -Wextra -fstack-protector @@ -89,13 +106,53 @@ elseif(CMAKE_COMPILER_IS_GNUCC) target_compile_options( ${CMAKE_PROJECT_NAME} PRIVATE + $<$:-Wall> $<$:-pedantic> ) endif() -#check for brotli feature enablement +#check for brotli encoding only feature enablement if(ENABLE_BROTLI) + + #macros for brotli + set(BROTLI_DEFINITIONS) + + set(LIBM_LIBRARY) + check_function_exists(log2 LOG2_RES) + + if(NOT LOG2_RES) + + set(orig_req_libs "${CMAKE_REQUIRED_LIBRARIES}") + set(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES};m") + + check_function_exists(log2 LOG2_LIBM_RES) + + if(LOG2_LIBM_RES) + set(LIBM_LIBRARY "m") + list(APPEND BROTLI_DEFINITIONS -DBROTLI_HAVE_LOG2=1) + else() + list(APPEND BROTLI_DEFINITIONS -DBROTLI_HAVE_LOG2=0) + endif() + + set(CMAKE_REQUIRED_LIBRARIES "${orig_req_libs}") + unset(LOG2_LIBM_RES) + unset(orig_req_libs) + else() + list(APPEND BROTLI_DEFINITIONS -DBROTLI_HAVE_LOG2=1) + endif() + unset(LOG2_RES) + + #target definitions from brotli cmakelists + if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") + list(APPEND BROTLI_DEFINITIONS -DOS_LINUX) + elseif(${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD") + list(APPEND BROTLI_DEFINITIONS -DOS_FREEBSD) + elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + list(APPEND BROTLI_DEFINITIONS -DOS_MACOSX) + set(CMAKE_MACOS_RPATH TRUE) + endif() + #add the include directory for brotli so we can include the header files include_directories(../third-party/brotli/c/include) @@ -107,33 +164,119 @@ if(ENABLE_BROTLI) #add brotli as a static library to link later add_library(lib_brotli STATIC ${BROTLI_SOURCES} ${BROTLI_ENC_SOURCES}) - - #define the brotli feature macro to enable brotli support - add_definitions(-DVNLIB_COMPRESSOR_BROTLI_ENABLED) target_link_libraries(${CMAKE_PROJECT_NAME} lib_brotli) + + #add the definitions to the brotli project only + target_compile_definitions(lib_brotli PRIVATE ${BROTLI_DEFINITIONS}) + endif() -#check for zlib feature enablement +#check for zlib feature enablement, supports madler and cloudflare forks if(ENABLE_ZLIB) + #add the include directory for zlib so we can include the header files include_directories(../third-party/zlib) + set(ZLIB_DEFINITIONS) + set(Z_C_FLAGS) + #we only need to add the zlib deflate source files to the project set(ZLIB_SOURCES ../third-party/zlib/deflate.c ../third-party/zlib/adler32.c - ../third-party/zlib/adler32_simd.c ../third-party/zlib/crc32.c ../third-party/zlib/zutil.c ../third-party/zlib/trees.c ) + + check_type_size(off64_t OFF64_T) + check_function_exists(fseeko HAVE_FSEEKO) + + if(HAVE_OFF64_T) + list(APPEND ZLIB_DEFINITIONS -D_LARGEFILE64_SOURCE=1) + endif() + + #add fseeko if we have it + if(NOT HAVE_FSEEKO) + list(APPEND ZLIB_DEFINITIONS -DNO_FSEEKO) + endif() + + if(MSVC) + + set(CMAKE_DEBUG_POSTFIX "d") + + list(APPEND ZLIB_DEFINITIONS + -D_CRT_SECURE_NO_DEPRECATE + -D_CRT_NONSTDC_NO_DEPRECATE + ) + #setup avx compiler support on Windows + check_c_compiler_flag(/arch:AVX HAS_AVX) + if (HAS_AVX) + list(APPEND Z_C_FLAGS /arch:AVX) + endif() + + elseif(UNIX) + + + #for cloudflare intrinsic detections + check_c_compiler_flag(-march=armv8-a+crc ARM_CRC) + check_c_compiler_flag(-msse2 HAS_SSE2) + check_c_compiler_flag(-mssse3 HAS_SSSE3) + check_c_compiler_flag(-msse4.2 HAS_SSE42) + check_c_compiler_flag(-mpclmul HAS_PCLMUL) + + if(ARM_CRC) + list(APPEND Z_C_FLAGS -march=armv8-a+crc) + else() + if(HAS_SSE2) + list(APPEND Z_C_FLAGS -msse2) + list(APPEND ZLIB_DEFINITIONS -DHAS_SSE2) + #excluding inflate specific optimizations + endif() + if(HAS_SSSE3) + list(APPEND Z_C_FLAGS -mssse3) + + #add cloudflare intrinsic optimizations + if(EXISTS "../third-party/zlib/adler32_simd.c") + list(APPEND ZLIB_DEFINITIONS -DHAS_SSSE3 -DADLER32_SIMD_SSSE3) + list(APPEND ZLIB_SOURCES ../third-party/zlib/adler32_simd.c) + endif() + + endif() + if(HAS_SSE42) + list(APPEND Z_C_FLAGS -msse4.2) + list(APPEND ZLIB_DEFINITIONS -DHAS_SSE42) + endif() + if(HAS_PCLMUL) + list(APPEND Z_C_FLAGS -mpclmul) + + #add cloudflare intrinsic optimizations for PCMLONGMUL + if(EXISTS "../third-party/zlib/crc32_simd.c") + list(APPEND ZLIB_DEFINITIONS -DHAS_PCLMUL) + list(APPEND ZLIB_SOURCES ../third-party/zlib/crc32_simd.c) + endif() + + endif() + endif() + + + endif() + #add zlib as a library to link later add_library(lib_deflate STATIC ${ZLIB_SOURCES}) - #define the zlib feature macro to enable zlib support - add_definitions(-DVNLIB_COMPRESSOR_ZLIB_ENABLED) + if(MSVC) + #allways targeting x64 machines + set_target_properties(lib_deflate PROPERTIES STATIC_LIBRARY_FLAGS "/machine:x64") + endif() + + #add the definitions to the zlib project only + target_compile_definitions(lib_deflate PRIVATE ${ZLIB_DEFINITIONS}) + + #only target zlib project + target_compile_options(lib_deflate PRIVATE ${Z_C_FLAGS}) target_link_libraries(${CMAKE_PROJECT_NAME} lib_deflate) endif() diff --git a/lib/Net.Compression/vnlib_compress/compression.c b/lib/Net.Compression/vnlib_compress/compression.c index 1139762..0d57096 100644 --- a/lib/Net.Compression/vnlib_compress/compression.c +++ b/lib/Net.Compression/vnlib_compress/compression.c @@ -19,6 +19,15 @@ * along with vnlib_compress. If not, see http://www.gnu.org/licenses/. */ +/* +* Notes: +* This api is desgined to be friendly to many types of callers +* without needing to worry about the platform integer size. I would +* like to return errors as negative values, and I am requiring block +* operations on block sizes under INT64_MAX. This allows 64bit support +* while allowing negative error codes as return values. I think +* this is a good compromise. +*/ #include "compression.h" @@ -31,39 +40,13 @@ #include "feature_zlib.h" #endif /* VNLIB_COMPRESSOR_GZIP_ENABLED */ + /* -* Configure DLLMAIN on Windows +* Public API functions */ - -#if false -#define WIN32_LEAN_AND_MEAN -#include - -BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved) -{ - /* - * Taken from the malloc.c file for initializing the library. - * and thread events - */ - switch (ul_reason_for_call) - { - case DLL_PROCESS_ATTACH: - break; - case DLL_THREAD_ATTACH: - break; - case DLL_THREAD_DETACH: - break; - case DLL_PROCESS_DETACH: - break; - } - return TRUE; -} - -#endif /* DLLMAIN */ - VNLIB_EXPORT CompressorType VNLIB_CC GetSupportedCompressors(void); -VNLIB_EXPORT int VNLIB_CC GetCompressorBlockSize(_In_ void* compressor); +VNLIB_EXPORT int64_t VNLIB_CC GetCompressorBlockSize(_In_ void* compressor); VNLIB_EXPORT CompressorType VNLIB_CC GetCompressorType(_In_ void* compressor); @@ -71,11 +54,11 @@ VNLIB_EXPORT CompressionLevel VNLIB_CC GetCompressorLevel(_In_ void* compressor) VNLIB_EXPORT void* VNLIB_CC AllocateCompressor(CompressorType type, CompressionLevel level); -VNLIB_EXPORT int VNLIB_CC FreeCompressor(void* compressor); +VNLIB_EXPORT int VNLIB_CC FreeCompressor(_In_ void* compressor); -VNLIB_EXPORT int VNLIB_CC GetCompressedSize(void* compressor, int inputLength, int flush); +VNLIB_EXPORT int64_t VNLIB_CC GetCompressedSize(_In_ void* compressor, uint64_t inputLength, int32_t flush); -VNLIB_EXPORT int VNLIB_CC CompressBlock(void* compressor, CompressionOperation* operation); +VNLIB_EXPORT int VNLIB_CC CompressBlock(_In_ void* compressor, CompressionOperation* operation); /* Gets the supported compressors, this is defined at compile time and is a convenience method for @@ -126,14 +109,14 @@ VNLIB_EXPORT CompressionLevel VNLIB_CC GetCompressorLevel(_In_ void* compressor) return ((CompressorState*)compressor)->level; } -VNLIB_EXPORT int VNLIB_CC GetCompressorBlockSize(_In_ void* compressor) +VNLIB_EXPORT int64_t VNLIB_CC GetCompressorBlockSize(_In_ void* compressor) { if (!compressor) { return ERR_INVALID_PTR; } - return ((CompressorState*)compressor)->blockSize; + return (int64_t)((CompressorState*)compressor)->blockSize; } @@ -189,6 +172,7 @@ VNLIB_EXPORT void* VNLIB_CC AllocateCompressor(CompressorType type, CompressionL /* * Unsupported compressor type allow error to propagate */ + case COMP_TYPE_LZ4: case COMP_TYPE_NONE: default: break; @@ -230,7 +214,7 @@ VNLIB_EXPORT void* VNLIB_CC AllocateCompressor(CompressorType type, CompressionL } } -VNLIB_EXPORT int VNLIB_CC FreeCompressor(void* compressor) +VNLIB_EXPORT int VNLIB_CC FreeCompressor(_In_ void* compressor) { CompressorState* comp; int errorCode; @@ -289,16 +273,21 @@ VNLIB_EXPORT int VNLIB_CC FreeCompressor(void* compressor) return errorCode; } -VNLIB_EXPORT int VNLIB_CC GetCompressedSize(_In_ void* compressor, int inputLength, int flush) +VNLIB_EXPORT int64_t VNLIB_CC GetCompressedSize(_In_ void* compressor, uint64_t inputLength, int32_t flush) { CompressorState* comp; - int result; + int64_t result; if (!compressor) { return ERR_INVALID_PTR; } + if (inputLength > INT64_MAX) + { + return ERR_OVERFLOW; + } + comp = (CompressorState*)compressor; switch (comp->type) @@ -321,10 +310,10 @@ VNLIB_EXPORT int VNLIB_CC GetCompressedSize(_In_ void* compressor, int inputLeng #endif - /* - * Set the result as an error code, since the compressor - * type is not supported. - */ + /* + * Set the result as an error code, since the compressor + * type is not supported. + */ case COMP_TYPE_NONE: case COMP_TYPE_LZ4: default: diff --git a/lib/Net.Compression/vnlib_compress/compression.h b/lib/Net.Compression/vnlib_compress/compression.h index 930ef1c..a4fc65f 100644 --- a/lib/Net.Compression/vnlib_compress/compression.h +++ b/lib/Net.Compression/vnlib_compress/compression.h @@ -45,6 +45,8 @@ #define ERR_COMP_LEVEL_NOT_SUPPORTED -10 #define ERR_INVALID_INPUT_DATA -11 #define ERR_INVALID_OUTPUT_DATA -12 +#define ERR_COMPRESSION_FAILED -13 +#define ERR_OVERFLOW -14 /* * Enumerated list of supported compression types for user selection @@ -95,6 +97,12 @@ typedef enum CompressorStatus { } CompressorStatus; typedef struct CompressorStateStruct{ + + /* + Pointer to the underlying compressor implementation. + */ + void* compressor; + /* Indicates the type of underlying compressor. */ @@ -109,18 +117,8 @@ typedef struct CompressorStateStruct{ /* Indicates the suggested block size for the underlying compressor. */ - int blockSize; + uint32_t blockSize; - /* - Pointer to the underlying compressor implementation. - */ - void* compressor; - - /* - Counts the number of pending bytes since the last successful flush - operation. - */ - uint32_t pendingBytes; } CompressorState; @@ -131,28 +129,28 @@ typedef struct CompressorStateStruct{ typedef struct CompressionOperationStruct { /* - * If the operation is a flush operation - */ - const int flush; - - /* - * Input stream data - */ + * Input stream data + */ const void* bytesIn; - const int bytesInLength; + /* + * Output buffer/data stream + */ + void* bytesOut; /* - * Output buffer/data stream + * If the operation is a flush operation */ - void* bytesOut; - const int bytesOutLength; + const int32_t flush; + + const uint32_t bytesInLength; + const uint32_t bytesOutLength; /* * Results of the streaming operation */ - int bytesRead; - int bytesWritten; + uint32_t bytesRead; + uint32_t bytesWritten; } CompressionOperation; diff --git a/lib/Net.Compression/vnlib_compress/feature_brotli.c b/lib/Net.Compression/vnlib_compress/feature_brotli.c index d83e2b4..f528cf1 100644 --- a/lib/Net.Compression/vnlib_compress/feature_brotli.c +++ b/lib/Net.Compression/vnlib_compress/feature_brotli.c @@ -30,12 +30,21 @@ int BrAllocCompressor(CompressorState* state) { BrotliEncoderState* comp; + + /* + * Never allow no compression, it is not supported by the br encoder + */ + + if (state->level == COMP_LEVEL_NO_COMPRESSION) + { + return ERR_COMP_LEVEL_NOT_SUPPORTED; + } comp = BrotliEncoderCreateInstance(0, 0, 0); if (!comp) { - return ERR_BR_INVALID_STATE; + return ERR_OUT_OF_MEMORY; } state->compressor = comp; @@ -49,7 +58,6 @@ int BrAllocCompressor(CompressorState* state) BrotliEncoderSetParameter(comp, BROTLI_PARAM_MODE, BROTLI_MODE_GENERIC); BrotliEncoderSetParameter(comp, BROTLI_PARAM_LGWIN, BR_DEFAULT_WINDOW); - /* * Capture the block size as a size hint if it is greater than 0 @@ -78,10 +86,10 @@ int BrAllocCompressor(CompressorState* state) BrotliEncoderSetParameter(comp, BROTLI_PARAM_QUALITY, BR_COMP_LEVEL_SMALLEST_SIZE); break; + case COMP_LEVEL_NO_COMPRESSION: default: BrotliEncoderSetParameter(comp, BROTLI_PARAM_QUALITY, BR_COMP_LEVEL_DEFAULT); break; - } return TRUE; @@ -163,29 +171,36 @@ int BrCompressBlock(CompressorState* state, CompressionOperation* operation) &nextOut, &totalOut ); + + /* + * check for possible overflow and retrun error + */ + if (availableIn > operation->bytesInLength || availableOut > operation->bytesOutLength) + { + return ERR_COMPRESSION_FAILED; + } /* - * Regardless of the operation success we should return the - * results to the caller. Br encoder sets the number of + * Regardless of the operation success we should return the + * results to the caller. Br encoder sets the number of * bytes remaining in the input/output spans */ - - operation->bytesRead = operation->bytesInLength - (int)availableIn; - operation->bytesWritten = operation->bytesOutLength - (int)availableOut; + operation->bytesRead = operation->bytesInLength - (uint32_t)availableIn; + operation->bytesWritten = operation->bytesOutLength - (uint32_t)availableOut; return brResult; } -int BrGetCompressedSize(CompressorState* state, int length) +int64_t BrGetCompressedSize(CompressorState* state, uint64_t length) { - size_t compressedSize; - /* * When the flush flag is set, the caller is requesting the * entire size of the compressed data, which can include metadata */ + size_t size; + validateCompState(state) if (length <= 0) @@ -193,7 +208,12 @@ int BrGetCompressedSize(CompressorState* state, int length) return 0; } - compressedSize = BrotliEncoderMaxCompressedSize(length); + size = BrotliEncoderMaxCompressedSize(length); + + if (size > INT64_MAX) + { + return ERR_OVERFLOW; + } - return (int)compressedSize; + return (int64_t)size; } \ No newline at end of file diff --git a/lib/Net.Compression/vnlib_compress/feature_brotli.h b/lib/Net.Compression/vnlib_compress/feature_brotli.h index b5a9ed6..e04ce7a 100644 --- a/lib/Net.Compression/vnlib_compress/feature_brotli.h +++ b/lib/Net.Compression/vnlib_compress/feature_brotli.h @@ -41,6 +41,6 @@ void BrFreeCompressor(CompressorState* state); int BrCompressBlock(CompressorState* state, CompressionOperation* operation); -int BrGetCompressedSize(CompressorState* state, int length); +int64_t BrGetCompressedSize(CompressorState* state, uint64_t length); #endif /* !BROTLI_STUB_H_ */ \ No newline at end of file diff --git a/lib/Net.Compression/vnlib_compress/feature_zlib.c b/lib/Net.Compression/vnlib_compress/feature_zlib.c index 210e212..3dc2e3e 100644 --- a/lib/Net.Compression/vnlib_compress/feature_zlib.c +++ b/lib/Net.Compression/vnlib_compress/feature_zlib.c @@ -31,8 +31,7 @@ int DeflateAllocCompressor(CompressorState* state) -{ - +{ int result, compLevel; z_stream* stream; @@ -42,6 +41,11 @@ int DeflateAllocCompressor(CompressorState* state) */ stream = (z_stream*)vncalloc(1, sizeof(z_stream)); + if (!stream) + { + return ERR_OUT_OF_MEMORY; + } + stream->zalloc = Z_NULL; stream->zfree = Z_NULL; stream->opaque = Z_NULL; @@ -208,6 +212,20 @@ int DeflateCompressBlock(CompressorState* state, CompressionOperation* operation result = deflate(stream, operation->flush ? Z_FINISH : Z_NO_FLUSH); + /* + * Allways clear stream fields as they are assigned on every call + */ + stream->next_in = NULL; + stream->next_out = NULL; + + /* + * check for result overflow and return the error code + */ + if (stream->avail_in > operation->bytesInLength || stream->avail_out > operation->bytesOutLength) + { + return ERR_COMPRESSION_FAILED; + } + /* * Regardless of the return value, we should always update the * the number of bytes read and written. @@ -218,17 +236,14 @@ int DeflateCompressBlock(CompressorState* state, CompressionOperation* operation operation->bytesRead = operation->bytesInLength - stream->avail_in; operation->bytesWritten = operation->bytesOutLength - stream->avail_out; - - /*Clear all stream fields after checking results */ + stream->avail_in = 0; - stream->next_in = NULL; stream->avail_out = 0; - stream->next_out = NULL; return result; } -int DeflateGetCompressedSize(CompressorState* state, int length, int flush) +int64_t DeflateGetCompressedSize(CompressorState* state, uint64_t length, int32_t flush) { uint64_t compressedSize; @@ -247,11 +262,11 @@ int DeflateGetCompressedSize(CompressorState* state, int length, int flush) if(flush) { /* - * If the flush flag is set, we need to add the size of the - * pending data in the stream + * TODO: actualy determine the size of the compressed data + * when the flush flag is set. */ - compressedSize = deflateBound(state->compressor, length + state->pendingBytes); + compressedSize = deflateBound(state->compressor, length); } else { @@ -259,10 +274,10 @@ int DeflateGetCompressedSize(CompressorState* state, int length, int flush) } /* Verify the results to make sure the value doesnt overflow */ - if (compressedSize > INT32_MAX) + if (compressedSize > INT64_MAX) { return ERR_GZ_OVERFLOW; } - return (int)compressedSize; + return (int64_t)compressedSize; } \ No newline at end of file diff --git a/lib/Net.Compression/vnlib_compress/feature_zlib.h b/lib/Net.Compression/vnlib_compress/feature_zlib.h index 0d4b6f6..7db456c 100644 --- a/lib/Net.Compression/vnlib_compress/feature_zlib.h +++ b/lib/Net.Compression/vnlib_compress/feature_zlib.h @@ -45,6 +45,6 @@ int DeflateFreeCompressor(CompressorState* state); int DeflateCompressBlock(CompressorState* state, CompressionOperation* operation); -int DeflateGetCompressedSize(CompressorState* state, int length, int flush); +int64_t DeflateGetCompressedSize(CompressorState* state, uint64_t length, int32_t flush); #endif diff --git a/lib/Net.Compression/vnlib_compress/util.h b/lib/Net.Compression/vnlib_compress/util.h index 583e10b..4a11708 100644 --- a/lib/Net.Compression/vnlib_compress/util.h +++ b/lib/Net.Compression/vnlib_compress/util.h @@ -50,7 +50,7 @@ #ifndef _In_ #define _In_ -#endif // !_In_ +#endif /* diff --git a/lib/Utils/tests/.runsettings b/lib/Utils/tests/.runsettings index cf85d64..977a261 100644 --- a/lib/Utils/tests/.runsettings +++ b/lib/Utils/tests/.runsettings @@ -2,7 +2,7 @@ - F:\Programming\VNLib\core\lib\WinRpMalloc\src\x64\Debug\rpmalloc.dll + 1 -- cgit