diff options
-rw-r--r-- | .editorconfig | 132 | ||||
-rw-r--r-- | .gitignore | 1 | ||||
-rw-r--r-- | lib/Net.Compression/VNLib.Net.Compression/CompressionExtensions.cs | 52 | ||||
-rw-r--r-- | lib/Net.Compression/VNLib.Net.Compression/INativeCompressor.cs | 14 | ||||
-rw-r--r-- | lib/Net.Compression/VNLib.Net.Compression/NativeCompressionLib.cs | 19 | ||||
-rw-r--r-- | lib/Net.Compression/VNLib.Net.CompressionTests/CompressorManagerTests.cs | 134 |
6 files changed, 215 insertions, 137 deletions
diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index 37d91eb..0000000 --- a/.editorconfig +++ /dev/null @@ -1,132 +0,0 @@ -[*.cs] - -# IDE0023: Use block body for conversion operator -csharp_style_expression_bodied_operators = when_on_single_line:suggestion -csharp_indent_labels = one_less_than_current -csharp_using_directive_placement = outside_namespace:silent -csharp_prefer_simple_using_statement = true:suggestion -csharp_prefer_braces = true:silent -csharp_style_namespace_declarations = block_scoped:silent -csharp_style_prefer_method_group_conversion = true:silent -csharp_style_prefer_top_level_statements = true:silent -csharp_style_expression_bodied_methods = when_on_single_line:suggestion -csharp_style_expression_bodied_constructors = when_on_single_line:suggestion -csharp_style_expression_bodied_properties = true:silent -csharp_style_expression_bodied_indexers = true:silent -csharp_style_expression_bodied_accessors = true:silent -csharp_style_expression_bodied_lambdas = true:silent -csharp_style_expression_bodied_local_functions = false:silent -csharp_space_around_binary_operators = before_and_after -csharp_style_throw_expression = true:suggestion -csharp_style_prefer_null_check_over_type_check = true:suggestion -csharp_prefer_simple_default_expression = true:suggestion -csharp_style_prefer_index_operator = true:suggestion -csharp_style_prefer_local_over_anonymous_function = true:suggestion -csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion -csharp_style_prefer_range_operator = true:suggestion -csharp_style_prefer_tuple_swap = true:suggestion -csharp_style_prefer_utf8_string_literals = true:suggestion -csharp_style_inlined_variable_declaration = true:suggestion -csharp_style_deconstructed_variable_declaration = true:suggestion -csharp_style_unused_value_assignment_preference = discard_variable:suggestion -csharp_style_unused_value_expression_statement_preference = discard_variable:silent -csharp_prefer_static_local_function = true:suggestion -csharp_style_prefer_readonly_struct = true:suggestion -csharp_style_prefer_readonly_struct_member = true:suggestion -csharp_style_allow_embedded_statements_on_same_line_experimental = true:silent -csharp_style_allow_blank_line_after_colon_in_constructor_initializer_experimental = true:silent -csharp_style_allow_blank_lines_between_consecutive_braces_experimental = true:silent -csharp_style_allow_blank_line_after_token_in_arrow_expression_clause_experimental = true:silent -csharp_style_allow_blank_line_after_token_in_conditional_expression_experimental = true:silent -csharp_style_conditional_delegate_call = true:suggestion -csharp_style_prefer_switch_expression = true:suggestion -csharp_style_prefer_pattern_matching = true:silent -csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion -csharp_style_pattern_matching_over_as_with_null_check = true:suggestion -csharp_style_prefer_not_pattern = true:suggestion -csharp_style_prefer_extended_property_pattern = true:suggestion -csharp_style_var_for_built_in_types = false:silent -csharp_style_var_when_type_is_apparent = false:silent -csharp_style_var_elsewhere = false:silent - -[*.{cs,vb}] -#### Naming styles #### - -# Naming rules - -dotnet_naming_rule.interface_should_be_begins_with_i.severity = suggestion -dotnet_naming_rule.interface_should_be_begins_with_i.symbols = interface -dotnet_naming_rule.interface_should_be_begins_with_i.style = begins_with_i - -dotnet_naming_rule.types_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.types_should_be_pascal_case.symbols = types -dotnet_naming_rule.types_should_be_pascal_case.style = pascal_case - -dotnet_naming_rule.non_field_members_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.non_field_members_should_be_pascal_case.symbols = non_field_members -dotnet_naming_rule.non_field_members_should_be_pascal_case.style = pascal_case - -# Symbol specifications - -dotnet_naming_symbols.interface.applicable_kinds = interface -dotnet_naming_symbols.interface.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected -dotnet_naming_symbols.interface.required_modifiers = - -dotnet_naming_symbols.types.applicable_kinds = class, struct, interface, enum -dotnet_naming_symbols.types.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected -dotnet_naming_symbols.types.required_modifiers = - -dotnet_naming_symbols.non_field_members.applicable_kinds = property, event, method -dotnet_naming_symbols.non_field_members.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected -dotnet_naming_symbols.non_field_members.required_modifiers = - -# Naming styles - -dotnet_naming_style.begins_with_i.required_prefix = I -dotnet_naming_style.begins_with_i.required_suffix = -dotnet_naming_style.begins_with_i.word_separator = -dotnet_naming_style.begins_with_i.capitalization = pascal_case - -dotnet_naming_style.pascal_case.required_prefix = -dotnet_naming_style.pascal_case.required_suffix = -dotnet_naming_style.pascal_case.word_separator = -dotnet_naming_style.pascal_case.capitalization = pascal_case - -dotnet_naming_style.pascal_case.required_prefix = -dotnet_naming_style.pascal_case.required_suffix = -dotnet_naming_style.pascal_case.word_separator = -dotnet_naming_style.pascal_case.capitalization = pascal_case -dotnet_style_operator_placement_when_wrapping = beginning_of_line -tab_width = 4 -indent_size = 4 -end_of_line = crlf -dotnet_style_coalesce_expression = true:suggestion -dotnet_style_null_propagation = true:suggestion -dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion -dotnet_style_prefer_auto_properties = true:silent -dotnet_style_object_initializer = true:suggestion -dotnet_style_collection_initializer = true:suggestion -dotnet_style_prefer_simplified_boolean_expressions = true:suggestion -dotnet_style_prefer_conditional_expression_over_assignment = true:silent -dotnet_style_prefer_conditional_expression_over_return = true:silent -dotnet_style_explicit_tuple_names = true:suggestion -dotnet_style_prefer_inferred_tuple_names = true:suggestion -dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion -dotnet_style_prefer_compound_assignment = true:suggestion -dotnet_style_prefer_simplified_interpolation = true:suggestion -dotnet_style_namespace_match_folder = true:suggestion -dotnet_style_readonly_field = true:suggestion -dotnet_style_predefined_type_for_locals_parameters_members = true:silent -dotnet_style_predefined_type_for_member_access = true:silent -dotnet_style_require_accessibility_modifiers = for_non_interface_members:silent -dotnet_style_allow_multiple_blank_lines_experimental = true:silent -dotnet_style_allow_statement_immediately_after_block_experimental = true:silent -dotnet_code_quality_unused_parameters = all:suggestion -dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent -dotnet_style_qualification_for_field = false:silent -dotnet_style_qualification_for_property = false:silent -dotnet_style_qualification_for_method = false:silent -dotnet_style_qualification_for_event = false:silent @@ -364,3 +364,4 @@ MigrationBackup/ *.filters **/build/* +/.editorconfig diff --git a/lib/Net.Compression/VNLib.Net.Compression/CompressionExtensions.cs b/lib/Net.Compression/VNLib.Net.Compression/CompressionExtensions.cs index ab451d9..4509b67 100644 --- a/lib/Net.Compression/VNLib.Net.Compression/CompressionExtensions.cs +++ b/lib/Net.Compression/VNLib.Net.Compression/CompressionExtensions.cs @@ -24,6 +24,7 @@ using System; using System.Buffers; +using System.Runtime.InteropServices; using VNLib.Net.Http; @@ -79,5 +80,54 @@ namespace VNLib.Net.Compression BytesWritten = (int)op->bytesWritten }; } + + /// <summary> + /// Compresses a block using the compressor context pointer provided + /// </summary> + /// <param name="nativeLib"></param> + /// <param name="comp">A pointer to the compressor context</param> + /// <param name="output">A buffer to write the result to</param> + /// <param name="input">The input block of memory to compress</param> + /// <param name="finalBlock">A value that indicates if a flush is requested</param> + /// <returns>The results of the compression operation</returns> + public static unsafe CompressionResult CompressBlock(this LibraryWrapper nativeLib, IntPtr comp, Span<byte> output, ReadOnlySpan<byte> input, bool finalBlock) + { + /* + * Since .NET only supports int32 size memory blocks + * we dont need to worry about integer overflow. + * + * Output sizes can never be larger than input + * sizes (read/written) + */ + + fixed(byte* inputPtr = &MemoryMarshal.GetReference(input), + outPtr = &MemoryMarshal.GetReference(output)) + { + //Create the operation struct + CompressionOperation operation; + CompressionOperation* op = &operation; + + op->flush = finalBlock ? 1 : 0; + op->bytesRead = 0; + op->bytesWritten = 0; + + //Configure the input and output buffers + op->inputBuffer = inputPtr; + op->inputSize = (uint)input.Length; + + op->outputBuffer = outPtr; + op->outputSize = (uint)output.Length; + + //Call the native compress function + nativeLib!.CompressBlock(comp, &operation); + + //Return the number of bytes written + return new() + { + BytesRead = (int)op->bytesRead, + BytesWritten = (int)op->bytesWritten + }; + } + } } -}
\ No newline at end of file +} diff --git a/lib/Net.Compression/VNLib.Net.Compression/INativeCompressor.cs b/lib/Net.Compression/VNLib.Net.Compression/INativeCompressor.cs index c522860..ceaa87f 100644 --- a/lib/Net.Compression/VNLib.Net.Compression/INativeCompressor.cs +++ b/lib/Net.Compression/VNLib.Net.Compression/INativeCompressor.cs @@ -54,6 +54,13 @@ namespace VNLib.Net.Compression int Flush(Memory<byte> buffer); /// <summary> + /// Flushes all remaining data in the compressor to the output buffer + /// </summary> + /// <param name="buffer">The output buffer to write flushed compressor data to</param> + /// <returns>The number of bytes written to the output buffer</returns> + int Flush(Span<byte> buffer); + + /// <summary> /// Compresses the input block and writes the compressed data to the output block /// </summary> /// <param name="input">The input buffer to compress</param> @@ -61,6 +68,13 @@ namespace VNLib.Net.Compression CompressionResult Compress(ReadOnlyMemory<byte> input, Memory<byte> output); /// <summary> + /// Compresses the input block and writes the compressed data to the output block + /// </summary> + /// <param name="input">The input buffer to compress</param> + /// <param name="output">The output buffer to write compressed data to</param> + CompressionResult Compress(ReadOnlySpan<byte> input, Span<byte> output); + + /// <summary> /// Gets the compressor block size if configured /// </summary> /// <returns>The ideal input buffer size for compressing blocks, or <![CDATA[<1]]> if block size is unlimited</returns> diff --git a/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionLib.cs b/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionLib.cs index 10bad84..438f777 100644 --- a/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionLib.cs +++ b/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionLib.cs @@ -99,6 +99,14 @@ namespace VNLib.Net.Compression } ///<inheritdoc/> + public CompressionResult Compress(ReadOnlySpan<byte> input, Span<byte> output) + { + CompressorHandle.ThrowIfClosed(); + IntPtr compressor = CompressorHandle.DangerousGetHandle(); + return LibComp.CompressBlock(compressor, output, input, false); + } + + ///<inheritdoc/> public void Dispose() => CompressorHandle.Dispose(); ///<inheritdoc/> @@ -106,7 +114,16 @@ namespace VNLib.Net.Compression { CompressorHandle.ThrowIfClosed(); IntPtr compressor = CompressorHandle.DangerousGetHandle(); - CompressionResult result = LibComp.CompressBlock(compressor, buffer, ReadOnlyMemory<byte>.Empty, true); + CompressionResult result = LibComp.CompressBlock(compressor, buffer, default, true); + return result.BytesWritten; + } + + ///<inheritdoc/> + public int Flush(Span<byte> buffer) + { + CompressorHandle.ThrowIfClosed(); + IntPtr compressor = CompressorHandle.DangerousGetHandle(); + CompressionResult result = LibComp.CompressBlock(compressor, buffer, default, true); return result.BytesWritten; } diff --git a/lib/Net.Compression/VNLib.Net.CompressionTests/CompressorManagerTests.cs b/lib/Net.Compression/VNLib.Net.CompressionTests/CompressorManagerTests.cs index 8d8deeb..95c1cc1 100644 --- a/lib/Net.Compression/VNLib.Net.CompressionTests/CompressorManagerTests.cs +++ b/lib/Net.Compression/VNLib.Net.CompressionTests/CompressorManagerTests.cs @@ -6,6 +6,7 @@ using System.Text.Json; using System.Diagnostics; using System.IO.Compression; using System.Security.Cryptography; +using System.Runtime.InteropServices; using VNLib.Utils.IO; using VNLib.Utils.Memory; @@ -60,6 +61,103 @@ namespace VNLib.Net.Compression.Tests TestCompressionForSupportedMethods(cp); } + [TestMethod()] + public void CompressorPerformanceTest() + { + //Set test level + const CompressionLevel testLevel = CompressionLevel.Fastest; + const int testItterations = 5; + + PrintSystemInformation(); + + //Load native library + using NativeCompressionLib lib = NativeCompressionLib.LoadLibrary(LIB_PATH, System.Runtime.InteropServices.DllImportSearchPath.SafeDirectories); + + //Huge array of random data to compress + byte[] testData = RandomNumberGenerator.GetBytes(10 * 1024 * 1024); + + LibTestComp cp = new(lib, testLevel); + + CompressionMethod supported = cp.GetSupportedMethods(); + + for (int itterations = 0; itterations < testItterations; itterations++) + { + if ((supported & CompressionMethod.Gzip) > 0) + { + TestSingleCompressor(cp, CompressionMethod.Gzip, testLevel, testData); + } + + if ((supported & CompressionMethod.Deflate) > 0) + { + TestSingleCompressor(cp, CompressionMethod.Deflate, testLevel, testData); + } + + if ((supported & CompressionMethod.Brotli) > 0) + { + TestSingleCompressor(cp, CompressionMethod.Brotli, testLevel, testData); + } + } + } + + private static void TestSingleCompressor(LibTestComp comp, CompressionMethod method, CompressionLevel level, byte[] testData) + { + byte[] outputBlock = new byte[8 * 1024]; + long ms; + + Stopwatch stopwatch = new (); + { + //Start sw + stopwatch.Start(); + comp.InitCompressor(method); + try + { + ForwardOnlyReader<byte> reader = new(outputBlock); + + while (true) + { + CompressionResult result = comp.CompressBlock(reader.Window, outputBlock); + reader.Advance(result.BytesRead); + + if (reader.WindowSize == 0) + { + break; + } + } + + //Flush all data + while (comp.Flush(outputBlock) != 0) + { } + } + finally + { + //Include deinit + comp.DeinitCompressor(); + stopwatch.Stop(); + ms = stopwatch.ElapsedTicks / (TimeSpan.TicksPerMillisecond / 1000); + } + } + + using (Stream compStream = GetEncodeStream(Stream.Null, method, level)) + { + stopwatch.Restart(); + try + { + //Write the block to the compression stream + compStream.Write(testData, 0, testData.Length); + } + finally + { + stopwatch.Stop(); + } + } + + long streamMicroseconds = stopwatch.ElapsedTicks / (TimeSpan.TicksPerMillisecond / 1000); + + string winner = ms < streamMicroseconds ? "native" : "stream"; + + Debug.WriteLine($"{method}: {testData.Length} bytes, {ms}misec vs {streamMicroseconds}misec. Winner {winner}"); + } + private static CompressorManager InitCompressorUnderTest() { CompressorManager manager = new(); @@ -128,7 +226,7 @@ namespace VNLib.Net.Compression.Tests Assert.ThrowsException<NotSupportedException>(() => { compressor.InitCompressor(CompressionMethod.None); }); //test out of range, this should be a native lib error - Assert.ThrowsException<NotSupportedException>(() => { compressor.InitCompressor((CompressionMethod)24); }); + Assert.ThrowsException<NotSupportedException>(() => { compressor.InitCompressor((CompressionMethod)4500); }); //Test all 3 compression methods CompressionMethod supported = compressor.GetSupportedMethods(); @@ -254,6 +352,34 @@ namespace VNLib.Net.Compression.Tests }; } + private static Stream GetEncodeStream(Stream output, CompressionMethod method, CompressionLevel level) + { + return method switch + { + CompressionMethod.Gzip => new GZipStream(output, level, true), + CompressionMethod.Deflate => new DeflateStream(output, level, true), + CompressionMethod.Brotli => new BrotliStream(output, level, true), + _ => throw new ArgumentException("Unsupported compression method", nameof(method)), + }; + } + + private static void PrintSystemInformation() + { + + + string sysInfo = @$" +OS: {RuntimeInformation.OSDescription} +Framework: {RuntimeInformation.FrameworkDescription} +Processor: {RuntimeInformation.ProcessArchitecture} +Platform ID: {Environment.OSVersion.Platform} +Is 64 bit: {Environment.Is64BitOperatingSystem} +Is 64 bit process: {Environment.Is64BitProcess} +Processor Count: {Environment.ProcessorCount} +Page Size: {Environment.SystemPageSize} +"; + Debug.WriteLine(sysInfo); + } + interface ITestCompressor { int InitCompressor(CompressionMethod method); @@ -281,12 +407,14 @@ namespace VNLib.Net.Compression.Tests } - sealed record class LibTestComp(NativeCompressionLib Library, CompressionLevel level) : ITestCompressor + sealed record class LibTestComp(NativeCompressionLib Library, CompressionLevel Level) : ITestCompressor { private INativeCompressor? _comp; public CompressionResult CompressBlock(ReadOnlyMemory<byte> input, Memory<byte> output) => _comp!.Compress(input, output); + public CompressionResult CompressBlock(ReadOnlySpan<byte> input, Span<byte> output) => _comp!.Compress(input, output); + public void DeinitCompressor() { _comp!.Dispose(); @@ -299,7 +427,7 @@ namespace VNLib.Net.Compression.Tests public int InitCompressor(CompressionMethod method) { - _comp = Library.AllocCompressor(method, level); + _comp = Library.AllocCompressor(method, Level); return (int)_comp.GetBlockSize(); } } |