diff options
Diffstat (limited to 'lib/Net.Compression')
4 files changed, 214 insertions, 5 deletions
diff --git a/lib/Net.Compression/VNLib.Net.Compression/CompressionExtensions.cs b/lib/Net.Compression/VNLib.Net.Compression/CompressionExtensions.cs index ab451d9..4509b67 100644 --- a/lib/Net.Compression/VNLib.Net.Compression/CompressionExtensions.cs +++ b/lib/Net.Compression/VNLib.Net.Compression/CompressionExtensions.cs @@ -24,6 +24,7 @@ using System; using System.Buffers; +using System.Runtime.InteropServices; using VNLib.Net.Http; @@ -79,5 +80,54 @@ namespace VNLib.Net.Compression BytesWritten = (int)op->bytesWritten }; } + + /// <summary> + /// Compresses a block using the compressor context pointer provided + /// </summary> + /// <param name="nativeLib"></param> + /// <param name="comp">A pointer to the compressor context</param> + /// <param name="output">A buffer to write the result to</param> + /// <param name="input">The input block of memory to compress</param> + /// <param name="finalBlock">A value that indicates if a flush is requested</param> + /// <returns>The results of the compression operation</returns> + public static unsafe CompressionResult CompressBlock(this LibraryWrapper nativeLib, IntPtr comp, Span<byte> output, ReadOnlySpan<byte> input, bool finalBlock) + { + /* + * Since .NET only supports int32 size memory blocks + * we dont need to worry about integer overflow. + * + * Output sizes can never be larger than input + * sizes (read/written) + */ + + fixed(byte* inputPtr = &MemoryMarshal.GetReference(input), + outPtr = &MemoryMarshal.GetReference(output)) + { + //Create the operation struct + CompressionOperation operation; + CompressionOperation* op = &operation; + + op->flush = finalBlock ? 1 : 0; + op->bytesRead = 0; + op->bytesWritten = 0; + + //Configure the input and output buffers + op->inputBuffer = inputPtr; + op->inputSize = (uint)input.Length; + + op->outputBuffer = outPtr; + op->outputSize = (uint)output.Length; + + //Call the native compress function + nativeLib!.CompressBlock(comp, &operation); + + //Return the number of bytes written + return new() + { + BytesRead = (int)op->bytesRead, + BytesWritten = (int)op->bytesWritten + }; + } + } } -}
\ No newline at end of file +} diff --git a/lib/Net.Compression/VNLib.Net.Compression/INativeCompressor.cs b/lib/Net.Compression/VNLib.Net.Compression/INativeCompressor.cs index c522860..ceaa87f 100644 --- a/lib/Net.Compression/VNLib.Net.Compression/INativeCompressor.cs +++ b/lib/Net.Compression/VNLib.Net.Compression/INativeCompressor.cs @@ -54,6 +54,13 @@ namespace VNLib.Net.Compression int Flush(Memory<byte> buffer); /// <summary> + /// Flushes all remaining data in the compressor to the output buffer + /// </summary> + /// <param name="buffer">The output buffer to write flushed compressor data to</param> + /// <returns>The number of bytes written to the output buffer</returns> + int Flush(Span<byte> buffer); + + /// <summary> /// Compresses the input block and writes the compressed data to the output block /// </summary> /// <param name="input">The input buffer to compress</param> @@ -61,6 +68,13 @@ namespace VNLib.Net.Compression CompressionResult Compress(ReadOnlyMemory<byte> input, Memory<byte> output); /// <summary> + /// Compresses the input block and writes the compressed data to the output block + /// </summary> + /// <param name="input">The input buffer to compress</param> + /// <param name="output">The output buffer to write compressed data to</param> + CompressionResult Compress(ReadOnlySpan<byte> input, Span<byte> output); + + /// <summary> /// Gets the compressor block size if configured /// </summary> /// <returns>The ideal input buffer size for compressing blocks, or <![CDATA[<1]]> if block size is unlimited</returns> diff --git a/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionLib.cs b/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionLib.cs index 10bad84..438f777 100644 --- a/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionLib.cs +++ b/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionLib.cs @@ -99,6 +99,14 @@ namespace VNLib.Net.Compression } ///<inheritdoc/> + public CompressionResult Compress(ReadOnlySpan<byte> input, Span<byte> output) + { + CompressorHandle.ThrowIfClosed(); + IntPtr compressor = CompressorHandle.DangerousGetHandle(); + return LibComp.CompressBlock(compressor, output, input, false); + } + + ///<inheritdoc/> public void Dispose() => CompressorHandle.Dispose(); ///<inheritdoc/> @@ -106,7 +114,16 @@ namespace VNLib.Net.Compression { CompressorHandle.ThrowIfClosed(); IntPtr compressor = CompressorHandle.DangerousGetHandle(); - CompressionResult result = LibComp.CompressBlock(compressor, buffer, ReadOnlyMemory<byte>.Empty, true); + CompressionResult result = LibComp.CompressBlock(compressor, buffer, default, true); + return result.BytesWritten; + } + + ///<inheritdoc/> + public int Flush(Span<byte> buffer) + { + CompressorHandle.ThrowIfClosed(); + IntPtr compressor = CompressorHandle.DangerousGetHandle(); + CompressionResult result = LibComp.CompressBlock(compressor, buffer, default, true); return result.BytesWritten; } diff --git a/lib/Net.Compression/VNLib.Net.CompressionTests/CompressorManagerTests.cs b/lib/Net.Compression/VNLib.Net.CompressionTests/CompressorManagerTests.cs index 8d8deeb..95c1cc1 100644 --- a/lib/Net.Compression/VNLib.Net.CompressionTests/CompressorManagerTests.cs +++ b/lib/Net.Compression/VNLib.Net.CompressionTests/CompressorManagerTests.cs @@ -6,6 +6,7 @@ using System.Text.Json; using System.Diagnostics; using System.IO.Compression; using System.Security.Cryptography; +using System.Runtime.InteropServices; using VNLib.Utils.IO; using VNLib.Utils.Memory; @@ -60,6 +61,103 @@ namespace VNLib.Net.Compression.Tests TestCompressionForSupportedMethods(cp); } + [TestMethod()] + public void CompressorPerformanceTest() + { + //Set test level + const CompressionLevel testLevel = CompressionLevel.Fastest; + const int testItterations = 5; + + PrintSystemInformation(); + + //Load native library + using NativeCompressionLib lib = NativeCompressionLib.LoadLibrary(LIB_PATH, System.Runtime.InteropServices.DllImportSearchPath.SafeDirectories); + + //Huge array of random data to compress + byte[] testData = RandomNumberGenerator.GetBytes(10 * 1024 * 1024); + + LibTestComp cp = new(lib, testLevel); + + CompressionMethod supported = cp.GetSupportedMethods(); + + for (int itterations = 0; itterations < testItterations; itterations++) + { + if ((supported & CompressionMethod.Gzip) > 0) + { + TestSingleCompressor(cp, CompressionMethod.Gzip, testLevel, testData); + } + + if ((supported & CompressionMethod.Deflate) > 0) + { + TestSingleCompressor(cp, CompressionMethod.Deflate, testLevel, testData); + } + + if ((supported & CompressionMethod.Brotli) > 0) + { + TestSingleCompressor(cp, CompressionMethod.Brotli, testLevel, testData); + } + } + } + + private static void TestSingleCompressor(LibTestComp comp, CompressionMethod method, CompressionLevel level, byte[] testData) + { + byte[] outputBlock = new byte[8 * 1024]; + long ms; + + Stopwatch stopwatch = new (); + { + //Start sw + stopwatch.Start(); + comp.InitCompressor(method); + try + { + ForwardOnlyReader<byte> reader = new(outputBlock); + + while (true) + { + CompressionResult result = comp.CompressBlock(reader.Window, outputBlock); + reader.Advance(result.BytesRead); + + if (reader.WindowSize == 0) + { + break; + } + } + + //Flush all data + while (comp.Flush(outputBlock) != 0) + { } + } + finally + { + //Include deinit + comp.DeinitCompressor(); + stopwatch.Stop(); + ms = stopwatch.ElapsedTicks / (TimeSpan.TicksPerMillisecond / 1000); + } + } + + using (Stream compStream = GetEncodeStream(Stream.Null, method, level)) + { + stopwatch.Restart(); + try + { + //Write the block to the compression stream + compStream.Write(testData, 0, testData.Length); + } + finally + { + stopwatch.Stop(); + } + } + + long streamMicroseconds = stopwatch.ElapsedTicks / (TimeSpan.TicksPerMillisecond / 1000); + + string winner = ms < streamMicroseconds ? "native" : "stream"; + + Debug.WriteLine($"{method}: {testData.Length} bytes, {ms}misec vs {streamMicroseconds}misec. Winner {winner}"); + } + private static CompressorManager InitCompressorUnderTest() { CompressorManager manager = new(); @@ -128,7 +226,7 @@ namespace VNLib.Net.Compression.Tests Assert.ThrowsException<NotSupportedException>(() => { compressor.InitCompressor(CompressionMethod.None); }); //test out of range, this should be a native lib error - Assert.ThrowsException<NotSupportedException>(() => { compressor.InitCompressor((CompressionMethod)24); }); + Assert.ThrowsException<NotSupportedException>(() => { compressor.InitCompressor((CompressionMethod)4500); }); //Test all 3 compression methods CompressionMethod supported = compressor.GetSupportedMethods(); @@ -254,6 +352,34 @@ namespace VNLib.Net.Compression.Tests }; } + private static Stream GetEncodeStream(Stream output, CompressionMethod method, CompressionLevel level) + { + return method switch + { + CompressionMethod.Gzip => new GZipStream(output, level, true), + CompressionMethod.Deflate => new DeflateStream(output, level, true), + CompressionMethod.Brotli => new BrotliStream(output, level, true), + _ => throw new ArgumentException("Unsupported compression method", nameof(method)), + }; + } + + private static void PrintSystemInformation() + { + + + string sysInfo = @$" +OS: {RuntimeInformation.OSDescription} +Framework: {RuntimeInformation.FrameworkDescription} +Processor: {RuntimeInformation.ProcessArchitecture} +Platform ID: {Environment.OSVersion.Platform} +Is 64 bit: {Environment.Is64BitOperatingSystem} +Is 64 bit process: {Environment.Is64BitProcess} +Processor Count: {Environment.ProcessorCount} +Page Size: {Environment.SystemPageSize} +"; + Debug.WriteLine(sysInfo); + } + interface ITestCompressor { int InitCompressor(CompressionMethod method); @@ -281,12 +407,14 @@ namespace VNLib.Net.Compression.Tests } - sealed record class LibTestComp(NativeCompressionLib Library, CompressionLevel level) : ITestCompressor + sealed record class LibTestComp(NativeCompressionLib Library, CompressionLevel Level) : ITestCompressor { private INativeCompressor? _comp; public CompressionResult CompressBlock(ReadOnlyMemory<byte> input, Memory<byte> output) => _comp!.Compress(input, output); + public CompressionResult CompressBlock(ReadOnlySpan<byte> input, Span<byte> output) => _comp!.Compress(input, output); + public void DeinitCompressor() { _comp!.Dispose(); @@ -299,7 +427,7 @@ namespace VNLib.Net.Compression.Tests public int InitCompressor(CompressionMethod method) { - _comp = Library.AllocCompressor(method, level); + _comp = Library.AllocCompressor(method, Level); return (int)_comp.GetBlockSize(); } } |