Compression stream and br patch

8 files changed, 106 insertions, 88 deletions

diff --git a/Module.Taskfile.yaml b/Module.Taskfile.yaml
index d729ea3..256e308 100644
--- a/Module.Taskfile.yaml
+++ b/Module.Taskfile.yaml
@@ -23,14 +23,13 @@ tasks:
      - git remote update
      - git reset --hard  
      - git pull origin {{.BRANCH_NAME}} --verify-signatures 
+     #re-write semver after hard reset
+     - dotnet-gitversion.exe /updateprojectfiles
 
 #called by build pipeline to build module
   build:
     cmds:
      - echo "building module {{.MODULE_NAME}}" 
-
-    #re-write semver after hard reset, before build
-     - dotnet-gitversion.exe /updateprojectfiles
      
      #build debug mode first
      - task: build_debug
diff --git a/lib/Net.Compression/VNLib.Net.Compression/CompressorManager.cs b/lib/Net.Compression/VNLib.Net.Compression/CompressorManager.cs
index 0695791..5a86c62 100644
--- a/lib/Net.Compression/VNLib.Net.Compression/CompressorManager.cs
+++ b/lib/Net.Compression/VNLib.Net.Compression/CompressorManager.cs
@@ -174,7 +174,7 @@ namespace VNLib.Net.Compression
             return CompressBlock(compressor.Instance, output, input, false);
         } 
         
-        private unsafe CompressionResult CompressBlock(IntPtr comp, ReadOnlyMemory<byte> output, ReadOnlyMemory<byte> input, bool finalBlock)
+        private unsafe CompressionResult CompressBlock(IntPtr comp, Memory<byte> output, ReadOnlyMemory<byte> input, bool finalBlock)
         {
             //get pointers to the input and output buffers
             using MemoryHandle inPtr = input.Pin();
diff --git a/lib/Net.Compression/vnlib_compress/feature_brotli.c b/lib/Net.Compression/vnlib_compress/feature_brotli.c
index a8b6fed..d83e2b4 100644
--- a/lib/Net.Compression/vnlib_compress/feature_brotli.c
+++ b/lib/Net.Compression/vnlib_compress/feature_brotli.c
@@ -131,7 +131,7 @@ int BrCompressBlock(CompressorState* state, CompressionOperation* operation)
 
 	if (operation->flush) 
 	{
-		brOperation = BROTLI_OPERATION_FLUSH;
+		brOperation = BROTLI_OPERATION_FINISH;
 	}
 	else
 	{
diff --git a/lib/Net.Http/src/Core/Response/ResponseWriter.cs b/lib/Net.Http/src/Core/Response/ResponseWriter.cs
index dcb220f..97b734a 100644
--- a/lib/Net.Http/src/Core/Response/ResponseWriter.cs
+++ b/lib/Net.Http/src/Core/Response/ResponseWriter.cs
@@ -35,6 +35,8 @@ using System.Threading;
 using System.Threading.Tasks;
 using System.Runtime.CompilerServices;
 
+using VNLib.Utils.IO;
+using VNLib.Utils.Memory;
 using VNLib.Net.Http.Core.Response;
 using VNLib.Net.Http.Core.Compression;
 
@@ -200,6 +202,8 @@ namespace VNLib.Net.Http.Core
             }
         }
 
+        ForwardOnlyMemoryReader<byte> _streamReader;
+
         ///<inheritdoc/>        
         async Task IHttpResponseBody.WriteEntityAsync(IResponseCompressor comp, IResponseDataWriter writer, Memory<byte> buffer)
         {
@@ -229,7 +233,7 @@ namespace VNLib.Net.Http.Core
                     buffer = buffer[..comp.BlockSize];
                 }
 
-                //Read in loop
+                //Process in loop
                 do
                 {
                     //read
@@ -241,12 +245,19 @@ namespace VNLib.Net.Http.Core
                         break;
                     }
 
-                    //Compress the buffered data and flush if required
-                    if (CompressNextSegment(buffer, read, comp, writer))
+                    //Track read bytes and loop uil all bytes are read
+                    _streamReader = new(buffer[..read]);
+                  
+                    do
                     {
-                        //Time to flush
-                        await writer.FlushAsync(false);
-                    }
+                        //Compress the buffered data and flush if required
+                        if (CompressNextSegment(ref _streamReader, comp, writer))
+                        {
+                            //Time to flush
+                            await writer.FlushAsync(false);
+                        }
+
+                    } while (_streamReader.WindowSize > 0);
 
                 } while (true);
 
@@ -309,13 +320,16 @@ namespace VNLib.Net.Http.Core
             return writer.Advance(res.BytesWritten) == 0;
         }
 
-        private static bool CompressNextSegment(Memory<byte> readSegment, int read, IResponseCompressor comp, IResponseDataWriter writer)
+        private static bool CompressNextSegment(ref ForwardOnlyMemoryReader<byte> reader, IResponseCompressor comp, IResponseDataWriter writer)
         {
             //Get output buffer
             Memory<byte> output = writer.GetMemory();
 
             //Compress the trimmed block
-            CompressionResult res = comp.CompressBlock(readSegment[..read], output);
+            CompressionResult res = comp.CompressBlock(reader.Window, output);
+
+            //Commit input bytes
+            reader.Advance(res.BytesRead);
 
             return writer.Advance(res.BytesWritten) == 0;
         }
@@ -329,6 +343,7 @@ namespace VNLib.Net.Http.Core
             HasData = false;
             Length = 0;
             _readSegment = default;
+            _streamReader = default;
 
             //Clear rseponse containers
             _streamResponse?.Dispose();
diff --git a/lib/Utils/src/ERRNO.cs b/lib/Utils/src/ERRNO.cs
index 03ebd17..3bc1296 100644
--- a/lib/Utils/src/ERRNO.cs
+++ b/lib/Utils/src/ERRNO.cs
@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2022 Vaughn Nugent
+* Copyright (c) 2023 Vaughn Nugent
 * 
 * Library: VNLib
 * Package: VNLib.Utils
@@ -44,83 +44,80 @@ namespace VNLib.Utils
         public static readonly ERRNO E_FAIL = false;
         
         private readonly nint ErrorCode;
+
         /// <summary>
         /// Creates a new <see cref="ERRNO"/> from the specified error value
         /// </summary>
         /// <param name="errno">The value of the error to represent</param>
         public ERRNO(nint errno) => ErrorCode = errno;
+
         /// <summary>
         /// Creates a new <see cref="ERRNO"/> from an <see cref="int"/> error code. null = 0 = false
         /// </summary>
         /// <param name="errorVal">Error code</param>
         public static implicit operator ERRNO(int errorVal) => new (errorVal);
+
         /// <summary>
         /// Creates a new <see cref="ERRNO"/> from an <see cref="int"/> error code. null = 0 = false
         /// </summary>
         /// <param name="errorVal">Error code</param>
         public static explicit operator ERRNO(int? errorVal) => new(errorVal ?? 0);
+
         /// <summary>
         /// Creates a new <see cref="ERRNO"/> from a booleam, 1 if true, 0 if false
         /// </summary>
         /// <param name="errorVal"></param>
         public static implicit operator ERRNO(bool errorVal) => new(errorVal ? 1 : 0);
+
         /// <summary>
         /// Creates a new <see cref="ERRNO"/> from a pointer value
         /// </summary>
         /// <param name="errno">The pointer value representing an error code</param>
         public static implicit operator ERRNO(nint errno) => new(errno);
+
         /// <summary>
         /// Error value as integer. Value of supplied error code or if cast from boolean 1 if true, 0 if false
         /// </summary>
         /// <param name="errorVal"><see cref="ERRNO"/> to get error code from</param>
         public static implicit operator int(ERRNO errorVal) => (int)errorVal.ErrorCode;
+
         /// <summary>
         /// C style boolean conversion. false if 0, true otherwise 
         /// </summary>
         /// <param name="errorVal"></param>
-        public static implicit operator bool(ERRNO errorVal) => errorVal != 0;       
+        public static implicit operator bool(ERRNO errorVal) => errorVal != 0;      
+        
         /// <summary>
         /// Creates a new <see cref="IntPtr"/> from the value if the stored (nint) error code 
         /// </summary>
         /// <param name="errno">The <see cref="ERRNO"/> contating the pointer value</param>
         public static implicit operator IntPtr(ERRNO errno) => new(errno.ErrorCode);
+
         /// <summary>
         /// Creates a new <c>nint</c> from the value if the stored error code 
         /// </summary>
         /// <param name="errno">The <see cref="ERRNO"/> contating the pointer value</param>
         public static implicit operator nint(ERRNO errno) => errno.ErrorCode;
 
-        public static ERRNO operator +(ERRNO err, int add) => new(err.ErrorCode + add);
-        public static ERRNO operator +(ERRNO err, nint add) => new(err.ErrorCode + add);
-        public static ERRNO operator ++(ERRNO err) => new(err.ErrorCode + 1);
-        public static ERRNO operator --(ERRNO err) => new(err.ErrorCode - 1);
-        public static ERRNO operator -(ERRNO err, int subtract) => new(err.ErrorCode - subtract);
-        public static ERRNO operator -(ERRNO err, nint subtract) => new(err.ErrorCode - subtract);
-
-        public static bool operator >(ERRNO err, ERRNO other) => err.ErrorCode > other.ErrorCode;
-        public static bool operator <(ERRNO err, ERRNO other) => err.ErrorCode < other.ErrorCode;
-        public static bool operator >=(ERRNO err, ERRNO other) => err.ErrorCode >= other.ErrorCode;
-        public static bool operator <=(ERRNO err, ERRNO other) => err.ErrorCode <= other.ErrorCode;
-
-        public static bool operator >(ERRNO err, int other) => err.ErrorCode > other;
-        public static bool operator <(ERRNO err, int other) => err.ErrorCode < other;
-        public static bool operator >=(ERRNO err, int other) => err.ErrorCode >= other;
-        public static bool operator <=(ERRNO err, int other) => err.ErrorCode <= other;
-
-        public static bool operator >(ERRNO err, nint other) => err.ErrorCode > other;
-        public static bool operator <(ERRNO err, nint other) => err.ErrorCode < other;
-        public static bool operator >=(ERRNO err, nint other) => err.ErrorCode >= other;
-        public static bool operator <=(ERRNO err, nint other) => err.ErrorCode <= other;
-
-        public static bool operator ==(ERRNO err, ERRNO other) => err.ErrorCode == other.ErrorCode;
-        public static bool operator !=(ERRNO err, ERRNO other) => err.ErrorCode != other.ErrorCode;
-        public static bool operator ==(ERRNO err, int other) => err.ErrorCode == other;
-        public static bool operator !=(ERRNO err, int other) => err.ErrorCode != other;
-        public static bool operator ==(ERRNO err, nint other) => err.ErrorCode == other;
-        public static bool operator !=(ERRNO err, nint other) => err.ErrorCode != other;
-
+        /// <summary>
+        /// Compares the value of this error code to another and returns true if they are equal
+        /// </summary>
+        /// <param name="other">The value to compare</param>
+        /// <returns>True if the ERRNO value is equal to the current value</returns>
         public readonly bool Equals(ERRNO other) => ErrorCode == other.ErrorCode;
+
+        /// <summary>
+        /// Compares the value of this error code to another and returns true if they are equal. 
+        /// You should avoid this overload as it will box the value type.
+        /// </summary>
+        /// <param name="obj">The instance to compare</param>
+        /// <returns>True if the ERRNO value is equal to the current value</returns>
         public readonly override bool Equals(object? obj) => obj is ERRNO other && Equals(other);
+
+        /// <summary>
+        /// Returns the hash code of the underlying value
+        /// </summary>
+        /// <returns>The hashcode of the current value</returns>
         public readonly override int GetHashCode() => ErrorCode.GetHashCode();
 
         /// <summary>
@@ -144,32 +141,50 @@ namespace VNLib.Utils
         /// The integer error value of the current instance in radix 10
         /// </summary>
         /// <returns>The radix 10 formatted error code</returns>
-        public readonly override string ToString()
-        {
-            //Return the string of the error code number
-            return ErrorCode.ToString();
-        }
+        public readonly override string ToString() => ErrorCode.ToString();
+
         /// <summary>
         /// Formats the internal nint error code as a string in specified format
         /// </summary>
         /// <param name="format">The format to use</param>
         /// <returns>The formatted error code</returns>
-        public readonly string ToString(string format)
-        {
-            //Return the string of the error code number
-            return ErrorCode.ToString(format);
-        }
+        public readonly string ToString(string format) => ErrorCode.ToString(format);
 
         ///<inheritdoc/>
-        public readonly bool TryFormat(Span<char> destination, out int charsWritten, ReadOnlySpan<char> format, IFormatProvider provider)
-        {
-            return ErrorCode.TryFormat(destination, out charsWritten, format, provider);
-        }
-        
+        public readonly bool TryFormat(Span<char> destination, out int charsWritten, ReadOnlySpan<char> format, IFormatProvider? provider) => ErrorCode.TryFormat(destination, out charsWritten, format, provider);
+
         ///<inheritdoc/>
-        public readonly string ToString(string format, IFormatProvider formatProvider)
-        {
-            return ErrorCode.ToString(format, formatProvider);
-        }
+        public readonly string ToString(string format, IFormatProvider? formatProvider) => ErrorCode.ToString(format, formatProvider);
+
+        public static ERRNO operator +(ERRNO err, int add) => new(err.ErrorCode + add);
+        public static ERRNO operator +(ERRNO err, nint add) => new(err.ErrorCode + add);
+        public static ERRNO operator ++(ERRNO err) => new(err.ErrorCode + 1);
+        public static ERRNO operator --(ERRNO err) => new(err.ErrorCode - 1);
+        public static ERRNO operator -(ERRNO err, int subtract) => new(err.ErrorCode - subtract);
+        public static ERRNO operator -(ERRNO err, nint subtract) => new(err.ErrorCode - subtract);
+
+        public static bool operator >(ERRNO err, ERRNO other) => err.ErrorCode > other.ErrorCode;
+        public static bool operator <(ERRNO err, ERRNO other) => err.ErrorCode < other.ErrorCode;
+        public static bool operator >=(ERRNO err, ERRNO other) => err.ErrorCode >= other.ErrorCode;
+        public static bool operator <=(ERRNO err, ERRNO other) => err.ErrorCode <= other.ErrorCode;
+
+        public static bool operator >(ERRNO err, int other) => err.ErrorCode > other;
+        public static bool operator <(ERRNO err, int other) => err.ErrorCode < other;
+        public static bool operator >=(ERRNO err, int other) => err.ErrorCode >= other;
+        public static bool operator <=(ERRNO err, int other) => err.ErrorCode <= other;
+
+        public static bool operator >(ERRNO err, nint other) => err.ErrorCode > other;
+        public static bool operator <(ERRNO err, nint other) => err.ErrorCode < other;
+        public static bool operator >=(ERRNO err, nint other) => err.ErrorCode >= other;
+        public static bool operator <=(ERRNO err, nint other) => err.ErrorCode <= other;
+
+        public static bool operator ==(ERRNO err, ERRNO other) => err.ErrorCode == other.ErrorCode;
+        public static bool operator !=(ERRNO err, ERRNO other) => err.ErrorCode != other.ErrorCode;
+        public static bool operator ==(ERRNO err, int other) => err.ErrorCode == other;
+        public static bool operator !=(ERRNO err, int other) => err.ErrorCode != other;
+        public static bool operator ==(ERRNO err, nint other) => err.ErrorCode == other;
+        public static bool operator !=(ERRNO err, nint other) => err.ErrorCode != other;
+
+        
     }
 }
diff --git a/lib/Utils/src/Memory/MemoryUtil.cs b/lib/Utils/src/Memory/MemoryUtil.cs
index 7369fc0..f4482c0 100644
--- a/lib/Utils/src/Memory/MemoryUtil.cs
+++ b/lib/Utils/src/Memory/MemoryUtil.cs
@@ -232,14 +232,11 @@ namespace VNLib.Utils.Memory
             }
             
             uint byteSize = ByteCount<T>((uint)block.Length);
-            
-            checked
+
+            fixed (void* ptr = &MemoryMarshal.GetReference(block))
             {
-                fixed (void* ptr = &MemoryMarshal.GetReference(block))
-                {
-                    //Calls memset
-                    Unsafe.InitBlock(ptr, 0, byteSize);
-                }
+                //Calls memset
+                Unsafe.InitBlock(ptr, 0, byteSize);
             }
         }
 
@@ -257,14 +254,11 @@ namespace VNLib.Utils.Memory
             }
             
             uint byteSize = ByteCount<T>((uint)block.Length);
-            
-            checked
-            {
-                //Pin memory and get pointer
-                using MemoryHandle handle = block.Pin();
-                //Calls memset
-                Unsafe.InitBlock(handle.Pointer, 0, byteSize);
-            }
+
+            //Pin memory and get pointer
+            using MemoryHandle handle = block.Pin();
+            //Calls memset
+            Unsafe.InitBlock(handle.Pointer, 0, byteSize);
         }
 
         /*
@@ -341,12 +335,9 @@ namespace VNLib.Utils.Memory
         /// </summary>
         /// <typeparam name="T">The structure type</typeparam>
         /// <param name="structPtr">The pointer to the allocated structure</param>
-        public static void ZeroStruct<T>(T* structPtr) where T : unmanaged
-        {
-            //Zero block
-            Unsafe.InitBlock(structPtr, 0, (uint)sizeof(T));
-        }
-     
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void ZeroStruct<T>(T* structPtr) where T : unmanaged => Unsafe.InitBlock(structPtr, 0, (uint)sizeof(T));
+
         #endregion
 
         #region Copy
diff --git a/lib/Utils/src/Memory/SubSequence.cs b/lib/Utils/src/Memory/SubSequence.cs
index def14ed..1db0ba5 100644
--- a/lib/Utils/src/Memory/SubSequence.cs
+++ b/lib/Utils/src/Memory/SubSequence.cs
@@ -77,6 +77,8 @@ namespace VNLib.Utils.Memory
         /// <param name="offset">The relative offset from the current window offset</param>
         /// <param name="size">The size of the block</param>
         /// <returns>A <see cref="SubSequence{T}"/> of the current sequence</returns>
+        /// <exception cref="OverflowException"></exception>
+        /// <exception cref="ArgumentOutOfRangeException"></exception>
         public readonly SubSequence<T> Slice(nuint offset, int size)
         {
             //Calc offset
diff --git a/lib/Utils/src/VNLib.Utils.csproj b/lib/Utils/src/VNLib.Utils.csproj
index 90a809d..12e6a5e 100644
--- a/lib/Utils/src/VNLib.Utils.csproj
+++ b/lib/Utils/src/VNLib.Utils.csproj
@@ -25,10 +25,6 @@
     <PackageProjectUrl>https://www.vaughnnugent.com/resources/software/modules/VNLib.Core</PackageProjectUrl>
     <RepositoryUrl>https://github.com/VnUgE/VNLib.Core/tree/main/lib/Utils</RepositoryUrl>
   </PropertyGroup>
-  
-  <ItemGroup>
-    <None Include="..\.editorconfig" Link=".editorconfig" />
-  </ItemGroup>
 
   <ItemGroup>
     <None Include="..\README.md">