From 7be5d6648e633ba46a270ca5784de6f4a5a4e0a9 Mon Sep 17 00:00:00 2001 From: vnugent Date: Sun, 28 Jul 2024 19:15:04 -0400 Subject: Squashed commit of the following: commit a4dacd2909426bf628c1eee1253cc5c8a01e2691 Author: vnugent Date: Sat Jul 27 22:41:04 2024 -0400 package updates commit f836e09981866f5c9f2ae46990d11b186a7d12bb Author: vnugent Date: Wed Jul 24 19:15:54 2024 -0400 chore: Remove argon2 docs & optional tcp resuse commit b9b892ab2143b0ab92e4dcf0a8b043c5c6c17271 Author: vnugent Date: Sun Jul 21 20:57:01 2024 -0400 fix spelling Enqueue and deprecate mispelled version commit 21ffa816f18be4b765ad740ed5d93346ec3b1fda Author: vnugent Date: Sat Jul 20 19:44:31 2024 -0400 static arugment list parsing functions commit 85cd6793818a3edd0a963bb4829a960ee6b0e022 Author: vnugent Date: Mon Jul 15 18:58:06 2024 -0400 chore: Just some minor checks and adjustments commit abfb5761ee381b7e1e5342a5525ceca8c8fd81dd Author: vnugent Date: Thu Jul 4 23:57:37 2024 -0400 analyzer pass commit 4a96dbb924f2b5bf80293e4054f221efe67151dd Author: vnugent Date: Thu Jul 4 22:45:28 2024 -0400 package updates commit 38ad7d923fa8d9e463d4aaa8e35f021086a03f2d Author: vnugent Date: Thu Jul 4 16:20:48 2024 -0400 mimalloc merge upstream upgrades commit 981ba286e4793de95bf65e6588313411344c4d53 Author: vnugent Date: Thu Jul 4 16:04:03 2024 -0400 refactor: Refactor extensions with perf updates commit 6b8c67888731f7dd210acdb2b1160cdbdbe30d47 Author: vnugent Date: Fri Jun 28 15:48:22 2024 -0400 refactor: Update service stack to reflect new loading patterns commit 12391e9a207b60b41a074600fc2373ad3eb1c3ab Author: vnugent Date: Wed Jun 26 21:01:15 2024 -0400 feat(server): Server arch update, Memory struct access commit 92e182ceaf843f8d859d38faa8b2c0ff53207ff6 Author: vnugent Date: Fri Jun 21 16:02:34 2024 -0400 feat: Multi transport listeners commit ee3620b8168a42c8e571e853c751ad5999a9b907 Author: vnugent Date: Tue Jun 18 21:17:28 2024 -0400 feat: Add file path caching support commit ff0926be56fc6eafdce36411847d73bf4ce9f183 Author: vnugent Date: Sun Jun 16 13:08:31 2024 -0400 feat: Allow multiple plugin loading directories commit 07ddf6738d32127926d07b1366e56d2a2308b53b Author: vnugent Date: Sun Jun 16 01:12:07 2024 -0400 perf: Absolutely yuge perf boosts commit ff15c05a9c3e632c39f3889820fb7d889342b452 Author: vnugent Date: Fri Jun 14 14:16:24 2024 -0400 fix: Improper request buffer property assignment commit 7d2987f1d4048c30808a85798e32c99747f6cfe3 Author: vnugent Date: Thu Jun 13 21:57:34 2024 -0400 perf: Async pre-buffer to avoid sync buffer commit 75c1d0cbf9a5a7856c544671a45f1b4312ffe7ce Author: vnugent Date: Tue Jun 11 22:11:45 2024 -0400 feat: Add a default site adapater and interceptors commit a7c739b7db9a17622cee751fe0e8a10e4b84b48b Author: vnugent Date: Sun Jun 9 13:05:12 2024 -0400 chore: Package updated commit b4b506a4b6c7c1e90b5b0980e4cfe0460e7546a2 Author: vnugent Date: Sat Jun 8 21:54:52 2024 -0400 some minor touchups commit 2160510fcc22a8574b0090fd91ca29072f45ab59 Author: vnugent Date: Fri May 31 15:12:20 2024 -0400 refactor: Immutable tcp listeners commit 51cb4eb93e4f1b4c47d35b105e72af1fe771abcc Author: vnugent Date: Thu May 30 17:31:16 2024 -0400 refactor: minor non-breaking changes to VNEncoding commit 768ddc1eb949266d693f96c67d734e881bd59374 Merge: 9a835fe 1b590c2 Author: vnugent Date: Wed May 22 17:50:57 2024 -0400 Merge branch 'main' into develop commit 9a835fe12c9586ab8dd44d7c96fef4a2d6017e4b Author: vnugent Date: Fri May 17 18:27:03 2024 -0400 chore: Update mimmaloc v2.1.6, update fPIC & cleanup commit 3b7004b88acfc7f7baa3a8857a5a2f7cf3dd560e Author: vnugent Date: Fri May 17 16:03:28 2024 -0400 feat: Added ReadFileDataAsync function commit 9a964795757bf0da4dd7fcab15ad304f4ea3fdf1 Author: vnugent Date: Wed May 15 21:57:39 2024 -0400 refactor: Harden some argon2 password hashing commit 4035c838c1508af0aa7e767a97431a692958ce1c Author: vnugent Date: Sun May 12 16:55:32 2024 -0400 perf: Utils + http perf mods commit f4f0d4f74250257991c57bfae74c4852c7e1ae46 Author: vnugent Date: Thu May 2 15:22:53 2024 -0400 feat: Buff middleware handlers | | Added implicit support for middleware post processing of files before the filehandler closes the connection. Also cleaned up some project file stuff commit f0b7dca107659df1d7d4631fdbd2aae9d716d053 Merge: 8c4a45e 107b058 Author: vnugent Date: Sat Apr 20 12:24:05 2024 -0400 Merge branch 'main' into develop commit 8c4a45e384accf92b1b6d748530e8d46f7de40d6 Author: vnugent Date: Sat Apr 20 11:10:30 2024 -0400 refactor: Overhaul C libraries and fix builds commit 42ff77080d10b0fc9fecbbc46141e8e23a1d066a Author: vnugent Date: Sat Apr 20 00:45:57 2024 -0400 fix!: Middlware array, multiple cookie set, and cookie check commit 97e82b9d66f387f9e6d21d88ddc7a8ab8693149c Merge: 4ca5791 e07537a Author: vnugent Date: Tue Apr 2 13:34:22 2024 -0400 Merge branch 'main' into develop commit 4ca5791ed67b9834bdbd010206b30373e4705e9b Author: vnugent Date: Tue Apr 2 13:32:12 2024 -0400 fix: Missed ! on null pointer check commit 9b4036377c52200c6488c98180d69a0e63321f97 Author: vnugent Date: Tue Apr 2 13:22:29 2024 -0400 fix: Fix _In_ macro for compression public api commit 53a7b4b5c5b67b4a4e06e1d9098cac4bcd6afd7c Merge: 448a93b 21130c8 Author: vnugent Date: Sun Mar 31 17:01:15 2024 -0400 Merge branch 'main' into develop commit 448a93bb1d18d032087afe2476ffccb98634a54c Author: vnugent Date: Sun Mar 31 16:56:51 2024 -0400 ci: fix third-party dir cleanup commit 9afed1427472da1ea13079f98dbe27339e55ee7d Author: vnugent Date: Sun Mar 31 16:43:15 2024 -0400 perf: Deprecate unsafememoryhandle span extensions commit 3ff90da4f02af47ea6d233fdd4445337ebe36452 Author: vnugent Date: Sat Mar 30 21:36:18 2024 -0400 refactor: Updates, advanced tracing, http optimizations commit 8d6b79b5ae309b36f265ba81529bcef8bfcd7414 Merge: 6c1667b 5585915 Author: vnugent Date: Sun Mar 24 21:01:31 2024 -0400 Merge branch 'main' into develop commit 6c1667be23597513537f8190e2f55d65eb9b7c7a Author: vnugent Date: Fri Mar 22 12:01:53 2024 -0400 refactor: Overhauled native library loading and lazy init commit ebf688f2f974295beabf7b5def7e6f6f150551d0 Author: vnugent Date: Wed Mar 20 22:16:17 2024 -0400 refactor: Update compression header files and macros + Ci build commit 9c7b564911080ccd5cbbb9851a0757b05e1e9047 Author: vnugent Date: Tue Mar 19 21:54:49 2024 -0400 refactor: JWK overhaul & add length getter to FileUpload commit 6d8c3444e09561e5957491b3cc1ae858e0abdd14 Author: vnugent Date: Mon Mar 18 16:13:20 2024 -0400 feat: Add FNV1a software checksum and basic correction tests commit 00d182088cecefc08ca80b1faee9bed3f215f40b Author: vnugent Date: Fri Mar 15 01:05:27 2024 -0400 chore: #6 Use utils filewatcher instead of built-in commit d513c10d9895c6693519ef1d459c6a5a76929436 Author: vnugent Date: Sun Mar 10 21:58:14 2024 -0400 source tree project location updated --- lib/Utils/src/IO/ArrayPoolStreamBuffer.cs | 26 +++---- lib/Utils/src/IO/IVnTextReader.cs | 7 +- lib/Utils/src/IO/VnMemoryStream.cs | 109 ++++++++++++++++++++++++----- lib/Utils/src/IO/VnStreamWriter.cs | 36 ++++------ lib/Utils/src/IO/VnTextReaderExtensions.cs | 4 +- 5 files changed, 118 insertions(+), 64 deletions(-) (limited to 'lib/Utils/src/IO') diff --git a/lib/Utils/src/IO/ArrayPoolStreamBuffer.cs b/lib/Utils/src/IO/ArrayPoolStreamBuffer.cs index b62412f..a943a5b 100644 --- a/lib/Utils/src/IO/ArrayPoolStreamBuffer.cs +++ b/lib/Utils/src/IO/ArrayPoolStreamBuffer.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Utils @@ -28,7 +28,13 @@ using System.Buffers; namespace VNLib.Utils.IO { - internal class ArrayPoolStreamBuffer : ISlindingWindowBuffer + /// + /// Creates a new from the + /// given array instance and it came from. + /// + /// The rented array to use + /// The pool to return the array to when completed + internal class ArrayPoolStreamBuffer(T[] array, ArrayPool pool) : ISlindingWindowBuffer { /// /// The shared instance to allocate buffers @@ -36,20 +42,8 @@ namespace VNLib.Utils.IO /// public static IStreamBufferFactory Shared { get; } = new DefaultFactory(); - private readonly ArrayPool _pool; - private T[] _buffer; - - /// - /// Creates a new from the - /// given array instance and it came from. - /// - /// The rented array to use - /// The pool to return the array to when completed - public ArrayPoolStreamBuffer(T[] array, ArrayPool pool) - { - _pool = pool; - _buffer = array; - } + private readonly ArrayPool _pool = pool; + private T[] _buffer = array; /// public int WindowStartPos { get; set; } diff --git a/lib/Utils/src/IO/IVnTextReader.cs b/lib/Utils/src/IO/IVnTextReader.cs index 625ba78..93de2d1 100644 --- a/lib/Utils/src/IO/IVnTextReader.cs +++ b/lib/Utils/src/IO/IVnTextReader.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2022 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Utils @@ -23,7 +23,6 @@ */ using System; -using System.IO; using System.Text; namespace VNLib.Utils.IO @@ -33,10 +32,6 @@ namespace VNLib.Utils.IO /// public interface IVnTextReader { - /// - /// The base stream to read data from - /// - Stream BaseStream { get; } /// /// The character encoding used by the TextReader /// diff --git a/lib/Utils/src/IO/VnMemoryStream.cs b/lib/Utils/src/IO/VnMemoryStream.cs index f4bf970..3f14061 100644 --- a/lib/Utils/src/IO/VnMemoryStream.cs +++ b/lib/Utils/src/IO/VnMemoryStream.cs @@ -41,15 +41,21 @@ namespace VNLib.Utils.IO /// public sealed class VnMemoryStream : Stream, ICloneable { + public const int DefaultBufferSize = 4096; + private nint _position; private nint _length; private bool _isReadonly; //Memory private readonly IResizeableMemoryHandle _buffer; + //Default owns handle private readonly bool OwnsHandle = true; + //Lazy loaded memory wrapper + private MemoryManager? _memoryWrapper; + /// /// Creates a new pointing to the begining of memory, and consumes the handle. /// @@ -78,15 +84,19 @@ namespace VNLib.Utils.IO ArgumentNullException.ThrowIfNull(handle); return handle.CanRealloc || readOnly - ? new VnMemoryStream(handle, length, readOnly, ownsHandle) + ? new VnMemoryStream(handle, existingManager: null, length, readOnly, ownsHandle) : throw new ArgumentException("The supplied memory handle must be resizable on a writable stream", nameof(handle)); } /// /// Converts a writable to readonly to allow shallow copies /// + /// + /// This funciton will convert the stream passed into it to a readonly stream. + /// The function passes through the input stream as the return value + /// /// The stream to make readonly - /// The readonly stream + /// A reference to the modified input stream public static VnMemoryStream CreateReadonly(VnMemoryStream stream) { ArgumentNullException.ThrowIfNull(stream); @@ -101,15 +111,21 @@ namespace VNLib.Utils.IO /// global heap instance. /// public VnMemoryStream() : this(MemoryUtil.Shared) { } - + /// /// Create a new memory stream where buffers will be allocated from the specified heap /// /// to allocate memory from /// /// - public VnMemoryStream(IUnmangedHeap heap) : this(heap, 0, false) { } - + public VnMemoryStream(IUnmangedHeap heap) : this(heap, DefaultBufferSize, false) { } + + /// + /// Creates a new memory stream using the + /// global heap instance. + /// + public VnMemoryStream(nuint bufferSize, bool zero) : this(MemoryUtil.Shared, bufferSize, zero) { } + /// /// Creates a new memory stream and pre-allocates the internal /// buffer of the specified size on the specified heap to avoid resizing. @@ -163,7 +179,14 @@ namespace VNLib.Utils.IO /// The length property of the stream /// Is the stream readonly (should mostly be true!) /// Does the new stream own the memory -> - private VnMemoryStream(IResizeableMemoryHandle buffer, nint length, bool readOnly, bool ownsHandle) + /// A reference to an existing memory manager class + private VnMemoryStream( + IResizeableMemoryHandle buffer, + MemoryManager? existingManager, + nint length, + bool readOnly, + bool ownsHandle + ) { Debug.Assert(length >= 0, "Length must be positive"); Debug.Assert(buffer.CanRealloc || readOnly, "The supplied buffer is not resizable on a writable stream"); @@ -172,6 +195,7 @@ namespace VNLib.Utils.IO _buffer = buffer; //Consume the handle _length = length; //Store length of the buffer _isReadonly = readOnly; + _memoryWrapper = existingManager; } /// @@ -189,7 +213,7 @@ namespace VNLib.Utils.IO //Create a new readonly copy (stream does not own the handle) return !_isReadonly ? throw new NotSupportedException("This stream is not readonly. Cannot create shallow copy on a mutable stream") - : new VnMemoryStream(_buffer, _length, true, false); + : new VnMemoryStream(_buffer, _memoryWrapper, _length, readOnly: true, ownsHandle: false); } /// @@ -247,27 +271,31 @@ namespace VNLib.Utils.IO cancellationToken.ThrowIfCancellationRequested(); + //Memory manager requires 32bit or less in length if(_length < Int32.MaxValue) { - //Safe to alloc a memory manager to do copy - using MemoryManager asMemManager = _buffer.ToMemoryManager(false); + //Get/alloc the internal memory manager and get the block + ReadOnlyMemory asMemory = AsMemory(); + + Debug.Assert(asMemory.Length >= LenToPosDiff, "Internal memory block smaller than desired for stream copy"); /* * CopyTo starts at the current position, as if calling Read() * so the reader must be offset to match and the _length gives us the * actual length of the stream and therefor the segment size - */ + */ - while(LenToPosDiff > 0) + while (LenToPosDiff > 0) { int blockSize = Math.Min((int)LenToPosDiff, bufferSize); - Memory window = asMemManager.Memory.Slice((int)_position, blockSize); + + ReadOnlyMemory window = asMemory.Slice((int)_position, blockSize); //write async await destination.WriteAsync(window, cancellationToken); //Update position - _position+= bufferSize; + _position += bufferSize; } } else @@ -354,7 +382,7 @@ namespace VNLib.Utils.IO } /// - public override unsafe int ReadByte() + public override int ReadByte() { if (LenToPosDiff == 0) { @@ -362,7 +390,7 @@ namespace VNLib.Utils.IO } //get the value at the current position - ref byte ptr = ref _buffer.GetOffsetByteRef((nuint)_position); + ref byte ptr = ref _buffer.GetOffsetByteRef(_position); //Increment position _position++; @@ -486,7 +514,7 @@ namespace VNLib.Utils.IO throw new NotSupportedException("Write operation is not allowed on readonly stream!"); } //Calculate the new final position - nint newPos = (_position + buffer.Length); + nint newPos = checked(_position + buffer.Length); //Determine if the buffer needs to be expanded if (buffer.Length > LenToPosDiff) { @@ -495,8 +523,16 @@ namespace VNLib.Utils.IO //Update length _length = newPos; } + //Copy the input buffer to the internal buffer - MemoryUtil.Copy(buffer, 0, _buffer, (nuint)_position, buffer.Length); + MemoryUtil.Copy( + source: buffer, + sourceOffset: 0, + dest: _buffer, + destOffset: (nuint)_position, + count: buffer.Length + ); + //Update the position _position = newPos; } @@ -550,6 +586,7 @@ namespace VNLib.Utils.IO /// /// Returns a window over the data within the entire stream + /// that is equal in length to the stream length. /// /// A of the data within the entire stream /// @@ -560,6 +597,40 @@ namespace VNLib.Utils.IO //Get span with no offset return _buffer.AsSpan(0, len); } + + /// + /// Returns a structure which is a window of the buffered + /// data as it currently sits. For writeable straems, you must call this function + /// every time the size of the stream changes. The memory structure is just a "pointer" to + /// the internal buffer. + /// + /// + /// A memory snapshot of the stream. + /// + /// + /// This function causes an internal allocation on the first call. After the first call + /// to this function, all calls are thread-safe. + /// + public ReadOnlyMemory AsMemory() + { + /* + * Safe cast stram length to int, because memory window requires a 32bit + * integer. Also will throw before allocating the mmemory manager + */ + + int len = Convert.ToInt32(_length); + + //Defer/lazy init the memory manager + MemoryManager asMemory = LazyInitializer.EnsureInitialized(ref _memoryWrapper, AllocMemManager); + + Debug.Assert(asMemory != null); + + /* + * Buffer window may be larger than the actual stream legnth, so + * slice the memory to the actual length of the stream + */ + return asMemory.Memory[..len]; + } /// /// If the current stream is a readonly stream, creates a shallow copy for reading only. @@ -568,5 +639,7 @@ namespace VNLib.Utils.IO /// public object Clone() => GetReadonlyShallowCopy(); + private MemoryManager AllocMemManager() => _buffer.ToMemoryManager(false); + } -} \ No newline at end of file +} diff --git a/lib/Utils/src/IO/VnStreamWriter.cs b/lib/Utils/src/IO/VnStreamWriter.cs index ddebc07..7d43f48 100644 --- a/lib/Utils/src/IO/VnStreamWriter.cs +++ b/lib/Utils/src/IO/VnStreamWriter.cs @@ -41,21 +41,29 @@ namespace VNLib.Utils.IO /// Provides a memory optimized implementation. Optimized for writing /// to network streams /// - public class VnStreamWriter : TextWriter + /// + /// Creates a new that writes encoded data to the base stream + /// and uses the specified buffer. + /// + /// The underlying stream to write data to + /// The to use when writing to the stream + /// The internal to use + /// + public class VnStreamWriter(Stream baseStream, Encoding encoding, ISlindingWindowBuffer buffer) : TextWriter { - private readonly Encoder Enc; + private readonly Encoder Enc = encoding.GetEncoder(); - private readonly ISlindingWindowBuffer _buffer; + private readonly ISlindingWindowBuffer _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); private bool closed; /// /// Gets the underlying stream that interfaces with the backing store /// - public virtual Stream BaseStream { get; } + public virtual Stream BaseStream { get; } = baseStream ?? throw new ArgumentNullException(nameof(buffer)); /// - public override Encoding Encoding { get; } + public override Encoding Encoding { get; } = encoding ?? throw new ArgumentNullException(nameof(encoding)); /// /// Line termination to use when writing lines to the output @@ -97,24 +105,6 @@ namespace VNLib.Utils.IO { } - /// - /// Creates a new that writes encoded data to the base stream - /// and uses the specified buffer. - /// - /// The underlying stream to write data to - /// The to use when writing to the stream - /// The internal to use - /// - public VnStreamWriter(Stream baseStream, Encoding encoding, ISlindingWindowBuffer buffer) - { - BaseStream = baseStream ?? throw new ArgumentNullException(nameof(buffer)); - Encoding = encoding ?? throw new ArgumentNullException(nameof(encoding)); - _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); - - //Get an encoder - Enc = encoding.GetEncoder(); - } - /// public void Write(byte value) { diff --git a/lib/Utils/src/IO/VnTextReaderExtensions.cs b/lib/Utils/src/IO/VnTextReaderExtensions.cs index 9ca5ae5..5dc6117 100644 --- a/lib/Utils/src/IO/VnTextReaderExtensions.cs +++ b/lib/Utils/src/IO/VnTextReaderExtensions.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Utils @@ -66,6 +66,7 @@ namespace VNLib.Utils.IO /// Allows reading lines of data from the stream without allocations public static ERRNO ReadLine(this T reader, Span charBuffer) where T : class, IVnTextReader { + ArgumentNullException.ThrowIfNull(reader); return ReadLineInternal(ref reader, charBuffer); } @@ -118,6 +119,7 @@ namespace VNLib.Utils.IO /// You should use the property to know how much remaining data is buffered public static int ReadRemaining(this T reader, Span buffer) where T : class, IVnTextReader { + ArgumentNullException.ThrowIfNull(reader); return ReadRemainingInternal(ref reader, buffer); } -- cgit