From 07ddf6738d32127926d07b1366e56d2a2308b53b Mon Sep 17 00:00:00 2001 From: vnugent Date: Sun, 16 Jun 2024 01:12:07 -0400 Subject: perf: Absolutely yuge perf boosts --- lib/Net.Http/src/Core/HttpContext.cs | 16 +- lib/Net.Http/src/Core/HttpEncodedSegment.cs | 16 +- lib/Net.Http/src/Core/HttpEvent.cs | 9 +- lib/Net.Http/src/Core/IHttpContextInformation.cs | 7 - lib/Net.Http/src/Core/Request/HttpInputStream.cs | 35 +- lib/Net.Http/src/Core/Request/HttpRequest.cs | 43 ++- .../src/Core/Request/HttpRequestExtensions.cs | 53 +-- .../src/Core/RequestParse/Http11ParseExtensions.cs | 89 +++-- .../src/Core/Response/HeaderDataAccumulator.cs | 14 +- lib/Net.Http/src/Core/Response/HttpResponse.cs | 44 +-- .../src/Core/Response/ReusableResponseStream.cs | 46 --- lib/Net.Http/src/Core/Response/TransportManager.cs | 99 ++++++ .../src/ITransportInterface.cs | 13 + .../src/ReusableNetworkStream.cs | 33 +- .../src/SocketPipeLineWorker.cs | 54 +-- lib/Utils/src/Extensions/MemoryExtensions.cs | 102 +----- lib/Utils/src/IO/VnMemoryStream.cs | 4 +- lib/Utils/src/Memory/MemoryUtilAlloc.cs | 378 ++++++++++++++------- lib/Utils/src/Memory/UnmanagedHeapBase.cs | 16 +- lib/Utils/tests/Memory/MemoryUtilTests.cs | 6 +- 20 files changed, 634 insertions(+), 443 deletions(-) delete mode 100644 lib/Net.Http/src/Core/Response/ReusableResponseStream.cs create mode 100644 lib/Net.Http/src/Core/Response/TransportManager.cs diff --git a/lib/Net.Http/src/Core/HttpContext.cs b/lib/Net.Http/src/Core/HttpContext.cs index 8880d49..18aef69 100644 --- a/lib/Net.Http/src/Core/HttpContext.cs +++ b/lib/Net.Http/src/Core/HttpContext.cs @@ -23,7 +23,6 @@ */ using System; -using System.IO; using System.Text; using System.Threading; using System.Diagnostics; @@ -81,6 +80,7 @@ namespace VNLib.Net.Http.Core /// public IAlternateProtocol? AlternateProtocol { get; set; } + private readonly TransportManager Transport; private readonly ManagedHttpCompressor? _compressor; private ITransportContext? _ctx; @@ -105,12 +105,14 @@ namespace VNLib.Net.Http.Core _compressor = null; } + Transport = new(); + //Init buffer manager, if compression is supported, we need to alloc a buffer for the compressor Buffers = new(server.Config.BufferConfig, _compressor != null); - Request = new (this, server.Config.MaxUploadsPerRequest); + Request = new (Transport, server.Config.MaxUploadsPerRequest); - Response = new (this, Buffers); + Response = new (this, Transport, Buffers); ResponseBody = new ResponseWriter(); } @@ -134,9 +136,7 @@ namespace VNLib.Net.Http.Core /// public ref readonly HttpEncodedSegment FinalChunkSegment => ref ParentServer.Config.FinalChunkBytes; - - /// - public Stream GetTransport() => _ctx!.ConnectionStream; + int _bytesRead; @@ -212,7 +212,7 @@ namespace VNLib.Net.Http.Core Buffers.AllocateBuffer(ParentServer.Config.MemoryPool); //Init new connection - Response.OnNewConnection(ctx.ConnectionStream); + Transport.OnNewConnection(ctx.ConnectionStream); } /// @@ -252,6 +252,8 @@ namespace VNLib.Net.Http.Core bool IReusable.Release() { + Transport.OnRelease(); + _ctx = null; AlternateProtocol = null; diff --git a/lib/Net.Http/src/Core/HttpEncodedSegment.cs b/lib/Net.Http/src/Core/HttpEncodedSegment.cs index bcb1f3b..cfeb52e 100644 --- a/lib/Net.Http/src/Core/HttpEncodedSegment.cs +++ b/lib/Net.Http/src/Core/HttpEncodedSegment.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Net.Http @@ -51,12 +51,15 @@ namespace VNLib.Net.Http.Core /// Performs a dangerous reference based copy-to (aka memmove) /// /// The output buffer to write the encoded segment to - internal readonly int DangerousCopyTo(Span output) + /// Points to the first byte in the buffer to write to + internal readonly int DangerousCopyTo(Span output, int offset) { Debug.Assert(output.Length >= Length, "Output span was empty and could not be written to"); + Debug.Assert(offset >= 0, "Buffer underrun detected"); + Debug.Assert(offset + Length <= output.Length, "Output span was too small to hold the encoded segment"); //Get reference of output buffer span - return DangerousCopyTo(ref MemoryMarshal.GetReference(output)); + return DangerousCopyTo(ref MemoryMarshal.GetReference(output), (nuint)offset); } /// @@ -73,14 +76,15 @@ namespace VNLib.Net.Http.Core //Ensure enough space is available if(offset + Length <= buffer.Size) { + //More efficient to get the offset ref from the buffer directly ref byte dst = ref buffer.DangerousGetBinRef(offset); - return DangerousCopyTo(ref dst); + return DangerousCopyTo(ref dst, destOffset: 0); } throw new ArgumentOutOfRangeException(nameof(offset), "Buffer is too small to hold the encoded segment"); } - private readonly int DangerousCopyTo(ref byte output) + private readonly int DangerousCopyTo(ref byte output, nuint destOffset) { Debug.Assert(!Unsafe.IsNullRef(ref output), "Output span was empty and could not be written to"); @@ -88,7 +92,7 @@ namespace VNLib.Net.Http.Core ref byte src = ref MemoryMarshal.GetArrayDataReference(Buffer); //Call memmove with the buffer offset and desired length - MemoryUtil.SmallMemmove(ref src, Offset, ref output, 0, Length); + MemoryUtil.SmallMemmove(ref src, Offset, ref output, destOffset, Length); return Length; } diff --git a/lib/Net.Http/src/Core/HttpEvent.cs b/lib/Net.Http/src/Core/HttpEvent.cs index 37c5ab5..270e7f8 100644 --- a/lib/Net.Http/src/Core/HttpEvent.cs +++ b/lib/Net.Http/src/Core/HttpEvent.cs @@ -91,11 +91,11 @@ namespace VNLib.Net.Http //If stream is empty, ignore it, the server will default to 0 content length and avoid overhead if (length == 0) { + //Stream is disposed because it is assumed we now own the lifecycle of the stream stream.Dispose(); return; } - - //Set status code + Context.Response.SetStatusCode(code); //Finally store the stream input @@ -120,11 +120,11 @@ namespace VNLib.Net.Http //If stream is empty, ignore it, the server will default to 0 content length and avoid overhead if (entity.Remaining == 0) { + //Stream is disposed because it is assumed we now own the lifecycle of the stream entity.Close(); return; } - - //Set status code + Context.Response.SetStatusCode(code); //Store the memory reader input @@ -150,6 +150,7 @@ namespace VNLib.Net.Http //If stream is empty, ignore it, the server will default to 0 content length and avoid overhead if (length == 0) { + //Stream is disposed because it is assumed we now own the lifecycle of the stream stream.Dispose(); return; } diff --git a/lib/Net.Http/src/Core/IHttpContextInformation.cs b/lib/Net.Http/src/Core/IHttpContextInformation.cs index 38e86b3..cb51e9f 100644 --- a/lib/Net.Http/src/Core/IHttpContextInformation.cs +++ b/lib/Net.Http/src/Core/IHttpContextInformation.cs @@ -22,7 +22,6 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -using System.IO; using System.Text; namespace VNLib.Net.Http.Core @@ -48,11 +47,5 @@ namespace VNLib.Net.Http.Core /// The current connection's http version /// HttpVersion CurrentVersion { get; } - - /// - /// Gets the transport stream for the current connection. - /// - /// The current transport stream - Stream GetTransport(); } } \ No newline at end of file diff --git a/lib/Net.Http/src/Core/Request/HttpInputStream.cs b/lib/Net.Http/src/Core/Request/HttpInputStream.cs index ccaa336..9ad0218 100644 --- a/lib/Net.Http/src/Core/Request/HttpInputStream.cs +++ b/lib/Net.Http/src/Core/Request/HttpInputStream.cs @@ -32,13 +32,14 @@ using System.Runtime.CompilerServices; using VNLib.Utils; using VNLib.Utils.Memory; using VNLib.Utils.Extensions; +using VNLib.Net.Http.Core.Response; namespace VNLib.Net.Http.Core { /// /// Specialized stream to allow reading a request entity body with a fixed content length. /// - internal sealed class HttpInputStream(IHttpContextInformation ContextInfo) : Stream + internal sealed class HttpInputStream(TransportManager transport) : Stream { private StreamState _state; private InitDataBuffer? _initalData; @@ -74,7 +75,6 @@ namespace VNLib.Net.Http.Core internal ref InitDataBuffer? Prepare(long contentLength) { _state.ContentLength = contentLength; - _state.InputStream = ContextInfo.GetTransport(); return ref _initalData; } @@ -153,10 +153,8 @@ namespace VNLib.Net.Http.Core //See if data is still remaining to be read from transport (reamining size is also the amount of data that can be read) if (writer.RemainingSize > 0) { - //Read from transport - ERRNO read = _state.InputStream!.Read(writer.Remaining); + ERRNO read = transport.Stream!.Read(writer.Remaining); - //Update writer position writer.Advance(read); _state.Position += read; @@ -189,27 +187,23 @@ namespace VNLib.Net.Http.Core { //Read as much as possible from internal buffer ERRNO read = _initalData.Value.Read(writer.Remaining.Span); - - //Advance writer + writer.Advance(read); - - //Update position + _state.Position += read; } //See if data is still remaining to be read from transport (reamining size is also the amount of data that can be read) if (writer.RemainingSize > 0) - { - //Read from transport - int read = await _state.InputStream!.ReadAsync(writer.Remaining, cancellationToken).ConfigureAwait(true); - - //Update writer position + { + int read = await transport.Stream.ReadAsync(writer.Remaining, cancellationToken) + .ConfigureAwait(true); + writer.Advance(read); _state.Position += read; } - - //Return number of bytes written to the buffer + return writer.Written; } @@ -249,13 +243,11 @@ namespace VNLib.Net.Http.Core while (bytesToRead > 0) { //Read data to the discard buffer until reading is completed (read == 0) - read = await _state.InputStream!.ReadAsync(HttpServer.WriteOnlyScratchBuffer.Slice(0, bytesToRead), CancellationToken.None) + read = await transport.Stream!.ReadAsync(HttpServer.WriteOnlyScratchBuffer.Slice(0, bytesToRead), CancellationToken.None) .ConfigureAwait(true); - - //Update position + _state.Position += read; - - //Recalculate the number of bytes to read + bytesToRead = (int)Math.Min(HttpServer.WriteOnlyScratchBuffer.Length, Remaining); } } @@ -286,7 +278,6 @@ namespace VNLib.Net.Http.Core private struct StreamState { - public Stream? InputStream; public long Position; public long ContentLength; } diff --git a/lib/Net.Http/src/Core/Request/HttpRequest.cs b/lib/Net.Http/src/Core/Request/HttpRequest.cs index 2c9eed0..9263e0f 100644 --- a/lib/Net.Http/src/Core/Request/HttpRequest.cs +++ b/lib/Net.Http/src/Core/Request/HttpRequest.cs @@ -29,10 +29,11 @@ using System.Runtime.CompilerServices; using VNLib.Utils; using VNLib.Utils.Memory; using VNLib.Utils.Extensions; +using VNLib.Net.Http.Core.Response; namespace VNLib.Net.Http.Core { - internal sealed class HttpRequest(IHttpContextInformation contextInfo, ushort maxUploads) : IHttpLifeCycle + internal sealed class HttpRequest(TransportManager transport, ushort maxUploads) : IHttpLifeCycle #if DEBUG ,IStringSerializeable #endif @@ -48,7 +49,7 @@ namespace VNLib.Net.Http.Core /// A transport stream wrapper that is positioned for reading /// the entity body from the input stream /// - public readonly HttpInputStream InputStream = new(contextInfo); + public readonly HttpInputStream InputStream = new(transport); /* * Evil mutable structure that stores the http request state. @@ -63,6 +64,7 @@ namespace VNLib.Net.Http.Core */ private HttpRequestState _state; private readonly FileUpload[] _uploads = new FileUpload[maxUploads]; + private readonly FileUpload[] _singleUpload = new FileUpload[1]; /// /// Gets a mutable structure ref only used to initalize the request @@ -120,13 +122,15 @@ namespace VNLib.Net.Http.Core } private void FreeUploadBuffers() - { - //Dispose all initialized files + { + //Dispose all initialized files, should be much faster than using Array.Clear(); for (int i = 0; i < _uploads.Length; i++) { _uploads[i].Free(); _uploads[i] = default; } + + _singleUpload[0] = default; } /// @@ -136,21 +140,31 @@ namespace VNLib.Net.Http.Core public bool CanAddUpload() => _state.UploadCount < _uploads.Length; /// - /// Attempts to add a file upload to the request if there - /// is room for it. If there is no room, it will be ignored. - /// See to check if another upload can be added. + /// Attempts to obtain a reference to the next available + /// file upload in the request. If there are no more uploads + /// available, a null reference is returned. /// - /// The file upload structure to add to the list - public void AddFileUpload(in FileUpload upload) + /// A reference within the upload array to add the file + public ref FileUpload AddFileUpload() { //See if there is room for another upload if (CanAddUpload()) { - //Add file to upload array and increment upload count - _uploads[_state.UploadCount++] = upload; + //get ref to current position and increment the upload count + return ref _uploads[_state.UploadCount++]; } + + return ref Unsafe.NullRef(); } + /// + /// Attempts to add a file upload to the request if there + /// is room for it. If there is no room, it will be ignored. + /// See to check if another upload can be added. + /// + /// The file upload structure to add to the list + public void AddFileUpload(in FileUpload upload) => AddFileUpload() = upload; + /// /// Creates a new array and copies the uploads to it. /// @@ -162,6 +176,13 @@ namespace VNLib.Net.Http.Core return Array.Empty(); } + //Shortcut for a single upload request (hotpath optimization) + if (_state.UploadCount == 1) + { + _singleUpload[0] = _uploads[0]; + return _singleUpload; + } + //Create new array to hold uploads FileUpload[] uploads = GC.AllocateUninitializedArray(_state.UploadCount, false); diff --git a/lib/Net.Http/src/Core/Request/HttpRequestExtensions.cs b/lib/Net.Http/src/Core/Request/HttpRequestExtensions.cs index 878622e..64e08c5 100644 --- a/lib/Net.Http/src/Core/Request/HttpRequestExtensions.cs +++ b/lib/Net.Http/src/Core/Request/HttpRequestExtensions.cs @@ -184,7 +184,7 @@ namespace VNLib.Net.Http.Core //Default case is store as a file default: //add upload (if it fails thats fine, no memory to clean up) - request.AddFileUpload(new(request.InputStream, false, request.State.ContentType, null)); + request.AddFileUpload(new(request.InputStream, DisposeStream: false, request.State.ContentType, FileName: null)); break; } @@ -222,7 +222,7 @@ namespace VNLib.Net.Http.Core Encoding encoding ) { - int length = 0; + int charsRead = 0; do { //read async @@ -238,18 +238,18 @@ namespace VNLib.Net.Http.Core int numChars = encoding.GetCharCount(binBuffer.Span[..read]); //Re-alloc buffer and guard for overflow - charBuffer.ResizeIfSmaller(checked(numChars + length)); - - //Decode and update position - _ = encoding.GetChars(binBuffer.Span[..read], charBuffer.Span.Slice(length, numChars)); - - //Update char count - length += numChars; + charBuffer.ResizeIfSmaller(checked(numChars + charsRead)); + + _ = encoding.GetChars( + bytes: binBuffer.Span[..read], + chars: charBuffer.Span.Slice(charsRead, numChars) + ); + + charsRead += numChars; } while (true); - - //Return the number of characters read - return length; + + return charsRead; } /* @@ -299,8 +299,9 @@ namespace VNLib.Net.Http.Core switch (headerType) { case HttpHelpers.ContentDisposition: - //Parse the content dispostion + HttpHelpers.ParseDisposition(headerValue, out DispType, out Name, out FileName); + break; case HttpRequestHeader.ContentType: //The header value for content type should be an MIME content type @@ -320,12 +321,13 @@ namespace VNLib.Net.Http.Core //Only add the upload if the request can accept more uploads, otherwise drop it if (state.Request.CanAddUpload()) { - ReadOnlySpan fileData = reader.Window.TrimCRLF(); - - FileUpload upload = UploadFromString(fileData, state, FileName, ctHeaderVal); - - //Store the file in the uploads - state.Request.AddFileUpload(in upload); + UploadFromString( + data: reader.Window.TrimCRLF(), + context: state, + filename: FileName, + contentType: ctHeaderVal, + upload: ref state.Request.AddFileUpload() + ); } } @@ -344,9 +346,16 @@ namespace VNLib.Net.Http.Core /// The string data to copy /// The connection context /// The name of the file - /// The content type of the file data + /// The content type of the file data + /// A reference to the file upload to assign /// The container - private static FileUpload UploadFromString(ReadOnlySpan data, HttpContext context, string filename, ContentType ct) + private static void UploadFromString( + ReadOnlySpan data, + HttpContext context, + string filename, + ContentType contentType, + ref FileUpload upload + ) { IHttpContextInformation info = context; IHttpMemoryPool pool = context.ParentServer.Config.MemoryPool; @@ -365,7 +374,7 @@ namespace VNLib.Net.Http.Core VnMemoryStream vms = VnMemoryStream.FromHandle(buffHandle, true, bytes, true); //Create new upload wrapper that owns the stream - return new(vms, true, ct, filename); + upload = new(vms, true, contentType, filename); } catch { diff --git a/lib/Net.Http/src/Core/RequestParse/Http11ParseExtensions.cs b/lib/Net.Http/src/Core/RequestParse/Http11ParseExtensions.cs index cabb723..8e4e0e2 100644 --- a/lib/Net.Http/src/Core/RequestParse/Http11ParseExtensions.cs +++ b/lib/Net.Http/src/Core/RequestParse/Http11ParseExtensions.cs @@ -49,11 +49,26 @@ namespace VNLib.Net.Http.Core /// public ref struct Http1ParseState { - internal UriBuilder? Location; - internal bool IsAbsoluteRequestUrl; + internal Uri? AbsoluteUri; + internal UriSegments Location; internal long ContentLength; } + /* + * Reduces load when parsing uri components + * and allows a one-time vaidation once the uri + * is compiled + */ + internal ref struct UriSegments + { + public string Scheme; + public string Host; + public string Path; + public string Query; + + public int Port; + } + /// /// Reads the first line from the transport stream using the specified buffer @@ -120,29 +135,23 @@ namespace VNLib.Net.Http.Core return HttpStatusCode.HttpVersionNotSupported; } - //Set keepalive flag if http11 + //Http 1.1 spec defaults to keepalive if the connection header is not set to close reqState.KeepAlive = reqState.HttpVersion == HttpVersion.Http11; - //Get the location segment from the request line pathAndQuery = requestLine[(index + 1)..endloc].TrimCRLF(); //Process an absolute uri, if (pathAndQuery.Contains("://", StringComparison.Ordinal)) { //Convert the location string to a .net string and init the location builder (will perform validation when the Uri propery is used) - parseState.Location = new(pathAndQuery.ToString()); - parseState.IsAbsoluteRequestUrl = true; + parseState.AbsoluteUri = new(pathAndQuery.ToString()); return 0; } - //Try to capture a realative uri + //Must be a relaive uri that starts with / else if (pathAndQuery.Length > 0 && pathAndQuery[0] == '/') { - //Create a default location uribuilder - parseState.Location = new() - { - //Set a default scheme - Scheme = usingTls ? Uri.UriSchemeHttps : Uri.UriSchemeHttp, - }; + //Set default scheme + parseState.Location.Scheme = usingTls ? Uri.UriSchemeHttps : Uri.UriSchemeHttp; //Need to manually parse the query string int q = pathAndQuery.IndexOf('?'); @@ -161,6 +170,7 @@ namespace VNLib.Net.Http.Core } return 0; } + //Cannot service an unknonw location return HttpStatusCode.BadRequest; } @@ -334,16 +344,20 @@ namespace VNLib.Net.Http.Core } //Verify that the host matches the host header if absolue uri is set - if (parseState.IsAbsoluteRequestUrl) + if (parseState.AbsoluteUri != null) { - if (!hostOnly.Equals(parseState.Location!.Host, StringComparison.OrdinalIgnoreCase)) + if (!hostOnly.Equals(parseState.Location.Host, StringComparison.OrdinalIgnoreCase)) { return HttpStatusCode.BadRequest; } } - //store the host value - parseState.Location!.Host = hostOnly; + /* + * Uri segments are only assigned/used if an absolute + * uri was not set in the request line. + */ + + parseState.Location.Host = hostOnly; //If the port span is empty, no colon was found or the port is invalid if (!port.IsEmpty) @@ -444,16 +458,14 @@ namespace VNLib.Net.Http.Core //Validate explicit range if(!HttpRange.IsValidRangeValue(startRangeValue, endRangeValue)) { - //Ignore and continue parsing headers + //If range is invalid were supposed to ignore it and continue break; } - - //Set full http range + reqState.Range = HttpRange.FullRange(startRangeValue, endRangeValue); } else { - //From-end range reqState.Range = HttpRange.FromEnd(endRangeValue); } } @@ -509,18 +521,43 @@ namespace VNLib.Net.Http.Core return HttpStatusCode.BadRequest; } + //Store absolute uri if set + if(parseState.AbsoluteUri != null) + { + reqState.Location = parseState.AbsoluteUri; + } //Check the final location to make sure data was properly sent - if (string.IsNullOrWhiteSpace(parseState.Location?.Host) + else if(string.IsNullOrWhiteSpace(parseState.Location.Host) || string.IsNullOrWhiteSpace(parseState.Location.Scheme) || string.IsNullOrWhiteSpace(parseState.Location.Path) - ) + ) { return HttpStatusCode.BadRequest; } + else + { + /* + * Double allocations are not ideal here, but for now, its the + * safest way to build and validate a foreign uri. Its better + * than it was. + * + * A string could be build from heap memory then passed to the + * uri constructor for validation, but this will work for now. + */ + + //Build the final uri if successfully parsed into segments + reqState.Location = new UriBuilder( + scheme: parseState.Location.Scheme, + host: parseState.Location.Host, + port: parseState.Location.Port, + path: parseState.Location.Path, + extraValue: null + ) + { + Query = parseState.Location.Query, + }.Uri; + } - //Store the finalized location - reqState.Location = parseState.Location.Uri; - return 0; } diff --git a/lib/Net.Http/src/Core/Response/HeaderDataAccumulator.cs b/lib/Net.Http/src/Core/Response/HeaderDataAccumulator.cs index 0611095..a28ce09 100644 --- a/lib/Net.Http/src/Core/Response/HeaderDataAccumulator.cs +++ b/lib/Net.Http/src/Core/Response/HeaderDataAccumulator.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Net.Http @@ -35,16 +35,10 @@ namespace VNLib.Net.Http.Core.Response /// /// Specialized data accumulator for compiling response headers /// - internal readonly struct HeaderDataAccumulator + internal readonly struct HeaderDataAccumulator(IResponseHeaderAccBuffer accBuffer, IHttpContextInformation ctx) { - private readonly IResponseHeaderAccBuffer _buffer; - private readonly IHttpContextInformation _contextInfo; - - public HeaderDataAccumulator(IResponseHeaderAccBuffer accBuffer, IHttpContextInformation ctx) - { - _buffer = accBuffer; - _contextInfo = ctx; - } + private readonly IResponseHeaderAccBuffer _buffer = accBuffer; + private readonly IHttpContextInformation _contextInfo = ctx; /// /// Gets the accumulated response data as its memory buffer, and resets the internal accumulator diff --git a/lib/Net.Http/src/Core/Response/HttpResponse.cs b/lib/Net.Http/src/Core/Response/HttpResponse.cs index e354998..a038c9b 100644 --- a/lib/Net.Http/src/Core/Response/HttpResponse.cs +++ b/lib/Net.Http/src/Core/Response/HttpResponse.cs @@ -23,7 +23,6 @@ */ using System; -using System.IO; using System.Net; using System.Threading; using System.Diagnostics; @@ -39,7 +38,8 @@ using VNLib.Net.Http.Core.Buffering; namespace VNLib.Net.Http.Core.Response { - internal sealed class HttpResponse(IHttpContextInformation ContextInfo, IHttpBufferManager manager) : IHttpLifeCycle + + internal sealed class HttpResponse(IHttpContextInformation ContextInfo, TransportManager transport, IHttpBufferManager manager) : IHttpLifeCycle #if DEBUG , IStringSerializeable #endif @@ -47,8 +47,8 @@ namespace VNLib.Net.Http.Core.Response const int DefaultCookieCapacity = 2; private readonly Dictionary Cookies = new(DefaultCookieCapacity, StringComparer.OrdinalIgnoreCase); - private readonly DirectStream ReusableDirectStream = new(); - private readonly ChunkedStream ReusableChunkedStream = new(manager.ChunkAccumulatorBuffer, ContextInfo); + private readonly DirectStream ReusableDirectStream = new(transport); + private readonly ChunkedStream ReusableChunkedStream = new(manager.ChunkAccumulatorBuffer, transport, ContextInfo); private readonly HeaderDataAccumulator Writer = new(manager.ResponseHeaderBuffer, ContextInfo); private int _headerWriterPosition; @@ -165,9 +165,6 @@ namespace VNLib.Net.Http.Core.Response //Update sent headers HeadersSent = true; - //Get the transport stream to write the response data to - Stream transport = ContextInfo.GetTransport(); - /* * ASYNC NOTICE: It is safe to get the memory block then return the task * because the response writer is not cleaned up until the OnComplete() @@ -175,7 +172,7 @@ namespace VNLib.Net.Http.Core.Response */ //Write the response data to the base stream - return responseBlock.IsEmpty ? ValueTask.CompletedTask : transport.WriteAsync(responseBlock); + return responseBlock.IsEmpty ? ValueTask.CompletedTask : transport.Stream.WriteAsync(responseBlock); } /// @@ -261,15 +258,14 @@ namespace VNLib.Net.Http.Core.Response //reset after getting the written buffer _headerWriterPosition = 0; - - //get base stream - Stream bs = ContextInfo.GetTransport(); - + //Write the response data to the base stream - await bs.WriteAsync(responseBlock); + await transport.Stream.WriteAsync(responseBlock); - //Flush the base stream to send the data to the client - await bs.FlushAsync(); + /* + * Force flush should send data to client + */ + await transport.FlushAsync(); } /// @@ -298,19 +294,9 @@ namespace VNLib.Net.Http.Core.Response /// public void OnRelease() { - ReusableChunkedStream.OnRelease(); - ReusableDirectStream.OnRelease(); Cookies.TrimExcess(DefaultCookieCapacity); } - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void OnNewConnection(Stream transport) - { - ReusableChunkedStream.OnNewConnection(transport); - ReusableDirectStream.OnNewConnection(transport); - } - /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public void OnNewRequest() @@ -340,16 +326,16 @@ namespace VNLib.Net.Http.Core.Response ReusableChunkedStream.OnComplete(); } - private sealed class DirectStream : ReusableResponseStream, IDirectResponsWriter + private sealed class DirectStream(TransportManager transport) : IDirectResponsWriter { /// - public ValueTask WriteAsync(ReadOnlyMemory buffer) => transport!.WriteAsync(buffer); + public ValueTask WriteAsync(ReadOnlyMemory buffer) => transport!.Stream.WriteAsync(buffer); } /// /// Writes chunked HTTP message bodies to an underlying streamwriter /// - private sealed class ChunkedStream(IChunkAccumulatorBuffer buffer, IHttpContextInformation context) : ReusableResponseStream, IResponseDataWriter + private sealed class ChunkedStream(IChunkAccumulatorBuffer buffer, TransportManager transport, IHttpContextInformation context) : IResponseDataWriter { private readonly ChunkDataAccumulator _chunkAccumulator = new(buffer, context); @@ -389,7 +375,7 @@ namespace VNLib.Net.Http.Core.Response _accumulatedBytes = 0; //Write remaining data to stream - return transport!.WriteAsync(chunkData, CancellationToken.None); + return transport.Stream.WriteAsync(chunkData, CancellationToken.None); } #endregion diff --git a/lib/Net.Http/src/Core/Response/ReusableResponseStream.cs b/lib/Net.Http/src/Core/Response/ReusableResponseStream.cs deleted file mode 100644 index 3070c82..0000000 --- a/lib/Net.Http/src/Core/Response/ReusableResponseStream.cs +++ /dev/null @@ -1,46 +0,0 @@ -/* -* Copyright (c) 2023 Vaughn Nugent -* -* Library: VNLib -* Package: VNLib.Net.Http -* File: ReusableResponseStream.cs -* -* ReusableResponseStream.cs is part of VNLib.Net.Http which is part -* of the larger VNLib collection of libraries and utilities. -* -* VNLib.Net.Http is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* VNLib.Net.Http is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see https://www.gnu.org/licenses/. -*/ - -using System.IO; - -namespace VNLib.Net.Http.Core.Response -{ - - internal abstract class ReusableResponseStream - { - protected Stream? transport; - - /// - /// Called when a new connection is established - /// - /// - public virtual void OnNewConnection(Stream transport) => this.transport = transport; - - /// - /// Called when the connection is released - /// - public virtual void OnRelease() => transport = null; - - } -} \ No newline at end of file diff --git a/lib/Net.Http/src/Core/Response/TransportManager.cs b/lib/Net.Http/src/Core/Response/TransportManager.cs new file mode 100644 index 0000000..45efc4b --- /dev/null +++ b/lib/Net.Http/src/Core/Response/TransportManager.cs @@ -0,0 +1,99 @@ +/* +* Copyright (c) 2024 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Net.Http +* File: TransportManager.cs +* +* TransportManager.cs is part of VNLib.Net.Http which is part of +* the larger VNLib collection of libraries and utilities. +* +* VNLib.Net.Http is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* VNLib.Net.Http is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +using System.IO; +using System.Buffers; +using System.Diagnostics; +using System.Threading.Tasks; + +namespace VNLib.Net.Http.Core.Response +{ + internal sealed class TransportManager + { + public bool IsBufferWriter; + +#nullable disable + +#if DEBUG + + private Stream _stream; + private IBufferWriter _asWriter; + + public Stream Stream + { + get + { + Debug.Assert(_stream != null, "Transport stream was accessed but was set to null"); + return _stream; + } + set => _stream = value; + } + + public IBufferWriter Writer + { + get + { + Debug.Assert(_asWriter != null, "Transport buffer writer accessed but the writer is null"); + return _asWriter; + } + set => _asWriter = value; + } + +#else + public Stream Stream; + public IBufferWriter Writer; +#endif + +#nullable restore + + public Task FlushAsync() => Stream.FlushAsync(); + + /// + /// Assigns a new transport stream to the wrapper + /// as a new connection is assigned to write responses to + /// + /// The transport stream to wrap + public void OnNewConnection(Stream transportStream) + { + Stream = transportStream; + + //Capture a buffer writer if the incoming stream supports direct writing + if (transportStream is IBufferWriter bw) + { + Writer = bw; + IsBufferWriter = true; + } + } + + /// + /// Closes the current connection and resets the transport stream + /// + public void OnRelease() + { + Stream = null; + Writer = null; + IsBufferWriter = false; + } + } +} \ No newline at end of file diff --git a/lib/Net.Transport.SimpleTCP/src/ITransportInterface.cs b/lib/Net.Transport.SimpleTCP/src/ITransportInterface.cs index 5d16c69..b5afef1 100644 --- a/lib/Net.Transport.SimpleTCP/src/ITransportInterface.cs +++ b/lib/Net.Transport.SimpleTCP/src/ITransportInterface.cs @@ -24,6 +24,7 @@ using System; +using System.Buffers; using System.Threading; using System.Threading.Tasks; @@ -68,5 +69,17 @@ namespace VNLib.Net.Transport.Tcp /// The number of bytes received int Recv(Span buffer, int timeout); + /// + /// Gets as transport buffer writer for more effecient writes + /// + IBufferWriter SendBuffer { get; } + + /// + /// Flushes the send buffer + /// + /// + /// + /// A task that completes when pending write data has been sent + ValueTask FlushSendAsync(int timeout, CancellationToken cancellation); } } \ No newline at end of file diff --git a/lib/Net.Transport.SimpleTCP/src/ReusableNetworkStream.cs b/lib/Net.Transport.SimpleTCP/src/ReusableNetworkStream.cs index 00b5d7c..1867748 100644 --- a/lib/Net.Transport.SimpleTCP/src/ReusableNetworkStream.cs +++ b/lib/Net.Transport.SimpleTCP/src/ReusableNetworkStream.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2023 Vaughn Nugent +* Copyright (c) 2024 Vaughn Nugent * * Library: VNLib * Package: VNLib.Net.Transport.SimpleTCP @@ -33,6 +33,7 @@ using System; +using System.Buffers; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -45,7 +46,7 @@ namespace VNLib.Net.Transport.Tcp /// /// A reusable stream that marshals data between the socket pipeline and the application /// - internal sealed class ReusableNetworkStream : Stream + internal sealed class ReusableNetworkStream : Stream, IBufferWriter { #region stream basics public override bool CanRead => true; @@ -83,18 +84,31 @@ namespace VNLib.Net.Transport.Tcp //Timer used to cancel pipeline recv timeouts private readonly ITransportInterface Transport; - - internal ReusableNetworkStream(ITransportInterface transport) - { - Transport = transport; - } + + internal ReusableNetworkStream(ITransportInterface transport) => Transport = transport; /// public override void Close() { } /// - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + public override Task FlushAsync(CancellationToken cancellationToken) + => Transport.FlushSendAsync(_sendTimeoutMs, cancellationToken).AsTask(); + + + /* + * Expose the buffer writer interface on the stream + * for more efficient publishing + */ + + /// + public void Advance(int count) => Transport.SendBuffer.Advance(count); + + /// + public Memory GetMemory(int sizeHint = 0) => Transport.SendBuffer.GetMemory(sizeHint); + + /// + public Span GetSpan(int sizeHint = 0) => Transport.SendBuffer.GetSpan(sizeHint); /// public override void Flush() @@ -136,5 +150,6 @@ namespace VNLib.Net.Transport.Tcp */ public override ValueTask DisposeAsync() => ValueTask.CompletedTask; + } -} \ No newline at end of file +} diff --git a/lib/Net.Transport.SimpleTCP/src/SocketPipeLineWorker.cs b/lib/Net.Transport.SimpleTCP/src/SocketPipeLineWorker.cs index e47dd37..27d1e27 100644 --- a/lib/Net.Transport.SimpleTCP/src/SocketPipeLineWorker.cs +++ b/lib/Net.Transport.SimpleTCP/src/SocketPipeLineWorker.cs @@ -234,6 +234,7 @@ namespace VNLib.Net.Transport.Tcp private FlushResult _recvFlushRes; + public async Task RecvDoWorkAsync(TIO sock, int bytesTransferred, int recvBufferSize) where TIO : ISocketIo @@ -335,6 +336,28 @@ namespace VNLib.Net.Transport.Tcp await RecvPipe.Reader.CompleteAsync(); } + /// + public IBufferWriter SendBuffer => SendPipe.Writer; + + /// + public ValueTask FlushSendAsync(int timeout, CancellationToken cancellation) + { + //See if timer is required + if (timeout < 1) + { + NoOpTimerWrapper noOpTimer = default; + + //no timer + return SendWithTimerInternalAsync(in noOpTimer, cancellation); + } + else + { + TpTimerWrapper sendTimer = new(SendTimer, timeout); + + //Pass new send timer to send method + return SendWithTimerInternalAsync(in sendTimer, cancellation); + } + } private static async Task AwaitFlushTask(ValueTask valueTask, TTimer timer) where TTimer : INetTimer @@ -350,6 +373,14 @@ namespace VNLib.Net.Transport.Tcp } } + private ValueTask SendAsync(ReadOnlySpan data, int timeout, CancellationToken cancellation) + { + //Publish send data to send pipe + CopyAndPublishDataOnSendPipe(data, _sysSocketBufferSize, SendPipe.Writer); + + return FlushSendAsync(timeout, cancellation); + } + private ValueTask SendWithTimerInternalAsync(in TTimer timer, CancellationToken cancellation) where TTimer : INetTimer { @@ -388,28 +419,6 @@ namespace VNLib.Net.Transport.Tcp } } - private ValueTask SendAsync(ReadOnlySpan data, int timeout, CancellationToken cancellation) - { - //Publish send data to send pipe - CopyAndPublishDataOnSendPipe(data, _sysSocketBufferSize, SendPipe.Writer); - - //See if timer is required - if (timeout < 1) - { - NoOpTimerWrapper noOpTimer = default; - - //no timer - return SendWithTimerInternalAsync(in noOpTimer, cancellation); - } - else - { - TpTimerWrapper sendTimer = new(SendTimer, timeout); - - //Pass new send timer to send method - return SendWithTimerInternalAsync(in sendTimer, cancellation); - } - } - ValueTask ITransportInterface.SendAsync(ReadOnlyMemory data, int timeout, CancellationToken cancellation) { return SendAsync(data.Span, timeout, cancellation); @@ -542,7 +551,6 @@ namespace VNLib.Net.Transport.Tcp RecvTimer.Stop(); } } - private static class ThrowHelpers { diff --git a/lib/Utils/src/Extensions/MemoryExtensions.cs b/lib/Utils/src/Extensions/MemoryExtensions.cs index c433527..99d4cf1 100644 --- a/lib/Utils/src/Extensions/MemoryExtensions.cs +++ b/lib/Utils/src/Extensions/MemoryExtensions.cs @@ -48,19 +48,8 @@ namespace VNLib.Utils.Extensions /// The minimum size array to allocate /// Should elements from 0 to size be set to default(T) /// A new encapsulating the rented array - public static UnsafeMemoryHandle UnsafeAlloc(this ArrayPool pool, int size, bool zero = false) where T : unmanaged - { - ArgumentNullException.ThrowIfNull(pool); - - T[] array = pool.Rent(size); - - if (zero) - { - MemoryUtil.InitializeBlock(array, (uint)size); - } - - return new(pool, array, size); - } + public static UnsafeMemoryHandle UnsafeAlloc(this ArrayPool pool, int size, bool zero = false) where T : unmanaged + => MemoryUtil.UnsafeAlloc(pool, size, zero); /// /// Rents a new array and stores it as a resource within an to return the @@ -72,19 +61,7 @@ namespace VNLib.Utils.Extensions /// Should elements from 0 to size be set to default(T) /// A new encapsulating the rented array public static IMemoryHandle SafeAlloc(this ArrayPool pool, int size, bool zero = false) where T : struct - { - ArgumentNullException.ThrowIfNull(pool); - - T[] array = pool.Rent(size); - - if (zero) - { - MemoryUtil.InitializeBlock(array, (uint)size); - } - - //Use the array pool buffer wrapper to return the array to the pool when the handle is disposed - return new ArrayPoolBuffer(pool, array, size); - } + => MemoryUtil.SafeAlloc(pool, size, zero); /// /// Retreives a buffer that is at least the reqested length, and clears the array from 0-size. @@ -110,13 +87,6 @@ namespace VNLib.Utils.Extensions return arr; } - /// - /// Copies the characters within the memory handle to a - /// - /// The string representation of the buffer - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static string ToString(this T charBuffer) where T : IMemoryHandle => charBuffer.Span.ToString(); - /// /// Wraps the instance in System.Buffers.MemoryManager /// wrapper to provide buffers from umanaged handles. @@ -131,7 +101,8 @@ namespace VNLib.Utils.Extensions /// NOTE: This wrapper now manages the lifetime of the current handle /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static MemoryManager ToMemoryManager(this IMemoryHandle handle, bool ownsHandle) => new SysBufferMemoryManager(handle, ownsHandle); + public static MemoryManager ToMemoryManager(this IMemoryHandle handle, bool ownsHandle) + => new SysBufferMemoryManager(handle, ownsHandle); /// /// Allows direct allocation of a fixed size from a instance @@ -168,7 +139,8 @@ namespace VNLib.Utils.Extensions /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetIntLength(this IMemoryHandle handle) => Convert.ToInt32(handle.Length); + public static int GetIntLength(this IMemoryHandle handle) + => Convert.ToInt32(handle.Length); /// /// Gets the integer length (number of elements) of the @@ -181,7 +153,8 @@ namespace VNLib.Utils.Extensions /// //Method only exists for consistancy since unsafe handles are always 32bit [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetIntLength(this in UnsafeMemoryHandle handle) where T : unmanaged => handle.IntLength; + public static int GetIntLength(this in UnsafeMemoryHandle handle) where T : unmanaged + => handle.IntLength; /// /// Gets an offset pointer from the base postion to the number of bytes specified. Performs bounds checks @@ -440,16 +413,8 @@ namespace VNLib.Utils.Extensions /// /// /// - public static unsafe MemoryHandle Alloc(this IUnmangedHeap heap, nuint elements, bool zero = false) where T : unmanaged - { - ArgumentNullException.ThrowIfNull(heap); - //Minimum of one element - elements = Math.Max(elements, 1); - //If zero flag is set then specify zeroing memory - IntPtr block = heap.Alloc(elements, (nuint)sizeof(T), zero); - //Return handle wrapper - return new MemoryHandle(heap, block, elements, zero); - } + public static unsafe MemoryHandle Alloc(this IUnmangedHeap heap, nuint elements, bool zero = false) where T : unmanaged + => MemoryUtil.SafeAlloc(heap, elements, zero); /// /// Allocates a block of unmanaged memory of the number of elements to store of an unmanged type @@ -464,10 +429,7 @@ namespace VNLib.Utils.Extensions /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public static MemoryHandle Alloc(this IUnmangedHeap heap, nint elements, bool zero = false) where T : unmanaged - { - ArgumentOutOfRangeException.ThrowIfNegative(elements); - return Alloc(heap, (nuint)elements, zero); - } + => MemoryUtil.SafeAlloc(heap, elements, zero); /// /// Allocates a buffer from the current heap and initialzies it by copying the initial data buffer @@ -481,10 +443,8 @@ namespace VNLib.Utils.Extensions [MethodImpl(MethodImplOptions.AggressiveInlining)] public static MemoryHandle AllocAndCopy(this IUnmangedHeap heap, ReadOnlySpan initialData) where T : unmanaged { - //Aloc block - MemoryHandle handle = heap.Alloc(initialData.Length); - - //Copy initial data + MemoryHandle handle = Alloc(heap, initialData.Length); + MemoryUtil.Copy(initialData, 0, handle, 0, initialData.Length); return handle; @@ -502,10 +462,8 @@ namespace VNLib.Utils.Extensions [MethodImpl(MethodImplOptions.AggressiveInlining)] public static MemoryHandle AllocAndCopy(this IUnmangedHeap heap, ReadOnlyMemory initialData) where T : unmanaged { - //Aloc block - MemoryHandle handle = heap.Alloc(initialData.Length); - - //Copy initial data + MemoryHandle handle = Alloc(heap, initialData.Length); + MemoryUtil.Copy(initialData, 0, handle, 0, initialData.Length); return handle; @@ -542,25 +500,8 @@ namespace VNLib.Utils.Extensions /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static UnsafeMemoryHandle UnsafeAlloc(this IUnmangedHeap heap, int elements, bool zero = false) where T : unmanaged - { - ArgumentNullException.ThrowIfNull(heap); - - if (elements < 1) - { - //Return an empty handle - return new UnsafeMemoryHandle(); - } - - //Get element size - nuint elementSize = (nuint)Unsafe.SizeOf(); - - //If zero flag is set then specify zeroing memory (safe case because of the above check) - IntPtr block = heap.Alloc((nuint)elements, elementSize, zero); - - //handle wrapper - return new (heap, block, elements); - } + public unsafe static UnsafeMemoryHandle UnsafeAlloc(this IUnmangedHeap heap, int elements, bool zero = false) where T : unmanaged + => MemoryUtil.UnsafeAlloc(heap, elements, zero); #region VnBufferWriter @@ -774,13 +715,6 @@ namespace VNLib.Utils.Extensions return charCount; } - /// - /// Converts the buffer data to a - /// - /// A instance that owns the underlying string memory - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static PrivateString ToPrivate(this ref ForwardOnlyWriter buffer) => new(buffer.ToString(), true); - /// /// Gets a over the modified section of the internal buffer /// diff --git a/lib/Utils/src/IO/VnMemoryStream.cs b/lib/Utils/src/IO/VnMemoryStream.cs index 2e5ec40..23df869 100644 --- a/lib/Utils/src/IO/VnMemoryStream.cs +++ b/lib/Utils/src/IO/VnMemoryStream.cs @@ -41,6 +41,8 @@ namespace VNLib.Utils.IO /// public sealed class VnMemoryStream : Stream, ICloneable { + public const int DefaultBufferSize = 4096; + private nint _position; private nint _length; private bool _isReadonly; @@ -108,7 +110,7 @@ namespace VNLib.Utils.IO /// to allocate memory from /// /// - public VnMemoryStream(IUnmangedHeap heap) : this(heap, 0, false) { } + public VnMemoryStream(IUnmangedHeap heap) : this(heap, DefaultBufferSize, false) { } /// /// Creates a new memory stream and pre-allocates the internal diff --git a/lib/Utils/src/Memory/MemoryUtilAlloc.cs b/lib/Utils/src/Memory/MemoryUtilAlloc.cs index e2e7434..6e4f9b0 100644 --- a/lib/Utils/src/Memory/MemoryUtilAlloc.cs +++ b/lib/Utils/src/Memory/MemoryUtilAlloc.cs @@ -27,14 +27,83 @@ using System.Buffers; using System.Diagnostics; using System.Runtime.CompilerServices; -using VNLib.Utils.Extensions; - namespace VNLib.Utils.Memory { public static unsafe partial class MemoryUtil { #region alloc + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool UseUnmanagedHeap(IUnmangedHeap heap, nuint elements) + { + /* + * We may allocate from the share heap only if the heap is not using locks + * or if the element size could cause performance issues because its too large + * to use a managed array. + * + * We want to avoid allocations, that may end up in the LOH if we can + */ + + return (heap.CreationFlags & HeapCreation.UseSynchronization) == 0 + || ByteCount((uint)elements) > MAX_UNSAFE_POOL_SIZE; + } + + /// + /// Allocates a block of unmanaged memory of the number of elements to store of an unmanged type + /// + /// Unmanaged data type to create a block of + /// + /// The size of the block (number of elements) + /// A flag that zeros the allocated block before returned + /// The unmanaged + /// + /// + /// + public static UnsafeMemoryHandle UnsafeAlloc(IUnmangedHeap heap, int elements, bool zero = false) where T : unmanaged + { + ArgumentNullException.ThrowIfNull(heap); + ArgumentOutOfRangeException.ThrowIfNegative(elements); + + if (elements == 0) + { + //Return an empty handle + return default; + } + + //If zero flag is set then specify zeroing memory (safe case because of the above check) + IntPtr block = heap.Alloc((nuint)elements, (nuint)sizeof(T), zero); + + return new(heap, block, elements); + } + + /// + /// Rents a new array and stores it as a resource within an to return the + /// array when work is completed + /// + /// + /// + /// The minimum size array to allocate + /// Should elements from 0 to size be set to default(T) + /// A new encapsulating the rented array + public static UnsafeMemoryHandle UnsafeAlloc(ArrayPool pool, int size, bool zero = false) where T : unmanaged + { + ArgumentNullException.ThrowIfNull(pool); + + if (size <= 0) + { + return default; + } + + T[] array = pool.Rent(size); + + if (zero) + { + InitializeBlock(array, (uint)size); + } + + return new(pool, array, size); + } + /// /// Allocates a block of unmanaged, or pooled manaaged memory depending on /// compilation flags and runtime unamanged allocators. @@ -43,40 +112,20 @@ namespace VNLib.Utils.Memory /// The number of elements of the type within the block /// Flag to zero elements during allocation before the method returns /// A handle to the block of memory - /// + /// /// public static UnsafeMemoryHandle UnsafeAlloc(int elements, bool zero = false) where T : unmanaged { - if (elements < 0) - { - throw new ArgumentException("Number of elements must be a positive integer", nameof(elements)); - } + ArgumentOutOfRangeException.ThrowIfNegative(elements); if (elements == 0) { return default; } - /* - * We may allocate from the share heap only if the heap is not using locks - * or if the element size could cause performance issues because its too large - * to use a managed array. - * - * We want to avoid allocations, that may end up in the LOH if we can - */ - - if ((Shared.CreationFlags & HeapCreation.UseSynchronization) == 0 || ByteCount((uint)elements) > MAX_UNSAFE_POOL_SIZE) - { - // Alloc from heap - IntPtr block = Shared.Alloc((uint)elements, (uint)sizeof(T), zero); - //Init new handle - return new(Shared, block, elements); - } - else - { - //Rent the array from the pool - return ArrayPool.Shared.UnsafeAlloc(elements, zero); - } + return UseUnmanagedHeap(Shared, (uint)elements) + ? UnsafeAlloc(Shared, elements, zero) + : UnsafeAlloc(ArrayPool.Shared, elements, zero); } /// @@ -88,18 +137,81 @@ namespace VNLib.Utils.Memory /// The number of elements of the type within the block /// Flag to zero elements during allocation before the method returns /// A handle to the block of memory - /// + /// /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static UnsafeMemoryHandle UnsafeAllocNearestPage(int elements, bool zero = false) where T : unmanaged { - if (elements < 0) + ArgumentOutOfRangeException.ThrowIfNegative(elements); + return UnsafeAlloc(elements: (int)NearestPage(elements), zero); + } + + /// + /// Allocates a block of unmanaged memory of the number of elements to store of an unmanged type + /// + /// Unmanaged data type to create a block of + /// + /// The size of the block (number of elements) + /// A flag that zeros the allocated block before returned + /// The unmanaged + /// + /// + /// + public static MemoryHandle SafeAlloc(IUnmangedHeap heap, nuint elements, bool zero = false) where T : unmanaged + { + ArgumentNullException.ThrowIfNull(heap); + + //Return empty handle if no elements were specified + if (elements == 0) { - throw new ArgumentException("Number of elements must be a positive integer", nameof(elements)); + return new MemoryHandle(); } + + IntPtr block = heap.Alloc(elements, (nuint)sizeof(T), zero); - //Round to nearest page (in bytes) - nint np = NearestPage(elements); - return UnsafeAlloc((int)np, zero); + return new MemoryHandle(heap, block, elements, zero); + } + + /// + /// Allocates a block of unmanaged memory of the number of elements to store of an unmanged type + /// + /// Unmanaged data type to create a block of + /// + /// The size of the block (number of elements) + /// A flag that zeros the allocated block before returned + /// The unmanaged + /// + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static MemoryHandle SafeAlloc(IUnmangedHeap heap, nint elements, bool zero = false) where T : unmanaged + { + ArgumentOutOfRangeException.ThrowIfNegative(elements); + return SafeAlloc(heap, (nuint)elements, zero); + } + + /// + /// Rents a new array and stores it as a resource within an to return the + /// array when work is completed + /// + /// + /// + /// The minimum size array to allocate + /// Should elements from 0 to size be set to default(T) + /// A new encapsulating the rented array + public static ArrayPoolBuffer SafeAlloc(ArrayPool pool, int size, bool zero = false) where T : struct + { + ArgumentNullException.ThrowIfNull(pool); + + T[] array = pool.Rent(size); + + if (zero) + { + InitializeBlock(array, (uint)size); + } + + //Use the array pool buffer wrapper to return the array to the pool when the handle is disposed + return new ArrayPoolBuffer(pool, array, size); } /// @@ -110,33 +222,58 @@ namespace VNLib.Utils.Memory /// The number of elements of the type within the block /// Flag to zero elements during allocation before the method returns /// A handle to the block of memory - /// + /// /// - public static IMemoryHandle SafeAlloc(int elements, bool zero = false) where T : unmanaged + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static IMemoryHandle SafeAlloc(nuint elements, bool zero = false) where T : unmanaged { - if (elements < 0) - { - throw new ArgumentException("Number of elements must be a positive integer", nameof(elements)); - } - - /* - * We may allocate from the share heap only if the heap is not using locks - * or if the element size could cause performance issues because its too large - * to use a managed array. - * - * We want to avoid allocations, that may end up in the LOH if we can - */ + ArgumentOutOfRangeException.ThrowIfNegative(elements); - if ((Shared.CreationFlags & HeapCreation.UseSynchronization) == 0 || ByteCount((uint)elements) > MAX_UNSAFE_POOL_SIZE) + if (UseUnmanagedHeap(Shared, elements)) { - return Shared.Alloc(elements, zero); + return SafeAlloc(Shared, elements, zero); } else { - return new ArrayPoolBuffer(ArrayPool.Shared, elements, zero); + //Should never happen because max pool size guards against this + Debug.Assert(elements <= int.MaxValue); + + return SafeAlloc(ArrayPool.Shared, (int)elements, zero); } } + /// + /// Allocates a block of unmanaged, or pooled manaaged memory depending on + /// compilation flags and runtime unamanged allocators. + /// + /// The unamanged type to allocate + /// The number of elements of the type within the block + /// Flag to zero elements during allocation before the method returns + /// A handle to the block of memory + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static IMemoryHandle SafeAlloc(int elements, bool zero = false) where T : unmanaged + { + ArgumentOutOfRangeException.ThrowIfNegative(elements); + return SafeAlloc((nuint)elements, zero); + } + + /// + /// Allocates a block of unmanaged, or pooled manaaged memory depending on + /// compilation flags and runtime unamanged allocators, rounded up to the + /// neareset memory page. + /// + /// The unamanged type to allocate + /// The number of elements of the type within the block + /// Flag to zero elements during allocation before the method returns + /// A handle to the block of memory + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static IMemoryHandle SafeAllocNearestPage(nuint elements, bool zero = false) where T : unmanaged + => SafeAlloc(elements: NearestPage(elements), zero); + /// /// Allocates a block of unmanaged, or pooled manaaged memory depending on /// compilation flags and runtime unamanged allocators, rounded up to the @@ -146,20 +283,51 @@ namespace VNLib.Utils.Memory /// The number of elements of the type within the block /// Flag to zero elements during allocation before the method returns /// A handle to the block of memory - /// + /// /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static IMemoryHandle SafeAllocNearestPage(int elements, bool zero = false) where T : unmanaged { - if (elements < 0) - { - throw new ArgumentException("Number of elements must be a positive integer", nameof(elements)); - } + ArgumentOutOfRangeException.ThrowIfNegative(elements); + return SafeAllocNearestPage((nuint)elements, zero); + } - //Round to nearest page (in bytes) - nint np = NearestPage(elements); - return SafeAlloc((int)np, zero); + /// + /// Allocates a block of unmanaged, or pooled manaaged memory depending on + /// compilation flags and runtime unamanged allocators, rounded up to the + /// neareset memory page. + /// + /// The unamanged type to allocate + /// The number of elements of the type within the block + /// Flag to zero elements during allocation before the method returns + /// The heap to allocate the block of memory from + /// A handle to the block of memory + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static MemoryHandle SafeAllocNearestPage(IUnmangedHeap heap, int elements, bool zero = false) where T : unmanaged + { + ArgumentOutOfRangeException.ThrowIfNegative(elements); + + return SafeAllocNearestPage(heap, (nuint)elements, zero); } + /// + /// Allocates a block of unmanaged, or pooled manaaged memory depending on + /// compilation flags and runtime unamanged allocators, rounded up to the + /// neareset memory page. + /// + /// The unamanged type to allocate + /// The number of elements of the type within the block + /// Flag to zero elements during allocation before the method returns + /// The heap to allocate the block of memory from + /// A handle to the block of memory + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static MemoryHandle SafeAllocNearestPage(IUnmangedHeap heap, nuint elements, bool zero = false) where T : unmanaged + => SafeAlloc(heap, elements: NearestPage(elements), zero); + /// /// Allocates a structure of the specified type on the specified /// unmanged heap and optionally zero's it's memory @@ -191,13 +359,8 @@ namespace VNLib.Utils.Memory /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ref T StructAllocRef(IUnmangedHeap heap, bool zero) where T : unmanaged - { - //Alloc structure - T* ptr = StructAlloc(heap, zero); - //Get a reference and assign it - return ref Unsafe.AsRef(ptr); - } + public static ref T StructAllocRef(IUnmangedHeap heap, bool zero) where T : unmanaged + => ref Unsafe.AsRef(StructAlloc(heap, zero)); /// /// Frees a structure allocated with @@ -207,7 +370,8 @@ namespace VNLib.Utils.Memory /// A pointer to the unmanaged structure to free /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void StructFree(IUnmangedHeap heap, T* structPtr) where T : unmanaged => StructFree(heap, (void*)structPtr); + public static void StructFree(IUnmangedHeap heap, T* structPtr) where T : unmanaged + => StructFree(heap, (void*)structPtr); /// /// Frees a structure allocated with @@ -218,7 +382,8 @@ namespace VNLib.Utils.Memory /// A reference to the unmanaged structure to free /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void StructFreeRef(IUnmangedHeap heap, ref T structRef) where T : unmanaged => StructFree(heap, Unsafe.AsPointer(ref structRef)); + public static void StructFreeRef(IUnmangedHeap heap, ref T structRef) where T : unmanaged + => StructFree(heap, Unsafe.AsPointer(ref structRef)); /// /// Frees a structure allocated with @@ -233,7 +398,7 @@ namespace VNLib.Utils.Memory //Get intpointer IntPtr ptr = (IntPtr)structPtr; - //Free + bool isFree = heap.Free(ref ptr); Debug.Assert(isFree, $"Structure free failed for heap {heap.GetHashCode()}, struct address {ptr:x}"); } @@ -249,39 +414,20 @@ namespace VNLib.Utils.Memory /// The number of elements of the type within the block /// Flag to zero elements during allocation before the method returns /// A handle to the block of memory - /// + /// /// public static UnsafeMemoryHandle UnsafeAlloc(int elements, bool zero = false) { - if (elements < 0) - { - throw new ArgumentException("Number of elements must be a positive integer", nameof(elements)); - } + ArgumentOutOfRangeException.ThrowIfNegative(elements); if(elements == 0) { return default; } - /* - * We may allocate from the share heap only if the heap is not using locks - * or if the element size could cause performance issues because its too large - * to use a managed array. - * - * We want to avoid allocations, that may end up in the LOH if we can - */ - - if ((Shared.CreationFlags & HeapCreation.UseSynchronization) == 0 || elements > MAX_UNSAFE_POOL_SIZE) - { - // Alloc from heap - IntPtr block = Shared.Alloc((uint)elements, 1, zero); - //Init new handle - return new(Shared, block, elements); - } - else - { - return ArrayPool.Shared.UnsafeAlloc(elements, zero); - } + return UseUnmanagedHeap(Shared, (uint)elements) + ? UnsafeAlloc(Shared, elements, zero) + : UnsafeAlloc(ArrayPool.Shared, elements, zero); } /// @@ -292,19 +438,15 @@ namespace VNLib.Utils.Memory /// The number of elements of the type within the block /// Flag to zero elements during allocation before the method returns /// A handle to the block of memory - /// + /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public static UnsafeMemoryHandle UnsafeAllocNearestPage(int elements, bool zero = false) { - if (elements < 0) - { - throw new ArgumentException("Number of elements must be a positive integer", nameof(elements)); - } + ArgumentOutOfRangeException.ThrowIfNegative(elements); //Round to nearest page (in bytes) - nint np = NearestPage(elements); - return UnsafeAlloc((int)np, zero); + return UnsafeAlloc(elements: (int)NearestPage(elements), zero); } /// @@ -314,31 +456,15 @@ namespace VNLib.Utils.Memory /// The number of elements of the type within the block /// Flag to zero elements during allocation before the method returns /// A handle to the block of memory - /// + /// /// public static IMemoryHandle SafeAlloc(int elements, bool zero = false) { - if (elements < 0) - { - throw new ArgumentException("Number of elements must be a positive integer", nameof(elements)); - } - - /* - * We may allocate from the share heap only if the heap is not using locks - * or if the element size could cause performance issues because its too large - * to use a managed array. - * - * We want to avoid allocations, that may end up in the LOH if we can - */ + ArgumentOutOfRangeException.ThrowIfNegative(elements); - if ((Shared.CreationFlags & HeapCreation.UseSynchronization) == 0 || elements > MAX_UNSAFE_POOL_SIZE) - { - return Shared.Alloc(elements, zero); - } - else - { - return new ArrayPoolBuffer(ArrayPool.Shared, elements, zero); - } + return UseUnmanagedHeap(Shared, (uint)elements) + ? SafeAlloc(Shared, (nuint)elements, zero) + : SafeAlloc(ArrayPool.Shared, elements, zero); } /// @@ -349,18 +475,12 @@ namespace VNLib.Utils.Memory /// The number of elements of the type within the block /// Flag to zero elements during allocation before the method returns /// A handle to the block of memory - /// + /// /// public static IMemoryHandle SafeAllocNearestPage(int elements, bool zero = false) { - if (elements < 0) - { - throw new ArgumentException("Number of elements must be a positive integer", nameof(elements)); - } - - //Round to nearest page (in bytes) - nint np = NearestPage(elements); - return SafeAlloc((int)np, zero); + ArgumentOutOfRangeException.ThrowIfNegative(elements); + return SafeAlloc(elements: (int)NearestPage(elements), zero); } #endregion diff --git a/lib/Utils/src/Memory/UnmanagedHeapBase.cs b/lib/Utils/src/Memory/UnmanagedHeapBase.cs index 7f42761..a9730b7 100644 --- a/lib/Utils/src/Memory/UnmanagedHeapBase.cs +++ b/lib/Utils/src/Memory/UnmanagedHeapBase.cs @@ -23,6 +23,7 @@ */ using System; +using System.Diagnostics; using System.Runtime.InteropServices; using Microsoft.Win32.SafeHandles; @@ -111,25 +112,32 @@ namespace VNLib.Utils.Memory bool result; //If disposed, set the block handle to zero and exit to avoid raising exceptions during finalization - if (IsClosed || IsInvalid) + if (IsClosed) { block = LPVOID.Zero; return true; } + /* + * Checking for invalid is not really necesasry because + * the only way the handle can be invalidated is + * if some derrived class mutates the handle value + * and doesn't close the handle + */ + Debug.Assert(IsInvalid == false); + if ((flags & HeapCreation.UseSynchronization) > 0) { //wait for lock lock (HeapLock) - { - //Free block + { result = FreeBlock(block); //Release lock before releasing handle } } else { - //No lock + //No lock needed result = FreeBlock(block); } diff --git a/lib/Utils/tests/Memory/MemoryUtilTests.cs b/lib/Utils/tests/Memory/MemoryUtilTests.cs index 63342b5..68bc35c 100644 --- a/lib/Utils/tests/Memory/MemoryUtilTests.cs +++ b/lib/Utils/tests/Memory/MemoryUtilTests.cs @@ -202,7 +202,7 @@ namespace VNLib.Utils.Memory.Tests { } //test against negative number - Assert.ThrowsException(() => MemoryUtil.UnsafeAlloc(-1)); + Assert.ThrowsException(() => MemoryUtil.UnsafeAlloc(-1)); //Alloc large block test (100mb) const int largTestSize = 100000 * 1024; @@ -257,7 +257,7 @@ namespace VNLib.Utils.Memory.Tests } //Negative value - Assert.ThrowsException(() => _ = MemoryUtil.UnsafeAlloc(-1)); + Assert.ThrowsException(() => _ = MemoryUtil.UnsafeAlloc(-1)); /* @@ -315,7 +315,7 @@ namespace VNLib.Utils.Memory.Tests } //Negative value - Assert.ThrowsException(() => _ = MemoryUtil.SafeAlloc(-1)); + Assert.ThrowsException(() => _ = MemoryUtil.SafeAlloc(-1)); /* * Alloc random sized blocks in a loop, confirm they are empty -- cgit