http overhaul, fix range, remove range auto-processing, fix some perf bottlenecks from profiling

5 files changed, 174 insertions, 21 deletions

diff --git a/lib/Plugins.Essentials/src/Endpoints/ResourceEndpointBase.cs b/lib/Plugins.Essentials/src/Endpoints/ResourceEndpointBase.cs
index 447d17b..7c72bc4 100644
--- a/lib/Plugins.Essentials/src/Endpoints/ResourceEndpointBase.cs
+++ b/lib/Plugins.Essentials/src/Endpoints/ResourceEndpointBase.cs
@@ -375,10 +375,32 @@ namespace VNLib.Plugins.Essentials.Endpoints
         /// <param name="code">The status code to return to the client</param>
         /// <param name="ct">The response content type</param>
         /// <param name="response">The stream response to return to the user</param>
+        /// <param name="length">The explicit length of the stream</param>
+        /// <returns>The <see cref="VfReturnType.VirtualSkip"/> operation result</returns>
+        public static VfReturnType VirtualClose(HttpEntity entity, HttpStatusCode code, ContentType ct, Stream response, long length)
+        {
+            entity.CloseResponse(code, ct, response, length);
+            return VfReturnType.VirtualSkip;
+        }
+
+        /// <summary>
+        /// Shortcut helper methods to a virtual skip response with a given status code,
+        /// and memory content to return to the client
+        /// </summary>
+        /// <param name="entity">The entity to close the connection for</param>
+        /// <param name="code">The status code to return to the client</param>
+        /// <param name="ct">The response content type</param>
+        /// <param name="response">The stream response to return to the user</param>
+        /// <param name="length">The explicit length of the stream</param>
         /// <returns>The <see cref="VfReturnType.VirtualSkip"/> operation result</returns>
         public static VfReturnType VirtualClose(HttpEntity entity, HttpStatusCode code, ContentType ct, Stream response)
         {
-            entity.CloseResponse(code, ct, response);
+            ArgumentNullException.ThrowIfNull(response, nameof(response));
+            if (!response.CanSeek)
+            {
+                throw new IOException("The stream must be seekable when using implicit length");
+            }
+            entity.CloseResponse(code, ct, response, response.Length);
             return VfReturnType.VirtualSkip;
         }
 
diff --git a/lib/Plugins.Essentials/src/EventProcessor.cs b/lib/Plugins.Essentials/src/EventProcessor.cs
index 3ea61c0..861f318 100644
--- a/lib/Plugins.Essentials/src/EventProcessor.cs
+++ b/lib/Plugins.Essentials/src/EventProcessor.cs
@@ -434,19 +434,84 @@ namespace VNLib.Plugins.Essentials
 
                     //try to open the selected file for reading and allow sharing
                     FileStream fs = new (filename, FileMode.Open, FileAccess.Read, FileShare.Read);
-                    
-                    //Check for range 
-                    if (entity.Server.Range != null && entity.Server.Range.Item1 > 0)
-                    {
-                        //Seek the stream to the specified position
-                        fs.Seek(entity.Server.Range.Item1, SeekOrigin.Begin);
-                        entity.CloseResponse(HttpStatusCode.PartialContent, fileType, fs);
-                    }
-                    else
+
+                    long endOffset = checked((long)entity.Server.Range.End);
+                    long startOffset = checked((long)entity.Server.Range.Start);
+
+                    //Follows rfc7233 -> https://www.rfc-editor.org/rfc/rfc7233#section-1.2
+                    switch (entity.Server.Range.RangeType)
                     {
-                        //send the whole file
-                        entity.CloseResponse(HttpStatusCode.OK, fileType, fs);
+                        case HttpRangeType.FullRange:
+                            if (endOffset > fs.Length || endOffset - startOffset < 0)
+                            {
+                                //Set acceptable range size
+                                entity.Server.Headers[HttpResponseHeader.ContentRange] = $"bytes */{fs.Length}";
+
+                                //The start offset is greater than the file length, return range not satisfiable
+                                entity.CloseResponse(HttpStatusCode.RequestedRangeNotSatisfiable);
+                            }
+                            else
+                            {
+                                //Seek the stream to the specified start position
+                                fs.Seek(startOffset, SeekOrigin.Begin);
+
+                                //Set range header, by passing the actual full content size
+                                entity.SetContentRangeHeader(entity.Server.Range, fs.Length);
+
+                                //Send the response, with actual response length (diff between stream length and position)
+                                entity.CloseResponse(HttpStatusCode.PartialContent, fileType, fs, endOffset - startOffset + 1);
+                            }
+                            break;
+                        case HttpRangeType.FromStart:
+                            if (startOffset > fs.Length)
+                            {
+                                //Set acceptable range size
+                                entity.Server.Headers[HttpResponseHeader.ContentRange] = $"bytes */{fs.Length}";
+
+                                //The start offset is greater than the file length, return range not satisfiable
+                                entity.CloseResponse(HttpStatusCode.RequestedRangeNotSatisfiable);
+                            }
+                            else
+                            {
+                                //Seek the stream to the specified start position
+                                fs.Seek(startOffset, SeekOrigin.Begin);
+
+                                //Set range header, by passing the actual full content size
+                                entity.SetContentRangeHeader(entity.Server.Range, fs.Length);
+
+                                //Send the response, with actual response length (diff between stream length and position)
+                                entity.CloseResponse(HttpStatusCode.PartialContent, fileType, fs, fs.Length - fs.Position);
+                            }
+                            break;
+
+                        case HttpRangeType.FromEnd:
+                            if (endOffset > fs.Length)
+                            {
+                                //Set acceptable range size
+                                entity.Server.Headers[HttpResponseHeader.ContentRange] = $"bytes */{fs.Length}";
+
+                                //The end offset is greater than the file length, return range not satisfiable
+                                entity.CloseResponse(HttpStatusCode.RequestedRangeNotSatisfiable);
+                            }
+                            else
+                            {
+                                //Seek the stream to the specified end position, server auto range will handle the rest
+                                fs.Seek(-endOffset, SeekOrigin.End);
+
+                                //Set range header, by passing the actual full content size
+                                entity.SetContentRangeHeader(entity.Server.Range, fs.Length);
+
+                                //Send the response, with actual response length (diff between stream length and position)
+                                entity.CloseResponse(HttpStatusCode.PartialContent, fileType, fs, fs.Length - fs.Position);
+                            }
+                            break;
+                        //No range or invalid range (the server is supposed to ignore invalid ranges)
+                        default:
+                            //send the whole file
+                            entity.CloseResponse(HttpStatusCode.OK, fileType, fs, fs.Length);
+                            break;
                     }
+                    
                 }
                 else
                 {
diff --git a/lib/Plugins.Essentials/src/Extensions/ConnectionInfoExtensions.cs b/lib/Plugins.Essentials/src/Extensions/ConnectionInfoExtensions.cs
index 5e0b04d..5c36465 100644
--- a/lib/Plugins.Essentials/src/Extensions/ConnectionInfoExtensions.cs
+++ b/lib/Plugins.Essentials/src/Extensions/ConnectionInfoExtensions.cs
@@ -30,6 +30,7 @@ using System.Diagnostics.CodeAnalysis;
 using System.Runtime.CompilerServices;
 
 using VNLib.Net.Http;
+using VNLib.Utils.Memory;
 using VNLib.Utils.Extensions;
 
 namespace VNLib.Plugins.Essentials.Extensions
@@ -154,6 +155,61 @@ namespace VNLib.Plugins.Essentials.Extensions
             server.Headers[HttpResponseHeader.LastModified] = value.ToString("R");
         }
 
+
+        /// <summary>
+        /// Sets the content-range header to the specified parameters
+        /// </summary>
+        /// <param name="entity"></param>
+        /// <param name="range">The http range used to return set the response header</param>
+        /// <param name="length">The total content length</param>
+        /// <exception cref="ArgumentOutOfRangeException"></exception>
+        public static void SetContentRangeHeader(this IHttpEvent entity, in HttpRange range, long length)
+        {
+            if(length < 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(length), "Length must be greater than or equal to zero");
+            }
+
+            ulong start;
+            ulong end;
+
+            //Determine start and end range from actual length and range
+            switch (range.RangeType)
+            {
+                case HttpRangeType.FullRange:
+                    start = range.Start;
+                    end = range.End;
+                    break;
+
+                case HttpRangeType.FromStart:
+                    start = range.Start;
+                    end = (ulong)length - 1;
+                    break;
+
+                case HttpRangeType.FromEnd:
+                    start = (ulong)length - range.End;
+                    end = (ulong)length - 1;
+                    break;
+
+                default:
+                    throw new InvalidOperationException("Invalid range type");
+            }
+
+
+            //Alloc enough space to hold the string
+            Span<char> buffer = stackalloc char[64];
+            ForwardOnlyWriter<char> rangeBuilder = new(buffer);
+            //Build the range header in this format "bytes <begin>-<end>/<total>"
+            rangeBuilder.Append("bytes ");
+            rangeBuilder.Append(start);
+            rangeBuilder.Append('-');
+            rangeBuilder.Append(end);
+            rangeBuilder.Append('/');
+            rangeBuilder.Append(length);
+            //Print to a string and set the content range header
+            entity.Server.Headers[HttpResponseHeader.ContentRange] = rangeBuilder.ToString();
+        }
+
         /// <summary>
         /// Is the connection requesting cors
         /// </summary>
diff --git a/lib/Plugins.Essentials/src/Extensions/EssentialHttpEventExtensions.cs b/lib/Plugins.Essentials/src/Extensions/EssentialHttpEventExtensions.cs
index 976eed5..b09924f 100644
--- a/lib/Plugins.Essentials/src/Extensions/EssentialHttpEventExtensions.cs
+++ b/lib/Plugins.Essentials/src/Extensions/EssentialHttpEventExtensions.cs
@@ -250,12 +250,13 @@ namespace VNLib.Plugins.Essentials.Extensions
         /// <param name="data">The data to straem to the client as an attatcment</param>
         /// <param name="ct">The <see cref="ContentType"/> that represents the file</param>
         /// <param name="fileName">The name of the file to attach</param>
+        /// <param name="length">Explicit length of the stream data</param>
         /// <exception cref="ContentTypeUnacceptableException"></exception>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void CloseResponseAttachment(this IHttpEvent ev, HttpStatusCode code, ContentType ct, Stream data, string fileName)
+        public static void CloseResponseAttachment(this IHttpEvent ev, HttpStatusCode code, ContentType ct, Stream data, string fileName, long length)
         {
             //Close with file
-            ev.CloseResponse(code, ct, data);
+            ev.CloseResponse(code, ct, data, length);
             //Set content dispostion as attachment (only if successfull)
             ev.Server.Headers["Content-Disposition"] = $"attachment; filename=\"{fileName}\"";
         }
@@ -304,9 +305,10 @@ namespace VNLib.Plugins.Essentials.Extensions
             //Get content type from filename
             ContentType ct = HttpHelpers.GetContentTypeFromFile(file.Name);            
             //Set the input as a stream
-            ev.CloseResponse(code, ct, file);
+            ev.CloseResponse(code, ct, file, file.Length);
         }
-        
+
+       
         /// <summary>
         /// Close a response to a connection with a character buffer using the server wide
         /// <see cref="ConnectionInfo.Encoding"/> encoding
@@ -321,7 +323,7 @@ namespace VNLib.Plugins.Essentials.Extensions
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static void CloseResponse(this IHttpEvent ev, HttpStatusCode code, ContentType type, ReadOnlySpan<char> data)
         {
-            //Get a memory stream using UTF8 encoding
+            //Get a memory stream using server built-in encoding
             CloseResponse(ev, code, type, data, ev.Server.Encoding);
         }
         
@@ -344,7 +346,7 @@ namespace VNLib.Plugins.Essentials.Extensions
             }
 
             //Validate encoding
-            _ = encoding ?? throw new ArgumentNullException(nameof(encoding));
+            ArgumentNullException.ThrowIfNull(encoding, nameof(encoding));
             
             //Get new simple memory response
             IMemoryResponseReader reader = new SimpleMemoryResponse(data, encoding);
@@ -790,6 +792,13 @@ namespace VNLib.Plugins.Essentials.Extensions
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static string ContentTypeString(this in FileUpload upload) => HttpHelpers.GetContentTypeString(upload.ContentType);
 
+        /// <summary>
+        /// Sets the <see cref="HttpControlMask.CompressionDisabed"/> flag on the current 
+        /// <see cref="IHttpEvent"/> instance to disable dynamic compression on the response.
+        /// </summary>
+        /// <param name="entity"></param>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static void DisableCompression(this IHttpEvent entity) => entity.SetControlFlag(HttpControlMask.CompressionDisabed);
 
         /// <summary>
         /// Attempts to upgrade the connection to a websocket, if the setup fails, it sets up the response to the client accordingly.
diff --git a/lib/Plugins.Essentials/src/HttpEntity.cs b/lib/Plugins.Essentials/src/HttpEntity.cs
index 64f18ec..f48198b 100644
--- a/lib/Plugins.Essentials/src/HttpEntity.cs
+++ b/lib/Plugins.Essentials/src/HttpEntity.cs
@@ -183,14 +183,15 @@ namespace VNLib.Plugins.Essentials
         ///<inheritdoc/>
         ///<exception cref="ContentTypeUnacceptableException"></exception>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public void CloseResponse(HttpStatusCode code, ContentType type, Stream stream)
+        public void CloseResponse(HttpStatusCode code, ContentType type, Stream stream, long length)
         {
-            Entity.CloseResponse(code, type, stream);
             //Verify content type matches
             if (!Server.Accepts(type))
             {
                 throw new ContentTypeUnacceptableException("The client does not accept the content type of the response");
             }
+
+            Entity.CloseResponse(code, type, stream, length);
         }
 
         ///<inheritdoc/>
@@ -209,7 +210,7 @@ namespace VNLib.Plugins.Essentials
 
         ///<inheritdoc/>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public void DisableCompression() => Entity.DisableCompression();
+        public void SetControlFlag(ulong mask) => Entity.SetControlFlag(mask);
 
         /*
          * Do not directly expose dangerous methods, but allow them to be called