diff options
30 files changed, 2897 insertions, 153 deletions
@@ -369,3 +369,4 @@ MigrationBackup/ *.filters *.ps1 **/.task/* +/lib/Net.Compression/vnlib_compress/build diff --git a/lib/Net.Compression/LICENSE.txt b/lib/Net.Compression/LICENSE.txt new file mode 100644 index 0000000..8a2d49d --- /dev/null +++ b/lib/Net.Compression/LICENSE.txt @@ -0,0 +1,293 @@ +Copyright (c) 2023 Vaughn Nugent + +Contact information + Name: Vaughn Nugent + Email: public[at]vaughnnugent[dot]com + Website: https://www.vaughnnugent.com + +The software in this repository is licensed under the GNU GPL version 2.0 (or any later version). + +SPDX-License-Identifier: GPL-2.0-or-later + +License-Text: + +GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS
\ No newline at end of file diff --git a/lib/Net.Compression/VNLib.Net.Compression/CompressionOperation.cs b/lib/Net.Compression/VNLib.Net.Compression/CompressionOperation.cs new file mode 100644 index 0000000..6093e01 --- /dev/null +++ b/lib/Net.Compression/VNLib.Net.Compression/CompressionOperation.cs @@ -0,0 +1,73 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Net.Compression +* File: CompressionOperation.cs +* +* CompressorManager.cs is part of VNLib.Net.Compression which is part of +* the larger VNLib collection of libraries and utilities. +* +* VNLib.Net.Compression is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* VNLib.Net.Compression is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with VNLib.Net.Compression. If not, see http://www.gnu.org/licenses/. +*/ + +using System.Runtime.InteropServices; + +namespace VNLib.Net.Compression +{ + /// <summary> + /// Matches the native compression operation struct + /// </summary> + [StructLayout(LayoutKind.Sequential)] + internal unsafe ref struct CompressionOperation + { + #region readonly + + /// <summary> + /// A value that indicates a flush operation, 0 for no flush, above 0 for flush + /// </summary> + public int flush; + + /// <summary> + /// A pointer to the input buffer + /// </summary> + public void* inputBuffer; + /// <summary> + /// The size of the input buffer + /// </summary> + public int inputSize; + + /// <summary> + /// A pointer to the output buffer + /// </summary> + public void* outputBuffer; + /// <summary> + /// The size of the output buffer + /// </summary> + public int outputSize; + + #endregion + + + /// <summary> + /// An output variable, the number of bytes read from the input buffer + /// </summary> + public int bytesRead; + + /// <summary> + /// An output variable, the number of bytes written to the output buffer + /// </summary> + public int bytesWritten; + } +}
\ No newline at end of file diff --git a/lib/Net.Compression/VNLib.Net.Compression/CompressorManager.cs b/lib/Net.Compression/VNLib.Net.Compression/CompressorManager.cs new file mode 100644 index 0000000..5c4f4fd --- /dev/null +++ b/lib/Net.Compression/VNLib.Net.Compression/CompressorManager.cs @@ -0,0 +1,258 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Net.Compression +* File: CompressorManager.cs +* +* CompressorManager.cs is part of VNLib.Net.Compression which is part of +* the larger VNLib collection of libraries and utilities. +* +* VNLib.Net.Compression is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* VNLib.Net.Compression is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with VNLib.Net.Compression. If not, see http://www.gnu.org/licenses/. +*/ + +/* + * Notes: + * + * This library implements the IHttpCompressorManager for dynamic library + * loading by the VNLib.Webserver (or any other application that implements + * the IHttpCompressorManager interface). + * + * It implements an unspecified method called OnLoad that is exepcted to be + * called by the VNLib.Webserver during load time. This method is used to + * initialize the compressor and load the native library. + */ + +using System; +using System.Buffers; +using System.Diagnostics; +using System.IO.Compression; +using System.Text.Json; +using System.Runtime.CompilerServices; + +using VNLib.Net.Http; +using VNLib.Utils.Memory; +using VNLib.Utils.Logging; + +namespace VNLib.Net.Compression +{ + public sealed class CompressorManager : IHttpCompressorManager + { + const string NATIVE_LIB_NAME = "vnlib_compress.dll"; + const int MIN_BUF_SIZE_DEFAULT = 8192; + + private LibraryWrapper? _nativeLib; + private CompressionLevel _compLevel; + private int minOutBufferSize; + + /// <summary> + /// Called by the VNLib.Webserver during startup to initiialize the compressor. + /// </summary> + /// <param name="log">The application log provider</param> + /// <param name="configJsonString">The raw json configuration data</param> + public void OnLoad(ILogProvider? log, JsonElement? config) + { + _compLevel = CompressionLevel.Optimal; + minOutBufferSize = MIN_BUF_SIZE_DEFAULT; + string libPath = NATIVE_LIB_NAME; + + if(config.HasValue) + { + //Get the compression element + if(config.Value.TryGetProperty("vnlib.net.compression", out JsonElement compEl)) + { + //Try to get the user specified compression level + if(compEl.TryGetProperty("level", out JsonElement lEl)) + { + _compLevel = (CompressionLevel)lEl.GetUInt16(); + } + + //Allow the user to specify the path to the native library + if(compEl.TryGetProperty("lib_path", out JsonElement libEl)) + { + libPath = libEl.GetString() ?? NATIVE_LIB_NAME; + } + + if(compEl.TryGetProperty("min_out_buf_size", out JsonElement minBufEl)) + { + minOutBufferSize = minBufEl.GetInt32(); + } + } + } + + log?.Debug("Attempting to load native compression library from: {lib}", libPath); + + //Load the native library + _nativeLib = LibraryWrapper.LoadLibrary(libPath); + + log?.Debug("Loaded native compression library with compression level {l}", _compLevel.ToString()); + } + + ///<inheritdoc/> + public CompressionMethod GetSupportedMethods() + { + if(_nativeLib == null) + { + throw new InvalidOperationException("The native library has not been loaded yet."); + } + + return _nativeLib.GetSupportedMethods(); + } + + ///<inheritdoc/> + public object AllocCompressor() + { + return new Compressor(); + } + + ///<inheritdoc/> + public int InitCompressor(object compressorState, CompressionMethod compMethod) + { + //For now do not allow empty compression methods, later we should allow this to be used as a passthrough + if(compMethod == CompressionMethod.None) + { + throw new ArgumentException("Compression method cannot be None", nameof(compMethod)); + } + + Compressor compressor = Unsafe.As<Compressor>(compressorState) ?? throw new ArgumentNullException(nameof(compressorState)); + + //Instance should be null during initialization calls + Debug.Assert(compressor.Instance == IntPtr.Zero); + + //Alloc the compressor + compressor.Instance = _nativeLib!.AllocateCompressor(compMethod, _compLevel); + + //Return the compressor block size + return _nativeLib!.GetBlockSize(compressor.Instance); + } + + ///<inheritdoc/> + public void DeinitCompressor(object compressorState) + { + Compressor compressor = Unsafe.As<Compressor>(compressorState) ?? throw new ArgumentNullException(nameof(compressorState)); + + if(compressor.Instance == IntPtr.Zero) + { + throw new InvalidOperationException("This compressor instance has not been initialized, cannot free compressor"); + } + + //Free the output buffer + if(compressor.OutputBuffer != null) + { + ArrayPool<byte>.Shared.Return(compressor.OutputBuffer, true); + compressor.OutputBuffer = null; + } + + //Free compressor instance + _nativeLib!.FreeCompressor(compressor.Instance); + + //Clear pointer after successful free + compressor.Instance = IntPtr.Zero; + } + + ///<inheritdoc/> + public ReadOnlyMemory<byte> Flush(object compressorState) + { + Compressor compressor = Unsafe.As<Compressor>(compressorState) ?? throw new ArgumentNullException(nameof(compressorState)); + + if (compressor.Instance == IntPtr.Zero) + { + throw new InvalidOperationException("This compressor instance has not been initialized, cannot free compressor"); + } + + //rent a new buffer of the minimum size if not already allocated + compressor.OutputBuffer ??= ArrayPool<byte>.Shared.Rent(minOutBufferSize); + + //Force a flush until no more data is available + int bytesWritten = CompressBlock(compressor.Instance, compressor.OutputBuffer, default, true); + + return compressor.OutputBuffer.AsMemory(0, bytesWritten); + } + + ///<inheritdoc/> + public ReadOnlyMemory<byte> CompressBlock(object compressorState, ReadOnlyMemory<byte> input, bool finalBlock) + { + Compressor compressor = Unsafe.As<Compressor>(compressorState) ?? throw new ArgumentNullException(nameof(compressorState)); + + if (compressor.Instance == IntPtr.Zero) + { + throw new InvalidOperationException("This compressor instance has not been initialized, cannot free compressor"); + } + + /* + * We only alloc the buffer on the first call because we can assume this is the + * largest input data the compressor will see, and the block size should be used + * as a reference for callers. If its too small it will just have to be flushed + */ + + //See if the compressor has a buffer allocated + if (compressor.OutputBuffer == null) + { + //Determine the required buffer size + int bufferSize = _nativeLib!.GetOutputSize(compressor.Instance, input.Length, finalBlock ? 1 : 0); + + //clamp the buffer size to the minimum output buffer size + bufferSize = Math.Max(bufferSize, minOutBufferSize); + + //rent a new buffer + compressor.OutputBuffer = ArrayPool<byte>.Shared.Rent(bufferSize); + } + + //Compress the block + int bytesWritten = CompressBlock(compressor.Instance, compressor.OutputBuffer, input, finalBlock); + + return compressor.OutputBuffer.AsMemory(0, bytesWritten); + } + + private unsafe int CompressBlock(IntPtr comp, byte[] output, ReadOnlyMemory<byte> input, bool finalBlock) + { + //get pointers to the input and output buffers + using MemoryHandle inPtr = input.Pin(); + using MemoryHandle outPtr = MemoryUtil.PinArrayAndGetHandle(output, 0); + + //Create the operation struct + CompressionOperation operation; + CompressionOperation* op = &operation; + + op->flush = finalBlock ? 1 : 0; + op->bytesRead = 0; + op->bytesWritten = 0; + + //Configure the input and output buffers + op->inputBuffer = inPtr.Pointer; + op->inputSize = input.Length; + + op->outputBuffer = outPtr.Pointer; + op->outputSize = output.Length; + + //Call the native compress function + _nativeLib!.CompressBlock(comp, &operation); + + //Return the number of bytes written + return op->bytesWritten; + } + + + /* + * A class to contain the compressor state + */ + private sealed class Compressor + { + public IntPtr Instance; + + public byte[]? OutputBuffer; + } + + } +}
\ No newline at end of file diff --git a/lib/Net.Compression/VNLib.Net.Compression/LibraryWrapper.cs b/lib/Net.Compression/VNLib.Net.Compression/LibraryWrapper.cs new file mode 100644 index 0000000..0f86107 --- /dev/null +++ b/lib/Net.Compression/VNLib.Net.Compression/LibraryWrapper.cs @@ -0,0 +1,277 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Net.Compression +* File: LibraryWrapper.cs +* +* LibraryWrapper.cs is part of VNLib.Net.Compression which is part of +* the larger VNLib collection of libraries and utilities. +* +* VNLib.Net.Compression is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* VNLib.Net.Compression is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with VNLib.Net.Compression. If not, see http://www.gnu.org/licenses/. +*/ + + +using System; +using System.IO.Compression; +using System.Runtime.InteropServices; +using System.Runtime.CompilerServices; + +using VNLib.Utils; +using VNLib.Utils.Native; +using VNLib.Utils.Extensions; + +using VNLib.Net.Http; + +namespace VNLib.Net.Compression +{ + /* + * Configure the delegate methods for the native library + * + * All calling conventions are set to Cdecl because the native + * library is compiled with Cdecl on all platforms. + */ + + [SafeMethodName("GetSupportedCompressors")] + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + delegate CompressionMethod GetSupportedMethodsDelegate(); + + [SafeMethodName("GetCompressorBlockSize")] + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + delegate int GetBlockSizeDelegate(IntPtr compressor); + + [SafeMethodName("GetCompressorType")] + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + delegate CompressionMethod GetCompressorTypeDelegate(IntPtr compressor); + + [SafeMethodName("GetCompressorLevel")] + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + delegate CompressionLevel GetCompressorLevelDelegate(IntPtr compressor); + + [SafeMethodName("AllocateCompressor")] + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + delegate IntPtr AllocateCompressorDelegate(CompressionMethod type, CompressionLevel level); + + [SafeMethodName("FreeCompressor")] + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + delegate int FreeCompressorDelegate(IntPtr compressor); + + [SafeMethodName("GetCompressedSize")] + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + delegate int GetCompressedSizeDelegate(IntPtr compressor, int uncompressedSize, int flush); + + [SafeMethodName("CompressBlock")] + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + unsafe delegate int CompressBlockDelegate(IntPtr compressor, CompressionOperation* operation); + + /// <summary> + /// <para> + /// Represents a wrapper that provides access to the native compression library + /// specified by a file path. + /// </para> + /// <para> + /// NOTE: This library is not meant to be freed, its meant to be loaded at runtime + /// and used for the lifetime of the application. + /// </para> + /// </summary> + internal sealed class LibraryWrapper + { + private readonly SafeLibraryHandle _lib; + private readonly MethodTable _methodTable; + + public string LibFilePath { get; } + + private LibraryWrapper(SafeLibraryHandle lib, string path, in MethodTable methodTable) + { + _lib = lib; + _methodTable = methodTable; + LibFilePath = path; + } + + /// <summary> + /// Loads the native library at the specified path into the current process + /// </summary> + /// <param name="filePath">The path to the native library to load</param> + /// <returns>The native library wrapper</returns> + public static LibraryWrapper LoadLibrary(string filePath) + { + //Load the library into the current process + SafeLibraryHandle lib = SafeLibraryHandle.LoadLibrary(filePath, DllImportSearchPath.SafeDirectories); + + try + { + //build the method table + MethodTable methods = new() + { + GetMethods = lib.DangerousGetMethod<GetSupportedMethodsDelegate>(), + + GetBlockSize = lib.DangerousGetMethod<GetBlockSizeDelegate>(), + + GetCompType = lib.DangerousGetMethod<GetCompressorTypeDelegate>(), + + GetCompLevel = lib.DangerousGetMethod<GetCompressorLevelDelegate>(), + + Alloc = lib.DangerousGetMethod<AllocateCompressorDelegate>(), + + Free = lib.DangerousGetMethod<FreeCompressorDelegate>(), + + GetOutputSize = lib.DangerousGetMethod<GetCompressedSizeDelegate>(), + + Compress = lib.DangerousGetMethod<CompressBlockDelegate>() + }; + + return new (lib, filePath, in methods); + } + catch + { + lib.Dispose(); + throw; + } + } + + /// <summary> + /// Gets an enum value of the supported compression methods by the underlying library + /// </summary> + /// <returns>The supported compression methods</returns> + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public CompressionMethod GetSupportedMethods() => _methodTable.GetMethods(); + + /// <summary> + /// Gets the block size of the specified compressor + /// </summary> + /// <param name="compressor">A pointer to the compressor instance </param> + /// <returns>A integer value of the compressor block size</returns> + /// <exception cref="NativeCompressionException"></exception> + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetBlockSize(IntPtr compressor) + { + int result = _methodTable.GetBlockSize(compressor); + ThrowHelper.ThrowIfError((ERRNO)result); + return result; + } + + /// <summary> + /// Gets the compressor type of the specified compressor + /// </summary> + /// <param name="compressor">A pointer to the compressor instance</param> + /// <returns>A enum value that represents the compressor type</returns> + /// <exception cref="NativeCompressionException"></exception> + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public CompressionMethod GetCompressorType(IntPtr compressor) + { + CompressionMethod result = _methodTable.GetCompType(compressor); + ThrowHelper.ThrowIfError((int)result); + return result; + } + + /// <summary> + /// Gets the compression level of the specified compressor + /// </summary> + /// <param name="compressor">A pointer to the compressor instance</param> + /// <returns>The <see cref="CompressionLevel"/> of the current compressor</returns> + /// <exception cref="NativeCompressionException"></exception> + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public CompressionLevel GetCompressorLevel(IntPtr compressor) + { + CompressionLevel result = _methodTable.GetCompLevel(compressor); + ThrowHelper.ThrowIfError((int)result); + return result; + } + + /// <summary> + /// Allocates a new compressor instance of the specified type and compression level + /// </summary> + /// <param name="type">The compressor type to allocate</param> + /// <param name="level">The desired compression level</param> + /// <returns>A pointer to the newly allocated compressor instance</returns> + /// <exception cref="NotSupportedException"></exception> + /// <exception cref="NativeCompressionException"></exception> + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public IntPtr AllocateCompressor(CompressionMethod type, CompressionLevel level) + { + IntPtr result = _methodTable.Alloc(type, level); + ThrowHelper.ThrowIfError(result); + return result; + } + + /// <summary> + /// Frees the specified compressor instance + /// </summary> + /// <param name="compressor">A pointer to the valid compressor instance to free</param> + /// <returns>A value indicating the result of the free operation</returns> + /// <exception cref="NativeCompressionException"></exception> + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void FreeCompressor(IntPtr compressor) + { + int result = _methodTable.Free(compressor); + ThrowHelper.ThrowIfError(result); + if(result == 0) + { + throw new NativeCompressionException("Failed to free the compressor instance"); + } + } + + /// <summary> + /// Determines the output size of a given input size and flush mode for the specified compressor + /// </summary> + /// <param name="compressor">A pointer to the compressor instance</param> + /// <param name="inputSize">The size of the input block to compress</param> + /// <param name="flush">A value that specifies a flush operation</param> + /// <returns>Returns the size of the required output buffer</returns> + /// <exception cref="NotSupportedException"></exception> + /// <exception cref="NativeCompressionException"></exception> + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetOutputSize(IntPtr compressor, int inputSize, int flush) + { + int result = _methodTable.GetOutputSize(compressor, inputSize, flush); + ThrowHelper.ThrowIfError(result); + return result; + } + + /// <summary> + /// Compresses a block of data using the specified compressor instance + /// </summary> + /// <param name="compressor">The compressor instance used to compress data</param> + /// <param name="operation">A pointer to the compression operation structure</param> + /// <returns>The result of the operation</returns> + /// <exception cref="NotSupportedException"></exception> + /// <exception cref="NativeLibraryException"></exception> + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public unsafe int CompressBlock(IntPtr compressor, CompressionOperation* operation) + { + int result = _methodTable.Compress(compressor, operation); + ThrowHelper.ThrowIfError(result); + return result; + } + + private readonly struct MethodTable + { + public GetSupportedMethodsDelegate GetMethods { get; init; } + + public GetBlockSizeDelegate GetBlockSize { get; init; } + + public GetCompressorTypeDelegate GetCompType { get; init; } + + public GetCompressorLevelDelegate GetCompLevel { get; init; } + + public AllocateCompressorDelegate Alloc { get; init; } + + public FreeCompressorDelegate Free { get; init; } + + public GetCompressedSizeDelegate GetOutputSize { get; init; } + + public CompressBlockDelegate Compress { get; init; } + } + } +}
\ No newline at end of file diff --git a/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionException.cs b/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionException.cs new file mode 100644 index 0000000..2f523e2 --- /dev/null +++ b/lib/Net.Compression/VNLib.Net.Compression/NativeCompressionException.cs @@ -0,0 +1,42 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Net.Compression +* File: NativeCompressionException.cs +* +* NativeCompressionException.cs is part of VNLib.Net.Compression which is part of +* the larger VNLib collection of libraries and utilities. +* +* VNLib.Net.Compression is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* VNLib.Net.Compression is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with VNLib.Net.Compression. If not, see http://www.gnu.org/licenses/. +*/ + +using System; + +using VNLib.Utils.Native; + +namespace VNLib.Net.Compression +{ + internal sealed class NativeCompressionException : NativeLibraryException + { + public NativeCompressionException() + { } + + public NativeCompressionException(string message) : base(message) + { } + + public NativeCompressionException(string message, Exception innerException) : base(message, innerException) + { } + } +}
\ No newline at end of file diff --git a/lib/Net.Compression/VNLib.Net.Compression/ThrowHelper.cs b/lib/Net.Compression/VNLib.Net.Compression/ThrowHelper.cs new file mode 100644 index 0000000..2793526 --- /dev/null +++ b/lib/Net.Compression/VNLib.Net.Compression/ThrowHelper.cs @@ -0,0 +1,91 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Net.Compression +* File: ThrowHelper.cs +* +* ThrowHelper.cs is part of VNLib.Net.Compression which is part of +* the larger VNLib collection of libraries and utilities. +* +* VNLib.Net.Compression is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* VNLib.Net.Compression is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with VNLib.Net.Compression. If not, see http://www.gnu.org/licenses/. +*/ + +using System; + +using VNLib.Utils; + +namespace VNLib.Net.Compression +{ + internal static class ThrowHelper + { + /* + * Error codes correspond to constants + * in the native compression library + */ + enum NativeErrorType + { + ErrInvalidPtr = -1, + ErrOutOfMemory = -2, + + ErrCompTypeNotSupported = -9, + ErrCompLevelNotSupported = -10, + ErrInvalidInput = -11, + ErrInvalidOutput = -12, + + ErrGzInvalidState = -16, + ErrGzOverflow = -17, + + ErrBrInvalidState = -24 + } + + /// <summary> + /// Determines if the specified result is an error and throws an exception if it is + /// </summary> + /// <param name="result"></param> + /// <exception cref="NativeCompressionException"></exception> + public static void ThrowIfError(ERRNO result) + { + //Check for no error + if(result > 0) + { + return; + } + + switch ((NativeErrorType)(int)result) + { + case NativeErrorType.ErrInvalidPtr: + throw new NativeCompressionException("A pointer to a compressor instance was null"); + case NativeErrorType.ErrOutOfMemory: + throw new NativeCompressionException("An operation falied because the system is out of memory"); + case NativeErrorType.ErrCompTypeNotSupported: + throw new NotSupportedException("The desired compression method is not supported by the native library"); + case NativeErrorType.ErrCompLevelNotSupported: + throw new NotSupportedException("The desired compression level is not supported by the native library"); + case NativeErrorType.ErrInvalidInput: + throw new NativeCompressionException("The input buffer was null and the input size was greater than 0"); + case NativeErrorType.ErrInvalidOutput: + throw new NativeCompressionException("The output buffer was null and the output size was greater than 0"); + case NativeErrorType.ErrGzInvalidState: + throw new NativeCompressionException("A gzip operation failed because the compressor state is invalid (null compressor pointer)"); + case NativeErrorType.ErrGzOverflow: + throw new NativeCompressionException("A gzip operation failed because the output buffer is too small"); + case NativeErrorType.ErrBrInvalidState: + throw new NativeCompressionException("A brotli operation failed because the compressor state is invalid (null compressor pointer)"); + default: + break; + } + } + } +}
\ No newline at end of file diff --git a/lib/Net.Compression/VNLib.Net.Compression/VNLib.Net.Compression.csproj b/lib/Net.Compression/VNLib.Net.Compression/VNLib.Net.Compression.csproj new file mode 100644 index 0000000..e91c640 --- /dev/null +++ b/lib/Net.Compression/VNLib.Net.Compression/VNLib.Net.Compression.csproj @@ -0,0 +1,37 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net6.0</TargetFramework> + <RootNamespace>VNLib.Net.Compression</RootNamespace> + <AssemblyName>VNLib.Net.Compression</AssemblyName> + <AllowUnsafeBlocks>true</AllowUnsafeBlocks> + <Nullable>enable</Nullable> + <GenerateDocumentationFile>True</GenerateDocumentationFile> + <AnalysisLevel>latest-all</AnalysisLevel> + <RunAnalyzersDuringBuild>false</RunAnalyzersDuringBuild> + + <!--Enable dynamic loading for debugging--> + <EnableDynamicLoading>true</EnableDynamicLoading> + </PropertyGroup> + + <PropertyGroup> + <PackageId>VNLib.Net.Compression</PackageId> + <Authors>Vaughn Nugent</Authors> + <Company>Vaughn Nugent</Company> + <Product>VNLib Native Http Compression Provider</Product> + <Copyright>Copyright © 2023 Vaughn Nugent</Copyright> + <Description> + .NET/6.0 dynamically loadable managed wrapper library for loading vnlib_compress native library. It provides + an implementation of the IHttpCompressorManager interface for use with the VNLib.Net.Http library and servers + wishing to support dynamic response compression. + </Description> + <PackageProjectUrl>https://www.vaughnnugent.com/resources/software/modules/VNLib.Core</PackageProjectUrl> + <RepositoryUrl>https://github.com/VnUgE/VNLib.Core/tree/main/lib/Net.Compression</RepositoryUrl> + </PropertyGroup> + + <ItemGroup> + <ProjectReference Include="..\..\Net.Http\src\VNLib.Net.Http.csproj" /> + <ProjectReference Include="..\..\Utils\src\VNLib.Utils.csproj" /> + </ItemGroup> + +</Project> diff --git a/lib/Net.Compression/VNLib.Net.CompressionTests/CompressorManagerTests.cs b/lib/Net.Compression/VNLib.Net.CompressionTests/CompressorManagerTests.cs new file mode 100644 index 0000000..2dea9d7 --- /dev/null +++ b/lib/Net.Compression/VNLib.Net.CompressionTests/CompressorManagerTests.cs @@ -0,0 +1,212 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.IO.Compression; +using System.Security.Cryptography; + +using VNLib.Utils.IO; +using VNLib.Net.Http; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace VNLib.Net.Compression.Tests +{ + [TestClass()] + public class CompressorManagerTests + { + const string LIB_PATH = @"F:\Programming\VNLib\VNLib.Net.Compression\native\vnlib_compress\build\Debug\vnlib_compress.dll"; + + + [TestMethod()] + public void OnLoadTest() + { + CompressorManager manager = InitCompressorUnderTest(); + + //Allocate a compressor instance + object? compressor = manager.AllocCompressor(); + + Assert.IsNotNull(compressor); + + //Test all 3 compression methods + TestCompressorMethod(manager, compressor, CompressionMethod.Brotli); + TestCompressorMethod(manager, compressor, CompressionMethod.Gzip); + TestCompressorMethod(manager, compressor, CompressionMethod.Deflate); + } + + [TestMethod()] + public void InitCompressorTest() + { + CompressorManager manager = InitCompressorUnderTest(); + + //Allocate a compressor instance + object compressor = manager.AllocCompressor(); + + Assert.IsNotNull(compressor); + + Assert.ThrowsException<ArgumentNullException>(() => manager.InitCompressor(null!, CompressionMethod.Deflate)); + Assert.ThrowsException<ArgumentNullException>(() => manager.DeinitCompressor(null!)); + + //Make sure error occurs with non-supported comp + Assert.ThrowsException<ArgumentException>(() => { manager.InitCompressor(compressor, CompressionMethod.None); }); + + //test out of range, this should be a native lib error + Assert.ThrowsException<NotSupportedException>(() => { manager.InitCompressor(compressor, (CompressionMethod)24); }); + + //Test all 3 compression methods + CompressionMethod supported = manager.GetSupportedMethods(); + + if ((supported & CompressionMethod.Gzip) > 0) + { + //Make sure no error occurs with supported comp + manager.InitCompressor(compressor, CompressionMethod.Gzip); + manager.DeinitCompressor(compressor); + } + + if((supported & CompressionMethod.Brotli) > 0) + { + //Make sure no error occurs with supported comp + manager.InitCompressor(compressor, CompressionMethod.Brotli); + manager.DeinitCompressor(compressor); + } + + if((supported & CompressionMethod.Deflate) > 0) + { + //Make sure no error occurs with supported comp + manager.InitCompressor(compressor, CompressionMethod.Deflate); + manager.DeinitCompressor(compressor); + } + } + + private static CompressorManager InitCompressorUnderTest() + { + CompressorManager manager = new(); + + //Get the json config string + string config = GetCompConfig(); + + using JsonDocument doc = JsonDocument.Parse(config); + + //Attempt to load the native library + manager.OnLoad(null, doc.RootElement); + + //Get supported methods + CompressionMethod methods = manager.GetSupportedMethods(); + + //Verify that at least one method is supported + Assert.IsFalse(methods == CompressionMethod.None); + + return manager; + } + + private static string GetCompConfig() + { + using VnMemoryStream ms = new(); + using (Utf8JsonWriter writer = new(ms)) + { + writer.WriteStartObject(); + + writer.WriteStartObject("vnlib.net.compression"); + + writer.WriteNumber("level", 1); + writer.WriteString("lib_path", LIB_PATH); + + writer.WriteEndObject(); + writer.WriteEndObject(); + } + + return Encoding.UTF8.GetString(ms.AsSpan()); + } + + private static void TestCompressorMethod(CompressorManager manager, object compressor, CompressionMethod method) + { + /* + * This test method initalizes a new compressor instance of the desired type + * creates a test data buffer, compresses it using the compressor instance + * then decompresses the compressed data using a managed decompressor as + * a reference and compares the results. + * + * The decompression must be able to recover the original data. + */ + + //Time to initialize the compressor + int blockSize = manager.InitCompressor(compressor, method); + + Assert.IsTrue(blockSize == 0); + + try + { + using VnMemoryStream outputStream = new(); + + //Create a buffer to compress + byte[] buffer = new byte[4096]; + + //fill with random data + RandomNumberGenerator.Fill(buffer); + + //try to compress the data in chunks + for(int i = 0; i < 4; i++) + { + //Get 4th of a buffer + ReadOnlyMemory<byte> chunk = buffer.AsMemory(i * 1024, 1024); + + //Compress data + ReadOnlyMemory<byte> output = manager.CompressBlock(compressor, chunk, i == 3); + + //Write the compressed data to the output stream + outputStream.Write(output.Span); + } + + //flush the compressor + while(true) + { + ReadOnlyMemory<byte> output = manager.Flush(compressor); + if(output.IsEmpty) + { + break; + } + outputStream.Write(output.Span); + } + + //Verify the data + byte[] decompressed = DecompressData(outputStream, method); + + Assert.IsTrue(buffer.SequenceEqual(decompressed)); + } + finally + { + //Always deinitialize the compressor when done + manager.DeinitCompressor(compressor); + } + } + + + private static byte[] DecompressData(VnMemoryStream inputStream, CompressionMethod method) + { + inputStream.Position = 0; + + //Stream to write output data to + using VnMemoryStream output = new(); + + //Get the requested stream type to decompress the data + using (Stream gz = GetDecompStream(inputStream, method)) + { + gz.CopyTo(output); + } + + return output.ToArray(); + } + + private static Stream GetDecompStream(Stream input, CompressionMethod method) + { + return method switch + { + CompressionMethod.Gzip => new GZipStream(input, CompressionMode.Decompress, true), + CompressionMethod.Deflate => new DeflateStream(input, CompressionMode.Decompress, true), + CompressionMethod.Brotli => new BrotliStream(input, CompressionMode.Decompress, true), + _ => throw new ArgumentException("Unsupported compression method", nameof(method)), + }; + } + } +}
\ No newline at end of file diff --git a/lib/Net.Compression/VNLib.Net.CompressionTests/VNLib.Net.CompressionTests.csproj b/lib/Net.Compression/VNLib.Net.CompressionTests/VNLib.Net.CompressionTests.csproj new file mode 100644 index 0000000..e235ac5 --- /dev/null +++ b/lib/Net.Compression/VNLib.Net.CompressionTests/VNLib.Net.CompressionTests.csproj @@ -0,0 +1,22 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net6.0</TargetFramework> + <Nullable>enable</Nullable> + + <IsPackable>false</IsPackable> + <IsTestProject>true</IsTestProject> + </PropertyGroup> + + <ItemGroup> + <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" /> + <PackageReference Include="MSTest.TestAdapter" Version="2.2.10" /> + <PackageReference Include="MSTest.TestFramework" Version="2.2.10" /> + <PackageReference Include="coverlet.collector" Version="3.2.0" /> + </ItemGroup> + + <ItemGroup> + <ProjectReference Include="..\VNLib.Net.Compression\VNLib.Net.Compression.csproj" /> + </ItemGroup> + +</Project> diff --git a/lib/Net.Compression/build.readme.md b/lib/Net.Compression/build.readme.md new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/lib/Net.Compression/build.readme.md diff --git a/lib/Net.Compression/readme.md b/lib/Net.Compression/readme.md new file mode 100644 index 0000000..6b7f5f1 --- /dev/null +++ b/lib/Net.Compression/readme.md @@ -0,0 +1,18 @@ +# VNLib.Net.Compression + +Provides a cross platform (w/ cmake) native compression DLL for Brotli, Deflate, and Gzip compression encodings for dynamic HTTP response streaming of arbitrary data. This directory also provides a managed implementation with support for runtime loading. + +The native library relies on source code (which are statically compiled) for Brotli and Zlib. The original repositories for both libraries will do, but I use the Cloudflare fork of Zlib for testing. You should consult my documentation below for how and where to get the source for these libraries. + + +### Builds +Debug build w/ symbols & xml docs, release builds, NuGet packages, and individually packaged source code are available on my website (link below). All tar-gzip (.tgz) files will have an associated .sha384 appended checksum of the desired download file. + +### Docs and Guides +Documentation, specifications, and setup guides are available on my website. + +[Docs and Articles](https://www.vaughnnugent.com/resources/software/articles?tags=docs,_vnlib.net.compression) +[Builds and Source](https://www.vaughnnugent.com/resources/software/modules/VNLib.Core) + +## License +The software in this repository is licensed under the GNU GPL version 2.0 (or any later version). See the LICENSE files for more information.
\ No newline at end of file diff --git a/lib/Net.Compression/third-party/readme.md b/lib/Net.Compression/third-party/readme.md new file mode 100644 index 0000000..f10eb0e --- /dev/null +++ b/lib/Net.Compression/third-party/readme.md @@ -0,0 +1,3 @@ +# Third party + +This directory should contain the required third party libraries. Their should be a directory called `zlib` which includes the zlib source files, and a directory called `brotli` which includes the brotli source files. See the [readme](../readme.md) for more information diff --git a/lib/Net.Compression/vnlib_compress/CMakeLists.txt b/lib/Net.Compression/vnlib_compress/CMakeLists.txt new file mode 100644 index 0000000..bc3c1df --- /dev/null +++ b/lib/Net.Compression/vnlib_compress/CMakeLists.txt @@ -0,0 +1,140 @@ +cmake_minimum_required(VERSION 3.0) + +project(vnlib_compress) + +#export header files to the main project +file(GLOB COMP_HEADERS *.h) + +#Add indepednent source files to the project +set(VNLIB_COMPRESS_SOURCES compression.c) + +#add feature specific source files to the project +if(ENABLE_BROTLI) + set(VNLIB_FEATURE_BR_SOURCES feature_brotli.c) +endif() + +if(ENABLE_ZLIB) + set(VNLIB_FEATURE_ZLIB_SOURCES feature_zlib.c) +endif() + +#create my shared library +add_library(${CMAKE_PROJECT_NAME} SHARED ${VNLIB_COMPRESS_SOURCES} ${COMP_HEADERS} ${VNLIB_FEATURE_BR_SOURCES} ${VNLIB_FEATURE_ZLIB_SOURCES}) + +#Setup the compiler options + +enable_language(C) +set(CMAKE_CXX_STANDARD 90) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +#enable position independent code (for shared libraries with exports) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +#strict error checking for main project +set(CMAKE_COMPILE_WARNING_AS_ERROR ON) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +#force the compiler to use the C90/89 standard +target_compile_features(${CMAKE_PROJECT_NAME} PRIVATE c_std_90) + +#setup flags for windows compilation +if(MSVC) + + #global windows cl flags + add_compile_options( + /Qspectre + /sdl + /TC + /GS + + #for debug configs + $<$<CONFIG:Debug>:/Wall> + $<$<CONFIG:Debug>:/options:strict> + $<$<CONFIG:Debug>:/FC> + $<$<CONFIG:Debug>:/showIncludes> + ) + + #only target our project + target_compile_options( + ${CMAKE_PROJECT_NAME} + PRIVATE + + $<$<CONFIG:Debug>:/WX> #warnings as errors (only for our project) + $<$<CONFIG:Debug>:/Zi> + $<$<CONFIG:Debug>:/Zo> + ) + + #set build macros + add_compile_definitions( + $<$<CONFIG:DEBUG>:DEBUG> + $<$<CONFIG:RELEASE>:RELEASE> + ) + +#configure gcc flags +elseif(CMAKE_COMPILER_IS_GNUCC) + + add_compile_options( + ${CMAKE_PROJECT_NAME} + PUBLIC + -Wextra + -fstack-protector + + $<$<CONFIG:Debug>:-g> + $<$<CONFIG:Debug>:-Og> + $<$<CONFIG:Debug>:-Wall> + $<$<CONFIG:Debug>:-Werror> + ) + + #only target our project + target_compile_options( + ${CMAKE_PROJECT_NAME} + PRIVATE + $<$<CONFIG:Debug>:-pedantic> + ) + +endif() + +#check for brotli feature enablement +if(ENABLE_BROTLI) + #add the include directory for brotli so we can include the header files + include_directories(../third-party/brotli/c/include) + + #get common sources + file(GLOB BROTLI_SOURCES ../third-party/brotli/c/common/*.c) + + #we need to add the brotli encoder source files to the project + file(GLOB BROTLI_ENC_SOURCES ../third-party/brotli/c/enc/*.c) + + #add brotli as a static library to link later + add_library(lib_brotli STATIC ${BROTLI_SOURCES} ${BROTLI_ENC_SOURCES}) + + #define the brotli feature macro to enable brotli support + add_definitions(-DVNLIB_COMPRESSOR_BROTLI_ENABLED) + + target_link_libraries(${CMAKE_PROJECT_NAME} lib_brotli) +endif() + +#check for zlib feature enablement +if(ENABLE_ZLIB) + #add the include directory for zlib so we can include the header files + include_directories(../third-party/zlib) + + #we only need to add the zlib deflate source files to the project + set(ZLIB_SOURCES + ../third-party/zlib/deflate.c + ../third-party/zlib/adler32.c + ../third-party/zlib/adler32_simd.c + ../third-party/zlib/crc32.c + ../third-party/zlib/zutil.c + ../third-party/zlib/trees.c + ) + + #add zlib as a library to link later + add_library(lib_deflate STATIC ${ZLIB_SOURCES}) + + #define the zlib feature macro to enable zlib support + add_definitions(-DVNLIB_COMPRESSOR_ZLIB_ENABLED) + + target_link_libraries(${CMAKE_PROJECT_NAME} lib_deflate) +endif() + diff --git a/lib/Net.Compression/vnlib_compress/compression.c b/lib/Net.Compression/vnlib_compress/compression.c new file mode 100644 index 0000000..0e563ff --- /dev/null +++ b/lib/Net.Compression/vnlib_compress/compression.c @@ -0,0 +1,416 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: VNLib +* Package: vnlib_compress +* File: compression.c +* +* vnlib_compress is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* vnlib_compress is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with vnlib_compress. If not, see http://www.gnu.org/licenses/. +*/ + + +#include "compression.h" + +#ifdef VNLIB_COMPRESSOR_BROTLI_ENABLED +#include "feature_brotli.h" +#endif /* VNLIB_COMPRESSOR_BROTLI_ENABLED */ + + +#ifdef VNLIB_COMPRESSOR_ZLIB_ENABLED +#include "feature_zlib.h" +#endif /* VNLIB_COMPRESSOR_GZIP_ENABLED */ + +/* +* Configure DLLMAIN on Windows +*/ + +#if false +#define WIN32_LEAN_AND_MEAN +#include <Windows.h> + +BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved) +{ + /* + * Taken from the malloc.c file for initializing the library. + * and thread events + */ + switch (ul_reason_for_call) + { + case DLL_PROCESS_ATTACH: + break; + case DLL_THREAD_ATTACH: + break; + case DLL_THREAD_DETACH: + break; + case DLL_PROCESS_DETACH: + break; + } + return TRUE; +} + +#endif /* DLLMAIN */ + +VNLIB_EXPORT CompressorType VNLIB_CC GetSupportedCompressors(void); + +VNLIB_EXPORT int VNLIB_CC GetCompressorBlockSize(void* compressor); + +VNLIB_EXPORT CompressorType VNLIB_CC GetCompressorType(void* compressor); + +VNLIB_EXPORT CompressionLevel VNLIB_CC GetCompressorLevel(void* compressor); + +VNLIB_EXPORT void* VNLIB_CC AllocateCompressor(CompressorType type, CompressionLevel level); + +VNLIB_EXPORT int VNLIB_CC FreeCompressor(void* compressor); + +VNLIB_EXPORT int VNLIB_CC GetCompressedSize(void* compressor, int inputLength, int flush); + +VNLIB_EXPORT int VNLIB_CC CompressBlock(void* compressor, CompressionOperation* operation); + +/* + Gets the supported compressors, this is defined at compile time and is a convenience method for + the user to know what compressors are supported at runtime. +*/ +VNLIB_EXPORT CompressorType VNLIB_CC GetSupportedCompressors(void) +{ + /* + * Supported compressors are defined at compile time + */ + CompressorType supported; + + supported = COMP_TYPE_NONE; + +#ifdef VNLIB_COMPRESSOR_ZLIB_ENABLED + supported |= COMP_TYPE_GZIP; + supported |= COMP_TYPE_DEFLATE; +#endif + +#ifdef VNLIB_COMPRESSOR_LZ4_ENABLED + supported |= COMP_TYPE_LZ4; +#endif + +#ifdef VNLIB_COMPRESSOR_BROTLI_ENABLED + supported |= COMP_TYPE_BROTLI; +#endif + + return supported; +} + +VNLIB_EXPORT CompressorType VNLIB_CC GetCompressorType(void* compressor) +{ + if (!compressor) + { + return ERR_INVALID_PTR; + } + + return ((CompressorState*)compressor)->type; +} + +VNLIB_EXPORT CompressionLevel VNLIB_CC GetCompressorLevel(void* compressor) +{ + if (!compressor) + { + return ERR_INVALID_PTR; + } + + return ((CompressorState*)compressor)->level; +} + +VNLIB_EXPORT int VNLIB_CC GetCompressorBlockSize(void* compressor) +{ + if (!compressor) + { + return ERR_INVALID_PTR; + } + + return ((CompressorState*)compressor)->blockSize; +} + + + +VNLIB_EXPORT void* VNLIB_CC AllocateCompressor(CompressorType type, CompressionLevel level) +{ + int result; + CompressorState* state; + + /* Validate input arguments */ + if (level < 0 || level > 9) + { + return (void*)ERR_COMP_LEVEL_NOT_SUPPORTED; + } + + state = (CompressorState*)vncalloc(1, sizeof(CompressorState)); + + if (!state) + { + return (void*)ERR_OUT_OF_MEMORY; + } + + /* Configure the comp state */ + state->type = type; + state->level = level; + + result = ERR_COMP_TYPE_NOT_SUPPORTED; + + /* + * Compressor types are defined at compile time + * and callers are allowed to choose which to allocate + */ + + switch (type) + { + +#ifdef VNLIB_COMPRESSOR_BROTLI_ENABLED + + case COMP_TYPE_BROTLI: + result = BrAllocCompressor(state); + break; + +#endif + +#ifdef VNLIB_COMPRESSOR_ZLIB_ENABLED + + case COMP_TYPE_DEFLATE: + case COMP_TYPE_GZIP: + result = DeflateAllocCompressor(state); + break; +#endif + + /* + * Unsupported compressor type allow error to propagate + */ + case COMP_TYPE_NONE: + default: + break; + } + + + + /* + If result was successfull return the context pointer, if + the creation failed, free the state if it was allocated + and return the error code. + */ + + if (result > 0) + { + return (void*)state; + } + else + { + vnfree(state); + + /* + * Using strict/pedantic error checking int gcc will cause a warning + * when casting an int to a void* pointer. We are returning an error code + * and it is expected behavior + */ +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" + return (void*)result; +#pragma GCC diagnostic pop +#elif defined(_MSC_VER) + #pragma warning(push) + #pragma warning(disable: 4312) + return (void*)result; + #pragma warning(pop) +#else + return (void*)result; +#endif + } +} + +VNLIB_EXPORT int VNLIB_CC FreeCompressor(void* compressor) +{ + CompressorState* comp; + int errorCode; + + if (!compressor) + { + return ERR_INVALID_PTR; + } + + comp = (CompressorState*)compressor; + errorCode = TRUE; + + switch (comp->type) + { + +#ifdef VNLIB_COMPRESSOR_BROTLI_ENABLED + + case COMP_TYPE_BROTLI: + BrFreeCompressor(comp); + break; + +#endif + +#ifdef VNLIB_COMPRESSOR_ZLIB_ENABLED + + case COMP_TYPE_DEFLATE: + case COMP_TYPE_GZIP: + + /* + * Releasing a deflate compressor will cause a deflate + * end call, which can fail, we should send the error + * to the caller and clean up as best we can. + */ + + errorCode = DeflateFreeCompressor(comp); + break; + +#endif + /* + * If compression type is none, there is nothing to do + * since its not technically an error, so just return + * true. + */ + case COMP_TYPE_NONE: + case COMP_TYPE_LZ4: + default: + break; + + } + + /* + * Free the compressor state + */ + + vnfree(comp); + return errorCode; +} + +VNLIB_EXPORT int VNLIB_CC GetCompressedSize(void* compressor, int inputLength, int flush) +{ + CompressorState* comp; + int result; + + if (!compressor) + { + return ERR_INVALID_PTR; + } + + comp = (CompressorState*)compressor; + + switch (comp->type) + { + +#ifdef VNLIB_COMPRESSOR_BROTLI_ENABLED + + case COMP_TYPE_BROTLI: + result = BrGetCompressedSize(comp, inputLength); + break; + +#endif + +#ifdef VNLIB_COMPRESSOR_ZLIB_ENABLED + + case COMP_TYPE_DEFLATE: + case COMP_TYPE_GZIP: + result = DeflateGetCompressedSize(comp, inputLength, flush); + break; + +#endif + + /* + * Set the result as an error code, since the compressor + * type is not supported. + */ + case COMP_TYPE_NONE: + case COMP_TYPE_LZ4: + default: + result = ERR_COMP_TYPE_NOT_SUPPORTED; + break; + } + + return result; +} + +/* +* Compresses the data contained in the operation structure, ingests and compresses +* the data then writes it to the output buffer. The result of the operation is +* returned as an error code. Positive integers indicate success, negative integers +* indicate failure. +* @param compressor +*/ +VNLIB_EXPORT int VNLIB_CC CompressBlock(void* compressor, CompressionOperation* operation) +{ + int result; + CompressorState* comp; + + comp = (CompressorState*)compressor; + + /* + * Validate input arguments + */ + + if (!comp) + { + return ERR_INVALID_PTR; + } + + if (!operation) + { + return ERR_INVALID_PTR; + } + + /* + * Validate buffers, if the buffer length is greate than 0 + * it must point to a valid buffer + */ + + if (operation->bytesInLength > 0 && !operation->bytesIn) + { + return ERR_INVALID_INPUT_DATA; + } + + if (operation->bytesOutLength > 0 && !operation->bytesOut) + { + return ERR_INVALID_OUTPUT_DATA; + } + + /* + * Determine the compressor type and call the appropriate + * compression function + */ + + switch (comp->type) + { + + /* Brolti support */ +#ifdef VNLIB_COMPRESSOR_BROTLI_ENABLED + + case COMP_TYPE_BROTLI: + result = BrCompressBlock(comp, operation); + break; +#endif + + /* Deflate support */ +#ifdef VNLIB_COMPRESSOR_ZLIB_ENABLED + + case COMP_TYPE_DEFLATE: + case COMP_TYPE_GZIP: + result = DeflateCompressBlock(comp, operation); + break; + +#endif + + case COMP_TYPE_NONE: + case COMP_TYPE_LZ4: + default: + result = ERR_COMP_TYPE_NOT_SUPPORTED; + break; + } + + return result; +}
\ No newline at end of file diff --git a/lib/Net.Compression/vnlib_compress/compression.h b/lib/Net.Compression/vnlib_compress/compression.h new file mode 100644 index 0000000..153b7fc --- /dev/null +++ b/lib/Net.Compression/vnlib_compress/compression.h @@ -0,0 +1,161 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: VNLib +* Package: vnlib_compress +* File: compression.h +* +* vnlib_compress is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* vnlib_compress is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with vnlib_compress. If not, see http://www.gnu.org/licenses/. +*/ + + +/* +* Implementation notes: +* +* This library is designed to be a wrapper around the various compression libraries +* used for dynamic HTTP compression. Is is designed to be exported as a DLL or a shared +* library and written in portable C code. +* +* Compressors are standalone instances created by callers and used to perform compression +* operations. Compressors are created, used, and destroyed by callers. This library is meant +* to unify compression to a single api. The goal is performance and portability, so it can +* be easily used by portable runtimes. +*/ + +#pragma once + +#ifndef COMPRESSION_H_ +#define COMPRESSION_H_ + +#include "util.h" +#include <stdint.h> +#include <stddef.h> +#include <stdlib.h> + +#define ERR_COMP_TYPE_NOT_SUPPORTED -9 +#define ERR_COMP_LEVEL_NOT_SUPPORTED -10 +#define ERR_INVALID_INPUT_DATA -11 +#define ERR_INVALID_OUTPUT_DATA -12 + +/* +* Enumerated list of supported compression types for user selection +* at runtime. +*/ +typedef enum CompressorType +{ + COMP_TYPE_NONE = 0x00, + COMP_TYPE_GZIP = 0x01, + COMP_TYPE_DEFLATE = 0x02, + COMP_TYPE_BROTLI = 0x04, + COMP_TYPE_LZ4 = 0x08 +} CompressorType; + + +/* + Specifies values that indicate whether a compression operation emphasizes speed + or compression size. +*/ +typedef enum CompressionLevel +{ + /* + The compression operation should be optimally compressed, even if the operation + takes a longer time to complete. + */ + COMP_LEVEL_OPTIMAL = 0, + /* + The compression operation should complete as quickly as possible, even if the + resulting file is not optimally compressed. + */ + COMP_LEVEL_FASTEST = 1, + /* + No compression should be performed on the file. + */ + COMP_LEVEL_NO_COMPRESSION = 2, + /* + The compression operation should create output as small as possible, even if + the operation takes a longer time to complete. + */ + COMP_LEVEL_SMALLEST_SIZE = 3 +} CompressionLevel; + + +typedef enum CompressorStatus { + COMPRESSOR_STATUS_READY = 0x00, + COMPRESSOR_STATUS_INITALIZED = 0x01, + COMPRESSOR_STATUS_NEEDS_FLUSH = 0x02 +} CompressorStatus; + +typedef struct CompressorStateStruct{ + /* + Indicates the type of underlying compressor. + */ + CompressorType type; + + /* + The user specified compression level, the underlying compressor will decide + how to handle this value. + */ + CompressionLevel level; + + /* + Indicates the suggested block size for the underlying compressor. + */ + int blockSize; + + /* + Pointer to the underlying compressor implementation. + */ + void* compressor; + + /* + Counts the number of pending bytes since the last successful flush + operation. + */ + uint32_t pendingBytes; + +} CompressorState; + +/* +* An extern caller generated structure passed to calls for +* stream compression operations. +*/ +typedef struct CompressionOperationStruct { + + /* + * If the operation is a flush operation + */ + const int flush; + + /* + * Input stream data + */ + const uint8_t* bytesIn; + const int bytesInLength; + + /* + * Output buffer/data stream + */ + uint8_t* bytesOut; + const int bytesOutLength; + + /* + * Results of the streaming operation + */ + + int bytesRead; + int bytesWritten; + +} CompressionOperation; + +#endif /* !VNLIB_COMPRESS_MAIN_H_ */
\ No newline at end of file diff --git a/lib/Net.Compression/vnlib_compress/feature_brotli.c b/lib/Net.Compression/vnlib_compress/feature_brotli.c new file mode 100644 index 0000000..a8b6fed --- /dev/null +++ b/lib/Net.Compression/vnlib_compress/feature_brotli.c @@ -0,0 +1,199 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: VNLib +* Package: vnlib_compress +* File: feature_brotli.c +* +* vnlib_compress is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* vnlib_compress is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with vnlib_compress. If not, see http://www.gnu.org/licenses/. +*/ + +#include "feature_brotli.h" +#include <brotli/encode.h> + +#define validateCompState(state) \ + if (!state) return ERR_INVALID_PTR; \ + if (!state->compressor) return ERR_BR_INVALID_STATE; \ + + +int BrAllocCompressor(CompressorState* state) +{ + BrotliEncoderState* comp; + + comp = BrotliEncoderCreateInstance(0, 0, 0); + + if (!comp) + { + return ERR_BR_INVALID_STATE; + } + + state->compressor = comp; + + /* + * Setting parameters will only return false if the parameter type is + * invalid, or the compressor state is not valid + * + * Setup some defaults + */ + + BrotliEncoderSetParameter(comp, BROTLI_PARAM_MODE, BROTLI_MODE_GENERIC); + BrotliEncoderSetParameter(comp, BROTLI_PARAM_LGWIN, BR_DEFAULT_WINDOW); + + + /* + * Capture the block size as a size hint if it is greater than 0 + */ + if (state->blockSize > 0) + { + BrotliEncoderSetParameter(comp, BROTLI_PARAM_SIZE_HINT, state->blockSize); + } + + /* + * Setup compressor quality level based on the requested compression level + */ + + switch (state->level) + { + + case COMP_LEVEL_FASTEST: + BrotliEncoderSetParameter(comp, BROTLI_PARAM_QUALITY, BR_COMP_LEVEL_FASTEST); + break; + + case COMP_LEVEL_OPTIMAL: + BrotliEncoderSetParameter(comp, BROTLI_PARAM_QUALITY, BR_COMP_LEVEL_OPTIMAL); + break; + + case COMP_LEVEL_SMALLEST_SIZE: + BrotliEncoderSetParameter(comp, BROTLI_PARAM_QUALITY, BR_COMP_LEVEL_SMALLEST_SIZE); + break; + + default: + BrotliEncoderSetParameter(comp, BROTLI_PARAM_QUALITY, BR_COMP_LEVEL_DEFAULT); + break; + + } + + return TRUE; +} + +void BrFreeCompressor(CompressorState* state) +{ + /* + * Free the compressor instance if it exists + */ + if (state->compressor) + { + BrotliEncoderDestroyInstance((BrotliEncoderState*)state->compressor); + state->compressor = NULL; + } +} + +int BrCompressBlock(CompressorState* state, CompressionOperation* operation) +{ + BrotliEncoderOperation brOperation; + BROTLI_BOOL brResult; + + size_t availableIn, availableOut, totalOut; + const uint8_t* nextIn; + uint8_t* nextOut; + + /* Validate inputs */ + validateCompState(state) + + /* Clear the result read / written fields */ + operation->bytesRead = 0; + operation->bytesWritten = 0; + + /* + * If the input is empty a flush is not requested, they we are waiting for + * more input and this was just an empty call. Should be a no-op + */ + + if (operation->bytesInLength == 0 && operation->flush < 1) + { + return TRUE; + } + + /* + * Determine the operation to perform + */ + + if (operation->flush) + { + brOperation = BROTLI_OPERATION_FLUSH; + } + else + { + brOperation = BROTLI_OPERATION_PROCESS; + } + + /* + * Update lengths and data pointers from input/output spans + * for stream variables + */ + + availableIn = operation->bytesInLength; + availableOut = operation->bytesOutLength; + nextIn = operation->bytesIn; + nextOut = operation->bytesOut; + + + /* + * Compress block as stream and store the result + * directly on the result output to pass back to the caller + */ + + brResult = BrotliEncoderCompressStream( + state->compressor, + brOperation, + &availableIn, + &nextIn, + &availableOut, + &nextOut, + &totalOut + ); + + /* + * Regardless of the operation success we should return the + * results to the caller. Br encoder sets the number of + * bytes remaining in the input/output spans + */ + + operation->bytesRead = operation->bytesInLength - (int)availableIn; + operation->bytesWritten = operation->bytesOutLength - (int)availableOut; + + return brResult; +} + + +int BrGetCompressedSize(CompressorState* state, int length) +{ + size_t compressedSize; + + /* + * When the flush flag is set, the caller is requesting the + * entire size of the compressed data, which can include metadata + */ + + validateCompState(state) + + if (length <= 0) + { + return 0; + } + + compressedSize = BrotliEncoderMaxCompressedSize(length); + + return (int)compressedSize; +}
\ No newline at end of file diff --git a/lib/Net.Compression/vnlib_compress/feature_brotli.h b/lib/Net.Compression/vnlib_compress/feature_brotli.h new file mode 100644 index 0000000..b5a9ed6 --- /dev/null +++ b/lib/Net.Compression/vnlib_compress/feature_brotli.h @@ -0,0 +1,46 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: VNLib +* Package: vnlib_compress +* File: feature_brotli.h +* +* vnlib_compress is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* vnlib_compress is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with vnlib_compress. If not, see http://www.gnu.org/licenses/. +*/ + +#pragma once + +#ifndef BROTLI_STUB_H_ +#define BROTLI_STUB_H_ + +#include "compression.h" + +#define ERR_BR_INVALID_STATE -24 + +#define BR_COMP_LEVEL_FASTEST 1 +#define BR_COMP_LEVEL_OPTIMAL 11 +#define BR_COMP_LEVEL_SMALLEST_SIZE 9 +#define BR_COMP_LEVEL_DEFAULT 5 + +#define BR_DEFAULT_WINDOW 22 + +int BrAllocCompressor(CompressorState* state); + +void BrFreeCompressor(CompressorState* state); + +int BrCompressBlock(CompressorState* state, CompressionOperation* operation); + +int BrGetCompressedSize(CompressorState* state, int length); + +#endif /* !BROTLI_STUB_H_ */
\ No newline at end of file diff --git a/lib/Net.Compression/vnlib_compress/feature_zlib.c b/lib/Net.Compression/vnlib_compress/feature_zlib.c new file mode 100644 index 0000000..210e212 --- /dev/null +++ b/lib/Net.Compression/vnlib_compress/feature_zlib.c @@ -0,0 +1,268 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: VNLib +* Package: vnlib_compress +* File: feature_zlib.c +* +* vnlib_compress is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* vnlib_compress is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with vnlib_compress. If not, see http://www.gnu.org/licenses/. +*/ + +/* +* Include the stub header and also the zlib header +*/ +#include "feature_zlib.h" +#include <zlib.h> + +#define validateCompState(state) \ + if (!state) return ERR_INVALID_PTR; \ + if (!state->compressor) return ERR_GZ_INVALID_STATE; \ + + +int DeflateAllocCompressor(CompressorState* state) +{ + + int result, compLevel; + z_stream* stream; + + /* + * Allocate the z-stream state on the heap so we can + * store it in the compressor state + */ + stream = (z_stream*)vncalloc(1, sizeof(z_stream)); + + stream->zalloc = Z_NULL; + stream->zfree = Z_NULL; + stream->opaque = Z_NULL; + + /* + * Initialize the z-stream state with the + * desired compression level + */ + + + switch (state->level) + { + case COMP_LEVEL_NO_COMPRESSION: + compLevel = Z_NO_COMPRESSION; + break; + + case COMP_LEVEL_FASTEST: + compLevel = Z_BEST_SPEED; + break; + + case COMP_LEVEL_OPTIMAL: + compLevel = Z_BEST_COMPRESSION; + break; + + case COMP_LEVEL_SMALLEST_SIZE: + compLevel = Z_BEST_COMPRESSION; + break; + + /* + Default compression level + */ + default: + compLevel = Z_DEFAULT_COMPRESSION; + break; + } + + /* + * If gzip is enabled, we need to configure the deflatenit2, with + * the max window size to 16 + */ + + if(state->type & COMP_TYPE_GZIP) + { + result = deflateInit2( + stream, + compLevel, + Z_DEFLATED, + GZ_ENABLE_GZIP_WINDOW, + GZ_DEFAULT_MEM_LEVEL, + Z_DEFAULT_STRATEGY + ); + } + else + { + /* Enable raw deflate */ + result = deflateInit2( + stream, + compLevel, + Z_DEFLATED, + GZ_ENABLE_RAW_DEFLATE_WINDOW, + GZ_DEFAULT_MEM_LEVEL, + Z_DEFAULT_STRATEGY + ); + } + + /* + * Inspect the result of the initialization, + * of the init failed, free the stream and return + * the error code + */ + + if (result != Z_OK) + { + vnfree(stream); + return result; + } + + /* + * Assign the z-stream state to the compressor state, all done! + */ + state->compressor = stream; + return TRUE; +} + +int DeflateFreeCompressor(CompressorState* state) +{ + int result; + + /* + * Free the z-stream state, only if the compressor is initialized + */ + if (state->compressor) + { + /* + * Attempt to end the deflate stream, and store the status code + */ + + result = deflateEnd(state->compressor); + + /* + * We can always free the z-stream state, even if the deflate + * stream failed to end. + */ + + vnfree(state->compressor); + state->compressor = NULL; + + /* + * A data error is acceptable when calling end in this library + * since at that point all resources have been cleaned, and zlib + * is simply returning a warning that the stream was not properly + * terminated. + * + * We assum that calls to free are meant to clean up resources regarless + * of their status + */ + return result == Z_OK || result == Z_DATA_ERROR; + } + + return TRUE; +} + +int DeflateCompressBlock(CompressorState* state, CompressionOperation* operation) +{ + z_stream* stream; + int result; + + validateCompState(state) + + /* Clear the result read/written fields */ + operation->bytesRead = 0; + operation->bytesWritten = 0; + + /* + * If the input is empty a flush is not requested, they we are waiting for + * more input and this was just an empty call. Should be a no-op + */ + + if (operation->bytesInLength == 0 && operation->flush < 1) + { + return TRUE; + } + + stream = (z_stream*)state->compressor; + + /* + * Overwrite the stream state with the operation parameters from + * this next compression operation. + * + * The caller stores the stream positions in its application memory. + */ + stream->avail_in = operation->bytesInLength; + stream->next_in = (Bytef*)operation->bytesIn; + + stream->avail_out = operation->bytesOutLength; + stream->next_out = (Bytef*)operation->bytesOut; + + /* + * In this library we only use the flush flag as a boolean value. + * Callers only set the flush flag when the operation has completed + * and the compressor is expected to flush its internal buffers. + * (aka finish) + */ + + result = deflate(stream, operation->flush ? Z_FINISH : Z_NO_FLUSH); + + /* + * Regardless of the return value, we should always update the + * the number of bytes read and written. + * + * The result is the number total bytes minus the number of + * bytes remaining in the stream. + */ + + operation->bytesRead = operation->bytesInLength - stream->avail_in; + operation->bytesWritten = operation->bytesOutLength - stream->avail_out; + + /*Clear all stream fields after checking results */ + stream->avail_in = 0; + stream->next_in = NULL; + stream->avail_out = 0; + stream->next_out = NULL; + + return result; +} + +int DeflateGetCompressedSize(CompressorState* state, int length, int flush) +{ + uint64_t compressedSize; + + /* + * When the flush flag is set, the caller is requesting the + * entire size of the compressed data, which can include metadata + */ + + validateCompState(state) + + if (length <= 0) + { + return 0; + } + + if(flush) + { + /* + * If the flush flag is set, we need to add the size of the + * pending data in the stream + */ + + compressedSize = deflateBound(state->compressor, length + state->pendingBytes); + } + else + { + compressedSize = deflateBound(state->compressor, length); + } + + /* Verify the results to make sure the value doesnt overflow */ + if (compressedSize > INT32_MAX) + { + return ERR_GZ_OVERFLOW; + } + + return (int)compressedSize; +}
\ No newline at end of file diff --git a/lib/Net.Compression/vnlib_compress/feature_zlib.h b/lib/Net.Compression/vnlib_compress/feature_zlib.h new file mode 100644 index 0000000..0d4b6f6 --- /dev/null +++ b/lib/Net.Compression/vnlib_compress/feature_zlib.h @@ -0,0 +1,50 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: VNLib +* Package: vnlib_compress +* File: feature_zlib.h +* +* vnlib_compress is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* vnlib_compress is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with vnlib_compress. If not, see http://www.gnu.org/licenses/. +*/ + +#pragma once + +#ifndef ZLIB_STUB_H_ +#define ZLIB_STUB_H_ + +#include "compression.h" + +#define ERR_GZ_INVALID_STATE -16 +#define ERR_GZ_OVERFLOW -17 + +/* Allow user to define their own memory level value */ +#ifndef GZ_DEFAULT_MEM_LEVEL +#define GZ_DEFAULT_MEM_LEVEL 8 +#endif + +/* Specifies the window value to enable GZIP */ +#define GZ_ENABLE_GZIP_WINDOW 15 + 16 +#define GZ_ENABLE_RAW_DEFLATE_WINDOW -15 + + +int DeflateAllocCompressor(CompressorState* state); + +int DeflateFreeCompressor(CompressorState* state); + +int DeflateCompressBlock(CompressorState* state, CompressionOperation* operation); + +int DeflateGetCompressedSize(CompressorState* state, int length, int flush); + +#endif diff --git a/lib/Net.Compression/vnlib_compress/util.h b/lib/Net.Compression/vnlib_compress/util.h new file mode 100644 index 0000000..6e7b59e --- /dev/null +++ b/lib/Net.Compression/vnlib_compress/util.h @@ -0,0 +1,65 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: VNLib +* Package: vnlib_compress +* File: util.h +* +* vnlib_compress is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* vnlib_compress is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with vnlib_compress. If not, see http://www.gnu.org/licenses/. +*/ + +#pragma once + +#ifndef UTIL_H_ +#define UTIL_H_ + +/* +* Stub missing types and constants for GCC +*/ +#if defined(__GNUC__) +#define inline __inline__ +#define VNLIB_EXPORT __attribute__((visibility("default"))) +#define VNLIB_CC +#elif defined(_MSC_VER) +#define VNLIB_EXPORT __declspec(dllexport) +#define VNLIB_CC __cdecl +#endif /* WIN32 */ + +#define ERR_INVALID_PTR -1 +#define ERR_OUT_OF_MEMORY -2 + +#define TRUE 1; +#define FALSE 0; + +#ifndef NULL +#define NULL 0 +#endif /* !NULL */ + +/* +* Stub method for malloc. All calls to vnmalloc should be freed with vnfree. +*/ +#define vnmalloc(size) malloc(size) + +/* +* Stub method for free +*/ +#define vnfree(ptr) free(ptr) + +/* +* Stub method for calloc. All calls to vncalloc should be freed with vnfree. +*/ +#define vncalloc(num, size) calloc(num, size) + + +#endif /* !UTIL_H_ */
\ No newline at end of file diff --git a/lib/Net.Compression/vnlib_compress/vnlib_compress.vcxitems b/lib/Net.Compression/vnlib_compress/vnlib_compress.vcxitems new file mode 100644 index 0000000..9249ad9 --- /dev/null +++ b/lib/Net.Compression/vnlib_compress/vnlib_compress.vcxitems @@ -0,0 +1,30 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Label="Globals"> + <MSBuildAllProjects Condition="'$(MSBuildVersion)' == '' Or '$(MSBuildVersion)' < '16.0'">$(MSBuildAllProjects);$(MSBuildThisFileFullPath)</MSBuildAllProjects> + <HasSharedItems>true</HasSharedItems> + <ItemsProjectGuid>{f2e07583-6244-41a4-84a3-e29fd257ee7c}</ItemsProjectGuid> + </PropertyGroup> + <ItemDefinitionGroup> + <ClCompile> + <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(MSBuildThisFileDirectory)</AdditionalIncludeDirectories> + </ClCompile> + </ItemDefinitionGroup> + <ItemGroup> + <ProjectCapability Include="SourceItemsFromImports" /> + </ItemGroup> + <ItemGroup> + <ClCompile Include="$(MSBuildThisFileDirectory)feature_brotli.c" /> + <ClCompile Include="$(MSBuildThisFileDirectory)compression.c" /> + <ClCompile Include="$(MSBuildThisFileDirectory)feature_zlib.c" /> + </ItemGroup> + <ItemGroup> + <ClInclude Include="$(MSBuildThisFileDirectory)feature_brotli.h" /> + <ClInclude Include="$(MSBuildThisFileDirectory)compression.h" /> + <ClInclude Include="$(MSBuildThisFileDirectory)util.h" /> + <ClInclude Include="$(MSBuildThisFileDirectory)feature_zlib.h" /> + </ItemGroup> + <ItemGroup> + <Text Include="$(MSBuildThisFileDirectory)CMakeLists.txt" /> + </ItemGroup> +</Project>
\ No newline at end of file diff --git a/lib/Net.Http/src/Core/Compression/IHttpCompressorManager.cs b/lib/Net.Http/src/Core/Compression/IHttpCompressorManager.cs index 982fa28..80763f8 100644 --- a/lib/Net.Http/src/Core/Compression/IHttpCompressorManager.cs +++ b/lib/Net.Http/src/Core/Compression/IHttpCompressorManager.cs @@ -72,6 +72,13 @@ namespace VNLib.Net.Http ReadOnlyMemory<byte> CompressBlock(object compressorState, ReadOnlyMemory<byte> input, bool finalBlock); /// <summary> + /// Flushes any stored compressor data that still needs to be sent to the client. + /// </summary> + /// <param name="compressorState">The compressor state instance</param> + /// <returns>The remaining data stored in the compressor state, may be empty if no data is pending</returns> + ReadOnlyMemory<byte> Flush(object compressorState); + + /// <summary> /// Initializes the compressor state for a compression operation /// </summary> /// <param name="compressorState">The user-defined compression state</param> diff --git a/lib/Net.Http/src/Core/Compression/IResponseCompressor.cs b/lib/Net.Http/src/Core/Compression/IResponseCompressor.cs index 0beea28..a68a838 100644 --- a/lib/Net.Http/src/Core/Compression/IResponseCompressor.cs +++ b/lib/Net.Http/src/Core/Compression/IResponseCompressor.cs @@ -50,10 +50,16 @@ namespace VNLib.Net.Http.Core.Compression /// </summary> /// <param name="compMethod">The compression mode to use</param> /// <param name="output">The stream to write compressed data to</param> - void Init(Stream output, CompressionMethod compMethod); + void Init(Stream output, CompressionMethod compMethod); /// <summary> - /// Compresses a block of data and writes it to the output stream + /// Gets a value that indicates if the compressor requires more flushing to occur + /// </summary> + bool IsFlushRequired(); + + /// <summary> + /// Compresses a block of data and writes it to the output stream. If an empty flush is + /// commited, then the input buffer will be empty. /// </summary> /// <param name="buffer">The block of memory to write to compress</param> /// <param name="finalBlock">A value that indicates if this block is the final block</param> diff --git a/lib/Net.Http/src/Core/Compression/ManagedHttpCompressor.cs b/lib/Net.Http/src/Core/Compression/ManagedHttpCompressor.cs index e580b2d..b3f971a 100644 --- a/lib/Net.Http/src/Core/Compression/ManagedHttpCompressor.cs +++ b/lib/Net.Http/src/Core/Compression/ManagedHttpCompressor.cs @@ -29,7 +29,6 @@ using System.Threading.Tasks; namespace VNLib.Net.Http.Core.Compression { - internal sealed class ManagedHttpCompressor : IResponseCompressor { //Store the compressor @@ -49,13 +48,31 @@ namespace VNLib.Net.Http.Core.Compression private object? _compressor; private Stream? _stream; + private ReadOnlyMemory<byte> _lastFlush; + private bool initialized; ///<inheritdoc/> public int BlockSize { get; private set; } + public bool IsFlushRequired() + { + //See if a flush is required + _lastFlush = _provider.Flush(_compressor!); + return _lastFlush.Length > 0; + } + ///<inheritdoc/> public ValueTask CompressBlockAsync(ReadOnlyMemory<byte> buffer, bool finalBlock) { + /* + * If input buffer is empty and flush data is available, + * write the last flush data to the stream + */ + if(buffer.Length == 0 && _lastFlush.Length > 0) + { + return _stream!.WriteAsync(_lastFlush); + } + //Compress the block ReadOnlyMemory<byte> result = _provider.CompressBlock(_compressor!, buffer, finalBlock); @@ -68,7 +85,14 @@ namespace VNLib.Net.Http.Core.Compression { //Remove stream ref and de-init the compressor _stream = null; - _provider.DeinitCompressor(_compressor!); + _lastFlush = default; + + //Deinit compressor if initialized + if (initialized) + { + _provider.DeinitCompressor(_compressor!); + initialized = false; + } } ///<inheritdoc/> @@ -76,10 +100,12 @@ namespace VNLib.Net.Http.Core.Compression { //Defer alloc the compressor _compressor ??= _provider.AllocCompressor(); + + //Init the compressor and get the block size + BlockSize = _provider.InitCompressor(_compressor, compMethod); - //Store the stream and init the compressor _stream = output; - BlockSize = _provider.InitCompressor(_compressor, compMethod); + initialized = true; } } }
\ No newline at end of file diff --git a/lib/Net.Http/src/Core/Response/ResponseWriter.cs b/lib/Net.Http/src/Core/Response/ResponseWriter.cs index 7a448a1..77dc619 100644 --- a/lib/Net.Http/src/Core/Response/ResponseWriter.cs +++ b/lib/Net.Http/src/Core/Response/ResponseWriter.cs @@ -91,11 +91,12 @@ namespace VNLib.Net.Http.Core #pragma warning disable CA2007 // Consider calling ConfigureAwait on the awaited task + ReadOnlyMemory<byte> _readSegment; + ///<inheritdoc/> async Task IHttpResponseBody.WriteEntityAsync(Stream dest, long count, Memory<byte> buffer) { int remaining; - ReadOnlyMemory<byte> segment; //Write a sliding window response if (_memoryResponse != null) @@ -107,16 +108,16 @@ namespace VNLib.Net.Http.Core while (remaining > 0) { //Get remaining segment - segment = _memoryResponse.GetRemainingConstrained(remaining); + _readSegment = _memoryResponse.GetRemainingConstrained(remaining); //Write segment to output stream - await dest.WriteAsync(segment); + await dest.WriteAsync(_readSegment); //Advance by the written ammount - _memoryResponse.Advance(segment.Length); + _memoryResponse.Advance(_readSegment.Length); //Update remaining - remaining -= segment.Length; + remaining -= _readSegment.Length; } } else @@ -135,8 +136,6 @@ namespace VNLib.Net.Http.Core ///<inheritdoc/> async Task IHttpResponseBody.WriteEntityAsync(Stream dest, Memory<byte> buffer) { - ReadOnlyMemory<byte> segment; - //Write a sliding window response if (_memoryResponse != null) { @@ -144,13 +143,13 @@ namespace VNLib.Net.Http.Core while (_memoryResponse.Remaining > 0) { //Get remaining segment - segment = _memoryResponse.GetMemory(); + _readSegment = _memoryResponse.GetMemory(); //Write segment to output stream - await dest.WriteAsync(segment); + await dest.WriteAsync(_readSegment); //Advance by the written ammount - _memoryResponse.Advance(segment.Length); + _memoryResponse.Advance(_readSegment.Length); } } else @@ -172,7 +171,6 @@ namespace VNLib.Net.Http.Core //Locals bool remaining; int read; - ReadOnlyMemory<byte> segment; //Write a sliding window response if (_memoryResponse != null) @@ -197,16 +195,16 @@ namespace VNLib.Net.Http.Core //Write response body from memory do { - segment = _memoryResponse.GetRemainingConstrained(dest.BlockSize); + _readSegment = _memoryResponse.GetRemainingConstrained(dest.BlockSize); //Advance by the trimmed segment length - _memoryResponse.Advance(segment.Length); + _memoryResponse.Advance(_readSegment.Length); //Check if data is remaining after an advance remaining = _memoryResponse.Remaining > 0; //Compress the trimmed block - await dest.CompressBlockAsync(segment, !remaining); + await dest.CompressBlockAsync(_readSegment, !remaining); } while (remaining); } @@ -214,16 +212,16 @@ namespace VNLib.Net.Http.Core { do { - segment = _memoryResponse.GetMemory(); + _readSegment = _memoryResponse.GetMemory(); //Advance by the segment length, this should be safe even if its zero - _memoryResponse.Advance(segment.Length); + _memoryResponse.Advance(_readSegment.Length); //Check if data is remaining after an advance remaining = _memoryResponse.Remaining > 0; //Write to output - await dest.CompressBlockAsync(segment, !remaining); + await dest.CompressBlockAsync(_readSegment, !remaining); } while (remaining); } @@ -264,6 +262,13 @@ namespace VNLib.Net.Http.Core //remove ref so its not disposed again _streamResponse = null; } + + //Continue flusing flushing the compressor if required + while(dest.IsFlushRequired()) + { + //Flush the compressor + await dest.CompressBlockAsync(ReadOnlyMemory<byte>.Empty, true); + } } #pragma warning restore CA2007 // Consider calling ConfigureAwait on the awaited task @@ -274,6 +279,7 @@ namespace VNLib.Net.Http.Core //Clear has data flag HasData = false; Length = 0; + _readSegment = default; //Clear rseponse containers _streamResponse?.Dispose(); diff --git a/lib/Net.Http/src/IMemoryResponseEntity.cs b/lib/Net.Http/src/IMemoryResponseReader.cs index aa77f58..a54cec7 100644 --- a/lib/Net.Http/src/IMemoryResponseEntity.cs +++ b/lib/Net.Http/src/IMemoryResponseReader.cs @@ -1,11 +1,11 @@ /* -* Copyright (c) 2022 Vaughn Nugent +* Copyright (c) 2023 Vaughn Nugent * * Library: VNLib * Package: VNLib.Net.Http -* File: IMemoryResponseEntity.cs +* File: IMemoryResponseReader.cs * -* IMemoryResponseEntity.cs is part of VNLib.Net.Http which is part of the larger +* IMemoryResponseReader.cs is part of VNLib.Net.Http which is part of the larger * VNLib collection of libraries and utilities. * * VNLib.Net.Http is free software: you can redistribute it and/or modify diff --git a/lib/Net.Messaging.FBM/src/Helpers.cs b/lib/Net.Messaging.FBM/src/Helpers.cs index cce1d27..4fa00fa 100644 --- a/lib/Net.Messaging.FBM/src/Helpers.cs +++ b/lib/Net.Messaging.FBM/src/Helpers.cs @@ -58,6 +58,11 @@ namespace VNLib.Net.Messaging.FBM public static ReadOnlyMemory<byte> Termination { get; } = new byte[] { 0xFF, 0xF1 }; /// <summary> + /// Allocates a random integer to use as a message id + /// </summary> + public static int RandomMessageId => RandomNumberGenerator.GetInt32(1, int.MaxValue); + + /// <summary> /// Parses the header line for a message-id /// </summary> /// <param name="line">A sequence of bytes that make up a header line</param> @@ -106,13 +111,7 @@ namespace VNLib.Net.Messaging.FBM accumulator.Append(buffer); WriteTermination(accumulator); - } - - - /// <summary> - /// Alloctes a random integer to use as a message id - /// </summary> - public static int RandomMessageId => RandomNumberGenerator.GetInt32(1, int.MaxValue); + } /// <summary> /// Gets the remaining data after the current position of the stream. @@ -120,10 +119,7 @@ namespace VNLib.Net.Messaging.FBM /// <param name="response">The stream to segment</param> /// <returns>The remaining data segment</returns> [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ReadOnlySpan<byte> GetRemainingData(VnMemoryStream response) - { - return response.AsSpan()[(int)response.Position..]; - } + public static ReadOnlySpan<byte> GetRemainingData(VnMemoryStream response) => response.AsSpan()[(int)response.Position..]; /// <summary> /// Reads the next available line from the response message @@ -212,10 +208,7 @@ namespace VNLib.Net.Messaging.FBM /// <param name="line"></param> /// <returns>The <see cref="HeaderCommand"/> enum value from hte first byte of the message</returns> [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static HeaderCommand GetHeaderCommand(ReadOnlySpan<byte> line) - { - return (HeaderCommand)line[0]; - } + public static HeaderCommand GetHeaderCommand(ReadOnlySpan<byte> line) => (HeaderCommand)line[0]; /// <summary> /// Gets the value of the header following the colon bytes in the specifed @@ -239,8 +232,7 @@ namespace VNLib.Net.Messaging.FBM //Decode the characters and return the char count _ = encoding.GetChars(value, output); return charCount; - } - + } /// <summary> /// Ends the header section of the request and appends the message body to @@ -257,7 +249,6 @@ namespace VNLib.Net.Messaging.FBM buffer.Append(body); } - /// <summary> /// Rounds the requested byte size up to the 1kb /// number of bytes @@ -274,7 +265,6 @@ namespace VNLib.Net.Messaging.FBM return kbs * 1024; } - /// <summary> /// Writes a line termination to the message buffer /// </summary> @@ -292,6 +282,11 @@ namespace VNLib.Net.Messaging.FBM /// <exception cref="ArgumentException"></exception> public static void WriteHeader(IDataAccumulator<byte> buffer, byte header, ReadOnlySpan<char> value, Encoding encoding) { + if(header == 0) + { + throw new ArgumentException("A header command of 0 is illegal", nameof(header)); + } + //Write header command enum value buffer.Append(header); //Convert the characters to binary and write to the buffer diff --git a/lib/Net.Messaging.FBM/src/Server/FBMListener.cs b/lib/Net.Messaging.FBM/src/Server/FBMListener.cs index 3417abc..fd8b025 100644 --- a/lib/Net.Messaging.FBM/src/Server/FBMListener.cs +++ b/lib/Net.Messaging.FBM/src/Server/FBMListener.cs @@ -54,110 +54,7 @@ namespace VNLib.Net.Messaging.FBM.Server /// and raises events on requests. /// </summary> public class FBMListener - { - private sealed class ListeningSession - { - private readonly ReusableStore<FBMContext> CtxStore; - private readonly CancellationTokenSource Cancellation; - private readonly CancellationTokenRegistration Registration; - private readonly FBMListenerSessionParams Params; - - - public readonly object? UserState; - - public readonly SemaphoreSlim ResponseLock; - - public readonly WebSocketSession Socket; - - public readonly RequestHandler OnRecieved; - - public CancellationToken CancellationToken => Cancellation.Token; - - - public ListeningSession(WebSocketSession session, RequestHandler onRecieved, in FBMListenerSessionParams args, object? userState) - { - Params = args; - Socket = session; - UserState = userState; - OnRecieved = onRecieved; - - //Create cancellation and register for session close - Cancellation = new(); - Registration = session.Token.Register(Cancellation.Cancel); - - ResponseLock = new(1); - CtxStore = ObjectRental.CreateReusable(ContextCtor); - } - - private FBMContext ContextCtor() => new(Params.MaxHeaderBufferSize, Params.ResponseBufferSize, Params.HeaderEncoding); - - /// <summary> - /// Cancels any pending opreations relating to the current session - /// </summary> - public void CancelSession() - { - Cancellation.Cancel(); - - //If dispose happens without any outstanding requests, we can dispose the session - if (_counter == 0) - { - CleanupInternal(); - } - } - - private void CleanupInternal() - { - Registration.Dispose(); - CtxStore.Dispose(); - Cancellation.Dispose(); - ResponseLock.Dispose(); - } - - - private uint _counter; - - /// <summary> - /// Rents a new <see cref="FBMContext"/> instance from the pool - /// and increments the counter - /// </summary> - /// <returns>The rented instance</returns> - /// <exception cref="ObjectDisposedException"></exception> - public FBMContext RentContext() - { - - if (Cancellation.IsCancellationRequested) - { - throw new ObjectDisposedException("The instance has been disposed"); - } - - //Rent context - FBMContext ctx = CtxStore.Rent(); - //Increment counter - Interlocked.Increment(ref _counter); - - return ctx; - } - - /// <summary> - /// Returns a previously rented context to the pool - /// and decrements the counter. If the session has been - /// cancelled, when the counter reaches 0, cleanup occurs - /// </summary> - /// <param name="ctx">The context to return</param> - public void ReturnContext(FBMContext ctx) - { - //Return the context - CtxStore.Return(ctx); - - uint current = Interlocked.Decrement(ref _counter); - - //No more contexts in use, dispose internals - if (Cancellation.IsCancellationRequested && current == 0) - { - CleanupInternal(); - } - } - } + { public const int SEND_SEMAPHORE_TIMEOUT_MS = 10 * 1000; @@ -189,7 +86,8 @@ namespace VNLib.Net.Messaging.FBM.Server /// <returns>A <see cref="Task"/> that completes when the connection closes</returns> public async Task ListenAsync(WebSocketSession wss, RequestHandler handler, FBMListenerSessionParams args, object? userState) { - ListeningSession session = new(wss, handler, args, userState); + ListeningSession session = new(wss, handler, in args, userState); + //Alloc a recieve buffer using IMemoryOwner<byte> recvBuffer = Heap.DirectAlloc<byte>(args.RecvBufferSize); @@ -385,5 +283,109 @@ namespace VNLib.Net.Messaging.FBM.Server { return Task.CompletedTask; } + + private sealed class ListeningSession + { + private readonly ReusableStore<FBMContext> CtxStore; + private readonly CancellationTokenSource Cancellation; + private readonly CancellationTokenRegistration Registration; + private readonly FBMListenerSessionParams Params; + + + public readonly object? UserState; + + public readonly SemaphoreSlim ResponseLock; + + public readonly WebSocketSession Socket; + + public readonly RequestHandler OnRecieved; + + public CancellationToken CancellationToken => Cancellation.Token; + + + public ListeningSession(WebSocketSession session, RequestHandler onRecieved, in FBMListenerSessionParams args, object? userState) + { + Params = args; + Socket = session; + UserState = userState; + OnRecieved = onRecieved; + + //Create cancellation and register for session close + Cancellation = new(); + Registration = session.Token.Register(Cancellation.Cancel); + + ResponseLock = new(1); + CtxStore = ObjectRental.CreateReusable(ContextCtor); + } + + private FBMContext ContextCtor() => new(Params.MaxHeaderBufferSize, Params.ResponseBufferSize, Params.HeaderEncoding); + + /// <summary> + /// Cancels any pending opreations relating to the current session + /// </summary> + public void CancelSession() + { + Cancellation.Cancel(); + + //If dispose happens without any outstanding requests, we can dispose the session + if (_counter == 0) + { + CleanupInternal(); + } + } + + private void CleanupInternal() + { + Registration.Dispose(); + CtxStore.Dispose(); + Cancellation.Dispose(); + ResponseLock.Dispose(); + } + + + private uint _counter; + + /// <summary> + /// Rents a new <see cref="FBMContext"/> instance from the pool + /// and increments the counter + /// </summary> + /// <returns>The rented instance</returns> + /// <exception cref="ObjectDisposedException"></exception> + public FBMContext RentContext() + { + + if (Cancellation.IsCancellationRequested) + { + throw new ObjectDisposedException("The instance has been disposed"); + } + + //Rent context + FBMContext ctx = CtxStore.Rent(); + //Increment counter + Interlocked.Increment(ref _counter); + + return ctx; + } + + /// <summary> + /// Returns a previously rented context to the pool + /// and decrements the counter. If the session has been + /// cancelled, when the counter reaches 0, cleanup occurs + /// </summary> + /// <param name="ctx">The context to return</param> + public void ReturnContext(FBMContext ctx) + { + //Return the context + CtxStore.Return(ctx); + + uint current = Interlocked.Decrement(ref _counter); + + //No more contexts in use, dispose internals + if (Cancellation.IsCancellationRequested && current == 0) + { + CleanupInternal(); + } + } + } } } diff --git a/lib/Plugins/src/Attributes/ServiceConfiguratorAttribute.cs b/lib/Plugins/src/Attributes/ServiceConfiguratorAttribute.cs index e922e9d..123fb66 100644 --- a/lib/Plugins/src/Attributes/ServiceConfiguratorAttribute.cs +++ b/lib/Plugins/src/Attributes/ServiceConfiguratorAttribute.cs @@ -29,8 +29,8 @@ namespace VNLib.Plugins.Attributes { /// <summary> /// <para> - /// Set this attribute on an <see cref="IPlugin"/> instance method to define the service configuration - /// method. When declated, allows the plugin to expose shared types to the host + /// Declare this attribute on an <see cref="IPlugin"/> instance method to define the service configuration + /// method. When declared, allows the plugin to expose shared types to the host /// </para> /// <para> /// This method may be runtime dependant, it may not be called on all platforms, and it @@ -47,7 +47,7 @@ namespace VNLib.Plugins.Attributes /// </summary> [AttributeUsage(AttributeTargets.Method, AllowMultiple = false, Inherited = true)] public sealed class ServiceConfiguratorAttribute : Attribute - {} + { } /// <summary> /// A safe delegate that matches the signature of the <see cref="ServiceConfiguratorAttribute"/> |