aboutsummaryrefslogtreecommitdiff
path: root/lib/Utils
diff options
context:
space:
mode:
Diffstat (limited to 'lib/Utils')
-rw-r--r--lib/Utils/src/Extensions/MemoryExtensions.cs21
-rw-r--r--lib/Utils/src/IO/VnMemoryStream.cs38
-rw-r--r--lib/Utils/src/Memory/Caching/ObjectRentalBase.cs70
-rw-r--r--lib/Utils/src/Memory/Caching/ReusableStore.cs61
-rw-r--r--lib/Utils/src/Memory/Caching/ThreadLocalReusableStore.cs64
-rw-r--r--lib/Utils/src/Memory/Diagnostics/TrackedHeapWrapper.cs14
-rw-r--r--lib/Utils/src/Memory/MemoryUtil.cs35
-rw-r--r--lib/Utils/src/VnEncoding.cs97
-rw-r--r--lib/Utils/tests/Async/AsyncAccessSerializerTests.cs27
-rw-r--r--lib/Utils/tests/IO/VnMemoryStreamTests.cs99
-rw-r--r--lib/Utils/tests/Memory/MemoryUtilTests.cs2
11 files changed, 342 insertions, 186 deletions
diff --git a/lib/Utils/src/Extensions/MemoryExtensions.cs b/lib/Utils/src/Extensions/MemoryExtensions.cs
index 32bb3d4..6525db4 100644
--- a/lib/Utils/src/Extensions/MemoryExtensions.cs
+++ b/lib/Utils/src/Extensions/MemoryExtensions.cs
@@ -438,6 +438,27 @@ namespace VNLib.Utils.Extensions
}
/// <summary>
+ /// Allocates a buffer from the current heap and initialzies it by copying the initial data buffer
+ /// </summary>
+ /// <typeparam name="T"></typeparam>
+ /// <param name="heap"></param>
+ /// <param name="initialData">The initial data to set the buffer to</param>
+ /// <returns>The initalized <see cref="MemoryHandle{T}"/> block</returns>
+ /// <exception cref="OutOfMemoryException"></exception>
+ /// <exception cref="ObjectDisposedException"></exception>
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static MemoryHandle<T> AllocAndCopy<T>(this IUnmangedHeap heap, ReadOnlyMemory<T> initialData) where T : unmanaged
+ {
+ //Aloc block
+ MemoryHandle<T> handle = heap.Alloc<T>(initialData.Length);
+
+ //Copy initial data
+ MemoryUtil.Copy(initialData, handle, 0);
+
+ return handle;
+ }
+
+ /// <summary>
/// Copies data from the input buffer to the current handle and resizes the handle to the
/// size of the buffer
/// </summary>
diff --git a/lib/Utils/src/IO/VnMemoryStream.cs b/lib/Utils/src/IO/VnMemoryStream.cs
index d97036d..eed4ca2 100644
--- a/lib/Utils/src/IO/VnMemoryStream.cs
+++ b/lib/Utils/src/IO/VnMemoryStream.cs
@@ -94,7 +94,7 @@ namespace VNLib.Utils.IO
/// buffer of the specified size on the specified heap to avoid resizing.
/// </summary>
/// <param name="heap"><see cref="Win32PrivateHeap"/> to allocate memory from</param>
- /// <param name="bufferSize">Number of bytes (length) of the stream if known</param>
+ /// <param name="bufferSize">The initial internal buffer size, does not effect the length/size of the stream, helps pre-alloc</param>
/// <param name="zero">Zero memory allocations during buffer expansions</param>
/// <exception cref="OutOfMemoryException"></exception>
/// <exception cref="ArgumentNullException"></exception>
@@ -119,7 +119,22 @@ namespace VNLib.Utils.IO
_length = data.Length;
_position = 0;
}
-
+
+ /// <summary>
+ /// Creates a new memory stream from the data provided
+ /// </summary>
+ /// <param name="heap"><see cref="Win32PrivateHeap"/> to allocate memory from</param>
+ /// <param name="data">Initial data</param>
+ public VnMemoryStream(IUnmangedHeap heap, ReadOnlyMemory<byte> data)
+ {
+ _ = heap ?? throw new ArgumentNullException(nameof(heap));
+ //Alloc the internal buffer to match the data stream
+ _buffer = heap.AllocAndCopy(data);
+ //Set length
+ _length = data.Length;
+ _position = 0;
+ }
+
/// <summary>
/// WARNING: Dangerous constructor, make sure read-only and owns hanlde are set accordingly
/// </summary>
@@ -302,6 +317,23 @@ namespace VNLib.Utils.IO
return bytesToRead;
}
+ ///<inheritdoc/>
+ public override unsafe int ReadByte()
+ {
+ if (LenToPosDiff == 0)
+ {
+ return -1;
+ }
+
+ //get the value at the current position
+ byte* ptr = _buffer.GetOffset(_position);
+
+ //Increment position
+ _position++;
+
+ //Return value
+ return ptr[0];
+ }
/*
* Async reading will always run synchronously in a memory stream,
@@ -469,7 +501,7 @@ namespace VNLib.Utils.IO
/// If the current stream is a readonly stream, creates a shallow copy for reading only.
/// </summary>
/// <returns>New stream shallow copy of the internal stream</returns>
- /// <exception cref="InvalidOperationException"></exception>
+ /// <exception cref="NotSupportedException"></exception>
public object Clone() => GetReadonlyShallowCopy();
/*
diff --git a/lib/Utils/src/Memory/Caching/ObjectRentalBase.cs b/lib/Utils/src/Memory/Caching/ObjectRentalBase.cs
index 305d93f..ca07885 100644
--- a/lib/Utils/src/Memory/Caching/ObjectRentalBase.cs
+++ b/lib/Utils/src/Memory/Caching/ObjectRentalBase.cs
@@ -39,8 +39,9 @@ namespace VNLib.Utils.Memory.Caching
public static ObjectRental<TNew> Create<TNew>(int quota = 0) where TNew : class, new()
{
static TNew constructor() => new();
- return new ObjectRental<TNew>(constructor, null, null, quota);
+ return Create(constructor, null, null, quota);
}
+
/// <summary>
/// Creates a new <see cref="ObjectRental{T}"/> store with generic rental and return callback handlers
/// </summary>
@@ -50,18 +51,17 @@ namespace VNLib.Utils.Memory.Caching
public static ObjectRental<TNew> Create<TNew>(Action<TNew>? rentCb, Action<TNew>? returnCb, int quota = 0) where TNew : class, new()
{
static TNew constructor() => new();
- return new ObjectRental<TNew>(constructor, rentCb, returnCb, quota);
+ return Create(constructor, rentCb, returnCb, quota);
}
+
/// <summary>
/// Creates a new <see cref="ObjectRental{T}"/> store with a generic constructor function
/// </summary>
/// <param name="constructor">The function invoked to create a new instance when required</param>
/// <param name="quota">The maximum number of elements that will be cached</param>
/// <returns></returns>
- public static ObjectRental<TNew> Create<TNew>(Func<TNew> constructor, int quota = 0) where TNew: class
- {
- return new ObjectRental<TNew>(constructor, null, null, quota);
- }
+ public static ObjectRental<TNew> Create<TNew>(Func<TNew> constructor, int quota = 0) where TNew : class => Create(constructor, null, null, quota);
+
/// <summary>
/// Creates a new <see cref="ObjectRental{T}"/> store with generic rental and return callback handlers
/// </summary>
@@ -69,10 +69,8 @@ namespace VNLib.Utils.Memory.Caching
/// <param name="rentCb">Function responsible for preparing an instance to be rented</param>
/// <param name="returnCb">Function responsible for cleaning up an instance before reuse</param>
/// <param name="quota">The maximum number of elements that will be cached</param>
- public static ObjectRental<TNew> Create<TNew>(Func<TNew> constructor, Action<TNew>? rentCb, Action<TNew>? returnCb, int quota = 0) where TNew : class
- {
- return new ObjectRental<TNew>(constructor, rentCb, returnCb, quota);
- }
+ public static ObjectRental<TNew> Create<TNew>(Func<TNew> constructor, Action<TNew>? rentCb, Action<TNew>? returnCb, int quota = 0) where TNew : class
+ => new(constructor, rentCb, returnCb, quota);
/// <summary>
/// Creates a new <see cref="ThreadLocalObjectStorage{TNew}"/> store with generic rental and return callback handlers
@@ -82,10 +80,9 @@ namespace VNLib.Utils.Memory.Caching
/// <param name="rentCb">Function responsible for preparing an instance to be rented</param>
/// <param name="returnCb">Function responsible for cleaning up an instance before reuse</param>
/// <returns>The initialized store</returns>
- public static ThreadLocalObjectStorage<TNew> CreateThreadLocal<TNew>(Func<TNew> constructor, Action<TNew>? rentCb, Action<TNew>? returnCb) where TNew : class
- {
- return new ThreadLocalObjectStorage<TNew>(constructor, rentCb, returnCb);
- }
+ public static ThreadLocalObjectStorage<TNew> CreateThreadLocal<TNew>(Func<TNew> constructor, Action<TNew>? rentCb, Action<TNew>? returnCb) where TNew : class
+ => new (constructor, rentCb, returnCb);
+
/// <summary>
/// Creates a new <see cref="ThreadLocalObjectStorage{T}"/> store with generic rental and return callback handlers
/// </summary>
@@ -94,62 +91,75 @@ namespace VNLib.Utils.Memory.Caching
public static ThreadLocalObjectStorage<TNew> CreateThreadLocal<TNew>(Action<TNew>? rentCb, Action<TNew>? returnCb) where TNew : class, new()
{
static TNew constructor() => new();
- return new ThreadLocalObjectStorage<TNew>(constructor, rentCb, returnCb);
+ return CreateThreadLocal(constructor, rentCb, returnCb);
}
+
/// <summary>
/// Creates a new <see cref="ThreadLocalObjectStorage{T}"/> store
/// </summary>
public static ThreadLocalObjectStorage<TNew> CreateThreadLocal<TNew>() where TNew : class, new()
{
static TNew constructor() => new();
- return new ThreadLocalObjectStorage<TNew>(constructor, null, null);
+ return CreateThreadLocal(constructor, null, null);
}
+
/// <summary>
/// Creates a new <see cref="ThreadLocalObjectStorage{T}"/> store with a generic constructor function
/// </summary>
/// <param name="constructor">The function invoked to create a new instance when required</param>
/// <returns></returns>
- public static ThreadLocalObjectStorage<TNew> CreateThreadLocal<TNew>(Func<TNew> constructor) where TNew : class
- {
- return new ThreadLocalObjectStorage<TNew>(constructor, null, null);
- }
+ public static ThreadLocalObjectStorage<TNew> CreateThreadLocal<TNew>(Func<TNew> constructor) where TNew : class => CreateThreadLocal(constructor, null, null);
/// <summary>
- /// Creates a new <see cref="ReusableStore{T}"/> instance with a parameterless constructor
+ /// Creates a new <see cref="ObjectRental{T}"/> instance with a parameterless constructor
/// </summary>
/// <typeparam name="T">The <see cref="IReusable"/> type</typeparam>
/// <param name="quota">The maximum number of elements that will be cached</param>
/// <returns></returns>
- public static ReusableStore<T> CreateReusable<T>(int quota = 0) where T : class, IReusable, new()
+ public static ObjectRental<T> CreateReusable<T>(int quota = 0) where T : class, IReusable, new()
{
static T constructor() => new();
- return new(constructor, quota);
+ return CreateReusable(constructor, quota);
}
+
/// <summary>
- /// Creates a new <see cref="ReusableStore{T}"/> instance with the specified constructor
+ /// Creates a new <see cref="ObjectRental{T}"/> instance with the specified constructor
/// </summary>
/// <typeparam name="T">The <see cref="IReusable"/> type</typeparam>
/// <param name="constructor">The constructor function invoked to create new instances of the <see cref="IReusable"/> type</param>
/// <param name="quota">The maximum number of elements that will be cached</param>
/// <returns></returns>
- public static ReusableStore<T> CreateReusable<T>(Func<T> constructor, int quota = 0) where T : class, IReusable => new(constructor, quota);
+ public static ObjectRental<T> CreateReusable<T>(Func<T> constructor, int quota = 0) where T : class, IReusable
+ {
+ //Rent/return callbacks
+ static void Rent(T item) => item.Prepare();
+ static void Return(T item) => item.Release();
+
+ return Create(constructor, Rent, Return, quota);
+ }
/// <summary>
- /// Creates a new <see cref="ThreadLocalReusableStore{T}"/> instance with a parameterless constructor
+ /// Creates a new <see cref="ThreadLocalObjectStorage{T}"/> instance with a parameterless constructor
/// </summary>
/// <typeparam name="T">The <see cref="IReusable"/> type</typeparam>
/// <returns></returns>
- public static ThreadLocalReusableStore<T> CreateThreadLocalReusable<T>() where T : class, IReusable, new()
+ public static ThreadLocalObjectStorage<T> CreateThreadLocalReusable<T>() where T : class, IReusable, new()
{
static T constructor() => new();
- return new(constructor);
+ return CreateThreadLocalReusable(constructor);
}
+
/// <summary>
- /// Creates a new <see cref="ThreadLocalReusableStore{T}"/> instance with the specified constructor
+ /// Creates a new <see cref="ThreadLocalObjectStorage{T}"/> instance with the specified constructor
/// </summary>
/// <typeparam name="T">The <see cref="IReusable"/> type</typeparam>
/// <param name="constructor">The constructor function invoked to create new instances of the <see cref="IReusable"/> type</param>
/// <returns></returns>
- public static ThreadLocalReusableStore<T> CreateThreadLocalReusable<T>(Func<T> constructor) where T : class, IReusable => new(constructor);
+ public static ThreadLocalObjectStorage<T> CreateThreadLocalReusable<T>(Func<T> constructor) where T : class, IReusable
+ {
+ static void Rent(T item) => item.Prepare();
+ static void Return(T item) => item.Release();
+ return new ThreadLocalObjectStorage<T>(constructor, Rent, Return);
+ }
}
}
diff --git a/lib/Utils/src/Memory/Caching/ReusableStore.cs b/lib/Utils/src/Memory/Caching/ReusableStore.cs
deleted file mode 100644
index aacd012..0000000
--- a/lib/Utils/src/Memory/Caching/ReusableStore.cs
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
-* Copyright (c) 2022 Vaughn Nugent
-*
-* Library: VNLib
-* Package: VNLib.Utils
-* File: ReusableStore.cs
-*
-* ReusableStore.cs is part of VNLib.Utils which is part of the larger
-* VNLib collection of libraries and utilities.
-*
-* VNLib.Utils is free software: you can redistribute it and/or modify
-* it under the terms of the GNU General Public License as published
-* by the Free Software Foundation, either version 2 of the License,
-* or (at your option) any later version.
-*
-* VNLib.Utils is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-* General Public License for more details.
-*
-* You should have received a copy of the GNU General Public License
-* along with VNLib.Utils. If not, see http://www.gnu.org/licenses/.
-*/
-
-using System;
-
-namespace VNLib.Utils.Memory.Caching
-{
-
- /// <summary>
- /// A reusable object store that extends <see cref="ObjectRental{T}"/>, that allows for objects to be reused heavily
- /// </summary>
- /// <typeparam name="T">A reusable object</typeparam>
- public class ReusableStore<T> : ObjectRental<T> where T : class, IReusable
- {
- internal ReusableStore(Func<T> constructor, int quota) :base(constructor, null, null, quota)
- {}
- ///<inheritdoc/>
- public override T Rent()
- {
- //Rent the object (or create it)
- T rental = base.Rent();
- //Invoke prepare function
- rental.Prepare();
- //return object
- return rental;
- }
- ///<inheritdoc/>
- public override void Return(T item)
- {
- /*
- * Clean up the item by invoking the cleanup function,
- * and only return the item for reuse if the caller allows
- */
- if (item.Release())
- {
- base.Return(item);
- }
- }
- }
-} \ No newline at end of file
diff --git a/lib/Utils/src/Memory/Caching/ThreadLocalReusableStore.cs b/lib/Utils/src/Memory/Caching/ThreadLocalReusableStore.cs
deleted file mode 100644
index 83cd4d6..0000000
--- a/lib/Utils/src/Memory/Caching/ThreadLocalReusableStore.cs
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
-* Copyright (c) 2022 Vaughn Nugent
-*
-* Library: VNLib
-* Package: VNLib.Utils
-* File: ThreadLocalReusableStore.cs
-*
-* ThreadLocalReusableStore.cs is part of VNLib.Utils which is part of the larger
-* VNLib collection of libraries and utilities.
-*
-* VNLib.Utils is free software: you can redistribute it and/or modify
-* it under the terms of the GNU General Public License as published
-* by the Free Software Foundation, either version 2 of the License,
-* or (at your option) any later version.
-*
-* VNLib.Utils is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-* General Public License for more details.
-*
-* You should have received a copy of the GNU General Public License
-* along with VNLib.Utils. If not, see http://www.gnu.org/licenses/.
-*/
-
-using System;
-
-namespace VNLib.Utils.Memory.Caching
-{
- /// <summary>
- /// A reusable object store that extends <see cref="ThreadLocalObjectStorage{T}"/>, that allows for objects to be reused heavily
- /// in a thread-local cache
- /// </summary>
- /// <typeparam name="T">A reusable object</typeparam>
- public class ThreadLocalReusableStore<T> : ThreadLocalObjectStorage<T> where T: class, IReusable
- {
- /// <summary>
- /// Creates a new <see cref="ThreadLocalReusableStore{T}"/> instance
- /// </summary>
- internal ThreadLocalReusableStore(Func<T> constructor):base(constructor, null, null)
- { }
- ///<inheritdoc/>
- public override T Rent()
- {
- //Rent the object (or create it)
- T rental = base.Rent();
- //Invoke prepare function
- rental.Prepare();
- //return object
- return rental;
- }
- ///<inheritdoc/>
- public override void Return(T item)
- {
- /*
- * Clean up the item by invoking the cleanup function,
- * and only return the item for reuse if the caller allows
- */
- if (item.Release())
- {
- base.Return(item);
- }
- }
- }
-} \ No newline at end of file
diff --git a/lib/Utils/src/Memory/Diagnostics/TrackedHeapWrapper.cs b/lib/Utils/src/Memory/Diagnostics/TrackedHeapWrapper.cs
index 41b08c1..820c819 100644
--- a/lib/Utils/src/Memory/Diagnostics/TrackedHeapWrapper.cs
+++ b/lib/Utils/src/Memory/Diagnostics/TrackedHeapWrapper.cs
@@ -34,6 +34,7 @@ namespace VNLib.Utils.Memory.Diagnostics
public class TrackedHeapWrapper : VnDisposeable, IUnmangedHeap
{
private readonly IUnmangedHeap _heap;
+ private readonly bool _ownsHeap;
private readonly object _statsLock;
private readonly ConcurrentDictionary<IntPtr, ulong> _table;
@@ -62,13 +63,15 @@ namespace VNLib.Utils.Memory.Diagnostics
/// Creates a new diagnostics wrapper for the heap
/// </summary>
/// <param name="heap">The heap to gather statistics on</param>
- public TrackedHeapWrapper(IUnmangedHeap heap)
+ /// <param name="ownsHeap">If true, the wrapper will dispose the heap when disposed</param>
+ public TrackedHeapWrapper(IUnmangedHeap heap, bool ownsHeap)
{
_statsLock = new();
_table = new();
_heap = heap;
- //Default min block size to 0
+ //Default min block size to max
_minBlockSize = ulong.MaxValue;
+ _ownsHeap = ownsHeap;
}
/// <summary>
@@ -124,8 +127,11 @@ namespace VNLib.Utils.Memory.Diagnostics
///<inheritdoc/>
protected override void Free()
{
- Heap.Dispose();
- }
+ if(_ownsHeap)
+ {
+ _heap.Dispose();
+ }
+ }
///<inheritdoc/>
public bool Free(ref IntPtr block)
diff --git a/lib/Utils/src/Memory/MemoryUtil.cs b/lib/Utils/src/Memory/MemoryUtil.cs
index f671da0..dc69e76 100644
--- a/lib/Utils/src/Memory/MemoryUtil.cs
+++ b/lib/Utils/src/Memory/MemoryUtil.cs
@@ -68,6 +68,11 @@ namespace VNLib.Utils.Memory
public const string SHARED_HEAP_RAW_FLAGS = "VNLIB_SHARED_HEAP_RAW_FLAGS";
/// <summary>
+ /// The environment variable name used to specify the shared heap type
+ /// </summary>
+ public const string SHARED_HEAP_GLOBAL_ZERO = "VNLIB_SHARED_HEAP_GLOBAL_ZERO";
+
+ /// <summary>
/// Initial shared heap size (bytes)
/// </summary>
public const nuint SHARED_HEAP_INIT_SIZE = 20971520;
@@ -102,10 +107,12 @@ namespace VNLib.Utils.Memory
{
//Get env for heap diag
_ = ERRNO.TryParse(Environment.GetEnvironmentVariable(SHARED_HEAP_ENABLE_DIAGNOISTICS_ENV), out ERRNO diagEnable);
+ _ = ERRNO.TryParse(Environment.GetEnvironmentVariable(SHARED_HEAP_GLOBAL_ZERO), out ERRNO globalZero);
Trace.WriteIf(diagEnable, "Shared heap diagnostics enabled");
+ Trace.WriteIf(globalZero, "Shared heap global zero enabled");
- Lazy<IUnmangedHeap> heap = new (() => InitHeapInternal(true, diagEnable), LazyThreadSafetyMode.PublicationOnly);
+ Lazy<IUnmangedHeap> heap = new (() => InitHeapInternal(true, diagEnable, globalZero), LazyThreadSafetyMode.PublicationOnly);
//Cleanup the heap on process exit
AppDomain.CurrentDomain.DomainUnload += DomainUnloaded;
@@ -143,12 +150,13 @@ namespace VNLib.Utils.Memory
/// Initializes a new <see cref="IUnmangedHeap"/> determined by compilation/runtime flags
/// and operating system type for the current proccess.
/// </summary>
+ /// <param name="globalZero">If true, sets the <see cref="HeapCreation.GlobalZero"/> flag</param>
/// <returns>An <see cref="IUnmangedHeap"/> for the current process</returns>
/// <exception cref="SystemException"></exception>
/// <exception cref="DllNotFoundException"></exception>
- public static IUnmangedHeap InitializeNewHeapForProcess() => InitHeapInternal(false, false);
+ public static IUnmangedHeap InitializeNewHeapForProcess(bool globalZero = false) => InitHeapInternal(false, false, globalZero);
- private static IUnmangedHeap InitHeapInternal(bool isShared, bool enableStats)
+ private static IUnmangedHeap InitHeapInternal(bool isShared, bool enableStats, bool globalZero)
{
bool IsWindows = OperatingSystem.IsWindows();
@@ -167,6 +175,9 @@ namespace VNLib.Utils.Memory
*/
cFlags |= isShared ? HeapCreation.IsSharedHeap : HeapCreation.None;
+ //Set global zero flag if requested
+ cFlags |= globalZero ? HeapCreation.GlobalZero : HeapCreation.None;
+
IUnmangedHeap heap;
ERRNO userFlags = 0;
@@ -207,7 +218,7 @@ namespace VNLib.Utils.Memory
}
//Enable heap statistics
- return enableStats ? new TrackedHeapWrapper(heap) : heap;
+ return enableStats ? new TrackedHeapWrapper(heap, true) : heap;
}
/// <summary>
@@ -230,7 +241,7 @@ namespace VNLib.Utils.Memory
{
return;
}
-
+
uint byteSize = ByteCount<T>((uint)block.Length);
fixed (void* ptr = &MemoryMarshal.GetReference(block))
@@ -305,6 +316,14 @@ namespace VNLib.Utils.Memory
}
/// <summary>
+ /// Zeroes a block of memory of the given unmanaged type
+ /// </summary>
+ /// <typeparam name="T">The unmanaged type to zero</typeparam>
+ /// <param name="block">A pointer to the block of memory to zero</param>
+ /// <param name="itemCount">The number of elements in the block to zero</param>
+ public static void InitializeBlock<T>(IntPtr block, int itemCount) where T : unmanaged => InitializeBlock((T*)block, itemCount);
+
+ /// <summary>
/// Zeroes a block of memory pointing to the structure
/// </summary>
/// <typeparam name="T">The structure type</typeparam>
@@ -498,7 +517,7 @@ namespace VNLib.Utils.Memory
CheckBounds(dest, destOffset, count);
//Check if 64bit
- if(sizeof(nuint) == 8)
+ if(sizeof(void*) == 8)
{
//Get the number of bytes to copy
nuint byteCount = ByteCount<T>(count);
@@ -510,7 +529,7 @@ namespace VNLib.Utils.Memory
T* src = (T*)srcHandle.Pointer + offset;
//pin array
- fixed (T* dst = dest)
+ fixed (T* dst = &MemoryMarshal.GetArrayDataReference(dest))
{
//Offset dest ptr
T* dstOffset = dst + destOffset;
@@ -592,7 +611,7 @@ namespace VNLib.Utils.Memory
{
if (offset + count > handle.Length)
{
- throw new ArgumentOutOfRangeException("The offset or count is outside of the range of the block of memory");
+ throw new ArgumentOutOfRangeException(nameof(offset), "Offset or count are beyond the range of the supplied memory handle");
}
}
diff --git a/lib/Utils/src/VnEncoding.cs b/lib/Utils/src/VnEncoding.cs
index b8f18bd..e945135 100644
--- a/lib/Utils/src/VnEncoding.cs
+++ b/lib/Utils/src/VnEncoding.cs
@@ -28,6 +28,7 @@ using System.Text;
using System.Buffers;
using System.Text.Json;
using System.Threading;
+using System.Diagnostics;
using System.Buffers.Text;
using System.Threading.Tasks;
using System.Runtime.InteropServices;
@@ -37,7 +38,6 @@ using VNLib.Utils.IO;
using VNLib.Utils.Memory;
using VNLib.Utils.Extensions;
-
namespace VNLib.Utils
{
/// <summary>
@@ -778,7 +778,7 @@ namespace VNLib.Utils
{
return Convert.TryToBase64Chars(buffer, base64, out int charsWritten, options: options) ? charsWritten : ERRNO.E_FAIL;
}
-
+
/*
* Calc base64 padding chars excluding the length mod 4 = 0 case
@@ -983,6 +983,99 @@ namespace VNLib.Utils
}
/// <summary>
+ /// Attempts to base64url encode the binary buffer to it's base64url encoded representation
+ /// in place, aka does not allocate a temporary buffer. The buffer must be large enough to
+ /// encode the data, if not the operation will fail. The data in this span will be overwritten
+ /// to do the conversion
+ /// </summary>
+ /// <param name="rawData">The raw data buffer that will be used to encode data aswell as read it</param>
+ /// <param name="length">The length of the binary data to encode</param>
+ /// <param name="includePadding">A value specifying whether base64 padding should be encoded</param>
+ /// <returns>The base64url encoded string</returns>
+ /// <exception cref="ArgumentException"></exception>
+ public static string ToBase64UrlSafeStringInPlace(Span<byte> rawData, int length, bool includePadding)
+ {
+ //Encode in place
+ if (Base64.EncodeToUtf8InPlace(rawData, length, out int converted) != OperationStatus.Done)
+ {
+ throw new ArgumentException("The input buffer was not large enough to encode in-place", nameof(rawData));
+ }
+
+ //trim to converted size
+ Span<byte> base64 = rawData[..converted];
+
+ //Make url safe
+ Base64ToUrlSafeInPlace(base64);
+
+ //Remove padding
+ if (!includePadding)
+ {
+ base64 = base64.TrimEnd((byte)0x3d);
+ }
+
+ //Convert to string
+ return Encoding.UTF8.GetString(base64);
+ }
+
+ /// <summary>
+ /// Converts binary data to it's base64url encoded representation and may allocate a temporary
+ /// heap buffer.
+ /// </summary>
+ /// <param name="rawData">The binary data to encode</param>
+ /// <param name="includePadding">A value that indicates if the base64 padding characters should be included</param>
+ /// <returns>The base64url encoded string</returns>
+ /// <exception cref="ArgumentException"></exception>
+ public static string ToBase64UrlSafeString(ReadOnlySpan<byte> rawData, bool includePadding)
+ {
+ int maxBufSize = Base64.GetMaxEncodedToUtf8Length(rawData.Length);
+
+ if(maxBufSize > MAX_STACKALLOC)
+ {
+ //alloc buffer
+ using UnsafeMemoryHandle<byte> buffer = MemoryUtil.UnsafeAllocNearestPage(maxBufSize);
+
+ return ConvertToBase64UrlStringInternal(rawData, buffer.Span, includePadding);
+ }
+ else
+ {
+ //Stack alloc buffer
+ Span<byte> buffer = stackalloc byte[maxBufSize];
+
+ return ConvertToBase64UrlStringInternal(rawData, buffer, includePadding);
+ }
+ }
+
+ private static string ConvertToBase64UrlStringInternal(ReadOnlySpan<byte> rawData, Span<byte> buffer, bool includePadding)
+ {
+ //Conver to base64
+ OperationStatus status = Base64.EncodeToUtf8(rawData, buffer, out _, out int written, true);
+
+ //Check for invalid states
+ Debug.Assert(status != OperationStatus.DestinationTooSmall, "Buffer allocation was too small for the conversion");
+ Debug.Assert(status != OperationStatus.NeedMoreData, "Need more data status was returned but is not valid for an encoding operation");
+
+ //Should never occur, but just in case, this is an input error
+ if (status == OperationStatus.InvalidData)
+ {
+ throw new ArgumentException("Your input data contained values that could not be converted to base64", nameof(rawData));
+ }
+
+ Span<byte> base64 = buffer[..written];
+
+ //Make url safe
+ Base64ToUrlSafeInPlace(base64);
+
+ //Remove padding
+ if (!includePadding)
+ {
+ base64 = base64.TrimEnd((byte)0x3d);
+ }
+
+ //Convert to string
+ return Encoding.UTF8.GetString(base64);
+ }
+
+ /// <summary>
/// Encodes the binary input buffer to its base64url safe utf8 encoding, and writes the output
/// to the supplied buffer. Be sure to call <see cref="Base64.GetMaxEncodedToUtf8Length(int)"/>
/// to allocate the correct size buffer for encoding
diff --git a/lib/Utils/tests/Async/AsyncAccessSerializerTests.cs b/lib/Utils/tests/Async/AsyncAccessSerializerTests.cs
index 7119d21..3c9bde7 100644
--- a/lib/Utils/tests/Async/AsyncAccessSerializerTests.cs
+++ b/lib/Utils/tests/Async/AsyncAccessSerializerTests.cs
@@ -2,12 +2,12 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
+using System.Linq;
using System.Threading;
+using System.Diagnostics;
using System.Threading.Tasks;
using System.Collections.Generic;
-using System.Diagnostics;
using System.Runtime.CompilerServices;
-using System.Linq;
namespace VNLib.Utils.Async.Tests
{
@@ -101,7 +101,7 @@ namespace VNLib.Utils.Async.Tests
int maxCount = 64;
- Task[] asyncArr = new int[maxCount].Select(async p =>
+ Task[] asyncArr = new int[maxCount].Select(p => Task.Run(async () =>
{
//Take a lock then random delay, then release
Task entry = serializer.WaitAsync(DEFAULT_KEY);
@@ -119,7 +119,7 @@ namespace VNLib.Utils.Async.Tests
serializer.Release(DEFAULT_KEY);
- }).ToArray();
+ })).ToArray();
Task.WaitAll(asyncArr);
}
@@ -149,7 +149,7 @@ namespace VNLib.Utils.Async.Tests
using CancellationTokenSource cts = new(500);
- Task[] asyncArr = new int[maxCount].Select(async p =>
+ Task[] asyncArr = new int[maxCount].Select(p => Task.Run(async () =>
{
//Take a lock then random delay, then release
await serializer.WaitAsync(DEFAULT_KEY, cts.Token);
@@ -159,7 +159,7 @@ namespace VNLib.Utils.Async.Tests
serializer.Release(DEFAULT_KEY);
- }).ToArray();
+ })).ToArray();
Task.WaitAll(asyncArr);
@@ -175,18 +175,19 @@ namespace VNLib.Utils.Async.Tests
//Alloc serailzer base on string
IAsyncAccessSerializer<string> serializer = new AsyncAccessSerializer<string>(100, 100, StringComparer.Ordinal);
- int maxCount = 128;
+ const int maxCount = 128;
+ const int itterations = 20;
string test = "";
Stopwatch timer = new();
using CancellationTokenSource cts = new(500);
- for (int i = 0; i < 10; i++)
+ for (int i = 0; i < itterations; i++)
{
test = "";
timer.Restart();
- Task[] asyncArr = new int[maxCount].Select(async p =>
+ Task[] asyncArr = new int[maxCount].Select(p => Task.Run(async () =>
{
//Take a lock then random delay, then release
await serializer.WaitAsync(DEFAULT_KEY, cts.Token);
@@ -196,7 +197,7 @@ namespace VNLib.Utils.Async.Tests
serializer.Release(DEFAULT_KEY);
- }).ToArray();
+ })).ToArray();
Task.WaitAll(asyncArr);
@@ -208,12 +209,12 @@ namespace VNLib.Utils.Async.Tests
using SemaphoreSlim slim = new(1,1);
- for (int i = 0; i < 10; i++)
+ for (int i = 0; i < itterations; i++)
{
test = "";
timer.Restart();
- Task[] asyncArr = new int[maxCount].Select(async p =>
+ Task[] asyncArr = new int[maxCount].Select(p => Task.Run(async () =>
{
//Take a lock then random delay, then release
await slim.WaitAsync(cts.Token);
@@ -222,7 +223,7 @@ namespace VNLib.Utils.Async.Tests
test += "0";
slim.Release();
- }).ToArray();
+ })).ToArray();
Task.WaitAll(asyncArr);
diff --git a/lib/Utils/tests/IO/VnMemoryStreamTests.cs b/lib/Utils/tests/IO/VnMemoryStreamTests.cs
new file mode 100644
index 0000000..3eb95ce
--- /dev/null
+++ b/lib/Utils/tests/IO/VnMemoryStreamTests.cs
@@ -0,0 +1,99 @@
+using System;
+
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+using VNLib.Utils.Memory;
+using VNLib.Utils.Extensions;
+
+namespace VNLib.Utils.IO.Tests
+{
+ [TestClass()]
+ public class VnMemoryStreamTests
+ {
+ [TestMethod()]
+ public void VnMemoryStreamConstructorTest()
+ {
+ using (VnMemoryStream vms = new())
+ {
+ Assert.IsTrue(vms.Length == 0);
+ Assert.IsTrue(vms.Position == 0);
+ Assert.IsTrue(vms.CanSeek == true);
+ Assert.IsTrue(vms.CanRead == true);
+ Assert.IsTrue(vms.CanWrite == true);
+ }
+
+ //Test heap
+ IUnmangedHeap privateHeap = MemoryUtil.InitializeNewHeapForProcess();
+
+ using (VnMemoryStream vms = new(privateHeap, 1024, false))
+ {
+ Assert.IsTrue(vms.Length == 0);
+ Assert.IsTrue(vms.Position == 0);
+ Assert.IsTrue(vms.CanSeek == true);
+ Assert.IsTrue(vms.CanRead == true);
+ Assert.IsTrue(vms.CanWrite == true);
+ }
+
+
+ //Create from mem handle
+ MemoryHandle<byte> handle = privateHeap.Alloc<byte>(byte.MaxValue);
+
+ using (VnMemoryStream vms = VnMemoryStream.ConsumeHandle(handle, handle.GetIntLength(), false))
+ {
+ Assert.IsTrue(vms.Length == byte.MaxValue);
+ Assert.IsTrue(vms.Position == 0);
+ Assert.IsTrue(vms.CanSeek == true);
+ Assert.IsTrue(vms.CanRead == true);
+ Assert.IsTrue(vms.CanWrite == true);
+ }
+
+ //From existing data
+ ReadOnlySpan<byte> testSpan = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 };
+ using (VnMemoryStream vms = new (privateHeap, testSpan))
+ {
+ Assert.IsTrue(vms.Length == testSpan.Length);
+ Assert.IsTrue(vms.Position == 0);
+
+ //Check values copied
+ while (vms.Position < vms.Length)
+ {
+ byte test = testSpan[(int)vms.Position];
+ Assert.IsTrue(vms.ReadByte() == test);
+ }
+ }
+
+ ReadOnlyMemory<byte> testMemory = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 };
+ using (VnMemoryStream vms = new (privateHeap, testMemory))
+ {
+ Assert.IsTrue(vms.Length == testMemory.Length);
+ Assert.IsTrue(vms.Position == 0);
+
+ //Check values copied
+ while(vms.Position < vms.Length)
+ {
+ byte test = testMemory.Span[(int)vms.Position];
+ Assert.IsTrue(vms.ReadByte() == test);
+ }
+ }
+ }
+
+ [TestMethod()]
+ public void VnMemoryStreamReadonlyTest()
+ {
+ using VnMemoryStream vms = new(MemoryUtil.Shared, 0, false);
+
+ Assert.IsTrue(vms.CanWrite == true);
+
+ //Convert to readonly
+ _ = VnMemoryStream.CreateReadonly(vms);
+
+ Assert.IsTrue(vms.CanSeek == true);
+ Assert.IsTrue(vms.CanRead == true);
+ Assert.IsTrue(vms.CanWrite == false);
+
+ //Try to write
+ Assert.ThrowsException<NotSupportedException>(() => vms.WriteByte(0));
+
+ }
+ }
+} \ No newline at end of file
diff --git a/lib/Utils/tests/Memory/MemoryUtilTests.cs b/lib/Utils/tests/Memory/MemoryUtilTests.cs
index 2166eea..64f94ff 100644
--- a/lib/Utils/tests/Memory/MemoryUtilTests.cs
+++ b/lib/Utils/tests/Memory/MemoryUtilTests.cs
@@ -388,7 +388,7 @@ namespace VNLib.Utils.Memory.Tests
IUnmangedHeap heap = MemoryUtil.InitializeNewHeapForProcess();
//Init wrapper and dispose
- using TrackedHeapWrapper wrapper = new(heap);
+ using TrackedHeapWrapper wrapper = new(heap, true);
//Confirm 0 stats
HeapStatistics preTest = wrapper.GetCurrentStats();
generated by cgit (git 2.46.0) at 2024-10-01 19:28:53 +0000