diff options
| author |  vnugent <public@vaughnnugent.com> | 2023-10-20 12:18:44 -0400 |
|---|---|---|
| committer | vnugent <public@vaughnnugent.com> | 2023-10-20 12:18:44 -0400 |
| commit | d997950a29ec3ce29cd652298e678d708218fdad (patch) | |
| tree | 65733374ba223b08c99c98629a51c3789c0aa2e6 | |
| parent | f44cdf8f2685c37e5a1d77018a5227942b578863 (diff) | |
compression rpmalloc static linking, encoder memory callbacks, and more tweaks
29 files changed, 668 insertions, 180 deletions
diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..37d91eb --- /dev/null +++ b/.editorconfig @@ -0,0 +1,132 @@ +[*.cs] + +# IDE0023: Use block body for conversion operator +csharp_style_expression_bodied_operators = when_on_single_line:suggestion +csharp_indent_labels = one_less_than_current +csharp_using_directive_placement = outside_namespace:silent +csharp_prefer_simple_using_statement = true:suggestion +csharp_prefer_braces = true:silent +csharp_style_namespace_declarations = block_scoped:silent +csharp_style_prefer_method_group_conversion = true:silent +csharp_style_prefer_top_level_statements = true:silent +csharp_style_expression_bodied_methods = when_on_single_line:suggestion +csharp_style_expression_bodied_constructors = when_on_single_line:suggestion +csharp_style_expression_bodied_properties = true:silent +csharp_style_expression_bodied_indexers = true:silent +csharp_style_expression_bodied_accessors = true:silent +csharp_style_expression_bodied_lambdas = true:silent +csharp_style_expression_bodied_local_functions = false:silent +csharp_space_around_binary_operators = before_and_after +csharp_style_throw_expression = true:suggestion +csharp_style_prefer_null_check_over_type_check = true:suggestion +csharp_prefer_simple_default_expression = true:suggestion +csharp_style_prefer_index_operator = true:suggestion +csharp_style_prefer_local_over_anonymous_function = true:suggestion +csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion +csharp_style_prefer_range_operator = true:suggestion +csharp_style_prefer_tuple_swap = true:suggestion +csharp_style_prefer_utf8_string_literals = true:suggestion +csharp_style_inlined_variable_declaration = true:suggestion +csharp_style_deconstructed_variable_declaration = true:suggestion +csharp_style_unused_value_assignment_preference = discard_variable:suggestion +csharp_style_unused_value_expression_statement_preference = discard_variable:silent +csharp_prefer_static_local_function = true:suggestion +csharp_style_prefer_readonly_struct = true:suggestion +csharp_style_prefer_readonly_struct_member = true:suggestion +csharp_style_allow_embedded_statements_on_same_line_experimental = true:silent +csharp_style_allow_blank_line_after_colon_in_constructor_initializer_experimental = true:silent +csharp_style_allow_blank_lines_between_consecutive_braces_experimental = true:silent +csharp_style_allow_blank_line_after_token_in_arrow_expression_clause_experimental = true:silent +csharp_style_allow_blank_line_after_token_in_conditional_expression_experimental = true:silent +csharp_style_conditional_delegate_call = true:suggestion +csharp_style_prefer_switch_expression = true:suggestion +csharp_style_prefer_pattern_matching = true:silent +csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion +csharp_style_pattern_matching_over_as_with_null_check = true:suggestion +csharp_style_prefer_not_pattern = true:suggestion +csharp_style_prefer_extended_property_pattern = true:suggestion +csharp_style_var_for_built_in_types = false:silent +csharp_style_var_when_type_is_apparent = false:silent +csharp_style_var_elsewhere = false:silent + +[*.{cs,vb}] +#### Naming styles #### + +# Naming rules + +dotnet_naming_rule.interface_should_be_begins_with_i.severity = suggestion +dotnet_naming_rule.interface_should_be_begins_with_i.symbols = interface +dotnet_naming_rule.interface_should_be_begins_with_i.style = begins_with_i + +dotnet_naming_rule.types_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.types_should_be_pascal_case.symbols = types +dotnet_naming_rule.types_should_be_pascal_case.style = pascal_case + +dotnet_naming_rule.non_field_members_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.non_field_members_should_be_pascal_case.symbols = non_field_members +dotnet_naming_rule.non_field_members_should_be_pascal_case.style = pascal_case + +# Symbol specifications + +dotnet_naming_symbols.interface.applicable_kinds = interface +dotnet_naming_symbols.interface.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected +dotnet_naming_symbols.interface.required_modifiers = + +dotnet_naming_symbols.types.applicable_kinds = class, struct, interface, enum +dotnet_naming_symbols.types.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected +dotnet_naming_symbols.types.required_modifiers = + +dotnet_naming_symbols.non_field_members.applicable_kinds = property, event, method +dotnet_naming_symbols.non_field_members.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected +dotnet_naming_symbols.non_field_members.required_modifiers = + +# Naming styles + +dotnet_naming_style.begins_with_i.required_prefix = I +dotnet_naming_style.begins_with_i.required_suffix = +dotnet_naming_style.begins_with_i.word_separator = +dotnet_naming_style.begins_with_i.capitalization = pascal_case + +dotnet_naming_style.pascal_case.required_prefix = +dotnet_naming_style.pascal_case.required_suffix = +dotnet_naming_style.pascal_case.word_separator = +dotnet_naming_style.pascal_case.capitalization = pascal_case + +dotnet_naming_style.pascal_case.required_prefix = +dotnet_naming_style.pascal_case.required_suffix = +dotnet_naming_style.pascal_case.word_separator = +dotnet_naming_style.pascal_case.capitalization = pascal_case +dotnet_style_operator_placement_when_wrapping = beginning_of_line +tab_width = 4 +indent_size = 4 +end_of_line = crlf +dotnet_style_coalesce_expression = true:suggestion +dotnet_style_null_propagation = true:suggestion +dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion +dotnet_style_prefer_auto_properties = true:silent +dotnet_style_object_initializer = true:suggestion +dotnet_style_collection_initializer = true:suggestion +dotnet_style_prefer_simplified_boolean_expressions = true:suggestion +dotnet_style_prefer_conditional_expression_over_assignment = true:silent +dotnet_style_prefer_conditional_expression_over_return = true:silent +dotnet_style_explicit_tuple_names = true:suggestion +dotnet_style_prefer_inferred_tuple_names = true:suggestion +dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion +dotnet_style_prefer_compound_assignment = true:suggestion +dotnet_style_prefer_simplified_interpolation = true:suggestion +dotnet_style_namespace_match_folder = true:suggestion +dotnet_style_readonly_field = true:suggestion +dotnet_style_predefined_type_for_locals_parameters_members = true:silent +dotnet_style_predefined_type_for_member_access = true:silent +dotnet_style_require_accessibility_modifiers = for_non_interface_members:silent +dotnet_style_allow_multiple_blank_lines_experimental = true:silent +dotnet_style_allow_statement_immediately_after_block_experimental = true:silent +dotnet_code_quality_unused_parameters = all:suggestion +dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent +dotnet_style_qualification_for_field = false:silent +dotnet_style_qualification_for_property = false:silent +dotnet_style_qualification_for_method = false:silent +dotnet_style_qualification_for_event = false:silent diff --git a/lib/Net.Compression/vnlib_compress/CMakeLists.txt b/lib/Net.Compression/vnlib_compress/CMakeLists.txt index 2e8f61b..90fe75d 100644 --- a/lib/Net.Compression/vnlib_compress/CMakeLists.txt +++ b/lib/Net.Compression/vnlib_compress/CMakeLists.txt @@ -11,6 +11,7 @@ set(VNLIB_COMPRESS_SOURCES compression.c) #set options for enable botli and zlib option(ENABLE_BROTLI "Enable brotli compression" ON) option(ENABLE_ZLIB "Enable zlib compression" ON) +option(ENABLE_RPMALLOC "Link local vnlib_rpmalloc allocator" ON) #add feature specific source files to the project if(ENABLE_BROTLI) @@ -32,7 +33,7 @@ add_library(${CMAKE_PROJECT_NAME}_static STATIC ${VNLIB_COMPRESS_SOURCES} ${COMP #if on unix lib will be appended, so we can adjust if(UNIX) - set_target_properties(${CMAKE_PROJECT_NAME} ${CMAKE_PROJECT_NAME}_static PROPERTIES OUTPUT_NAME vn_compress) + set_target_properties(${CMAKE_PROJECT_NAME} ${CMAKE_PROJECT_NAME}_static PROPERTIES OUTPUT_NAME _vncompress) endif() @@ -280,3 +281,53 @@ if(ENABLE_ZLIB) target_link_libraries(${CMAKE_PROJECT_NAME} lib_deflate) endif() +#Add support for rpmalloc memmory allocator +if(ENABLE_RPMALLOC) + + #Include the nativeheap api header + include_directories(../../Utils.Memory/NativeHeapApi/src) + + if(MSVC) + + #find the rpmalloc static library for windows builds + find_library(VNLIB_RPMALLOC_DEBUG_LIB + NAMES vnlib_rpmalloc_static + PATHS ../../Utils.Memory/vnlib_rpmalloc/build/Debug + ) + + find_library(VNLIB_RPMALLOC_RELEASE_LIB + NAMES vnlib_rpmalloc_static + PATHS ../../Utils.Memory/vnlib_rpmalloc/build/Release + ) + + #target static libraries for all configs + target_link_libraries(${CMAKE_PROJECT_NAME} + $<$<CONFIG:Debug>:${VNLIB_RPMALLOC_DEBUG_LIB}> + #set release for all release configs + $<$<CONFIG:MinSizeRel>:${VNLIB_RPMALLOC_RELEASE_LIB}> + $<$<CONFIG:RelWithDepInfo>:${VNLIB_RPMALLOC_RELEASE_LIB}> + $<$<CONFIG:Release>:${VNLIB_RPMALLOC_RELEASE_LIB}> + ) + + elseif(UNIX) + + #find the rpmalloc library for unix builds + find_library(VNLIB_RPMALLOC_LIB + NAMES _vnrpmalloc + PATHS ../../Utils.Memory/vnlib_rpmalloc/build + ) + + #add the rpmalloc library to the project + target_link_libraries(${CMAKE_PROJECT_NAME} ${VNLIB_RPMALLOC_LIB}) + + else() + + #failure + message(FATAL_ERROR "Unsupported platform configuration for rpmalloc") + + endif() + + #add defintion to enable custom malloc heap overrides + target_compile_definitions(${CMAKE_PROJECT_NAME} PRIVATE VNLIB_CUSTOM_MALLOC_ENABLE) + +endif() diff --git a/lib/Net.Compression/vnlib_compress/feature_brotli.c b/lib/Net.Compression/vnlib_compress/feature_brotli.c index 752082a..a300500 100644 --- a/lib/Net.Compression/vnlib_compress/feature_brotli.c +++ b/lib/Net.Compression/vnlib_compress/feature_brotli.c @@ -26,6 +26,26 @@ if (!state) return ERR_INVALID_PTR; \ if (!state->compressor) return ERR_BR_INVALID_STATE; \ +/* +* Stream memory management functions +*/ +static void* _brAllocCallback(void* opaque, size_t size) +{ + (void)opaque; + return vnmalloc(size, 1); +} + +static void _brFreeCallback(void* opaque, void* address) +{ + (void)opaque; + + //Brotli may pass a null address to the free callback + if (address) + { + vnfree(address); + } +} + int BrAllocCompressor(CompressorState* state) { @@ -40,7 +60,11 @@ int BrAllocCompressor(CompressorState* state) return ERR_COMP_LEVEL_NOT_SUPPORTED; } - comp = BrotliEncoderCreateInstance(0, 0, 0); + comp = BrotliEncoderCreateInstance( + &_brAllocCallback, + &_brFreeCallback, + NULL + ); if (!comp) { @@ -62,7 +86,7 @@ int BrAllocCompressor(CompressorState* state) /* * Capture the block size as a size hint if it is greater than 0 */ - if (state->blockSize > 0) + if (state->blockSize > 0) { BrotliEncoderSetParameter(comp, BROTLI_PARAM_SIZE_HINT, state->blockSize); } diff --git a/lib/Net.Compression/vnlib_compress/feature_zlib.c b/lib/Net.Compression/vnlib_compress/feature_zlib.c index 432b860..5dbd7ad 100644 --- a/lib/Net.Compression/vnlib_compress/feature_zlib.c +++ b/lib/Net.Compression/vnlib_compress/feature_zlib.c @@ -25,10 +25,25 @@ #include "feature_zlib.h" #include <zlib.h> + #define validateCompState(state) \ if (!state) return ERR_INVALID_PTR; \ if (!state->compressor) return ERR_GZ_INVALID_STATE; \ +/* +* Stream memory management functions +*/ +static void* _gzAllocCallback(void* opaque, uint32_t items, uint32_t size) +{ + (void)opaque; + return vnmalloc(items, size); +} + +static void _gzFreeCallback(void* opaque, void* address) +{ + (void)opaque; + vnfree(address); +} int DeflateAllocCompressor(CompressorState* state) { @@ -46,8 +61,8 @@ int DeflateAllocCompressor(CompressorState* state) return ERR_OUT_OF_MEMORY; } - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; + stream->zalloc = &_gzAllocCallback; + stream->zfree = &_gzFreeCallback; stream->opaque = Z_NULL; /* diff --git a/lib/Net.Compression/vnlib_compress/util.h b/lib/Net.Compression/vnlib_compress/util.h index 4a11708..5d87c8f 100644 --- a/lib/Net.Compression/vnlib_compress/util.h +++ b/lib/Net.Compression/vnlib_compress/util.h @@ -41,8 +41,8 @@ #define ERR_INVALID_PTR -1 #define ERR_OUT_OF_MEMORY -2 -#define TRUE 1; -#define FALSE 0; +#define TRUE 1 +#define FALSE 0 #ifndef NULL #define NULL 0 @@ -52,11 +52,54 @@ #define _In_ #endif +#if defined(VNLIB_CUSTOM_MALLOC_ENABLE) + +/* +* Add rpmalloc overrides +*/ + +#include <NativeHeapApi.h> + +/* +* Add debug runtime assertions +*/ +#ifdef DEBUG + #define NDEBUG + #include <assert.h> +#else + #define assert(x) +#endif + +/* +* Add overrides for malloc, calloc, and free that use +* the nativeheap api to allocate memory +*/ + +static inline void* vnmalloc(size_t num, size_t size) +{ + return heapAlloc(heapGetSharedHeapHandle(), num, size, FALSE); +} + +static inline void* vncalloc(size_t num, size_t size) +{ + return heapAlloc(heapGetSharedHeapHandle(), num , size, TRUE); +} + +static inline void vnfree(void* ptr) +{ + ERRNO result; + result = heapFree(heapGetSharedHeapHandle(), ptr); + + //track failed free results + assert(result > 0); +} + +#else /* * Stub method for malloc. All calls to vnmalloc should be freed with vnfree. */ -#define vnmalloc(size) malloc(size) +#define vnmalloc(num, size) malloc(num * size) /* * Stub method for free @@ -68,5 +111,6 @@ */ #define vncalloc(num, size) calloc(num, size) +#endif #endif /* !UTIL_H_ */
\ No newline at end of file diff --git a/lib/Net.Http/src/Core/Request/HttpRequestExtensions.cs b/lib/Net.Http/src/Core/Request/HttpRequestExtensions.cs index 0da2e98..dd21707 100644 --- a/lib/Net.Http/src/Core/Request/HttpRequestExtensions.cs +++ b/lib/Net.Http/src/Core/Request/HttpRequestExtensions.cs @@ -368,7 +368,7 @@ namespace VNLib.Net.Http.Core bytes = info.Encoding.GetBytes(data, buffHandle); //Create a new memory stream encapsulating the file data - VnMemoryStream vms = VnMemoryStream.ConsumeHandle(buffHandle, bytes, true); + VnMemoryStream vms = VnMemoryStream.FromHandle(buffHandle, true, bytes, true); //Create new upload wrapper that owns the stream return new(vms, true, ct, filename); diff --git a/lib/Plugins.Essentials/src/EventProcessor.cs b/lib/Plugins.Essentials/src/EventProcessor.cs index f292068..3ea61c0 100644 --- a/lib/Plugins.Essentials/src/EventProcessor.cs +++ b/lib/Plugins.Essentials/src/EventProcessor.cs @@ -569,7 +569,11 @@ namespace VNLib.Plugins.Essentials public bool FindResourceInRoot(string resourcePath, bool fullyQualified, out string path) { //Special case where user's can specify a fullly qualified path (meant to reach a remote file, eg UNC/network share or other disk) - if (fullyQualified && Path.IsPathRooted(resourcePath) && Path.IsPathFullyQualified(resourcePath) && FileOperations.FileExists(resourcePath)) + if (fullyQualified + && Path.IsPathRooted(resourcePath) + && Path.IsPathFullyQualified(resourcePath) + && FileOperations.FileExists(resourcePath) + ) { path = resourcePath; return true; diff --git a/lib/Plugins.Essentials/src/Extensions/JsonResponse.cs b/lib/Plugins.Essentials/src/Extensions/JsonResponse.cs index 549d746..b418b6f 100644 --- a/lib/Plugins.Essentials/src/Extensions/JsonResponse.cs +++ b/lib/Plugins.Essentials/src/Extensions/JsonResponse.cs @@ -57,8 +57,8 @@ namespace VNLib.Plugins.Essentials.Extensions //Alloc buffer _handle = MemoryUtil.Shared.Alloc<byte>(4096, false); - //Consume handle for stream, but make sure not to dispose the stream - _asStream = VnMemoryStream.ConsumeHandle(_handle, 0, false); + //Create stream around handle and not own it + _asStream = VnMemoryStream.FromHandle(_handle, false, 0, false); //Get memory owner from handle _memoryOwner = _handle.ToMemoryManager(false); diff --git a/lib/Utils.Memory/NativeHeapApi/src/NativeHeapApi.h b/lib/Utils.Memory/NativeHeapApi/src/NativeHeapApi.h index 6f9ad63..eec4a8a 100644 --- a/lib/Utils.Memory/NativeHeapApi/src/NativeHeapApi.h +++ b/lib/Utils.Memory/NativeHeapApi/src/NativeHeapApi.h @@ -21,6 +21,8 @@ #pragma once +#include <stdint.h> + #ifndef NATIVE_HEAP_API #define NATIVE_HEAP_API @@ -35,7 +37,6 @@ #endif // WIN32 #endif // !HEAP_METHOD_CC - /* * Decorator for exporting methods for dll usage */ @@ -47,8 +48,7 @@ #endif #endif // HEAP_METHOD_EXPORT! -#ifndef WIN32 -typedef unsigned long DWORD; +#ifdef WIN32 typedef void* LPVOID; #endif // !WIN32 @@ -83,16 +83,27 @@ typedef enum HeapCreationFlags typedef void* ERRNO; /// <summary> +/// A pointer to a heap structure that was stored during heap creation +/// </summary> +typedef void* HeapHandle; + +/// <summary> /// A structure for heap initialization /// </summary> typedef struct UnmanagedHeapDescriptor { - LPVOID HeapPointer; + HeapHandle HeapPointer; + ERRNO Flags; HeapCreationFlags CreationFlags; - ERRNO Flags; } UnmanagedHeapDescriptor; /// <summary> +/// Gets the shared heap handle for the process/library +/// </summary> +/// <returns>A pointer to the shared heap</returns> +HEAP_METHOD_EXPORT HeapHandle HEAP_METHOD_CC heapGetSharedHeapHandle(void); + +/// <summary> /// The heap creation method. You must set the flags->HeapPointer = your heap /// structure /// </summary> @@ -104,7 +115,7 @@ HEAP_METHOD_EXPORT ERRNO HEAP_METHOD_CC heapCreate(UnmanagedHeapDescriptor* flag /// Destroys a previously created heap /// </summary> /// <param name="heap">The pointer to your custom heap structure from heap creation</param> -HEAP_METHOD_EXPORT ERRNO HEAP_METHOD_CC heapDestroy(LPVOID heap); +HEAP_METHOD_EXPORT ERRNO HEAP_METHOD_CC heapDestroy(HeapHandle heap); /// <summary> /// Allocates a block from the desired heap and returns a pointer @@ -115,7 +126,7 @@ HEAP_METHOD_EXPORT ERRNO HEAP_METHOD_CC heapDestroy(LPVOID heap); /// <param name="alignment">The alignment (or size) of each element in bytes</param> /// <param name="zero">A flag to zero the block before returning the block</param> /// <returns>A pointer to the allocated block</returns> -HEAP_METHOD_EXPORT LPVOID HEAP_METHOD_CC heapAlloc(LPVOID heap, size_t elements, size_t alignment, BOOL zero); +HEAP_METHOD_EXPORT void* HEAP_METHOD_CC heapAlloc(HeapHandle heap, uint64_t elements, uint64_t alignment, int zero); /// <summary> /// Reallocates a block on the desired heap and returns a pointer to the new block. If reallocation @@ -128,7 +139,7 @@ HEAP_METHOD_EXPORT LPVOID HEAP_METHOD_CC heapAlloc(LPVOID heap, size_t elements, /// <param name="alignment">The element size or block alignment</param> /// <param name="zero">A flag to zero the block (or the new size) before returning.</param> /// <returns>A pointer to the reallocated block, or zero if the operation failed or is not supported</returns> -HEAP_METHOD_EXPORT LPVOID HEAP_METHOD_CC heapRealloc(LPVOID heap, LPVOID block, size_t elements, size_t alignment, BOOL zero); +HEAP_METHOD_EXPORT void* HEAP_METHOD_CC heapRealloc(HeapHandle heap, void* block, uint64_t elements, uint64_t alignment, int zero); /// <summary> /// Frees a previously allocated block on the desired heap. @@ -136,6 +147,6 @@ HEAP_METHOD_EXPORT LPVOID HEAP_METHOD_CC heapRealloc(LPVOID heap, LPVOID block, /// <param name="heap">A pointer to your heap structure</param> /// <param name="block">A pointer to the block to free</param> /// <returns>A value that indicates the result of the operation, nonzero if success, 0 if a failure occurred </returns> -HEAP_METHOD_EXPORT ERRNO HEAP_METHOD_CC heapFree(LPVOID heap, LPVOID block); +HEAP_METHOD_EXPORT ERRNO HEAP_METHOD_CC heapFree(HeapHandle heap, void* block); #endif // !NATIVE_HEAP_API
\ No newline at end of file diff --git a/lib/Utils.Memory/vnlib_rpmalloc/CMakeLists.txt b/lib/Utils.Memory/vnlib_rpmalloc/CMakeLists.txt index b2cb287..75efde6 100644 --- a/lib/Utils.Memory/vnlib_rpmalloc/CMakeLists.txt +++ b/lib/Utils.Memory/vnlib_rpmalloc/CMakeLists.txt @@ -25,7 +25,7 @@ add_library(${CMAKE_PROJECT_NAME}_static STATIC ${VNLIB_RPMALLOC_SOURCES} ${HEAD #if on unix lib will be appended, so we can adjust if(UNIX) - set_target_properties(${CMAKE_PROJECT_NAME} ${CMAKE_PROJECT_NAME}_static PROPERTIES OUTPUT_NAME _rpmalloc) + set_target_properties(${CMAKE_PROJECT_NAME} ${CMAKE_PROJECT_NAME}_static PROPERTIES OUTPUT_NAME _vnrpmalloc) endif() #Setup the compiler options diff --git a/lib/Utils.Memory/vnlib_rpmalloc/Taskfile.yaml b/lib/Utils.Memory/vnlib_rpmalloc/Taskfile.yaml index 635c006..481b437 100644 --- a/lib/Utils.Memory/vnlib_rpmalloc/Taskfile.yaml +++ b/lib/Utils.Memory/vnlib_rpmalloc/Taskfile.yaml @@ -31,9 +31,12 @@ tasks: TAR_FILES: "license.txt readme.txt rpmalloc_license.txt" cmds: - #make bin dir + #make bin dir - cmd: powershell -Command "New-Item -Type Directory -Force -Path './bin'" ignore_error: true + + #pack source code + - task: packsource #get licenses for debug - task: licenses @@ -63,6 +66,14 @@ tasks: - powershell -Command "Copy-Item -Path ./license -Destination '{{.TARGET}}/license.txt'" #add readme file - powershell -Command "Copy-Item -Path ./build.readme.txt -Destination '{{.TARGET}}/readme.txt'" + + packsource: + dir: '{{.USER_WORKING_DIR}}' + internal: true + cmds: + #copy source code to target + - powershell -Command "Get-ChildItem -Exclude *.yaml,build,bin | Resolve-Path -Relative | tar --files-from - -cvzf 'bin/src.tgz'" + clean: ignore_error: true diff --git a/lib/Utils.Memory/vnlib_rpmalloc/vnlib_rpmalloc.c b/lib/Utils.Memory/vnlib_rpmalloc/vnlib_rpmalloc.c index 20e1554..b1faf85 100644 --- a/lib/Utils.Memory/vnlib_rpmalloc/vnlib_rpmalloc.c +++ b/lib/Utils.Memory/vnlib_rpmalloc/vnlib_rpmalloc.c @@ -22,13 +22,29 @@ * along with vnlib_rpmalloc. If not, see http://www.gnu.org/licenses/. */ -#include "vnlib_rpmalloc.h" +/* +* Top level linux definitions +*/ +#ifdef __GNUC__ + +#define _GNU_SOURCE +#define TRUE 1 +#define FALSE 0 + +#endif + #include <NativeHeapApi.h> #include <rpmalloc.h> -#include <stdint.h> #if defined(_WIN32) +/* +* setup windows api incudes +*/ + +#define WIN32_LEAN_AND_MEAN +#include <Windows.h> + BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved) { (void)hModule; @@ -57,11 +73,6 @@ BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserv #else -/* -* Nuts and bolts from rpmalloc/malloc.c for pthread hooks -* for thread and process heap initializations -*/ - #include <pthread.h> #include <stdlib.h> #include <stdint.h> @@ -128,11 +139,17 @@ int pthread_create(pthread_t* thread, #endif -#define GLOBAL_HEAP_HANDLE_VALUE -10 +#define SHARED_HEAP_HANDLE_VALUE ((HeapHandle)1) #define GLOBAL_HEAP_INIT_CHECK if (!rpmalloc_is_thread_initialized()) { rpmalloc_thread_initialize(); } //Define the heap methods +HEAP_METHOD_EXPORT HeapHandle HEAP_METHOD_CC heapGetSharedHeapHandle(void) +{ + //Return the shared heap pointer + return SHARED_HEAP_HANDLE_VALUE; +} + HEAP_METHOD_EXPORT ERRNO HEAP_METHOD_CC heapCreate(UnmanagedHeapDescriptor* flags) { //Check flags @@ -141,8 +158,8 @@ HEAP_METHOD_EXPORT ERRNO HEAP_METHOD_CC heapCreate(UnmanagedHeapDescriptor* flag //User requested the global heap, synchronziation is not required, so we can clear the sync flag flags->CreationFlags &= ~(HEAP_CREATION_SERIALZE_ENABLED); - //Set the heap pointer as the global heap value - flags->HeapPointer = (LPVOID)GLOBAL_HEAP_HANDLE_VALUE; + //For shared heap set pointer to null + flags->HeapPointer = heapGetSharedHeapHandle(); //Success return (ERRNO)TRUE; @@ -158,32 +175,29 @@ HEAP_METHOD_EXPORT ERRNO HEAP_METHOD_CC heapCreate(UnmanagedHeapDescriptor* flag } -HEAP_METHOD_EXPORT ERRNO HEAP_METHOD_CC heapDestroy(LPVOID heap) +HEAP_METHOD_EXPORT ERRNO HEAP_METHOD_CC heapDestroy(HeapHandle heap) { - //Destroy the heap - if ((int64_t)heap == GLOBAL_HEAP_HANDLE_VALUE) + //Destroy non-shared heaps + if (heap != SHARED_HEAP_HANDLE_VALUE) { - //Gloal heap, do nothing, and allow the entrypoint cleanup - return (ERRNO)TRUE; - } + //Free all before destroy + rpmalloc_heap_free_all(heap); - //Free all before destroy - rpmalloc_heap_free_all(heap); - - //Destroy the heap - rpmalloc_heap_release(heap); + //Destroy the heap + rpmalloc_heap_release(heap); + } return (ERRNO)TRUE; } -HEAP_METHOD_EXPORT LPVOID HEAP_METHOD_CC heapAlloc(LPVOID heap, size_t elements, size_t alignment, BOOL zero) +HEAP_METHOD_EXPORT void* HEAP_METHOD_CC heapAlloc(HeapHandle heap, size_t elements, size_t alignment, int zero) { //Multiply for element size size_t size = elements * alignment; //Check for global heap - if ((int64_t)heap == GLOBAL_HEAP_HANDLE_VALUE) + if (heap == SHARED_HEAP_HANDLE_VALUE) { /* * When called from the dotnet CLR the thread may not call the DLL @@ -219,14 +233,14 @@ HEAP_METHOD_EXPORT LPVOID HEAP_METHOD_CC heapAlloc(LPVOID heap, size_t elements, } -HEAP_METHOD_EXPORT LPVOID HEAP_METHOD_CC heapRealloc(LPVOID heap, LPVOID block, size_t elements, size_t alignment, BOOL zero) +HEAP_METHOD_EXPORT void* HEAP_METHOD_CC heapRealloc(HeapHandle heap, void* block, size_t elements, size_t alignment, int zero) { //Multiply for element size size_t size = elements * alignment; (void)zero; //Check for global heap - if ((int64_t)heap == GLOBAL_HEAP_HANDLE_VALUE) + if (heap == SHARED_HEAP_HANDLE_VALUE) { /* * When called from the dotnet CLR the thread may not call the DLL @@ -246,10 +260,10 @@ HEAP_METHOD_EXPORT LPVOID HEAP_METHOD_CC heapRealloc(LPVOID heap, LPVOID block, } -HEAP_METHOD_EXPORT ERRNO HEAP_METHOD_CC heapFree(LPVOID heap, LPVOID block) +HEAP_METHOD_EXPORT ERRNO HEAP_METHOD_CC heapFree(HeapHandle heap, void* block) { //Check for global heap - if ((int64_t)heap == GLOBAL_HEAP_HANDLE_VALUE) + if (heap == SHARED_HEAP_HANDLE_VALUE) { /* * If free happens on a different thread, we must allocate the heap diff --git a/lib/Utils.Memory/vnlib_rpmalloc/vnlib_rpmalloc.h b/lib/Utils.Memory/vnlib_rpmalloc/vnlib_rpmalloc.h deleted file mode 100644 index dd46c91..0000000 --- a/lib/Utils.Memory/vnlib_rpmalloc/vnlib_rpmalloc.h +++ /dev/null @@ -1,25 +0,0 @@ - -#pragma once -#ifndef VNLIB_RPMALLOC_H - -#if defined(_WIN32) || defined(_WIN64) - -#define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers -// Windows Header Files -#include <windows.h> - -#else - -#define _GNU_SOURCE // for RTLD_NEXT - -#include <stddef.h> - -#define TRUE 1 -#define FALSE 0 - -//Windows type aliases for non-win -typedef int BOOL; - -#endif - -#endif // !VNLIB_RPMALLOC_H
\ No newline at end of file diff --git a/lib/Utils.Memory/vnlib_rpmalloc/vnlib_rpmalloc.vcxitems b/lib/Utils.Memory/vnlib_rpmalloc/vnlib_rpmalloc.vcxitems index 2e22921..6fc55c7 100644 --- a/lib/Utils.Memory/vnlib_rpmalloc/vnlib_rpmalloc.vcxitems +++ b/lib/Utils.Memory/vnlib_rpmalloc/vnlib_rpmalloc.vcxitems @@ -19,9 +19,6 @@ <Text Include="$(MSBuildThisFileDirectory)license.txt" /> </ItemGroup> <ItemGroup> - <ClInclude Include="$(MSBuildThisFileDirectory)vnlib_rpmalloc.h" /> - </ItemGroup> - <ItemGroup> <ClCompile Include="$(MSBuildThisFileDirectory)vnlib_rpmalloc.c" /> </ItemGroup> <ItemGroup> diff --git a/lib/Utils/src/Async/AsyncQueue.cs b/lib/Utils/src/Async/AsyncQueue.cs index ba45513..45f1219 100644 --- a/lib/Utils/src/Async/AsyncQueue.cs +++ b/lib/Utils/src/Async/AsyncQueue.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2022 Vaughn Nugent +* Copyright (c) 2023 Vaughn Nugent * * Library: VNLib * Package: VNLib.Utils @@ -30,11 +30,12 @@ using System.Diagnostics.CodeAnalysis; namespace VNLib.Utils.Async { + /// <summary> /// Provides a <see cref="Channel{T}"/> based asynchronous queue /// </summary> /// <typeparam name="T">The event object type</typeparam> - public class AsyncQueue<T> + public class AsyncQueue<T> : IAsyncQueue<T> { private readonly Channel<T> _channel; @@ -45,12 +46,13 @@ namespace VNLib.Utils.Async /// </summary> /// <param name="capacity">The maxium number of items to allow in the queue</param> public AsyncQueue(int capacity):this(false, false, capacity) - {} + { } + /// <summary> /// Initalizes a new multi-threaded unbound channel queue /// </summary> public AsyncQueue():this(false, false) - {} + { } /// <summary> /// Initalizes a new queue that allows specifying concurrency requirements @@ -105,39 +107,22 @@ namespace VNLib.Utils.Async _channel = Channel.CreateBounded<T>(options); } - /// <summary> - /// Attemts to enqeue an item if the queue has the capacity - /// </summary> - /// <param name="item">The item to eqneue</param> - /// <returns>True if the queue can accept another item, false otherwise</returns> + /// <inheritdoc/> public bool TryEnque(T item) => _channel.Writer.TryWrite(item); - /// <summary> - /// Enqueues an item to the end of the queue and notifies a waiter that an item was enqueued - /// </summary> - /// <param name="item">The item to enqueue</param> - /// <param name="cancellationToken"></param> + + /// <inheritdoc/> /// <exception cref="ObjectDisposedException"></exception> public ValueTask EnqueueAsync(T item, CancellationToken cancellationToken = default) => _channel.Writer.WriteAsync(item, cancellationToken); - /// <summary> - /// Asynchronously waits for an item to be Enqueued to the end of the queue. - /// </summary> - /// <returns>The item at the begining of the queue</returns> + + /// <inheritdoc/> /// <exception cref="ObjectDisposedException"></exception> public ValueTask<T> DequeueAsync(CancellationToken cancellationToken = default) => _channel.Reader.ReadAsync(cancellationToken); - /// <summary> - /// Removes the object at the beginning of the queue and stores it to the result parameter. Without waiting for a change - /// event. - /// </summary> - /// <param name="result">The item that was at the begining of the queue</param> - /// <returns>True if the queue could be read synchronously, false if the lock could not be entered, or the queue contains no items</returns> + + /// <inheritdoc/> /// <exception cref="ObjectDisposedException"></exception> public bool TryDequeue([MaybeNullWhen(false)] out T result) => _channel.Reader.TryRead(out result); - /// <summary> - /// Peeks the object at the beginning of the queue and stores it to the result parameter. Without waiting for a change - /// event. - /// </summary> - /// <param name="result">The item that was at the begining of the queue</param> - /// <returns>True if the queue could be read synchronously, false if the lock could not be entered, or the queue contains no items</returns> + + /// <inheritdoc/> /// <exception cref="ObjectDisposedException"></exception> public bool TryPeek([MaybeNullWhen(false)] out T result) => _channel.Reader.TryPeek(out result); } diff --git a/lib/Utils/src/Async/IAsyncEventSource.cs b/lib/Utils/src/Async/IAsyncEventSource.cs new file mode 100644 index 0000000..968f94c --- /dev/null +++ b/lib/Utils/src/Async/IAsyncEventSource.cs @@ -0,0 +1,45 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Utils +* File: IAsyncEventSource.cs +* +* IAsyncEventSource.cs is part of VNLib.Utils which is part of the larger +* VNLib collection of libraries and utilities. +* +* VNLib.Utils is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* VNLib.Utils is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with VNLib.Utils. If not, see http://www.gnu.org/licenses/. +*/ + +namespace VNLib.Utils.Async +{ + /// <summary> + /// A type that publishes events to asynchronous event queues + /// </summary> + /// <typeparam name="T">The event item type to publish</typeparam> + public interface IAsyncEventSource<T> + { + /// <summary> + /// Subscribes a new queue to publish events to + /// </summary> + /// <param name="queue">The queue instance to publish new events to</param> + void Subscribe(IAsyncQueue<T> queue); + + /// <summary> + /// Unsubscribes a previously subscribed queue from receiving events + /// </summary> + /// <param name="queue">The queue instance to unregister from events</param> + void Unsubscribe(IAsyncQueue<T> queue); + } +} diff --git a/lib/Utils/src/Async/IAsyncQueue.cs b/lib/Utils/src/Async/IAsyncQueue.cs new file mode 100644 index 0000000..ab786f1 --- /dev/null +++ b/lib/Utils/src/Async/IAsyncQueue.cs @@ -0,0 +1,76 @@ +/* +* Copyright (c) 2023 Vaughn Nugent +* +* Library: VNLib +* Package: VNLib.Utils +* File: IAsyncQueue.cs +* +* IAsyncQueue.cs is part of VNLib.Utils which is part of the larger +* VNLib collection of libraries and utilities. +* +* VNLib.Utils is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published +* by the Free Software Foundation, either version 2 of the License, +* or (at your option) any later version. +* +* VNLib.Utils is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with VNLib.Utils. If not, see http://www.gnu.org/licenses/. +*/ + +using System.Threading; +using System.Threading.Tasks; +using System.Diagnostics.CodeAnalysis; + +namespace VNLib.Utils.Async +{ + /// <summary> + /// Provides a generic asynchronous queue + /// </summary> + /// <typeparam name="T">The item message type</typeparam> +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix + public interface IAsyncQueue<T> +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix + { + /// <summary> + /// Attemts to enqueue an item if the queue has the capacity + /// </summary> + /// <param name="item">The item to eqneue</param> + /// <returns>True if the queue can accept another item, false otherwise</returns> + bool TryEnque(T item); + + /// <summary> + /// Enqueues an item to the end of the queue and notifies a waiter that an item was enqueued + /// </summary> + /// <param name="item">The item to enqueue</param> + /// <param name="cancellationToken">A token to cancel the operation</param> + ValueTask EnqueueAsync(T item, CancellationToken cancellationToken = default); + + /// <summary> + /// Asynchronously waits for an item to be Enqueued to the end of the queue. + /// </summary> + /// <param name="cancellationToken">A token to cancel the operation</param> + /// <returns>The item at the begining of the queue</returns> + ValueTask<T> DequeueAsync(CancellationToken cancellationToken = default); + + /// <summary> + /// Removes the object at the beginning of the queue and stores it to the result parameter. Without waiting for a change + /// event. + /// </summary> + /// <param name="result">The item that was at the begining of the queue</param> + /// <returns>True if the queue could be read synchronously, false if the lock could not be entered, or the queue contains no items</returns> + bool TryDequeue([MaybeNullWhen(false)] out T result); + + /// <summary> + /// Peeks the object at the beginning of the queue and stores it to the result parameter. Without waiting for a change + /// event. + /// </summary> + /// <param name="result">The item that was at the begining of the queue</param> + /// <returns>True if the queue could be read synchronously, false if the lock could not be entered, or the queue contains no items</returns> + bool TryPeek([MaybeNullWhen(false)] out T result); + } +} diff --git a/lib/Utils/src/Extensions/IoExtensions.cs b/lib/Utils/src/Extensions/IoExtensions.cs index d727fed..ffa374b 100644 --- a/lib/Utils/src/Extensions/IoExtensions.cs +++ b/lib/Utils/src/Extensions/IoExtensions.cs @@ -26,15 +26,19 @@ using System; using System.IO; using System.Buffers; using System.Threading; +using System.Diagnostics; using System.Threading.Tasks; using System.Runtime.Versioning; +using System.IO.IsolatedStorage; using System.Runtime.CompilerServices; using VNLib.Utils.IO; using VNLib.Utils.Memory; - using static VNLib.Utils.Memory.MemoryUtil; +//Disable configure await warning +#pragma warning disable CA2007 + namespace VNLib.Utils.Extensions { /// <summary> @@ -103,6 +107,7 @@ namespace VNLib.Utils.Extensions //Wait for copy to complete await CopyToAsync(source, dest, buffer.Memory, token); } + /// <summary> /// Provides an async wrapper for copying data from the current stream to another with a /// buffer from the <paramref name="heap"/> @@ -186,6 +191,7 @@ namespace VNLib.Utils.Extensions dest.Write(buffer.Span[..read]); } while (true); } + /// <summary> /// Copies data from one stream to another, using self managed buffers. May allocate up to 2MB. /// </summary> @@ -440,5 +446,75 @@ namespace VNLib.Utils.Extensions return Parent.WriteFileAsync(path, data, contentType, cancellation); } } + + /// <summary> + /// The maximum buffer size to use when copying files + /// </summary> + public const long MaxCopyBufferSize = 0x10000; //64k + + /// <summary> + /// The minimum buffer size to use when copying files + /// </summary> + public const long MinCopyBufferSize = 0x1000; //4k + + /// <summary> + /// Creates a new <see cref="ISimpleFilesystem"/> wrapper for the given <see cref="IsolatedStorageDirectory"/> + /// <para> + /// Buffers are clamped to <see cref="MaxCopyBufferSize"/> and <see cref="MinCopyBufferSize"/> + /// </para> + /// </summary> + /// <param name="dir"></param> + /// <returns>A <see cref="ISimpleFilesystem"/> wrapper around the <see cref="IsolatedStorageDirectory"/></returns> + public static ISimpleFilesystem CreateSimpleFs(this IsolatedStorageDirectory dir) => new IsolatedStorageSimpleFs(dir); + + private sealed record class IsolatedStorageSimpleFs(IsolatedStorageDirectory Directory) : ISimpleFilesystem + { + ///<inheritdoc/> + public string GetExternalFilePath(string filePath) => Directory.GetFullFilePath(filePath); + + ///<inheritdoc/> + public Task DeleteFileAsync(string filePath, CancellationToken cancellation) + { + Directory.DeleteFile(filePath); + return Task.CompletedTask; + } + + ///<inheritdoc/> + public async Task WriteFileAsync(string filePath, Stream data, string contentType, CancellationToken cancellation) + { + //For when I forget and increase the buffer size + Debug.Assert(MaxCopyBufferSize < int.MaxValue, "MaxCopyBufferSize is too large to be cast to an int"); + + //Try to calculate a buffer size + long bufferSize = data.CanSeek ? Math.Clamp(data.Length, MinCopyBufferSize, MaxCopyBufferSize) : MinCopyBufferSize; + + //Open the local file + await using IsolatedStorageFileStream output = Directory.OpenFile(filePath, FileMode.OpenOrCreate, FileAccess.ReadWrite); + + //Copy file + await CopyToAsync(data, output, (int)bufferSize, Shared, cancellation); + + //All done + } + + ///<inheritdoc/> + public async Task<long> ReadFileAsync(string filePath, Stream output, CancellationToken cancellation) + { + //For when I forget and increase the buffer size + Debug.Assert(MaxCopyBufferSize < int.MaxValue, "MaxCopyBufferSize is too large to be cast to an int"); + + //Open the local file + await using IsolatedStorageFileStream local = Directory.OpenFile(filePath, FileMode.Open, FileAccess.Read); + + //Try to calculate a buffer size + long bufferSize = Math.Clamp(local.Length, MinCopyBufferSize, MaxCopyBufferSize); + + //Copy file + await CopyToAsync(local, output, (int)bufferSize, Shared, cancellation); + + return local.Length; + } + + } } }
\ No newline at end of file diff --git a/lib/Utils/src/IO/IsolatedStorageDirectory.cs b/lib/Utils/src/IO/IsolatedStorageDirectory.cs index c06018e..6ed9339 100644 --- a/lib/Utils/src/IO/IsolatedStorageDirectory.cs +++ b/lib/Utils/src/IO/IsolatedStorageDirectory.cs @@ -1,5 +1,5 @@ /* -* Copyright (c) 2022 Vaughn Nugent +* Copyright (c) 2023 Vaughn Nugent * * Library: VNLib * Package: VNLib.Utils @@ -35,18 +35,23 @@ namespace VNLib.Utils.IO { private readonly string DirectoryPath; private readonly IsolatedStorageFile Storage; + /// <summary> /// Creates a new <see cref="IsolatedStorageDirectory"/> within the specified file using the directory name. /// </summary> /// <param name="storage">A configured and open <see cref="IsolatedStorageFile"/></param> /// <param name="dir">The directory name to open or create within the store</param> + /// <exception cref="ArgumentException"></exception> public IsolatedStorageDirectory(IsolatedStorageFile storage, string dir) { - this.Storage = storage; - this.DirectoryPath = dir; + Storage = storage ?? throw new ArgumentNullException(nameof(storage)); + DirectoryPath = dir ?? throw new ArgumentNullException(nameof(dir)); + //If the directory doesnt exist, create it - if (!this.Storage.DirectoryExists(dir)) - this.Storage.CreateDirectory(dir); + if (!Storage.DirectoryExists(dir)) + { + Storage.CreateDirectory(dir); + } } private IsolatedStorageDirectory(IsolatedStorageDirectory parent, string dirName) @@ -54,9 +59,9 @@ namespace VNLib.Utils.IO //Store ref to parent dir Parent = parent; //Referrence store - this.Storage = parent.Storage; + Storage = parent.Storage; //Add the name of this dir to the end of the specified dir path - this.DirectoryPath = Path.Combine(parent.DirectoryPath, dirName); + DirectoryPath = Path.Combine(parent.DirectoryPath, dirName); } /// <summary> @@ -67,19 +72,15 @@ namespace VNLib.Utils.IO /// <exception cref="IsolatedStorageException"></exception> /// <exception cref="ObjectDisposedException"></exception> /// <exception cref="DirectoryNotFoundException"></exception> - public IsolatedStorageFileStream CreateFile(string fileName) - { - return this.Storage.CreateFile(Path.Combine(DirectoryPath, fileName)); - } + public IsolatedStorageFileStream CreateFile(string fileName) => Storage.CreateFile(GetFullFilePath(fileName)); + /// <summary> /// Removes a file from the current directory /// </summary> /// <param name="fileName">The path of the file to remove</param> /// <exception cref="IsolatedStorageException"></exception> - public void DeleteFile(string fileName) - { - this.Storage.DeleteFile(Path.Combine(this.DirectoryPath, fileName)); - } + public void DeleteFile(string fileName) => Storage.DeleteFile(GetFullFilePath(fileName)); + /// <summary> /// Opens a file that exists within the current directory /// </summary> @@ -87,10 +88,9 @@ namespace VNLib.Utils.IO /// <param name="mode">File mode</param> /// <param name="access">File access</param> /// <returns>The open <see cref="IsolatedStorageFileStream"/> from the current directory</returns> - public IsolatedStorageFileStream OpenFile(string fileName, FileMode mode, FileAccess access) - { - return this.Storage.OpenFile(Path.Combine(DirectoryPath, fileName), mode, access); - } + public IsolatedStorageFileStream OpenFile(string fileName, FileMode mode, FileAccess access) + => Storage.OpenFile(GetFullFilePath(fileName), mode, access); + /// <summary> /// Opens a file that exists within the current directory /// </summary> @@ -99,10 +99,8 @@ namespace VNLib.Utils.IO /// <param name="access">File access</param> /// <param name="share">The file shareing mode</param> /// <returns>The open <see cref="IsolatedStorageFileStream"/> from the current directory</returns> - public IsolatedStorageFileStream OpenFile(string fileName, FileMode mode, FileAccess access, FileShare share) - { - return this.Storage.OpenFile(Path.Combine(DirectoryPath, fileName), mode, access, share); - } + public IsolatedStorageFileStream OpenFile(string fileName, FileMode mode, FileAccess access, FileShare share) + => Storage.OpenFile(GetFullFilePath(fileName), mode, access, share); /// <summary> /// Determiens if the specified file path refers to an existing file within the directory @@ -113,25 +111,22 @@ namespace VNLib.Utils.IO /// <exception cref="ObjectDisposedException"></exception> /// <exception cref="IsolatedStorageException"></exception> /// <exception cref="InvalidOperationException"></exception> - public bool FileExists(string fileName) - { - return this.Storage.FileExists(Path.Combine(this.DirectoryPath, fileName)); - } + public bool FileExists(string fileName) => Storage.FileExists(GetFullFilePath(fileName)); /// <summary> /// Removes the directory and its contents from the store /// </summary> - public override void Remove() - { - Storage.DeleteDirectory(this.DirectoryPath); - } + public override void Remove() => Storage.DeleteDirectory(DirectoryPath); ///<inheritdoc/> public override long AvailableFreeSpace => Storage.AvailableFreeSpace; + ///<inheritdoc/> public override long Quota => Storage.Quota; + ///<inheritdoc/> public override long UsedSize => Storage.UsedSize; + ///<inheritdoc/> public override bool IncreaseQuotaTo(long newQuotaSize) => Storage.IncreaseQuotaTo(newQuotaSize); @@ -139,9 +134,7 @@ namespace VNLib.Utils.IO /// The parent <see cref="IsolatedStorageDirectory"/> this directory is a child within. null if there are no parent directories /// above this dir /// </summary> - public IsolatedStorageDirectory? Parent { get; } -#nullable disable /// <summary> /// Creates a child directory within the current directory @@ -150,9 +143,23 @@ namespace VNLib.Utils.IO /// <returns>A new <see cref="IsolatedStorageDirectory"/> for which <see cref="IsolatedStorageFileStream"/>s can be opened/created</returns> /// <exception cref="ArgumentException"></exception> /// <exception cref="ArgumentNullException"></exception> - public IsolatedStorageDirectory CreateChildDirectory(string directoryName) + public IsolatedStorageDirectory CreateChildDirectory(string directoryName) => new (this, directoryName); + + /// <summary> + /// Gets the IsolatedStorage file path localized to the current directory, including th path + /// of any parent directories. NOTE: it's not possible to get the full filesystem path due to + /// isolated storage security restrictions. + /// </summary> + /// <param name="filePath"> + /// The relative path to the file within the directory to recover the file path from + /// </param> + /// <returns>The localized relative file path within the current directory</returns> + /// <exception cref="ArgumentException"></exception> + public string GetFullFilePath(string filePath) { - return new IsolatedStorageDirectory(this, directoryName); + return Path.IsPathRooted(filePath) + ? throw new ArgumentException("The file path may not be fully rooted, it must be a relative", nameof(filePath)) + : Path.Combine(DirectoryPath, filePath); } } }
\ No newline at end of file diff --git a/lib/Utils/src/IO/VnMemoryStream.cs b/lib/Utils/src/IO/VnMemoryStream.cs index d269134..97cef03 100644 --- a/lib/Utils/src/IO/VnMemoryStream.cs +++ b/lib/Utils/src/IO/VnMemoryStream.cs @@ -56,12 +56,25 @@ namespace VNLib.Utils.IO /// <param name="readOnly">Should the stream be readonly?</param> /// <exception cref="ArgumentException"></exception> /// <returns>A <see cref="VnMemoryStream"/> wrapper to access the handle data</returns> - public static VnMemoryStream ConsumeHandle(MemoryHandle<byte> handle, nint length, bool readOnly) + public static VnMemoryStream ConsumeHandle(MemoryHandle<byte> handle, nint length, bool readOnly) => FromHandle(handle, true, length, readOnly); + + /// <summary> + /// Creates a new <see cref="VnMemoryStream"/> from the supplied memory handle + /// of the initial length. This function also accepts a value that indicates if this stream + /// owns the memory handle, which will cause it to be disposed when the stream is disposed. + /// </summary> + /// <param name="handle"><see cref="MemoryHandle{T}"/> to consume</param> + /// <param name="length">The initial length of the stream</param> + /// <param name="readOnly">Should the stream be readonly?</param> + /// <param name="ownsHandle">A value that indicates if the current stream owns the memory handle</param> + /// <exception cref="ArgumentException"></exception> + /// <returns>A <see cref="VnMemoryStream"/> wrapper to access the handle data</returns> + public static VnMemoryStream FromHandle(MemoryHandle<byte> handle, bool ownsHandle, nint length, bool readOnly) { handle.ThrowIfClosed(); - return new VnMemoryStream(handle, length, readOnly, true); + return new VnMemoryStream(handle, length, readOnly, ownsHandle); } - + /// <summary> /// Converts a writable <see cref="VnMemoryStream"/> to readonly to allow shallow copies /// </summary> @@ -258,6 +271,7 @@ namespace VNLib.Utils.IO /// </para> /// </summary> public override bool CanRead => true; + /// <summary> /// <inheritdoc/> /// <para> @@ -265,13 +279,16 @@ namespace VNLib.Utils.IO /// </para> /// </summary> public override bool CanSeek => true; + /// <summary> /// True unless the stream is (or has been converted to) a readonly /// stream. /// </summary> public override bool CanWrite => !_isReadonly; + ///<inheritdoc/> public override long Length => _length; + ///<inheritdoc/> public override bool CanTimeout => false; @@ -281,6 +298,7 @@ namespace VNLib.Utils.IO get => _position; set => Seek(value, SeekOrigin.Begin); } + /// <summary> /// Closes the stream and frees the internal allocated memory blocks /// </summary> @@ -292,13 +310,17 @@ namespace VNLib.Utils.IO _buffer.Dispose(); } } + ///<inheritdoc/> public override void Flush() { } + // Override to reduce base class overhead ///<inheritdoc/> public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + ///<inheritdoc/> public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count)); + ///<inheritdoc/> public override int Read(Span<byte> buffer) { @@ -317,6 +339,7 @@ namespace VNLib.Utils.IO return bytesToRead; } + ///<inheritdoc/> public override unsafe int ReadByte() { @@ -346,6 +369,7 @@ namespace VNLib.Utils.IO int read = Read(buffer.Span); return ValueTask.FromResult(read); } + ///<inheritdoc/> public override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { @@ -353,6 +377,7 @@ namespace VNLib.Utils.IO int read = Read(buffer.AsSpan(offset, count)); return Task.FromResult(read); } + ///<inheritdoc/> public override long Seek(long offset, SeekOrigin origin) { @@ -384,8 +409,7 @@ namespace VNLib.Utils.IO default: throw new ArgumentException("Stream operation is not supported on current stream"); } - } - + } /// <summary> /// Resizes the internal buffer to the exact size (in bytes) of the @@ -423,8 +447,10 @@ namespace VNLib.Utils.IO //Make sure the position is not pointing outside of the buffer after resize _position = Math.Min(_position, _length); } + ///<inheritdoc/> public override void Write(byte[] buffer, int offset, int count) => Write(buffer.AsSpan(offset, count)); + ///<inheritdoc/> public override void Write(ReadOnlySpan<byte> buffer) { @@ -447,6 +473,7 @@ namespace VNLib.Utils.IO //Update the position _position = newPos; } + ///<inheritdoc/> public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { @@ -454,6 +481,7 @@ namespace VNLib.Utils.IO Write(buffer, offset, count); return Task.CompletedTask; } + ///<inheritdoc/> public override ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default) { @@ -461,6 +489,7 @@ namespace VNLib.Utils.IO Write(buffer.Span); return ValueTask.CompletedTask; } + ///<inheritdoc/> public override void WriteByte(byte value) { @@ -504,19 +533,5 @@ namespace VNLib.Utils.IO /// <exception cref="NotSupportedException"></exception> public object Clone() => GetReadonlyShallowCopy(); - /* - * Override the Dispose async method to avoid the base class overhead - * and task allocation since this will always be a syncrhonous - * operation (freeing memory) - */ - - ///<inheritdoc/> - public override ValueTask DisposeAsync() - { - //Dispose and return completed task - base.Dispose(true); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } } }
\ No newline at end of file diff --git a/lib/Utils/src/IO/VnStreamWriter.cs b/lib/Utils/src/IO/VnStreamWriter.cs index 03b0a6e..5ec65fc 100644 --- a/lib/Utils/src/IO/VnStreamWriter.cs +++ b/lib/Utils/src/IO/VnStreamWriter.cs @@ -33,6 +33,8 @@ using System.Runtime.CompilerServices; using VNLib.Utils.Memory; +#pragma warning disable CA2215, CA2007, CS8765 // Dispose methods should call base class dispose + namespace VNLib.Utils.IO { /// <summary> diff --git a/lib/Utils/src/Memory/HeapCreation.cs b/lib/Utils/src/Memory/HeapCreation.cs index 2d30c29..9ef9fdb 100644 --- a/lib/Utils/src/Memory/HeapCreation.cs +++ b/lib/Utils/src/Memory/HeapCreation.cs @@ -49,6 +49,6 @@ namespace VNLib.Utils.Memory /// <summary> /// Specifies that the requested heap will be a shared heap for the process/library /// </summary> - IsSharedHeap = 0x04 + Shared = 0x04 } }
\ No newline at end of file diff --git a/lib/Utils/src/Memory/MemoryUtil.cs b/lib/Utils/src/Memory/MemoryUtil.cs index 6af2392..8cc9736 100644 --- a/lib/Utils/src/Memory/MemoryUtil.cs +++ b/lib/Utils/src/Memory/MemoryUtil.cs @@ -109,8 +109,8 @@ namespace VNLib.Utils.Memory _ = ERRNO.TryParse(Environment.GetEnvironmentVariable(SHARED_HEAP_ENABLE_DIAGNOISTICS_ENV), out ERRNO diagEnable); _ = ERRNO.TryParse(Environment.GetEnvironmentVariable(SHARED_HEAP_GLOBAL_ZERO), out ERRNO globalZero); - Trace.WriteIf(diagEnable, "Shared heap diagnostics enabled"); - Trace.WriteIf(globalZero, "Shared heap global zero enabled"); + Trace.WriteLineIf(diagEnable, "Shared heap diagnostics enabled"); + Trace.WriteLineIf(globalZero, "Shared heap global zero enabled"); Lazy<IUnmangedHeap> heap = new (() => InitHeapInternal(true, diagEnable, globalZero), LazyThreadSafetyMode.PublicationOnly); @@ -173,7 +173,7 @@ namespace VNLib.Utils.Memory * The heap impl may reset the synchronziation flag if it does not * need serialziation */ - cFlags |= isShared ? HeapCreation.IsSharedHeap : HeapCreation.None; + cFlags |= isShared ? HeapCreation.Shared : HeapCreation.None; //Set global zero flag if requested cFlags |= globalZero ? HeapCreation.GlobalZero : HeapCreation.None; diff --git a/lib/Utils/src/Memory/NativeHeap.cs b/lib/Utils/src/Memory/NativeHeap.cs index 4b76866..7a3d4dd 100644 --- a/lib/Utils/src/Memory/NativeHeap.cs +++ b/lib/Utils/src/Memory/NativeHeap.cs @@ -177,9 +177,9 @@ namespace VNLib.Utils.Memory { public IntPtr HeapPointer; - public HeapCreation CreationFlags; - public ERRNO Flags; + + public HeapCreation CreationFlags; } readonly record struct HeapMethods diff --git a/lib/Utils/src/Memory/ProcessHeap.cs b/lib/Utils/src/Memory/ProcessHeap.cs index 84c9e0e..3d581cd 100644 --- a/lib/Utils/src/Memory/ProcessHeap.cs +++ b/lib/Utils/src/Memory/ProcessHeap.cs @@ -45,11 +45,11 @@ namespace VNLib.Utils.Memory /// <summary> /// <inheritdoc/> /// <para> - /// Is always <see cref="HeapCreation.IsSharedHeap"/> as this heap is the default + /// Is always <see cref="HeapCreation.Shared"/> as this heap is the default /// process heap. Meaining memory will be shared across the process /// </para> /// </summary> - public HeapCreation CreationFlags { get; } = HeapCreation.IsSharedHeap; + public HeapCreation CreationFlags { get; } = HeapCreation.Shared; /// <summary> /// Initalizes a new global (cross platform) process heap diff --git a/lib/Utils/src/Memory/Win32PrivateHeap.cs b/lib/Utils/src/Memory/Win32PrivateHeap.cs index 0a00a77..41fe33a 100644 --- a/lib/Utils/src/Memory/Win32PrivateHeap.cs +++ b/lib/Utils/src/Memory/Win32PrivateHeap.cs @@ -110,7 +110,7 @@ namespace VNLib.Utils.Memory /// </remarks> public static Win32PrivateHeap Create(nuint initialSize, HeapCreation cFlags, nuint maxHeapSize = 0, DWORD flags = HEAP_NO_FLAGS) { - if (cFlags.HasFlag(HeapCreation.IsSharedHeap)) + if (cFlags.HasFlag(HeapCreation.Shared)) { //Clear the synchronization flag because we don't need it for a process heap cFlags &= ~HeapCreation.UseSynchronization; diff --git a/lib/Utils/src/VnEncoding.cs b/lib/Utils/src/VnEncoding.cs index e945135..e24f7df 100644 --- a/lib/Utils/src/VnEncoding.cs +++ b/lib/Utils/src/VnEncoding.cs @@ -51,10 +51,15 @@ namespace VNLib.Utils /// </summary> /// <param name="data">Data to be encoded</param> /// <param name="encoding"><see cref="Encoding"/> to encode data with</param> + /// <param name="heap">Heap to allocate memory from</param> /// <returns>A <see cref="Stream"/> contating the encoded data</returns> - public static VnMemoryStream GetMemoryStream(ReadOnlySpan<char> data, Encoding encoding) + public static VnMemoryStream GetMemoryStream(ReadOnlySpan<char> data, Encoding encoding, IUnmangedHeap? heap = null) { _ = encoding ?? throw new ArgumentNullException(nameof(encoding)); + + //Assign default heap if not specified + heap ??= MemoryUtil.Shared; + //Create new memory handle to copy data to MemoryHandle<byte>? handle = null; try @@ -62,7 +67,7 @@ namespace VNLib.Utils //get number of bytes int byteCount = encoding.GetByteCount(data); //resize the handle to fit the data - handle = MemoryUtil.Shared.Alloc<byte>(byteCount); + handle = heap.Alloc<byte>(byteCount); //encode int size = encoding.GetBytes(data, handle); //Consume the handle into a new vnmemstream and return it diff --git a/lib/Utils/tests/IO/VnMemoryStreamTests.cs b/lib/Utils/tests/IO/VnMemoryStreamTests.cs index 3eb95ce..9742197 100644 --- a/lib/Utils/tests/IO/VnMemoryStreamTests.cs +++ b/lib/Utils/tests/IO/VnMemoryStreamTests.cs @@ -23,7 +23,7 @@ namespace VNLib.Utils.IO.Tests } //Test heap - IUnmangedHeap privateHeap = MemoryUtil.InitializeNewHeapForProcess(); + using IUnmangedHeap privateHeap = MemoryUtil.InitializeNewHeapForProcess(); using (VnMemoryStream vms = new(privateHeap, 1024, false)) { @@ -38,7 +38,7 @@ namespace VNLib.Utils.IO.Tests //Create from mem handle MemoryHandle<byte> handle = privateHeap.Alloc<byte>(byte.MaxValue); - using (VnMemoryStream vms = VnMemoryStream.ConsumeHandle(handle, handle.GetIntLength(), false)) + using (VnMemoryStream vms = VnMemoryStream.FromHandle(handle, true, handle.GetIntLength(), false)) { Assert.IsTrue(vms.Length == byte.MaxValue); Assert.IsTrue(vms.Position == 0); diff --git a/lib/Utils/tests/Memory/NativeHeapTests.cs b/lib/Utils/tests/Memory/NativeHeapTests.cs index dde12cd..291faa5 100644 --- a/lib/Utils/tests/Memory/NativeHeapTests.cs +++ b/lib/Utils/tests/Memory/NativeHeapTests.cs @@ -10,10 +10,10 @@ namespace VNLib.Utils.Memory.Tests const string RpMallocLibPath = "../../../../../Utils.Memory/vnlib_rpmalloc/build/Debug/vnlib_rpmalloc.dll"; [TestMethod()] - public void LoadHeapTest() + public void LoadInTreeRpmallocTest() { - //Try to load the global heap - using NativeHeap heap = NativeHeap.LoadHeap(RpMallocLibPath, System.Runtime.InteropServices.DllImportSearchPath.SafeDirectories, HeapCreation.None, 0); + //Try to load the shared heap + using NativeHeap heap = NativeHeap.LoadHeap(RpMallocLibPath, System.Runtime.InteropServices.DllImportSearchPath.SafeDirectories, HeapCreation.Shared, 0); Assert.IsFalse(heap.IsInvalid); @@ -26,7 +26,6 @@ namespace VNLib.Utils.Memory.Tests //confirm the pointer it zeroed Assert.IsTrue(block == IntPtr.Zero); - } } }
\ No newline at end of file |