diff options
| author |  vnugent <public@vaughnnugent.com> | 2024-07-04 16:20:48 -0400 |
|---|---|---|
| committer | vnugent <public@vaughnnugent.com> | 2024-07-04 16:20:48 -0400 |
| commit | 38ad7d923fa8d9e463d4aaa8e35f021086a03f2d (patch) | |
| tree | f1f7e95ddb10f3411b7da59b9bd1d5a986a1087a | |
| parent | 981ba286e4793de95bf65e6588313411344c4d53 (diff) | |
mimalloc merge upstream upgrades
39 files changed, 2247 insertions, 2393 deletions
diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/.gitattributes b/lib/Utils.Memory/vnlib_mimalloc/vendor/.gitattributes deleted file mode 100644 index f083b10..0000000 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/.gitattributes +++ /dev/null @@ -1,13 +0,0 @@ -# default behavior is to always use unix style line endings -* text eol=lf -*.png binary -*.pdn binary -*.jpg binary -*.sln binary -*.suo binary -*.vcproj binary -*.patch binary -*.dll binary -*.lib binary -*.exe binary -bin export-ignore diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/.gitignore b/lib/Utils.Memory/vnlib_mimalloc/vendor/.gitignore deleted file mode 100644 index f8b7f5e..0000000 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -ide/vs20??/*.db -ide/vs20??/*.opendb -ide/vs20??/*.user -ide/vs20??/*.vcxproj.filters -ide/vs20??/.vs -ide/vs20??/VTune* -out/ -docs/ -*.zip diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/CMakeLists.txt b/lib/Utils.Memory/vnlib_mimalloc/vendor/CMakeLists.txt index 2cc2fc4..7a73055 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/CMakeLists.txt +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.13) +cmake_minimum_required(VERSION 3.18) project(libmimalloc C CXX) set(CMAKE_C_STANDARD 11) @@ -35,6 +35,7 @@ option(MI_NO_THP "Disable transparent huge pages support on Linux/And option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) option(MI_USE_LIBATOMIC "Explicitly link with -latomic (on older systems) (deprecated and detected automatically)" OFF) +include(CheckLinkerFlag) # requires cmake 3.18 include(CheckIncludeFiles) include(GNUInstallDirs) include("cmake/mimalloc-config-version.cmake") @@ -60,7 +61,7 @@ set(mi_sources set(mi_cflags "") set(mi_cflags_static "") # extra flags for a static library build set(mi_cflags_dynamic "") # extra flags for a shared-object library build -set(mi_defines "") +set(mi_defines "") set(mi_libraries "") # ----------------------------------------------------------------------------- @@ -257,6 +258,7 @@ if(MI_DEBUG_UBSAN) if(CMAKE_BUILD_TYPE MATCHES "Debug") if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") message(STATUS "Build with undefined-behavior sanitizer (MI_DEBUG_UBSAN=ON)") + list(APPEND mi_defines MI_UBSAN=1) list(APPEND mi_cflags -fsanitize=undefined -g -fno-sanitize-recover=undefined) list(APPEND mi_libraries -fsanitize=undefined) if (NOT MI_USE_CXX) @@ -326,7 +328,7 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM list(APPEND mi_cflags_dynamic -ftls-model=initial-exec) message(STATUS "Use local dynamic TLS for the static build (since MI_LIBC_MUSL=ON)") else() - list(APPEND mi_cflags -ftls-model=initial-exec) + list(APPEND mi_cflags -ftls-model=initial-exec) endif() endif() if(MI_OVERRIDE) @@ -338,29 +340,45 @@ if (MSVC AND MSVC_VERSION GREATER_EQUAL 1914) list(APPEND mi_cflags /Zc:__cplusplus) endif() +if(MINGW) + add_definitions(-D_WIN32_WINNT=0x600) +endif() + # extra needed libraries + +# we prefer -l<lib> test over `find_library` as sometimes core libraries +# like `libatomic` are not on the system path (see issue #898) +function(find_link_library libname outlibname) + check_linker_flag(C "-l${libname}" mi_has_lib${libname}) + if (mi_has_lib${libname}) + message(VERBOSE "link library: -l${libname}") + set(${outlibname} ${libname} PARENT_SCOPE) + else() + find_library(MI_LIBPATH libname) + if (MI_LIBPATH) + message(VERBOSE "link library ${libname} at ${MI_LIBPATH}") + set(${outlibname} ${MI_LIBPATH} PARENT_SCOPE) + else() + message(VERBOSE "link library not found: ${libname}") + set(${outlibname} "" PARENT_SCOPE) + endif() + endif() +endfunction() + if(WIN32) list(APPEND mi_libraries psapi shell32 user32 advapi32 bcrypt) - set(pc_libraries "-lpsapi -lshell32 -luser32 -ladvapi32 -lbcrypt") else() - set(pc_libraries "") - find_library(MI_LIBPTHREAD pthread) - if (MI_LIBPTHREAD) - list(APPEND mi_libraries ${MI_LIBPTHREAD}) - set(pc_libraries "${pc_libraries} -pthread") - endif() - find_library(MI_LIBRT rt) - if(MI_LIBRT) - list(APPEND mi_libraries ${MI_LIBRT}) - set(pc_libraries "${pc_libraries} -lrt") + find_link_library("pthread" MI_LIB_PTHREAD) + if(MI_LIB_PTHREAD) + list(APPEND mi_libraries "${MI_LIB_PTHREAD}") endif() - find_library(MI_LIBATOMIC atomic) - if (NOT MI_LIBATOMIC AND MI_USE_LIBATOMIC) - set(MI_LIBATOMIC atomic) + find_link_library("rt" MI_LIB_RT) + if(MI_LIB_RT) + list(APPEND mi_libraries "${MI_LIB_RT}") endif() - if (MI_LIBATOMIC) - list(APPEND mi_libraries ${MI_LIBATOMIC}) - set(pc_libraries "${pc_libraries} -latomic") + find_link_library("atomic" MI_LIB_ATOMIC) + if(MI_LIB_ATOMIC) + list(APPEND mi_libraries "${MI_LIB_ATOMIC}") endif() endif() @@ -369,7 +387,8 @@ endif() # ----------------------------------------------------------------------------- # dynamic/shared library and symlinks always go to /usr/local/lib equivalent -set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}") +set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}") +set(mi_install_bindir "${CMAKE_INSTALL_BINDIR}") # static libraries and object files, includes, and cmake config files # are either installed at top level, or use versioned directories for side-by-side installation (default) @@ -453,10 +472,10 @@ if(MI_BUILD_SHARED) add_custom_command(TARGET mimalloc POST_BUILD COMMAND "${CMAKE_COMMAND}" -E copy "${CMAKE_CURRENT_SOURCE_DIR}/bin/mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll" $<TARGET_FILE_DIR:mimalloc> COMMENT "Copy mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll to output directory") - install(FILES "$<TARGET_FILE_DIR:mimalloc>/mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll" DESTINATION ${mi_install_libdir}) + install(FILES "$<TARGET_FILE_DIR:mimalloc>/mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll" DESTINATION ${mi_install_bindir}) endif() - install(TARGETS mimalloc EXPORT mimalloc DESTINATION ${mi_install_libdir} LIBRARY) + install(TARGETS mimalloc EXPORT mimalloc ARCHIVE DESTINATION ${mi_install_libdir} RUNTIME DESTINATION ${mi_install_bindir} LIBRARY DESTINATION ${mi_install_libdir}) install(EXPORT mimalloc DESTINATION ${mi_install_cmakedir}) endif() @@ -522,6 +541,15 @@ if (MI_BUILD_OBJECT) endif() # pkg-config file support +set(pc_libraries "") +foreach(item IN LISTS mi_libraries) + if(item MATCHES " *[-].*") + set(pc_libraries "${pc_libraries} ${item}") + else() + set(pc_libraries "${pc_libraries} -l${item}") + endif() +endforeach() + include("cmake/JoinPaths.cmake") join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}") join_paths(libdir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_LIBDIR}") @@ -530,6 +558,8 @@ configure_file(mimalloc.pc.in mimalloc.pc @ONLY) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/mimalloc.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/") + + # ----------------------------------------------------------------------------- # API surface testing # ----------------------------------------------------------------------------- diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/azure-pipelines.yml b/lib/Utils.Memory/vnlib_mimalloc/vendor/azure-pipelines.yml deleted file mode 100644 index 0247c76..0000000 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/azure-pipelines.yml +++ /dev/null @@ -1,197 +0,0 @@ -# Starter pipeline -# Start with a minimal pipeline that you can customize to build and deploy your code. -# Add steps that build, run tests, deploy, and more: -# https://aka.ms/yaml - -trigger: - branches: - include: - - master - - dev - - dev-slice - tags: - include: - - v* - -jobs: -- job: - displayName: Windows - pool: - vmImage: - windows-2022 - strategy: - matrix: - Debug: - BuildType: debug - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON - MSBuildConfiguration: Debug - Release: - BuildType: release - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release - MSBuildConfiguration: Release - Secure: - BuildType: secure - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON - MSBuildConfiguration: Release - steps: - - task: CMake@1 - inputs: - workingDirectory: $(BuildType) - cmakeArgs: .. $(cmakeExtraArgs) - - task: MSBuild@1 - inputs: - solution: $(BuildType)/libmimalloc.sln - configuration: '$(MSBuildConfiguration)' - msbuildArguments: -m - - script: ctest --verbose --timeout 120 -C $(MSBuildConfiguration) - workingDirectory: $(BuildType) - displayName: CTest - #- script: $(BuildType)\$(BuildType)\mimalloc-test-stress - # displayName: TestStress - #- upload: $(Build.SourcesDirectory)/$(BuildType) - # artifact: mimalloc-windows-$(BuildType) - -- job: - displayName: Linux - pool: - vmImage: - ubuntu-22.04 - strategy: - matrix: - Debug: - CC: gcc - CXX: g++ - BuildType: debug - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON - Release: - CC: gcc - CXX: g++ - BuildType: release - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release - Secure: - CC: gcc - CXX: g++ - BuildType: secure - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON - Debug++: - CC: gcc - CXX: g++ - BuildType: debug-cxx - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON - Debug Clang: - CC: clang - CXX: clang++ - BuildType: debug-clang - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON - Release Clang: - CC: clang - CXX: clang++ - BuildType: release-clang - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release - Secure Clang: - CC: clang - CXX: clang++ - BuildType: secure-clang - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON - Debug++ Clang: - CC: clang - CXX: clang++ - BuildType: debug-clang-cxx - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON - Debug ASAN Clang: - CC: clang - CXX: clang++ - BuildType: debug-asan-clang - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_TRACK_ASAN=ON - Debug UBSAN Clang: - CC: clang - CXX: clang++ - BuildType: debug-ubsan-clang - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_DEBUG_UBSAN=ON - Debug TSAN Clang++: - CC: clang - CXX: clang++ - BuildType: debug-tsan-clang-cxx - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_USE_CXX=ON -DMI_DEBUG_TSAN=ON - - steps: - - task: CMake@1 - inputs: - workingDirectory: $(BuildType) - cmakeArgs: .. $(cmakeExtraArgs) - - script: make -j$(nproc) -C $(BuildType) - displayName: Make - - script: ctest --verbose --timeout 180 - workingDirectory: $(BuildType) - displayName: CTest -# - upload: $(Build.SourcesDirectory)/$(BuildType) -# artifact: mimalloc-ubuntu-$(BuildType) - -- job: - displayName: macOS - pool: - vmImage: - macOS-latest - strategy: - matrix: - Debug: - BuildType: debug - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON - Release: - BuildType: release - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release - Secure: - BuildType: secure - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON - steps: - - task: CMake@1 - inputs: - workingDirectory: $(BuildType) - cmakeArgs: .. $(cmakeExtraArgs) - - script: make -j$(sysctl -n hw.ncpu) -C $(BuildType) - displayName: Make - # - script: MIMALLOC_VERBOSE=1 ./mimalloc-test-api - # workingDirectory: $(BuildType) - # displayName: TestAPI - # - script: MIMALLOC_VERBOSE=1 ./mimalloc-test-stress - # workingDirectory: $(BuildType) - # displayName: TestStress - - script: ctest --verbose --timeout 120 - workingDirectory: $(BuildType) - displayName: CTest - -# - upload: $(Build.SourcesDirectory)/$(BuildType) -# artifact: mimalloc-macos-$(BuildType) - -# - job: -# displayName: Windows-2017 -# pool: -# vmImage: -# vs2017-win2016 -# strategy: -# matrix: -# Debug: -# BuildType: debug -# cmakeExtraArgs: -A x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -# MSBuildConfiguration: Debug -# Release: -# BuildType: release -# cmakeExtraArgs: -A x64 -DCMAKE_BUILD_TYPE=Release -# MSBuildConfiguration: Release -# Secure: -# BuildType: secure -# cmakeExtraArgs: -A x64 -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON -# MSBuildConfiguration: Release -# steps: -# - task: CMake@1 -# inputs: -# workingDirectory: $(BuildType) -# cmakeArgs: .. $(cmakeExtraArgs) -# - task: MSBuild@1 -# inputs: -# solution: $(BuildType)/libmimalloc.sln -# configuration: '$(MSBuildConfiguration)' -# - script: | -# cd $(BuildType) -# ctest --verbose --timeout 120 -# displayName: CTest diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/cmake/mimalloc-config-version.cmake b/lib/Utils.Memory/vnlib_mimalloc/vendor/cmake/mimalloc-config-version.cmake index 9b19b56..f92d52e 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/cmake/mimalloc-config-version.cmake +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/cmake/mimalloc-config-version.cmake @@ -1,6 +1,6 @@ -set(mi_version_major 2) -set(mi_version_minor 1) -set(mi_version_patch 6) +set(mi_version_major 1) +set(mi_version_minor 8) +set(mi_version_patch 8) set(mi_version ${mi_version_major}.${mi_version_minor}) set(PACKAGE_VERSION ${mi_version}) diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc-override.h b/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc-override.h index c63b0b9..48a8a62 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc-override.h +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc-override.h @@ -24,7 +24,7 @@ not accidentally mix pointers from different allocators). #define free(p) mi_free(p) #define strdup(s) mi_strdup(s) -#define strndup(s,n) mi_strndup(s,n) +#define strndup(s,n) mi_strndup(s,n) #define realpath(f,n) mi_realpath(f,n) // Microsoft extensions @@ -43,6 +43,7 @@ not accidentally mix pointers from different allocators). #define reallocf(p,n) mi_reallocf(p,n) #define malloc_size(p) mi_usable_size(p) #define malloc_usable_size(p) mi_usable_size(p) +#define malloc_good_size(sz) mi_malloc_good_size(sz) #define cfree(p) mi_free(p) #define valloc(n) mi_valloc(n) diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc.h b/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc.h index 8446d99..bc743fd 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc.h +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 216 // major + 2 digits minor +#define MI_MALLOC_VERSION 188 // major + 2 digits minor // ------------------------------------------------------ // Compiler specific attributes @@ -97,7 +97,6 @@ terms of the MIT license. A copy of the license can be found in the file #include <stddef.h> // size_t #include <stdbool.h> // bool -#include <stdint.h> // INTPTR_MAX #ifdef __cplusplus extern "C" { @@ -259,11 +258,12 @@ typedef struct mi_heap_area_s { size_t used; // number of allocated blocks size_t block_size; // size in bytes of each block size_t full_block_size; // size in bytes of a full block including padding and metadata. + int heap_tag; // heap tag associated with this area } mi_heap_area_t; typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg); -mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block_visit_fun* visitor, void* arg); +mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); // Experimental mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept; @@ -289,8 +289,25 @@ mi_decl_export bool mi_manage_os_memory_ex(void* start, size_t size, bool is_co mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id); #endif + +// Experimental: allow sub-processes whose memory segments stay separated (and no reclamation between them) +// Used for example for separate interpreter's in one process. +typedef void* mi_subproc_id_t; +mi_decl_export mi_subproc_id_t mi_subproc_main(void); +mi_decl_export mi_subproc_id_t mi_subproc_new(void); +mi_decl_export void mi_subproc_delete(mi_subproc_id_t subproc); +mi_decl_export void mi_subproc_add_current_thread(mi_subproc_id_t subproc); // this should be called right after a thread is created (and no allocation has taken place yet) + +// Experimental: visit abandoned heap areas (from threads that have been terminated) +mi_decl_export bool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); + +// Experimental: create a new heap with a specified heap tag. Set `allow_destroy` to false to allow the thread +// to reclaim abandoned memory (with a compatible heap_tag and arena_id) but in that case `mi_heap_destroy` will +// fall back to `mi_heap_delete`. +mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_ex(int heap_tag, bool allow_destroy, mi_arena_id_t arena_id); + // deprecated -mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept; +mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept; // ------------------------------------------------------ @@ -328,7 +345,7 @@ typedef enum mi_option_e { mi_option_allow_large_os_pages, // allow large (2 or 4 MiB) OS pages, implies eager commit. If false, also disables THP for the process. mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB pages) at startup mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node - mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup + mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use `mi_option_get_size`) mi_option_deprecated_segment_cache, mi_option_deprecated_page_reset, mi_option_abandoned_page_purge, // immediately purge delayed purges on thread termination @@ -342,11 +359,13 @@ typedef enum mi_option_e { mi_option_max_warnings, // issue at most N warning messages mi_option_max_segment_reclaim, // max. percentage of the abandoned segments can be reclaimed per try (=10%) mi_option_destroy_on_exit, // if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe - mi_option_arena_reserve, // initial memory size in KiB for arena reservation (= 1 GiB on 64-bit) + mi_option_arena_reserve, // initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use `mi_option_get_size`) mi_option_arena_purge_mult, // multiplier for `purge_delay` for the purging delay for arenas (=10) mi_option_purge_extend_delay, mi_option_abandoned_reclaim_on_free, // allow to reclaim an abandoned segment on a free (=1) mi_option_disallow_arena_alloc, // 1 = do not use arena's for allocation (except if using specific arena id's) + mi_option_retry_on_oom, // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows) + mi_option_visit_abandoned, // allow visiting heap blocks from abandoned threads (=0) _mi_option_last, // legacy option names mi_option_large_os_pages = mi_option_allow_large_os_pages, @@ -516,7 +535,7 @@ template<class T, bool _mi_destroy> struct _mi_heap_stl_allocator_common : publi protected: std::shared_ptr<mi_heap_t> heap; template<class U, bool D> friend struct _mi_heap_stl_allocator_common; - + _mi_heap_stl_allocator_common() { mi_heap_t* hp = mi_heap_new(); this->heap.reset(hp, (_mi_destroy ? &heap_destroy : &heap_delete)); /* calls heap_delete/destroy when the refcount drops to zero */ @@ -533,7 +552,7 @@ private: template<class T> struct mi_heap_stl_allocator : public _mi_heap_stl_allocator_common<T, false> { using typename _mi_heap_stl_allocator_common<T, false>::size_type; mi_heap_stl_allocator() : _mi_heap_stl_allocator_common<T, false>() { } // creates fresh heap that is deleted when the destructor is called - mi_heap_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, false>(hp) { } // no delete nor destroy on the passed in heap + mi_heap_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, false>(hp) { } // no delete nor destroy on the passed in heap template<class U> mi_heap_stl_allocator(const mi_heap_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, false>(x) { } mi_heap_stl_allocator select_on_container_copy_construction() const { return *this; } @@ -550,7 +569,7 @@ template<class T1, class T2> bool operator!=(const mi_heap_stl_allocator<T1>& x, template<class T> struct mi_heap_destroy_stl_allocator : public _mi_heap_stl_allocator_common<T, true> { using typename _mi_heap_stl_allocator_common<T, true>::size_type; mi_heap_destroy_stl_allocator() : _mi_heap_stl_allocator_common<T, true>() { } // creates fresh heap that is destroyed when the destructor is called - mi_heap_destroy_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, true>(hp) { } // no delete nor destroy on the passed in heap + mi_heap_destroy_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, true>(hp) { } // no delete nor destroy on the passed in heap template<class U> mi_heap_destroy_stl_allocator(const mi_heap_destroy_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, true>(x) { } mi_heap_destroy_stl_allocator select_on_container_copy_construction() const { return *this; } diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/atomic.h b/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/atomic.h index 807c4da..3a0d489 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/atomic.h +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/atomic.h @@ -8,6 +8,17 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_ATOMIC_H #define MIMALLOC_ATOMIC_H +// include windows.h or pthreads.h +#if defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include <windows.h> +#elif !defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__)) +#define MI_USE_PTHREADS +#include <pthread.h> +#endif + // -------------------------------------------------------------------------------------------- // Atomics // We need to be portable between C, C++, and MSVC. @@ -24,9 +35,9 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_atomic(name) std::atomic_##name #define mi_memory_order(name) std::memory_order_##name #if (__cplusplus >= 202002L) // c++20, see issue #571 -#define MI_ATOMIC_VAR_INIT(x) x + #define MI_ATOMIC_VAR_INIT(x) x #elif !defined(ATOMIC_VAR_INIT) -#define MI_ATOMIC_VAR_INIT(x) x + #define MI_ATOMIC_VAR_INIT(x) x #else #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) #endif @@ -132,11 +143,7 @@ static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) { #elif defined(_MSC_VER) -// MSVC C compilation wrapper that uses Interlocked operations to model C11 atomics. -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif -#include <windows.h> +// Legacy MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics. #include <intrin.h> #ifdef _WIN64 typedef LONG64 msc_intptr_t; @@ -201,7 +208,7 @@ static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_ #else uintptr_t x = *p; if (mo > mi_memory_order_relaxed) { - while (!mi_atomic_compare_exchange_weak_explicit(p, &x, x, mo, mi_memory_order_relaxed)) { /* nothing */ }; + while (!mi_atomic_compare_exchange_weak_explicit((_Atomic(uintptr_t)*)p, &x, x, mo, mi_memory_order_relaxed)) { /* nothing */ }; } return x; #endif @@ -302,11 +309,16 @@ static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) { return (intptr_t)mi_atomic_addi(p, -sub); } + +// ---------------------------------------------------------------------- +// Once and Guard +// ---------------------------------------------------------------------- + typedef _Atomic(uintptr_t) mi_atomic_once_t; // Returns true only on the first invocation static inline bool mi_atomic_once( mi_atomic_once_t* once ) { - if (mi_atomic_load_relaxed(once) != 0) return false; // quick test + if (mi_atomic_load_relaxed(once) != 0) return false; // quick test uintptr_t expected = 0; return mi_atomic_cas_strong_acq_rel(once, &expected, (uintptr_t)1); // try to set to 1 } @@ -322,17 +334,16 @@ typedef _Atomic(uintptr_t) mi_atomic_guard_t; +// ---------------------------------------------------------------------- // Yield +// ---------------------------------------------------------------------- + #if defined(__cplusplus) #include <thread> static inline void mi_atomic_yield(void) { std::this_thread::yield(); } #elif defined(_WIN32) -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif -#include <windows.h> static inline void mi_atomic_yield(void) { YieldProcessor(); } @@ -390,4 +401,107 @@ static inline void mi_atomic_yield(void) { #endif +// ---------------------------------------------------------------------- +// Locks are only used for abandoned segment visiting in `arena.c` +// ---------------------------------------------------------------------- + +#if defined(_WIN32) + +#define mi_lock_t CRITICAL_SECTION + +static inline bool mi_lock_try_acquire(mi_lock_t* lock) { + return TryEnterCriticalSection(lock); +} +static inline bool mi_lock_acquire(mi_lock_t* lock) { + EnterCriticalSection(lock); + return true; +} +static inline void mi_lock_release(mi_lock_t* lock) { + LeaveCriticalSection(lock); +} +static inline void mi_lock_init(mi_lock_t* lock) { + InitializeCriticalSection(lock); +} +static inline void mi_lock_done(mi_lock_t* lock) { + DeleteCriticalSection(lock); +} + + +#elif defined(MI_USE_PTHREADS) + +#define mi_lock_t pthread_mutex_t + +static inline bool mi_lock_try_acquire(mi_lock_t* lock) { + return (pthread_mutex_trylock(lock) == 0); +} +static inline bool mi_lock_acquire(mi_lock_t* lock) { + return (pthread_mutex_lock(lock) == 0); +} +static inline void mi_lock_release(mi_lock_t* lock) { + pthread_mutex_unlock(lock); +} +static inline void mi_lock_init(mi_lock_t* lock) { + pthread_mutex_init(lock, NULL); +} +static inline void mi_lock_done(mi_lock_t* lock) { + pthread_mutex_destroy(lock); +} + +/* +#elif defined(__cplusplus) + +#include <mutex> +#define mi_lock_t std::mutex + +static inline bool mi_lock_try_acquire(mi_lock_t* lock) { + return lock->lock_try_acquire(); +} +static inline bool mi_lock_acquire(mi_lock_t* lock) { + lock->lock(); + return true; +} +static inline void mi_lock_release(mi_lock_t* lock) { + lock->unlock(); +} +static inline void mi_lock_init(mi_lock_t* lock) { + (void)(lock); +} +static inline void mi_lock_done(mi_lock_t* lock) { + (void)(lock); +} +*/ + +#else + +// fall back to poor man's locks. +// this should only be the case in a single-threaded environment (like __wasi__) + +#define mi_lock_t _Atomic(uintptr_t) + +static inline bool mi_lock_try_acquire(mi_lock_t* lock) { + uintptr_t expected = 0; + return mi_atomic_cas_strong_acq_rel(lock, &expected, (uintptr_t)1); +} +static inline bool mi_lock_acquire(mi_lock_t* lock) { + for (int i = 0; i < 1000; i++) { // for at most 1000 tries? + if (mi_lock_try_acquire(lock)) return true; + mi_atomic_yield(); + } + return true; +} +static inline void mi_lock_release(mi_lock_t* lock) { + mi_atomic_store_release(lock, (uintptr_t)0); +} +static inline void mi_lock_init(mi_lock_t* lock) { + mi_lock_release(lock); +} +static inline void mi_lock_done(mi_lock_t* lock) { + (void)(lock); +} + +#endif + + + + #endif // __MIMALLOC_ATOMIC_H diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/internal.h b/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/internal.h index 44f4caf..6e87d5a 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/internal.h +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/internal.h @@ -14,8 +14,8 @@ terms of the MIT license. A copy of the license can be found in the file // functions and macros. // -------------------------------------------------------------------------- -#include "mimalloc/types.h" -#include "mimalloc/track.h" +#include "types.h" +#include "track.h" #if (MI_DEBUG>0) #define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) @@ -53,11 +53,6 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_externc #endif -// pthreads -#if !defined(_WIN32) && !defined(__wasi__) -#define MI_USE_PTHREADS -#include <pthread.h> -#endif // "options.c" void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); @@ -84,10 +79,12 @@ extern mi_decl_cache_align const mi_page_t _mi_page_empty; bool _mi_is_main_thread(void); size_t _mi_current_thread_count(void); bool _mi_preloading(void); // true while the C runtime is not initialized yet -mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; -mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap void _mi_thread_done(mi_heap_t* heap); void _mi_thread_data_collect(void); +void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap); +mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; +mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap +mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id); // os.c void _mi_os_init(void); // called from process init @@ -100,7 +97,6 @@ size_t _mi_os_good_alloc_size(size_t size); bool _mi_os_has_overcommit(void); bool _mi_os_has_virtual_reserve(void); -bool _mi_os_purge(void* p, size_t size, mi_stats_t* stats); bool _mi_os_reset(void* addr, size_t size, mi_stats_t* tld_stats); bool _mi_os_commit(void* p, size_t size, bool* is_zero, mi_stats_t* stats); bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); @@ -130,15 +126,22 @@ void _mi_arena_unsafe_destroy_all(mi_stats_t* stats); bool _mi_arena_segment_clear_abandoned(mi_segment_t* segment); void _mi_arena_segment_mark_abandoned(mi_segment_t* segment); -size_t _mi_arena_segment_abandoned_count(void); -typedef struct mi_arena_field_cursor_s { // abstract - mi_arena_id_t start; - int count; - size_t bitmap_idx; +void* _mi_arena_meta_zalloc(size_t size, mi_memid_t* memid); +void _mi_arena_meta_free(void* p, mi_memid_t memid, size_t size); + +typedef struct mi_arena_field_cursor_s { // abstract struct + size_t os_list_count; // max entries to visit in the OS abandoned list + size_t start; // start arena idx (may need to be wrapped) + size_t end; // end arena idx (exclusive, may need to be wrapped) + size_t bitmap_idx; // current bit idx for an arena + mi_subproc_t* subproc; // only visit blocks in this sub-process + bool visit_all; // ensure all abandoned blocks are seen (blocking) + bool hold_visit_lock; // if the subproc->abandoned_os_visit_lock is held } mi_arena_field_cursor_t; -void _mi_arena_field_cursor_init(mi_heap_t* heap, mi_arena_field_cursor_t* current); +void _mi_arena_field_cursor_init(mi_heap_t* heap, mi_subproc_t* subproc, bool visit_all, mi_arena_field_cursor_t* current); mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_field_cursor_t* previous); +void _mi_arena_field_cursor_done(mi_arena_field_cursor_t* current); // "segment-map.c" void _mi_segment_map_allocated_at(const mi_segment_t* segment); @@ -148,8 +151,7 @@ void _mi_segment_map_freed_at(const mi_segment_t* segment); mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld); void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld); void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld); -bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segments_tld_t* tld); -void _mi_segment_collect(mi_segment_t* segment, bool force, mi_segments_tld_t* tld); +uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size); #if MI_HUGE_PAGE_ABANDON void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block); @@ -157,11 +159,10 @@ void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_block_t* block); #endif -uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size); // page start for any page +void _mi_segments_collect(bool force, mi_segments_tld_t* tld); void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld); -void _mi_abandoned_await_readers(void); -void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld); bool _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment); +bool _mi_segment_visit_blocks(mi_segment_t* segment, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); // "page.c" void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept mi_attr_malloc; @@ -186,11 +187,15 @@ size_t _mi_bin_size(uint8_t bin); // for stats uint8_t _mi_bin(size_t size); // for stats // "heap.c" +void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool noreclaim, uint8_t tag); void _mi_heap_destroy_pages(mi_heap_t* heap); void _mi_heap_collect_abandon(mi_heap_t* heap); void _mi_heap_set_default_direct(mi_heap_t* heap); bool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid); void _mi_heap_unsafe_destroy_all(void); +mi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag); +void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page); +bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg); // "stats.c" void _mi_stats_done(mi_stats_t* stats); @@ -317,28 +322,11 @@ static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) { } } -// Align downwards -static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) { - mi_assert_internal(alignment != 0); - uintptr_t mask = alignment - 1; - if ((alignment & mask) == 0) { // power of two? - return (sz & ~mask); - } - else { - return ((sz / alignment) * alignment); - } -} - // Align a pointer upwards static inline void* mi_align_up_ptr(void* p, size_t alignment) { return (void*)_mi_align_up((uintptr_t)p, alignment); } -// Align a pointer downwards -static inline void* mi_align_down_ptr(void* p, size_t alignment) { - return (void*)_mi_align_down((uintptr_t)p, alignment); -} - // Divide upwards: `s <= _mi_divide_up(s,d)*d < s+d`. static inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) { @@ -346,6 +334,14 @@ static inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) { return (divider == 0 ? size : ((size + divider - 1) / divider)); } + +// clamp an integer +static inline size_t _mi_clamp(size_t sz, size_t min, size_t max) { + if (sz < min) return min; + else if (sz > max) return max; + else return sz; +} + // Is memory zero initialized? static inline bool mi_mem_is_zero(const void* p, size_t size) { for (size_t i = 0; i < size; i++) { @@ -354,7 +350,6 @@ static inline bool mi_mem_is_zero(const void* p, size_t size) { return true; } - // Align a byte size to a size in _machine words_, // i.e. byte size == `wsize*sizeof(void*)`. static inline size_t _mi_wsize_from_size(size_t size) { @@ -379,10 +374,10 @@ static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { } #else /* __builtin_umul_overflow is unavailable */ static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { - #define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX) + #define MI_MUL_COULD_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX) *total = count * size; // note: gcc/clang optimize this to directly check the overflow flag - return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW) && size > 0 && (SIZE_MAX / size) < count); + return ((size >= MI_MUL_COULD_OVERFLOW || count >= MI_MUL_COULD_OVERFLOW) && size > 0 && (SIZE_MAX / size) < count); } #endif @@ -449,44 +444,29 @@ static inline mi_segment_t* _mi_ptr_segment(const void* p) { #endif } -static inline mi_page_t* mi_slice_to_page(mi_slice_t* s) { - mi_assert_internal(s->slice_offset== 0 && s->slice_count > 0); - return (mi_page_t*)(s); -} - -static inline mi_slice_t* mi_page_to_slice(mi_page_t* p) { - mi_assert_internal(p->slice_offset== 0 && p->slice_count > 0); - return (mi_slice_t*)(p); -} - // Segment belonging to a page static inline mi_segment_t* _mi_page_segment(const mi_page_t* page) { mi_assert_internal(page!=NULL); mi_segment_t* segment = _mi_ptr_segment(page); - mi_assert_internal(segment == NULL || ((mi_slice_t*)page >= segment->slices && (mi_slice_t*)page < segment->slices + segment->slice_entries)); + mi_assert_internal(segment == NULL || page == &segment->pages[page->segment_idx]); return segment; } -static inline mi_slice_t* mi_slice_first(const mi_slice_t* slice) { - mi_slice_t* start = (mi_slice_t*)((uint8_t*)slice - slice->slice_offset); - mi_assert_internal(start >= _mi_ptr_segment(slice)->slices); - mi_assert_internal(start->slice_offset == 0); - mi_assert_internal(start + start->slice_count > slice); - return start; +// used internally +static inline size_t _mi_segment_page_idx_of(const mi_segment_t* segment, const void* p) { + // if (segment->page_size > MI_SEGMENT_SIZE) return &segment->pages[0]; // huge pages + ptrdiff_t diff = (uint8_t*)p - (uint8_t*)segment; + mi_assert_internal(diff >= 0 && (size_t)diff <= MI_SEGMENT_SIZE /* for huge alignment it can be equal */); + size_t idx = (size_t)diff >> segment->page_shift; + mi_assert_internal(idx < segment->capacity); + mi_assert_internal(segment->page_kind <= MI_PAGE_MEDIUM || idx == 0); + return idx; } -// Get the page containing the pointer (performance critical as it is called in mi_free) +// Get the page containing the pointer static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const void* p) { - mi_assert_internal(p > (void*)segment); - ptrdiff_t diff = (uint8_t*)p - (uint8_t*)segment; - mi_assert_internal(diff > 0 && diff <= (ptrdiff_t)MI_SEGMENT_SIZE); - size_t idx = (size_t)diff >> MI_SEGMENT_SLICE_SHIFT; - mi_assert_internal(idx <= segment->slice_entries); - mi_slice_t* slice0 = (mi_slice_t*)&segment->slices[idx]; - mi_slice_t* slice = mi_slice_first(slice0); // adjust to the block that holds the page data - mi_assert_internal(slice->slice_offset == 0); - mi_assert_internal(slice >= segment->slices && slice < segment->slices + segment->slice_entries); - return mi_slice_to_page(slice); + size_t idx = _mi_segment_page_idx_of(segment, p); + return &((mi_segment_t*)segment)->pages[idx]; } // Quick page start for initialized pages @@ -509,8 +489,8 @@ static inline size_t mi_page_block_size(const mi_page_t* page) { } static inline bool mi_page_is_huge(const mi_page_t* page) { - mi_assert_internal((page->is_huge && _mi_page_segment(page)->kind == MI_SEGMENT_HUGE) || - (!page->is_huge && _mi_page_segment(page)->kind != MI_SEGMENT_HUGE)); + mi_assert_internal((page->is_huge && _mi_page_segment(page)->page_kind == MI_PAGE_HUGE) || + (!page->is_huge && _mi_page_segment(page)->page_kind != MI_PAGE_HUGE)); return page->is_huge; } @@ -522,11 +502,7 @@ static inline size_t mi_page_usable_block_size(const mi_page_t* page) { // size of a segment static inline size_t mi_segment_size(mi_segment_t* segment) { - return segment->segment_slices * MI_SEGMENT_SLICE_SIZE; -} - -static inline uint8_t* mi_segment_end(mi_segment_t* segment) { - return (uint8_t*)segment + mi_segment_size(segment); + return segment->segment_size; } // Thread free access @@ -546,6 +522,7 @@ static inline mi_heap_t* mi_page_heap(const mi_page_t* page) { static inline void mi_page_set_heap(mi_page_t* page, mi_heap_t* heap) { mi_assert_internal(mi_page_thread_free_flag(page) != MI_DELAYED_FREEING); mi_atomic_store_release(&page->xheap,(uintptr_t)heap); + if (heap != NULL) { page->heap_tag = heap->tag; } } // Thread free flag helpers @@ -647,13 +624,12 @@ static inline bool mi_is_in_same_segment(const void* p, const void* q) { } static inline bool mi_is_in_same_page(const void* p, const void* q) { - mi_segment_t* segment = _mi_ptr_segment(p); - if (_mi_ptr_segment(q) != segment) return false; - // assume q may be invalid // return (_mi_segment_page_of(segment, p) == _mi_segment_page_of(segment, q)); - mi_page_t* page = _mi_segment_page_of(segment, p); - size_t psize; - uint8_t* start = _mi_segment_page_start(segment, page, &psize); - return (start <= (uint8_t*)q && (uint8_t*)q < start + psize); + mi_segment_t* segmentp = _mi_ptr_segment(p); + mi_segment_t* segmentq = _mi_ptr_segment(q); + if (segmentp != segmentq) return false; + size_t idxp = _mi_segment_page_idx_of(segmentp, p); + size_t idxq = _mi_segment_page_idx_of(segmentq, q); + return (idxp == idxq); } static inline uintptr_t mi_rotl(uintptr_t x, uintptr_t shift) { @@ -725,50 +701,6 @@ static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, c } -// ------------------------------------------------------------------- -// commit mask -// ------------------------------------------------------------------- - -static inline void mi_commit_mask_create_empty(mi_commit_mask_t* cm) { - for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { - cm->mask[i] = 0; - } -} - -static inline void mi_commit_mask_create_full(mi_commit_mask_t* cm) { - for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { - cm->mask[i] = ~((size_t)0); - } -} - -static inline bool mi_commit_mask_is_empty(const mi_commit_mask_t* cm) { - for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { - if (cm->mask[i] != 0) return false; - } - return true; -} - -static inline bool mi_commit_mask_is_full(const mi_commit_mask_t* cm) { - for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { - if (cm->mask[i] != ~((size_t)0)) return false; - } - return true; -} - -// defined in `segment.c`: -size_t _mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total); -size_t _mi_commit_mask_next_run(const mi_commit_mask_t* cm, size_t* idx); - -#define mi_commit_mask_foreach(cm,idx,count) \ - idx = 0; \ - while ((count = _mi_commit_mask_next_run(cm,&idx)) > 0) { - -#define mi_commit_mask_foreach_end() \ - idx += count; \ - } - - - /* ----------------------------------------------------------- memory id's ----------------------------------------------------------- */ diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/prim.h b/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/prim.h index 4d813b7..640c966 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/prim.h +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/prim.h @@ -26,7 +26,7 @@ typedef struct mi_os_mem_config_s { size_t large_page_size; // 0 if not supported, usually 2MiB (4MiB on Windows) size_t alloc_granularity; // smallest allocation size (usually 4KiB, on Windows 64KiB) bool has_overcommit; // can we reserve more memory than can be actually committed? - bool must_free_whole; // must allocated blocks be freed as a whole (false for mmap, true for VirtualAlloc) + bool has_partial_free; // can allocated blocks be freed partially? (true for mmap, false for VirtualAlloc) bool has_virtual_reserve; // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory) } mi_os_mem_config_t; @@ -115,6 +115,7 @@ void _mi_prim_thread_done_auto_done(void); void _mi_prim_thread_associate_default_heap(mi_heap_t* heap); + //------------------------------------------------------------------- // Thread id: `_mi_prim_thread_id()` // @@ -198,7 +199,7 @@ static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexce tcb[slot] = value; #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781 MI_UNUSED(ofs); - pthread_setspecific(slot, value); + pthread_setspecific(slot, value); #endif } @@ -208,13 +209,18 @@ static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexce // but unfortunately, it seems we cannot test for this reliably at this time (see issue #883) // Nevertheless, it seems needed on older graviton platforms (see issue #851). // For now, we only enable this for specific platforms. -#if defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__) /* special case aarch64 for older gcc versions (issue #851) */ \ - && !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly (<https://github.com/microsoft/mimalloc/issues/343#issuecomment-763272369>)*/ \ +#if !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly (<https://github.com/microsoft/mimalloc/issues/343#issuecomment-763272369>)*/ \ + && !defined(MI_LIBC_MUSL) \ && (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot (<https://lore.kernel.org/linux-arm-kernel/202110280952.352F66D8@keescook/T/>) */ -#define MI_USE_BUILTIN_THREAD_POINTER 1 + #if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \ + || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \ + || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__))) + #define MI_USE_BUILTIN_THREAD_POINTER 1 + #endif #endif + // defined in `init.c`; do not use these directly extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from extern bool _mi_process_is_initialized; // has mi_process_init been called? @@ -222,22 +228,24 @@ extern bool _mi_process_is_initialized; // has mi_process_init been static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept; // Get a unique id for the current thread. -#if defined(_WIN32) +#if defined(MI_PRIM_THREAD_ID) + +static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { + return MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488) +} + +#elif defined(_WIN32) -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif -#include <windows.h> static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { // Windows: works on Intel and ARM in both 32- and 64-bit return (uintptr_t)NtCurrentTeb(); } -#elif MI_USE_BUILTIN_THREAD_POINTER +#elif MI_USE_BUILTIN_THREAD_POINTER static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { // Works on most Unix based platforms with recent compilers - return (uintptr_t)__builtin_thread_pointer(); + return (uintptr_t)__builtin_thread_pointer(); } #elif defined(MI_HAS_TLS_SLOT) @@ -359,4 +367,6 @@ static inline mi_heap_t* mi_prim_get_default_heap(void) { + + #endif // MIMALLOC_PRIM_H diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/track.h b/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/track.h index a659d94..4b5709e 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/track.h +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/track.h @@ -34,7 +34,7 @@ The corresponding `mi_track_free` still uses the block start pointer and origina The `mi_track_resize` is currently unused but could be called on reallocations within a block. `mi_track_init` is called at program start. -The following macros are for tools like asan and valgrind to track whether memory is +The following macros are for tools like asan and valgrind to track whether memory is defined, undefined, or not accessible at all: #define mi_track_mem_defined(p,size) @@ -82,10 +82,6 @@ defined, undefined, or not accessible at all: #define MI_TRACK_HEAP_DESTROY 1 #define MI_TRACK_TOOL "ETW" -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif -#include <windows.h> #include "../src/prim/windows/etw.h" #define mi_track_init() EventRegistermicrosoft_windows_mimalloc(); @@ -96,7 +92,7 @@ defined, undefined, or not accessible at all: // no tracking #define MI_TRACK_ENABLED 0 -#define MI_TRACK_HEAP_DESTROY 0 +#define MI_TRACK_HEAP_DESTROY 0 #define MI_TRACK_TOOL "none" #define mi_track_malloc_size(p,reqsize,size,zero) diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/types.h b/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/types.h index cc807ee..31ed35f 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/types.h +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/include/mimalloc/types.h @@ -13,9 +13,8 @@ terms of the MIT license. A copy of the license can be found in the file // mi_heap_t : all data for a thread-local heap, contains // lists of all managed heap pages. // mi_segment_t : a larger chunk of memory (32GiB) from where pages -// are allocated. A segment is divided in slices (64KiB) from -// which pages are allocated. -// mi_page_t : a "mimalloc" page (usually 64KiB or 512KiB) from +// are allocated. +// mi_page_t : a mimalloc page (usually 64KiB or 512KiB) from // where objects are allocated. // Note: we write "OS page" for OS memory pages while // using plain "page" for mimalloc pages (`mi_page_t`). @@ -24,7 +23,7 @@ terms of the MIT license. A copy of the license can be found in the file #include <stddef.h> // ptrdiff_t #include <stdint.h> // uintptr_t, uint16_t, etc -#include "mimalloc/atomic.h" // _Atomic +#include "atomic.h" // _Atomic #ifdef _MSC_VER #pragma warning(disable:4214) // bitfield is not int @@ -160,66 +159,55 @@ typedef int32_t mi_ssize_t; // ------------------------------------------------------ // Main tuning parameters for segment and page sizes -// Sizes for 64-bit (usually divide by two for 32-bit) -#ifndef MI_SEGMENT_SLICE_SHIFT -#define MI_SEGMENT_SLICE_SHIFT (13 + MI_INTPTR_SHIFT) // 64KiB (32KiB on 32-bit) -#endif - -#ifndef MI_SEGMENT_SHIFT -#if MI_INTPTR_SIZE > 4 -#define MI_SEGMENT_SHIFT ( 9 + MI_SEGMENT_SLICE_SHIFT) // 32MiB -#else -#define MI_SEGMENT_SHIFT ( 7 + MI_SEGMENT_SLICE_SHIFT) // 4MiB on 32-bit -#endif -#endif - +// Sizes for 64-bit, divide by two for 32-bit #ifndef MI_SMALL_PAGE_SHIFT -#define MI_SMALL_PAGE_SHIFT (MI_SEGMENT_SLICE_SHIFT) // 64KiB +#define MI_SMALL_PAGE_SHIFT (13 + MI_INTPTR_SHIFT) // 64KiB #endif #ifndef MI_MEDIUM_PAGE_SHIFT -#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512KiB +#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512KiB +#endif +#ifndef MI_LARGE_PAGE_SHIFT +#define MI_LARGE_PAGE_SHIFT ( 3 + MI_MEDIUM_PAGE_SHIFT) // 4MiB +#endif +#ifndef MI_SEGMENT_SHIFT +#define MI_SEGMENT_SHIFT ( MI_LARGE_PAGE_SHIFT) // 4MiB -- must be equal to `MI_LARGE_PAGE_SHIFT` #endif // Derived constants #define MI_SEGMENT_SIZE (MI_ZU(1)<<MI_SEGMENT_SHIFT) -#define MI_SEGMENT_ALIGN MI_SEGMENT_SIZE +#define MI_SEGMENT_ALIGN (MI_SEGMENT_SIZE) #define MI_SEGMENT_MASK ((uintptr_t)(MI_SEGMENT_ALIGN - 1)) -#define MI_SEGMENT_SLICE_SIZE (MI_ZU(1)<< MI_SEGMENT_SLICE_SHIFT) -#define MI_SLICES_PER_SEGMENT (MI_SEGMENT_SIZE / MI_SEGMENT_SLICE_SIZE) // 1024 #define MI_SMALL_PAGE_SIZE (MI_ZU(1)<<MI_SMALL_PAGE_SHIFT) #define MI_MEDIUM_PAGE_SIZE (MI_ZU(1)<<MI_MEDIUM_PAGE_SHIFT) +#define MI_LARGE_PAGE_SIZE (MI_ZU(1)<<MI_LARGE_PAGE_SHIFT) -#define MI_SMALL_OBJ_SIZE_MAX (MI_SMALL_PAGE_SIZE/4) // 8KiB on 64-bit -#define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128KiB on 64-bit -#define MI_MEDIUM_OBJ_WSIZE_MAX (MI_MEDIUM_OBJ_SIZE_MAX/MI_INTPTR_SIZE) -#define MI_LARGE_OBJ_SIZE_MAX (MI_SEGMENT_SIZE/2) // 32MiB on 64-bit +#define MI_SMALL_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_SMALL_PAGE_SIZE) +#define MI_MEDIUM_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_MEDIUM_PAGE_SIZE) +#define MI_LARGE_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_LARGE_PAGE_SIZE) + +// The max object size are checked to not waste more than 12.5% internally over the page sizes. +// (Except for large pages since huge objects are allocated in 4MiB chunks) +#define MI_SMALL_OBJ_SIZE_MAX (MI_SMALL_PAGE_SIZE/4) // 16KiB +#define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128KiB +#define MI_LARGE_OBJ_SIZE_MAX (MI_LARGE_PAGE_SIZE/2) // 2MiB #define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX/MI_INTPTR_SIZE) // Maximum number of size classes. (spaced exponentially in 12.5% increments) #define MI_BIN_HUGE (73U) -#if (MI_MEDIUM_OBJ_WSIZE_MAX >= 655360) +#if (MI_LARGE_OBJ_WSIZE_MAX >= 655360) #error "mimalloc internal: define more bins" #endif // Maximum block size for which blocks are guaranteed to be block size aligned. (see `segment.c:_mi_segment_page_start`) -#define MI_MAX_ALIGN_GUARANTEE (MI_MEDIUM_OBJ_SIZE_MAX) +#define MI_MAX_ALIGN_GUARANTEE (MI_MEDIUM_OBJ_SIZE_MAX) // Alignments over MI_BLOCK_ALIGNMENT_MAX are allocated in dedicated huge page segments -#define MI_BLOCK_ALIGNMENT_MAX (MI_SEGMENT_SIZE >> 1) - -// Maximum slice count (255) for which we can find the page for interior pointers -#define MI_MAX_SLICE_OFFSET_COUNT ((MI_BLOCK_ALIGNMENT_MAX / MI_SEGMENT_SLICE_SIZE) - 1) +#define MI_BLOCK_ALIGNMENT_MAX (MI_SEGMENT_SIZE >> 1) -// we never allocate more than PTRDIFF_MAX (see also <https://sourceware.org/ml/libc-announce/2019/msg00001.html>) -// on 64-bit+ systems we also limit the maximum allocation size such that the slice count fits in 32-bits. (issue #877) -#if (PTRDIFF_MAX > INT32_MAX) && (PTRDIFF_MAX >= (MI_SEGMENT_SLIZE_SIZE * UINT32_MAX)) -#define MI_MAX_ALLOC_SIZE (MI_SEGMENT_SLICE_SIZE * (UINT32_MAX-1)) -#else +// We never allocate more than PTRDIFF_MAX (see also <https://sourceware.org/ml/libc-announce/2019/msg00001.html>) #define MI_MAX_ALLOC_SIZE PTRDIFF_MAX -#endif - // ------------------------------------------------------ // Mimalloc pages contain allocated blocks @@ -291,8 +279,8 @@ typedef uintptr_t mi_thread_free_t; // Notes: // - Access is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc` // - Using `uint16_t` does not seem to slow things down -// - The size is 12 words on 64-bit which helps the page index calculations -// (and 14 words on 32-bit, and encoded free lists add 2 words) +// - The size is 10 words on 64-bit which helps the page index calculations +// (and 12 words on 32-bit, and encoded free lists add 2 words) // - `xthread_free` uses the bottom bits as a delayed-free flags to optimize // concurrent frees where only the first concurrent free adds to the owning // heap `thread_delayed_free` list (see `free.c:mi_free_block_mt`). @@ -302,12 +290,12 @@ typedef uintptr_t mi_thread_free_t; // will be freed correctly even if only other threads free blocks. typedef struct mi_page_s { // "owned" by the segment - uint32_t slice_count; // slices in this page (0 if not a page) - uint32_t slice_offset; // distance from the actual page data slice (0 if a page) + uint8_t segment_idx; // index in the segment `pages` array, `page == &segment->pages[page->segment_idx]` + uint8_t segment_in_use:1; // `true` if the segment allocated this page uint8_t is_committed:1; // `true` if the page virtual memory is committed uint8_t is_zero_init:1; // `true` if the page was initially zero initialized - uint8_t is_huge:1; // `true` if the page is in a huge segment (`segment->kind == MI_SEGMENT_HUGE`) - // padding + uint8_t is_huge:1; // `true` if the page is in a huge segment + // layout like this to optimize access in `mi_malloc` and `mi_free` uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear` uint16_t reserved; // number of blocks reserved in memory @@ -319,6 +307,7 @@ typedef struct mi_page_s { mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) uint16_t used; // number of blocks in use (including blocks in `thread_free`) uint8_t block_size_shift; // if not zero, then `(1 << block_size_shift) == block_size` (only used for fast path in `free.c:_mi_page_ptr_unalign`) + uint8_t heap_tag; // tag of the owning heap, used to separate heaps by object type // padding size_t block_size; // size available in each block (always `>0`) uint8_t* page_start; // start of the page area containing the blocks @@ -330,11 +319,12 @@ typedef struct mi_page_s { _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads _Atomic(uintptr_t) xheap; - struct mi_page_s* next; // next page owned by this thread with the same `block_size` - struct mi_page_s* prev; // previous page owned by this thread with the same `block_size` + struct mi_page_s* next; // next page owned by the heap with the same `block_size` + struct mi_page_s* prev; // previous page owned by the heap with the same `block_size` - // 64-bit 11 words, 32-bit 13 words, (+2 for secure) + #if MI_INTPTR_SIZE==4 // pad to 12 words on 32-bit void* padding[1]; + #endif } mi_page_t; @@ -347,44 +337,10 @@ typedef enum mi_page_kind_e { MI_PAGE_SMALL, // small blocks go into 64KiB pages inside a segment MI_PAGE_MEDIUM, // medium blocks go into 512KiB pages inside a segment MI_PAGE_LARGE, // larger blocks go into a single page spanning a whole segment - MI_PAGE_HUGE // a huge page is a single page in a segment of variable size + MI_PAGE_HUGE // a huge page is a single page in a segment of variable size (but still 2MiB aligned) // used for blocks `> MI_LARGE_OBJ_SIZE_MAX` or an aligment `> MI_BLOCK_ALIGNMENT_MAX`. } mi_page_kind_t; -typedef enum mi_segment_kind_e { - MI_SEGMENT_NORMAL, // MI_SEGMENT_SIZE size with pages inside. - MI_SEGMENT_HUGE, // segment with just one huge page inside. -} mi_segment_kind_t; - -// ------------------------------------------------------ -// A segment holds a commit mask where a bit is set if -// the corresponding MI_COMMIT_SIZE area is committed. -// The MI_COMMIT_SIZE must be a multiple of the slice -// size. If it is equal we have the most fine grained -// decommit (but setting it higher can be more efficient). -// The MI_MINIMAL_COMMIT_SIZE is the minimal amount that will -// be committed in one go which can be set higher than -// MI_COMMIT_SIZE for efficiency (while the decommit mask -// is still tracked in fine-grained MI_COMMIT_SIZE chunks) -// ------------------------------------------------------ - -#define MI_MINIMAL_COMMIT_SIZE (1*MI_SEGMENT_SLICE_SIZE) -#define MI_COMMIT_SIZE (MI_SEGMENT_SLICE_SIZE) // 64KiB -#define MI_COMMIT_MASK_BITS (MI_SEGMENT_SIZE / MI_COMMIT_SIZE) -#define MI_COMMIT_MASK_FIELD_BITS MI_SIZE_BITS -#define MI_COMMIT_MASK_FIELD_COUNT (MI_COMMIT_MASK_BITS / MI_COMMIT_MASK_FIELD_BITS) - -#if (MI_COMMIT_MASK_BITS != (MI_COMMIT_MASK_FIELD_COUNT * MI_COMMIT_MASK_FIELD_BITS)) -#error "the segment size must be exactly divisible by the (commit size * size_t bits)" -#endif - -typedef struct mi_commit_mask_s { - size_t mask[MI_COMMIT_MASK_FIELD_COUNT]; -} mi_commit_mask_t; - -typedef mi_page_t mi_slice_t; -typedef int64_t mi_msecs_t; - // --------------------------------------------------------------- // a memory id tracks the provenance of arena/OS allocated memory @@ -428,49 +384,42 @@ typedef struct mi_memid_s { } mi_memid_t; -// ----------------------------------------------------------------------------------------- -// Segments are large allocated memory blocks (8mb on 64 bit) from arenas or the OS. -// -// Inside segments we allocated fixed size mimalloc pages (`mi_page_t`) that contain blocks. -// The start of a segment is this structure with a fixed number of slice entries (`slices`) -// usually followed by a guard OS page and the actual allocation area with pages. -// While a page is not allocated, we view it's data as a `mi_slice_t` (instead of a `mi_page_t`). -// Of any free area, the first slice has the info and `slice_offset == 0`; for any subsequent -// slices part of the area, the `slice_offset` is the byte offset back to the first slice -// (so we can quickly find the page info on a free, `internal.h:_mi_segment_page_of`). -// For slices, the `block_size` field is repurposed to signify if a slice is used (`1`) or not (`0`). -// Small and medium pages use a fixed amount of slices to reduce slice fragmentation, while -// large and huge pages span a variable amount of slices. +// --------------------------------------------------------------- +// Segments contain mimalloc pages +// --------------------------------------------------------------- +typedef struct mi_subproc_s mi_subproc_t; + +// Segments are large allocated memory blocks (2MiB on 64 bit) from the OS. +// Inside segments we allocated fixed size _pages_ that contain blocks. typedef struct mi_segment_s { // constant fields - mi_memid_t memid; // memory id for arena/OS allocation - bool allow_decommit; // can we decommmit the memory - bool allow_purge; // can we purge the memory (reset or decommit) - size_t segment_size; + mi_memid_t memid; // memory id to track provenance + bool allow_decommit; + bool allow_purge; + size_t segment_size; // for huge pages this may be different from `MI_SEGMENT_SIZE` + mi_subproc_t* subproc; // segment belongs to sub process // segment fields - mi_msecs_t purge_expire; // purge slices in the `purge_mask` after this time - mi_commit_mask_t purge_mask; // slices that can be purged - mi_commit_mask_t commit_mask; // slices that are currently committed + struct mi_segment_s* next; // must be the first (non-constant) segment field -- see `segment.c:segment_init` + struct mi_segment_s* prev; + bool was_reclaimed; // true if it was reclaimed (used to limit on-free reclamation) - // from here is zero initialized - struct mi_segment_s* next; // the list of freed segments in the cache (must be first field, see `segment.c:mi_segment_init`) - bool was_reclaimed; // true if it was reclaimed (used to limit on-free reclamation) + size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`) + size_t abandoned_visits; // count how often this segment is visited for reclaiming (to force reclaim if it is too long) - size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`) - size_t abandoned_visits; // count how often this segment is visited during abondoned reclamation (to force reclaim if it takes too long) - size_t used; // count of pages in use - uintptr_t cookie; // verify addresses in debug mode: `mi_ptr_cookie(segment) == segment->cookie` + size_t used; // count of pages in use (`used <= capacity`) + size_t capacity; // count of available pages (`#free + used`) + size_t segment_info_size;// space we are using from the first page for segment meta-data and possible guard pages. + uintptr_t cookie; // verify addresses in secure mode: `_mi_ptr_cookie(segment) == segment->cookie` - size_t segment_slices; // for huge segments this may be different from `MI_SLICES_PER_SEGMENT` - size_t segment_info_slices; // initial count of slices that we are using for segment info and possible guard pages. + struct mi_segment_s* abandoned_os_next; // only used for abandoned segments outside arena's, and only if `mi_option_visit_abandoned` is enabled + struct mi_segment_s* abandoned_os_prev; // layout like this to optimize access in `mi_free` - mi_segment_kind_t kind; - size_t slice_entries; // entries in the `slices` array, at most `MI_SLICES_PER_SEGMENT` _Atomic(mi_threadid_t) thread_id; // unique id of the thread owning this segment - - mi_slice_t slices[MI_SLICES_PER_SEGMENT+1]; // one extra final entry for huge blocks with large alignment + size_t page_shift; // `1 << page_shift` == the page sizes == `page->block_size * page->reserved` (unless the first page, then `-segment_info_size`). + mi_page_kind_t page_kind; // kind of pages: small, medium, large, or huge + mi_page_t pages[1]; // up to `MI_SMALL_PAGES_PER_SEGMENT` pages } mi_segment_t; @@ -538,6 +487,7 @@ struct mi_heap_s { size_t page_retired_max; // largest retired index into the `pages` array. mi_heap_t* next; // list of heaps per thread bool no_reclaim; // `true` if this heap should not reclaim abandoned pages + uint8_t tag; // custom tag, can be used for separating heaps based on the object types mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin") }; @@ -615,7 +565,7 @@ typedef struct mi_stats_s { mi_stat_count_t threads; mi_stat_count_t normal; mi_stat_count_t huge; - mi_stat_count_t large; + mi_stat_count_t giant; mi_stat_count_t malloc; mi_stat_count_t segments_cache; mi_stat_counter_t pages_extended; @@ -627,7 +577,6 @@ typedef struct mi_stats_s { mi_stat_counter_t searches; mi_stat_counter_t normal_count; mi_stat_counter_t huge_count; - mi_stat_counter_t large_count; mi_stat_counter_t arena_count; mi_stat_counter_t arena_crossover_count; mi_stat_counter_t arena_rollback_count; @@ -657,18 +606,32 @@ void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); // ------------------------------------------------------ +// Sub processes do not reclaim or visit segments +// from other sub processes +// ------------------------------------------------------ + +struct mi_subproc_s { + _Atomic(size_t) abandoned_count; // count of abandoned segments for this sub-process + _Atomic(size_t) abandoned_os_list_count; // count of abandoned segments in the os-list + mi_lock_t abandoned_os_lock; // lock for the abandoned os segment list (outside of arena's) (this lock protect list operations) + mi_lock_t abandoned_os_visit_lock; // ensure only one thread per subproc visits the abandoned os list + mi_segment_t* abandoned_os_list; // doubly-linked list of abandoned segments outside of arena's (in OS allocated memory) + mi_segment_t* abandoned_os_list_tail; // the tail-end of the list + mi_memid_t memid; // provenance of this memory block +}; + +// ------------------------------------------------------ // Thread Local data // ------------------------------------------------------ -// A "span" is is an available range of slices. The span queues keep -// track of slice spans of at most the given `slice_count` (but more than the previous size class). -typedef struct mi_span_queue_s { - mi_slice_t* first; - mi_slice_t* last; - size_t slice_count; -} mi_span_queue_t; +// Milliseconds as in `int64_t` to avoid overflows +typedef int64_t mi_msecs_t; -#define MI_SEGMENT_BIN_MAX (35) // 35 == mi_segment_bin(MI_SLICES_PER_SEGMENT) +// Queue of segments +typedef struct mi_segment_queue_s { + mi_segment_t* first; + mi_segment_t* last; +} mi_segment_queue_t; // OS thread local data typedef struct mi_os_tld_s { @@ -676,17 +639,19 @@ typedef struct mi_os_tld_s { mi_stats_t* stats; // points to tld stats } mi_os_tld_t; - // Segments thread local data typedef struct mi_segments_tld_s { - mi_span_queue_t spans[MI_SEGMENT_BIN_MAX+1]; // free slice spans inside segments + mi_segment_queue_t small_free; // queue of segments with free small pages + mi_segment_queue_t medium_free; // queue of segments with free medium pages + mi_page_queue_t pages_purge; // queue of freed pages that are delay purged size_t count; // current number of segments; size_t peak_count; // peak number of segments size_t current_size; // current size of all segments size_t peak_size; // peak size of all segments size_t reclaim_count;// number of reclaimed (abandoned) segments + mi_subproc_t* subproc; // sub-process this thread belongs to. mi_stats_t* stats; // points to tld stats - mi_os_tld_t* os; // points to os stats + mi_os_tld_t* os; // points to os tld } mi_segments_tld_t; // Thread local data diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/readme.md b/lib/Utils.Memory/vnlib_mimalloc/vendor/readme.md index 9197458..a0296b4 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/readme.md +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/readme.md @@ -12,8 +12,8 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the runtime systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release tag: `v2.1.6` (2024-05-13). -Latest v1 tag: `v1.8.6` (2024-05-13). +Latest release tag: `v2.1.7` (2024-05-21). +Latest v1 tag: `v1.8.7` (2024-05-21). mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: @@ -82,6 +82,8 @@ memory usage and fragmentation compared to mimalloc `v1.x` (especially for large workloads). Should otherwise have similar performance (see [below](#performance)); please report if you observe any significant performance regression. +* 2024-05-21, `v1.8.7`, `v2.1.7`: Fix build issues on less common platforms. Started upstreaming patches + from the CPython [integration](https://github.com/python/cpython/issues/113141#issuecomment-2119255217). Upstream `vcpkg` patches. * 2024-05-13, `v1.8.6`, `v2.1.6`: Fix build errors on various (older) platforms. Refactored aligned allocation. * 2024-04-22, `v1.8.4`, `v2.1.4`: Fixes various bugs and build issues. Add `MI_LIBC_MUSL` cmake flag for musl builds. Free-ing code is refactored into a separate module (`free.c`). Mimalloc page info is simplified with the block size diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/alloc-aligned.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/alloc-aligned.c index ba629ef..20c3604 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/alloc-aligned.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/alloc-aligned.c @@ -76,7 +76,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t // now zero the block if needed if (alignment > MI_BLOCK_ALIGNMENT_MAX) { - // for the tracker, on huge aligned allocations only the memory from the start of the large block is defined + // for the tracker, on huge aligned allocations only from the start of the large block is defined mi_track_mem_undefined(aligned_p, size); if (zero) { _mi_memzero_aligned(aligned_p, mi_usable_size(aligned_p)); @@ -171,11 +171,6 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, return mi_heap_malloc_aligned_at(heap, size, alignment, 0); } -// ensure a definition is emitted -#if defined(__cplusplus) -void* _mi_extern_heap_malloc_aligned = (void*)&mi_heap_malloc_aligned; -#endif - // ------------------------------------------------------ // Aligned Allocation // ------------------------------------------------------ diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/alloc.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/alloc.c index 86aaae7..1eee1f2 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/alloc.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/alloc.c @@ -28,7 +28,7 @@ terms of the MIT license. A copy of the license can be found in the file // Fast allocation in a page: just pop from the free list. // Fall back to generic allocation only if the list is empty. // Note: in release mode the (inlined) routine is about 7 instructions with a single test. -extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept +extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept { mi_assert_internal(page->block_size == 0 /* empty heap */ || mi_page_block_size(page) >= size); mi_block_t* const block = page->free; @@ -74,7 +74,7 @@ extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_ #if (MI_STAT>0) const size_t bsize = mi_page_usable_block_size(page); - if (bsize <= MI_MEDIUM_OBJ_SIZE_MAX) { + if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { mi_heap_stat_increase(heap, normal, bsize); mi_heap_stat_counter_increase(heap, normal_count, 1); #if (MI_STAT>1) @@ -125,7 +125,7 @@ static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, #endif mi_page_t* page = _mi_heap_get_free_small_page(heap, size + MI_PADDING_SIZE); - void* const p = _mi_page_malloc_zero(heap, page, size + MI_PADDING_SIZE, zero); + void* const p = _mi_page_malloc_zero(heap, page, size + MI_PADDING_SIZE, zero); mi_track_malloc(p,size,zero); #if MI_STAT>1 @@ -362,7 +362,7 @@ mi_decl_nodiscard mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_ #ifndef PATH_MAX #define PATH_MAX MAX_PATH #endif -#include <windows.h> + mi_decl_nodiscard mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept { // todo: use GetFullPathNameW to allow longer file names char buf[PATH_MAX]; @@ -530,7 +530,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, si } mi_decl_nodiscard mi_decl_restrict void* mi_new_n(size_t count, size_t size) { - return mi_heap_alloc_new_n(mi_prim_get_default_heap(), size, count); + return mi_heap_alloc_new_n(mi_prim_get_default_heap(), count, size); } @@ -584,6 +584,7 @@ mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { #ifdef __cplusplus void* _mi_externs[] = { (void*)&_mi_page_malloc, + (void*)&_mi_page_malloc_zero, (void*)&_mi_heap_malloc_zero, (void*)&_mi_heap_malloc_zero_ex, (void*)&mi_malloc, @@ -591,7 +592,7 @@ void* _mi_externs[] = { (void*)&mi_zalloc_small, (void*)&mi_heap_malloc, (void*)&mi_heap_zalloc, - (void*)&mi_heap_malloc_small, + (void*)&mi_heap_malloc_small // (void*)&mi_heap_alloc_new, // (void*)&mi_heap_alloc_new_n }; diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/arena-abandon.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/arena-abandon.c new file mode 100644 index 0000000..eaa8c7c --- /dev/null +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/arena-abandon.c @@ -0,0 +1,356 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2024, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#if !defined(MI_IN_ARENA_C) +#error "this file should be included from 'arena.c' (so mi_arena_t is visible)" +// add includes help an IDE +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "bitmap.h" +#endif + +// Minimal exports for arena-abandoned. +size_t mi_arena_id_index(mi_arena_id_t id); +mi_arena_t* mi_arena_from_index(size_t idx); +size_t mi_arena_get_count(void); +void* mi_arena_block_start(mi_arena_t* arena, mi_bitmap_index_t bindex); +bool mi_arena_memid_indices(mi_memid_t memid, size_t* arena_index, mi_bitmap_index_t* bitmap_index); + +/* ----------------------------------------------------------- + Abandoned blocks/segments: + + _mi_arena_segment_clear_abandoned + _mi_arena_segment_mark_abandoned + + This is used to atomically abandon/reclaim segments + (and crosses the arena API but it is convenient to have here). + + Abandoned segments still have live blocks; they get reclaimed + when a thread frees a block in it, or when a thread needs a fresh + segment. + + Abandoned segments are atomically marked in the `block_abandoned` + bitmap of arenas. Any segments allocated outside arenas are put + in the sub-process `abandoned_os_list`. This list is accessed + using locks but this should be uncommon and generally uncontended. + Reclaim and visiting either scan through the `block_abandoned` + bitmaps of the arena's, or visit the `abandoned_os_list` + + A potentially nicer design is to use arena's for everything + and perhaps have virtual arena's to map OS allocated memory + but this would lack the "density" of our current arena's. TBC. +----------------------------------------------------------- */ + + +// reclaim a specific OS abandoned segment; `true` on success. +// sets the thread_id. +static bool mi_arena_segment_os_clear_abandoned(mi_segment_t* segment, bool take_lock) { + mi_assert(segment->memid.memkind != MI_MEM_ARENA); + // not in an arena, remove from list of abandoned os segments + mi_subproc_t* const subproc = segment->subproc; + if (take_lock && !mi_lock_try_acquire(&subproc->abandoned_os_lock)) { + return false; // failed to acquire the lock, we just give up + } + // remove atomically from the abandoned os list (if possible!) + bool reclaimed = false; + mi_segment_t* const next = segment->abandoned_os_next; + mi_segment_t* const prev = segment->abandoned_os_prev; + if (next != NULL || prev != NULL || subproc->abandoned_os_list == segment) { + #if MI_DEBUG>3 + // find ourselves in the abandoned list (and check the count) + bool found = false; + size_t count = 0; + for (mi_segment_t* current = subproc->abandoned_os_list; current != NULL; current = current->abandoned_os_next) { + if (current == segment) { found = true; } + count++; + } + mi_assert_internal(found); + mi_assert_internal(count == mi_atomic_load_relaxed(&subproc->abandoned_os_list_count)); + #endif + // remove (atomically) from the list and reclaim + if (prev != NULL) { prev->abandoned_os_next = next; } + else { subproc->abandoned_os_list = next; } + if (next != NULL) { next->abandoned_os_prev = prev; } + else { subproc->abandoned_os_list_tail = prev; } + segment->abandoned_os_next = NULL; + segment->abandoned_os_prev = NULL; + mi_atomic_decrement_relaxed(&subproc->abandoned_count); + mi_atomic_decrement_relaxed(&subproc->abandoned_os_list_count); + if (take_lock) { // don't reset the thread_id when iterating + mi_atomic_store_release(&segment->thread_id, _mi_thread_id()); + } + reclaimed = true; + } + if (take_lock) { mi_lock_release(&segment->subproc->abandoned_os_lock); } + return reclaimed; +} + +// reclaim a specific abandoned segment; `true` on success. +// sets the thread_id. +bool _mi_arena_segment_clear_abandoned(mi_segment_t* segment) { + if mi_unlikely(segment->memid.memkind != MI_MEM_ARENA) { + return mi_arena_segment_os_clear_abandoned(segment, true /* take lock */); + } + // arena segment: use the blocks_abandoned bitmap. + size_t arena_idx; + size_t bitmap_idx; + mi_arena_memid_indices(segment->memid, &arena_idx, &bitmap_idx); + mi_arena_t* arena = mi_arena_from_index(arena_idx); + mi_assert_internal(arena != NULL); + // reclaim atomically + bool was_marked = _mi_bitmap_unclaim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx); + if (was_marked) { + mi_assert_internal(mi_atomic_load_acquire(&segment->thread_id) == 0); + mi_atomic_decrement_relaxed(&segment->subproc->abandoned_count); + mi_atomic_store_release(&segment->thread_id, _mi_thread_id()); + } + // mi_assert_internal(was_marked); + mi_assert_internal(!was_marked || _mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); + //mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx)); + return was_marked; +} + + +// mark a specific OS segment as abandoned +static void mi_arena_segment_os_mark_abandoned(mi_segment_t* segment) { + mi_assert(segment->memid.memkind != MI_MEM_ARENA); + // not in an arena; we use a list of abandoned segments + mi_subproc_t* const subproc = segment->subproc; + if (!mi_lock_acquire(&subproc->abandoned_os_lock)) { + _mi_error_message(EFAULT, "internal error: failed to acquire the abandoned (os) segment lock to mark abandonment"); + // we can continue but cannot visit/reclaim such blocks.. + } + else { + // push on the tail of the list (important for the visitor) + mi_segment_t* prev = subproc->abandoned_os_list_tail; + mi_assert_internal(prev == NULL || prev->abandoned_os_next == NULL); + mi_assert_internal(segment->abandoned_os_prev == NULL); + mi_assert_internal(segment->abandoned_os_next == NULL); + if (prev != NULL) { prev->abandoned_os_next = segment; } + else { subproc->abandoned_os_list = segment; } + subproc->abandoned_os_list_tail = segment; + segment->abandoned_os_prev = prev; + segment->abandoned_os_next = NULL; + mi_atomic_increment_relaxed(&subproc->abandoned_os_list_count); + mi_atomic_increment_relaxed(&subproc->abandoned_count); + // and release the lock + mi_lock_release(&subproc->abandoned_os_lock); + } + return; +} + +// mark a specific segment as abandoned +// clears the thread_id. +void _mi_arena_segment_mark_abandoned(mi_segment_t* segment) +{ + mi_assert_internal(segment->used == segment->abandoned); + mi_atomic_store_release(&segment->thread_id, 0); // mark as abandoned for multi-thread free's + if mi_unlikely(segment->memid.memkind != MI_MEM_ARENA) { + mi_arena_segment_os_mark_abandoned(segment); + return; + } + // segment is in an arena, mark it in the arena `blocks_abandoned` bitmap + size_t arena_idx; + size_t bitmap_idx; + mi_arena_memid_indices(segment->memid, &arena_idx, &bitmap_idx); + mi_arena_t* arena = mi_arena_from_index(arena_idx); + mi_assert_internal(arena != NULL); + // set abandonment atomically + mi_subproc_t* const subproc = segment->subproc; // don't access the segment after setting it abandoned + const bool was_unmarked = _mi_bitmap_claim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx, NULL); + if (was_unmarked) { mi_atomic_increment_relaxed(&subproc->abandoned_count); } + mi_assert_internal(was_unmarked); + mi_assert_internal(_mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); +} + + +/* ----------------------------------------------------------- + Iterate through the abandoned blocks/segments using a cursor. + This is used for reclaiming and abandoned block visiting. +----------------------------------------------------------- */ + +// start a cursor at a randomized arena +void _mi_arena_field_cursor_init(mi_heap_t* heap, mi_subproc_t* subproc, bool visit_all, mi_arena_field_cursor_t* current) { + mi_assert_internal(heap == NULL || heap->tld->segments.subproc == subproc); + current->bitmap_idx = 0; + current->subproc = subproc; + current->visit_all = visit_all; + current->hold_visit_lock = false; + const size_t abandoned_count = mi_atomic_load_relaxed(&subproc->abandoned_count); + const size_t abandoned_list_count = mi_atomic_load_relaxed(&subproc->abandoned_os_list_count); + const size_t max_arena = mi_arena_get_count(); + if (heap != NULL && heap->arena_id != _mi_arena_id_none()) { + // for a heap that is bound to one arena, only visit that arena + current->start = mi_arena_id_index(heap->arena_id); + current->end = current->start + 1; + current->os_list_count = 0; + } + else { + // otherwise visit all starting at a random location + if (abandoned_count > abandoned_list_count && max_arena > 0) { + current->start = (heap == NULL || max_arena == 0 ? 0 : (mi_arena_id_t)(_mi_heap_random_next(heap) % max_arena)); + current->end = current->start + max_arena; + } + else { + current->start = 0; + current->end = 0; + } + current->os_list_count = abandoned_list_count; // max entries to visit in the os abandoned list + } + mi_assert_internal(current->start <= max_arena); +} + +void _mi_arena_field_cursor_done(mi_arena_field_cursor_t* current) { + if (current->hold_visit_lock) { + mi_lock_release(¤t->subproc->abandoned_os_visit_lock); + current->hold_visit_lock = false; + } +} + +static mi_segment_t* mi_arena_segment_clear_abandoned_at(mi_arena_t* arena, mi_subproc_t* subproc, mi_bitmap_index_t bitmap_idx) { + // try to reclaim an abandoned segment in the arena atomically + if (!_mi_bitmap_unclaim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx)) return NULL; + mi_assert_internal(_mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); + mi_segment_t* segment = (mi_segment_t*)mi_arena_block_start(arena, bitmap_idx); + mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id) == 0); + // check that the segment belongs to our sub-process + // note: this is the reason we need the `abandoned_visit` lock in the case abandoned visiting is enabled. + // without the lock an abandoned visit may otherwise fail to visit all abandoned segments in the sub-process. + // for regular reclaim it is fine to miss one sometimes so without abandoned visiting we don't need the `abandoned_visit` lock. + if (segment->subproc != subproc) { + // it is from another sub-process, re-mark it and continue searching + const bool was_zero = _mi_bitmap_claim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx, NULL); + mi_assert_internal(was_zero); MI_UNUSED(was_zero); + return NULL; + } + else { + // success, we unabandoned a segment in our sub-process + mi_atomic_decrement_relaxed(&subproc->abandoned_count); + return segment; + } +} + +static mi_segment_t* mi_arena_segment_clear_abandoned_next_field(mi_arena_field_cursor_t* previous) { + const size_t max_arena = mi_arena_get_count(); + size_t field_idx = mi_bitmap_index_field(previous->bitmap_idx); + size_t bit_idx = mi_bitmap_index_bit_in_field(previous->bitmap_idx) + 1; + // visit arena's (from the previous cursor) + for (; previous->start < previous->end; previous->start++, field_idx = 0, bit_idx = 0) { + // index wraps around + size_t arena_idx = (previous->start >= max_arena ? previous->start % max_arena : previous->start); + mi_arena_t* arena = mi_arena_from_index(arena_idx); + if (arena != NULL) { + bool has_lock = false; + // visit the abandoned fields (starting at previous_idx) + for (; field_idx < arena->field_count; field_idx++, bit_idx = 0) { + size_t field = mi_atomic_load_relaxed(&arena->blocks_abandoned[field_idx]); + if mi_unlikely(field != 0) { // skip zero fields quickly + // we only take the arena lock if there are actually abandoned segments present + if (!has_lock && mi_option_is_enabled(mi_option_visit_abandoned)) { + has_lock = (previous->visit_all ? mi_lock_acquire(&arena->abandoned_visit_lock) : mi_lock_try_acquire(&arena->abandoned_visit_lock)); + if (!has_lock) { + if (previous->visit_all) { + _mi_error_message(EFAULT, "internal error: failed to visit all abandoned segments due to failure to acquire the visitor lock"); + } + // skip to next arena + break; + } + } + mi_assert_internal(has_lock || !mi_option_is_enabled(mi_option_visit_abandoned)); + // visit each set bit in the field (todo: maybe use `ctz` here?) + for (; bit_idx < MI_BITMAP_FIELD_BITS; bit_idx++) { + // pre-check if the bit is set + size_t mask = ((size_t)1 << bit_idx); + if mi_unlikely((field & mask) == mask) { + previous->bitmap_idx = mi_bitmap_index_create(field_idx, bit_idx); + mi_segment_t* const segment = mi_arena_segment_clear_abandoned_at(arena, previous->subproc, previous->bitmap_idx); + if (segment != NULL) { + //mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx)); + if (has_lock) { mi_lock_release(&arena->abandoned_visit_lock); } + return segment; + } + } + } + } + } + if (has_lock) { mi_lock_release(&arena->abandoned_visit_lock); } + } + } + return NULL; +} + +static mi_segment_t* mi_arena_segment_clear_abandoned_next_list(mi_arena_field_cursor_t* previous) { + // go through the abandoned_os_list + // we only allow one thread per sub-process to do to visit guarded by the `abandoned_os_visit_lock`. + // The lock is released when the cursor is released. + if (!previous->hold_visit_lock) { + previous->hold_visit_lock = (previous->visit_all ? mi_lock_acquire(&previous->subproc->abandoned_os_visit_lock) + : mi_lock_try_acquire(&previous->subproc->abandoned_os_visit_lock)); + if (!previous->hold_visit_lock) { + if (previous->visit_all) { + _mi_error_message(EFAULT, "internal error: failed to visit all abandoned segments due to failure to acquire the OS visitor lock"); + } + return NULL; // we cannot get the lock, give up + } + } + // One list entry at a time + while (previous->os_list_count > 0) { + previous->os_list_count--; + const bool has_lock = mi_lock_acquire(&previous->subproc->abandoned_os_lock); // this could contend with concurrent OS block abandonment and reclaim from `free` + if (has_lock) { + mi_segment_t* segment = previous->subproc->abandoned_os_list; + // pop from head of the list, a subsequent mark will push at the end (and thus we iterate through os_list_count entries) + if (segment == NULL || mi_arena_segment_os_clear_abandoned(segment, false /* we already have the lock */)) { + mi_lock_release(&previous->subproc->abandoned_os_lock); + return segment; + } + // already abandoned, try again + mi_lock_release(&previous->subproc->abandoned_os_lock); + } + else { + _mi_error_message(EFAULT, "failed to acquire abandoned OS list lock during abandoned block visit\n"); + return NULL; + } + } + // done + mi_assert_internal(previous->os_list_count == 0); + return NULL; +} + + +// reclaim abandoned segments +// this does not set the thread id (so it appears as still abandoned) +mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_field_cursor_t* previous) { + if (previous->start < previous->end) { + // walk the arena + mi_segment_t* segment = mi_arena_segment_clear_abandoned_next_field(previous); + if (segment != NULL) { return segment; } + } + // no entries in the arena's anymore, walk the abandoned OS list + mi_assert_internal(previous->start == previous->end); + return mi_arena_segment_clear_abandoned_next_list(previous); +} + + +bool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + // (unfortunately) the visit_abandoned option must be enabled from the start. + // This is to avoid taking locks if abandoned list visiting is not required (as for most programs) + if (!mi_option_is_enabled(mi_option_visit_abandoned)) { + _mi_error_message(EFAULT, "internal error: can only visit abandoned blocks when MIMALLOC_VISIT_ABANDONED=ON"); + return false; + } + mi_arena_field_cursor_t current; + _mi_arena_field_cursor_init(NULL, _mi_subproc_from_id(subproc_id), true /* visit all (blocking) */, ¤t); + mi_segment_t* segment; + bool ok = true; + while (ok && (segment = _mi_arena_segment_clear_abandoned_next(¤t)) != NULL) { + ok = _mi_segment_visit_blocks(segment, heap_tag, visit_blocks, visitor, arg); + _mi_arena_segment_mark_abandoned(segment); + } + _mi_arena_field_cursor_done(¤t); + return ok; +} diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/arena.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/arena.c index 62bea78..3bb8f50 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/arena.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/arena.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2019-2023, Microsoft Research, Daan Leijen +Copyright (c) 2019-2024, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -11,68 +11,66 @@ large blocks (>= MI_ARENA_MIN_BLOCK_SIZE, 4MiB). In contrast to the rest of mimalloc, the arenas are shared between threads and need to be accessed using atomic operations. -Arenas are used to for huge OS page (1GiB) reservations or for reserving +Arenas are also used to for huge OS page (1GiB) reservations or for reserving OS memory upfront which can be improve performance or is sometimes needed on embedded devices. We can also employ this with WASI or `sbrk` systems to reserve large arenas upfront and be able to reuse the memory more effectively. The arena allocation needs to be thread safe and we use an atomic bitmap to allocate. -----------------------------------------------------------------------------*/ + #include "mimalloc.h" #include "mimalloc/internal.h" #include "mimalloc/atomic.h" +#include "bitmap.h" -#include <string.h> // memset -#include <errno.h> // ENOMEM - -#include "bitmap.h" // atomic bitmap /* ----------------------------------------------------------- Arena allocation ----------------------------------------------------------- */ -// Block info: bit 0 contains the `in_use` bit, the upper bits the -// size in count of arena blocks. -typedef uintptr_t mi_block_info_t; -#define MI_ARENA_BLOCK_SIZE (MI_SEGMENT_SIZE) // 64MiB (must be at least MI_SEGMENT_ALIGN) -#define MI_ARENA_MIN_OBJ_SIZE (MI_ARENA_BLOCK_SIZE/2) // 32MiB -#define MI_MAX_ARENAS (112) // not more than 126 (since we use 7 bits in the memid and an arena index + 1) - // A memory arena descriptor typedef struct mi_arena_s { - mi_arena_id_t id; // arena id; 0 for non-specific - mi_memid_t memid; // memid of the memory area - _Atomic(uint8_t*) start; // the start of the memory area - size_t block_count; // size of the area in arena blocks (of `MI_ARENA_BLOCK_SIZE`) - size_t field_count; // number of bitmap fields (where `field_count * MI_BITMAP_FIELD_BITS >= block_count`) - size_t meta_size; // size of the arena structure itself (including its bitmaps) - mi_memid_t meta_memid; // memid of the arena structure itself (OS or static allocation) - int numa_node; // associated NUMA node - bool exclusive; // only allow allocations if specifically for this arena - bool is_large; // memory area consists of large- or huge OS pages (always committed) - _Atomic(size_t) search_idx; // optimization to start the search for free blocks - _Atomic(mi_msecs_t) purge_expire; // expiration time when blocks should be decommitted from `blocks_decommit`. - mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero? - mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted) - mi_bitmap_field_t* blocks_purge; // blocks that can be (reset) decommitted. (can be NULL for memory that cannot be (reset) decommitted) - mi_bitmap_field_t* blocks_abandoned; // blocks that start with an abandoned segment. (This crosses API's but it is convenient to have here) - mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`) + mi_arena_id_t id; // arena id; 0 for non-specific + mi_memid_t memid; // memid of the memory area + _Atomic(uint8_t*)start; // the start of the memory area + size_t block_count; // size of the area in arena blocks (of `MI_ARENA_BLOCK_SIZE`) + size_t field_count; // number of bitmap fields (where `field_count * MI_BITMAP_FIELD_BITS >= block_count`) + size_t meta_size; // size of the arena structure itself (including its bitmaps) + mi_memid_t meta_memid; // memid of the arena structure itself (OS or static allocation) + int numa_node; // associated NUMA node + bool exclusive; // only allow allocations if specifically for this arena + bool is_large; // memory area consists of large- or huge OS pages (always committed) + mi_lock_t abandoned_visit_lock; // lock is only used when abandoned segments are being visited + _Atomic(size_t)search_idx; // optimization to start the search for free blocks + _Atomic(mi_msecs_t)purge_expire; // expiration time when blocks should be decommitted from `blocks_decommit`. + mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero? + mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted) + mi_bitmap_field_t* blocks_purge; // blocks that can be (reset) decommitted. (can be NULL for memory that cannot be (reset) decommitted) + mi_bitmap_field_t* blocks_abandoned; // blocks that start with an abandoned segment. (This crosses API's but it is convenient to have here) + mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`) + // do not add further fields here as the dirty, committed, purged, and abandoned bitmaps follow the inuse bitmap fields. } mi_arena_t; +#define MI_ARENA_BLOCK_SIZE (MI_SEGMENT_SIZE) // 64MiB (must be at least MI_SEGMENT_ALIGN) +#define MI_ARENA_MIN_OBJ_SIZE (MI_ARENA_BLOCK_SIZE/2) // 32MiB +#define MI_MAX_ARENAS (132) // Limited as the reservation exponentially increases (and takes up .bss) + // The available arenas static mi_decl_cache_align _Atomic(mi_arena_t*) mi_arenas[MI_MAX_ARENAS]; static mi_decl_cache_align _Atomic(size_t) mi_arena_count; // = 0 - -//static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int numa_node, bool exclusive, mi_memid_t memid, mi_arena_id_t* arena_id) mi_attr_noexcept; +#define MI_IN_ARENA_C +#include "arena-abandon.c" +#undef MI_IN_ARENA_C /* ----------------------------------------------------------- Arena id's id = arena_index + 1 ----------------------------------------------------------- */ -static size_t mi_arena_id_index(mi_arena_id_t id) { +size_t mi_arena_id_index(mi_arena_id_t id) { return (size_t)(id <= 0 ? MI_MAX_ARENAS : id - 1); } @@ -99,10 +97,16 @@ bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_id_t request_arena_i } } -bool _mi_arena_memid_is_os_allocated(mi_memid_t memid) { - return (memid.memkind == MI_MEM_OS); +size_t mi_arena_get_count(void) { + return mi_atomic_load_relaxed(&mi_arena_count); +} + +mi_arena_t* mi_arena_from_index(size_t idx) { + mi_assert_internal(idx < mi_arena_get_count()); + return mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[idx]); } + /* ----------------------------------------------------------- Arena allocations get a (currently) 16-bit memory id where the lower 8 bits are the arena id, and the upper bits the block index. @@ -128,7 +132,7 @@ static mi_memid_t mi_memid_create_arena(mi_arena_id_t id, bool is_exclusive, mi_ return memid; } -static bool mi_arena_memid_indices(mi_memid_t memid, size_t* arena_index, mi_bitmap_index_t* bitmap_index) { +bool mi_arena_memid_indices(mi_memid_t memid, size_t* arena_index, mi_bitmap_index_t* bitmap_index) { mi_assert_internal(memid.memkind == MI_MEM_ARENA); *arena_index = mi_arena_id_index(memid.mem.arena.id); *bitmap_index = memid.mem.arena.block_index; @@ -139,23 +143,24 @@ static bool mi_arena_memid_indices(mi_memid_t memid, size_t* arena_index, mi_bit /* ----------------------------------------------------------- Special static area for mimalloc internal structures - to avoid OS calls (for example, for the arena metadata) + to avoid OS calls (for example, for the arena metadata (~= 256b)) ----------------------------------------------------------- */ -#define MI_ARENA_STATIC_MAX (MI_INTPTR_SIZE*MI_KiB) // 8 KiB on 64-bit +#define MI_ARENA_STATIC_MAX ((MI_INTPTR_SIZE/2)*MI_KiB) // 4 KiB on 64-bit -static uint8_t mi_arena_static[MI_ARENA_STATIC_MAX]; -static _Atomic(size_t) mi_arena_static_top; +static mi_decl_cache_align uint8_t mi_arena_static[MI_ARENA_STATIC_MAX]; // must be cache aligned, see issue #895 +static mi_decl_cache_align _Atomic(size_t) mi_arena_static_top; static void* mi_arena_static_zalloc(size_t size, size_t alignment, mi_memid_t* memid) { *memid = _mi_memid_none(); if (size == 0 || size > MI_ARENA_STATIC_MAX) return NULL; - if ((mi_atomic_load_relaxed(&mi_arena_static_top) + size) > MI_ARENA_STATIC_MAX) return NULL; + const size_t toplow = mi_atomic_load_relaxed(&mi_arena_static_top); + if ((toplow + size) > MI_ARENA_STATIC_MAX) return NULL; // try to claim space - if (alignment == 0) { alignment = 1; } + if (alignment < MI_MAX_ALIGN_SIZE) { alignment = MI_MAX_ALIGN_SIZE; } const size_t oversize = size + alignment - 1; - if (oversize > MI_ARENA_STATIC_MAX) return NULL; + if (toplow + oversize > MI_ARENA_STATIC_MAX) return NULL; const size_t oldtop = mi_atomic_add_acq_rel(&mi_arena_static_top, oversize); size_t top = oldtop + oversize; if (top > MI_ARENA_STATIC_MAX) { @@ -169,11 +174,11 @@ static void* mi_arena_static_zalloc(size_t size, size_t alignment, mi_memid_t* m memid->initially_zero = true; const size_t start = _mi_align_up(oldtop, alignment); uint8_t* const p = &mi_arena_static[start]; - _mi_memzero(p, size); + _mi_memzero_aligned(p, size); return p; } -static void* mi_arena_meta_zalloc(size_t size, mi_memid_t* memid, mi_stats_t* stats) { +void* _mi_arena_meta_zalloc(size_t size, mi_memid_t* memid) { *memid = _mi_memid_none(); // try static @@ -181,7 +186,7 @@ static void* mi_arena_meta_zalloc(size_t size, mi_memid_t* memid, mi_stats_t* st if (p != NULL) return p; // or fall back to the OS - p = _mi_os_alloc(size, memid, stats); + p = _mi_os_alloc(size, memid, &_mi_stats_main); if (p == NULL) return NULL; // zero the OS memory if needed @@ -192,16 +197,16 @@ static void* mi_arena_meta_zalloc(size_t size, mi_memid_t* memid, mi_stats_t* st return p; } -static void mi_arena_meta_free(void* p, mi_memid_t memid, size_t size, mi_stats_t* stats) { +void _mi_arena_meta_free(void* p, mi_memid_t memid, size_t size) { if (mi_memkind_is_os(memid.memkind)) { - _mi_os_free(p, size, memid, stats); + _mi_os_free(p, size, memid, &_mi_stats_main); } else { mi_assert(memid.memkind == MI_MEM_STATIC); } } -static void* mi_arena_block_start(mi_arena_t* arena, mi_bitmap_index_t bindex) { +void* mi_arena_block_start(mi_arena_t* arena, mi_bitmap_index_t bindex) { return (arena->start + mi_arena_block_size(mi_bitmap_index_bit(bindex))); } @@ -291,7 +296,7 @@ static void* mi_arena_try_alloc_at_id(mi_arena_id_t arena_id, bool match_numa_no mi_assert_internal(size <= mi_arena_block_size(bcount)); // Check arena suitability - mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[arena_index]); + mi_arena_t* arena = mi_arena_from_index(arena_index); if (arena == NULL) return NULL; if (!allow_large && arena->is_large) return NULL; if (!mi_arena_id_is_suitable(arena->id, arena->exclusive, req_arena_id)) return NULL; @@ -359,8 +364,14 @@ static bool mi_arena_reserve(size_t req_size, bool allow_large, mi_arena_id_t re arena_reserve = arena_reserve/4; // be conservative if virtual reserve is not supported (for WASM for example) } arena_reserve = _mi_align_up(arena_reserve, MI_ARENA_BLOCK_SIZE); + arena_reserve = _mi_align_up(arena_reserve, MI_SEGMENT_SIZE); if (arena_count >= 8 && arena_count <= 128) { - arena_reserve = ((size_t)1<<(arena_count/8)) * arena_reserve; // scale up the arena sizes exponentially + // scale up the arena sizes exponentially every 8 entries (128 entries get to 589TiB) + const size_t multiplier = (size_t)1 << _mi_clamp(arena_count/8, 0, 16 ); + size_t reserve = 0; + if (!mi_mul_overflow(multiplier, arena_reserve, &reserve)) { + arena_reserve = reserve; + } } if (arena_reserve < req_size) return false; // should be able to at least handle the current allocation size @@ -505,7 +516,7 @@ static bool mi_arena_purge_range(mi_arena_t* arena, size_t idx, size_t startidx, size_t bitidx = startidx; bool all_purged = false; while (bitidx < endidx) { - // count consequetive ones in the purge mask + // count consecutive ones in the purge mask size_t count = 0; while (bitidx + count < endidx && (purge & ((size_t)1 << (bitidx + count))) != 0) { count++; @@ -542,11 +553,12 @@ static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force, mi if (purge != 0) { size_t bitidx = 0; while (bitidx < MI_BITMAP_FIELD_BITS) { - // find consequetive range of ones in the purge mask + // find consecutive range of ones in the purge mask size_t bitlen = 0; while (bitidx + bitlen < MI_BITMAP_FIELD_BITS && (purge & ((size_t)1 << (bitidx + bitlen))) != 0) { bitlen++; } + // temporarily claim the purge range as "in-use" to be thread-safe with allocation // try to claim the longest range of corresponding in_use bits const mi_bitmap_index_t bitmap_index = mi_bitmap_index_create(i, bitidx); while( bitlen > 0 ) { @@ -615,6 +627,9 @@ void _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memi if (size==0) return; const bool all_committed = (committed_size == size); + // need to set all memory to undefined as some parts may still be marked as no_access (like padding etc.) + mi_track_mem_undefined(p,size); + if (mi_memkind_is_os(memid.memkind)) { // was a direct OS allocation, pass through if (!all_committed && committed_size > 0) { @@ -644,9 +659,6 @@ void _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memi return; } - // need to set all memory to undefined as some parts may still be marked as no_access (like padding etc.) - mi_track_mem_undefined(p,size); - // potentially decommit if (arena->memid.is_pinned || arena->blocks_committed == NULL) { mi_assert_internal(all_committed); @@ -696,6 +708,7 @@ static void mi_arenas_unsafe_destroy(void) { for (size_t i = 0; i < max_arena; i++) { mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[i]); if (arena != NULL) { + mi_lock_done(&arena->abandoned_visit_lock); if (arena->start != NULL && mi_memkind_is_os(arena->memid.memkind)) { mi_atomic_store_ptr_release(mi_arena_t, &mi_arenas[i], NULL); _mi_os_free(arena->start, mi_arena_size(arena), arena->memid, &_mi_stats_main); @@ -703,7 +716,7 @@ static void mi_arenas_unsafe_destroy(void) { else { new_max_arena = i; } - mi_arena_meta_free(arena, arena->meta_memid, arena->meta_size, &_mi_stats_main); + _mi_arena_meta_free(arena, arena->meta_memid, arena->meta_size); } } @@ -728,7 +741,7 @@ void _mi_arena_unsafe_destroy_all(mi_stats_t* stats) { bool _mi_arena_contains(const void* p) { const size_t max_arena = mi_atomic_load_relaxed(&mi_arena_count); for (size_t i = 0; i < max_arena; i++) { - mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[i]); + mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[i]); if (arena != NULL && arena->start <= (const uint8_t*)p && arena->start + mi_arena_block_size(arena->block_count) > (const uint8_t*)p) { return true; } @@ -737,139 +750,6 @@ bool _mi_arena_contains(const void* p) { } /* ----------------------------------------------------------- - Abandoned blocks/segments. - This is used to atomically abandon/reclaim segments - (and crosses the arena API but it is convenient to have here). - Abandoned segments still have live blocks; they get reclaimed - when a thread frees a block in it, or when a thread needs a fresh - segment; these threads scan the abandoned segments through - the arena bitmaps. ------------------------------------------------------------ */ - -// Maintain a count of all abandoned segments -static mi_decl_cache_align _Atomic(size_t)abandoned_count; - -size_t _mi_arena_segment_abandoned_count(void) { - return mi_atomic_load_relaxed(&abandoned_count); -} - -// reclaim a specific abandoned segment; `true` on success. -// sets the thread_id. -bool _mi_arena_segment_clear_abandoned(mi_segment_t* segment ) -{ - if (segment->memid.memkind != MI_MEM_ARENA) { - // not in an arena, consider it un-abandoned now. - // but we need to still claim it atomically -- we use the thread_id for that. - size_t expected = 0; - if (mi_atomic_cas_strong_acq_rel(&segment->thread_id, &expected, _mi_thread_id())) { - mi_atomic_decrement_relaxed(&abandoned_count); - return true; - } - else { - return false; - } - } - // arena segment: use the blocks_abandoned bitmap. - size_t arena_idx; - size_t bitmap_idx; - mi_arena_memid_indices(segment->memid, &arena_idx, &bitmap_idx); - mi_assert_internal(arena_idx < MI_MAX_ARENAS); - mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[arena_idx]); - mi_assert_internal(arena != NULL); - bool was_marked = _mi_bitmap_unclaim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx); - if (was_marked) { - mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id) == 0); - mi_atomic_decrement_relaxed(&abandoned_count); - mi_atomic_store_release(&segment->thread_id, _mi_thread_id()); - } - // mi_assert_internal(was_marked); - mi_assert_internal(!was_marked || _mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); - //mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx)); - return was_marked; -} - -// mark a specific segment as abandoned -// clears the thread_id. -void _mi_arena_segment_mark_abandoned(mi_segment_t* segment) -{ - mi_atomic_store_release(&segment->thread_id, 0); - mi_assert_internal(segment->used == segment->abandoned); - if (segment->memid.memkind != MI_MEM_ARENA) { - // not in an arena; count it as abandoned and return - mi_atomic_increment_relaxed(&abandoned_count); - return; - } - size_t arena_idx; - size_t bitmap_idx; - mi_arena_memid_indices(segment->memid, &arena_idx, &bitmap_idx); - mi_assert_internal(arena_idx < MI_MAX_ARENAS); - mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[arena_idx]); - mi_assert_internal(arena != NULL); - const bool was_unmarked = _mi_bitmap_claim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx, NULL); - if (was_unmarked) { mi_atomic_increment_relaxed(&abandoned_count); } - mi_assert_internal(was_unmarked); - mi_assert_internal(_mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); -} - -// start a cursor at a randomized arena -void _mi_arena_field_cursor_init(mi_heap_t* heap, mi_arena_field_cursor_t* current) { - const size_t max_arena = mi_atomic_load_relaxed(&mi_arena_count); - current->start = (max_arena == 0 ? 0 : (mi_arena_id_t)( _mi_heap_random_next(heap) % max_arena)); - current->count = 0; - current->bitmap_idx = 0; -} - -// reclaim abandoned segments -// this does not set the thread id (so it appears as still abandoned) -mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_field_cursor_t* previous ) -{ - const int max_arena = (int)mi_atomic_load_relaxed(&mi_arena_count); - if (max_arena <= 0 || mi_atomic_load_relaxed(&abandoned_count) == 0) return NULL; - - int count = previous->count; - size_t field_idx = mi_bitmap_index_field(previous->bitmap_idx); - size_t bit_idx = mi_bitmap_index_bit_in_field(previous->bitmap_idx) + 1; - // visit arena's (from previous) - for (; count < max_arena; count++, field_idx = 0, bit_idx = 0) { - mi_arena_id_t arena_idx = previous->start + count; - if (arena_idx >= max_arena) { arena_idx = arena_idx % max_arena; } // wrap around - mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[arena_idx]); - if (arena != NULL) { - // visit the abandoned fields (starting at previous_idx) - for ( ; field_idx < arena->field_count; field_idx++, bit_idx = 0) { - size_t field = mi_atomic_load_relaxed(&arena->blocks_abandoned[field_idx]); - if mi_unlikely(field != 0) { // skip zero fields quickly - // visit each set bit in the field (todo: maybe use `ctz` here?) - for ( ; bit_idx < MI_BITMAP_FIELD_BITS; bit_idx++) { - // pre-check if the bit is set - size_t mask = ((size_t)1 << bit_idx); - if mi_unlikely((field & mask) == mask) { - mi_bitmap_index_t bitmap_idx = mi_bitmap_index_create(field_idx, bit_idx); - // try to reclaim it atomically - if (_mi_bitmap_unclaim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx)) { - mi_atomic_decrement_relaxed(&abandoned_count); - previous->bitmap_idx = bitmap_idx; - previous->count = count; - mi_assert_internal(_mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); - mi_segment_t* segment = (mi_segment_t*)mi_arena_block_start(arena, bitmap_idx); - mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id) == 0); - //mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx)); - return segment; - } - } - } - } - } - } - } - // no more found - previous->bitmap_idx = 0; - previous->count = 0; - return NULL; -} - - -/* ----------------------------------------------------------- Add an arena. ----------------------------------------------------------- */ @@ -905,7 +785,7 @@ static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int const size_t bitmaps = (memid.is_pinned ? 3 : 5); const size_t asize = sizeof(mi_arena_t) + (bitmaps*fields*sizeof(mi_bitmap_field_t)); mi_memid_t meta_memid; - mi_arena_t* arena = (mi_arena_t*)mi_arena_meta_zalloc(asize, &meta_memid, &_mi_stats_main); // TODO: can we avoid allocating from the OS? + mi_arena_t* arena = (mi_arena_t*)_mi_arena_meta_zalloc(asize, &meta_memid); if (arena == NULL) return false; // already zero'd due to zalloc @@ -922,7 +802,8 @@ static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int arena->is_large = is_large; arena->purge_expire = 0; arena->search_idx = 0; - // consequetive bitmaps + mi_lock_init(&arena->abandoned_visit_lock); + // consecutive bitmaps arena->blocks_dirty = &arena->blocks_inuse[fields]; // just after inuse bitmap arena->blocks_abandoned = &arena->blocks_inuse[2 * fields]; // just after dirty bitmap arena->blocks_committed = (arena->memid.is_pinned ? NULL : &arena->blocks_inuse[3*fields]); // just after abandoned bitmap @@ -1025,7 +906,7 @@ void mi_debug_show_arenas(bool show_inuse, bool show_abandoned, bool show_purge) mi_debug_show_bitmap(" ", "committed blocks", arena->block_count, arena->blocks_committed, arena->field_count); } if (show_abandoned) { - abandoned_total += mi_debug_show_bitmap(" ", "abandoned blocks", arena->block_count, arena->blocks_abandoned, arena->field_count); + abandoned_total += mi_debug_show_bitmap(" ", "abandoned blocks", arena->block_count, arena->blocks_abandoned, arena->field_count); } if (show_purge && arena->blocks_purge != NULL) { purge_total += mi_debug_show_bitmap(" ", "purgeable blocks", arena->block_count, arena->blocks_purge, arena->field_count); @@ -1104,3 +985,4 @@ int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserv return err; } + diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/bitmap.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/bitmap.c index 4b6be66..976ba72 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/bitmap.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/bitmap.c @@ -34,17 +34,17 @@ static inline size_t mi_bitmap_mask_(size_t count, size_t bitidx) { } + /* ----------------------------------------------------------- Claim a bit sequence atomically ----------------------------------------------------------- */ // Try to atomically claim a sequence of `count` bits in a single // field at `idx` in `bitmap`. Returns `true` on success. -inline bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_t count, mi_bitmap_index_t* bitmap_idx) +bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_t count, mi_bitmap_index_t* bitmap_idx) { mi_assert_internal(bitmap_idx != NULL); mi_assert_internal(count <= MI_BITMAP_FIELD_BITS); - mi_assert_internal(count > 0); mi_bitmap_field_t* field = &bitmap[idx]; size_t map = mi_atomic_load_relaxed(field); if (map==MI_BITMAP_FIELD_FULL) return false; // short cut @@ -94,9 +94,9 @@ inline bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, cons return false; } -// Find `count` bits of 0 and set them to 1 atomically; returns `true` on success. + // Starts at idx, and wraps around to search in all `bitmap_fields` fields. -// `count` can be at most MI_BITMAP_FIELD_BITS and will never cross fields. +// For now, `count` can be at most MI_BITMAP_FIELD_BITS and will never cross fields. bool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx) { size_t idx = start_field_idx; for (size_t visited = 0; visited < bitmap_fields; visited++, idx++) { @@ -108,24 +108,6 @@ bool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fiel return false; } -// Like _mi_bitmap_try_find_from_claim but with an extra predicate that must be fullfilled -bool _mi_bitmap_try_find_from_claim_pred(mi_bitmap_t bitmap, const size_t bitmap_fields, - const size_t start_field_idx, const size_t count, - mi_bitmap_pred_fun_t pred_fun, void* pred_arg, - mi_bitmap_index_t* bitmap_idx) { - size_t idx = start_field_idx; - for (size_t visited = 0; visited < bitmap_fields; visited++, idx++) { - if (idx >= bitmap_fields) idx = 0; // wrap - if (_mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx)) { - if (pred_fun == NULL || pred_fun(*bitmap_idx, pred_arg)) { - return true; - } - // predicate returned false, unclaim and look further - _mi_bitmap_unclaim(bitmap, bitmap_fields, count, *bitmap_idx); - } - } - return false; -} // Set `count` bits at `bitmap_idx` to 0 atomically // Returns `true` if all `count` bits were 1 previously. @@ -246,7 +228,7 @@ static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bit // intermediate fields while (++field < final_field) { - newmap = MI_BITMAP_FIELD_FULL; + newmap = mi_bitmap_mask_(MI_BITMAP_FIELD_BITS, 0); map = 0; if (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)) { goto rollback; } } @@ -269,7 +251,7 @@ rollback: // (we just failed to claim `field` so decrement first) while (--field > initial_field) { newmap = 0; - map = MI_BITMAP_FIELD_FULL; + map = mi_bitmap_mask_(MI_BITMAP_FIELD_BITS, 0); mi_assert_internal(mi_atomic_load_relaxed(field) == map); mi_atomic_store_release(field, newmap); } diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/bitmap.h b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/bitmap.h index d8316b8..a1e7686 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/bitmap.h +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/bitmap.h @@ -40,11 +40,6 @@ static inline mi_bitmap_index_t mi_bitmap_index_create(size_t idx, size_t bitidx return (idx*MI_BITMAP_FIELD_BITS) + bitidx; } -// Create a bit index. -static inline mi_bitmap_index_t mi_bitmap_index_create_from_bit(size_t full_bitidx) { - return mi_bitmap_index_create(full_bitidx / MI_BITMAP_FIELD_BITS, full_bitidx % MI_BITMAP_FIELD_BITS); -} - // Get the field index from a bit index. static inline size_t mi_bitmap_index_field(mi_bitmap_index_t bitmap_idx) { return (bitmap_idx / MI_BITMAP_FIELD_BITS); @@ -72,10 +67,6 @@ bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_ // For now, `count` can be at most MI_BITMAP_FIELD_BITS and will never cross fields. bool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx); -// Like _mi_bitmap_try_find_from_claim but with an extra predicate that must be fullfilled -typedef bool (mi_cdecl *mi_bitmap_pred_fun_t)(mi_bitmap_index_t bitmap_idx, void* pred_arg); -bool _mi_bitmap_try_find_from_claim_pred(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_pred_fun_t pred_fun, void* pred_arg, mi_bitmap_index_t* bitmap_idx); - // Set `count` bits at `bitmap_idx` to 0 atomically // Returns `true` if all `count` bits were 1 previously. bool _mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/free.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/free.c index b9cb634..c6221fe 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/free.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/free.c @@ -9,7 +9,6 @@ terms of the MIT license. A copy of the license can be found in the file // add includes help an IDE #include "mimalloc.h" #include "mimalloc/internal.h" -#include "mimalloc/atomic.h" #include "mimalloc/prim.h" // _mi_prim_thread_id() #endif @@ -36,12 +35,10 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool mi_check_padding(page, block); if (track_stats) { mi_stat_free(page, block); } #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN - if (!mi_page_is_huge(page)) { // huge page content may be already decommitted - memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); - } + memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); #endif if (track_stats) { mi_track_free_size(block, mi_page_usable_size_of(page, block)); } // faster then mi_usable_size as we already know the page and that p is unaligned - + // actual free: push on the local free list mi_block_set_next(page, block, page->local_free); page->local_free = block; @@ -110,16 +107,10 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms #if (MI_DEBUG>0) if mi_unlikely(!mi_is_in_heap_region(p)) { - #if (MI_INTPTR_SIZE == 8 && defined(__linux__)) - if (((uintptr_t)p >> 40) != 0x7F) { // linux tends to align large blocks above 0x7F000000000 (issue #640) - #else - { - #endif - _mi_warning_message("%s: pointer might not point to a valid heap region: %p\n" - "(this may still be a valid very large allocation (over 64MiB))\n", msg, p); - if mi_likely(_mi_ptr_cookie(segment) == segment->cookie) { - _mi_warning_message("(yes, the previous pointer %p was valid after all)\n", p); - } + _mi_warning_message("%s: pointer might not point to a valid heap region: %p\n" + "(this may still be a valid very large allocation (over 64MiB))\n", msg, p); + if mi_likely(_mi_ptr_cookie(segment) == segment->cookie) { + _mi_warning_message("(yes, the previous pointer %p was valid after all)\n", p); } } #endif @@ -248,7 +239,8 @@ static void mi_decl_noinline mi_free_block_mt(mi_page_t* page, mi_segment_t* seg { // the segment is abandoned, try to reclaim it into our heap if (_mi_segment_attempt_reclaim(mi_heap_get_default(), segment)) { - mi_assert_internal(_mi_prim_thread_id() == mi_atomic_load_relaxed(&segment->thread_id)); + mi_assert_internal(_mi_thread_id() == mi_atomic_load_relaxed(&segment->thread_id)); + mi_assert_internal(mi_heap_get_default()->tld->segments.subproc == segment->subproc); mi_free(block); // recursively free as now it will be a local free in our heap return; } @@ -265,7 +257,7 @@ static void mi_decl_noinline mi_free_block_mt(mi_page_t* page, mi_segment_t* seg // for small size, ensure we can fit the delayed thread pointers without triggering overflow detection _mi_padding_shrink(page, block, sizeof(mi_block_t)); - if (segment->kind == MI_SEGMENT_HUGE) { + if (segment->page_kind == MI_PAGE_HUGE) { #if MI_HUGE_PAGE_ABANDON // huge page segments are always abandoned and can be freed immediately _mi_segment_huge_page_free(segment, page, block); @@ -501,26 +493,24 @@ static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) { // only maintain stats for smaller objects if requested #if (MI_STAT>0) static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { - #if (MI_STAT < 2) +#if (MI_STAT < 2) MI_UNUSED(block); - #endif +#endif mi_heap_t* const heap = mi_heap_get_default(); const size_t bsize = mi_page_usable_block_size(page); - #if (MI_STAT>1) +#if (MI_STAT>1) const size_t usize = mi_page_usable_size_of(page, block); mi_heap_stat_decrease(heap, malloc, usize); - #endif - if (bsize <= MI_MEDIUM_OBJ_SIZE_MAX) { +#endif + if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { mi_heap_stat_decrease(heap, normal, bsize); - #if (MI_STAT > 1) +#if (MI_STAT > 1) mi_heap_stat_decrease(heap, normal_bins[_mi_bin(bsize)], 1); - #endif - } - else if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { - mi_heap_stat_decrease(heap, large, bsize); +#endif } else { - mi_heap_stat_decrease(heap, huge, bsize); + const size_t bpsize = mi_page_block_size(page); // match stat in page.c:mi_huge_page_alloc + mi_heap_stat_decrease(heap, huge, bpsize); } } #else diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/heap.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/heap.c index 6c56edd..0d716f9 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/heap.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/heap.c @@ -32,7 +32,7 @@ static bool mi_heap_visit_pages(mi_heap_t* heap, heap_page_visitor_fun* fn, void #if MI_DEBUG>1 size_t total = heap->page_count; size_t count = 0; - #endif + #endif for (size_t i = 0; i <= MI_BIN_FULL; i++) { mi_page_queue_t* pq = &heap->pages[i]; @@ -95,11 +95,6 @@ static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t mi_assert_internal(mi_heap_page_is_valid(heap, pq, page, NULL, NULL)); mi_collect_t collect = *((mi_collect_t*)arg_collect); _mi_page_free_collect(page, collect >= MI_FORCE); - if (collect == MI_FORCE) { - // note: call before a potential `_mi_page_free` as the segment may be freed if this was the last used page in that segment. - mi_segment_t* segment = _mi_page_segment(page); - _mi_segment_collect(segment, true /* force? */, &heap->tld->segments); - } if (mi_page_all_free(page)) { // no more used blocks, free the page. // note: this will free retired pages as well. @@ -128,18 +123,21 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) const bool force = (collect >= MI_FORCE); _mi_deferred_free(heap, force); + // python/cpython#112532: we may be called from a thread that is not the owner of the heap + const bool is_main_thread = (_mi_is_main_thread() && heap->thread_id == _mi_thread_id()); + // note: never reclaim on collect but leave it to threads that need storage to reclaim - const bool force_main = - #ifdef NDEBUG + if ( + #ifdef NDEBUG collect == MI_FORCE - #else + #else collect >= MI_FORCE - #endif - && _mi_is_main_thread() && mi_heap_is_backing(heap) && !heap->no_reclaim; - - if (force_main) { + #endif + && is_main_thread && mi_heap_is_backing(heap) && !heap->no_reclaim) + { // the main thread is abandoned (end-of-program), try to reclaim all abandoned segments. // if all memory is freed by now, all segments should be freed. + // note: this only collects in the current subprocess _mi_abandoned_reclaim_all(heap, &heap->tld->segments); } @@ -159,12 +157,11 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) mi_heap_visit_pages(heap, &mi_heap_page_collect, &collect, NULL); mi_assert_internal( collect != MI_ABANDON || mi_atomic_load_ptr_acquire(mi_block_t,&heap->thread_delayed_free) == NULL ); - // collect abandoned segments (in particular, purge expired parts of segments in the abandoned segment list) - // note: forced purge can be quite expensive if many threads are created/destroyed so we do not force on abandonment - _mi_abandoned_collect(heap, collect == MI_FORCE /* force? */, &heap->tld->segments); - + // collect segments (purge pages, this can be expensive so don't force on abandonment) + _mi_segments_collect(collect == MI_FORCE, &heap->tld->segments); + // if forced, collect thread data cache on program-exit (or shared library unload) - if (force && _mi_is_main_thread() && mi_heap_is_backing(heap)) { + if (force && is_main_thread && mi_heap_is_backing(heap)) { _mi_thread_data_collect(); // collect thread data cache } @@ -208,27 +205,43 @@ mi_heap_t* mi_heap_get_backing(void) { return bheap; } -mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) { - mi_heap_t* bheap = mi_heap_get_backing(); - mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? - if (heap == NULL) return NULL; +void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool noreclaim, uint8_t tag) { _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t)); - heap->tld = bheap->tld; - heap->thread_id = _mi_thread_id(); - heap->arena_id = arena_id; - _mi_random_split(&bheap->random, &heap->random); - heap->cookie = _mi_heap_random_next(heap) | 1; + heap->tld = tld; + heap->thread_id = _mi_thread_id(); + heap->arena_id = arena_id; + heap->no_reclaim = noreclaim; + heap->tag = tag; + if (heap == tld->heap_backing) { + _mi_random_init(&heap->random); + } + else { + _mi_random_split(&tld->heap_backing->random, &heap->random); + } + heap->cookie = _mi_heap_random_next(heap) | 1; heap->keys[0] = _mi_heap_random_next(heap); heap->keys[1] = _mi_heap_random_next(heap); - heap->no_reclaim = true; // don't reclaim abandoned pages or otherwise destroy is unsafe // push on the thread local heaps list heap->next = heap->tld->heaps; heap->tld->heaps = heap; +} + +mi_decl_nodiscard mi_heap_t* mi_heap_new_ex(int heap_tag, bool allow_destroy, mi_arena_id_t arena_id) { + mi_heap_t* bheap = mi_heap_get_backing(); + mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? + if (heap == NULL) return NULL; + mi_assert(heap_tag >= 0 && heap_tag < 256); + _mi_heap_init(heap, bheap->tld, arena_id, allow_destroy /* no reclaim? */, (uint8_t)heap_tag /* heap tag */); return heap; } +mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) { + return mi_heap_new_ex(0 /* default heap tag */, false /* don't allow `mi_heap_destroy` */, arena_id); +} + mi_decl_nodiscard mi_heap_t* mi_heap_new(void) { - return mi_heap_new_in_arena(_mi_arena_id_none()); + // don't reclaim abandoned memory or otherwise destroy is unsafe + return mi_heap_new_ex(0 /* default heap tag */, true /* no reclaim */, _mi_arena_id_none()); } bool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid) { @@ -281,6 +294,18 @@ static void mi_heap_free(mi_heap_t* heap) { mi_free(heap); } +// return a heap on the same thread as `heap` specialized for the specified tag (if it exists) +mi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag) { + if (heap->tag == tag) { + return heap; + } + for (mi_heap_t *curr = heap->tld->heaps; curr != NULL; curr = curr->next) { + if (curr->tag == tag) { + return curr; + } + } + return NULL; +} /* ----------------------------------------------------------- Heap destroy @@ -297,13 +322,8 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_ // stats const size_t bsize = mi_page_block_size(page); - if (bsize > MI_MEDIUM_OBJ_SIZE_MAX) { - if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { - mi_heap_stat_decrease(heap, large, bsize); - } - else { - mi_heap_stat_decrease(heap, huge, bsize); - } + if (bsize > MI_LARGE_OBJ_SIZE_MAX) { + mi_heap_stat_decrease(heap, huge, bsize); } #if (MI_STAT) _mi_page_free_collect(page, false); // update used count @@ -501,54 +521,96 @@ bool mi_check_owned(const void* p) { enable visiting all blocks of all heaps across threads ----------------------------------------------------------- */ -// Separate struct to keep `mi_page_t` out of the public interface -typedef struct mi_heap_area_ex_s { - mi_heap_area_t area; - mi_page_t* page; -} mi_heap_area_ex_t; +void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) { + const size_t bsize = mi_page_block_size(page); + const size_t ubsize = mi_page_usable_block_size(page); + area->reserved = page->reserved * bsize; + area->committed = page->capacity * bsize; + area->blocks = mi_page_start(page); + area->used = page->used; // number of blocks in use (#553) + area->block_size = ubsize; + area->full_block_size = bsize; + area->heap_tag = page->heap_tag; +} -static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_visit_fun* visitor, void* arg) { - mi_assert(xarea != NULL); - if (xarea==NULL) return true; - const mi_heap_area_t* area = &xarea->area; - mi_page_t* page = xarea->page; + +static void mi_get_fast_divisor(size_t divisor, uint64_t* magic, size_t* shift) { + mi_assert_internal(divisor > 0 && divisor <= UINT32_MAX); + *shift = 64 - mi_clz(divisor - 1); + *magic = ((((uint64_t)1 << 32) * (((uint64_t)1 << *shift) - divisor)) / divisor + 1); +} + +static size_t mi_fast_divide(size_t n, uint64_t magic, size_t shift) { + mi_assert_internal(n <= UINT32_MAX); + return ((((uint64_t)n * magic) >> 32) + n) >> shift; +} + +bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg) { + mi_assert(area != NULL); + if (area==NULL) return true; mi_assert(page != NULL); if (page == NULL) return true; - _mi_page_free_collect(page,true); + _mi_page_free_collect(page,true); // collect both thread_delayed and local_free mi_assert_internal(page->local_free == NULL); if (page->used == 0) return true; - const size_t bsize = mi_page_block_size(page); - const size_t ubsize = mi_page_usable_block_size(page); // without padding - size_t psize; - uint8_t* pstart = _mi_segment_page_start(_mi_page_segment(page), page, &psize); + size_t psize; + uint8_t* const pstart = _mi_segment_page_start(_mi_page_segment(page), page, &psize); + mi_heap_t* const heap = mi_page_heap(page); + const size_t bsize = mi_page_block_size(page); + const size_t ubsize = mi_page_usable_block_size(page); // without padding + // optimize page with one block if (page->capacity == 1) { - // optimize page with one block mi_assert_internal(page->used == 1 && page->free == NULL); return visitor(mi_page_heap(page), area, pstart, ubsize, arg); } + mi_assert(bsize <= UINT32_MAX); + + // optimize full pages + if (page->used == page->capacity) { + uint8_t* block = pstart; + for (size_t i = 0; i < page->capacity; i++) { + if (!visitor(heap, area, block, ubsize, arg)) return false; + block += bsize; + } + return true; + } // create a bitmap of free blocks. #define MI_MAX_BLOCKS (MI_SMALL_PAGE_SIZE / sizeof(void*)) - uintptr_t free_map[MI_MAX_BLOCKS / sizeof(uintptr_t)]; - memset(free_map, 0, sizeof(free_map)); + uintptr_t free_map[MI_MAX_BLOCKS / MI_INTPTR_BITS]; + const uintptr_t bmapsize = _mi_divide_up(page->capacity, MI_INTPTR_BITS); + memset(free_map, 0, bmapsize * sizeof(intptr_t)); + if (page->capacity % MI_INTPTR_BITS != 0) { + // mark left-over bits at the end as free + size_t shift = (page->capacity % MI_INTPTR_BITS); + uintptr_t mask = (UINTPTR_MAX << shift); + free_map[bmapsize - 1] = mask; + } + + // fast repeated division by the block size + uint64_t magic; + size_t shift; + mi_get_fast_divisor(bsize, &magic, &shift); #if MI_DEBUG>1 size_t free_count = 0; #endif - for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page,block)) { + for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page, block)) { #if MI_DEBUG>1 free_count++; #endif mi_assert_internal((uint8_t*)block >= pstart && (uint8_t*)block < (pstart + psize)); size_t offset = (uint8_t*)block - pstart; mi_assert_internal(offset % bsize == 0); - size_t blockidx = offset / bsize; // Todo: avoid division? - mi_assert_internal( blockidx < MI_MAX_BLOCKS); - size_t bitidx = (blockidx / sizeof(uintptr_t)); - size_t bit = blockidx - (bitidx * sizeof(uintptr_t)); + mi_assert_internal(offset <= UINT32_MAX); + size_t blockidx = mi_fast_divide(offset, magic, shift); + mi_assert_internal(blockidx == offset / bsize); + mi_assert_internal(blockidx < MI_MAX_BLOCKS); + size_t bitidx = (blockidx / MI_INTPTR_BITS); + size_t bit = blockidx - (bitidx * MI_INTPTR_BITS); free_map[bitidx] |= ((uintptr_t)1 << bit); } mi_assert_internal(page->capacity == (free_count + page->used)); @@ -557,42 +619,53 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v #if MI_DEBUG>1 size_t used_count = 0; #endif - for (size_t i = 0; i < page->capacity; i++) { - size_t bitidx = (i / sizeof(uintptr_t)); - size_t bit = i - (bitidx * sizeof(uintptr_t)); - uintptr_t m = free_map[bitidx]; - if (bit == 0 && m == UINTPTR_MAX) { - i += (sizeof(uintptr_t) - 1); // skip a run of free blocks + uint8_t* block = pstart; + for (size_t i = 0; i < bmapsize; i++) { + if (free_map[i] == 0) { + // every block is in use + for (size_t j = 0; j < MI_INTPTR_BITS; j++) { + #if MI_DEBUG>1 + used_count++; + #endif + if (!visitor(heap, area, block, ubsize, arg)) return false; + block += bsize; + } } - else if ((m & ((uintptr_t)1 << bit)) == 0) { - #if MI_DEBUG>1 - used_count++; - #endif - uint8_t* block = pstart + (i * bsize); - if (!visitor(mi_page_heap(page), area, block, ubsize, arg)) return false; + else { + // visit the used blocks in the mask + uintptr_t m = ~free_map[i]; + while (m != 0) { + #if MI_DEBUG>1 + used_count++; + #endif + size_t bitidx = mi_ctz(m); + if (!visitor(heap, area, block + (bitidx * bsize), ubsize, arg)) return false; + m &= m - 1; // clear least significant bit + } + block += bsize * MI_INTPTR_BITS; } } mi_assert_internal(page->used == used_count); return true; } -typedef bool (mi_heap_area_visit_fun)(const mi_heap_t* heap, const mi_heap_area_ex_t* area, void* arg); +// Separate struct to keep `mi_page_t` out of the public interface +typedef struct mi_heap_area_ex_s { + mi_heap_area_t area; + mi_page_t* page; +} mi_heap_area_ex_t; + +typedef bool (mi_heap_area_visit_fun)(const mi_heap_t* heap, const mi_heap_area_ex_t* area, void* arg); + static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* vfun, void* arg) { MI_UNUSED(heap); MI_UNUSED(pq); mi_heap_area_visit_fun* fun = (mi_heap_area_visit_fun*)vfun; mi_heap_area_ex_t xarea; - const size_t bsize = mi_page_block_size(page); - const size_t ubsize = mi_page_usable_block_size(page); xarea.page = page; - xarea.area.reserved = page->reserved * bsize; - xarea.area.committed = page->capacity * bsize; - xarea.area.blocks = mi_page_start(page); - xarea.area.used = page->used; // number of blocks in use (#553) - xarea.area.block_size = ubsize; - xarea.area.full_block_size = bsize; + _mi_heap_area_init(&xarea.area, page); return fun(heap, &xarea, arg); } @@ -613,7 +686,7 @@ static bool mi_heap_area_visitor(const mi_heap_t* heap, const mi_heap_area_ex_t* mi_visit_blocks_args_t* args = (mi_visit_blocks_args_t*)arg; if (!args->visitor(heap, &xarea->area, NULL, xarea->area.block_size, args->arg)) return false; if (args->visit_blocks) { - return mi_heap_area_visit_blocks(xarea, args->visitor, args->arg); + return _mi_heap_area_visit_blocks(&xarea->area, xarea->page, args->visitor, args->arg); } else { return true; diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/init.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/init.c index 3316106..ead5a14 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/init.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/init.c @@ -25,6 +25,7 @@ const mi_page_t _mi_page_empty = { NULL, // local_free 0, // used 0, // block size shift + 0, // heap tag 0, // block_size NULL, // page_start #if (MI_PADDING || MI_ENCODE_FREELIST) @@ -33,14 +34,13 @@ const mi_page_t _mi_page_empty = { MI_ATOMIC_VAR_INIT(0), // xthread_free MI_ATOMIC_VAR_INIT(0), // xheap NULL, NULL - #if MI_INTPTR_SIZE==8 - , { 0 } // padding + #if MI_INTPTR_SIZE==4 + , { NULL } #endif }; #define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty) -#if (MI_SMALL_WSIZE_MAX==128) #if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8) #define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } #elif (MI_PADDING>0) @@ -48,9 +48,7 @@ const mi_page_t _mi_page_empty = { #else #define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() } #endif -#else -#error "define right initialization sizes corresponding to MI_SMALL_WSIZE_MAX" -#endif + // Empty page queues for every bin #define QNULL(sz) { NULL, NULL, (sz)*sizeof(uintptr_t) } @@ -65,8 +63,8 @@ const mi_page_t _mi_page_empty = { QNULL( 10240), QNULL( 12288), QNULL( 14336), QNULL( 16384), QNULL( 20480), QNULL( 24576), QNULL( 28672), QNULL( 32768), /* 56 */ \ QNULL( 40960), QNULL( 49152), QNULL( 57344), QNULL( 65536), QNULL( 81920), QNULL( 98304), QNULL(114688), QNULL(131072), /* 64 */ \ QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \ - QNULL(MI_MEDIUM_OBJ_WSIZE_MAX + 1 /* 655360, Huge queue */), \ - QNULL(MI_MEDIUM_OBJ_WSIZE_MAX + 2) /* Full queue */ } + QNULL(MI_LARGE_OBJ_WSIZE_MAX + 1 /* 655360, Huge queue */), \ + QNULL(MI_LARGE_OBJ_WSIZE_MAX + 2) /* Full queue */ } #define MI_STAT_COUNT_NULL() {0,0,0,0} @@ -88,22 +86,9 @@ const mi_page_t _mi_page_empty = { MI_STAT_COUNT_NULL(), \ { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ - { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ - { 0, 0 } \ + { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 } \ MI_STAT_COUNT_END_NULL() - -// Empty slice span queues for every bin -#define SQNULL(sz) { NULL, NULL, sz } -#define MI_SEGMENT_SPAN_QUEUES_EMPTY \ - { SQNULL(1), \ - SQNULL( 1), SQNULL( 2), SQNULL( 3), SQNULL( 4), SQNULL( 5), SQNULL( 6), SQNULL( 7), SQNULL( 10), /* 8 */ \ - SQNULL( 12), SQNULL( 14), SQNULL( 16), SQNULL( 20), SQNULL( 24), SQNULL( 28), SQNULL( 32), SQNULL( 40), /* 16 */ \ - SQNULL( 48), SQNULL( 56), SQNULL( 64), SQNULL( 80), SQNULL( 96), SQNULL( 112), SQNULL( 128), SQNULL( 160), /* 24 */ \ - SQNULL( 192), SQNULL( 224), SQNULL( 256), SQNULL( 320), SQNULL( 384), SQNULL( 448), SQNULL( 512), SQNULL( 640), /* 32 */ \ - SQNULL( 768), SQNULL( 896), SQNULL( 1024) /* 35 */ } - - // -------------------------------------------------------- // Statically allocate an empty heap as the initial // thread local value for the default heap, @@ -124,22 +109,12 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = { 0, // page count MI_BIN_FULL, 0, // page retired min/max NULL, // next - false, + false, // can reclaim + 0, // tag MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY }; -#define tld_empty_stats ((mi_stats_t*)((uint8_t*)&tld_empty + offsetof(mi_tld_t,stats))) -#define tld_empty_os ((mi_os_tld_t*)((uint8_t*)&tld_empty + offsetof(mi_tld_t,os))) - -mi_decl_cache_align static const mi_tld_t tld_empty = { - 0, - false, - NULL, NULL, - { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, 0, tld_empty_stats, tld_empty_os }, // segments - { 0, tld_empty_stats }, // os - { MI_STATS_NULL } // stats -}; mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { return _mi_prim_thread_id(); @@ -150,15 +125,20 @@ mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty; extern mi_heap_t _mi_heap_main; -static mi_tld_t tld_main = { +static mi_decl_cache_align mi_subproc_t mi_subproc_default; + +static mi_decl_cache_align mi_tld_t tld_main = { 0, false, - &_mi_heap_main, & _mi_heap_main, - { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, 0, &tld_main.stats, &tld_main.os }, // segments + &_mi_heap_main, &_mi_heap_main, + { { NULL, NULL }, {NULL ,NULL}, {NULL ,NULL, 0}, + 0, 0, 0, 0, 0, &mi_subproc_default, + &tld_main.stats, &tld_main.os + }, // segments { 0, &tld_main.stats }, // os { MI_STATS_NULL } // stats }; -mi_heap_t _mi_heap_main = { +mi_decl_cache_align mi_heap_t _mi_heap_main = { &tld_main, MI_ATOMIC_VAR_INIT(NULL), 0, // thread id @@ -170,6 +150,7 @@ mi_heap_t _mi_heap_main = { MI_BIN_FULL, 0, // page retired min/max NULL, // next heap false, // can reclaim + 0, // tag MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY }; @@ -191,6 +172,8 @@ static void mi_heap_main_init(void) { _mi_heap_main.cookie = _mi_heap_random_next(&_mi_heap_main); _mi_heap_main.keys[0] = _mi_heap_random_next(&_mi_heap_main); _mi_heap_main.keys[1] = _mi_heap_random_next(&_mi_heap_main); + mi_lock_init(&mi_subproc_default.abandoned_os_lock); + mi_lock_init(&mi_subproc_default.abandoned_os_visit_lock); } } @@ -201,6 +184,58 @@ mi_heap_t* _mi_heap_main_get(void) { /* ----------------------------------------------------------- + Sub process +----------------------------------------------------------- */ + +mi_subproc_id_t mi_subproc_main(void) { + return NULL; +} + +mi_subproc_id_t mi_subproc_new(void) { + mi_memid_t memid = _mi_memid_none(); + mi_subproc_t* subproc = (mi_subproc_t*)_mi_arena_meta_zalloc(sizeof(mi_subproc_t), &memid); + if (subproc == NULL) return NULL; + subproc->memid = memid; + subproc->abandoned_os_list = NULL; + mi_lock_init(&subproc->abandoned_os_lock); + mi_lock_init(&subproc->abandoned_os_visit_lock); + return subproc; +} + +mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) { + return (subproc_id == NULL ? &mi_subproc_default : (mi_subproc_t*)subproc_id); +} + +void mi_subproc_delete(mi_subproc_id_t subproc_id) { + if (subproc_id == NULL) return; + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + // check if there are no abandoned segments still.. + bool safe_to_delete = false; + if (mi_lock_acquire(&subproc->abandoned_os_lock)) { + if (subproc->abandoned_os_list == NULL) { + safe_to_delete = true; + } + mi_lock_release(&subproc->abandoned_os_lock); + } + if (!safe_to_delete) return; + // safe to release + // todo: should we refcount subprocesses? + mi_lock_done(&subproc->abandoned_os_lock); + mi_lock_done(&subproc->abandoned_os_visit_lock); + _mi_arena_meta_free(subproc, subproc->memid, sizeof(mi_subproc_t)); +} + +void mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) { + mi_heap_t* heap = mi_heap_get_default(); + if (heap == NULL) return; + mi_assert(heap->tld->segments.subproc == &mi_subproc_default); + if (heap->tld->segments.subproc != &mi_subproc_default) return; + heap->tld->segments.subproc = _mi_subproc_from_id(subproc_id); +} + + + +/* ----------------------------------------------------------- Initialization and freeing of the thread local heaps ----------------------------------------------------------- */ @@ -217,7 +252,7 @@ typedef struct mi_thread_data_s { // destroy many OS threads, this may causes too much overhead // per thread so we maintain a small cache of recently freed metadata. -#define TD_CACHE_SIZE (16) +#define TD_CACHE_SIZE (32) static _Atomic(mi_thread_data_t*) td_cache[TD_CACHE_SIZE]; static mi_thread_data_t* mi_thread_data_zalloc(void) { @@ -288,7 +323,7 @@ void _mi_thread_data_collect(void) { } // Initialize the thread local default heap, called from `mi_thread_init` -static bool _mi_heap_init(void) { +static bool _mi_thread_heap_init(void) { if (mi_heap_is_initialized(mi_prim_get_default_heap())) return true; if (_mi_is_main_thread()) { // mi_assert_internal(_mi_heap_main.thread_id != 0); // can happen on freeBSD where alloc is called before any initialization @@ -304,26 +339,26 @@ static bool _mi_heap_init(void) { mi_tld_t* tld = &td->tld; mi_heap_t* heap = &td->heap; - _mi_memcpy_aligned(tld, &tld_empty, sizeof(*tld)); - _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(*heap)); - heap->thread_id = _mi_thread_id(); - _mi_random_init(&heap->random); - heap->cookie = _mi_heap_random_next(heap) | 1; - heap->keys[0] = _mi_heap_random_next(heap); - heap->keys[1] = _mi_heap_random_next(heap); - heap->tld = tld; - tld->heap_backing = heap; - tld->heaps = heap; - tld->segments.stats = &tld->stats; - tld->segments.os = &tld->os; - tld->os.stats = &tld->stats; + _mi_tld_init(tld, heap); // must be before `_mi_heap_init` + _mi_heap_init(heap, tld, _mi_arena_id_none(), false /* can reclaim */, 0 /* default tag */); _mi_heap_set_default_direct(heap); } return false; } +// initialize thread local data +void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap) { + _mi_memzero_aligned(tld,sizeof(mi_tld_t)); + tld->heap_backing = bheap; + tld->heaps = NULL; + tld->segments.subproc = &mi_subproc_default; + tld->segments.stats = &tld->stats; + tld->segments.os = &tld->os; + tld->os.stats = &tld->stats; +} + // Free the thread local default heap (called from `mi_thread_done`) -static bool _mi_heap_done(mi_heap_t* heap) { +static bool _mi_thread_heap_done(mi_heap_t* heap) { if (!mi_heap_is_initialized(heap)) return true; // reset default heap @@ -356,10 +391,7 @@ static bool _mi_heap_done(mi_heap_t* heap) { // free if not the main thread if (heap != &_mi_heap_main) { - // the following assertion does not always hold for huge segments as those are always treated - // as abondened: one may allocate it in one thread, but deallocate in another in which case - // the count can be too large or negative. todo: perhaps not count huge segments? see issue #363 - // mi_assert_internal(heap->tld->segments.count == 0 || heap->thread_id != _mi_thread_id()); + mi_assert_internal(heap->tld->segments.count == 0 || heap->thread_id != _mi_thread_id()); mi_thread_data_free((mi_thread_data_t*)heap); } else { @@ -420,7 +452,7 @@ void mi_thread_init(void) mi_attr_noexcept // initialize the thread local default heap // (this will call `_mi_heap_set_default_direct` and thus set the // fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called) - if (_mi_heap_init()) return; // returns true if already initialized + if (_mi_thread_heap_init()) return; // returns true if already initialized _mi_stat_increase(&_mi_stats_main.threads, 1); mi_atomic_increment_relaxed(&thread_count); @@ -452,7 +484,7 @@ void _mi_thread_done(mi_heap_t* heap) if (heap->thread_id != _mi_thread_id()) return; // abandon the thread local heap - if (_mi_heap_done(heap)) return; // returns true if already ran + if (_mi_thread_heap_done(heap)) return; // returns true if already ran } void _mi_heap_set_default_direct(mi_heap_t* heap) { @@ -613,7 +645,7 @@ void mi_process_init(void) mi_attr_noexcept { if (mi_option_is_enabled(mi_option_reserve_os_memory)) { long ksize = mi_option_get(mi_option_reserve_os_memory); if (ksize > 0) { - mi_reserve_os_memory((size_t)ksize*MI_KiB, true /* commit? */, true /* allow large pages? */); + mi_reserve_os_memory((size_t)ksize*MI_KiB, true, true); } } } diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/options.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/options.c index fba9076..462a7c7 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/options.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/options.c @@ -50,11 +50,11 @@ typedef struct mi_option_desc_s { static mi_option_desc_t options[_mi_option_last] = { // stable options - #if MI_DEBUG || defined(MI_SHOW_ERRORS) +#if MI_DEBUG || defined(MI_SHOW_ERRORS) { 1, UNINIT, MI_OPTION(show_errors) }, - #else +#else { 0, UNINIT, MI_OPTION(show_errors) }, - #endif +#endif { 0, UNINIT, MI_OPTION(show_stats) }, { 0, UNINIT, MI_OPTION(verbose) }, @@ -65,10 +65,10 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's { 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages {-1, UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N - { 0, UNINIT, MI_OPTION(reserve_os_memory) }, // reserve OS memory in advance + { 0, UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`) { 0, UNINIT, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread { 0, UNINIT, MI_OPTION(deprecated_page_reset) }, // reset page memory on free - { 0, UNINIT, MI_OPTION_LEGACY(abandoned_page_purge,abandoned_page_reset) }, // reset free page memory when a thread terminates + { 0, UNINIT, MI_OPTION(abandoned_page_purge) }, // purge free page memory when a thread terminates { 0, UNINIT, MI_OPTION(deprecated_segment_reset) }, // reset segment memory on free (needs eager commit) #if defined(__NetBSD__) { 0, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed @@ -79,19 +79,26 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes. { 0, UNINIT, MI_OPTION_LEGACY(disallow_os_alloc,limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas) { 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose - { 16, UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output - { 16, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output - { 10, UNINIT, MI_OPTION(max_segment_reclaim)}, // max. percentage of the abandoned segments per try. + { 32, UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output + { 32, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output + { 10, UNINIT, MI_OPTION(max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try. { 0, UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees! #if (MI_INTPTR_SIZE>4) - { 1024L * 1024L, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time + { 1024L*1024L, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`) #else - { 128L * 1024L, UNINIT, MI_OPTION(arena_reserve) }, + { 128L*1024L, UNINIT, MI_OPTION(arena_reserve) }, // =128MiB on 32-bit #endif - { 10, UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's + + { 10, UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's { 1, UNINIT, MI_OPTION_LEGACY(purge_extend_delay, decommit_extend_delay) }, { 1, UNINIT, MI_OPTION(abandoned_reclaim_on_free) },// reclaim an abandoned segment on a free { 0, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) + { 400, UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. +#if defined(MI_VISIT_ABANDONED) + { 1, INITIALIZED, MI_OPTION(visit_abandoned) }, // allow visiting heap blocks in abandonded segments; requires taking locks during reclaim. +#else + { 0, UNINIT, MI_OPTION(visit_abandoned) }, +#endif }; static void mi_option_init(mi_option_desc_t* desc); @@ -135,8 +142,12 @@ mi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long ma mi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) { mi_assert_internal(mi_option_has_size_in_kib(option)); - long x = mi_option_get(option); - return (x < 0 ? 0 : (size_t)x * MI_KiB); + const long x = mi_option_get(option); + size_t size = (x < 0 ? 0 : (size_t)x); + if (mi_option_has_size_in_kib(option)) { + size *= MI_KiB; + } + return size; } void mi_option_set(mi_option_t option, long value) { @@ -189,7 +200,7 @@ static void mi_cdecl mi_out_stderr(const char* msg, void* arg) { // an output function is registered it is called immediately with // the output up to that point. #ifndef MI_MAX_DELAY_OUTPUT -#define MI_MAX_DELAY_OUTPUT ((size_t)(32*1024)) +#define MI_MAX_DELAY_OUTPUT ((size_t)(16*1024)) #endif static char out_buf[MI_MAX_DELAY_OUTPUT+1]; static _Atomic(size_t) out_len; @@ -479,14 +490,20 @@ static void mi_option_init(mi_option_desc_t* desc) { else { char* end = buf; long value = strtol(buf, &end, 10); - if (desc->option == mi_option_reserve_os_memory || desc->option == mi_option_arena_reserve) { - // this option is interpreted in KiB to prevent overflow of `long` + if (mi_option_has_size_in_kib(desc->option)) { + // this option is interpreted in KiB to prevent overflow of `long` for large allocations + // (long is 32-bit on 64-bit windows, which allows for 4TiB max.) + size_t size = (value < 0 ? 0 : (size_t)value); + bool overflow = false; if (*end == 'K') { end++; } - else if (*end == 'M') { value *= MI_KiB; end++; } - else if (*end == 'G') { value *= MI_MiB; end++; } - else { value = (value + MI_KiB - 1) / MI_KiB; } - if (end[0] == 'I' && end[1] == 'B') { end += 2; } - else if (*end == 'B') { end++; } + else if (*end == 'M') { overflow = mi_mul_overflow(size,MI_KiB,&size); end++; } + else if (*end == 'G') { overflow = mi_mul_overflow(size,MI_MiB,&size); end++; } + else if (*end == 'T') { overflow = mi_mul_overflow(size,MI_GiB,&size); end++; } + else { size = (size + MI_KiB - 1) / MI_KiB; } + if (end[0] == 'I' && end[1] == 'B') { end += 2; } // KiB, MiB, GiB, TiB + else if (*end == 'B') { end++; } // Kb, Mb, Gb, Tb + if (overflow || size > MI_MAX_ALLOC_SIZE) { size = (MI_MAX_ALLOC_SIZE / MI_KiB); } + value = (size > LONG_MAX ? LONG_MAX : (long)size); } if (*end == 0) { desc->value = value; diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/os.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/os.c index dda6844..4babd8d 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/os.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/os.c @@ -11,9 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file /* ----------------------------------------------------------- - Initialization. - On windows initializes support for aligned allocation and - large OS pages (if MIMALLOC_LARGE_OS_PAGES is true). + Initialization. ----------------------------------------------------------- */ static mi_os_mem_config_t mi_os_mem_config = { @@ -21,7 +19,7 @@ static mi_os_mem_config_t mi_os_mem_config = { 0, // large page size (usually 2MiB) 4096, // allocation granularity true, // has overcommit? (if true we use MAP_NORESERVE on mmap systems) - false, // must free whole? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span) + false, // can we partially free allocated blocks? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span) true // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory) }; @@ -73,6 +71,21 @@ void _mi_os_init(void) { bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats); +static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) { + mi_assert_internal(alignment != 0); + uintptr_t mask = alignment - 1; + if ((alignment & mask) == 0) { // power of two? + return (sz & ~mask); + } + else { + return ((sz / alignment) * alignment); + } +} + +static void* mi_align_down_ptr(void* p, size_t alignment) { + return (void*)_mi_align_down((uintptr_t)p, alignment); +} + /* ----------------------------------------------------------- aligned hinting @@ -144,7 +157,8 @@ static void mi_os_prim_free(void* addr, size_t size, bool still_committed, mi_st _mi_stat_decrease(&stats->reserved, size); } -void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_stats_t* tld_stats) { +void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_stats_t* stats) { + if (stats == NULL) stats = &_mi_stats_main; if (mi_memkind_is_os(memid.memkind)) { size_t csize = _mi_os_good_alloc_size(size); void* base = addr; @@ -158,10 +172,10 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me // free it if (memid.memkind == MI_MEM_OS_HUGE) { mi_assert(memid.is_pinned); - mi_os_free_huge_os_pages(base, csize, tld_stats); + mi_os_free_huge_os_pages(base, csize, stats); } else { - mi_os_prim_free(base, csize, still_committed, tld_stats); + mi_os_prim_free(base, csize, still_committed, stats); } } else { @@ -170,8 +184,9 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me } } -void _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* tld_stats) { - _mi_os_free_ex(p, size, true, memid, tld_stats); +void _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* stats) { + if (stats == NULL) stats = &_mi_stats_main; + _mi_os_free_ex(p, size, true, memid, stats); } @@ -239,7 +254,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit if (size >= (SIZE_MAX - alignment)) return NULL; // overflow const size_t over_size = size + alignment; - if (mi_os_mem_config.must_free_whole) { // win32 virtualAlloc cannot free parts of an allocate block + if (!mi_os_mem_config.has_partial_free) { // win32 virtualAlloc cannot free parts of an allocated block // over-allocate uncommitted (virtual) memory p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, is_large, is_zero, stats); if (p == NULL) return NULL; @@ -260,7 +275,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit p = mi_os_prim_alloc(over_size, 1, commit, false, is_large, is_zero, stats); if (p == NULL) return NULL; - // and selectively unmap parts around the over-allocated area. (noop on sbrk) + // and selectively unmap parts around the over-allocated area. void* aligned_p = mi_align_up_ptr(p, alignment); size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p; size_t mid_size = _mi_align_up(size, _mi_os_page_size()); @@ -268,7 +283,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size); if (pre_size > 0) { mi_os_prim_free(p, pre_size, commit, stats); } if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, commit, stats); } - // we can return the aligned pointer on `mmap` (and sbrk) systems + // we can return the aligned pointer on `mmap` systems p = aligned_p; *base = aligned_p; // since we freed the pre part, `*base == p`. } @@ -286,6 +301,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit void* _mi_os_alloc(size_t size, mi_memid_t* memid, mi_stats_t* stats) { *memid = _mi_memid_none(); if (size == 0) return NULL; + if (stats == NULL) stats = &_mi_stats_main; size = _mi_os_good_alloc_size(size); bool os_is_large = false; bool os_is_zero = false; @@ -301,6 +317,7 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo MI_UNUSED(&_mi_os_get_aligned_hint); // suppress unused warnings *memid = _mi_memid_none(); if (size == 0) return NULL; + if (stats == NULL) stats = &_mi_stats_main; size = _mi_os_good_alloc_size(size); alignment = _mi_align_up(alignment, _mi_os_page_size()); @@ -329,6 +346,7 @@ void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offse mi_assert(offset <= size); mi_assert((alignment % _mi_os_page_size()) == 0); *memid = _mi_memid_none(); + if (stats == NULL) stats = &_mi_stats_main; if (offset > MI_SEGMENT_SIZE) return NULL; if (offset == 0) { // regular aligned allocation @@ -470,7 +488,7 @@ bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, mi_stats_t* stats) _mi_stat_increase(&stats->purged, size); if (mi_option_is_enabled(mi_option_purge_decommits) && // should decommit? - !_mi_preloading()) // don't decommit during preloading (unsafe) + !_mi_preloading()) // don't decommit during preloading (unsafe) { bool needs_recommit = true; mi_os_decommit_ex(p, size, &needs_recommit, stats); @@ -490,6 +508,7 @@ bool _mi_os_purge(void* p, size_t size, mi_stats_t * stats) { return _mi_os_purge_ex(p, size, true, stats); } + // Protect a region in memory to be not accessible. static bool mi_os_protectx(void* addr, size_t size, bool protect) { // page align conservatively within the range diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/page-queue.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/page-queue.c index ceea91e..02a8008 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/page-queue.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/page-queue.c @@ -12,7 +12,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MI_IN_PAGE_C #error "this file should be included from 'page.c'" // include to help an IDE -#include "mimalloc.h" +#include "mimalloc.h" #include "mimalloc/internal.h" #include "mimalloc/atomic.h" #endif @@ -38,15 +38,15 @@ terms of the MIT license. A copy of the license can be found in the file static inline bool mi_page_queue_is_huge(const mi_page_queue_t* pq) { - return (pq->block_size == (MI_MEDIUM_OBJ_SIZE_MAX+sizeof(uintptr_t))); + return (pq->block_size == (MI_LARGE_OBJ_SIZE_MAX+sizeof(uintptr_t))); } static inline bool mi_page_queue_is_full(const mi_page_queue_t* pq) { - return (pq->block_size == (MI_MEDIUM_OBJ_SIZE_MAX+(2*sizeof(uintptr_t)))); + return (pq->block_size == (MI_LARGE_OBJ_SIZE_MAX+(2*sizeof(uintptr_t)))); } static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) { - return (pq->block_size > MI_MEDIUM_OBJ_SIZE_MAX); + return (pq->block_size > MI_LARGE_OBJ_SIZE_MAX); } /* ----------------------------------------------------------- @@ -76,7 +76,7 @@ static inline uint8_t mi_bin(size_t size) { bin = (uint8_t)wsize; } #endif - else if (wsize > MI_MEDIUM_OBJ_WSIZE_MAX) { + else if (wsize > MI_LARGE_OBJ_WSIZE_MAX) { bin = MI_BIN_HUGE; } else { @@ -112,7 +112,7 @@ size_t _mi_bin_size(uint8_t bin) { // Good size for allocation size_t mi_good_size(size_t size) mi_attr_noexcept { - if (size <= MI_MEDIUM_OBJ_SIZE_MAX) { + if (size <= MI_LARGE_OBJ_SIZE_MAX) { return _mi_bin_size(mi_bin(size + MI_PADDING_SIZE)); } else { @@ -141,17 +141,13 @@ static bool mi_heap_contains_queue(const mi_heap_t* heap, const mi_page_queue_t* } #endif -static inline bool mi_page_is_large_or_huge(const mi_page_t* page) { - return (mi_page_block_size(page) > MI_MEDIUM_OBJ_SIZE_MAX || mi_page_is_huge(page)); -} - static mi_page_queue_t* mi_heap_page_queue_of(mi_heap_t* heap, const mi_page_t* page) { mi_assert_internal(heap!=NULL); uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page)))); mi_assert_internal(bin <= MI_BIN_FULL); mi_page_queue_t* pq = &heap->pages[bin]; mi_assert_internal((mi_page_block_size(page) == pq->block_size) || - (mi_page_is_large_or_huge(page) && mi_page_queue_is_huge(pq)) || + (mi_page_is_huge(page) && mi_page_queue_is_huge(pq)) || (mi_page_is_in_full(page) && mi_page_queue_is_full(pq))); return pq; } @@ -214,11 +210,10 @@ static bool mi_page_queue_is_empty(mi_page_queue_t* queue) { static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) { mi_assert_internal(page != NULL); mi_assert_expensive(mi_page_queue_contains(queue, page)); - mi_assert_internal(mi_page_block_size(page) == queue->block_size || - (mi_page_is_large_or_huge(page) && mi_page_queue_is_huge(queue)) || + mi_assert_internal(mi_page_block_size(page) == queue->block_size || + (mi_page_is_huge(page) && mi_page_queue_is_huge(queue)) || (mi_page_is_in_full(page) && mi_page_queue_is_full(queue))); mi_heap_t* heap = mi_page_heap(page); - if (page->prev != NULL) page->prev->next = page->next; if (page->next != NULL) page->next->prev = page->prev; if (page == queue->last) queue->last = page->prev; @@ -240,10 +235,10 @@ static void mi_page_queue_push(mi_heap_t* heap, mi_page_queue_t* queue, mi_page_ mi_assert_internal(mi_page_heap(page) == heap); mi_assert_internal(!mi_page_queue_contains(queue, page)); #if MI_HUGE_PAGE_ABANDON - mi_assert_internal(_mi_page_segment(page)->kind != MI_SEGMENT_HUGE); + mi_assert_internal(_mi_page_segment(page)->page_kind != MI_PAGE_HUGE); #endif mi_assert_internal(mi_page_block_size(page) == queue->block_size || - (mi_page_is_large_or_huge(page) && mi_page_queue_is_huge(queue)) || + (mi_page_is_huge(page) && mi_page_queue_is_huge(queue)) || (mi_page_is_in_full(page) && mi_page_queue_is_full(queue))); mi_page_set_in_full(page, mi_page_queue_is_full(queue)); @@ -274,8 +269,8 @@ static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* fro mi_assert_internal((bsize == to->block_size && bsize == from->block_size) || (bsize == to->block_size && mi_page_queue_is_full(from)) || (bsize == from->block_size && mi_page_queue_is_full(to)) || - (mi_page_is_large_or_huge(page) && mi_page_queue_is_huge(to)) || - (mi_page_is_large_or_huge(page) && mi_page_queue_is_full(to))); + (mi_page_is_huge(page) && mi_page_queue_is_huge(to)) || + (mi_page_is_huge(page) && mi_page_queue_is_full(to))); mi_heap_t* heap = mi_page_heap(page); if (page->prev != NULL) page->prev->next = page->next; diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/page.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/page.c index 871ed21..96d1b24 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/page.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/page.c @@ -82,9 +82,11 @@ static bool mi_page_is_valid_init(mi_page_t* page) { mi_assert_internal(page->used <= page->capacity); mi_assert_internal(page->capacity <= page->reserved); + // const size_t bsize = mi_page_block_size(page); + mi_segment_t* segment = _mi_page_segment(page); uint8_t* start = mi_page_start(page); - mi_assert_internal(start == _mi_segment_page_start(_mi_page_segment(page), page, NULL)); - mi_assert_internal(page->is_huge == (_mi_page_segment(page)->kind == MI_SEGMENT_HUGE)); + mi_assert_internal(start == _mi_segment_page_start(segment,page,NULL)); + mi_assert_internal(page->is_huge == (segment->page_kind == MI_PAGE_HUGE)); //mi_assert_internal(start + page->capacity*page->block_size == page->top); mi_assert_internal(mi_page_list_is_valid(page,page->free)); @@ -121,15 +123,14 @@ bool _mi_page_is_valid(mi_page_t* page) { #endif if (mi_page_heap(page)!=NULL) { mi_segment_t* segment = _mi_page_segment(page); - - mi_assert_internal(!_mi_process_is_initialized || segment->thread_id==0 || segment->thread_id == mi_page_heap(page)->thread_id); + mi_assert_internal(!_mi_process_is_initialized || segment->thread_id == mi_page_heap(page)->thread_id || segment->thread_id==0); #if MI_HUGE_PAGE_ABANDON - if (segment->kind != MI_SEGMENT_HUGE) + if (segment->page_kind != MI_PAGE_HUGE) #endif { mi_page_queue_t* pq = mi_page_queue_of(page); mi_assert_internal(mi_page_queue_contains(pq, page)); - mi_assert_internal(pq->block_size==mi_page_block_size(page) || mi_page_block_size(page) > MI_MEDIUM_OBJ_SIZE_MAX || mi_page_is_in_full(page)); + mi_assert_internal(pq->block_size==mi_page_block_size(page) || mi_page_block_size(page) > MI_LARGE_OBJ_SIZE_MAX || mi_page_is_in_full(page)); mi_assert_internal(mi_heap_contains_queue(mi_page_heap(page),pq)); } } @@ -256,11 +257,10 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { // called from segments when reclaiming abandoned pages void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page) { mi_assert_expensive(mi_page_is_valid_init(page)); - mi_assert_internal(mi_page_heap(page) == heap); mi_assert_internal(mi_page_thread_free_flag(page) != MI_NEVER_DELAYED_FREE); #if MI_HUGE_PAGE_ABANDON - mi_assert_internal(_mi_page_segment(page)->kind != MI_SEGMENT_HUGE); + mi_assert_internal(_mi_page_segment(page)->page_kind != MI_PAGE_HUGE); #endif // TODO: push on full queue immediately if it is full? @@ -274,7 +274,7 @@ static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size #if !MI_HUGE_PAGE_ABANDON mi_assert_internal(pq != NULL); mi_assert_internal(mi_heap_contains_queue(heap, pq)); - mi_assert_internal(page_alignment > 0 || block_size > MI_MEDIUM_OBJ_SIZE_MAX || block_size == pq->block_size); + mi_assert_internal(page_alignment > 0 || block_size > MI_LARGE_OBJ_SIZE_MAX || block_size == pq->block_size); #endif mi_page_t* page = _mi_segment_page_alloc(heap, block_size, page_alignment, &heap->tld->segments, &heap->tld->os); if (page == NULL) { @@ -284,7 +284,6 @@ static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size #if MI_HUGE_PAGE_ABANDON mi_assert_internal(pq==NULL || _mi_page_segment(page)->page_kind != MI_PAGE_HUGE); #endif - mi_assert_internal(page_alignment >0 || block_size > MI_MEDIUM_OBJ_SIZE_MAX || _mi_page_segment(page)->kind != MI_SEGMENT_HUGE); mi_assert_internal(pq!=NULL || mi_page_block_size(page) >= block_size); // a fresh page was found, initialize it const size_t full_block_size = (pq == NULL || mi_page_is_huge(page) ? mi_page_block_size(page) : block_size); // see also: mi_segment_huge_page_alloc @@ -416,11 +415,9 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { // no more aligned blocks in here mi_page_set_has_aligned(page, false); - mi_heap_t* heap = mi_page_heap(page); - // remove from the page list // (no need to do _mi_heap_delayed_free first as all blocks are already free) - mi_segments_tld_t* segments_tld = &heap->tld->segments; + mi_segments_tld_t* segments_tld = &mi_page_heap(page)->tld->segments; mi_page_queue_remove(pq, page); // and free it @@ -428,7 +425,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { _mi_segment_page_free(page, force, segments_tld); } -#define MI_MAX_RETIRE_SIZE MI_MEDIUM_OBJ_SIZE_MAX // should be less than size for MI_BIN_HUGE +#define MI_MAX_RETIRE_SIZE MI_LARGE_OBJ_SIZE_MAX // should be less than size for MI_BIN_HUGE #define MI_RETIRE_CYCLES (16) // Retire a page with no more used blocks @@ -466,6 +463,7 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { return; // don't free after all } } + _mi_page_free(page, pq, false); } @@ -600,7 +598,7 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co #if (MI_SECURE>0) #define MI_MIN_EXTEND (8*MI_SECURE) // extend at least by this many #else -#define MI_MIN_EXTEND (4) +#define MI_MIN_EXTEND (1) #endif // Extend the capacity (up to reserved) by initializing a free list @@ -609,7 +607,6 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co // allocations but this did not speed up any benchmark (due to an // extra test in malloc? or cache effects?) static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld) { - MI_UNUSED(tld); mi_assert_expensive(mi_page_is_valid_init(page)); #if (MI_SECURE<=2) mi_assert(page->free == NULL); @@ -618,6 +615,9 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld) #endif if (page->capacity >= page->reserved) return; + size_t page_size; + //uint8_t* page_start = + _mi_segment_page_start(_mi_page_segment(page), page, &page_size); mi_stat_counter_increase(tld->stats.pages_extended, 1); // calculate the extend count @@ -663,8 +663,6 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi size_t page_size; page->page_start = _mi_segment_page_start(segment, page, &page_size); mi_track_mem_noaccess(page->page_start,page_size); - mi_assert_internal(mi_page_block_size(page) <= page_size); - mi_assert_internal(page_size <= page->slice_count*MI_SEGMENT_SLICE_SIZE); mi_assert_internal(page_size / block_size < (1L<<16)); page->reserved = (uint16_t)(page_size / block_size); mi_assert_internal(page->reserved > 0); @@ -679,7 +677,6 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi mi_assert_expensive(mi_mem_is_zero(page->page_start, page_size)); } #endif - mi_assert_internal(page->is_committed); if (block_size > 0 && _mi_is_power_of_two(block_size)) { page->block_size_shift = (uint8_t)(mi_ctz((uintptr_t)block_size)); } @@ -753,7 +750,7 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p mi_heap_stat_counter_increase(heap, searches, count); if (page == NULL) { - _mi_heap_collect_retired(heap, false); // perhaps make a page available? + _mi_heap_collect_retired(heap, false); // perhaps make a page available page = mi_page_fresh(heap, pq); if (page == NULL && first_try) { // out-of-memory _or_ an abandoned page with free blocks was reclaimed, try once again @@ -824,46 +821,31 @@ void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noex General allocation ----------------------------------------------------------- */ -// Large and huge page allocation. -// Huge pages contain just one block, and the segment contains just that page (as `MI_SEGMENT_HUGE`). +// Huge pages contain just one block, and the segment contains just that page. // Huge pages are also use if the requested alignment is very large (> MI_BLOCK_ALIGNMENT_MAX) // so their size is not always `> MI_LARGE_OBJ_SIZE_MAX`. -static mi_page_t* mi_large_huge_page_alloc(mi_heap_t* heap, size_t size, size_t page_alignment) { +static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size, size_t page_alignment) { size_t block_size = _mi_os_good_alloc_size(size); mi_assert_internal(mi_bin(block_size) == MI_BIN_HUGE || page_alignment > 0); - bool is_huge = (block_size > MI_LARGE_OBJ_SIZE_MAX || page_alignment > 0); #if MI_HUGE_PAGE_ABANDON - mi_page_queue_t* pq = (is_huge ? NULL : mi_page_queue(heap, block_size)); + mi_page_queue_t* pq = NULL; #else - mi_page_queue_t* pq = mi_page_queue(heap, is_huge ? MI_LARGE_OBJ_SIZE_MAX+1 : block_size); - mi_assert_internal(!is_huge || mi_page_queue_is_huge(pq)); + mi_page_queue_t* pq = mi_page_queue(heap, MI_LARGE_OBJ_SIZE_MAX+1); // always in the huge queue regardless of the block size + mi_assert_internal(mi_page_queue_is_huge(pq)); #endif mi_page_t* page = mi_page_fresh_alloc(heap, pq, block_size, page_alignment); if (page != NULL) { + mi_assert_internal(mi_page_block_size(page) >= size); mi_assert_internal(mi_page_immediate_available(page)); - - if (is_huge) { - mi_assert_internal(mi_page_is_huge(page)); - mi_assert_internal(_mi_page_segment(page)->kind == MI_SEGMENT_HUGE); - mi_assert_internal(_mi_page_segment(page)->used==1); - #if MI_HUGE_PAGE_ABANDON - mi_assert_internal(_mi_page_segment(page)->thread_id==0); // abandoned, not in the huge queue - mi_page_set_heap(page, NULL); - #endif - } - else { - mi_assert_internal(!mi_page_is_huge(page)); - } - - const size_t bsize = mi_page_usable_block_size(page); // note: not `mi_page_block_size` to account for padding - if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { - mi_heap_stat_increase(heap, large, bsize); - mi_heap_stat_counter_increase(heap, large_count, 1); - } - else { - mi_heap_stat_increase(heap, huge, bsize); - mi_heap_stat_counter_increase(heap, huge_count, 1); - } + mi_assert_internal(mi_page_is_huge(page)); + mi_assert_internal(_mi_page_segment(page)->page_kind == MI_PAGE_HUGE); + mi_assert_internal(_mi_page_segment(page)->used==1); + #if MI_HUGE_PAGE_ABANDON + mi_assert_internal(_mi_page_segment(page)->thread_id==0); // abandoned, not in the huge queue + mi_page_set_heap(page, NULL); + #endif + mi_heap_stat_increase(heap, huge, mi_page_block_size(page)); + mi_heap_stat_counter_increase(heap, huge_count, 1); } return page; } @@ -874,13 +856,13 @@ static mi_page_t* mi_large_huge_page_alloc(mi_heap_t* heap, size_t size, size_t static mi_page_t* mi_find_page(mi_heap_t* heap, size_t size, size_t huge_alignment) mi_attr_noexcept { // huge allocation? const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` - if mi_unlikely(req_size > (MI_MEDIUM_OBJ_SIZE_MAX - MI_PADDING_SIZE) || huge_alignment > 0) { + if mi_unlikely(req_size > (MI_LARGE_OBJ_SIZE_MAX - MI_PADDING_SIZE) || huge_alignment > 0) { if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE) { _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size); return NULL; } else { - return mi_large_huge_page_alloc(heap,size,huge_alignment); + return mi_huge_page_alloc(heap,size,huge_alignment); } } else { diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/emscripten/prim.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/emscripten/prim.c index 1f60a1b..944c0cb 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/emscripten/prim.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/emscripten/prim.c @@ -51,7 +51,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config) { config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB config->alloc_granularity = 16; config->has_overcommit = false; - config->must_free_whole = true; + config->has_partial_free = false; config->has_virtual_reserve = false; } @@ -200,7 +200,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { // Thread init/done //---------------------------------------------------------------- -#ifdef __EMSCRIPTEN_SHARED_MEMORY__ +#if defined(MI_USE_PTHREADS) // use pthread local storage keys to detect thread ending // (and used with MI_TLS_PTHREADS for the default heap) diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/osx/alloc-override-zone.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/osx/alloc-override-zone.c index 9a31775..1515b88 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/osx/alloc-override-zone.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/osx/alloc-override-zone.c @@ -422,6 +422,7 @@ __attribute__((constructor(0))) #else __attribute__((constructor)) // seems not supported by g++-11 on the M1 #endif +__attribute__((used)) static void _mi_macos_override_malloc(void) { malloc_zone_t* purgeable_zone = NULL; diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/unix/prim.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/unix/prim.c index 8d40536..63a36f2 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/unix/prim.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/unix/prim.c @@ -22,18 +22,17 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc/internal.h" -#include "mimalloc/atomic.h" #include "mimalloc/prim.h" #include <sys/mman.h> // mmap #include <unistd.h> // sysconf #include <fcntl.h> // open, close, read, access - + #if defined(__linux__) #include <features.h> - #if defined(MI_NO_THP) - #include <sys/prctl.h> - #endif + //#if defined(MI_NO_THP) + #include <sys/prctl.h> // THP disable + //#endif #if defined(__GLIBC__) #include <linux/mman.h> // linux mmap flags #else @@ -57,7 +56,7 @@ terms of the MIT license. A copy of the license can be found in the file #include <sys/sysctl.h> #endif -#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) && !defined(__OpenBSD__) && !defined(__sun) +#if defined(__linux__) || defined(__FreeBSD__) #define MI_HAS_SYSCALL_H #include <sys/syscall.h> #endif @@ -65,39 +64,38 @@ terms of the MIT license. A copy of the license can be found in the file //------------------------------------------------------------------------------------ // Use syscalls for some primitives to allow for libraries that override open/read/close etc. -// and do allocation themselves; using syscalls prevents recursion when mimalloc is +// and do allocation themselves; using syscalls prevents recursion when mimalloc is // still initializing (issue #713) +// Declare inline to avoid unused function warnings. //------------------------------------------------------------------------------------ - #if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access) -static int mi_prim_open(const char* fpath, int open_flags) { +static inline int mi_prim_open(const char* fpath, int open_flags) { return syscall(SYS_open,fpath,open_flags,0); } -static ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) { +static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) { return syscall(SYS_read,fd,buf,bufsize); } -static int mi_prim_close(int fd) { +static inline int mi_prim_close(int fd) { return syscall(SYS_close,fd); } -static int mi_prim_access(const char *fpath, int mode) { +static inline int mi_prim_access(const char *fpath, int mode) { return syscall(SYS_access,fpath,mode); } -#elif !defined(__sun) && \ - (!defined(__APPLE__) || (MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_7)) // avoid unused warnings on macOS and Solaris +#else -static int mi_prim_open(const char* fpath, int open_flags) { +static inline int mi_prim_open(const char* fpath, int open_flags) { return open(fpath,open_flags); } -static ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) { +static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) { return read(fd,buf,bufsize); } -static int mi_prim_close(int fd) { +static inline int mi_prim_close(int fd) { return close(fd); } -static int mi_prim_access(const char *fpath, int mode) { +static inline int mi_prim_access(const char *fpath, int mode) { return access(fpath,mode); } @@ -130,12 +128,12 @@ static bool unix_detect_overcommit(void) { os_overcommit = (val != 0); } #else - // default: overcommit is true + // default: overcommit is true #endif return os_overcommit; } -void _mi_prim_mem_init( mi_os_mem_config_t* config ) +void _mi_prim_mem_init( mi_os_mem_config_t* config ) { long psize = sysconf(_SC_PAGESIZE); if (psize > 0) { @@ -144,7 +142,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) } config->large_page_size = 2*MI_MiB; // TODO: can we query the OS for this? config->has_overcommit = unix_detect_overcommit(); - config->must_free_whole = false; // mmap can free in parts + config->has_partial_free = true; // mmap can free in parts config->has_virtual_reserve = true; // todo: check if this true for NetBSD? (for anonymous mmap with PROT_NONE) // disable transparent huge pages for this process? @@ -197,12 +195,12 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p size_t n = mi_bsr(try_alignment); if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); - if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { + if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { int err = errno; - _mi_warning_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr); + _mi_trace_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr); } if (p!=MAP_FAILED) return p; - // fall back to regular mmap + // fall back to regular mmap } } #elif defined(MAP_ALIGN) // Solaris @@ -218,16 +216,16 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p void* hint = _mi_os_get_aligned_hint(try_alignment, size); if (hint != NULL) { p = mmap(hint, size, protect_flags, flags, fd, 0); - if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { + if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { #if MI_TRACK_ENABLED // asan sometimes does not instrument errno correctly? int err = 0; #else int err = errno; #endif - _mi_warning_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint); + _mi_trace_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint); } if (p!=MAP_FAILED) return p; - // fall back to regular mmap + // fall back to regular mmap } } #endif @@ -356,9 +354,9 @@ int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_la mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); mi_assert_internal(commit || !allow_large); mi_assert_internal(try_alignment > 0); - + *is_zero = true; - int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); + int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); *addr = unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large); return (*addr != NULL ? 0 : errno); } @@ -380,25 +378,29 @@ static void unix_mprotect_hint(int err) { #endif } + + + + int _mi_prim_commit(void* start, size_t size, bool* is_zero) { // commit: ensure we can access the area // note: we may think that *is_zero can be true since the memory // was either from mmap PROT_NONE, or from decommit MADV_DONTNEED, but // we sometimes call commit on a range with still partially committed // memory and `mprotect` does not zero the range. - *is_zero = false; + *is_zero = false; int err = mprotect(start, size, (PROT_READ | PROT_WRITE)); - if (err != 0) { - err = errno; + if (err != 0) { + err = errno; unix_mprotect_hint(err); } return err; } int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) { - int err = 0; + int err = 0; // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE) - err = unix_madvise(start, size, MADV_DONTNEED); + err = unix_madvise(start, size, MADV_DONTNEED); #if !MI_DEBUG && !MI_SECURE *needs_recommit = false; #else @@ -410,15 +412,15 @@ int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) { *needs_recommit = true; const int fd = unix_mmap_fd(); void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0); - if (p != start) { err = errno; } + if (p != start) { err = errno; } */ return err; } int _mi_prim_reset(void* start, size_t size) { - // We try to use `MADV_FREE` as that is the fastest. A drawback though is that it + // We try to use `MADV_FREE` as that is the fastest. A drawback though is that it // will not reduce the `rss` stats in tools like `top` even though the memory is available - // to other processes. With the default `MIMALLOC_PURGE_DECOMMITS=1` we ensure that by + // to other processes. With the default `MIMALLOC_PURGE_DECOMMITS=1` we ensure that by // default `MADV_DONTNEED` is used though. #if defined(MADV_FREE) static _Atomic(size_t) advice = MI_ATOMIC_VAR_INIT(MADV_FREE); @@ -438,7 +440,7 @@ int _mi_prim_reset(void* start, size_t size) { int _mi_prim_protect(void* start, size_t size, bool protect) { int err = mprotect(start, size, protect ? PROT_NONE : (PROT_READ | PROT_WRITE)); - if (err != 0) { err = errno; } + if (err != 0) { err = errno; } unix_mprotect_hint(err); return err; } @@ -479,7 +481,7 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo if (err != 0) { err = errno; _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%x))\n", numa_node, err, err); - } + } } return (*addr != NULL ? 0 : errno); } @@ -594,9 +596,9 @@ mi_msecs_t _mi_prim_clock_now(void) { // low resolution timer mi_msecs_t _mi_prim_clock_now(void) { #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0) - return (mi_msecs_t)clock(); + return (mi_msecs_t)clock(); #elif (CLOCKS_PER_SEC < 1000) - return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); + return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); #else return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000); #endif @@ -636,7 +638,7 @@ void _mi_prim_process_info(mi_process_info_t* pinfo) pinfo->stime = timeval_secs(&rusage.ru_stime); #if !defined(__HAIKU__) pinfo->page_faults = rusage.ru_majflt; -#endif +#endif #if defined(__HAIKU__) // Haiku does not have (yet?) a way to // get these stats per process @@ -763,7 +765,7 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { bool _mi_prim_random_buf(void* buf, size_t buf_len) { // We prefere CCRandomGenerateBytes as it returns an error code while arc4random_buf // may fail silently on macOS. See PR #390, and <https://opensource.apple.com/source/Libc/Libc-1439.40.11/gen/FreeBSD/arc4random.c.auto.html> - return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess); + return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess); } #elif defined(__ANDROID__) || defined(__DragonFly__) || \ @@ -862,7 +864,7 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { } } -#else +#else void _mi_prim_thread_init_auto_done(void) { // nothing diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/wasi/prim.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/wasi/prim.c index f74acd2..5d7a813 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/wasi/prim.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/wasi/prim.c @@ -9,7 +9,6 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc/internal.h" -#include "mimalloc/atomic.h" #include "mimalloc/prim.h" #include <stdio.h> // fputs @@ -22,8 +21,8 @@ terms of the MIT license. A copy of the license can be found in the file void _mi_prim_mem_init( mi_os_mem_config_t* config ) { config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB config->alloc_granularity = 16; - config->has_overcommit = false; - config->must_free_whole = true; + config->has_overcommit = false; + config->has_partial_free = false; config->has_virtual_reserve = false; } @@ -134,7 +133,7 @@ int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_la //--------------------------------------------- int _mi_prim_commit(void* addr, size_t size, bool* is_zero) { - MI_UNUSED(addr); MI_UNUSED(size); + MI_UNUSED(addr); MI_UNUSED(size); *is_zero = false; return 0; } @@ -199,9 +198,9 @@ mi_msecs_t _mi_prim_clock_now(void) { // low resolution timer mi_msecs_t _mi_prim_clock_now(void) { #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0) - return (mi_msecs_t)clock(); + return (mi_msecs_t)clock(); #elif (CLOCKS_PER_SEC < 1000) - return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); + return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); #else return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000); #endif diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/windows/prim.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/windows/prim.c index 2dd7c60..bd874f9 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/windows/prim.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/prim/windows/prim.c @@ -9,7 +9,6 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc/internal.h" -#include "mimalloc/atomic.h" #include "mimalloc/prim.h" #include <stdio.h> // fputs, stderr @@ -112,7 +111,7 @@ static bool win_enable_large_os_pages(size_t* large_page_size) void _mi_prim_mem_init( mi_os_mem_config_t* config ) { config->has_overcommit = false; - config->must_free_whole = true; + config->has_partial_free = false; config->has_virtual_reserve = true; // get the page size SYSTEM_INFO si; @@ -178,7 +177,7 @@ int _mi_prim_free(void* addr, size_t size ) { // VirtualAlloc //--------------------------------------------- -static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) { +static void* win_virtual_alloc_prim_once(void* addr, size_t size, size_t try_alignment, DWORD flags) { #if (MI_INTPTR_SIZE >= 8) // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations if (addr == NULL) { @@ -200,13 +199,53 @@ static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignmen param.Arg.Pointer = &reqs; void* p = (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, ¶m, 1); if (p != NULL) return p; - _mi_warning_message("unable to allocate aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\n", size, GetLastError(), addr, try_alignment, flags); + _mi_warning_message("unable to allocate aligned OS memory (0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\n", size, GetLastError(), addr, try_alignment, flags); // fall through on error } // last resort return VirtualAlloc(addr, size, flags, PAGE_READWRITE); } +static bool win_is_out_of_memory_error(DWORD err) { + switch (err) { + case ERROR_COMMITMENT_MINIMUM: + case ERROR_COMMITMENT_LIMIT: + case ERROR_PAGEFILE_QUOTA: + case ERROR_NOT_ENOUGH_MEMORY: + return true; + default: + return false; + } +} + +static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) { + long max_retry_msecs = mi_option_get_clamp(mi_option_retry_on_oom, 0, 2000); // at most 2 seconds + if (max_retry_msecs == 1) { max_retry_msecs = 100; } // if one sets the option to "true" + for (long tries = 1; tries <= 10; tries++) { // try at most 10 times (=2200ms) + void* p = win_virtual_alloc_prim_once(addr, size, try_alignment, flags); + if (p != NULL) { + // success, return the address + return p; + } + else if (max_retry_msecs > 0 && (try_alignment <= 2*MI_SEGMENT_ALIGN) && + (flags&MEM_COMMIT) != 0 && (flags&MEM_LARGE_PAGES) == 0 && + win_is_out_of_memory_error(GetLastError())) { + // if committing regular memory and being out-of-memory, + // keep trying for a bit in case memory frees up after all. See issue #894 + _mi_warning_message("out-of-memory on OS allocation, try again... (attempt %lu, 0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\n", tries, size, GetLastError(), addr, try_alignment, flags); + long sleep_msecs = tries*40; // increasing waits + if (sleep_msecs > max_retry_msecs) { sleep_msecs = max_retry_msecs; } + max_retry_msecs -= sleep_msecs; + Sleep(sleep_msecs); + } + else { + // otherwise return with an error + break; + } + } + return NULL; +} + static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) { mi_assert_internal(!(large_only && !allow_large)); static _Atomic(size_t) large_page_try_ok; // = 0; @@ -276,7 +315,7 @@ int _mi_prim_commit(void* addr, size_t size, bool* is_zero) { return 0; } -int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) { +int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) { BOOL ok = VirtualFree(addr, size, MEM_DECOMMIT); *needs_recommit = true; // for safety, assume always decommitted even in the case of an error. return (ok ? 0 : (int)GetLastError()); @@ -428,7 +467,6 @@ mi_msecs_t _mi_prim_clock_now(void) { // Process Info //---------------------------------------------------------------- -#include <windows.h> #include <psapi.h> static mi_msecs_t filetime_msecs(const FILETIME* ftime) { @@ -451,7 +489,7 @@ void _mi_prim_process_info(mi_process_info_t* pinfo) GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut); pinfo->utime = filetime_msecs(&ut); pinfo->stime = filetime_msecs(&st); - + // load psapi on demand if (pGetProcessMemoryInfo == NULL) { HINSTANCE hDll = LoadLibrary(TEXT("psapi.dll")); @@ -465,7 +503,7 @@ void _mi_prim_process_info(mi_process_info_t* pinfo) memset(&info, 0, sizeof(info)); if (pGetProcessMemoryInfo != NULL) { pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info)); - } + } pinfo->current_rss = (size_t)info.WorkingSetSize; pinfo->peak_rss = (size_t)info.PeakWorkingSetSize; pinfo->current_commit = (size_t)info.PagefileUsage; @@ -477,7 +515,7 @@ void _mi_prim_process_info(mi_process_info_t* pinfo) // Output //---------------------------------------------------------------- -void _mi_prim_out_stderr( const char* msg ) +void _mi_prim_out_stderr( const char* msg ) { // on windows with redirection, the C runtime cannot handle locale dependent output // after the main thread closes so we use direct console output. @@ -524,7 +562,6 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { } - //---------------------------------------------------------------- // Random //---------------------------------------------------------------- @@ -560,7 +597,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { } if (pBCryptGenRandom == NULL) return false; } - return (pBCryptGenRandom(NULL, (PUCHAR)buf, (ULONG)buf_len, BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0); + return (pBCryptGenRandom(NULL, (PUCHAR)buf, (ULONG)buf_len, BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0); } #endif // MI_USE_RTLGENRANDOM @@ -572,6 +609,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { #if !defined(MI_SHARED_LIB) // use thread local storage keys to detect thread ending +// note: another design could be to use special linker sections (see issue #869) #include <fibersapi.h> #if (_WIN32_WINNT < 0x600) // before Windows Vista WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback ); @@ -595,9 +633,9 @@ void _mi_prim_thread_init_auto_done(void) { } void _mi_prim_thread_done_auto_done(void) { - // call thread-done on all threads (except the main thread) to prevent + // call thread-done on all threads (except the main thread) to prevent // dangling callback pointer if statically linked with a DLL; Issue #208 - FlsFree(mi_fls_key); + FlsFree(mi_fls_key); } void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { @@ -620,3 +658,4 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { } #endif + diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/segment-map.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/segment-map.c index 1efb1e2..8927a8b 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/segment-map.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/segment-map.c @@ -16,140 +16,111 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/internal.h" #include "mimalloc/atomic.h" -#if (MI_INTPTR_SIZE>=8) && MI_TRACK_ASAN -#define MI_MAX_ADDRESS ((size_t)140 << 40) // 140TB (see issue #881) -#elif (MI_INTPTR_SIZE >= 8) -#define MI_MAX_ADDRESS ((size_t)40 << 40) // 40TB (to include huge page areas) +// Reduce total address space to reduce .bss (due to the `mi_segment_map`) +#if (MI_INTPTR_SIZE > 4) && MI_TRACK_ASAN +#define MI_SEGMENT_MAP_MAX_ADDRESS (128*1024ULL*MI_GiB) // 128 TiB (see issue #881) +#elif (MI_INTPTR_SIZE > 4) +#define MI_SEGMENT_MAP_MAX_ADDRESS (48*1024ULL*MI_GiB) // 48 TiB #else -#define MI_MAX_ADDRESS ((size_t)2 << 30) // 2Gb +#define MI_SEGMENT_MAP_MAX_ADDRESS (MAX_UINT32) #endif -#define MI_SEGMENT_MAP_BITS (MI_MAX_ADDRESS / MI_SEGMENT_SIZE) -#define MI_SEGMENT_MAP_SIZE (MI_SEGMENT_MAP_BITS / 8) -#define MI_SEGMENT_MAP_WSIZE (MI_SEGMENT_MAP_SIZE / MI_INTPTR_SIZE) +#define MI_SEGMENT_MAP_PART_SIZE (MI_INTPTR_SIZE*MI_KiB - 128) // 128 > sizeof(mi_memid_t) ! +#define MI_SEGMENT_MAP_PART_BITS (8*MI_SEGMENT_MAP_PART_SIZE) +#define MI_SEGMENT_MAP_PART_ENTRIES (MI_SEGMENT_MAP_PART_SIZE / MI_INTPTR_SIZE) +#define MI_SEGMENT_MAP_PART_BIT_SPAN (MI_SEGMENT_ALIGN) +#define MI_SEGMENT_MAP_PART_SPAN (MI_SEGMENT_MAP_PART_BITS * MI_SEGMENT_MAP_PART_BIT_SPAN) +#define MI_SEGMENT_MAP_MAX_PARTS ((MI_SEGMENT_MAP_MAX_ADDRESS / MI_SEGMENT_MAP_PART_SPAN) + 1) -static _Atomic(uintptr_t) mi_segment_map[MI_SEGMENT_MAP_WSIZE + 1]; // 2KiB per TB with 64MiB segments +// A part of the segment map. +typedef struct mi_segmap_part_s { + mi_memid_t memid; + _Atomic(uintptr_t) map[MI_SEGMENT_MAP_PART_ENTRIES]; +} mi_segmap_part_t; -static size_t mi_segment_map_index_of(const mi_segment_t* segment, size_t* bitidx) { +// Allocate parts on-demand to reduce .bss footprint +static _Atomic(mi_segmap_part_t*) mi_segment_map[MI_SEGMENT_MAP_MAX_PARTS]; // = { NULL, .. } + +static mi_segmap_part_t* mi_segment_map_index_of(const mi_segment_t* segment, bool create_on_demand, size_t* idx, size_t* bitidx) { // note: segment can be invalid or NULL. mi_assert_internal(_mi_ptr_segment(segment + 1) == segment); // is it aligned on MI_SEGMENT_SIZE? - if ((uintptr_t)segment >= MI_MAX_ADDRESS) { - *bitidx = 0; - return MI_SEGMENT_MAP_WSIZE; - } - else { - const uintptr_t segindex = ((uintptr_t)segment) / MI_SEGMENT_SIZE; - *bitidx = segindex % MI_INTPTR_BITS; - const size_t mapindex = segindex / MI_INTPTR_BITS; - mi_assert_internal(mapindex < MI_SEGMENT_MAP_WSIZE); - return mapindex; + *idx = 0; + *bitidx = 0; + if ((uintptr_t)segment >= MI_SEGMENT_MAP_MAX_ADDRESS) return NULL; + const uintptr_t segindex = ((uintptr_t)segment) / MI_SEGMENT_MAP_PART_SPAN; + if (segindex >= MI_SEGMENT_MAP_MAX_PARTS) return NULL; + mi_segmap_part_t* part = mi_atomic_load_ptr_relaxed(mi_segmap_part_t, &mi_segment_map[segindex]); + + // allocate on demand to reduce .bss footprint + if (part == NULL) { + if (!create_on_demand) return NULL; + mi_memid_t memid; + part = (mi_segmap_part_t*)_mi_os_alloc(sizeof(mi_segmap_part_t), &memid, NULL); + if (part == NULL) return NULL; + mi_segmap_part_t* expected = NULL; + if (!mi_atomic_cas_ptr_strong_release(mi_segmap_part_t, &mi_segment_map[segindex], &expected, part)) { + _mi_os_free(part, sizeof(mi_segmap_part_t), memid, NULL); + part = expected; + if (part == NULL) return NULL; + } } + mi_assert(part != NULL); + const uintptr_t offset = ((uintptr_t)segment) % MI_SEGMENT_MAP_PART_SPAN; + const uintptr_t bitofs = offset / MI_SEGMENT_MAP_PART_BIT_SPAN; + *idx = bitofs / MI_INTPTR_BITS; + *bitidx = bitofs % MI_INTPTR_BITS; + return part; } void _mi_segment_map_allocated_at(const mi_segment_t* segment) { + if (segment->memid.memkind == MI_MEM_ARENA) return; // we lookup segments first in the arena's and don't need the segment map + size_t index; size_t bitidx; - size_t index = mi_segment_map_index_of(segment, &bitidx); - mi_assert_internal(index <= MI_SEGMENT_MAP_WSIZE); - if (index==MI_SEGMENT_MAP_WSIZE) return; - uintptr_t mask = mi_atomic_load_relaxed(&mi_segment_map[index]); + mi_segmap_part_t* part = mi_segment_map_index_of(segment, true /* alloc map if needed */, &index, &bitidx); + if (part == NULL) return; // outside our address range.. + uintptr_t mask = mi_atomic_load_relaxed(&part->map[index]); uintptr_t newmask; do { newmask = (mask | ((uintptr_t)1 << bitidx)); - } while (!mi_atomic_cas_weak_release(&mi_segment_map[index], &mask, newmask)); + } while (!mi_atomic_cas_weak_release(&part->map[index], &mask, newmask)); } void _mi_segment_map_freed_at(const mi_segment_t* segment) { + if (segment->memid.memkind == MI_MEM_ARENA) return; + size_t index; size_t bitidx; - size_t index = mi_segment_map_index_of(segment, &bitidx); - mi_assert_internal(index <= MI_SEGMENT_MAP_WSIZE); - if (index == MI_SEGMENT_MAP_WSIZE) return; - uintptr_t mask = mi_atomic_load_relaxed(&mi_segment_map[index]); + mi_segmap_part_t* part = mi_segment_map_index_of(segment, false /* don't alloc if not present */, &index, &bitidx); + if (part == NULL) return; // outside our address range.. + uintptr_t mask = mi_atomic_load_relaxed(&part->map[index]); uintptr_t newmask; do { newmask = (mask & ~((uintptr_t)1 << bitidx)); - } while (!mi_atomic_cas_weak_release(&mi_segment_map[index], &mask, newmask)); + } while (!mi_atomic_cas_weak_release(&part->map[index], &mask, newmask)); } // Determine the segment belonging to a pointer or NULL if it is not in a valid segment. static mi_segment_t* _mi_segment_of(const void* p) { if (p == NULL) return NULL; mi_segment_t* segment = _mi_ptr_segment(p); // segment can be NULL + size_t index; size_t bitidx; - size_t index = mi_segment_map_index_of(segment, &bitidx); - // fast path: for any pointer to valid small/medium/large object or first MI_SEGMENT_SIZE in huge - const uintptr_t mask = mi_atomic_load_relaxed(&mi_segment_map[index]); + mi_segmap_part_t* part = mi_segment_map_index_of(segment, false /* dont alloc if not present */, &index, &bitidx); + if (part == NULL) return NULL; + const uintptr_t mask = mi_atomic_load_relaxed(&part->map[index]); if mi_likely((mask & ((uintptr_t)1 << bitidx)) != 0) { + bool cookie_ok = (_mi_ptr_cookie(segment) == segment->cookie); + mi_assert_internal(cookie_ok); MI_UNUSED(cookie_ok); return segment; // yes, allocated by us } - if (index==MI_SEGMENT_MAP_WSIZE) return NULL; - - // TODO: maintain max/min allocated range for efficiency for more efficient rejection of invalid pointers? - - // search downwards for the first segment in case it is an interior pointer - // could be slow but searches in MI_INTPTR_SIZE * MI_SEGMENT_SIZE (512MiB) steps trough - // valid huge objects - // note: we could maintain a lowest index to speed up the path for invalid pointers? - size_t lobitidx; - size_t loindex; - uintptr_t lobits = mask & (((uintptr_t)1 << bitidx) - 1); - if (lobits != 0) { - loindex = index; - lobitidx = mi_bsr(lobits); // lobits != 0 - } - else if (index == 0) { - return NULL; - } - else { - mi_assert_internal(index > 0); - uintptr_t lomask = mask; - loindex = index; - do { - loindex--; - lomask = mi_atomic_load_relaxed(&mi_segment_map[loindex]); - } while (lomask != 0 && loindex > 0); - if (lomask == 0) return NULL; - lobitidx = mi_bsr(lomask); // lomask != 0 - } - mi_assert_internal(loindex < MI_SEGMENT_MAP_WSIZE); - // take difference as the addresses could be larger than the MAX_ADDRESS space. - size_t diff = (((index - loindex) * (8*MI_INTPTR_SIZE)) + bitidx - lobitidx) * MI_SEGMENT_SIZE; - segment = (mi_segment_t*)((uint8_t*)segment - diff); - - if (segment == NULL) return NULL; - mi_assert_internal((void*)segment < p); - bool cookie_ok = (_mi_ptr_cookie(segment) == segment->cookie); - mi_assert_internal(cookie_ok); - if mi_unlikely(!cookie_ok) return NULL; - if (((uint8_t*)segment + mi_segment_size(segment)) <= (uint8_t*)p) return NULL; // outside the range - mi_assert_internal(p >= (void*)segment && (uint8_t*)p < (uint8_t*)segment + mi_segment_size(segment)); - return segment; + return NULL; } // Is this a valid pointer in our heap? -static bool mi_is_valid_pointer(const void* p) { - return ((_mi_segment_of(p) != NULL) || (_mi_arena_contains(p))); +static bool mi_is_valid_pointer(const void* p) { + // first check if it is in an arena, then check if it is OS allocated + return (_mi_arena_contains(p) || _mi_segment_of(p) != NULL); } mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { return mi_is_valid_pointer(p); } - -/* -// Return the full segment range belonging to a pointer -static void* mi_segment_range_of(const void* p, size_t* size) { - mi_segment_t* segment = _mi_segment_of(p); - if (segment == NULL) { - if (size != NULL) *size = 0; - return NULL; - } - else { - if (size != NULL) *size = segment->segment_size; - return segment; - } - mi_assert_expensive(page == NULL || mi_segment_is_valid(_mi_page_segment(page),tld)); - mi_assert_internal(page == NULL || (mi_segment_page_size(_mi_page_segment(page)) - (MI_SECURE == 0 ? 0 : _mi_os_page_size())) >= block_size); - mi_reset_delayed(tld); - mi_assert_internal(page == NULL || mi_page_not_in_queue(page, tld)); - return page; -} -*/ diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/segment.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/segment.c index 9ac22f1..54a917e 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/segment.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/segment.c @@ -11,362 +11,461 @@ terms of the MIT license. A copy of the license can be found in the file #include <string.h> // memset #include <stdio.h> -// ------------------------------------------------------------------- -// Segments -// mimalloc pages reside in segments. See `mi_segment_valid` for invariants. -// ------------------------------------------------------------------- +#define MI_PAGE_HUGE_ALIGN (256*1024) +static uint8_t* mi_segment_raw_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size); -static void mi_segment_try_purge(mi_segment_t* segment, bool force, mi_stats_t* stats); +/* -------------------------------------------------------------------------------- + Segment allocation + We allocate pages inside bigger "segments" (4MiB on 64-bit). This is to avoid + splitting VMA's on Linux and reduce fragmentation on other OS's. + Each thread owns its own segments. + + Currently we have: + - small pages (64KiB), 64 in one segment + - medium pages (512KiB), 8 in one segment + - large pages (4MiB), 1 in one segment + - huge segments have 1 page in one segment that can be larger than `MI_SEGMENT_SIZE`. + it is used for blocks `> MI_LARGE_OBJ_SIZE_MAX` or with alignment `> MI_BLOCK_ALIGNMENT_MAX`. + + The memory for a segment is usually committed on demand. + (i.e. we are careful to not touch the memory until we actually allocate a block there) + + If a thread ends, it "abandons" pages that still contain live blocks. + Such segments are abondoned and these can be reclaimed by still running threads, + (much like work-stealing). +-------------------------------------------------------------------------------- */ -// ------------------------------------------------------------------- -// commit mask -// ------------------------------------------------------------------- +/* ----------------------------------------------------------- + Queue of segments containing free pages +----------------------------------------------------------- */ -static bool mi_commit_mask_all_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) { - for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { - if ((commit->mask[i] & cm->mask[i]) != cm->mask[i]) return false; - } - return true; +#if (MI_DEBUG>=3) +static bool mi_segment_queue_contains(const mi_segment_queue_t* queue, const mi_segment_t* segment) { + mi_assert_internal(segment != NULL); + mi_segment_t* list = queue->first; + while (list != NULL) { + if (list == segment) break; + mi_assert_internal(list->next==NULL || list->next->prev == list); + mi_assert_internal(list->prev==NULL || list->prev->next == list); + list = list->next; + } + return (list == segment); } +#endif -static bool mi_commit_mask_any_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) { - for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { - if ((commit->mask[i] & cm->mask[i]) != 0) return true; - } - return false; +/* +static bool mi_segment_queue_is_empty(const mi_segment_queue_t* queue) { + return (queue->first == NULL); } +*/ -static void mi_commit_mask_create_intersect(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm, mi_commit_mask_t* res) { - for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { - res->mask[i] = (commit->mask[i] & cm->mask[i]); - } +static void mi_segment_queue_remove(mi_segment_queue_t* queue, mi_segment_t* segment) { + mi_assert_expensive(mi_segment_queue_contains(queue, segment)); + if (segment->prev != NULL) segment->prev->next = segment->next; + if (segment->next != NULL) segment->next->prev = segment->prev; + if (segment == queue->first) queue->first = segment->next; + if (segment == queue->last) queue->last = segment->prev; + segment->next = NULL; + segment->prev = NULL; } -static void mi_commit_mask_clear(mi_commit_mask_t* res, const mi_commit_mask_t* cm) { - for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { - res->mask[i] &= ~(cm->mask[i]); +static void mi_segment_enqueue(mi_segment_queue_t* queue, mi_segment_t* segment) { + mi_assert_expensive(!mi_segment_queue_contains(queue, segment)); + segment->next = NULL; + segment->prev = queue->last; + if (queue->last != NULL) { + mi_assert_internal(queue->last->next == NULL); + queue->last->next = segment; + queue->last = segment; + } + else { + queue->last = queue->first = segment; } } -static void mi_commit_mask_set(mi_commit_mask_t* res, const mi_commit_mask_t* cm) { - for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { - res->mask[i] |= cm->mask[i]; - } +static mi_segment_queue_t* mi_segment_free_queue_of_kind(mi_page_kind_t kind, mi_segments_tld_t* tld) { + if (kind == MI_PAGE_SMALL) return &tld->small_free; + else if (kind == MI_PAGE_MEDIUM) return &tld->medium_free; + else return NULL; } -static void mi_commit_mask_create(size_t bitidx, size_t bitcount, mi_commit_mask_t* cm) { - mi_assert_internal(bitidx < MI_COMMIT_MASK_BITS); - mi_assert_internal((bitidx + bitcount) <= MI_COMMIT_MASK_BITS); - if (bitcount == MI_COMMIT_MASK_BITS) { - mi_assert_internal(bitidx==0); - mi_commit_mask_create_full(cm); +static mi_segment_queue_t* mi_segment_free_queue(const mi_segment_t* segment, mi_segments_tld_t* tld) { + return mi_segment_free_queue_of_kind(segment->page_kind, tld); +} + +// remove from free queue if it is in one +static void mi_segment_remove_from_free_queue(mi_segment_t* segment, mi_segments_tld_t* tld) { + mi_segment_queue_t* queue = mi_segment_free_queue(segment, tld); // may be NULL + bool in_queue = (queue!=NULL && (segment->next != NULL || segment->prev != NULL || queue->first == segment)); + if (in_queue) { + mi_segment_queue_remove(queue, segment); } - else if (bitcount == 0) { - mi_commit_mask_create_empty(cm); +} + +static void mi_segment_insert_in_free_queue(mi_segment_t* segment, mi_segments_tld_t* tld) { + mi_segment_enqueue(mi_segment_free_queue(segment, tld), segment); +} + + +/* ----------------------------------------------------------- + Invariant checking +----------------------------------------------------------- */ + +#if (MI_DEBUG >= 2) || (MI_SECURE >= 2) +static size_t mi_segment_page_size(const mi_segment_t* segment) { + if (segment->capacity > 1) { + mi_assert_internal(segment->page_kind <= MI_PAGE_MEDIUM); + return ((size_t)1 << segment->page_shift); } else { - mi_commit_mask_create_empty(cm); - size_t i = bitidx / MI_COMMIT_MASK_FIELD_BITS; - size_t ofs = bitidx % MI_COMMIT_MASK_FIELD_BITS; - while (bitcount > 0) { - mi_assert_internal(i < MI_COMMIT_MASK_FIELD_COUNT); - size_t avail = MI_COMMIT_MASK_FIELD_BITS - ofs; - size_t count = (bitcount > avail ? avail : bitcount); - size_t mask = (count >= MI_COMMIT_MASK_FIELD_BITS ? ~((size_t)0) : (((size_t)1 << count) - 1) << ofs); - cm->mask[i] = mask; - bitcount -= count; - ofs = 0; - i++; - } + mi_assert_internal(segment->page_kind >= MI_PAGE_LARGE); + return segment->segment_size; } } +#endif -size_t _mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total) { - mi_assert_internal((total%MI_COMMIT_MASK_BITS)==0); - size_t count = 0; - for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { - size_t mask = cm->mask[i]; - if (~mask == 0) { - count += MI_COMMIT_MASK_FIELD_BITS; - } - else { - for (; mask != 0; mask >>= 1) { // todo: use popcount - if ((mask&1)!=0) count++; - } - } +#if (MI_DEBUG>=2) +static bool mi_pages_purge_contains(const mi_page_t* page, mi_segments_tld_t* tld) { + mi_page_t* p = tld->pages_purge.first; + while (p != NULL) { + if (p == page) return true; + p = p->next; } - // we use total since for huge segments each commit bit may represent a larger size - return ((total / MI_COMMIT_MASK_BITS) * count); + return false; } +#endif - -size_t _mi_commit_mask_next_run(const mi_commit_mask_t* cm, size_t* idx) { - size_t i = (*idx) / MI_COMMIT_MASK_FIELD_BITS; - size_t ofs = (*idx) % MI_COMMIT_MASK_FIELD_BITS; - size_t mask = 0; - // find first ones - while (i < MI_COMMIT_MASK_FIELD_COUNT) { - mask = cm->mask[i]; - mask >>= ofs; - if (mask != 0) { - while ((mask&1) == 0) { - mask >>= 1; - ofs++; - } - break; +#if (MI_DEBUG>=3) +static bool mi_segment_is_valid(const mi_segment_t* segment, mi_segments_tld_t* tld) { + mi_assert_internal(segment != NULL); + mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie); + mi_assert_internal(segment->used <= segment->capacity); + mi_assert_internal(segment->abandoned <= segment->used); + mi_assert_internal(segment->page_kind <= MI_PAGE_MEDIUM || segment->capacity == 1); // one large or huge page per segment + size_t nfree = 0; + for (size_t i = 0; i < segment->capacity; i++) { + const mi_page_t* const page = &segment->pages[i]; + if (!page->segment_in_use) { + nfree++; + } + if (page->segment_in_use) { + mi_assert_expensive(!mi_pages_purge_contains(page, tld)); } - i++; - ofs = 0; + mi_assert_internal(page->is_huge == (segment->page_kind == MI_PAGE_HUGE)); } - if (i >= MI_COMMIT_MASK_FIELD_COUNT) { - // not found - *idx = MI_COMMIT_MASK_BITS; - return 0; + mi_assert_internal(nfree + segment->used == segment->capacity); + // mi_assert_internal(segment->thread_id == _mi_thread_id() || (segment->thread_id==0)); // or 0 + mi_assert_internal(segment->page_kind == MI_PAGE_HUGE || + (mi_segment_page_size(segment) * segment->capacity == segment->segment_size)); + return true; +} +#endif + +static bool mi_page_not_in_queue(const mi_page_t* page, mi_segments_tld_t* tld) { + mi_assert_internal(page != NULL); + if (page->next != NULL || page->prev != NULL) { + mi_assert_internal(mi_pages_purge_contains(page, tld)); + return false; } else { - // found, count ones - size_t count = 0; - *idx = (i*MI_COMMIT_MASK_FIELD_BITS) + ofs; - do { - mi_assert_internal(ofs < MI_COMMIT_MASK_FIELD_BITS && (mask&1) == 1); - do { - count++; - mask >>= 1; - } while ((mask&1) == 1); - if ((((*idx + count) % MI_COMMIT_MASK_FIELD_BITS) == 0)) { - i++; - if (i >= MI_COMMIT_MASK_FIELD_COUNT) break; - mask = cm->mask[i]; - ofs = 0; - } - } while ((mask&1) == 1); - mi_assert_internal(count > 0); - return count; + // both next and prev are NULL, check for singleton list + return (tld->pages_purge.first != page && tld->pages_purge.last != page); } } -/* -------------------------------------------------------------------------------- - Segment allocation --------------------------------------------------------------------------------- */ - - /* ----------------------------------------------------------- - Slices + Guard pages ----------------------------------------------------------- */ - -static const mi_slice_t* mi_segment_slices_end(const mi_segment_t* segment) { - return &segment->slices[segment->slice_entries]; +static void mi_segment_protect_range(void* p, size_t size, bool protect) { + if (protect) { + _mi_os_protect(p, size); + } + else { + _mi_os_unprotect(p, size); + } } -static uint8_t* mi_slice_start(const mi_slice_t* slice) { - mi_segment_t* segment = _mi_ptr_segment(slice); - mi_assert_internal(slice >= segment->slices && slice < mi_segment_slices_end(segment)); - return ((uint8_t*)segment + ((slice - segment->slices)*MI_SEGMENT_SLICE_SIZE)); +static void mi_segment_protect(mi_segment_t* segment, bool protect, mi_os_tld_t* tld) { + // add/remove guard pages + if (MI_SECURE != 0) { + // in secure mode, we set up a protected page in between the segment info and the page data + const size_t os_psize = _mi_os_page_size(); + mi_assert_internal((segment->segment_info_size - os_psize) >= (sizeof(mi_segment_t) + ((segment->capacity - 1) * sizeof(mi_page_t)))); + mi_assert_internal(((uintptr_t)segment + segment->segment_info_size) % os_psize == 0); + mi_segment_protect_range((uint8_t*)segment + segment->segment_info_size - os_psize, os_psize, protect); + #if (MI_SECURE >= 2) + if (segment->capacity == 1) + #endif + { + // and protect the last (or only) page too + mi_assert_internal(MI_SECURE <= 1 || segment->page_kind >= MI_PAGE_LARGE); + uint8_t* start = (uint8_t*)segment + segment->segment_size - os_psize; + if (protect && !segment->memid.initially_committed) { + if (protect) { + // ensure secure page is committed + if (_mi_os_commit(start, os_psize, NULL, tld->stats)) { // if this fails that is ok (as it is an unaccessible page) + mi_segment_protect_range(start, os_psize, protect); + } + } + } + else { + mi_segment_protect_range(start, os_psize, protect); + } + } + #if (MI_SECURE >= 2) + else { + // or protect every page + const size_t page_size = mi_segment_page_size(segment); + for (size_t i = 0; i < segment->capacity; i++) { + if (segment->pages[i].is_committed) { + mi_segment_protect_range((uint8_t*)segment + (i+1)*page_size - os_psize, os_psize, protect); + } + } + } + #endif + } } - /* ----------------------------------------------------------- - Bins + Page reset ----------------------------------------------------------- */ -// Use bit scan forward to quickly find the first zero bit if it is available - -static inline size_t mi_slice_bin8(size_t slice_count) { - if (slice_count<=1) return slice_count; - mi_assert_internal(slice_count <= MI_SLICES_PER_SEGMENT); - slice_count--; - size_t s = mi_bsr(slice_count); // slice_count > 1 - if (s <= 2) return slice_count + 1; - size_t bin = ((s << 2) | ((slice_count >> (s - 2))&0x03)) - 4; - return bin; -} -static inline size_t mi_slice_bin(size_t slice_count) { - mi_assert_internal(slice_count*MI_SEGMENT_SLICE_SIZE <= MI_SEGMENT_SIZE); - mi_assert_internal(mi_slice_bin8(MI_SLICES_PER_SEGMENT) <= MI_SEGMENT_BIN_MAX); - size_t bin = mi_slice_bin8(slice_count); - mi_assert_internal(bin <= MI_SEGMENT_BIN_MAX); - return bin; +static void mi_page_purge(mi_segment_t* segment, mi_page_t* page, mi_segments_tld_t* tld) { + // todo: should we purge the guard page as well when MI_SECURE>=2 ? + mi_assert_internal(page->is_committed); + mi_assert_internal(!page->segment_in_use); + if (!segment->allow_purge) return; + mi_assert_internal(page->used == 0); + mi_assert_internal(page->free == NULL); + mi_assert_expensive(!mi_pages_purge_contains(page, tld)); + size_t psize; + void* start = mi_segment_raw_page_start(segment, page, &psize); + const bool needs_recommit = _mi_os_purge(start, psize, tld->stats); + if (needs_recommit) { page->is_committed = false; } } -static inline size_t mi_slice_index(const mi_slice_t* slice) { - mi_segment_t* segment = _mi_ptr_segment(slice); - ptrdiff_t index = slice - segment->slices; - mi_assert_internal(index >= 0 && index < (ptrdiff_t)segment->slice_entries); - return index; +static bool mi_page_ensure_committed(mi_segment_t* segment, mi_page_t* page, mi_segments_tld_t* tld) { + if (page->is_committed) return true; + mi_assert_internal(segment->allow_decommit); + mi_assert_expensive(!mi_pages_purge_contains(page, tld)); + + size_t psize; + uint8_t* start = mi_segment_raw_page_start(segment, page, &psize); + bool is_zero = false; + const size_t gsize = (MI_SECURE >= 2 ? _mi_os_page_size() : 0); + bool ok = _mi_os_commit(start, psize + gsize, &is_zero, tld->stats); + if (!ok) return false; // failed to commit! + page->is_committed = true; + page->used = 0; + page->free = NULL; + page->is_zero_init = is_zero; + if (gsize > 0) { + mi_segment_protect_range(start + psize, gsize, true); + } + return true; } /* ----------------------------------------------------------- - Slice span queues + The free page queue ----------------------------------------------------------- */ -static void mi_span_queue_push(mi_span_queue_t* sq, mi_slice_t* slice) { - // todo: or push to the end? - mi_assert_internal(slice->prev == NULL && slice->next==NULL); - slice->prev = NULL; // paranoia - slice->next = sq->first; - sq->first = slice; - if (slice->next != NULL) slice->next->prev = slice; - else sq->last = slice; - slice->block_size = 0; // free +// we re-use the `free` field for the expiration counter. Since this is a +// a pointer size field while the clock is always 64-bit we need to guard +// against overflow, we use substraction to check for expiry which works +// as long as the reset delay is under (2^30 - 1) milliseconds (~12 days) +static uint32_t mi_page_get_expire( mi_page_t* page ) { + return (uint32_t)((uintptr_t)page->free); } -static mi_span_queue_t* mi_span_queue_for(size_t slice_count, mi_segments_tld_t* tld) { - size_t bin = mi_slice_bin(slice_count); - mi_span_queue_t* sq = &tld->spans[bin]; - mi_assert_internal(sq->slice_count >= slice_count); - return sq; +static void mi_page_set_expire( mi_page_t* page, uint32_t expire ) { + page->free = (mi_block_t*)((uintptr_t)expire); } -static void mi_span_queue_delete(mi_span_queue_t* sq, mi_slice_t* slice) { - mi_assert_internal(slice->block_size==0 && slice->slice_count>0 && slice->slice_offset==0); - // should work too if the queue does not contain slice (which can happen during reclaim) - if (slice->prev != NULL) slice->prev->next = slice->next; - if (slice == sq->first) sq->first = slice->next; - if (slice->next != NULL) slice->next->prev = slice->prev; - if (slice == sq->last) sq->last = slice->prev; - slice->prev = NULL; - slice->next = NULL; - slice->block_size = 1; // no more free +static void mi_page_purge_set_expire(mi_page_t* page) { + mi_assert_internal(mi_page_get_expire(page)==0); + uint32_t expire = (uint32_t)_mi_clock_now() + mi_option_get(mi_option_purge_delay); + mi_page_set_expire(page, expire); } - -/* ----------------------------------------------------------- - Invariant checking ------------------------------------------------------------ */ - -static bool mi_slice_is_used(const mi_slice_t* slice) { - return (slice->block_size > 0); +// we re-use the `free` field for the expiration counter. Since this is a +// a pointer size field while the clock is always 64-bit we need to guard +// against overflow, we use substraction to check for expiry which work +// as long as the reset delay is under (2^30 - 1) milliseconds (~12 days) +static bool mi_page_purge_is_expired(mi_page_t* page, mi_msecs_t now) { + int32_t expire = (int32_t)mi_page_get_expire(page); + return (((int32_t)now - expire) >= 0); } +static void mi_segment_schedule_purge(mi_segment_t* segment, mi_page_t* page, mi_segments_tld_t* tld) { + mi_assert_internal(!page->segment_in_use); + mi_assert_internal(mi_page_not_in_queue(page,tld)); + mi_assert_expensive(!mi_pages_purge_contains(page, tld)); + mi_assert_internal(_mi_page_segment(page)==segment); + if (!segment->allow_purge) return; -#if (MI_DEBUG>=3) -static bool mi_span_queue_contains(mi_span_queue_t* sq, mi_slice_t* slice) { - for (mi_slice_t* s = sq->first; s != NULL; s = s->next) { - if (s==slice) return true; + if (mi_option_get(mi_option_purge_delay) == 0) { + // purge immediately? + mi_page_purge(segment, page, tld); + } + else if (mi_option_get(mi_option_purge_delay) > 0) { // no purging if the delay is negative + // otherwise push on the delayed page reset queue + mi_page_queue_t* pq = &tld->pages_purge; + // push on top + mi_page_purge_set_expire(page); + page->next = pq->first; + page->prev = NULL; + if (pq->first == NULL) { + mi_assert_internal(pq->last == NULL); + pq->first = pq->last = page; + } + else { + pq->first->prev = page; + pq->first = page; + } } - return false; } -static bool mi_segment_is_valid(mi_segment_t* segment, mi_segments_tld_t* tld) { - mi_assert_internal(segment != NULL); - mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie); - mi_assert_internal(segment->abandoned <= segment->used); - mi_assert_internal(segment->thread_id == 0 || segment->thread_id == _mi_thread_id()); - mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask)); // can only decommit committed blocks - //mi_assert_internal(segment->segment_info_size % MI_SEGMENT_SLICE_SIZE == 0); - mi_slice_t* slice = &segment->slices[0]; - const mi_slice_t* end = mi_segment_slices_end(segment); - size_t used_count = 0; - mi_span_queue_t* sq; - while(slice < end) { - mi_assert_internal(slice->slice_count > 0); - mi_assert_internal(slice->slice_offset == 0); - size_t index = mi_slice_index(slice); - size_t maxindex = (index + slice->slice_count >= segment->slice_entries ? segment->slice_entries : index + slice->slice_count) - 1; - if (mi_slice_is_used(slice)) { // a page in use, we need at least MAX_SLICE_OFFSET_COUNT valid back offsets - used_count++; - mi_assert_internal(slice->is_huge == (segment->kind == MI_SEGMENT_HUGE)); - for (size_t i = 0; i <= MI_MAX_SLICE_OFFSET_COUNT && index + i <= maxindex; i++) { - mi_assert_internal(segment->slices[index + i].slice_offset == i*sizeof(mi_slice_t)); - mi_assert_internal(i==0 || segment->slices[index + i].slice_count == 0); - mi_assert_internal(i==0 || segment->slices[index + i].block_size == 1); - } - // and the last entry as well (for coalescing) - const mi_slice_t* last = slice + slice->slice_count - 1; - if (last > slice && last < mi_segment_slices_end(segment)) { - mi_assert_internal(last->slice_offset == (slice->slice_count-1)*sizeof(mi_slice_t)); - mi_assert_internal(last->slice_count == 0); - mi_assert_internal(last->block_size == 1); +static void mi_page_purge_remove(mi_page_t* page, mi_segments_tld_t* tld) { + if (mi_page_not_in_queue(page,tld)) return; + + mi_page_queue_t* pq = &tld->pages_purge; + mi_assert_internal(pq!=NULL); + mi_assert_internal(!page->segment_in_use); + mi_assert_internal(mi_page_get_expire(page) != 0); + mi_assert_internal(mi_pages_purge_contains(page, tld)); + if (page->prev != NULL) page->prev->next = page->next; + if (page->next != NULL) page->next->prev = page->prev; + if (page == pq->last) pq->last = page->prev; + if (page == pq->first) pq->first = page->next; + page->next = page->prev = NULL; + mi_page_set_expire(page,0); +} + +static void mi_segment_remove_all_purges(mi_segment_t* segment, bool force_purge, mi_segments_tld_t* tld) { + if (segment->memid.is_pinned) return; // never reset in huge OS pages + for (size_t i = 0; i < segment->capacity; i++) { + mi_page_t* page = &segment->pages[i]; + if (!page->segment_in_use) { + mi_page_purge_remove(page, tld); + if (force_purge && page->is_committed) { + mi_page_purge(segment, page, tld); } } - else { // free range of slices; only last slice needs a valid back offset - mi_slice_t* last = &segment->slices[maxindex]; - if (segment->kind != MI_SEGMENT_HUGE || slice->slice_count <= (segment->slice_entries - segment->segment_info_slices)) { - mi_assert_internal((uint8_t*)slice == (uint8_t*)last - last->slice_offset); - } - mi_assert_internal(slice == last || last->slice_count == 0 ); - mi_assert_internal(last->block_size == 0 || (segment->kind==MI_SEGMENT_HUGE && last->block_size==1)); - if (segment->kind != MI_SEGMENT_HUGE && segment->thread_id != 0) { // segment is not huge or abandoned - sq = mi_span_queue_for(slice->slice_count,tld); - mi_assert_internal(mi_span_queue_contains(sq,slice)); - } + else { + mi_assert_internal(mi_page_not_in_queue(page,tld)); } - slice = &segment->slices[maxindex+1]; } - mi_assert_internal(slice == end); - mi_assert_internal(used_count == segment->used + 1); - return true; } -#endif + +static void mi_pages_try_purge(bool force, mi_segments_tld_t* tld) { + if (mi_option_get(mi_option_purge_delay) < 0) return; // purging is not allowed + + mi_msecs_t now = _mi_clock_now(); + mi_page_queue_t* pq = &tld->pages_purge; + // from oldest up to the first that has not expired yet + mi_page_t* page = pq->last; + while (page != NULL && (force || mi_page_purge_is_expired(page,now))) { + mi_page_t* const prev = page->prev; // save previous field + mi_page_purge_remove(page, tld); // remove from the list to maintain invariant for mi_page_purge + mi_page_purge(_mi_page_segment(page), page, tld); + page = prev; + } + // discard the reset pages from the queue + pq->last = page; + if (page != NULL){ + page->next = NULL; + } + else { + pq->first = NULL; + } +} + /* ----------------------------------------------------------- Segment size calculations ----------------------------------------------------------- */ -static size_t mi_segment_info_size(mi_segment_t* segment) { - return segment->segment_info_slices * MI_SEGMENT_SLICE_SIZE; +static size_t mi_segment_raw_page_size(const mi_segment_t* segment) { + return (segment->page_kind == MI_PAGE_HUGE ? segment->segment_size : (size_t)1 << segment->page_shift); } -static uint8_t* _mi_segment_page_start_from_slice(const mi_segment_t* segment, const mi_slice_t* slice, size_t block_size, size_t* page_size) -{ - const ptrdiff_t idx = slice - segment->slices; - const size_t psize = (size_t)slice->slice_count * MI_SEGMENT_SLICE_SIZE; - uint8_t* const pstart = (uint8_t*)segment + (idx*MI_SEGMENT_SLICE_SIZE); - // make the start not OS page aligned for smaller blocks to avoid page/cache effects - // note: the offset must always be a block_size multiple since we assume small allocations - // are aligned (see `mi_heap_malloc_aligned`). - size_t start_offset = 0; - if (block_size > 0 && block_size <= MI_MAX_ALIGN_GUARANTEE) { - // for small objects, ensure the page start is aligned with the block size (PR#66 by kickunderscore) - const size_t adjust = block_size - ((uintptr_t)pstart % block_size); - if (adjust < block_size && psize >= block_size + adjust) { - start_offset += adjust; - } +// Raw start of the page available memory; can be used on uninitialized pages (only `segment_idx` must be set) +// The raw start is not taking aligned block allocation into consideration. +static uint8_t* mi_segment_raw_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size) { + size_t psize = mi_segment_raw_page_size(segment); + uint8_t* p = (uint8_t*)segment + page->segment_idx * psize; + + if (page->segment_idx == 0) { + // the first page starts after the segment info (and possible guard page) + p += segment->segment_info_size; + psize -= segment->segment_info_size; } - if (block_size >= MI_INTPTR_SIZE) { - if (block_size <= 64) { start_offset += 3*block_size; } - else if (block_size <= 512) { start_offset += block_size; } + +#if (MI_SECURE > 1) // every page has an os guard page + psize -= _mi_os_page_size(); +#elif (MI_SECURE==1) // the last page has an os guard page at the end + if (page->segment_idx == segment->capacity - 1) { + psize -= _mi_os_page_size(); } - if (page_size != NULL) { *page_size = psize - start_offset; } - return (pstart + start_offset); +#endif + + if (page_size != NULL) *page_size = psize; + mi_assert_internal(page->block_size == 0 || _mi_ptr_page(p) == page); + mi_assert_internal(_mi_ptr_segment(p) == segment); + return p; } -// Start of the page available memory; can be used on uninitialized pages +// Start of the page available memory; can be used on uninitialized pages (only `segment_idx` must be set) uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size) { - const mi_slice_t* slice = mi_page_to_slice((mi_page_t*)page); - uint8_t* p = _mi_segment_page_start_from_slice(segment, slice, mi_page_block_size(page), page_size); - mi_assert_internal(mi_page_block_size(page) > 0 || _mi_ptr_page(p) == page); + size_t psize; + uint8_t* p = mi_segment_raw_page_start(segment, page, &psize); + const size_t block_size = mi_page_block_size(page); + if (/*page->segment_idx == 0 &&*/ block_size > 0 && block_size <= MI_MAX_ALIGN_GUARANTEE) { + // for small and medium objects, ensure the page start is aligned with the block size (PR#66 by kickunderscore) + mi_assert_internal(segment->page_kind <= MI_PAGE_MEDIUM); + size_t adjust = block_size - ((uintptr_t)p % block_size); + if (adjust < block_size && psize >= block_size + adjust) { + p += adjust; + psize -= adjust; + mi_assert_internal((uintptr_t)p % block_size == 0); + } + } + + if (page_size != NULL) *page_size = psize; + mi_assert_internal(_mi_ptr_page(p) == page); mi_assert_internal(_mi_ptr_segment(p) == segment); return p; } -static size_t mi_segment_calculate_slices(size_t required, size_t* pre_size, size_t* info_slices) { - size_t page_size = _mi_os_page_size(); - size_t isize = _mi_align_up(sizeof(mi_segment_t), page_size); +static size_t mi_segment_calculate_sizes(size_t capacity, size_t required, size_t* pre_size, size_t* info_size) +{ + const size_t minsize = sizeof(mi_segment_t) + ((capacity - 1) * sizeof(mi_page_t)) + 16 /* padding */; size_t guardsize = 0; + size_t isize = 0; - if (MI_SECURE>0) { + if (MI_SECURE == 0) { + // normally no guard pages + isize = _mi_align_up(minsize, 16 * MI_MAX_ALIGN_SIZE); + } + else { // in secure mode, we set up a protected page in between the segment info // and the page data (and one at the end of the segment) + const size_t page_size = _mi_os_page_size(); + isize = _mi_align_up(minsize, page_size); guardsize = page_size; - if (required > 0) { - required = _mi_align_up(required, MI_SEGMENT_SLICE_SIZE) + page_size; - } + required = _mi_align_up(required, page_size); } - if (pre_size != NULL) *pre_size = isize; - isize = _mi_align_up(isize + guardsize, MI_SEGMENT_SLICE_SIZE); - if (info_slices != NULL) *info_slices = isize / MI_SEGMENT_SLICE_SIZE; - size_t segment_size = (required==0 ? MI_SEGMENT_SIZE : _mi_align_up( required + isize + guardsize, MI_SEGMENT_SLICE_SIZE) ); - mi_assert_internal(segment_size % MI_SEGMENT_SLICE_SIZE == 0); - return (segment_size / MI_SEGMENT_SLICE_SIZE); + if (info_size != NULL) *info_size = isize; + if (pre_size != NULL) *pre_size = isize + guardsize; + return (required==0 ? MI_SEGMENT_SIZE : _mi_align_up( required + isize + 2*guardsize, MI_PAGE_HUGE_ALIGN) ); } @@ -385,421 +484,43 @@ static void mi_segments_track_size(long segment_size, mi_segments_tld_t* tld) { if (tld->current_size > tld->peak_size) tld->peak_size = tld->current_size; } -static void mi_segment_os_free(mi_segment_t* segment, mi_segments_tld_t* tld) { +static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_segments_tld_t* tld) { segment->thread_id = 0; _mi_segment_map_freed_at(segment); - mi_segments_track_size(-((long)mi_segment_size(segment)),tld); + mi_segments_track_size(-((long)segment_size),tld); if (segment->was_reclaimed) { tld->reclaim_count--; segment->was_reclaimed = false; } - if (MI_SECURE>0) { - // _mi_os_unprotect(segment, mi_segment_size(segment)); // ensure no more guard pages are set - // unprotect the guard pages; we cannot just unprotect the whole segment size as part may be decommitted - size_t os_pagesize = _mi_os_page_size(); - _mi_os_unprotect((uint8_t*)segment + mi_segment_info_size(segment) - os_pagesize, os_pagesize); - uint8_t* end = (uint8_t*)segment + mi_segment_size(segment) - os_pagesize; - _mi_os_unprotect(end, os_pagesize); - } - - // purge delayed decommits now? (no, leave it to the arena) - // mi_segment_try_purge(segment,true,tld->stats); - - const size_t size = mi_segment_size(segment); - const size_t csize = _mi_commit_mask_committed_size(&segment->commit_mask, size); - - _mi_abandoned_await_readers(); // wait until safe to free - _mi_arena_free(segment, mi_segment_size(segment), csize, segment->memid, tld->stats); -} - -/* ----------------------------------------------------------- - Commit/Decommit ranges ------------------------------------------------------------ */ - -static void mi_segment_commit_mask(mi_segment_t* segment, bool conservative, uint8_t* p, size_t size, uint8_t** start_p, size_t* full_size, mi_commit_mask_t* cm) { - mi_assert_internal(_mi_ptr_segment(p + 1) == segment); - mi_assert_internal(segment->kind != MI_SEGMENT_HUGE); - mi_commit_mask_create_empty(cm); - if (size == 0 || size > MI_SEGMENT_SIZE || segment->kind == MI_SEGMENT_HUGE) return; - const size_t segstart = mi_segment_info_size(segment); - const size_t segsize = mi_segment_size(segment); - if (p >= (uint8_t*)segment + segsize) return; - - size_t pstart = (p - (uint8_t*)segment); - mi_assert_internal(pstart + size <= segsize); - - size_t start; - size_t end; - if (conservative) { - // decommit conservative - start = _mi_align_up(pstart, MI_COMMIT_SIZE); - end = _mi_align_down(pstart + size, MI_COMMIT_SIZE); - mi_assert_internal(start >= segstart); - mi_assert_internal(end <= segsize); - } - else { - // commit liberal - start = _mi_align_down(pstart, MI_MINIMAL_COMMIT_SIZE); - end = _mi_align_up(pstart + size, MI_MINIMAL_COMMIT_SIZE); - } - if (pstart >= segstart && start < segstart) { // note: the mask is also calculated for an initial commit of the info area - start = segstart; - } - if (end > segsize) { - end = segsize; - } - - mi_assert_internal(start <= pstart && (pstart + size) <= end); - mi_assert_internal(start % MI_COMMIT_SIZE==0 && end % MI_COMMIT_SIZE == 0); - *start_p = (uint8_t*)segment + start; - *full_size = (end > start ? end - start : 0); - if (*full_size == 0) return; - - size_t bitidx = start / MI_COMMIT_SIZE; - mi_assert_internal(bitidx < MI_COMMIT_MASK_BITS); - - size_t bitcount = *full_size / MI_COMMIT_SIZE; // can be 0 - if (bitidx + bitcount > MI_COMMIT_MASK_BITS) { - _mi_warning_message("commit mask overflow: idx=%zu count=%zu start=%zx end=%zx p=0x%p size=%zu fullsize=%zu\n", bitidx, bitcount, start, end, p, size, *full_size); - } - mi_assert_internal((bitidx + bitcount) <= MI_COMMIT_MASK_BITS); - mi_commit_mask_create(bitidx, bitcount, cm); -} - -static bool mi_segment_commit(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { - mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask)); - - // commit liberal - uint8_t* start = NULL; - size_t full_size = 0; - mi_commit_mask_t mask; - mi_segment_commit_mask(segment, false /* conservative? */, p, size, &start, &full_size, &mask); - if (mi_commit_mask_is_empty(&mask) || full_size == 0) return true; - - if (!mi_commit_mask_all_set(&segment->commit_mask, &mask)) { - // committing - bool is_zero = false; - mi_commit_mask_t cmask; - mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask); - _mi_stat_decrease(&_mi_stats_main.committed, _mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for overlap - if (!_mi_os_commit(start, full_size, &is_zero, stats)) return false; - mi_commit_mask_set(&segment->commit_mask, &mask); - } - - // increase purge expiration when using part of delayed purges -- we assume more allocations are coming soon. - if (mi_commit_mask_any_set(&segment->purge_mask, &mask)) { - segment->purge_expire = _mi_clock_now() + mi_option_get(mi_option_purge_delay); - } - - // always clear any delayed purges in our range (as they are either committed now) - mi_commit_mask_clear(&segment->purge_mask, &mask); - return true; -} - -static bool mi_segment_ensure_committed(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { - mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask)); - // note: assumes commit_mask is always full for huge segments as otherwise the commit mask bits can overflow - if (mi_commit_mask_is_full(&segment->commit_mask) && mi_commit_mask_is_empty(&segment->purge_mask)) return true; // fully committed - mi_assert_internal(segment->kind != MI_SEGMENT_HUGE); - return mi_segment_commit(segment, p, size, stats); -} - -static bool mi_segment_purge(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { - mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask)); - if (!segment->allow_purge) return true; - - // purge conservative - uint8_t* start = NULL; - size_t full_size = 0; - mi_commit_mask_t mask; - mi_segment_commit_mask(segment, true /* conservative? */, p, size, &start, &full_size, &mask); - if (mi_commit_mask_is_empty(&mask) || full_size==0) return true; - - if (mi_commit_mask_any_set(&segment->commit_mask, &mask)) { - // purging - mi_assert_internal((void*)start != (void*)segment); - mi_assert_internal(segment->allow_decommit); - const bool decommitted = _mi_os_purge(start, full_size, stats); // reset or decommit - if (decommitted) { - mi_commit_mask_t cmask; - mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask); - _mi_stat_increase(&_mi_stats_main.committed, full_size - _mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for double counting - mi_commit_mask_clear(&segment->commit_mask, &mask); - } - } - - // always clear any scheduled purges in our range - mi_commit_mask_clear(&segment->purge_mask, &mask); - return true; -} - -static void mi_segment_schedule_purge(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { - if (!segment->allow_purge) return; - - if (mi_option_get(mi_option_purge_delay) == 0) { - mi_segment_purge(segment, p, size, stats); - } - else { - // register for future purge in the purge mask - uint8_t* start = NULL; - size_t full_size = 0; - mi_commit_mask_t mask; - mi_segment_commit_mask(segment, true /*conservative*/, p, size, &start, &full_size, &mask); - if (mi_commit_mask_is_empty(&mask) || full_size==0) return; - - // update delayed commit - mi_assert_internal(segment->purge_expire > 0 || mi_commit_mask_is_empty(&segment->purge_mask)); - mi_commit_mask_t cmask; - mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask); // only purge what is committed; span_free may try to decommit more - mi_commit_mask_set(&segment->purge_mask, &cmask); - mi_msecs_t now = _mi_clock_now(); - if (segment->purge_expire == 0) { - // no previous purgess, initialize now - segment->purge_expire = now + mi_option_get(mi_option_purge_delay); - } - else if (segment->purge_expire <= now) { - // previous purge mask already expired - if (segment->purge_expire + mi_option_get(mi_option_purge_extend_delay) <= now) { - mi_segment_try_purge(segment, true, stats); - } - else { - segment->purge_expire = now + mi_option_get(mi_option_purge_extend_delay); // (mi_option_get(mi_option_purge_delay) / 8); // wait a tiny bit longer in case there is a series of free's - } - } - else { - // previous purge mask is not yet expired, increase the expiration by a bit. - segment->purge_expire += mi_option_get(mi_option_purge_extend_delay); - } - } -} -static void mi_segment_try_purge(mi_segment_t* segment, bool force, mi_stats_t* stats) { - if (!segment->allow_purge || segment->purge_expire == 0 || mi_commit_mask_is_empty(&segment->purge_mask)) return; - mi_msecs_t now = _mi_clock_now(); - if (!force && now < segment->purge_expire) return; - - mi_commit_mask_t mask = segment->purge_mask; - segment->purge_expire = 0; - mi_commit_mask_create_empty(&segment->purge_mask); - - size_t idx; - size_t count; - mi_commit_mask_foreach(&mask, idx, count) { - // if found, decommit that sequence - if (count > 0) { - uint8_t* p = (uint8_t*)segment + (idx*MI_COMMIT_SIZE); - size_t size = count * MI_COMMIT_SIZE; - mi_segment_purge(segment, p, size, stats); - } + if (MI_SECURE != 0) { + mi_assert_internal(!segment->memid.is_pinned); + mi_segment_protect(segment, false, tld->os); // ensure no more guard pages are set } - mi_commit_mask_foreach_end() - mi_assert_internal(mi_commit_mask_is_empty(&segment->purge_mask)); -} -// called from `mi_heap_collect_ex` -// this can be called per-page so it is important that try_purge has fast exit path -void _mi_segment_collect(mi_segment_t* segment, bool force, mi_segments_tld_t* tld) { - mi_segment_try_purge(segment, force, tld->stats); -} - -/* ----------------------------------------------------------- - Span free ------------------------------------------------------------ */ - -static bool mi_segment_is_abandoned(mi_segment_t* segment) { - return (mi_atomic_load_relaxed(&segment->thread_id) == 0); -} - -// note: can be called on abandoned segments -static void mi_segment_span_free(mi_segment_t* segment, size_t slice_index, size_t slice_count, bool allow_purge, mi_segments_tld_t* tld) { - mi_assert_internal(slice_index < segment->slice_entries); - mi_span_queue_t* sq = (segment->kind == MI_SEGMENT_HUGE || mi_segment_is_abandoned(segment) - ? NULL : mi_span_queue_for(slice_count,tld)); - if (slice_count==0) slice_count = 1; - mi_assert_internal(slice_index + slice_count - 1 < segment->slice_entries); - - // set first and last slice (the intermediates can be undetermined) - mi_slice_t* slice = &segment->slices[slice_index]; - slice->slice_count = (uint32_t)slice_count; - mi_assert_internal(slice->slice_count == slice_count); // no overflow? - slice->slice_offset = 0; - if (slice_count > 1) { - mi_slice_t* last = &segment->slices[slice_index + slice_count - 1]; - last->slice_count = 0; - last->slice_offset = (uint32_t)(sizeof(mi_page_t)*(slice_count - 1)); - last->block_size = 0; - } - - // perhaps decommit - if (allow_purge) { - mi_segment_schedule_purge(segment, mi_slice_start(slice), slice_count * MI_SEGMENT_SLICE_SIZE, tld->stats); - } - - // and push it on the free page queue (if it was not a huge page) - if (sq != NULL) mi_span_queue_push( sq, slice ); - else slice->block_size = 0; // mark huge page as free anyways -} - -/* -// called from reclaim to add existing free spans -static void mi_segment_span_add_free(mi_slice_t* slice, mi_segments_tld_t* tld) { - mi_segment_t* segment = _mi_ptr_segment(slice); - mi_assert_internal(slice->xblock_size==0 && slice->slice_count>0 && slice->slice_offset==0); - size_t slice_index = mi_slice_index(slice); - mi_segment_span_free(segment,slice_index,slice->slice_count,tld); -} -*/ - -static void mi_segment_span_remove_from_queue(mi_slice_t* slice, mi_segments_tld_t* tld) { - mi_assert_internal(slice->slice_count > 0 && slice->slice_offset==0 && slice->block_size==0); - mi_assert_internal(_mi_ptr_segment(slice)->kind != MI_SEGMENT_HUGE); - mi_span_queue_t* sq = mi_span_queue_for(slice->slice_count, tld); - mi_span_queue_delete(sq, slice); -} - -// note: can be called on abandoned segments -static mi_slice_t* mi_segment_span_free_coalesce(mi_slice_t* slice, mi_segments_tld_t* tld) { - mi_assert_internal(slice != NULL && slice->slice_count > 0 && slice->slice_offset == 0); - mi_segment_t* const segment = _mi_ptr_segment(slice); - const bool is_abandoned = (segment->thread_id == 0); // mi_segment_is_abandoned(segment); - - // for huge pages, just mark as free but don't add to the queues - if (segment->kind == MI_SEGMENT_HUGE) { - // issue #691: segment->used can be 0 if the huge page block was freed while abandoned (reclaim will get here in that case) - mi_assert_internal((segment->used==0 && slice->block_size==0) || segment->used == 1); // decreased right after this call in `mi_segment_page_clear` - slice->block_size = 0; // mark as free anyways - // we should mark the last slice `xblock_size=0` now to maintain invariants but we skip it to - // avoid a possible cache miss (and the segment is about to be freed) - return slice; - } - - // otherwise coalesce the span and add to the free span queues - size_t slice_count = slice->slice_count; - mi_slice_t* next = slice + slice->slice_count; - mi_assert_internal(next <= mi_segment_slices_end(segment)); - if (next < mi_segment_slices_end(segment) && next->block_size==0) { - // free next block -- remove it from free and merge - mi_assert_internal(next->slice_count > 0 && next->slice_offset==0); - slice_count += next->slice_count; // extend - if (!is_abandoned) { mi_segment_span_remove_from_queue(next, tld); } - } - if (slice > segment->slices) { - mi_slice_t* prev = mi_slice_first(slice - 1); - mi_assert_internal(prev >= segment->slices); - if (prev->block_size==0) { - // free previous slice -- remove it from free and merge - mi_assert_internal(prev->slice_count > 0 && prev->slice_offset==0); - slice_count += prev->slice_count; - if (!is_abandoned) { mi_segment_span_remove_from_queue(prev, tld); } - slice = prev; - } + bool fully_committed = true; + size_t committed_size = 0; + const size_t page_size = mi_segment_raw_page_size(segment); + for (size_t i = 0; i < segment->capacity; i++) { + mi_page_t* page = &segment->pages[i]; + if (page->is_committed) { committed_size += page_size; } + if (!page->is_committed) { fully_committed = false; } } + MI_UNUSED(fully_committed); + mi_assert_internal((fully_committed && committed_size == segment_size) || (!fully_committed && committed_size < segment_size)); - // and add the new free page - mi_segment_span_free(segment, mi_slice_index(slice), slice_count, true, tld); - return slice; + _mi_arena_free(segment, segment_size, committed_size, segment->memid, tld->stats); } - - -/* ----------------------------------------------------------- - Page allocation ------------------------------------------------------------ */ - -// Note: may still return NULL if committing the memory failed -static mi_page_t* mi_segment_span_allocate(mi_segment_t* segment, size_t slice_index, size_t slice_count, mi_segments_tld_t* tld) { - mi_assert_internal(slice_index < segment->slice_entries); - mi_slice_t* const slice = &segment->slices[slice_index]; - mi_assert_internal(slice->block_size==0 || slice->block_size==1); - - // commit before changing the slice data - if (!mi_segment_ensure_committed(segment, _mi_segment_page_start_from_slice(segment, slice, 0, NULL), slice_count * MI_SEGMENT_SLICE_SIZE, tld->stats)) { - return NULL; // commit failed! - } - - // convert the slices to a page - slice->slice_offset = 0; - slice->slice_count = (uint32_t)slice_count; - mi_assert_internal(slice->slice_count == slice_count); - const size_t bsize = slice_count * MI_SEGMENT_SLICE_SIZE; - slice->block_size = bsize; - mi_page_t* page = mi_slice_to_page(slice); - mi_assert_internal(mi_page_block_size(page) == bsize); - - // set slice back pointers for the first MI_MAX_SLICE_OFFSET_COUNT entries - size_t extra = slice_count-1; - if (extra > MI_MAX_SLICE_OFFSET_COUNT) extra = MI_MAX_SLICE_OFFSET_COUNT; - if (slice_index + extra >= segment->slice_entries) extra = segment->slice_entries - slice_index - 1; // huge objects may have more slices than avaiable entries in the segment->slices - - mi_slice_t* slice_next = slice + 1; - for (size_t i = 1; i <= extra; i++, slice_next++) { - slice_next->slice_offset = (uint32_t)(sizeof(mi_slice_t)*i); - slice_next->slice_count = 0; - slice_next->block_size = 1; - } - - // and also for the last one (if not set already) (the last one is needed for coalescing and for large alignments) - // note: the cast is needed for ubsan since the index can be larger than MI_SLICES_PER_SEGMENT for huge allocations (see #543) - mi_slice_t* last = slice + slice_count - 1; - mi_slice_t* end = (mi_slice_t*)mi_segment_slices_end(segment); - if (last > end) last = end; - if (last > slice) { - last->slice_offset = (uint32_t)(sizeof(mi_slice_t) * (last - slice)); - last->slice_count = 0; - last->block_size = 1; - } - - // and initialize the page - page->is_committed = true; - page->is_huge = (segment->kind == MI_SEGMENT_HUGE); - segment->used++; - return page; -} - -static void mi_segment_slice_split(mi_segment_t* segment, mi_slice_t* slice, size_t slice_count, mi_segments_tld_t* tld) { - mi_assert_internal(_mi_ptr_segment(slice) == segment); - mi_assert_internal(slice->slice_count >= slice_count); - mi_assert_internal(slice->block_size > 0); // no more in free queue - if (slice->slice_count <= slice_count) return; - mi_assert_internal(segment->kind != MI_SEGMENT_HUGE); - size_t next_index = mi_slice_index(slice) + slice_count; - size_t next_count = slice->slice_count - slice_count; - mi_segment_span_free(segment, next_index, next_count, false /* don't purge left-over part */, tld); - slice->slice_count = (uint32_t)slice_count; -} - -static mi_page_t* mi_segments_page_find_and_allocate(size_t slice_count, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld) { - mi_assert_internal(slice_count*MI_SEGMENT_SLICE_SIZE <= MI_LARGE_OBJ_SIZE_MAX); - // search from best fit up - mi_span_queue_t* sq = mi_span_queue_for(slice_count, tld); - if (slice_count == 0) slice_count = 1; - while (sq <= &tld->spans[MI_SEGMENT_BIN_MAX]) { - for (mi_slice_t* slice = sq->first; slice != NULL; slice = slice->next) { - if (slice->slice_count >= slice_count) { - // found one - mi_segment_t* segment = _mi_ptr_segment(slice); - if (_mi_arena_memid_is_suitable(segment->memid, req_arena_id)) { - // found a suitable page span - mi_span_queue_delete(sq, slice); - - if (slice->slice_count > slice_count) { - mi_segment_slice_split(segment, slice, slice_count, tld); - } - mi_assert_internal(slice != NULL && slice->slice_count == slice_count && slice->block_size > 0); - mi_page_t* page = mi_segment_span_allocate(segment, mi_slice_index(slice), slice->slice_count, tld); - if (page == NULL) { - // commit failed; return NULL but first restore the slice - mi_segment_span_free_coalesce(slice, tld); - return NULL; - } - return page; - } - } - } - sq++; +// called from `heap_collect`. +void _mi_segments_collect(bool force, mi_segments_tld_t* tld) { + mi_pages_try_purge(force,tld); + #if MI_DEBUG>=2 + if (!_mi_is_main_thread()) { + mi_assert_internal(tld->pages_purge.first == NULL); + mi_assert_internal(tld->pages_purge.last == NULL); } - // could not find a page.. - return NULL; + #endif } @@ -807,242 +528,239 @@ static mi_page_t* mi_segments_page_find_and_allocate(size_t slice_count, mi_aren Segment allocation ----------------------------------------------------------- */ -static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment, bool eager_delayed, mi_arena_id_t req_arena_id, - size_t* psegment_slices, size_t* ppre_size, size_t* pinfo_slices, - bool commit, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) - +static mi_segment_t* mi_segment_os_alloc(bool eager_delayed, size_t page_alignment, mi_arena_id_t req_arena_id, + size_t pre_size, size_t info_size, bool commit, size_t segment_size, + mi_segments_tld_t* tld, mi_os_tld_t* tld_os) { mi_memid_t memid; bool allow_large = (!eager_delayed && (MI_SECURE == 0)); // only allow large OS pages once we are no longer lazy size_t align_offset = 0; - size_t alignment = MI_SEGMENT_ALIGN; - + size_t alignment = MI_SEGMENT_SIZE; if (page_alignment > 0) { - // mi_assert_internal(huge_page != NULL); - mi_assert_internal(page_alignment >= MI_SEGMENT_ALIGN); alignment = page_alignment; - const size_t info_size = (*pinfo_slices) * MI_SEGMENT_SLICE_SIZE; - align_offset = _mi_align_up( info_size, MI_SEGMENT_ALIGN ); - const size_t extra = align_offset - info_size; - // recalculate due to potential guard pages - *psegment_slices = mi_segment_calculate_slices(required + extra, ppre_size, pinfo_slices); - mi_assert_internal(*psegment_slices > 0 && *psegment_slices <= UINT32_MAX); + align_offset = _mi_align_up(pre_size, MI_SEGMENT_SIZE); + segment_size = segment_size + (align_offset - pre_size); // adjust the segment size } - const size_t segment_size = (*psegment_slices) * MI_SEGMENT_SLICE_SIZE; - mi_segment_t* segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, alignment, align_offset, commit, allow_large, req_arena_id, &memid, os_tld); + mi_segment_t* segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, alignment, align_offset, commit, allow_large, req_arena_id, &memid, tld_os); if (segment == NULL) { return NULL; // failed to allocate } - // ensure metadata part of the segment is committed - mi_commit_mask_t commit_mask; - if (memid.initially_committed) { - mi_commit_mask_create_full(&commit_mask); - } - else { - // at least commit the info slices - const size_t commit_needed = _mi_divide_up((*pinfo_slices)*MI_SEGMENT_SLICE_SIZE, MI_COMMIT_SIZE); - mi_assert_internal(commit_needed>0); - mi_commit_mask_create(0, commit_needed, &commit_mask); - mi_assert_internal(commit_needed*MI_COMMIT_SIZE >= (*pinfo_slices)*MI_SEGMENT_SLICE_SIZE); - if (!_mi_os_commit(segment, commit_needed*MI_COMMIT_SIZE, NULL, tld->stats)) { - _mi_arena_free(segment,segment_size,0,memid,tld->stats); + if (!memid.initially_committed) { + // ensure the initial info is committed + mi_assert_internal(!memid.is_pinned); + bool ok = _mi_os_commit(segment, pre_size, NULL, tld_os->stats); + if (!ok) { + // commit failed; we cannot touch the memory: free the segment directly and return `NULL` + _mi_arena_free(segment, segment_size, 0, memid, tld_os->stats); return NULL; } } - mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0); + MI_UNUSED(info_size); segment->memid = memid; segment->allow_decommit = !memid.is_pinned; segment->allow_purge = segment->allow_decommit && (mi_option_get(mi_option_purge_delay) >= 0); segment->segment_size = segment_size; - segment->commit_mask = commit_mask; - segment->purge_expire = 0; - mi_commit_mask_create_empty(&segment->purge_mask); - + segment->subproc = tld->subproc; mi_segments_track_size((long)(segment_size), tld); _mi_segment_map_allocated_at(segment); return segment; } - // Allocate a segment from the OS aligned to `MI_SEGMENT_SIZE` . -static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld, mi_os_tld_t* os_tld, mi_page_t** huge_page) +static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind, size_t page_shift, size_t page_alignment, + mi_arena_id_t req_arena_id, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { - mi_assert_internal((required==0 && huge_page==NULL) || (required>0 && huge_page != NULL)); + // required is only > 0 for huge page allocations + mi_assert_internal((required > 0 && page_kind > MI_PAGE_LARGE)|| (required==0 && page_kind <= MI_PAGE_LARGE)); // calculate needed sizes first - size_t info_slices; + size_t capacity; + if (page_kind == MI_PAGE_HUGE) { + mi_assert_internal(page_shift == MI_SEGMENT_SHIFT + 1 && required > 0); + capacity = 1; + } + else { + mi_assert_internal(required == 0 && page_alignment == 0); + size_t page_size = (size_t)1 << page_shift; + capacity = MI_SEGMENT_SIZE / page_size; + mi_assert_internal(MI_SEGMENT_SIZE % page_size == 0); + mi_assert_internal(capacity >= 1 && capacity <= MI_SMALL_PAGES_PER_SEGMENT); + } + size_t info_size; size_t pre_size; - size_t segment_slices = mi_segment_calculate_slices(required, &pre_size, &info_slices); - mi_assert_internal(segment_slices > 0 && segment_slices <= UINT32_MAX); - - // Commit eagerly only if not the first N lazy segments (to reduce impact of many threads that allocate just a little) - const bool eager_delay = (// !_mi_os_has_overcommit() && // never delay on overcommit systems - _mi_current_thread_count() > 1 && // do not delay for the first N threads - tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay)); - const bool eager = !eager_delay && mi_option_is_enabled(mi_option_eager_commit); - bool commit = eager || (required > 0); - - // Allocate the segment from the OS - mi_segment_t* segment = mi_segment_os_alloc(required, page_alignment, eager_delay, req_arena_id, - &segment_slices, &pre_size, &info_slices, commit, tld, os_tld); + const size_t init_segment_size = mi_segment_calculate_sizes(capacity, required, &pre_size, &info_size); + mi_assert_internal(init_segment_size >= required); + + // Initialize parameters + const bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && // don't delay for large objects + // !_mi_os_has_overcommit() && // never delay on overcommit systems + _mi_current_thread_count() > 1 && // do not delay for the first N threads + tld->peak_count < (size_t)mi_option_get(mi_option_eager_commit_delay)); + const bool eager = !eager_delayed && mi_option_is_enabled(mi_option_eager_commit); + const bool init_commit = eager; // || (page_kind >= MI_PAGE_LARGE); + + // Allocate the segment from the OS (segment_size can change due to alignment) + mi_segment_t* segment = mi_segment_os_alloc(eager_delayed, page_alignment, req_arena_id, pre_size, info_size, init_commit, init_segment_size, tld, os_tld); if (segment == NULL) return NULL; + mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0); + mi_assert_internal(segment->memid.is_pinned ? segment->memid.initially_committed : true); - // zero the segment info? -- not always needed as it may be zero initialized from the OS - if (!segment->memid.initially_zero) { - ptrdiff_t ofs = offsetof(mi_segment_t, next); - size_t prefix = offsetof(mi_segment_t, slices) - ofs; - size_t zsize = prefix + (sizeof(mi_slice_t) * (segment_slices + 1)); // one more - _mi_memzero((uint8_t*)segment + ofs, zsize); - } - - // initialize the rest of the segment info - const size_t slice_entries = (segment_slices > MI_SLICES_PER_SEGMENT ? MI_SLICES_PER_SEGMENT : segment_slices); - segment->segment_slices = segment_slices; - segment->segment_info_slices = info_slices; - segment->thread_id = _mi_thread_id(); - segment->cookie = _mi_ptr_cookie(segment); - segment->slice_entries = slice_entries; - segment->kind = (required == 0 ? MI_SEGMENT_NORMAL : MI_SEGMENT_HUGE); - - // _mi_memzero(segment->slices, sizeof(mi_slice_t)*(info_slices+1)); - _mi_stat_increase(&tld->stats->page_committed, mi_segment_info_size(segment)); - - // set up guard pages - size_t guard_slices = 0; - if (MI_SECURE>0) { - // in secure mode, we set up a protected page in between the segment info - // and the page data, and at the end of the segment. - size_t os_pagesize = _mi_os_page_size(); - mi_assert_internal(mi_segment_info_size(segment) - os_pagesize >= pre_size); - _mi_os_protect((uint8_t*)segment + mi_segment_info_size(segment) - os_pagesize, os_pagesize); - uint8_t* end = (uint8_t*)segment + mi_segment_size(segment) - os_pagesize; - mi_segment_ensure_committed(segment, end, os_pagesize, tld->stats); - _mi_os_protect(end, os_pagesize); - if (slice_entries == segment_slices) segment->slice_entries--; // don't use the last slice :-( - guard_slices = 1; - } - - // reserve first slices for segment info - mi_page_t* page0 = mi_segment_span_allocate(segment, 0, info_slices, tld); - mi_assert_internal(page0!=NULL); if (page0==NULL) return NULL; // cannot fail as we always commit in advance - mi_assert_internal(segment->used == 1); - segment->used = 0; // don't count our internal slices towards usage - - // initialize initial free pages - if (segment->kind == MI_SEGMENT_NORMAL) { // not a huge page - mi_assert_internal(huge_page==NULL); - mi_segment_span_free(segment, info_slices, segment->slice_entries - info_slices, false /* don't purge */, tld); + // zero the segment info (but not the `mem` fields) + ptrdiff_t ofs = offsetof(mi_segment_t, next); + _mi_memzero((uint8_t*)segment + ofs, info_size - ofs); + + // initialize pages info + const bool is_huge = (page_kind == MI_PAGE_HUGE); + for (size_t i = 0; i < capacity; i++) { + mi_assert_internal(i <= 255); + segment->pages[i].segment_idx = (uint8_t)i; + segment->pages[i].is_committed = segment->memid.initially_committed; + segment->pages[i].is_zero_init = segment->memid.initially_zero; + segment->pages[i].is_huge = is_huge; } - else { - mi_assert_internal(huge_page!=NULL); - mi_assert_internal(mi_commit_mask_is_empty(&segment->purge_mask)); - mi_assert_internal(mi_commit_mask_is_full(&segment->commit_mask)); - *huge_page = mi_segment_span_allocate(segment, info_slices, segment_slices - info_slices - guard_slices, tld); - mi_assert_internal(*huge_page != NULL); // cannot fail as we commit in advance + + // initialize + segment->page_kind = page_kind; + segment->capacity = capacity; + segment->page_shift = page_shift; + segment->segment_info_size = pre_size; + segment->thread_id = _mi_thread_id(); + segment->cookie = _mi_ptr_cookie(segment); + + // set protection + mi_segment_protect(segment, true, tld->os); + + // insert in free lists for small and medium pages + if (page_kind <= MI_PAGE_MEDIUM) { + mi_segment_insert_in_free_queue(segment, tld); } - mi_assert_expensive(mi_segment_is_valid(segment,tld)); return segment; } static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t* tld) { MI_UNUSED(force); - mi_assert_internal(segment != NULL); - mi_assert_internal(segment->next == NULL); - mi_assert_internal(segment->used == 0); - - // Remove the free pages - mi_slice_t* slice = &segment->slices[0]; - const mi_slice_t* end = mi_segment_slices_end(segment); - #if MI_DEBUG>1 - size_t page_count = 0; - #endif - while (slice < end) { - mi_assert_internal(slice->slice_count > 0); - mi_assert_internal(slice->slice_offset == 0); - mi_assert_internal(mi_slice_index(slice)==0 || slice->block_size == 0); // no more used pages .. - if (slice->block_size == 0 && segment->kind != MI_SEGMENT_HUGE) { - mi_segment_span_remove_from_queue(slice, tld); - } - #if MI_DEBUG>1 - page_count++; - #endif - slice = slice + slice->slice_count; - } - mi_assert_internal(page_count == 2); // first page is allocated by the segment itself + mi_assert(segment != NULL); + // don't purge as we are freeing now + mi_segment_remove_all_purges(segment, false /* don't force as we are about to free */, tld); + mi_segment_remove_from_free_queue(segment, tld); - // stats - _mi_stat_decrease(&tld->stats->page_committed, mi_segment_info_size(segment)); + mi_assert_expensive(!mi_segment_queue_contains(&tld->small_free, segment)); + mi_assert_expensive(!mi_segment_queue_contains(&tld->medium_free, segment)); + mi_assert(segment->next == NULL); + mi_assert(segment->prev == NULL); + _mi_stat_decrease(&tld->stats->page_committed, segment->segment_info_size); // return it to the OS - mi_segment_os_free(segment, tld); + mi_segment_os_free(segment, segment->segment_size, tld); +} + +/* ----------------------------------------------------------- + Free page management inside a segment +----------------------------------------------------------- */ + + +static bool mi_segment_has_free(const mi_segment_t* segment) { + return (segment->used < segment->capacity); +} + +static bool mi_segment_page_claim(mi_segment_t* segment, mi_page_t* page, mi_segments_tld_t* tld) { + mi_assert_internal(_mi_page_segment(page) == segment); + mi_assert_internal(!page->segment_in_use); + mi_page_purge_remove(page, tld); + + // check commit + if (!mi_page_ensure_committed(segment, page, tld)) return false; + + // set in-use before doing unreset to prevent delayed reset + page->segment_in_use = true; + segment->used++; + mi_assert_internal(page->segment_in_use && page->is_committed && page->used==0 && !mi_pages_purge_contains(page,tld)); + mi_assert_internal(segment->used <= segment->capacity); + if (segment->used == segment->capacity && segment->page_kind <= MI_PAGE_MEDIUM) { + // if no more free pages, remove from the queue + mi_assert_internal(!mi_segment_has_free(segment)); + mi_segment_remove_from_free_queue(segment, tld); + } + return true; } /* ----------------------------------------------------------- - Page Free + Free ----------------------------------------------------------- */ static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld); -// note: can be called on abandoned pages -static mi_slice_t* mi_segment_page_clear(mi_page_t* page, mi_segments_tld_t* tld) { - mi_assert_internal(page->block_size > 0); +// clear page data; can be called on abandoned segments +static void mi_segment_page_clear(mi_segment_t* segment, mi_page_t* page, mi_segments_tld_t* tld) +{ + mi_assert_internal(page->segment_in_use); mi_assert_internal(mi_page_all_free(page)); - mi_segment_t* segment = _mi_ptr_segment(page); - mi_assert_internal(segment->used > 0); + mi_assert_internal(page->is_committed); + mi_assert_internal(mi_page_not_in_queue(page, tld)); size_t inuse = page->capacity * mi_page_block_size(page); _mi_stat_decrease(&tld->stats->page_committed, inuse); _mi_stat_decrease(&tld->stats->pages, 1); - // reset the page memory to reduce memory pressure? - if (segment->allow_decommit && mi_option_is_enabled(mi_option_deprecated_page_reset)) { - size_t psize; - uint8_t* start = _mi_segment_page_start(segment, page, &psize); - _mi_os_reset(start, psize, tld->stats); - } - - // zero the page data, but not the segment fields page->is_zero_init = false; - ptrdiff_t ofs = offsetof(mi_page_t, capacity); + page->segment_in_use = false; + + // zero the page data, but not the segment fields and capacity, page start, and block_size (for page size calculations) + size_t block_size = page->block_size; + uint8_t block_size_shift = page->block_size_shift; + uint8_t heap_tag = page->heap_tag; + uint8_t* page_start = page->page_start; + uint16_t capacity = page->capacity; + uint16_t reserved = page->reserved; + ptrdiff_t ofs = offsetof(mi_page_t,capacity); _mi_memzero((uint8_t*)page + ofs, sizeof(*page) - ofs); - page->block_size = 1; - - // and free it - mi_slice_t* slice = mi_segment_span_free_coalesce(mi_page_to_slice(page), tld); + page->capacity = capacity; + page->reserved = reserved; + page->block_size = block_size; + page->block_size_shift = block_size_shift; + page->heap_tag = heap_tag; + page->page_start = page_start; segment->used--; - // cannot assert segment valid as it is called during reclaim - // mi_assert_expensive(mi_segment_is_valid(segment, tld)); - return slice; + + // schedule purge + mi_segment_schedule_purge(segment, page, tld); + + page->capacity = 0; // after purge these can be zero'd now + page->reserved = 0; } void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld) { mi_assert(page != NULL); - mi_segment_t* segment = _mi_page_segment(page); mi_assert_expensive(mi_segment_is_valid(segment,tld)); + mi_pages_try_purge(false /*force?*/, tld); // mark it as free now - mi_segment_page_clear(page, tld); - mi_assert_expensive(mi_segment_is_valid(segment, tld)); + mi_segment_page_clear(segment, page, tld); if (segment->used == 0) { // no more used pages; remove from the free list and free the segment mi_segment_free(segment, force, tld); } - else if (segment->used == segment->abandoned) { - // only abandoned pages; remove from free list and abandon - mi_segment_abandon(segment,tld); - } else { - // perform delayed purges - mi_segment_try_purge(segment, false /* force? */, tld->stats); + if (segment->used == segment->abandoned) { + // only abandoned pages; remove from free list and abandon + mi_segment_abandon(segment,tld); + } + else if (segment->used + 1 == segment->capacity) { + mi_assert_internal(segment->page_kind <= MI_PAGE_MEDIUM); // large and huge pages are always the single page in a segment + if (segment->page_kind <= MI_PAGE_MEDIUM) { + // move back to segments free list + mi_segment_insert_in_free_queue(segment,tld); + } + } } } @@ -1051,7 +769,7 @@ void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld) Abandonment When threads terminate, they can leave segments with -live blocks (reachable through other threads). Such segments +live blocks (reached through other threads). Such segments are "abandoned" and will be reclaimed by other threads to reuse their pages and/or free them eventually. The `thread_id` of such segments is 0. @@ -1065,11 +783,6 @@ by scanning the arena memory (segments outside arena memoryare only reclaimed by a free). ----------------------------------------------------------- */ -// legacy: Wait until there are no more pending reads on segments that used to be in the abandoned list -void _mi_abandoned_await_readers(void) { - // nothing needed -} - /* ----------------------------------------------------------- Abandon segment/page ----------------------------------------------------------- */ @@ -1077,33 +790,22 @@ void _mi_abandoned_await_readers(void) { static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) { mi_assert_internal(segment->used == segment->abandoned); mi_assert_internal(segment->used > 0); - mi_assert_internal(segment->abandoned_visits == 0); - mi_assert_expensive(mi_segment_is_valid(segment,tld)); + mi_assert_expensive(mi_segment_is_valid(segment, tld)); - // remove the free pages from the free page queues - mi_slice_t* slice = &segment->slices[0]; - const mi_slice_t* end = mi_segment_slices_end(segment); - while (slice < end) { - mi_assert_internal(slice->slice_count > 0); - mi_assert_internal(slice->slice_offset == 0); - if (slice->block_size == 0) { // a free page - mi_segment_span_remove_from_queue(slice,tld); - slice->block_size = 0; // but keep it free - } - slice = slice + slice->slice_count; - } + // Potentially force purge. Only abandoned segments in arena memory can be + // reclaimed without a free so if a segment is not from an arena we force purge here to be conservative. + mi_pages_try_purge(false /*force?*/,tld); + const bool force_purge = (segment->memid.memkind != MI_MEM_ARENA) || mi_option_is_enabled(mi_option_abandoned_page_purge); + mi_segment_remove_all_purges(segment, force_purge, tld); - // perform delayed decommits (forcing is much slower on mstress) - // Only abandoned segments in arena memory can be reclaimed without a free - // so if a segment is not from an arena we force purge here to be conservative. - const bool force_purge = (segment->memid.memkind != MI_MEM_ARENA) || mi_option_is_enabled(mi_option_abandoned_page_purge); - mi_segment_try_purge(segment, force_purge, tld->stats); + // remove the segment from the free page queue if needed + mi_segment_remove_from_free_queue(segment, tld); + mi_assert_internal(segment->next == NULL && segment->prev == NULL); // all pages in the segment are abandoned; add it to the abandoned list _mi_stat_increase(&tld->stats->segments_abandoned, 1); - mi_segments_track_size(-((long)mi_segment_size(segment)), tld); - segment->thread_id = 0; - segment->abandoned_visits = 1; // from 0 to 1 to signify it is abandoned + mi_segments_track_size(-((long)segment->segment_size), tld); + segment->abandoned_visits = 0; if (segment->was_reclaimed) { tld->reclaim_count--; segment->was_reclaimed = false; @@ -1116,10 +818,9 @@ void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld) { mi_assert_internal(mi_page_thread_free_flag(page)==MI_NEVER_DELAYED_FREE); mi_assert_internal(mi_page_heap(page) == NULL); mi_segment_t* segment = _mi_page_segment(page); - - mi_assert_expensive(mi_segment_is_valid(segment,tld)); + mi_assert_expensive(!mi_pages_purge_contains(page, tld)); + mi_assert_expensive(mi_segment_is_valid(segment, tld)); segment->abandoned++; - _mi_stat_increase(&tld->stats->pages_abandoned, 1); mi_assert_internal(segment->abandoned <= segment->used); if (segment->used == segment->abandoned) { @@ -1132,40 +833,24 @@ void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld) { Reclaim abandoned pages ----------------------------------------------------------- */ -static mi_slice_t* mi_slices_start_iterate(mi_segment_t* segment, const mi_slice_t** end) { - mi_slice_t* slice = &segment->slices[0]; - *end = mi_segment_slices_end(segment); - mi_assert_internal(slice->slice_count>0 && slice->block_size>0); // segment allocated page - slice = slice + slice->slice_count; // skip the first segment allocated page - return slice; -} - -// Possibly free pages and check if free space is available -static bool mi_segment_check_free(mi_segment_t* segment, size_t slices_needed, size_t block_size, mi_segments_tld_t* tld) +// Possibly clear pages and check if free space is available +static bool mi_segment_check_free(mi_segment_t* segment, size_t block_size, bool* all_pages_free) { - mi_assert_internal(mi_segment_is_abandoned(segment)); + mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id) == 0); bool has_page = false; - - // for all slices - const mi_slice_t* end; - mi_slice_t* slice = mi_slices_start_iterate(segment, &end); - while (slice < end) { - mi_assert_internal(slice->slice_count > 0); - mi_assert_internal(slice->slice_offset == 0); - if (mi_slice_is_used(slice)) { // used page + size_t pages_used = 0; + size_t pages_used_empty = 0; + for (size_t i = 0; i < segment->capacity; i++) { + mi_page_t* page = &segment->pages[i]; + if (page->segment_in_use) { + pages_used++; // ensure used count is up to date and collect potential concurrent frees - mi_page_t* const page = mi_slice_to_page(slice); _mi_page_free_collect(page, false); if (mi_page_all_free(page)) { - // if this page is all free now, free it without adding to any queues (yet) - mi_assert_internal(page->next == NULL && page->prev==NULL); - _mi_stat_decrease(&tld->stats->pages_abandoned, 1); - segment->abandoned--; - slice = mi_segment_page_clear(page, tld); // re-assign slice due to coalesce! - mi_assert_internal(!mi_slice_is_used(slice)); - if (slice->slice_count >= slices_needed) { - has_page = true; - } + // if everything free already, page can be reused for some block size + // note: don't clear the page yet as we can only OS reset it once it is reclaimed + pages_used_empty++; + has_page = true; } else if (mi_page_block_size(page) == block_size && mi_page_has_any_available(page)) { // a page has available free blocks of the right size @@ -1173,87 +858,99 @@ static bool mi_segment_check_free(mi_segment_t* segment, size_t slices_needed, s } } else { - // empty span - if (slice->slice_count >= slices_needed) { - has_page = true; - } + // whole empty page + has_page = true; } - slice = slice + slice->slice_count; + } + mi_assert_internal(pages_used == segment->used && pages_used >= pages_used_empty); + if (all_pages_free != NULL) { + *all_pages_free = ((pages_used - pages_used_empty) == 0); } return has_page; } -// Reclaim an abandoned segment; returns NULL if the segment was freed + +// Reclaim a segment; returns NULL if the segment was freed // set `right_page_reclaimed` to `true` if it reclaimed a page of the right `block_size` that was not full. static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, size_t requested_block_size, bool* right_page_reclaimed, mi_segments_tld_t* tld) { if (right_page_reclaimed != NULL) { *right_page_reclaimed = false; } // can be 0 still with abandoned_next, or already a thread id for segments outside an arena that are reclaimed on a free. mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id) == 0 || mi_atomic_load_relaxed(&segment->thread_id) == _mi_thread_id()); + mi_assert_internal(segment->subproc == heap->tld->segments.subproc); // only reclaim within the same subprocess mi_atomic_store_release(&segment->thread_id, _mi_thread_id()); segment->abandoned_visits = 0; segment->was_reclaimed = true; tld->reclaim_count++; - mi_segments_track_size((long)mi_segment_size(segment), tld); - mi_assert_internal(segment->next == NULL); + mi_segments_track_size((long)segment->segment_size, tld); + mi_assert_internal(segment->next == NULL && segment->prev == NULL); + mi_assert_expensive(mi_segment_is_valid(segment, tld)); _mi_stat_decrease(&tld->stats->segments_abandoned, 1); - // for all slices - const mi_slice_t* end; - mi_slice_t* slice = mi_slices_start_iterate(segment, &end); - while (slice < end) { - mi_assert_internal(slice->slice_count > 0); - mi_assert_internal(slice->slice_offset == 0); - if (mi_slice_is_used(slice)) { - // in use: reclaim the page in our heap - mi_page_t* page = mi_slice_to_page(slice); + for (size_t i = 0; i < segment->capacity; i++) { + mi_page_t* page = &segment->pages[i]; + if (page->segment_in_use) { mi_assert_internal(page->is_committed); + mi_assert_internal(mi_page_not_in_queue(page, tld)); mi_assert_internal(mi_page_thread_free_flag(page)==MI_NEVER_DELAYED_FREE); mi_assert_internal(mi_page_heap(page) == NULL); - mi_assert_internal(page->next == NULL && page->prev==NULL); - _mi_stat_decrease(&tld->stats->pages_abandoned, 1); segment->abandoned--; - // set the heap again and allow delayed free again - mi_page_set_heap(page, heap); + mi_assert(page->next == NULL); + _mi_stat_decrease(&tld->stats->pages_abandoned, 1); + // get the target heap for this thread which has a matching heap tag (so we reclaim into a matching heap) + mi_heap_t* target_heap = _mi_heap_by_tag(heap, page->heap_tag); // allow custom heaps to separate objects + if (target_heap == NULL) { + target_heap = heap; + _mi_error_message(EFAULT, "page with tag %u cannot be reclaimed by a heap with the same tag (using heap tag %u instead)\n", page->heap_tag, heap->tag ); + } + // associate the heap with this page, and allow heap thread delayed free again. + mi_page_set_heap(page, target_heap); _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, true); // override never (after heap is set) _mi_page_free_collect(page, false); // ensure used count is up to date if (mi_page_all_free(page)) { - // if everything free by now, free the page - slice = mi_segment_page_clear(page, tld); // set slice again due to coalesceing + // if everything free already, clear the page directly + mi_segment_page_clear(segment, page, tld); // reset is ok now } else { // otherwise reclaim it into the heap - _mi_page_reclaim(heap, page); - if (requested_block_size == mi_page_block_size(page) && mi_page_has_any_available(page)) { + _mi_page_reclaim(target_heap, page); + if (requested_block_size == mi_page_block_size(page) && mi_page_has_any_available(page) && heap == target_heap) { if (right_page_reclaimed != NULL) { *right_page_reclaimed = true; } } } } - else { - // the span is free, add it to our page queues - slice = mi_segment_span_free_coalesce(slice, tld); // set slice again due to coalesceing + /* expired + else if (page->is_committed) { // not in-use, and not reset yet + // note: do not reset as this includes pages that were not touched before + // mi_pages_purge_add(segment, page, tld); } - mi_assert_internal(slice->slice_count>0 && slice->slice_offset==0); - slice = slice + slice->slice_count; + */ } - - mi_assert(segment->abandoned == 0); - mi_assert_expensive(mi_segment_is_valid(segment, tld)); - if (segment->used == 0) { // due to page_clear + mi_assert_internal(segment->abandoned == 0); + if (segment->used == 0) { mi_assert_internal(right_page_reclaimed == NULL || !(*right_page_reclaimed)); mi_segment_free(segment, false, tld); return NULL; } else { + if (segment->page_kind <= MI_PAGE_MEDIUM && mi_segment_has_free(segment)) { + mi_segment_insert_in_free_queue(segment, tld); + } return segment; } } + // attempt to reclaim a particular segment (called from multi threaded free `alloc.c:mi_free_block_mt`) bool _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment) { if (mi_atomic_load_relaxed(&segment->thread_id) != 0) return false; // it is not abandoned - // don't reclaim more from a free than half the current segments + if (segment->subproc != heap->tld->segments.subproc) return false; // only reclaim within the same subprocess + if (!_mi_heap_memid_is_suitable(heap,segment->memid)) return false; // don't reclaim between exclusive and non-exclusive arena's + // don't reclaim more from a `free` call than half the current segments // this is to prevent a pure free-ing thread to start owning too many segments - if (heap->tld->segments.reclaim_count * 2 > heap->tld->segments.count) return false; + // (but not for out-of-arena segments as that is the main way to be reclaimed for those) + if (segment->memid.memkind == MI_MEM_ARENA && heap->tld->segments.reclaim_count * 2 > heap->tld->segments.count) { + return false; + } if (_mi_arena_segment_clear_abandoned(segment)) { // atomically unabandon mi_segment_t* res = mi_segment_reclaim(segment, heap, 0, NULL, &heap->tld->segments); mi_assert_internal(res == segment); @@ -1264,17 +961,19 @@ bool _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment) { void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld) { mi_segment_t* segment; - mi_arena_field_cursor_t current; _mi_arena_field_cursor_init(heap, ¤t); + mi_arena_field_cursor_t current; + _mi_arena_field_cursor_init(heap, tld->subproc, true /* visit all, blocking */, ¤t); while ((segment = _mi_arena_segment_clear_abandoned_next(¤t)) != NULL) { mi_segment_reclaim(segment, heap, 0, NULL, tld); } + _mi_arena_field_cursor_done(¤t); } -static long mi_segment_get_reclaim_tries(void) { +static long mi_segment_get_reclaim_tries(mi_segments_tld_t* tld) { // limit the tries to 10% (default) of the abandoned segments with at least 8 and at most 1024 tries. const size_t perc = (size_t)mi_option_get_clamp(mi_option_max_segment_reclaim, 0, 100); if (perc <= 0) return 0; - const size_t total_count = _mi_arena_segment_abandoned_count(); + const size_t total_count = mi_atomic_load_relaxed(&tld->subproc->abandoned_count); if (total_count == 0) return 0; const size_t relative_count = (total_count > 10000 ? (total_count / 100) * perc : (total_count * perc) / 100); // avoid overflow long max_tries = (long)(relative_count <= 1 ? 1 : (relative_count > 1024 ? 1024 : relative_count)); @@ -1282,23 +981,27 @@ static long mi_segment_get_reclaim_tries(void) { return max_tries; } -static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slices, size_t block_size, bool* reclaimed, mi_segments_tld_t* tld) +static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t block_size, mi_page_kind_t page_kind, bool* reclaimed, mi_segments_tld_t* tld) { *reclaimed = false; - long max_tries = mi_segment_get_reclaim_tries(); + long max_tries = mi_segment_get_reclaim_tries(tld); if (max_tries <= 0) return NULL; - mi_segment_t* segment; - mi_arena_field_cursor_t current; _mi_arena_field_cursor_init(heap, ¤t); + mi_segment_t* result = NULL; + mi_segment_t* segment = NULL; + mi_arena_field_cursor_t current; + _mi_arena_field_cursor_init(heap, tld->subproc, false /* non-blocking */, ¤t); while ((max_tries-- > 0) && ((segment = _mi_arena_segment_clear_abandoned_next(¤t)) != NULL)) { + mi_assert(segment->subproc == heap->tld->segments.subproc); // cursor only visits segments in our sub-process segment->abandoned_visits++; // todo: should we respect numa affinity for abondoned reclaim? perhaps only for the first visit? // todo: an arena exclusive heap will potentially visit many abandoned unsuitable segments and use many tries // Perhaps we can skip non-suitable ones in a better way? bool is_suitable = _mi_heap_memid_is_suitable(heap, segment->memid); - bool has_page = mi_segment_check_free(segment,needed_slices,block_size,tld); // try to free up pages (due to concurrent frees) - if (segment->used == 0) { + bool all_pages_free; + bool has_page = mi_segment_check_free(segment,block_size,&all_pages_free); // try to free up pages (due to concurrent frees) + if (all_pages_free) { // free the segment (by forced reclaim) to make it available to other threads. // note1: we prefer to free a segment as that might lead to reclaiming another // segment that is still partially used. @@ -1306,135 +1009,159 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice // freeing but that would violate some invariants temporarily) mi_segment_reclaim(segment, heap, 0, NULL, tld); } - else if (has_page && is_suitable) { - // found a large enough free span, or a page of the right block_size with free space + else if (has_page && segment->page_kind == page_kind && is_suitable) { + // found a free page of the right kind, or page of the right block_size with free space // we return the result of reclaim (which is usually `segment`) as it might free // the segment due to concurrent frees (in which case `NULL` is returned). - return mi_segment_reclaim(segment, heap, block_size, reclaimed, tld); + result = mi_segment_reclaim(segment, heap, block_size, reclaimed, tld); + break; } - else if (segment->abandoned_visits > 3 && is_suitable) { - // always reclaim on 3rd visit to limit the abandoned queue length. + else if (segment->abandoned_visits >= 3 && is_suitable) { + // always reclaim on 3rd visit to limit the list length. mi_segment_reclaim(segment, heap, 0, NULL, tld); } else { - // otherwise, push on the visited list so it gets not looked at too quickly again - mi_segment_try_purge(segment, false /* true force? */, tld->stats); // force purge if needed as we may not visit soon again + // otherwise, mark it back as abandoned + // todo: reset delayed pages in the segment? _mi_arena_segment_mark_abandoned(segment); } } - return NULL; + _mi_arena_field_cursor_done(¤t); + return result; } -void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld) -{ - mi_segment_t* segment; - mi_arena_field_cursor_t current; _mi_arena_field_cursor_init(heap, ¤t); - long max_tries = (force ? (long)_mi_arena_segment_abandoned_count() : 1024); // limit latency - while ((max_tries-- > 0) && ((segment = _mi_arena_segment_clear_abandoned_next(¤t)) != NULL)) { - mi_segment_check_free(segment,0,0,tld); // try to free up pages (due to concurrent frees) - if (segment->used == 0) { - // free the segment (by forced reclaim) to make it available to other threads. - // note: we could in principle optimize this by skipping reclaim and directly - // freeing but that would violate some invariants temporarily) - mi_segment_reclaim(segment, heap, 0, NULL, tld); - } - else { - // otherwise, purge if needed and push on the visited list - // note: forced purge can be expensive if many threads are destroyed/created as in mstress. - mi_segment_try_purge(segment, force, tld->stats); - _mi_arena_segment_mark_abandoned(segment); - } - } -} - /* ----------------------------------------------------------- Reclaim or allocate ----------------------------------------------------------- */ -static mi_segment_t* mi_segment_reclaim_or_alloc(mi_heap_t* heap, size_t needed_slices, size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) +static mi_segment_t* mi_segment_reclaim_or_alloc(mi_heap_t* heap, size_t block_size, mi_page_kind_t page_kind, size_t page_shift, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { + mi_assert_internal(page_kind <= MI_PAGE_LARGE); mi_assert_internal(block_size <= MI_LARGE_OBJ_SIZE_MAX); // 1. try to reclaim an abandoned segment bool reclaimed; - mi_segment_t* segment = mi_segment_try_reclaim(heap, needed_slices, block_size, &reclaimed, tld); + mi_segment_t* segment = mi_segment_try_reclaim(heap, block_size, page_kind, &reclaimed, tld); + mi_assert_internal(segment == NULL || _mi_arena_memid_is_suitable(segment->memid, heap->arena_id)); if (reclaimed) { // reclaimed the right page right into the heap - mi_assert_internal(segment != NULL); + mi_assert_internal(segment != NULL && segment->page_kind == page_kind && page_kind <= MI_PAGE_LARGE); return NULL; // pretend out-of-memory as the page will be in the page queue of the heap with available blocks } else if (segment != NULL) { - // reclaimed a segment with a large enough empty span in it + // reclaimed a segment with empty pages (of `page_kind`) in it return segment; } // 2. otherwise allocate a fresh segment - return mi_segment_alloc(0, 0, heap->arena_id, tld, os_tld, NULL); + return mi_segment_alloc(0, page_kind, page_shift, 0, heap->arena_id, tld, os_tld); } /* ----------------------------------------------------------- - Page allocation + Small page allocation ----------------------------------------------------------- */ -static mi_page_t* mi_segments_page_alloc(mi_heap_t* heap, mi_page_kind_t page_kind, size_t required, size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) -{ - mi_assert_internal(required <= MI_LARGE_OBJ_SIZE_MAX && page_kind <= MI_PAGE_LARGE); - - // find a free page - size_t page_size = _mi_align_up(required, (required > MI_MEDIUM_PAGE_SIZE ? MI_MEDIUM_PAGE_SIZE : MI_SEGMENT_SLICE_SIZE)); - size_t slices_needed = page_size / MI_SEGMENT_SLICE_SIZE; - mi_assert_internal(slices_needed * MI_SEGMENT_SLICE_SIZE == page_size); - mi_page_t* page = mi_segments_page_find_and_allocate(slices_needed, heap->arena_id, tld); //(required <= MI_SMALL_SIZE_MAX ? 0 : slices_needed), tld); - if (page==NULL) { - // no free page, allocate a new segment and try again - if (mi_segment_reclaim_or_alloc(heap, slices_needed, block_size, tld, os_tld) == NULL) { - // OOM or reclaimed a good page in the heap - return NULL; +static mi_page_t* mi_segment_find_free(mi_segment_t* segment, mi_segments_tld_t* tld) { + mi_assert_internal(mi_segment_has_free(segment)); + mi_assert_expensive(mi_segment_is_valid(segment, tld)); + for (size_t i = 0; i < segment->capacity; i++) { // TODO: use a bitmap instead of search? + mi_page_t* page = &segment->pages[i]; + if (!page->segment_in_use) { + bool ok = mi_segment_page_claim(segment, page, tld); + if (ok) return page; } - else { - // otherwise try again - return mi_segments_page_alloc(heap, page_kind, required, block_size, tld, os_tld); + } + mi_assert(false); + return NULL; +} + +// Allocate a page inside a segment. Requires that the page has free pages +static mi_page_t* mi_segment_page_alloc_in(mi_segment_t* segment, mi_segments_tld_t* tld) { + mi_assert_internal(mi_segment_has_free(segment)); + return mi_segment_find_free(segment, tld); +} + +static mi_page_t* mi_segment_page_try_alloc_in_queue(mi_heap_t* heap, mi_page_kind_t kind, mi_segments_tld_t* tld) { + // find an available segment the segment free queue + mi_segment_queue_t* const free_queue = mi_segment_free_queue_of_kind(kind, tld); + for (mi_segment_t* segment = free_queue->first; segment != NULL; segment = segment->next) { + if (_mi_arena_memid_is_suitable(segment->memid, heap->arena_id) && mi_segment_has_free(segment)) { + return mi_segment_page_alloc_in(segment, tld); } } - mi_assert_internal(page != NULL && page->slice_count*MI_SEGMENT_SLICE_SIZE == page_size); - mi_assert_internal(_mi_ptr_segment(page)->thread_id == _mi_thread_id()); - mi_segment_try_purge(_mi_ptr_segment(page), false, tld->stats); + return NULL; +} + +static mi_page_t* mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, mi_page_kind_t kind, size_t page_shift, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { + mi_page_t* page = mi_segment_page_try_alloc_in_queue(heap, kind, tld); + if (page == NULL) { + // possibly allocate or reclaim a fresh segment + mi_segment_t* const segment = mi_segment_reclaim_or_alloc(heap, block_size, kind, page_shift, tld, os_tld); + if (segment == NULL) return NULL; // return NULL if out-of-memory (or reclaimed) + mi_assert_internal(segment->page_kind==kind); + mi_assert_internal(segment->used < segment->capacity); + mi_assert_internal(_mi_arena_memid_is_suitable(segment->memid, heap->arena_id)); + page = mi_segment_page_try_alloc_in_queue(heap, kind, tld); // this should now succeed + } + mi_assert_internal(page != NULL); + #if MI_DEBUG>=2 && !MI_TRACK_ENABLED // && !MI_TSAN + // verify it is committed + mi_segment_raw_page_start(_mi_page_segment(page), page, NULL)[0] = 0; + #endif return page; } +static mi_page_t* mi_segment_small_page_alloc(mi_heap_t* heap, size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { + return mi_segment_page_alloc(heap, block_size, MI_PAGE_SMALL,MI_SMALL_PAGE_SHIFT,tld,os_tld); +} +static mi_page_t* mi_segment_medium_page_alloc(mi_heap_t* heap, size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { + return mi_segment_page_alloc(heap, block_size, MI_PAGE_MEDIUM, MI_MEDIUM_PAGE_SHIFT, tld, os_tld); +} /* ----------------------------------------------------------- - Huge page allocation + large page allocation ----------------------------------------------------------- */ +static mi_page_t* mi_segment_large_page_alloc(mi_heap_t* heap, size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { + mi_segment_t* segment = mi_segment_reclaim_or_alloc(heap,block_size,MI_PAGE_LARGE,MI_LARGE_PAGE_SHIFT,tld,os_tld); + if (segment == NULL) return NULL; + mi_page_t* page = mi_segment_find_free(segment, tld); + mi_assert_internal(page != NULL); +#if MI_DEBUG>=2 && !MI_TRACK_ENABLED // && !MI_TSAN + mi_segment_raw_page_start(segment, page, NULL)[0] = 0; +#endif + return page; +} + static mi_page_t* mi_segment_huge_page_alloc(size_t size, size_t page_alignment, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { - mi_page_t* page = NULL; - mi_segment_t* segment = mi_segment_alloc(size,page_alignment,req_arena_id,tld,os_tld,&page); - if (segment == NULL || page==NULL) return NULL; - mi_assert_internal(segment->used==1); - mi_assert_internal(mi_page_block_size(page) >= size); + mi_segment_t* segment = mi_segment_alloc(size, MI_PAGE_HUGE, MI_SEGMENT_SHIFT + 1, page_alignment, req_arena_id, tld, os_tld); + if (segment == NULL) return NULL; + mi_assert_internal(mi_segment_page_size(segment) - segment->segment_info_size - (2*(MI_SECURE == 0 ? 0 : _mi_os_page_size())) >= size); #if MI_HUGE_PAGE_ABANDON - segment->thread_id = 0; // huge segments are immediately abandoned + segment->thread_id = 0; // huge pages are immediately abandoned + mi_segments_track_size(-(long)segment->segment_size, tld); #endif + mi_page_t* page = mi_segment_find_free(segment, tld); + mi_assert_internal(page != NULL); + mi_assert_internal(page->is_huge); // for huge pages we initialize the block_size as we may // overallocate to accommodate large alignments. size_t psize; - uint8_t* start = _mi_segment_page_start(segment, page, &psize); + uint8_t* start = mi_segment_raw_page_start(segment, page, &psize); page->block_size = psize; - mi_assert_internal(page->is_huge); - // decommit the part of the prefix of a page that will not be used; this can be quite large (close to MI_SEGMENT_SIZE) - if (page_alignment > 0 && segment->allow_decommit) { + // reset the part of the page that will not be used; this can be quite large (close to MI_SEGMENT_SIZE) + if (page_alignment > 0 && segment->allow_decommit && page->is_committed) { uint8_t* aligned_p = (uint8_t*)_mi_align_up((uintptr_t)start, page_alignment); mi_assert_internal(_mi_is_aligned(aligned_p, page_alignment)); mi_assert_internal(psize - (aligned_p - start) >= size); - uint8_t* decommit_start = start + sizeof(mi_block_t); // for the free list + uint8_t* decommit_start = start + sizeof(mi_block_t); // for the free list ptrdiff_t decommit_size = aligned_p - decommit_start; - _mi_os_reset(decommit_start, decommit_size, &_mi_stats_main); // note: cannot use segment_decommit on huge segments + _mi_os_reset(decommit_start, decommit_size, os_tld->stats); // do not decommit as it may be in a region } return page; @@ -1444,7 +1171,7 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, size_t page_alignment, // free huge block from another thread void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block) { // huge page segments are always abandoned and can be freed immediately by any thread - mi_assert_internal(segment->kind==MI_SEGMENT_HUGE); + mi_assert_internal(segment->page_kind==MI_PAGE_HUGE); mi_assert_internal(segment == _mi_page_segment(page)); mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id)==0); @@ -1459,6 +1186,7 @@ void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block page->is_zero_init = false; mi_assert(page->used == 0); mi_tld_t* tld = heap->tld; + mi_segments_track_size((long)segment->segment_size, &tld->segments); _mi_segment_page_free(page, true, &tld->segments); } #if (MI_DEBUG!=0) @@ -1471,48 +1199,79 @@ void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block #else // reset memory of a huge block from another thread void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_block_t* block) { - MI_UNUSED(page); - mi_assert_internal(segment->kind == MI_SEGMENT_HUGE); + mi_assert_internal(segment->page_kind == MI_PAGE_HUGE); mi_assert_internal(segment == _mi_page_segment(page)); mi_assert_internal(page->used == 1); // this is called just before the free mi_assert_internal(page->free == NULL); - if (segment->allow_decommit) { - size_t csize = mi_usable_size(block); - if (csize > sizeof(mi_block_t)) { - csize = csize - sizeof(mi_block_t); + if (segment->allow_decommit && page->is_committed) { + size_t usize = mi_usable_size(block); + if (usize > sizeof(mi_block_t)) { + usize = usize - sizeof(mi_block_t); uint8_t* p = (uint8_t*)block + sizeof(mi_block_t); - _mi_os_reset(p, csize, &_mi_stats_main); // note: cannot use segment_decommit on huge segments + _mi_os_reset(p, usize, &_mi_stats_main); } } } #endif /* ----------------------------------------------------------- - Page allocation and free + Page allocation ----------------------------------------------------------- */ + mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { mi_page_t* page; if mi_unlikely(page_alignment > MI_BLOCK_ALIGNMENT_MAX) { mi_assert_internal(_mi_is_power_of_two(page_alignment)); mi_assert_internal(page_alignment >= MI_SEGMENT_SIZE); + //mi_assert_internal((MI_SEGMENT_SIZE % page_alignment) == 0); if (page_alignment < MI_SEGMENT_SIZE) { page_alignment = MI_SEGMENT_SIZE; } - page = mi_segment_huge_page_alloc(block_size,page_alignment,heap->arena_id,tld,os_tld); + page = mi_segment_huge_page_alloc(block_size, page_alignment, heap->arena_id, tld, os_tld); } else if (block_size <= MI_SMALL_OBJ_SIZE_MAX) { - page = mi_segments_page_alloc(heap,MI_PAGE_SMALL,block_size,block_size,tld,os_tld); + page = mi_segment_small_page_alloc(heap, block_size, tld, os_tld); } else if (block_size <= MI_MEDIUM_OBJ_SIZE_MAX) { - page = mi_segments_page_alloc(heap,MI_PAGE_MEDIUM,MI_MEDIUM_PAGE_SIZE,block_size,tld, os_tld); + page = mi_segment_medium_page_alloc(heap, block_size, tld, os_tld); } - else if (block_size <= MI_LARGE_OBJ_SIZE_MAX) { - page = mi_segments_page_alloc(heap,MI_PAGE_LARGE,block_size,block_size,tld, os_tld); + else if (block_size <= MI_LARGE_OBJ_SIZE_MAX /* || mi_is_good_fit(block_size, MI_LARGE_PAGE_SIZE - sizeof(mi_segment_t)) */ ) { + page = mi_segment_large_page_alloc(heap, block_size, tld, os_tld); } else { - page = mi_segment_huge_page_alloc(block_size,page_alignment,heap->arena_id,tld,os_tld); + page = mi_segment_huge_page_alloc(block_size, page_alignment, heap->arena_id, tld, os_tld); } - mi_assert_internal(page == NULL || _mi_heap_memid_is_suitable(heap, _mi_page_segment(page)->memid)); mi_assert_expensive(page == NULL || mi_segment_is_valid(_mi_page_segment(page),tld)); + mi_assert_internal(page == NULL || (mi_segment_page_size(_mi_page_segment(page)) - (MI_SECURE == 0 ? 0 : _mi_os_page_size())) >= block_size); + // mi_segment_try_purge(tld); + mi_assert_internal(page == NULL || mi_page_not_in_queue(page, tld)); + mi_assert_internal(page == NULL || _mi_page_segment(page)->subproc == tld->subproc); return page; } +/* ----------------------------------------------------------- + Visit blocks in a segment (only used for abandoned segments) +----------------------------------------------------------- */ + +static bool mi_segment_visit_page(mi_page_t* page, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + mi_heap_area_t area; + _mi_heap_area_init(&area, page); + if (!visitor(NULL, &area, NULL, area.block_size, arg)) return false; + if (visit_blocks) { + return _mi_heap_area_visit_blocks(&area, page, visitor, arg); + } + else { + return true; + } +} + +bool _mi_segment_visit_blocks(mi_segment_t* segment, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + for (size_t i = 0; i < segment->capacity; i++) { + mi_page_t* const page = &segment->pages[i]; + if (page->segment_in_use) { + if (heap_tag < 0 || (int)page->heap_tag == heap_tag) { + if (!mi_segment_visit_page(page, visit_blocks, visitor, arg)) return false; + } + } + } + return true; +} diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/static.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/static.c index bf025eb..9e06ce0 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/static.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/static.c @@ -31,7 +31,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "options.c" #include "os.c" #include "page.c" // includes page-queue.c -#include "random.c" +#include "random.c" #include "segment.c" #include "segment-map.c" #include "stats.c" diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/stats.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/stats.c index a936402..99cf89c 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/src/stats.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/src/stats.c @@ -106,7 +106,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { mi_stat_add(&stats->segments_cache, &src->segments_cache, 1); mi_stat_add(&stats->normal, &src->normal, 1); mi_stat_add(&stats->huge, &src->huge, 1); - mi_stat_add(&stats->large, &src->large, 1); + mi_stat_add(&stats->giant, &src->giant, 1); mi_stat_counter_add(&stats->pages_extended, &src->pages_extended, 1); mi_stat_counter_add(&stats->mmap_calls, &src->mmap_calls, 1); @@ -117,8 +117,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { mi_stat_counter_add(&stats->page_no_retire, &src->page_no_retire, 1); mi_stat_counter_add(&stats->searches, &src->searches, 1); mi_stat_counter_add(&stats->normal_count, &src->normal_count, 1); - mi_stat_counter_add(&stats->huge_count, &src->huge_count, 1); - mi_stat_counter_add(&stats->large_count, &src->large_count, 1); + mi_stat_counter_add(&stats->huge_count, &src->huge_count, 1); #if MI_STAT>1 for (size_t i = 0; i <= MI_BIN_HUGE; i++) { if (src->normal_bins[i].allocated > 0 || src->normal_bins[i].freed > 0) { @@ -314,11 +313,9 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) #endif #if MI_STAT mi_stat_print(&stats->normal, "normal", (stats->normal_count.count == 0 ? 1 : -(stats->normal.allocated / stats->normal_count.count)), out, arg); - mi_stat_print(&stats->large, "large", (stats->large_count.count == 0 ? 1 : -(stats->large.allocated / stats->large_count.count)), out, arg); - mi_stat_print(&stats->huge, "huge", (stats->huge_count.count == 0 ? 1 : -(stats->huge.allocated / stats->huge_count.count)), out, arg); + mi_stat_print(&stats->huge, "huge", (stats->huge_count.count == 0 ? 1 : -(stats->huge.allocated / stats->huge_count.count)), out, arg); mi_stat_count_t total = { 0,0,0,0 }; mi_stat_add(&total, &stats->normal, 1); - mi_stat_add(&total, &stats->large, 1); mi_stat_add(&total, &stats->huge, 1); mi_stat_print(&total, "total", 1, out, arg); #endif @@ -455,7 +452,7 @@ mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, s pinfo.page_faults = 0; _mi_prim_process_info(&pinfo); - + if (elapsed_msecs!=NULL) *elapsed_msecs = (pinfo.elapsed < 0 ? 0 : (pinfo.elapsed < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.elapsed : PTRDIFF_MAX)); if (user_msecs!=NULL) *user_msecs = (pinfo.utime < 0 ? 0 : (pinfo.utime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.utime : PTRDIFF_MAX)); if (system_msecs!=NULL) *system_msecs = (pinfo.stime < 0 ? 0 : (pinfo.stime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.stime : PTRDIFF_MAX)); diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/test/CMakeLists.txt b/lib/Utils.Memory/vnlib_mimalloc/vendor/test/CMakeLists.txt index e76ffa6..398637c 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/test/CMakeLists.txt +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/test/CMakeLists.txt @@ -16,7 +16,7 @@ if (NOT CMAKE_BUILD_TYPE) endif() # Import mimalloc (if installed) -find_package(mimalloc 2.0 REQUIRED NO_SYSTEM_ENVIRONMENT_PATH) +find_package(mimalloc 1.7 REQUIRED NO_SYSTEM_ENVIRONMENT_PATH) message(STATUS "Found mimalloc installed at: ${MIMALLOC_LIBRARY_DIR} (${MIMALLOC_VERSION_DIR})") # overriding with a dynamic library diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/test/main-override-static.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/test/main-override-static.c index e71be29..bf1cc41 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/test/main-override-static.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/test/main-override-static.c @@ -7,7 +7,6 @@ #include <mimalloc.h> #include <mimalloc-override.h> // redefines malloc etc. - static void double_free1(); static void double_free2(); static void corrupt_free(); @@ -19,8 +18,7 @@ static void test_reserved(void); static void negative_stat(void); static void alloc_huge(void); static void test_heap_walk(void); -static void test_heap_arena(void); -static void test_align(void); + int main() { mi_version(); @@ -36,10 +34,7 @@ int main() { // negative_stat(); // test_heap_walk(); // alloc_huge(); - // test_heap_walk(); - // test_heap_arena(); - // test_align(); - + void* p1 = malloc(78); void* p2 = malloc(24); free(p1); @@ -55,7 +50,7 @@ int main() { free(p1); free(p2); free(s); - + /* now test if override worked by allocating/freeing across the api's*/ //p1 = mi_malloc(32); //free(p1); @@ -70,13 +65,6 @@ int main() { return 0; } -static void test_align() { - void* p = mi_malloc_aligned(256, 256); - if (((uintptr_t)p % 256) != 0) { - fprintf(stderr, "%p is not 256 alignend!\n", p); - } -} - static void invalid_free() { free((void*)0xBADBEEF); realloc((void*)0xBADBEEF,10); @@ -228,20 +216,6 @@ static void test_heap_walk(void) { mi_heap_visit_blocks(heap, true, &test_visit, NULL); } -static void test_heap_arena(void) { - mi_arena_id_t arena_id; - int err = mi_reserve_os_memory_ex(100 * 1024 * 1024, false /* commit */, false /* allow large */, true /* exclusive */, &arena_id); - if (err) abort(); - mi_heap_t* heap = mi_heap_new_in_arena(arena_id); - for (int i = 0; i < 500000; i++) { - void* p = mi_heap_malloc(heap, 1024); - if (p == NULL) { - printf("out of memory after %d kb (expecting about 100_000kb)\n", i); - break; - } - } -} - // ---------------------------- // bin size experiments // ------------------------------ diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/test/main-override.cpp b/lib/Utils.Memory/vnlib_mimalloc/vendor/test/main-override.cpp index 582f24e..fc7f70f 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/test/main-override.cpp +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/test/main-override.cpp @@ -19,7 +19,7 @@ #endif #ifdef _WIN32 -#include <Windows.h> +#include <windows.h> static void msleep(unsigned long msecs) { Sleep(msecs); } #else #include <unistd.h> @@ -32,12 +32,9 @@ static void heap_late_free(); // issue #204 static void padding_shrink(); // issue #209 static void various_tests(); static void test_mt_shutdown(); -static void large_alloc(void); // issue #363 static void fail_aslr(); // issue #372 static void tsan_numa_test(); // issue #414 -static void strdup_test(); // issue #445 -static void bench_alloc_large(void); // issue #xxx -//static void test_large_migrate(void); // issue #691 +static void strdup_test(); // issue #445 static void heap_thread_free_huge(); static void test_std_string(); // issue #697 @@ -46,26 +43,21 @@ static void test_stl_allocators(); int main() { // mi_stats_reset(); // ignore earlier allocations - - // test_std_string(); + + test_std_string(); // heap_thread_free_huge(); /* - heap_thread_free_huge(); - heap_thread_free_large(); - heap_no_delete(); - heap_late_free(); - padding_shrink(); - various_tests(); - large_alloc(); - tsan_numa_test(); - strdup_test(); + heap_thread_free_large(); + heap_no_delete(); + heap_late_free(); + padding_shrink(); + various_tests(); + tsan_numa_test(); + strdup_test(); + test_stl_allocators(); + test_mt_shutdown(); */ - // test_stl_allocators(); - // test_mt_shutdown(); - // test_large_migrate(); - //fail_aslr(); - // bench_alloc_large(); // mi_stats_print(NULL); return 0; } @@ -185,42 +177,6 @@ static void test_stl_allocators() { #endif } -#if 0 -// issue #691 -static char* cptr; - -static void* thread1_allocate() -{ - cptr = mi_calloc_tp(char,22085632); - return NULL; -} - -static void* thread2_free() -{ - assert(cptr); - mi_free(cptr); - cptr = NULL; - return NULL; -} - -static void test_large_migrate(void) { - auto t1 = std::thread(thread1_allocate); - t1.join(); - auto t2 = std::thread(thread2_free); - t2.join(); - /* - pthread_t thread1, thread2; - - pthread_create(&thread1, NULL, &thread1_allocate, NULL); - pthread_join(thread1, NULL); - - pthread_create(&thread2, NULL, &thread2_free, NULL); - pthread_join(thread2, NULL); - */ - return; -} -#endif - // issue 445 static void strdup_test() { #ifdef _MSC_VER @@ -294,7 +250,7 @@ static void heap_thread_free_large_worker() { static void heap_thread_free_large() { for (int i = 0; i < 100; i++) { - shared_p = mi_malloc_aligned(2 * 1024 * 1024 + 1, 8); + shared_p = mi_malloc_aligned(2*1024*1024 + 1, 8); auto t1 = std::thread(heap_thread_free_large_worker); t1.join(); } @@ -305,13 +261,14 @@ static void heap_thread_free_huge_worker() { } static void heap_thread_free_huge() { - for (int i = 0; i < 100; i++) { + for (int i = 0; i < 10; i++) { shared_p = mi_malloc(1024 * 1024 * 1024); auto t1 = std::thread(heap_thread_free_huge_worker); t1.join(); } } + static void test_mt_shutdown() { const int threads = 5; @@ -336,18 +293,6 @@ static void test_mt_shutdown() std::cout << "done" << std::endl; } -// issue #363 -using namespace std; - -void large_alloc(void) -{ - char* a = new char[1ull << 25]; - thread th([&] { - delete[] a; - }); - th.join(); -} - // issue #372 static void fail_aslr() { size_t sz = (4ULL << 40); // 4TiB @@ -367,34 +312,3 @@ static void tsan_numa_test() { dummy_worker(); t1.join(); } - -// issue #? -#include <chrono> -#include <random> -#include <iostream> - -static void bench_alloc_large(void) { - static constexpr int kNumBuffers = 20; - static constexpr size_t kMinBufferSize = 5 * 1024 * 1024; - static constexpr size_t kMaxBufferSize = 25 * 1024 * 1024; - std::unique_ptr<char[]> buffers[kNumBuffers]; - - std::random_device rd; (void)rd; - std::mt19937 gen(42); //rd()); - std::uniform_int_distribution<> size_distribution(kMinBufferSize, kMaxBufferSize); - std::uniform_int_distribution<> buf_number_distribution(0, kNumBuffers - 1); - - static constexpr int kNumIterations = 2000; - const auto start = std::chrono::steady_clock::now(); - for (int i = 0; i < kNumIterations; ++i) { - int buffer_idx = buf_number_distribution(gen); - size_t new_size = size_distribution(gen); - buffers[buffer_idx] = std::make_unique<char[]>(new_size); - } - const auto end = std::chrono::steady_clock::now(); - const auto num_ms = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count(); - const auto us_per_allocation = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count() / kNumIterations; - std::cout << kNumIterations << " allocations Done in " << num_ms << "ms." << std::endl; - std::cout << "Avg " << us_per_allocation << " us per allocation" << std::endl; -} - diff --git a/lib/Utils.Memory/vnlib_mimalloc/vendor/test/test-stress.c b/lib/Utils.Memory/vnlib_mimalloc/vendor/test/test-stress.c index 15d0e25..f9b3c9d 100644 --- a/lib/Utils.Memory/vnlib_mimalloc/vendor/test/test-stress.c +++ b/lib/Utils.Memory/vnlib_mimalloc/vendor/test/test-stress.c @@ -25,20 +25,28 @@ terms of the MIT license. // > mimalloc-test-stress [THREADS] [SCALE] [ITER] // // argument defaults +#if defined(MI_TSAN) // with thread-sanitizer reduce the threads to test within the azure pipeline limits +static int THREADS = 8; +static int SCALE = 25; +static int ITER = 200; +#elif defined(MI_UBSAN) // with undefined behavious sanitizer reduce parameters to stay within the azure pipeline limits +static int THREADS = 8; +static int SCALE = 25; +static int ITER = 20; +#else static int THREADS = 32; // more repeatable if THREADS <= #processors static int SCALE = 25; // scaling factor - -#if defined(MI_TSAN) -static int ITER = 10; // N full iterations destructing and re-creating all threads (on tsan reduce for azure pipeline limits) -#else static int ITER = 50; // N full iterations destructing and re-creating all threads #endif -// static int THREADS = 8; // more repeatable if THREADS <= #processors -// static int SCALE = 100; // scaling factor + #define STRESS // undefine for leak test +#ifndef NDEBUG +#define HEAP_WALK // walk the heap objects? +#endif + static bool allow_large_objects = true; // allow very large objects? (set to `true` if SCALE>100) static size_t use_one_size = 0; // use single object size of `N * sizeof(uintptr_t)`? @@ -129,6 +137,16 @@ static void free_items(void* p) { custom_free(p); } +#ifdef HEAP_WALK +static bool visit_blocks(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg) { + (void)(heap); (void)(area); + size_t* total = (size_t*)arg; + if (block != NULL) { + *total += block_size; + } + return true; +} +#endif static void stress(intptr_t tid) { //bench_start_thread(); @@ -173,6 +191,13 @@ static void stress(intptr_t tid) { data[data_idx] = q; } } + + #ifdef HEAP_WALK + // walk the heap + size_t total = 0; + mi_heap_visit_blocks(mi_heap_get_default(), true, visit_blocks, &total); + #endif + // free everything that is left for (size_t i = 0; i < retain_top; i++) { free_items(retained[i]); @@ -190,7 +215,15 @@ static void run_os_threads(size_t nthreads, void (*entry)(intptr_t tid)); static void test_stress(void) { uintptr_t r = rand(); for (int n = 0; n < ITER; n++) { - run_os_threads(THREADS, &stress); + run_os_threads(THREADS, &stress); + #ifndef NDEBUG + // switch between arena and OS allocation for testing + mi_option_set_enabled(mi_option_disallow_arena_alloc, (n%2)==1); + #endif + #ifdef HEAP_WALK + size_t total = 0; + mi_abandoned_visit_blocks(mi_subproc_main(), -1, true, visit_blocks, &total); + #endif for (int i = 0; i < TRANSFERS; i++) { if (chance(50, &r) || n + 1 == ITER) { // free all on last run, otherwise free half of the transfers void* p = atomic_exchange_ptr(&transfer[i], NULL); @@ -200,7 +233,7 @@ static void test_stress(void) { #ifndef NDEBUG //mi_collect(false); //mi_debug_show_arenas(); - #endif + #endif #if !defined(NDEBUG) || defined(MI_TSAN) if ((n + 1) % 10 == 0) { printf("- iterations left: %3d\n", ITER - (n + 1)); } #endif @@ -230,9 +263,15 @@ static void test_leak(void) { #endif int main(int argc, char** argv) { + #ifdef HEAP_WALK + mi_option_enable(mi_option_visit_abandoned); + #endif + #ifndef NDEBUG + mi_option_set(mi_option_arena_reserve, 32 * 1024 /* in kib = 32MiB */); + #endif #ifndef USE_STD_MALLOC mi_stats_reset(); - #endif + #endif // > mimalloc-test-stress [THREADS] [SCALE] [ITER] if (argc >= 2) { @@ -262,23 +301,17 @@ int main(int argc, char** argv) { // Run ITER full iterations where half the objects in the transfer buffer survive to the next round. srand(0x7feb352d); - - //mi_reserve_os_memory(512ULL << 20, true, true); - -#if !defined(NDEBUG) && !defined(USE_STD_MALLOC) - mi_stats_reset(); -#endif - + // mi_stats_reset(); #ifdef STRESS - test_stress(); + test_stress(); #else - test_leak(); + test_leak(); #endif #ifndef USE_STD_MALLOC #ifndef NDEBUG - // mi_collect(true); mi_debug_show_arenas(true,true,true); + mi_collect(true); #endif mi_stats_print(NULL); #endif @@ -291,7 +324,7 @@ static void (*thread_entry_fun)(intptr_t) = &stress; #ifdef _WIN32 -#include <Windows.h> +#include <windows.h> static DWORD WINAPI thread_entry(LPVOID param) { thread_entry_fun((intptr_t)param); |