diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 000000000..97489cb60 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + groups: + minor-and-patch: + patterns: ["*"] + update-types: ["minor", "patch"] \ No newline at end of file diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0c994aad6..448a7ff3b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,7 +9,7 @@ permissions: contents: write env: - RELEASE: Release v3.4.5 + RELEASE: Release v3.5.2 FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true name: Release @@ -19,14 +19,14 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - branch: [v3.4.5, v2.4.5, v1.9.15] # [dev,dev2,dev3] + branch: [v3.5.2, v2.5.2, v1.15.2] # [dev,dev2,dev3] # we build on the oldest ubuntu version for better binary compatibility. os: [windows-latest, macOS-latest, macos-15-intel, ubuntu-22.04, ubuntu-22.04-arm] steps: # Checkout: https://github.com/actions/checkout - name: Checkout repository - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ matrix.branch }} fetch-tags: true @@ -45,7 +45,7 @@ jobs: # Create a release: https://github.com/softprops/action-gh-release (MIT license) - name: Release - uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 + uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 if: ${{ env.RELEASE != '' && env.RELEASE != 'no' }} with: name: ${{ env.RELEASE }} diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index ec5d51b2b..7079b90d0 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -1,7 +1,7 @@ on: workflow_dispatch: # allow running the workflow manually schedule: - - cron: "15 21 * * *" # minute, hour, day (1-31), month (1-12), day of the week (0 - 6 or SUN-SAT) + - cron: "01 21 * * *" # minute, hour, day (1-31), month (1-12), day of the week (0 - 6 or SUN-SAT) name: Close inactive issues jobs: @@ -11,7 +11,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0 + - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0 with: days-before-issue-stale: 360 days-before-issue-close: 14 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bfe0c9ed0..4bbf2db37 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -19,112 +19,154 @@ jobs: build: name: Test runs-on: ${{ matrix.os }} + env: + MIMALLOC_SHOW_ERRORS: 1 strategy: - fail-fast: false # one platform's failure should not hide the others' results + fail-fast: false matrix: tests: [basic,extra] os: [windows-latest, macos-latest, ubuntu-latest, macos-14, macos-15-intel, ubuntu-22.04-arm, windows-11-arm] + c_or_cxx: [c, cxx] exclude: - os: macos-14 tests: extra + - c_or_cxx: cxx + tests: extra + # this fork is always compiled as C++ on Windows (32-bit atomics that the MSVC C wrapper does not model) + - os: windows-latest + c_or_cxx: c + tests: basic + - os: windows-11-arm + c_or_cxx: c + tests: basic include: + - os: ubuntu-latest + tests: tsan - os: ubuntu-latest tests: freebsd-x64 # The Alpine (musl) jobs are off in this fork: setup-alpine uses an unpinned nested action # (webiny/action-post-run@3.1.0), which the oven-sh action policy rejects for the whole workflow. - os: windows-latest tests: mingw-ucrt64 + - os: windows-latest # clang-cl and clang + tests: clang + - os: windows-11-arm # clang-cl and clang + tests: clang # - os: ubuntu-latest # slow, so disable by default # tests: freebsd-arm64 # qemu steps: # Checkout: https://github.com/actions/checkout - name: Checkout repository - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # Basic - - name: Debug - if: matrix.tests == 'basic' + # Choose MI_USE_CXX setting + - name: Set default cmake options + id: cmake_args + shell: bash run: | - cmake . -B out/debug -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON - cmake --build out/debug --parallel 4 --config Debug - ctest --test-dir out/debug --verbose --timeout 240 -C Debug + if [ "${{ matrix.c_or_cxx }}" = "cxx" ]; then + echo "CMAKE_CXX_OPT_DEFAULT=-DMI_USE_CXX=ON" >> $GITHUB_OUTPUT + else # Default to C mode + echo "CMAKE_CXX_OPT_DEFAULT=-DMI_USE_CXX=OFF" >> $GITHUB_OUTPUT + fi + # Basic - name: Release if: matrix.tests == 'basic' run: | - cmake . -B out/release -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_SEE_ASM=ON - cmake --build out/release --parallel 4 --config Release - ctest --test-dir out/release --verbose --timeout 240 -C Release + cmake . -B out/release-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_SEE_ASM=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/release-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/release-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - - name: Release, asm - if: matrix.tests == 'basic' && runner.os == 'Linux' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + - name: Release, Asm + if: matrix.tests == 'basic' && runner.os == 'Linux' && matrix.c_or_cxx == 'c' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: ${{ matrix.os }}-alloc.c.s - path: out/release/CMakeFiles/mimalloc-static.dir/src/alloc*.s + name: ${{ matrix.os }}-${{ matrix.c_or_cxx }}-asm + path: out/release-${{ matrix.c_or_cxx }}/CMakeFiles/mimalloc-static.dir/src/*.s + - name: Debug + if: matrix.tests == 'basic' + run: | + env + cmake . -B out/debug-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/debug-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Secure if: matrix.tests == 'basic' run: | - cmake . -B out/secure -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON - cmake --build out/secure --parallel 4 --config Release - ctest --test-dir out/secure --verbose --timeout 240 -C Release + cmake . -B out/secure-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/secure-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/secure-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - # Basic, C++ - - name: Debug, C++ - if: matrix.tests == 'basic' && runner.os != 'Windows' # windows already uses C++ by default + # Extra Windows: x86, clang-cl, and clang + - name: Debug, Win32 + if: matrix.tests == 'basic' && runner.os == 'Windows' run: | - cmake . -B out/debug-cxx -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON - cmake --build out/debug-cxx --parallel 8 --config Debug - ctest --test-dir out/debug-cxx --verbose --timeout 240 -C Debug + cmake . -B out/debug-x86-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -A Win32 ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} + cmake --build out/debug-x86-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-x86-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - - name: Release, C++ - if: matrix.tests == 'basic' && runner.os != 'Windows' # windows already uses C++ by default + - name: Release, Win32 + if: matrix.tests == 'basic' && runner.os == 'Windows' run: | - cmake . -B out/release-cxx -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_USE_CXX=ON - cmake --build out/release-cxx --parallel 8 --config Release - ctest --test-dir out/release-cxx --verbose --timeout 240 -C Release + cmake . -B out/release-x86-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -A Win32 ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} + cmake --build out/release-x86-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/release-x86-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - # Extra, Windows x86 - - name: Debug, Win32 - if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') + - name: Debug, Win32, clang-cl + if: matrix.tests == 'clang' && runner.os == 'Windows' run: | - cmake . -B out/debug-x86 -DCMAKE_BUILD_TYPE=Debug -A Win32 - cmake --build out/debug-x86 --parallel 4 --config Debug - ctest --test-dir out/debug-x86 --verbose --timeout 240 -C Debug + cmake . -B out/debug-x86-clang-cl-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -A Win32 -T ClangCl ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} + cmake --build out/debug-x86-clang-cl-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-x86-clang-cl-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - - name: Release, Win32 - if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') + - name: Release, Win32, clang-cl + if: matrix.tests == 'clang' && runner.os == 'Windows' run: | - cmake . -B out/release-x86 -DCMAKE_BUILD_TYPE=Release -A Win32 - cmake --build out/release-x86 --parallel 4 --config Release - ctest --test-dir out/release-x86 --verbose --timeout 240 -C Release + cmake . -B out/release-x86-clang-cl-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -A Win32 -T ClangCl ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} + cmake --build out/release-x86-clang-cl-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/release-x86-clang-cl-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - - name: Debug, clang-cl - if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') + - name: Debug, Windows, clang-cl + if: matrix.tests == 'clang' && runner.os == 'Windows' run: | - cmake . -B out/debug-clang -DCMAKE_BUILD_TYPE=Debug -A Win32 -T ClangCl - cmake --build out/debug-clang --parallel 4 --config Debug - ctest --test-dir out/debug-clang --verbose --timeout 240 -C Debug + cmake . -B out/debug-clang-cl-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -T ClangCl ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} + cmake --build out/debug-clang-cl-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-clang-cl-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - - name: Release, clang-cl - if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') + - name: Release, Windows, clang-cl + if: matrix.tests == 'clang' && runner.os == 'Windows' run: | - cmake . -B out/release-clang -DCMAKE_BUILD_TYPE=Release -A Win32 -T ClangCl - cmake --build out/release-clang --parallel 4 --config Release - ctest --test-dir out/release-clang --verbose --timeout 240 -C Release + cmake . -B out/release-clang-cl-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -T ClangCl ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} + cmake --build out/release-clang-cl-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/release-clang-cl-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release + + - name: Debug, Windows, clang + if: matrix.tests == 'clang' && runner.os == 'Windows' + run: | + cmake . -B out/debug-clang-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=clang ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} + cmake --build out/debug-clang-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-clang-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug + + - name: Release, Windows, clang + if: matrix.tests == 'clang' && runner.os == 'Windows' + run: | + cmake . -B out/release-clang-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} + cmake --build out/release-clang-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/release-clang-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release # Extra, macOS fixed TLS - - name: Debug, TLS fixed + - name: Debug, macOS, TLS fixed if: matrix.tests == 'extra' && runner.os == 'macOS' run: | cmake . -B out/debug-tls-fixed -DCMAKE_BUILD_TYPE=Debug -DMI_TLS_MODEL_FIXED=ON cmake --build out/debug-tls-fixed --parallel 4 --config Debug ctest --test-dir out/debug-tls-fixed --verbose --timeout 240 -C Debug - - name: Release, TLS fixed + - name: Release, macOS, TLS fixed if: matrix.tests == 'extra' && runner.os == 'macOS' run: | cmake . -B out/release-tls-fixed -DCMAKE_BUILD_TYPE=Release -DMI_TLS_MODEL_FIXED=ON @@ -132,17 +174,17 @@ jobs: ctest --test-dir out/release-tls-fixed --verbose --timeout 240 -C Release # Extra, Linux pthread TLS - - name: Debug, TLS pthreads + - name: Debug, Linux, TLS pthreads if: matrix.tests == 'extra' && runner.os == 'linux' run: | cmake . -B out/debug-tls-pthreads -DCMAKE_BUILD_TYPE=Debug -DMI_TLS_MODEL_PTHREADS=ON cmake --build out/debug-tls-pthreads --parallel 4 --config Debug ctest --test-dir out/debug-tls-pthreads --verbose --timeout 240 -C Debug - - name: Release, TLS pthreads + - name: Release, Linux, TLS pthreads if: matrix.tests == 'extra' && runner.os == 'linux' run: | - cmake . -B out/release-tls-pthreads -DCMAKE_BUILD_TYPE=Release -DMI_TLS_MODEL_PTHREADS=ON + cmake . -B out/release-tls-pthreads -DCMAKE_BUILD_TYPE=Release -DMI_TLS_MODEL_PTHREADS=ON -DMI_SEE_ASM=ON cmake --build out/release-tls-pthreads --parallel 4 --config Release ctest --test-dir out/release-tls-pthreads --verbose --timeout 240 -C Release @@ -150,76 +192,99 @@ jobs: - name: Release, SIMD if: matrix.tests == 'extra' run: | - cmake . -B out/release-simd -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON + cmake . -B out/release-simd -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON cmake --build out/release-simd --parallel 8 --config Release ctest --test-dir out/release-simd --verbose --timeout 240 -C Release - - name: Release, C++, SIMD - if: matrix.tests == 'basic' && runner.os != 'Windows' && runner.os != 'macOS' - run: | - cmake . -B out/release-cxx -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON -DMI_USE_CXX=ON - cmake --build out/release-cxx --parallel 8 --config Release - ctest --test-dir out/release-cxx --verbose --timeout 240 -C Release - - name: Debug, C++, clang++ - if: matrix.tests == 'extra' && runner.os == 'Linux' + if: matrix.tests == 'extra' && runner.os == 'Linux' run: | - CC=clang CXX=clang++ cmake . -B out/debug-clang-cxx -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON + CC=clang CXX=clang++ cmake . -B out/debug-clang-cxx -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_USE_CXX=ON cmake --build out/debug-clang-cxx --parallel 8 --config Debug ctest --test-dir out/debug-clang-cxx --verbose --timeout 240 -C Debug - name: Release, C++, clang++ - if: matrix.tests == 'extra' && runner.os == 'Linux' + if: matrix.tests == 'extra' && runner.os == 'Linux' run: | CC=clang CXX=clang++ cmake . -B out/release-clang-cxx -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_USE_CXX=ON -DMI_SEE_ASM=ON cmake --build out/release-clang-cxx --parallel 8 --config Release ctest --test-dir out/release-clang-cxx --verbose --timeout 240 -C Release - - name: Release, C++, clang++, asm - if: matrix.tests == 'extra' && runner.os == 'Linux' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + - name: Release, C++, clang++, Asm + if: matrix.tests == 'extra' && runner.os == 'Linux' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: ${{ matrix.os }}-clang-cxx-alloc.s - path: out/release-clang-cxx/CMakeFiles/mimalloc-static.dir/src/alloc*.s + name: ${{ matrix.os }}-clang-cxx-asm + path: out/release-clang-cxx/CMakeFiles/mimalloc-static.dir/src/*.s - name: Debug, ASAN if: matrix.tests == 'extra' && runner.os == 'Linux' run: | - CC=clang CXX=clang++ cmake . -B out/debug-asan -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_TRACK_ASAN=ON + CC=clang CXX=clang++ cmake . -B out/debug-asan -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_TRACK=ASAN cmake --build out/debug-asan --parallel 8 --config Debug ctest --test-dir out/debug-asan --verbose --timeout 240 -C Debug - name: Debug, UBSAN - if: matrix.tests == 'extra' && runner.os == 'Linux' + if: matrix.tests == 'extra' && runner.os == 'Linux' run: | - CC=clang CXX=clang++ cmake . -B out/debug-ubsan -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_DEBUG_UBSAN=ON - cmake --build out/debug-ubsan --parallel 8 --config Debug - ctest --test-dir out/debug-ubsan --verbose --timeout 240 -C Debug + CC=clang CXX=clang++ cmake . -B out/debug-ubsan-cxx -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_DEBUG_UBSAN=ON + cmake --build out/debug-ubsan-cxx --parallel 8 --config Debug + ctest --test-dir out/debug-ubsan-cxx --verbose --timeout 240 -C Debug - name: Debug, TSAN - if: matrix.tests == 'extra' && runner.os == 'Linux' + if: matrix.tests == 'tsan' && runner.os == 'Linux' + env: + MI_TEST_LIGHT: 1 # the full TSAN stress does not fit the timeout on the 4 core runners run: | - CC=clang CXX=clang++ cmake . -B out/debug-tsan -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_DEBUG_TSAN=ON + CC=clang CXX=clang++ cmake . -B out/debug-tsan -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_DEBUG_TSAN=ON cmake --build out/debug-tsan --parallel 8 --config Debug ctest --test-dir out/debug-tsan --verbose --timeout 240 -C Debug - name: Debug, Guarded if: matrix.tests == 'extra' && runner.os != 'macOS' run: | - cmake . -B out/debug-guarded -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_GUARDED=ON + cmake . -B out/debug-guarded -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_GUARDED=FULL cmake --build out/debug-guarded --parallel 8 --config Debug ctest --test-dir out/debug-guarded --verbose --timeout 240 -C Debug env: - MIMALLOC_GUARDED_SAMPLE_RATE: 100 + MIMALLOC_GUARDED_SAMPLE_RATE: 1024 - name: Release, Guarded - if: matrix.tests == 'extra' && runner.os == 'Linux' && startsWith(github.ref_name,'dev3') + if: matrix.tests == 'extra' && runner.os == 'Linux' && (startsWith(github.ref_name,'dev3') || startsWith(github.ref_name,'bun') || github.event_name == 'pull_request') run: | - cmake . -B out/release-guarded -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_GUARDED=ON + cmake . -B out/release-guarded -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_GUARDED=FULL cmake --build out/release-guarded --parallel 8 --config Release ctest --test-dir out/release-guarded --verbose --timeout 240 -C Release env: - MIMALLOC_GUARDED_SAMPLE_RATE: 100 + MIMALLOC_GUARDED_SAMPLE_RATE: 128 + + - name: Debug, Use pagemap + if: matrix.tests == 'extra' && runner.os == 'Linux' && (startsWith(github.ref_name,'dev3') || startsWith(github.ref_name,'bun') || github.event_name == 'pull_request') + run: | + cmake . -B out/debug-pagemap -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_FREE_USE_PAGEMAP=ON + cmake --build out/debug-pagemap --parallel 8 --config Debug + ctest --test-dir out/debug-pagemap --verbose --timeout 240 -C Debug + + - name: Release, Use pagemap + if: matrix.tests == 'extra' && runner.os == 'Linux' && (startsWith(github.ref_name,'dev3') || startsWith(github.ref_name,'bun') || github.event_name == 'pull_request') + run: | + cmake . -B out/release-pagemap -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_FREE_USE_PAGEMAP=ON + cmake --build out/release-pagemap --parallel 8 --config Release + ctest --test-dir out/release-pagemap --verbose --timeout 240 -C Release + + - name: Debug, Secure Full + if: matrix.tests == 'extra' && runner.os != 'macOS' + run: | + cmake . -B out/debug-secure-full -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_SECURE=FULL + cmake --build out/debug-secure-full --parallel 8 --config Debug + ctest --test-dir out/debug-secure-full --verbose --timeout 240 -C Debug + + - name: Release, Secure Full + if: matrix.tests == 'extra' && runner.os != 'macOS' + run: | + cmake . -B out/release-secure-full -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_SECURE=FULL + cmake --build out/release-secure-full --parallel 8 --config Release + ctest --test-dir out/release-secure-full --verbose --timeout 240 -C Release # MSYS2 mingw64 - name: mingw-ucrt64 x64, Setup @@ -238,7 +303,7 @@ jobs: shell: msys2 {0} run: | env - cmake . -B out/debug-mingw-ucrt64-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + cmake . -B out/debug-mingw-ucrt64-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL cmake --build out/debug-mingw-ucrt64-x64 --parallel 4 --config Debug ctest --test-dir out/debug-mingw-ucrt64-x64 --verbose --timeout 240 -C Debug @@ -254,7 +319,7 @@ jobs: # VM: FreeBSD (arm64 and x64) - name: FreeBSD arm64, Setup if: matrix.tests == 'freebsd-arm64' - uses: vmactions/freebsd-vm@83b151f58c6047089f4c80eb5ba2039d158ce093 # v1 + uses: vmactions/freebsd-vm@f0552d3b69211736abd97f02ff3d4674c56b73b1 # v1.5.5 with: arch: arm64 custom-shell-name: freebsd-arm64 @@ -268,7 +333,7 @@ jobs: shell: freebsd-arm64 {0} run: | export MI_TEST_LIGHT=1 - cmake . -B out/debug-arm64 -G Ninja -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + cmake . -B out/debug-arm64 -G Ninja -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL cmake --build out/debug-arm64 --parallel 4 --config Debug ctest --test-dir out/debug-arm64 --verbose --timeout 240 -C Debug @@ -277,7 +342,7 @@ jobs: shell: freebsd-arm64 {0} run: | export MI_TEST_LIGHT=1 - cmake . -B out/release-arm64 -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON + cmake . -B out/release-arm64 -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON cmake --build out/release-arm64 --parallel 4 --config Release ctest --test-dir out/release-arm64 --verbose --timeout 240 -C Release @@ -286,14 +351,14 @@ jobs: shell: freebsd-arm64 {0} run: | export MI_TEST_LIGHT=1 - cmake . -B out/secure-arm64 -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + cmake . -B out/secure-arm64 -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON cmake --build out/secure-arm64 --parallel 4 --config Release ctest --test-dir out/secure-arm64 --verbose --timeout 240 -C Release - name: FreeBSD x64, Setup if: matrix.tests == 'freebsd-x64' - uses: vmactions/freebsd-vm@83b151f58c6047089f4c80eb5ba2039d158ce093 # v1 + uses: vmactions/freebsd-vm@f0552d3b69211736abd97f02ff3d4674c56b73b1 # v1.5.5 with: arch: x86_64 usesh: true @@ -306,7 +371,7 @@ jobs: if: matrix.tests == 'freebsd-x64' shell: freebsd-x64 {0} run: | - cmake . -B out/debug-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + cmake . -B out/debug-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL cmake --build out/debug-x64 --parallel 4 --config Debug ctest --test-dir out/debug-x64 --verbose --timeout 240 -C Debug @@ -314,7 +379,7 @@ jobs: if: matrix.tests == 'freebsd-x64' shell: freebsd-x64 {0} run: | - cmake . -B out/release-x64 -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON + cmake . -B out/release-x64 -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON cmake --build out/release-x64 --parallel 4 --config Release ctest --test-dir out/release-x64 --verbose --timeout 240 -C Release @@ -322,8 +387,6 @@ jobs: if: matrix.tests == 'freebsd-x64' shell: freebsd-x64 {0} run: | - cmake . -B out/secure-x64 -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + cmake . -B out/secure-x64 -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON cmake --build out/secure-x64 --parallel 4 --config Release ctest --test-dir out/secure-x64 --verbose --timeout 240 -C Release - - diff --git a/CMakeLists.txt b/CMakeLists.txt index b6a5c9de9..9db485cd5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,26 +4,44 @@ project(libmimalloc C) set(CMAKE_C_STANDARD 11) set(CMAKE_CXX_STANDARD 17) -option(MI_SECURE "Use security mitigations (like meta data guard pages, allocation randomization, double-free mitigation, and free-list corruption detection)" OFF) -option(MI_SECURE_FULL "Use full security mitigations including guard pages at the end of each mimalloc page (may be expensive)" OFF) +set(MI_SECURE "OFF" CACHE STRING "Use security mitigations (like meta data guard pages, allocation randomization, double-free mitigation, and free-list corruption detection): OFF, ON, FULL. The FULL setting adds guard pages at the end of each mimalloc page (which may be expensive)") +set_property(CACHE MI_SECURE PROPERTY STRINGS "OFF" "ON" "FULL") + option(MI_OVERRIDE "Override the standard malloc interface (i.e. define entry points for 'malloc', 'free', etc)" ON) option(MI_XMALLOC "Enable abort() call on memory allocation failure by default" OFF) option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) -option(MI_GUARDED "Build with guard pages behind certain object allocations (enabled by default in a debug build)" OFF) -option(MI_USE_CXX "Use the C++ compiler to compile the library (instead of the C compiler)" OFF) -option(MI_OPT_ARCH "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.1-a' (2016))" OFF) -option(MI_OPT_SIMD "Use SIMD instructions (requires MI_OPT_ARCH to be enabled)" OFF) -option(MI_FREE_IS_CHECKED "Always check for invalid pointers in 'free' (always enabled in debug and secure builds, or when MI_OVERRIDE is ON)" OFF) -option(MI_DEBUG "Enable assertion checks (enabled by default in a debug build)" OFF) -option(MI_DEBUG_INTERNAL "Enable assertion and internal invariant checks (enabled by default in a debug build)" OFF) -option(MI_DEBUG_FULL "Enable assertion checks and expensive internal heap invariant checking" OFF) +set(MI_GUARDED "DEFAULT" CACHE STRING "Build with ability to put guard pages behind certain object allocations") +set_property(CACHE MI_GUARDED PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") + +set(MI_USE_CXX "DEFAULT" CACHE STRING "Use the C++ compiler to compile the library (instead of the C compiler)") +set_property(CACHE MI_USE_CXX PROPERTY STRINGS "OFF" "ON" "DEFAULT") + +set(MI_OPT_ARCH "DEFAULT" CACHE STRING "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.1-a' (2016), for riscV: '-march=rv64gcb_zacas' (2023))") +set_property(CACHE MI_OPT_ARCH PROPERTY STRINGS "OFF" "ON" "avx2" "armv8.1" "armv8.3" "rva22" "rva23" "rv64gcb_zacas" "rv64gcb_zacas_zalasr" "DEFAULT") + +set(MI_DEBUG "DEFAULT" CACHE STRING "Enable assertion checks: OFF, ON, INTERNAL, FULL, DEFAULT. INTERNAL adds extra internal invariant checks, while FULL adds expensive heap invariant checks") +set_property(CACHE MI_DEBUG PROPERTY STRINGS "OFF" "ON" "INTERNAL" "FULL" "DEFAULT") + +set(MI_TRACK "OFF" CACHE STRING "Enable tracking: OFF, ASAN (address sanitizer), VALGRIND, ETW (Windows event tracing). Adds a small overhead.") +set_property(CACHE MI_TRACK PROPERTY STRINGS "OFF" "ASAN" "VALGRIND" "ETW") + +set(MI_STATS "DEFAULT" CACHE STRING "Enable statistics.") +set_property(CACHE MI_STATS PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") + +set(MI_PROFILE "DEFAULT" CACHE STRING "Enable profiling.") +set_property(CACHE MI_PROFILE PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") + +set(MI_FREE_IS_CHECKED "DEFAULT" CACHE STRING "Check for invalid pointers in 'mi_free': OFF, SHARED, ON, DEFAULT. 'SHARED' enables it only for the dynamic/shared library when MI_OVERRIDE is ON") +set_property(CACHE MI_FREE_IS_CHECKED PROPERTY STRINGS "OFF" "SHARED" "ON" "DEFAULT") + +set(MI_ALLOW_THP "DEFAULT" CACHE STRING "Enable transparent huge pages on Linux: OFF, ON, FULL, DEFAULT. 'FULL' enables THP and also avoids splitting up huge pages during purging.") +set_property(CACHE MI_ALLOW_THP PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") + +option(MI_OPT_SIMD "Use SIMD instructions (requires MI_OPT_ARCH to be enabled)" OFF) option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) -option(MI_TRACK_VALGRIND "Compile with Valgrind support (adds a small overhead)" OFF) -option(MI_TRACK_ASAN "Compile with address sanitizer support (adds a small overhead)" OFF) -option(MI_TRACK_ETW "Compile with Windows event tracing (ETW) support (adds a small overhead)" OFF) option(MI_BUILD_SHARED "Build shared library" ON) option(MI_BUILD_STATIC "Build static library" ON) @@ -34,40 +52,53 @@ option(MI_BUILD_TESTS "Build test executables" ON) option(MI_OSX_INTERPOSE "Use interpose to override standard malloc on macOS" ON) option(MI_OSX_ZONE "Use malloc zone to override standard malloc on macOS" ON) option(MI_WIN_REDIRECT "Use redirection module ('mimalloc-redirect') on Windows if compiling mimalloc as a DLL" ON) -option(MI_WIN_DIRECT_TLS "Use only direct TLS slots on Windows to avoid extra tests in the malloc fast path (only works if the program uses less than 64 TlsAlloc'd slots in total)" OFF) -option(MI_LOCAL_DYNAMIC_TLS "Use local-dynamic-tls, a slightly slower but dlopen-compatible thread local storage mechanism (Unix)" OFF) option(MI_LIBC_MUSL "Enable this when linking with musl libc" OFF) # advanced -option(MI_OPT_FREE_SMALL "Use optimized `mi_free_small` without pagemap lookup (can only be used if SECURE or GUARDED mode are off)" OFF) +option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF) +option(MI_NO_THP "Opt the memory mimalloc maps out of transparent huge pages on Linux/Android (sets MI_ALLOW_THP=OFF, i.e. MI_DEFAULT_ALLOW_THP=0)" OFF) +option(MI_FREE_USE_PAGEMAP "Always use the page map when free-ing a pointer -- enable only for systems that don't support large OS aligned allocations well." OFF) option(MI_PADDING "Enable padding to detect heap block overflow (always on in DEBUG or SECURE mode, or with Valgrind/ASAN)" OFF) option(MI_NO_PROCESS_DETACH "Skip automatic mi_process_done at process exit (embedder calls it manually or not at all)" OFF) option(MI_NO_PADDING "Force no use of padding even in DEBUG mode etc." OFF) option(MI_SKIP_COLLECT_ON_EXIT "Skip collecting memory on program exit" OFF) -option(MI_NO_THP "Opt the memory mimalloc maps out of transparent huge pages on Linux/Android (sets MI_DEFAULT_ALLOW_THP=0)" OFF) option(MI_EXTRA_CPPDEFS "Extra pre-processor definitions (use as `-DMI_EXTRA_CPPDEFS=\"opt1=val1;opt2=val2\"`)" "") option(MI_SEE_ASM "Generate assembly files" OFF) -option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF) # negated options for vcpkg features option(MI_NO_USE_CXX "Use plain C compilation (has priority over MI_USE_CXX)" OFF) -option(MI_NO_OPT_ARCH "Do not use architecture specific optimizations (like '-march=armv8.1-a' for example) (has priority over MI_OPT_ARCH)" OFF) +option(MI_NO_OPT_ARCH "Do not use architecture specific optimizations (like '-march=armv8.3-a' for example) (has priority over MI_OPT_ARCH)" OFF) option(MI_NO_DEBUG "Disable all assertion checks even in debug builds" OFF) -option(MI_NO_FREE_IS_CHECKED "Always disable checking for invalid pointers in release builds" OFF) # experimental -option(MI_WIN_INIT_USE_RAW_DLLMAIN "Use the raw DLL main entry point for mimalloc initialization; can be more robust but can also lead to link errors with other libraries" OFF) -option(MI_WIN_INIT_USE_TLS_DLLMAIN "Use legacy TLS entries with DllMain for mimalloc initialization" OFF) +option(MI_TLS_RECURSE_GUARD "Force a recursion check on thread local access (for MI_TLS_MODEL_LOCAL only)" OFF) + +set(MI_WIN_INIT "DEFAULT" CACHE STRING "Force Windows intialization strategy: CRT_TLS (default), RAW_DLLMAIN (use raw DLL main entry point), TLS_DLLMAIN (TLS entries with DllMain), FLS (legacy fiber API), DEFAULT") +set_property(CACHE MI_WIN_INIT PROPERTY STRINGS "CRT_TLS" "RAW_DLLMAIN" "TLS_DLLMAIN" "FLS" "DEFAULT") -option(MI_TLS_MODEL_LOCAL "Force TLS model based on thread local variables" OFF) -option(MI_TLS_MODEL_PTHREADS "Force TLS model based on pthreads" OFF) -option(MI_TLS_MODEL_FIXED "Force TLS model based on fixed thread local slots" OFF) -option(MI_TLS_RECURSE_GUARD "Force a recursion check on thread local access (for MI_TLS_MODEL_LOCAL only)" OFF) +set(MI_TLS_MODEL "DEFAULT" CACHE STRING "Force TLS model: LOCAL (thread local variables), PTHREADS (pthread locals), LOCAL_DYNAMIC (dlopen-compatible thread local storage mechanism (Unix)), WIN32 (direct Windows TLS slots), FIXED (fixed thread local slots), DEFAULT.") +set_property(CACHE MI_TLS_MODEL PROPERTY STRINGS "LOCAL" "PTHREADS" "LOCAL_DYNAMIC" "WIN32" "FIXED" "DEFAULT") # deprecated options -option(MI_WIN_USE_FLS "Use Fiber local storage on Windows to detect thread termination (deprecated)" OFF) -option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) -option(MI_USE_LIBATOMIC "Explicitly link with -latomic (on older systems) (deprecated and detected automatically)" OFF) +option(MI_USE_LIBATOMIC "Deprecated: Explicitly link with -latomic (on older systems) (deprecated and detected automatically)" OFF) +option(MI_DEBUG_INTERNAL "Deprecated: Enable assertion and internal invariant checks (use MI_DEBUG=INTERNAL instead)" OFF) +option(MI_DEBUG_FULL "Deprecated: Enable assertion checks and expensive internal heap invariant checking (use MI_DEBUG=FULL instead)" OFF) +option(MI_CHECK_FULL "Deprecated: Use full internal invariant checking in DEBUG mode (use MI_DEBUG=FULL instead)" OFF) +option(MI_SECURE_FULL "Deprecated: Use full security mitigations including guard pages at the end of each mimalloc page (may be expensive) (use MI_SECURE=FULL instead)" OFF) +option(MI_TLS_MODEL_LOCAL "Deprecated: Force TLS model based on thread local variables (use MI_TLS_MODEL=LOCAL instead)" OFF) +option(MI_TLS_MODEL_PTHREADS "Deprecated: Force TLS model based on pthreads (use MI_TLS_MODEL=PTHREADS instead)" OFF) +option(MI_TLS_MODEL_FIXED "Deprecated: Force TLS model based on fixed thread local slots (use MI_TLS_MODEL=FIXED instead)" OFF) +option(MI_LOCAL_DYNAMIC_TLS "Deprecated: Use local-dynamic-tls, a slightly slower but dlopen-compatible thread local storage mechanism (Unix) (use MI_TLS_MODEL=LOCAL_DYNAMIC instead)" OFF) +option(MI_WIN_USE_FLS "Deprecated: Use Fiber local storage on Windows to detect thread termination (use MI_WIN_INIT=FLS instead)" OFF) +option(MI_WIN_INIT_USE_RAW_DLLMAIN "Deprecated: Use the raw DLL main entry point for mimalloc initialization; can be more robust but can also lead to link errors with other libraries (use MI_WIN_INIT=RAW_DLLMAIN instead)" OFF) +option(MI_WIN_INIT_USE_TLS_DLLMAIN "Deprecated: Use legacy TLS entries with DllMain for mimalloc initialization (use MI_WIN_INIT=TLS_DLLMAIN instead)" OFF) +option(MI_WIN_DIRECT_TLS "Deprecated: Use only direct TLS slots on Windows to avoid extra tests in the malloc fast path (only works if the program uses less than 64 TlsAlloc'd slots in total)" OFF) +option(MI_TRACK_VALGRIND "Deprecated: Compile with Valgrind support (adds a small overhead) (use MI_TRACK=VALGRIND instead)" OFF) +option(MI_TRACK_ASAN "Deprecated: Compile with address sanitizer support (adds a small overhead) (use MI_TRACK=ASAN instead)" OFF) +option(MI_TRACK_ETW "Deprecated: Compile with Windows event tracing (ETW) support (adds a small overhead) (use MI_TRACK=ETW instead)" OFF) + +set(MI_OPT_FREE_SMALL "DEFAULT" CACHE STRING "Deprecated: Use aligned `mi_free_small` without pagemap lookup (can only be used if SECURE or GUARDED mode are off)") +set_property(CACHE MI_OPT_FREE_SMALL PROPERTY STRINGS "OFF" "ON" "DEFAULT") include(CheckLinkerFlag) # requires cmake 3.18 include(CheckIncludeFiles) @@ -82,21 +113,22 @@ set(mi_sources src/bitmap.c src/heap.c src/heap-snapshot.c - src/prof.c src/init.c src/libc.c src/options.c src/os.c src/page.c - src/page-map.c + src/page-map.c + src/prim/prim.c + src/prim/prim-tls.c src/random.c + src/sample-guarded.c + src/sample-profile.c src/scavenger.c src/stats.c src/subproc.c src/theap.c - src/threadlocal.c - src/prim/prim.c - src/prim/prim-tls.c) + src/threadlocal.c) set(mi_cflags "") set(mi_cflags_static "") # extra flags for a static library build @@ -122,12 +154,23 @@ if(EXISTS "${CMAKE_SOURCE_DIR}/.git/index") endif() endif() +# case insensitive comparison +function(STRIEQUAL str1 str2 is_equal) + string(TOUPPER "${str1}" str1_upper) + string(TOUPPER "${str2}" str2_upper) + if(str1_upper STREQUAL str2_upper) + set(${is_equal} TRUE PARENT_SCOPE) + else() + set(${is_equal} FALSE PARENT_SCOPE) + endif() +endfunction() + # ----------------------------------------------------------------------------- # Convenience: set default build type and compiler depending on the build directory # ----------------------------------------------------------------------------- message(STATUS "") -if(NOT CMAKE_BUILD_TYPE) +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) if ("${CMAKE_BINARY_DIR}" MATCHES ".*((D|d)ebug|asan|tsan|ubsan|valgrind)$") message(STATUS "No build type selected, default to 'Debug'") set(CMAKE_BUILD_TYPE "Debug") @@ -149,7 +192,18 @@ endif() # Determine architecture -set(MI_OPT_ARCH_FLAGS "") +if(CMAKE_SIZEOF_VOID_P EQUAL 16) + set(MI_ARCH_BITS "128") +elseif(CMAKE_SIZEOF_VOID_P EQUAL 8) + set(MI_ARCH_BITS "64") +elseif(CMAKE_SIZEOF_VOID_P EQUAL 4) + set(MI_ARCH_BITS "32") +elseif(CMAKE_SIZEOF_VOID_P EQUAL 2) + set(MI_ARCH_BITS "16") +else() + set(MI_ARCH_BITS "") +endif() + set(MI_ARCH "unknown") if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86|i[3456]86)$" OR CMAKE_GENERATOR_PLATFORM MATCHES "^(x86|Win32)$") set(MI_ARCH "x86") @@ -160,15 +214,12 @@ elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|armv[89].?|ARM64)$" OR CM elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm|armv[34567].?|ARM)$") set(MI_ARCH "arm32") elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(riscv|riscv32|riscv64)$") - if(CMAKE_SIZEOF_VOID_P==4) - set(MI_ARCH "riscv32") - else() - set(MI_ARCH "riscv64") - endif() + set(MI_ARCH "riscv${MI_ARCH_BITS}") else() set(MI_ARCH ${CMAKE_SYSTEM_PROCESSOR}) endif() -message(STATUS "Architecture: ${MI_ARCH}") # (${CMAKE_SYSTEM_PROCESSOR}, ${CMAKE_GENERATOR_PLATFORM}, ${CMAKE_GENERATOR})") + +message(STATUS "Architecture: ${MI_ARCH} (${MI_ARCH_BITS} bit)") # (${CMAKE_SYSTEM_PROCESSOR}, ${CMAKE_GENERATOR_PLATFORM}, ${CMAKE_GENERATOR})") # negative overrides (mainly to support vcpkg features) if(MI_NO_USE_CXX) @@ -177,8 +228,14 @@ endif() if(MI_NO_OPT_ARCH) set(MI_OPT_ARCH "OFF") -elseif(MI_ARCH STREQUAL "arm64") - set(MI_OPT_ARCH "ON") # enable armv8.1-a by default on arm64 unless MI_NO_OPT_ARCH is set +endif() + +if(MI_OPT_ARCH STREQUAL "DEFAULT") + if(MI_ARCH STREQUAL "arm64") + set(MI_OPT_ARCH "ON") # enable by default for arm64 (defaults to armv8.1-a which has fast atomics) + else() + set(MI_OPT_ARCH "OFF") + endif() endif() if(DEFINED ENV{MI_TEST_LIGHT}) @@ -194,10 +251,22 @@ if(CMAKE_C_COMPILER_ID STREQUAL "Clang" AND CMAKE_C_COMPILER_FRONTEND_VARIANT ST set(MI_CLANG_CL "ON") endif() -# force C++ compilation with msvc or clang-cl to use modern C++ atomics -if(CMAKE_C_COMPILER_ID MATCHES "MSVC|Intel" OR MI_CLANG_CL) - set(MI_USE_CXX "ON") -elseif(MI_DEBUG_UBSAN AND CMAKE_BUILD_TYPE MATCHES "Debug") # ubsan needs C++ +if(MI_USE_CXX STREQUAL "DEFAULT") + if(CMAKE_C_COMPILER_ID MATCHES "MSVC|Intel" OR MI_CLANG_CL) + # force C++ compilation with msvc or clang-cl to use modern C++ atomics + set(MI_USE_CXX "ON") + elseif(MI_DEBUG_UBSAN AND (CMAKE_BUILD_TYPE MATCHES "Debug" OR NOT CMAKE_CONFIGURATION_TYPES)) # ubsan needs C++ + set(MI_USE_CXX "ON") + else() + set(MI_USE_CXX "OFF") + endif() +endif() + +# The plain C wrapper for the MSVC atomics (`include/mimalloc/atomic.h`) only models word sized atomics, and this +# fork also uses 32-bit ones (the park and scavenger state). That wrapper is selected whenever `_MSC_VER` is defined +# in a C compilation: cl, clang-cl, and also plain clang when it targets the MSVC ABI. +if(NOT MI_USE_CXX AND WIN32 AND NOT (MINGW OR CYGWIN OR MSYS)) + message(WARNING "Cannot compile as C code with an MSVC compatible compiler (the C atomics wrapper is word-size only); compiling as C++ instead (MI_USE_CXX=ON)") set(MI_USE_CXX "ON") endif() @@ -207,6 +276,8 @@ if(MI_USE_CXX) message(STATUS "Use the C++ compiler to compile (MI_USE_CXX=ON)") if(CMAKE_CXX_COMPILER_ID MATCHES "Intel" AND NOT CMAKE_CXX_COMPILER_ID MATCHES "IntelLLVM") list(APPEND mi_cflags -Kc++) + elseif(MSVC AND MSVC_VERSION GREATER_EQUAL 1914) + list(APPEND mi_cflags /Zc:__cplusplus) endif() endif() @@ -268,63 +339,65 @@ if(WIN32) endif() if(MI_SECURE_FULL) - set(MI_SECURE ON) - message(STATUS "Set full secure build (MI_SECURE_FULL=ON)") + message(STATUS "MI_SECURE_FULL is deprecated, use MI_SECURE=FULL instead") + set(MI_SECURE "FULL") +endif() +if(MI_SECURE STREQUAL "FULL") + message(STATUS "Use full secure build (MI_SECURE=FULL, may be expensive)") list(APPEND mi_defines MI_SECURE=5) elseif(MI_SECURE) - message(STATUS "Set secure build (MI_SECURE=ON)") + message(STATUS "Use secure build (MI_SECURE=${MI_SECURE})") list(APPEND mi_defines MI_SECURE=4) endif() if(MI_TRACK_VALGRIND) + message(STATUS "Use of MI_TRACK_VALGRIND is deprecated, use MI_TRACK=VALGRIND instead") + set(MI_TRACK "VALGRIND") +elseif(MI_TRACK_ASAN) + message(STATUS "Use of MI_TRACK_ASAN is deprecated, use MI_TRACK=ASAN instead") + set(MI_TRACK "ASAN") +elseif(MI_TRACK_ETW) + message(STATUS "Use of MI_TRACK_ETW is deprecated, use MI_TRACK=ETW instead") + set(MI_TRACK "ETW") +endif() + +if(MI_TRACK STREQUAL "VALGRIND") CHECK_INCLUDE_FILES("valgrind/valgrind.h;valgrind/memcheck.h" MI_HAS_VALGRINDH) if (NOT MI_HAS_VALGRINDH) - set(MI_TRACK_VALGRIND OFF) + set(MI_TRACK OFF) message(WARNING "Cannot find the 'valgrind/valgrind.h' and 'valgrind/memcheck.h' -- install valgrind first?") - message(STATUS "Disabling Valgrind support (MI_TRACK_VALGRIND=OFF)") + message(STATUS "Disabling Valgrind support (MI_TRACK=OFF)") else() - message(STATUS "Compile with Valgrind support (MI_TRACK_VALGRIND=ON)") + message(STATUS "Compile with Valgrind support (MI_TRACK=VALGRIND)") list(APPEND mi_defines MI_TRACK_VALGRIND=1) endif() -endif() - -if(MI_TRACK_ASAN) +elseif(MI_TRACK STREQUAL "ASAN") if (APPLE AND MI_OVERRIDE) - set(MI_TRACK_ASAN OFF) - message(WARNING "Cannot enable address sanitizer support on macOS if MI_OVERRIDE is ON (MI_TRACK_ASAN=OFF)") - endif() - if (MI_TRACK_VALGRIND) - set(MI_TRACK_ASAN OFF) - message(WARNING "Cannot enable address sanitizer support with also Valgrind support enabled (MI_TRACK_ASAN=OFF)") - endif() - if(MI_TRACK_ASAN) + set(MI_TRACK OFF) + message(WARNING "Cannot enable address sanitizer support on macOS if MI_OVERRIDE is ON (MI_TRACK=OFF)") + else() CHECK_INCLUDE_FILES("sanitizer/asan_interface.h" MI_HAS_ASANH) if (NOT MI_HAS_ASANH) - set(MI_TRACK_ASAN OFF) + set(MI_TRACK OFF) message(WARNING "Cannot find the 'sanitizer/asan_interface.h' -- install address sanitizer support first") - message(STATUS "Compile **without** address sanitizer support (MI_TRACK_ASAN=OFF)") + message(STATUS "Compile **without** address sanitizer support (MI_TRACK=OFF)") else() - message(STATUS "Compile with address sanitizer support (MI_TRACK_ASAN=ON)") + message(STATUS "Compile with address sanitizer support (MI_TRACK=ASAN)") list(APPEND mi_defines MI_TRACK_ASAN=1) list(APPEND mi_cflags -fsanitize=address) list(APPEND mi_libraries -fsanitize=address) endif() endif() -endif() - -if(MI_TRACK_ETW) +elseif(MI_TRACK STREQUAL "ETW") if(NOT WIN32) - set(MI_TRACK_ETW OFF) - message(WARNING "Can only enable ETW support on Windows (MI_TRACK_ETW=OFF)") - endif() - if (MI_TRACK_VALGRIND OR MI_TRACK_ASAN) - set(MI_TRACK_ETW OFF) - message(WARNING "Cannot enable ETW support with also Valgrind or ASAN support enabled (MI_TRACK_ETW=OFF)") - endif() - if(MI_TRACK_ETW) - message(STATUS "Compile with Windows event tracing support (MI_TRACK_ETW=ON)") + set(MI_TRACK OFF) + message(WARNING "Can only enable ETW support on Windows (MI_TRACK=OFF)") + else() + message(STATUS "Compile with Windows event tracing support (MI_TRACK=ETW)") list(APPEND mi_defines MI_TRACK_ETW=1) endif() +elseif(MI_TRACK) + message(STATUS "Invalid option value: MI_TRACK=${MI_TRACK} (ignored)") endif() if(MI_SEE_ASM) @@ -340,10 +413,6 @@ if(MI_SEE_ASM) endif() endif() -if(MI_CHECK_FULL) - message(STATUS "The MI_CHECK_FULL option is deprecated, use MI_DEBUG_FULL instead") - set(MI_DEBUG_FULL "ON") -endif() if (MI_SKIP_COLLECT_ON_EXIT) message(STATUS "Skip collecting memory on program exit (MI_SKIP_COLLECT_ON_EXIT=ON)") @@ -355,34 +424,67 @@ if (MI_NO_PROCESS_DETACH) list(APPEND mi_defines MI_NO_PROCESS_DETACH=1) endif() -if(MI_DEBUG_FULL) - message(STATUS "Set debug level to full assertion and internal invariant checking (MI_DEBUG_FULL=ON, expensive)") - set(MI_DEBUG ON) - list(APPEND mi_defines MI_DEBUG=3) # full invariant checking (mi_assert, mi_assert_internal, and mi_assert_expensive) +if(MI_CHECK_FULL OR MI_DEBUG_FULL) + message(STATUS "Use of MI_CHECK_FULL/MI_DEBUG_FULL is deprecated, use MI_DEBUG=FULL instead") + if(MI_DEBUG STREQUAL "DEFAULT") + set(MI_DEBUG "FULL") + endif() elseif(MI_DEBUG_INTERNAL) - message(STATUS "Set debug level to internal assertion and invariant checking (MI_DEBUG_INTERNAL=ON)") - set(MI_DEBUG ON) + message(STATUS "Use of MI_DEBUG_INTERNAL is deprecated, use MI_DEBUG=INTERNAL instead") + if(MI_DEBUG STREQUAL "DEFAULT") + set(MI_DEBUG "INTERNAL") + endif() +endif() + +if(MI_DEBUG STREQUAL "DEFAULT") + if(CMAKE_CONFIGURATION_TYPES) + # Multi-config generators: Choose debug flags using generator expression + list(APPEND mi_defines $,MI_DEBUG=2,MI_DEBUG=0 NDEBUG=1>) + set(MI_DEBUG OFF) # don't go into further MI_DEBUG checks below... + else() + if(CMAKE_BUILD_TYPE MATCHES "Debug") + set(MI_DEBUG "INTERNAL") + else() + set(MI_DEBUG "OFF") + endif() + endif() +endif() + +if(MI_NO_DEBUG) + message(STATUS "Disable assertion checks (MI_NO_DEBUG=ON)") + list(APPEND mi_defines NDEBUG=1) + set(MI_DEBUG OFF) +elseif(MI_DEBUG STREQUAL "FULL") + message(STATUS "Set debug level to full assertion and internal invariant checking (MI_DEBUG=FULL, expensive)") + list(APPEND mi_defines MI_DEBUG=3) # full invariant checking (mi_assert, mi_assert_internal, and mi_assert_expensive) +elseif(MI_DEBUG STREQUAL "INTERNAL") + message(STATUS "Set debug level to internal assertion and invariant checking (MI_DEBUG=INTERNAL)") list(APPEND mi_defines MI_DEBUG=2) # invariant checking (mi_assert and mi_assert_internal) elseif(MI_DEBUG) - message(STATUS "Set debug level to assertion checking (MI_DEBUG=ON)") + message(STATUS "Set debug level to assertion checking (MI_DEBUG=${MI_DEBUG})") list(APPEND mi_defines MI_DEBUG=1) # assertion checking (mi_assert) -elseif(MI_NO_DEBUG) - message(STATUS "Disable assertion checks (MI_NO_DEBUG=ON)") +elseif(NOT CMAKE_CONFIGURATION_TYPES) + message(STATUS "Disable assertion checks (MI_DEBUG=${MI_DEBUG})") list(APPEND mi_defines NDEBUG=1) -elseif(CMAKE_BUILD_TYPE MATCHES "Debug") - message(STATUS "Set debug level to internal assertion and invariant checking (CMAKE_BUILD_TYPE=Debug)") - set(MI_DEBUG_INTERNAL ON) - set(MI_DEBUG ON) - list(APPEND mi_defines MI_DEBUG=2) # invariant checking (mi_assert and mi_assert_internal) endif() -if(MI_DEBUG AND !MI_GUARDED) - message(STATUS "Enable MI_GUARDED (since MI_DEBUG=ON)") - set(MI_GUARDED=ON) +if(MI_GUARDED STREQUAL "DEFAULT") + if(MI_DEBUG) + set(MI_GUARDED "FULL") + else() + set(MI_GUARDED "OFF") + endif() endif() -if(MI_GUARDED) - message(STATUS "Compile guard pages behind certain object allocations (MI_GUARDED=ON)") + +if(MI_GUARDED STREQUAL "FULL") + message(STATUS "Enable fine-grained sampling for guard pages (MI_GUARDED=FULL)") + list(APPEND mi_defines MI_GUARDED=2) +elseif(MI_GUARDED) + message(STATUS "Enable sampled guard pages (MI_GUARDED=${MI_GUARDED})") list(APPEND mi_defines MI_GUARDED=1) +else() + # message(STATUS "Disable guard pages (MI_GUARDED=${MI_GUARDED})") + list(APPEND mi_defines MI_GUARDED=0) endif() if(MI_NO_PADDING) @@ -395,27 +497,88 @@ else() endif() endif() -if(MI_DEBUG OR MI_SECURE) - set(MI_FREE_IS_CHECKED ON) +if(MI_FREE_IS_CHECKED STREQUAL "DEFAULT") + if (MI_SECURE) + set(MI_FREE_IS_CHECKED ON) + else() + set(MI_FREE_IS_CHECKED OFF) + endif() endif() -if(MI_FREE_IS_CHECKED) - message(STATUS "Check validity of pointers passed to 'mi_free' (MI_FREE_IS_CHECKED=ON)") + +if(MI_FREE_IS_CHECKED STREQUAL "ON") + message(STATUS "Check validity eof pointers passed to 'mi_free' (MI_FREE_IS_CHECKED=ON)") list(APPEND mi_defines MI_FREE_IS_CHECKED=1) -elseif(MI_OVERRIDE AND NOT MI_NO_FREE_IS_CHECKED) - message(STATUS "Check validity of pointers passed to 'mi_free' only in the dynamic link library (MI_OVERRIDE=ON, MI_FREE_IS_CHECKED=OFF)") +elseif(MI_SECURE) + message(STATUS "Check validity of pointers passed to 'mi_free' (MI_SECURE=ON, MI_FREE_IS_CHECKED=ON)") +elseif(MI_FREE_IS_CHECKED STREQUAL "SHARED") + message(STATUS "Check validity of pointers passed to 'mi_free' only in the dynamic/shared link library (MI_FREE_IS_CHECKED=SHARED)") list(APPEND mi_cflags_dynamic -DMI_FREE_IS_CHECKED=1) +elseif(MI_FREE_IS_CHECKED STREQUAL "OFF") + # message(STATUS "Do not check validity of pointers passed to 'mi_free' (MI_FREE_IS_CHECKED=OFF)") +else() + message(STATUS "Invalid value for MI_FREE_IS_CHECKED: ${MI_FREE_IS_CHECKED} (expecting: DEFAULT, OFF, SHARED, or ON)") +endif() + +if(MI_STATS STREQUAL "DEFAULT") + if(MI_DEBUG) + set(MI_STATS "FULL") + else() + set(MI_STATS "ON") + endif() endif() -if(NOT MI_OPT_FREE_SMALL AND NOT MI_DEBUG AND NOT MI_GUARDED AND NOT MI_SECURE) -set(MI_OPT_FREE_SMALL ON) +if(MI_STATS STREQUAL "FULL") + message(STATUS "Enable detailed statistics including precise requested bytes (expensive) (MI_STATS=FULL)") + list(APPEND mi_defines MI_STATS=2) +elseif(MI_STATS) + message(STATUS "Enable detailed statistics (MI_STATS=${MI_STATS})") + list(APPEND mi_defines MI_STATS=1) +else() + message(STATUS "Disable detailed statistics (MI_STATS=${MI_STATS})") + list(APPEND mi_defines MI_STATS=0) endif() -if(MI_OPT_FREE_SMALL) - if(MI_GUARDED OR MI_SECURE) + +if(MI_PROFILE STREQUAL "DEFAULT") + if(MI_DEBUG) + set(MI_PROFILE "FULL") + else() + set(MI_PROFILE "ON") + endif() +endif() + +if(MI_PROFILE STREQUAL "FULL") + message(STATUS "Enable fine-grained sampled profiling (MI_PROFILE=FULL)") + list(APPEND mi_defines MI_PROFILE=2) +elseif(MI_PROFILE) + message(STATUS "Enable sampled profiling (MI_PROFILE=${MI_PROFILE})") + list(APPEND mi_defines MI_PROFILE=1) +else() + message(STATUS "Disable profiling (MI_PROFILE=${MI_PROFILE})") + list(APPEND mi_defines MI_PROFILE=0) +endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "WASI") + message(STATUS "Enabling MI_FREE_USE_PAGEMAP since the target (WASI) does not support large OS aligned allocations well") + set(MI_FREE_USE_PAGEMAP ON) +endif() +if(MI_FREE_USE_PAGEMAP) + message(STATUS "Always use the pagemap in mi_free (MI_FREE_USE_PAGEMAP=ON)") + list(APPEND mi_defines MI_FREE_USE_PAGEMAP=1) +endif() + +# deprecated +if(MI_OPT_FREE_SMALL STREQUAL "DEFAULT") + # do nothing +elseif(MI_OPT_FREE_SMALL) + if(MI_GUARDED OR MI_SECURE) message(STATUS "Cannot enable MI_OPT_FREE_SMALL in secure or guarded mode (MI_OPT_FREE_SMALL=OFF)") else() - message(STATUS "Optimize `mi_free_small` to not need a pagemap lookup (MI_OPT_FREE_SMALL=ON)") + message(STATUS "Use aligned `mi_free_small` (MI_OPT_FREE_SMALL=ON)") list(APPEND mi_defines MI_OPT_FREE_SMALL=1) endif() +else() + message(STATUS "Disable aligned `mi_free_small` (MI_OPT_FREE_SMALL=OFF)") + list(APPEND mi_defines MI_OPT_FREE_SMALL=0) endif() if(MI_XMALLOC) @@ -440,7 +603,7 @@ if(MI_DEBUG_TSAN) endif() if(MI_DEBUG_UBSAN) - if(CMAKE_BUILD_TYPE MATCHES "Debug") + if(CMAKE_BUILD_TYPE MATCHES "Debug" OR CMAKE_CONFIGURATION_TYPES) if(MI_USE_CXX) if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") message(STATUS "Build with undefined-behavior sanitizer (MI_DEBUG_UBSAN=ON)") @@ -460,7 +623,23 @@ endif() if(CMAKE_SYSTEM_NAME MATCHES "Linux|Android") if(MI_NO_THP) - message(STATUS "Disable transparent huge pages support (MI_NO_THP=ON)") + set(MI_ALLOW_THP "OFF") + endif() + if(MI_ALLOW_THP STREQUAL "DEFAULT") + if(CMAKE_SYSTEM_NAME MATCHES "Android") + set(MI_ALLOW_THP "OFF") + else() + set(MI_ALLOW_THP "FULL") + endif() + endif() + if(MI_ALLOW_THP STREQUAL "FULL") + message(STATUS "Enable full transparent huge pages support, where purging avoids splitting huge pages (MI_ALLOW_THP=FULL)") + list(APPEND mi_defines MI_DEFAULT_ALLOW_THP=2) + elseif (MI_ALLOW_THP) + message(STATUS "Enable transparent huge pages support (MI_ALLOW_THP=${MI_ALLOW_THP}) (also consider using FULL for better performance)") + list(APPEND mi_defines MI_DEFAULT_ALLOW_THP=1) + else() + message(STATUS "Disable transparent huge pages support (MI_ALLOW_THP=${MI_ALLOW_THP})") list(APPEND mi_defines MI_DEFAULT_ALLOW_THP=0) endif() endif() @@ -470,19 +649,42 @@ if(MI_LIBC_MUSL) list(APPEND mi_defines MI_LIBC_MUSL=1) endif() -if(MI_TLS_MODEL_PTHREADS) - message(STATUS "Use TLS model based on pthreads (MI_TLS_MODEL_PTHREADS=ON)") +if(MI_TLS_MODEL STREQUAL "DEFAULT") + if(MI_TLS_MODEL_PTHREADS) + message(STATUS "Use of MI_TLS_MODEL_PTHREADS is deprecated, use MI_TLS_MODEL=PTHREADS instead") + set(MI_TLS_MODEL "PTHREADS") + elseif(MI_TLS_MODEL_FIXED) + message(STATUS "Use of MI_TLS_MODEL_FIXED is deprecated, use MI_TLS_MODEL=FIXED instead") + set(MI_TLS_MODEL "FIXED") + elseif(MI_TLS_MODEL_LOCAL) + message(STATUS "Use of MI_TLS_MODEL_LOCAL is deprecated, use MI_TLS_MODEL=LOCAL instead") + set(MI_TLS_MODEL "LOCAL") + elseif(MI_LOCAL_DYNAMIC_TLS) + message(STATUS "Use of MI_LOCAL_DYNAMIC_TLS is deprecated, use MI_TLS_MODEL=LOCAL_DYNAMIC instead") + set(MI_TLS_MODEL "LOCAL_DYNAMIC") + endif() +endif() +if(MI_TLS_MODEL STREQUAL "PTHREADS") + message(STATUS "Use TLS model based on pthreads (MI_TLS_MODEL=PTHREADS)") list(APPEND mi_defines MI_TLS_MODEL_PTHREADS=1) -elseif(MI_TLS_MODEL_FIXED) - message(STATUS "Use TLS model based on fixed thread local slots (MI_TLS_MODEL_FIXED=ON)") +elseif(MI_TLS_MODEL STREQUAL "FIXED") + message(STATUS "Use TLS model based on fixed thread local slots (MI_TLS_MODEL=FIXED)") message(STATUS "Note: fixed TLS slots do not allow multiple mimalloc instances in a single process.") - list(APPEND mi_defines MI_TLS_MODEL_FIXED=1) -elseif(MI_TLS_MODEL_LOCAL) - message(STATUS "Use TLS model based on thread local (MI_TLS_MODEL_LOCAL=ON)") + list(APPEND mi_defines MI_TLS_MODEL=FIXED=1) +elseif(MI_TLS_MODEL STREQUAL "LOCAL") + message(STATUS "Use TLS model based on thread local (MI_TLS_MODEL=LOCAL)") if(APPLE) - message(WARNING "On macOS the MI_TLS_MODEL_LOCAL does not work reliably when dynamically overriding mimalloc. MI_TLS_MODEL_PTHREADS is recommended instead.") + message(WARNING "On macOS the MI_TLS_MODEL=LOCAL does not work reliably when dynamically overriding mimalloc. MI_TLS_MODEL=PTHREADS is recommended instead.") endif() list(APPEND mi_defines MI_TLS_MODEL_LOCAL=1) +elseif(MI_TLS_MODEL STREQUAL "LOCAL_DYNAMIC") + message(STATUS "Use TLS model based on dlopen compatible dynamic thread locals (MI_TLS_MODEL=LOCAL_DYNAMIC)") + list(APPEND mi_defines MI_TLS_MODEL_LOCAL=1) +elseif(MI_TLS_MODEL STREQUAL "WIN32") + message(STATUS "Use TLS model based on direct Win32 TlsAlloc'd slots (MI_TLS_MODEL=WIN32)") + list(APPEND mi_defines MI_TLS_MODEL_WIN32=1) +elseif(NOT (MI_TLS_MODEL STREQUAL "DEFAULT")) + message(STATUS "Invalid option value: MI_TLS_MODEL=${MI_TLS_MODEL} (ignored)") endif() if(MI_TLS_RECURSE_GUARD) @@ -504,19 +706,31 @@ if(MI_WIN_DIRECT_TLS) list(APPEND mi_defines MI_WIN_DIRECT_TLS=1) endif() -if(MI_WIN_INIT_USE_RAW_DLLMAIN) - message(STATUS "Use raw Dll main entry on Windows to initialize and finalize mimalloc (MI_WIN_INIT_USE_RAW_DLLMAIN=ON)") +if(MI_WIN_INIT STREQUAL "DEFAULT") + if(MI_WIN_INIT_USE_RAW_DLLMAIN) + message(STATUS "Use of MI_WIN_INIT_USE_RAW_DLLMAIN is deprecated, use MI_WIN_INIT=RAW_DLLMAIN instead") + set(MI_WIN_INIT "RAW_DLLMAIN") + elseif(MI_WIN_INIT_USE_TLS_DLLMAIN) + message(STATUS "Use of MI_WIN_INIT_USE_TLS_DLLMAIN is deprecated, use MI_WIN_INIT=TLS_DLLMAIN instead") + set(MI_WIN_INIT "TLS_DLLMAIN") + elseif(MI_WIN_USE_FLS) + message(STATUS "Use of MI_WIN_USE_FLS is deprecated, use MI_WIN_INIT=FLS instead") + set(MI_WIN_INIT "FLS") + endif() +endif() + +if(MI_WIN_INIT STREQUAL "RAW_DLLMAIN") + message(STATUS "Use raw Dll main entry on Windows to initialize and finalize mimalloc (MI_WIN_INIT=RAW_DLLMAIN)") message(STATUS "Note: this can lead to link errors if other libraries also try to define a raw Dll main entry point") list(APPEND mi_defines MI_WIN_INIT_USE_RAW_DLLMAIN=1) -elseif(MI_WIN_INIT_USE_TLS_DLLMAIN) - message(STATUS "Use Dll main entry with TLS entries on Windows to initialize and finalize mimalloc (MI_WIN_INIT_USE_TLS_DLLMAIN=ON)") +elseif(MI_WIN_INIT STREQUAL "TLS_DLLMAIN") + message(STATUS "Use Dll main entry with TLS entries on Windows to initialize and finalize mimalloc (MI_WIN_INIT=TLS_DLLMAIN)") list(APPEND mi_defines MI_WIN_INIT_USE_TLS_DLLMAIN=1) -elseif(MI_WIN_USE_FLS) - message(STATUS "Deprecated: Use the Fiber API to detect thread termination (deprecated) (MI_WIN_USE_FLS=ON)") +elseif(MI_WIN_INIT STREQUAL "FLS") + message(STATUS "Use the Fiber API to detect thread termination (deprecated) (MI_WIN_INIT=FLS)") list(APPEND mi_defines MI_WIN_INIT_USE_FLS=1) -else() - # default uses CRT with TLS entries for initialization and finalization - # list(APPEND mi_defines MI_WIN_INIT_USE_CRT_TLS=1) +elseif(NOT (MI_WIN_INIT STREQUAL "DEFAULT")) + message(STATUS "Invalid option value: MI_WIN_INIT=${MI_WIN_INIT} (ignored)") endif() if(MI_ARCH MATCHES "riscv") @@ -530,6 +744,13 @@ if(MI_ARCH MATCHES "riscv") endif() endif() +if(NOT WIN32) + CHECK_INCLUDE_FILES("execinfo.h" MI_HAS_EXECINFOH) + if(MI_HAS_EXECINFOH) + list(APPEND mi_defines MI_HAS_EXECINFOH=1) + endif() +endif() + # On Haiku use `-DCMAKE_INSTALL_PREFIX` instead, issue #788 # if(CMAKE_SYSTEM_NAME MATCHES "Haiku") # SET(CMAKE_INSTALL_LIBDIR ~/config/non-packaged/lib) @@ -545,6 +766,9 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU" AND NOT MI_CLANG_CL) if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang") list(APPEND mi_cflags -Wno-static-in-inline -mno-outline) endif() + if(CMAKE_C_COMPILER_ID STREQUAL "AppleClang" AND MI_ARCH STREQUAL "arm64") + list(APPEND mi_cflags -Xclang -fno-split-cold-code) + endif() endif() if(CMAKE_C_COMPILER_ID MATCHES "Intel") @@ -552,7 +776,7 @@ if(CMAKE_C_COMPILER_ID MATCHES "Intel") endif() if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM_NAME MATCHES "Haiku" AND NOT MI_CLANG_CL) - if(MI_LOCAL_DYNAMIC_TLS) + if(MI_TLS_MODEL STREQUAL "LOCAL_DYNAMIC") list(APPEND mi_cflags -ftls-model=local-dynamic) else() if(MI_LIBC_MUSL) @@ -573,30 +797,62 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel") endif() # Compiler and architecture specific flags -if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM_NAME MATCHES "Haiku") - if(MI_OPT_ARCH) - if(APPLE AND CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang" AND CMAKE_OSX_ARCHITECTURES) # to support multi-arch binaries (#999) - if("arm64" IN_LIST CMAKE_OSX_ARCHITECTURES) - list(APPEND MI_OPT_ARCH_FLAGS "-Xarch_arm64;-march=armv8.1-a") - endif() - if("x86_64" IN_LIST CMAKE_OSX_ARCHITECTURES) - list(APPEND MI_OPT_ARCH_FLAGS "-Xarch_x86_64;-march=haswell;-Xarch_x86_64;-mavx2") - endif() - elseif(MI_ARCH STREQUAL "x64") - set(MI_OPT_ARCH_FLAGS "-march=haswell;-mavx2") # fast bit scan (since 2013) - elseif(MI_ARCH STREQUAL "arm64") - set(MI_OPT_ARCH_FLAGS "-march=armv8.1-a") # fast atomics (since 2016) +if(MI_OPT_ARCH MATCHES "^(1|ON|YES|TRUE|Y)$") + if(APPLE AND CMAKE_OSX_ARCHITECTURES AND CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang" ) + set(MI_OPT_ARCH "ON") + elseif(MI_ARCH STREQUAL "arm64") + if(APPLE) + set(MI_OPT_ARCH "armv8.3") # M1+ supports arm8.3-a (which includes LDAPR (load-aquire)) + else() + set(MI_OPT_ARCH "armv8.1") # enable armv8.1-a by default (which has fast atomics) endif() + elseif(MI_ARCH STREQUAL "arm32") + set(MI_OPT_ARCH "armv8.1") # enable armv8.1-a by default (which has fast atomics) + elseif(MI_ARCH STREQUAL "x64") + set(MI_OPT_ARCH "avx2") + elseif(MI_ARCH MATCHES "riscv") + set(MI_OPT_ARCH "rv${MI_ARCH_BITS}gcb_zacas") # fast bit scan and atomics (since 2023) + # set(MI_OPT_ARCH "rv${MI_ARCH_BITS}gcb_zacas_zalasr") # fast bit scan and atomics (since 2025) + else() + set(MI_OPT_ARCH OFF) endif() endif() -if (MSVC AND MSVC_VERSION GREATER_EQUAL 1914) # vs2017+ - list(APPEND mi_cflags /Zc:__cplusplus) - if(MI_OPT_ARCH AND NOT MI_CLANG_CL) - if(MI_ARCH STREQUAL "x64") +# Compiler and architecture specific flags +set(MI_OPT_ARCH_FLAGS "") +if(NOT MI_OPT_ARCH MATCHES "^(0|OFF|NO|FALSE|N)$") + # apple support for multi-arch binaries (#999) + if(APPLE AND CMAKE_OSX_ARCHITECTURES AND CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang") + if("arm64" IN_LIST CMAKE_OSX_ARCHITECTURES) + list(APPEND MI_OPT_ARCH_FLAGS "-Xarch_arm64;-march=armv8.3-a") + endif() + if("x86_64" IN_LIST CMAKE_OSX_ARCHITECTURES) + list(APPEND MI_OPT_ARCH_FLAGS "-Xarch_x86_64;-march=haswell;-Xarch_x86_64;-mavx2") + endif() + + # clang/gcc + elseif(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM_NAME MATCHES "Haiku") + if(MI_OPT_ARCH STREQUAL "avx2") + set(MI_OPT_ARCH_FLAGS "-march=haswell;-mavx2") # fast bit scan (since 2013) + elseif(MI_OPT_ARCH MATCHES "^armv[89].*$") + set(MI_OPT_ARCH_FLAGS "-march=${MI_OPT_ARCH}-a") # fast atomics + elseif(MI_OPT_ARCH MATCHES "^rva[0-9][0-9]$") + set(MI_OPT_ARCH_FLAGS "-march=${MI_OPT_ARCH}u${MI_ARCH_BITS}") + elseif(MI_OPT_ARCH MATCHES "^rv.*$") + set(MI_OPT_ARCH_FLAGS "-march=${MI_OPT_ARCH}") + else() + set(MI_OPT_ARCH_FLAGS "-march=${MI_OPT_ARCH}") + message(STATUS "warning: unknown MI_OPT_ARCH=${MI_OPT_ARCH} passed as '-march=${MI_OPT_ARCH}'") + endif() + + # msvc + elseif (MSVC AND MSVC_VERSION GREATER_EQUAL 1914 AND NOT MI_CLANG_CL) # vs2017+ + if(MI_OPT_ARCH STREQUAL "avx2") set(MI_OPT_ARCH_FLAGS "/arch:AVX2") - elseif(MI_ARCH STREQUAL "arm64") - set(MI_OPT_ARCH_FLAGS "/arch:armv8.1") # fast atomics (LSE) + elseif(MI_OPT_ARCH MATCHES "^armv[89].*$") + set(MI_OPT_ARCH_FLAGS "/arch:${MI_OPT_ARCH}") # fast atomics + else() + message(STATUS "warning: unknown MI_OPT_ARCH=${MI_OPT_ARCH} passed as '/arch:${MI_OPT_ARCH}'") endif() endif() endif() @@ -607,7 +863,7 @@ endif() if(MI_OPT_ARCH_FLAGS) list(APPEND mi_cflags ${MI_OPT_ARCH_FLAGS}) - message(STATUS "Architecture specific optimization is enabled (with ${MI_OPT_ARCH_FLAGS}) (MI_OPT_ARCH=ON)") + message(STATUS "Architecture specific optimization is enabled (with ${MI_OPT_ARCH_FLAGS}) (MI_OPT_ARCH=${MI_OPT_ARCH})") if (MI_OPT_SIMD) list(APPEND mi_defines "MI_OPT_SIMD=1") message(STATUS "SIMD instructions are enabled (MI_OPT_SIMD=ON)") @@ -616,6 +872,7 @@ elseif(MI_OPT_SIMD) message(STATUS "SIMD instructions are not enabled (either MI_OPT_ARCH=OFF or this architecture has no SIMD support)") endif() + # extra needed libraries # we prefer -l test over `find_library` as sometimes core libraries @@ -657,7 +914,7 @@ endif() # set language for source files now if(MI_USE_CXX) set_source_files_properties(${mi_sources} PROPERTIES LANGUAGE CXX ) - set_source_files_properties(src/static.c test/test-api.c test/test-api-fill.c test/test-stress.c test/test-stress-heaps.c test/test-stress-subprocs.c PROPERTIES LANGUAGE CXX ) + set_source_files_properties(src/static.c test/test-api.c test/test-api-fill.c test/test-stress.c test/test-stress-heaps.c test/test-stress-subprocs.c test/test-profile.c PROPERTIES LANGUAGE CXX ) # the white-box tests call internal functions, whose names are mangled now; the heap tests use C11 atomics, which MSVC lacks in C mode set_source_files_properties(test/test-theap-sentinel.c test/test-purge-holes.c test/test-commit-fail.c test/test-heap-aba.c test/test-heap-delete-race.c test/test-heap-teardown.c @@ -688,18 +945,32 @@ set(mi_libname "mimalloc") if(MI_SECURE) set(mi_libname "${mi_libname}-secure") endif() -if(MI_TRACK_VALGRIND) +if(MI_TRACK STREQUAL "VALGRIND") set(mi_libname "${mi_libname}-valgrind") endif() -if(MI_TRACK_ASAN) +if(MI_TRACK STREQUAL "ASAN") set(mi_libname "${mi_libname}-asan") endif() -string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC) -list(APPEND mi_defines "MI_CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE_LC}") #todo: multi-config project needs $ ? -if(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel|none)$") - list(APPEND mi_defines MI_BUILD_RELEASE) +if (CMAKE_CONFIGURATION_TYPES) + # multi-config builds + # generator expression lower-case build type + set(mi_gx_build_type_lc $>) + # generator expression for "is this a release build?" + set(mi_gx_release $,$,$,$>) + + list(APPEND mi_defines "MI_CMAKE_BUILD_TYPE=${mi_gx_build_type_lc}") + list(APPEND mi_defines $<${mi_gx_release}:MI_BUILD_RELEASE>) + # append build type (e.g. -debug) if not a release version + set(mi_libname "${mi_libname}$<$:-${mi_gx_build_type_lc}>") else() - set(mi_libname "${mi_libname}-${CMAKE_BUILD_TYPE_LC}") #append build type (e.g. -debug) if not a release version + # single-config builds + string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC) + list(APPEND mi_defines "MI_CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE_LC}") #todo: multi-config project needs $ ? + if(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel|none)$") + list(APPEND mi_defines MI_BUILD_RELEASE) + else() + set(mi_libname "${mi_libname}-${CMAKE_BUILD_TYPE_LC}") #append build type (e.g. -debug) if not a release version + endif() endif() string(REGEX REPLACE "^mimalloc[-]?" "" mi_libsuffix "${mi_libname}") @@ -720,7 +991,9 @@ endif() message(STATUS "") message(STATUS "Library name : ${mi_libname}") message(STATUS "Version : ${mi_version}.${mi_version_patch}") -message(STATUS "Build type : ${CMAKE_BUILD_TYPE_LC}") +if(CMAKE_BUILD_TYPE_LC) + message(STATUS "Build type : ${CMAKE_BUILD_TYPE_LC}") +endif() if(MI_USE_CXX) message(STATUS "C++ Compiler : ${CMAKE_CXX_COMPILER}") else() @@ -829,11 +1102,7 @@ if (MI_BUILD_OBJECT) ) # Copy the generated object file (`static.o`) to the output directory (as `mimalloc.o`) - if(CMAKE_GENERATOR MATCHES "^Visual Studio.*$") - set(mimalloc-obj-static "${CMAKE_CURRENT_BINARY_DIR}/mimalloc-obj.dir/$/static${CMAKE_C_OUTPUT_EXTENSION}") - else() - set(mimalloc-obj-static "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/mimalloc-obj.dir/src/static.c${CMAKE_C_OUTPUT_EXTENSION}") - endif() + set(mimalloc-obj-static "$") set(mimalloc-obj-out "${CMAKE_CURRENT_BINARY_DIR}/${mi_libname}${CMAKE_C_OUTPUT_EXTENSION}") add_custom_command(OUTPUT ${mimalloc-obj-out} DEPENDS mimalloc-obj mimalloc-static COMMAND "${CMAKE_COMMAND}" -E copy "${mimalloc-obj-static}" "${mimalloc-obj-out}") add_custom_target(mimalloc-obj-target ALL DEPENDS ${mimalloc-obj-out}) @@ -878,18 +1147,15 @@ if (MI_BUILD_TESTS) enable_testing() # static link tests - set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-teardown heap-delete-race heap-churn heap-release-mt heap-aba heap-burst-destroy abandoned-lazy fork-user-heap snapshot prof prof-adversarial purge-zero park-handoff forced-purge purge-behind-pass free-before-init) - if(NOT (MI_DEBUG_TSAN OR MI_TRACK_ASAN OR MI_DEBUG_UBSAN)) + set(mi_static_tests api api-fill stress-heaps stress-subprocs stress profile heap-mt heap-teardown heap-delete-race heap-churn heap-release-mt heap-aba heap-burst-destroy abandoned-lazy fork-user-heap snapshot purge-zero park-handoff forced-purge purge-behind-pass free-before-init) + if(NOT (MI_DEBUG_TSAN OR (MI_TRACK STREQUAL "ASAN") OR MI_DEBUG_UBSAN)) list(APPEND mi_static_tests thp-optout) # counts madvise calls by interposing it, which a sanitizer runtime does first endif() if(MI_BUILD_STATIC AND NOT MI_DEBUG_TSAN) # these reach into internals that the shared library (which the TSAN build links the tests against) does not export - list(APPEND mi_static_tests theap-sentinel) - if(NOT MI_GUARDED) # and these two know the page layout, which guard pages change - list(APPEND mi_static_tests purge-holes) - endif() + list(APPEND mi_static_tests theap-sentinel purge-holes) endif() - if (MI_DEBUG_FULL OR CMAKE_BUILD_TYPE MATCHES "Debug") + if (MI_DEBUG) list(APPEND mi_static_tests commit-fail) # uses mi_debug_fail_os_commit_after (MI_DEBUG>0 only) endif() foreach(TEST_NAME ${mi_static_tests}) @@ -904,14 +1170,23 @@ if (MI_BUILD_TESTS) else() message(STATUS "cannot build TSAN tests without MI_BUILD_SHARED being enabled") endif() - if(MI_GUARDED) - add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_GUARDED_SAMPLE_RATE=1 $) + set(mi_test_env "") + if(MI_GUARDED AND TEST_NAME STREQUAL "purge-holes") + list(APPEND mi_test_env MIMALLOC_GUARDED_SAMPLE_RATE=0) # it knows the page layout, which guard pages change (set explicitly: the CI sets a rate for the whole job) + elseif(MI_GUARDED AND NOT (TEST_NAME STREQUAL "profile")) + list(APPEND mi_test_env MIMALLOC_GUARDED_SAMPLE_RATE=1024) elseif(TEST_NAME STREQUAL "stress-heaps") - add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_ARENA_EAGER_COMMIT=0 MIMALLOC_PAGE_COMMIT_ON_DEMAND=1 $) - elseif(TEST_NAME STREQUAL "thp-optout") - add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_ALLOW_THP=0 $) # off from the start: covers the mappings made during initialization too + list(APPEND mi_test_env MIMALLOC_ARENA_EAGER_COMMIT=0 MIMALLOC_PAGE_COMMIT_ON_DEMAND=0) + endif() + if(TEST_NAME STREQUAL "thp-optout") + list(APPEND mi_test_env MIMALLOC_ALLOW_THP=0) # off from the start: covers the mappings made during initialization too elseif(TEST_NAME STREQUAL "fork-user-heap" AND MI_DEBUG_TSAN) - add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env TSAN_OPTIONS=die_after_fork=0 $) # the forked child starts the scavenger thread + list(APPEND mi_test_env TSAN_OPTIONS=die_after_fork=0) # the forked child starts the scavenger thread + endif() + if(NOT MI_PROFILE AND (TEST_NAME STREQUAL "profile")) + # don't add as a test + elseif(mi_test_env) + add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env ${mi_test_env} $) else() add_test(NAME test-${TEST_NAME} COMMAND mimalloc-test-${TEST_NAME}) endif() @@ -919,7 +1194,7 @@ if (MI_BUILD_TESTS) # the same hole-purging tests with the feature turned off: it must be a no-op if(TARGET mimalloc-test-purge-holes) - add_test(NAME test-purge-holes-off COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_PURGE_HOLES=0 $) + add_test(NAME test-purge-holes-off COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_PURGE_HOLES=0 MIMALLOC_GUARDED_SAMPLE_RATE=0 $) endif() # with no scavenger to hand off to, `mi_on_thread_idle_start` must do the work inline instead @@ -934,7 +1209,7 @@ if (MI_BUILD_TESTS) # allocator once more with emulated TLS, as malloc, in the TLS model Android uses, and link # test/test-emulated-tls.c against that build. Only compilers that can emulate TLS (clang); the # sanitizers bring their own malloc. - if(MI_OVERRIDE AND CMAKE_SYSTEM_NAME MATCHES "Linux|Android" AND NOT (MI_TRACK_ASAN OR MI_TRACK_VALGRIND OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) + if(MI_OVERRIDE AND CMAKE_SYSTEM_NAME MATCHES "Linux|Android" AND NOT (MI_TRACK OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) include(CheckCCompilerFlag) check_c_compiler_flag("-femulated-tls" MI_HAS_EMULATED_TLS) if(MI_HAS_EMULATED_TLS) @@ -960,8 +1235,18 @@ if (MI_BUILD_TESTS) target_compile_options(mi-heapview PRIVATE ${mi_cflags}) target_link_libraries(mi-heapview PRIVATE ${mi_libraries}) + # static override test + if(MI_BUILD_OBJECT AND NOT (MI_CYGWIN OR (MI_TRACK STREQUAL "ASAN") OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) + add_executable(mimalloc-test-stress-static test/test-stress.c) + target_compile_definitions(mimalloc-test-stress-static PRIVATE ${mi_defines} "USE_STD_MALLOC=1") + target_compile_options(mimalloc-test-stress-static PRIVATE ${mi_cflags}) + target_include_directories(mimalloc-test-stress-static PRIVATE include) + target_link_libraries(mimalloc-test-stress-static PRIVATE mimalloc-obj ${mi_libraries}) # override statically with an object file + add_test(NAME test-stress-static COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_VERBOSE=1 MIMALLOC_SHOW_STATS=1 $) + endif() + # dynamic override test - if(MI_BUILD_SHARED AND NOT (MI_CYGWIN OR MI_TRACK_ASAN OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) # AND NOT (APPLE AND MI_USE_CXX)) + if(MI_BUILD_SHARED AND NOT (MI_CYGWIN OR (MI_TRACK STREQUAL "ASAN") OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) # AND NOT (APPLE AND MI_USE_CXX)) add_executable(mimalloc-test-stress-dynamic test/test-stress.c) target_compile_definitions(mimalloc-test-stress-dynamic PRIVATE ${mi_defines} "USE_STD_MALLOC=1") target_compile_options(mimalloc-test-stress-dynamic PRIVATE ${mi_cflags}) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5094950e1..84ce387f9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -127,7 +127,7 @@ jobs: CC: clang CXX: clang++ BuildType: debug-asan-clang - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_TRACK_ASAN=ON + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_TRACK_ASAN=ON Debug UBSAN Clang: CC: clang CXX: clang++ @@ -156,6 +156,7 @@ jobs: displayName: CTest env: MIMALLOC_GUARDED_SAMPLE_RATE: 1000 + MIMALLOC_VERBOSE: 2 # - upload: $(Build.SourcesDirectory)/$(BuildType) # artifact: mimalloc-ubuntu-$(BuildType) @@ -188,6 +189,7 @@ jobs: - script: ctest --verbose --timeout 240 workingDirectory: $(BuildType) displayName: CTest + # - upload: $(Build.SourcesDirectory)/$(BuildType) # artifact: mimalloc-macos-$(BuildType) diff --git a/cmake/mimalloc-config-version.cmake b/cmake/mimalloc-config-version.cmake index 857ef9ab5..349535cce 100644 --- a/cmake/mimalloc-config-version.cmake +++ b/cmake/mimalloc-config-version.cmake @@ -1,6 +1,6 @@ set(mi_version_major 3) set(mi_version_minor 5) -set(mi_version_patch 0) +set(mi_version_patch 2) set(mi_version ${mi_version_major}.${mi_version_minor}) set(PACKAGE_VERSION ${mi_version}) diff --git a/contrib/vcpkg/portfile.cmake b/contrib/vcpkg/portfile.cmake index b5d219595..46dc48554 100644 --- a/contrib/vcpkg/portfile.cmake +++ b/contrib/vcpkg/portfile.cmake @@ -3,7 +3,7 @@ vcpkg_from_github( REPO microsoft/mimalloc HEAD_REF master - # The "REF" can be a commit hash, branch name (dev3), or a version (v3.5.0). + # The "REF" can be a commit hash, branch name (dev3), or a version (v3.5.1). REF "v${VERSION}" # The sha512 is the hash of the tar.gz bundle. diff --git a/contrib/vcpkg/vcpkg.json b/contrib/vcpkg/vcpkg.json index 39b0fadaf..f66b0ef8e 100644 --- a/contrib/vcpkg/vcpkg.json +++ b/contrib/vcpkg/vcpkg.json @@ -1,6 +1,6 @@ { "name": "mimalloc", - "version": "3.5.0", + "version": "3.5.1", "port-version": 0, "description": "Compact general purpose allocator with excellent performance", "homepage": "https://github.com/microsoft/mimalloc", diff --git a/doc/mimalloc-doc.h b/doc/mimalloc-doc.h index f3c71bd4f..03b0a44fb 100644 --- a/doc/mimalloc-doc.h +++ b/doc/mimalloc-doc.h @@ -124,9 +124,18 @@ Further information: /// Free previously allocated memory. /// The pointer `p` must have been allocated before (or be \a NULL). -/// @param p pointer to free, or \a NULL. +/// @param p pointer to the block that is freed, or \a NULL. +/// See also mi_cfree() for a checked free that validates the pointer +/// before free-ing. +/// @see mi_cfree() void mi_free(void* p); +/// Checked `free`: just as mi_free() but always checks if the pointer `p` belongs to our heap +/// @param p pointer to a block that is freed, or any other arbitrary pointer. +/// This is safe to use when a pointer might be invalid or allocated by +/// another allocator (in which case the pointer is ignored and not freed). +void mi_cfree(void* p); + /// Allocate \a size bytes. /// @param size number of bytes to allocate. /// @returns pointer to the allocated memory or \a NULL if out of memory. @@ -404,19 +413,20 @@ void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t of /// \} /// \defgroup constantsize Constant size allocation -/// Constant size allocation and freeing +/// Inlined definitions for onstant size allocation and freeing. +/// If the size is constant, it can pick statically between +/// mi_malloc() and mi_malloc_small(). /// For example: /// ``` /// long* p = (long*)mi_malloc_csize(sizeof(long)) /// ``` -/// On __v3__ these can improve performance as they will statically -/// use mi_malloc_small(), mi_free_small(), etc. when possible. +/// On __v3__ these can improve performance. /// This is especially useful for compilers and runtimes where the /// the size is often statically known. These functions can be used /// even when the size is not constant but then there is no performance benefit. /// \{ -/// @brief Allocate a constant size object. +/// @brief __v3__: Allocate a constant size object. /// @param size A (compile-time) constant size. /// @return A pointer to an allocated block of size \a size bytes. /// @see mi_free_csize() @@ -429,13 +439,22 @@ void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size); void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size); -/// @brief Free a pointer with a known constant size. +/// @brief __v3__: Free a pointer with a known constant size. /// @param p The pointer to the allocated block (or \a NULL ) -/// @param size The (compile time constant) size with which the pointer was allocated. +/// @param size The size with which the pointer was allocated. /// @see mi_malloc_csize() /// @see mi_free_size() +/// @see mi_free_size_nonnull() void mi_free_csize(void* p, size_t size); +/// @brief __v3__: Free a non-null pointer with a known constant size. +/// @param p The pointer to the allocated block (cannot be \a NULL !) +/// @param size The size with which the pointer was allocated. +/// @see mi_malloc_csize() +/// @see mi_free_size() +/// @see mi_free_size_nonnull() +void mi_free_csize_nonnull(void* p, size_t size); + /// \} /// \defgroup zeroinit Zero initialized re-allocation @@ -1056,12 +1075,13 @@ void* mi_zalloc_small(size_t size); /// __v3__: Can be used to free an object that was allocated with mi_malloc_small() or /// any allocation with a size < MI_SMALL_SIZE_MAX() -/// @param p Pointer that was returned from #mi_malloc_small et al. +/// @param p Pointer that was returned from #mi_malloc_small et al. (or \a NULL ) /// @details /// This function is meant for use in run-time systems for best /// performance and does not check if the pointer was _indeed_ allocated /// as a small object -- use with care! /// +/// @see mi_free_small_nonnull() /// @see mi_malloc_small() /// @see mi_zalloc_small() /// @see mi_heap_malloc_small() @@ -1070,6 +1090,23 @@ void* mi_zalloc_small(size_t size); /// @see mi_theap_zalloc_small() void mi_free_small(void* p); +/// __v3__: Can be used to free an object that was allocated with mi_malloc_small() or +/// any allocation with a size < MI_SMALL_SIZE_MAX() +/// @param p Pointer that was returned from #mi_malloc_small et al. Cannot be \a NULL ! +/// @details +/// This function is meant for use in run-time systems for best +/// performance and does not check if the pointer was _indeed_ allocated +/// as a small object or not NULL -- use with care! +/// +/// @see mi_free_small() +/// @see mi_malloc_small() +/// @see mi_zalloc_small() +/// @see mi_heap_malloc_small() +/// @see mi_heap_zalloc_small() +/// @see mi_theap_malloc_small() +/// @see mi_theap_zalloc_small() +void mi_free_small_nonnull(void* p); + /// \} @@ -1406,9 +1443,6 @@ void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize); /// /// \{ -/// Checked `free`: just as `mi_free` but always checks if the pointer `p` belongs to our heap -/// and is safe to use when a pointer might be invalid or allocated by another allocator. -void mi_cfree(void* p); void* mi__expand(void* p, size_t newsize); size_t mi_malloc_size(const void* p); @@ -1437,15 +1471,25 @@ int mi_reallocarr(void* p, size_t count, size_t size); void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment); void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset); -/// @brief Free an object that was allocated with a known size. +/// @brief Free a block that was allocated with a known size. /// @param p The pointer that was allocated with \a size bytes (or \a NULL ) /// @param size The size in bytes. -/// __v3__: this can improve performance for objects with a \size < MI_SMALL_SIZE_MAX() +/// __v3__: this can improve performance for objects with a \a size < MI_SMALL_SIZE_MAX() /// as it will use mi_free_small() internally. Always use this if possible. /// @see mi_free_tp() /// @see mi_free_csize() +/// @see mi_free_size_nonnull() void mi_free_size(void* p, size_t size); +/// @brief Free a non-NULL pointer that was allocated with a known size. +/// @param p The pointer that was allocated with \a size bytes (cannot be \a NULL !) +/// @param size The size in bytes. +/// __v3__: this can improve performance for objects with a \a size < MI_SMALL_SIZE_MAX() +/// as it will use mi_free_small_nonnull() internally. Always use this if possible. +/// @see mi_free_tp() +/// @see mi_free_csize() +void mi_free_size_nonnull(void* p, size_t size); + /// @brief Free an object that was allocated aligned. /// @param p The pointer to the object (or \a NULL ) /// @param size The size of the object as allocated. diff --git a/doc/release-notes.md b/doc/release-notes.md index 1e59f4a53..867484268 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -1,7 +1,7 @@ Binary and source releases for mimalloc. -- __v3__: recommended: latest mimalloc design that tends to use less memory then v2 but should perform similarly. -- __v2__: stable: most widely used. +- __v3__: recommended: latest mimalloc design that tends to use less memory then v2. +- __v2__: stable legacy. - __v1__: legacy. Notes: @@ -10,6 +10,6 @@ Notes: - Generally it is recommended to download sources (or use `vcpkg` etc.) and build mimalloc as part of your project. - Source releases can also be downloaded directly from github by the tag. - For example . + For example . - Binary releases include a release-, debug-, and secure build. - Linux binaries are built on Ubuntu 22. diff --git a/docs/annotated.html b/docs/annotated.html index 87d0fb2f6..cd3d28680 100644 --- a/docs/annotated.html +++ b/docs/annotated.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/bench.html b/docs/bench.html index 33217d900..e365f4a82 100644 --- a/docs/bench.html +++ b/docs/bench.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/build.html b/docs/build.html index 78e1cc335..2de2ebca0 100644 --- a/docs/build.html +++ b/docs/build.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/classes.html b/docs/classes.html index acb1e25d5..6ea4e4725 100644 --- a/docs/classes.html +++ b/docs/classes.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/doxygen_crawl.html b/docs/doxygen_crawl.html index aeea689b4..3c2434f1e 100644 --- a/docs/doxygen_crawl.html +++ b/docs/doxygen_crawl.html @@ -98,6 +98,7 @@ + @@ -113,6 +114,7 @@ + @@ -174,6 +176,7 @@ + @@ -250,9 +253,9 @@ - + diff --git a/docs/environment.html b/docs/environment.html index 4aafebc69..be9356a2f 100644 --- a/docs/environment.html +++ b/docs/environment.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/functions.html b/docs/functions.html index ff9b06858..fcb66777d 100644 --- a/docs/functions.html +++ b/docs/functions.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/functions_vars.html b/docs/functions_vars.html index caa47689a..72a3e8398 100644 --- a/docs/functions_vars.html +++ b/docs/functions_vars.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/group__aligned.html b/docs/group__aligned.html index 3de1f8731..43ab98c71 100644 --- a/docs/group__aligned.html +++ b/docs/group__aligned.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/group__analysis.html b/docs/group__analysis.html index 62d1187be..5cb834f2f 100644 --- a/docs/group__analysis.html +++ b/docs/group__analysis.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/group__arenas.html b/docs/group__arenas.html index b8fef1965..e9cf0a6c5 100644 --- a/docs/group__arenas.html +++ b/docs/group__arenas.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/group__constantsize.html b/docs/group__constantsize.html index dc7e32ef6..5be895f99 100644 --- a/docs/group__constantsize.html +++ b/docs/group__constantsize.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
@@ -110,13 +110,13 @@
-

Constant size allocation and freeing For example: +

Inlined definitions for onstant size allocation and freeing. More...

- + @@ -125,14 +125,17 @@ - + + + +

Functions

void * mi_malloc_csize (size_t size)
 Allocate a constant size object.
 v3: Allocate a constant size object.
 
void * mi_zalloc_csize (size_t size)
 
void * mi_theap_zalloc_csize (mi_theap_t *theap, size_t size)
 
void mi_free_csize (void *p, size_t size)
 Free a pointer with a known constant size.
 v3: Free a pointer with a known constant size.
 
void mi_free_csize_nonnull (void *p, size_t size)
 v3: Free a non-null pointer with a known constant size.
 

Detailed Description

-

Constant size allocation and freeing For example:

-
long* p = (long*)mi_malloc_csize(sizeof(long))
-
mi_malloc_csize
void * mi_malloc_csize(size_t size)
Allocate a constant size object.
-

On v3 these can improve performance as they will statically use mi_malloc_small(), mi_free_small(), etc. when possible. This is especially useful for compilers and runtimes where the the size is often statically known. These functions can be used even when the size is not constant but then there is no performance benefit.

+

Inlined definitions for onstant size allocation and freeing.

+

If the size is constant, it can pick statically between mi_malloc() and mi_malloc_small(). For example:

long* p = (long*)mi_malloc_csize(sizeof(long))
+
mi_malloc_csize
void * mi_malloc_csize(size_t size)
v3: Allocate a constant size object.
+

On v3 these can improve performance. This is especially useful for compilers and runtimes where the the size is often statically known. These functions can be used even when the size is not constant but then there is no performance benefit.

Function Documentation

◆ mi_free_csize()

@@ -153,17 +156,54 @@

-

Free a pointer with a known constant size.

+

v3: Free a pointer with a known constant size.

Parameters
- +
pThe pointer to the allocated block (or NULL )
sizeThe (compile time constant) size with which the pointer was allocated.
sizeThe size with which the pointer was allocated.
-
See also
mi_malloc_csize()
+
See also
mi_malloc_csize()
-mi_free_size()
+mi_free_size() +
+mi_free_size_nonnull()
+ +

+
+ +

◆ mi_free_csize_nonnull()

+ +
+
+ + + + + + + + + + + +
void mi_free_csize_nonnull (void * p,
size_t size )
+
+ +

v3: Free a non-null pointer with a known constant size.

+
Parameters
+ + + +
pThe pointer to the allocated block (cannot be NULL !)
sizeThe size with which the pointer was allocated.
+
+
+
See also
mi_malloc_csize()
+
+mi_free_size()
+
+mi_free_size_nonnull()
@@ -182,7 +222,7 @@

-

Allocate a constant size object.

+

v3: Allocate a constant size object.

Parameters
@@ -190,7 +230,7 @@

Returns
A pointer to an allocated block of size size bytes.
-
See also
mi_free_csize()
+
See also
mi_free_csize()
mi_malloc()
diff --git a/docs/group__constantsize.js b/docs/group__constantsize.js index bfaeb70ff..a3b6430e1 100644 --- a/docs/group__constantsize.js +++ b/docs/group__constantsize.js @@ -1,6 +1,7 @@ var group__constantsize = [ [ "mi_free_csize", "group__constantsize.html#ga7a421e302074e598527c3eb052a6a3c5", null ], + [ "mi_free_csize_nonnull", "group__constantsize.html#ga38a93d373a9dc8edc8bca83edb6af5c5", null ], [ "mi_malloc_csize", "group__constantsize.html#ga9cd44d925d34f7180922e12bbf594571", null ], [ "mi_theap_malloc_csize", "group__constantsize.html#ga105bd667da91a15d113fe79be04d625c", null ], [ "mi_theap_zalloc_csize", "group__constantsize.html#ga1addf755829380581dd9c3c9175e376f", null ], diff --git a/docs/group__cpp.html b/docs/group__cpp.html index 42c217152..ddc0ac046 100644 --- a/docs/group__cpp.html +++ b/docs/group__cpp.html @@ -34,7 +34,7 @@

+ + +
sizeA (compile-time) constant size.
-
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/group__extended.html b/docs/group__extended.html index e4eb5ef4b..7d3c8c67c 100644 --- a/docs/group__extended.html +++ b/docs/group__extended.html @@ -34,7 +34,7 @@
-
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
@@ -171,6 +171,9 @@
void mi_free_small (void *p)
 v3: Can be used to free an object that was allocated with mi_malloc_small() or any allocation with a size < MI_SMALL_SIZE_MAX()
 
void mi_free_small_nonnull (void *p)
 v3: Can be used to free an object that was allocated with mi_malloc_small() or any allocation with a size < MI_SMALL_SIZE_MAX()
 

Detailed Description

Internal functionality.

@@ -310,12 +313,53 @@

mi_malloc_small() or any allocation with a size < MI_SMALL_SIZE_MAX()

Parameters
- +
pPointer that was returned from mi_malloc_small et al.
pPointer that was returned from mi_malloc_small et al. (or NULL )

This function is meant for use in run-time systems for best performance and does not check if the pointer was indeed allocated as a small object – use with care!

-
See also
mi_malloc_small()
+
See also
mi_free_small_nonnull()
+
+mi_malloc_small()
+
+mi_zalloc_small()
+
+mi_heap_malloc_small()
+
+mi_heap_zalloc_small()
+
+mi_theap_malloc_small()
+
+mi_theap_zalloc_small()
+ +

+
+ +

◆ mi_free_small_nonnull()

+ +
+
+ + + + + + + +
void mi_free_small_nonnull (void * p)
+
+ +

v3: Can be used to free an object that was allocated with mi_malloc_small() or any allocation with a size < MI_SMALL_SIZE_MAX()

+
Parameters
+ + +
pPointer that was returned from mi_malloc_small et al. Cannot be NULL !
+
+
+

This function is meant for use in run-time systems for best performance and does not check if the pointer was indeed allocated as a small object or not NULL – use with care!

+
See also
mi_free_small()
+
+mi_malloc_small()
mi_zalloc_small()
diff --git a/docs/group__extended.js b/docs/group__extended.js index dd2242214..2934ab5be 100644 --- a/docs/group__extended.js +++ b/docs/group__extended.js @@ -6,6 +6,7 @@ var group__extended = [ "mi_output_fun", "group__extended.html#gadf31cea7d0332a81c8b882cbbdbadb8d", null ], [ "mi_collect", "group__extended.html#ga421430e2226d7d468529cec457396756", null ], [ "mi_free_small", "group__extended.html#ga2a8469764c38dcaf878f36e5f50e3610", null ], + [ "mi_free_small_nonnull", "group__extended.html#ga22890f54e5be433837069c8e97fdf379", null ], [ "mi_is_redirected", "group__extended.html#gaad25050b19f30cd79397b227e0157a3f", null ], [ "mi_malloc_small", "group__extended.html#ga7f050bc6b897da82692174f5fce59cde", null ], [ "mi_register_deferred_free", "group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece", null ], diff --git a/docs/group__heap.html b/docs/group__heap.html index 50a85dc5c..2d06f93fd 100644 --- a/docs/group__heap.html +++ b/docs/group__heap.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/group__malloc.html b/docs/group__malloc.html index 6b7796979..50f3615c3 100644 --- a/docs/group__malloc.html +++ b/docs/group__malloc.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
@@ -118,6 +118,9 @@ void mi_free (void *p)  Free previously allocated memory.
  +void mi_cfree (void *p) + Checked free: just as mi_free() but always checks if the pointer p belongs to our heap.
+  void * mi_malloc (size_t size)  Allocate size bytes.
  @@ -191,6 +194,31 @@

Returns
pointer to the allocated memory of size*count bytes, or NULL if either out of memory or when count*size overflows.

Returns a unique pointer if called with either size or count of 0.

See also
mi_zalloc()
+
+
+ +

◆ mi_cfree()

+ +
+
+ + + + + + + +
void mi_cfree (void * p)
+
+ +

Checked free: just as mi_free() but always checks if the pointer p belongs to our heap.

+
Parameters
+ + +
ppointer to a block that is freed, or any other arbitrary pointer. This is safe to use when a pointer might be invalid or allocated by another allocator (in which case the pointer is ignored and not freed).
+
+
+
@@ -242,10 +270,11 @@

Parameters
- +
ppointer to free, or NULL.
ppointer to the block that is freed, or NULL. See also mi_cfree() for a checked free that validates the pointer before free-ing.
+
See also
mi_cfree()

diff --git a/docs/group__malloc.js b/docs/group__malloc.js index 9fd99dfe4..28d281767 100644 --- a/docs/group__malloc.js +++ b/docs/group__malloc.js @@ -1,6 +1,7 @@ var group__malloc = [ [ "mi_calloc", "group__malloc.html#ga6686568014b54d1e6c7ac64a076e4f56", null ], + [ "mi_cfree", "group__malloc.html#ga705dc7a64bffacfeeb0141501a5c35d7", null ], [ "mi_expand", "group__malloc.html#ga19299856216cfbb08e2628593654dfb0", null ], [ "mi_free", "group__malloc.html#gaf2c7b89c327d1f60f59e68b9ea644d95", null ], [ "mi_good_size", "group__malloc.html#gac057927cd06c854b45fe7847e921bd47", null ], diff --git a/docs/group__options.html b/docs/group__options.html index 2824760be..5ad478ad4 100644 --- a/docs/group__options.html +++ b/docs/group__options.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/group__posix.html b/docs/group__posix.html index 96a40e751..e97ba3f22 100644 --- a/docs/group__posix.html +++ b/docs/group__posix.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
@@ -115,9 +115,6 @@ - - - @@ -157,8 +154,11 @@ - + + + + @@ -301,25 +301,6 @@

- - - -

◆ mi_cfree()

- -
-
-

Functions

void mi_cfree (void *p)
 Checked free: just as mi_free but always checks if the pointer p belongs to our heap and is safe to use when a pointer might be invalid or allocated by another allocator.
 
void * mi__expand (void *p, size_t newsize)
 
size_t mi_malloc_size (const void *p)
void * mi_aligned_offset_recalloc (void *p, size_t newcount, size_t size, size_t alignment, size_t offset)
 
void mi_free_size (void *p, size_t size)
 Free an object that was allocated with a known size.
 Free a block that was allocated with a known size.
 
void mi_free_size_nonnull (void *p, size_t size)
 Free a non-NULL pointer that was allocated with a known size.
 
void mi_free_size_aligned (void *p, size_t size, size_t alignment)
 Free an object that was allocated aligned.
 
- - - - - - -
void mi_cfree (void * p)
-
- -

Checked free: just as mi_free but always checks if the pointer p belongs to our heap and is safe to use when a pointer might be invalid or allocated by another allocator.

-
@@ -388,17 +369,19 @@

-

Free an object that was allocated with a known size.

+

Free a block that was allocated with a known size.

Parameters
- +
pThe pointer that was allocated with size bytes (or NULL )
sizeThe size in bytes. v3: this can improve performance for objects with a \size < MI_SMALL_SIZE_MAX() as it will use mi_free_small() internally. Always use this if possible.
sizeThe size in bytes. v3: this can improve performance for objects with a size < MI_SMALL_SIZE_MAX() as it will use mi_free_small() internally. Always use this if possible.
See also
mi_free_tp()
-mi_free_csize()
+mi_free_csize() +
+mi_free_size_nonnull()

@@ -431,11 +414,44 @@

pThe pointer to the object (or NULL ) sizeThe size of the object as allocated. - alignmentThe requested alignment at the allocation. Currently just defers to mi_free_size() + alignmentThe requested alignment at the allocation. Currently just defers to mi_free_size() +

+
+ +

◆ mi_free_size_nonnull()

+ +
+
+ + + + + + + + + + + +
void mi_free_size_nonnull (void * p,
size_t size )
+
+ +

Free a non-NULL pointer that was allocated with a known size.

+
Parameters
+ + + +
pThe pointer that was allocated with size bytes (cannot be NULL !)
sizeThe size in bytes. v3: this can improve performance for objects with a size < MI_SMALL_SIZE_MAX() as it will use mi_free_small_nonnull() internally. Always use this if possible.
+
+
+
See also
mi_free_tp()
+
+mi_free_csize()
+
diff --git a/docs/group__posix.js b/docs/group__posix.js index ea4ed269e..fa782adcb 100644 --- a/docs/group__posix.js +++ b/docs/group__posix.js @@ -5,11 +5,11 @@ var group__posix = [ "mi_aligned_alloc", "group__posix.html#ga430ed1513f0571ff83be00ec58a98ee0", null ], [ "mi_aligned_offset_recalloc", "group__posix.html#ga16570deddd559001b44953eedbad0084", null ], [ "mi_aligned_recalloc", "group__posix.html#gaf82cbb4b4f24acf723348628451798d3", null ], - [ "mi_cfree", "group__posix.html#ga705dc7a64bffacfeeb0141501a5c35d7", null ], [ "mi_dupenv_s", "group__posix.html#gab41369c1a1da7504013a7a0b1d4dd958", null ], [ "mi_free_aligned", "group__posix.html#ga0d28d5cf61e6bfbb18c63092939fe5c9", null ], [ "mi_free_size", "group__posix.html#gae01389eedab8d67341ff52e2aad80ebb", null ], [ "mi_free_size_aligned", "group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc", null ], + [ "mi_free_size_nonnull", "group__posix.html#ga796602dae2a89797026f72e0c0240f32", null ], [ "mi_malloc_good_size", "group__posix.html#ga9d23ac7885fed7413c11d8e0ffa31071", null ], [ "mi_malloc_size", "group__posix.html#ga4531c9e775bb3ae12db57c1ba8a5d7de", null ], [ "mi_malloc_usable_size", "group__posix.html#ga06d07cf357bbac5c73ba5d0c0c421e17", null ], diff --git a/docs/group__stats.html b/docs/group__stats.html index 529d18d89..2555bcac6 100644 --- a/docs/group__stats.html +++ b/docs/group__stats.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
@@ -260,7 +260,7 @@

if (mi_stats_get(&stats)) {

...
}
-
mi_stats_t_decl
#define mi_stats_t_decl(name)
Helper to declare a properly initialized mi_stats_t() local variable as name where the size and versi...
Definition mimalloc-doc.h:1099
+
mi_stats_t_decl
#define mi_stats_t_decl(name)
Helper to declare a properly initialized mi_stats_t() local variable as name where the size and versi...
Definition mimalloc-doc.h:1136
mi_stats_get
bool mi_stats_get(mi_stats_t *stats)
Get the statistics for the current subprocess aggregated over all its heaps.
diff --git a/docs/group__subproc.html b/docs/group__subproc.html index f3830c87f..88e1e8976 100644 --- a/docs/group__subproc.html +++ b/docs/group__subproc.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/group__theap.html b/docs/group__theap.html index e8abeb2c7..f84216e6a 100644 --- a/docs/group__theap.html +++ b/docs/group__theap.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/group__typed.html b/docs/group__typed.html index bbd77cafb..4b16d9c36 100644 --- a/docs/group__typed.html +++ b/docs/group__typed.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
@@ -155,7 +155,7 @@

Detailed Description

Typed allocation macros.

For example:

int* p = mi_malloc_tp(int)
-
mi_malloc_tp
#define mi_malloc_tp(tp)
Allocate a block of type tp.
Definition mimalloc-doc.h:364
+
mi_malloc_tp
#define mi_malloc_tp(tp)
Allocate a block of type tp.
Definition mimalloc-doc.h:373

On v3 these can improve performance as they use the mi_<malloc>_csize api's that in turn can use mi_malloc_small(), mi_free_small(), etc.

Macro Definition Documentation

@@ -204,11 +204,11 @@

Parameters
- +
tpThe type of the object that is freed.
pA pointer to an object of type tp (or NULL ) Can be more efficient as it internally uses mi_free_csize()
pA pointer to an object of type tp (or NULL ) Can be more efficient as it internally uses mi_free_csize()
-
See also
mi_free_csize()
+
See also
mi_free_csize()
mi_free_small()
@@ -406,7 +406,7 @@

Returns
A pointer to an object of type tp, or NULL if out of memory.

Example:

int* p = mi_malloc_tp(int)
-

Using this is more safe, but can also be more efficient since the allocation size is a compile-time constant.

See also
mi_malloc_csize()
+

Using this is more safe, but can also be more efficient since the allocation size is a compile-time constant.

See also
mi_malloc_csize()
mi_malloc_small()
diff --git a/docs/group__zeroinit.html b/docs/group__zeroinit.html index ecfa05d49..fde3d387d 100644 --- a/docs/group__zeroinit.html +++ b/docs/group__zeroinit.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/index.html b/docs/index.html index 25354e879..ba454f3e5 100644 --- a/docs/index.html +++ b/docs/index.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/mimalloc-doc_8h_source.html b/docs/mimalloc-doc_8h_source.html index 37d74e056..b02dec412 100644 --- a/docs/mimalloc-doc_8h_source.html +++ b/docs/mimalloc-doc_8h_source.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
@@ -119,531 +119,538 @@
10
123
124
-
128void mi_free(void* p);
-
129
-
134void* mi_malloc(size_t size);
-
135
-
140void* mi_zalloc(size_t size);
-
141
-
151void* mi_calloc(size_t count, size_t size);
-
152
-
166void* mi_realloc(void* p, size_t newsize);
-
167
-
181void* mi_expand(void* p, size_t newsize);
-
182
-
192void* mi_mallocn(size_t count, size_t size);
-
193
-
203void* mi_reallocn(void* p, size_t count, size_t size);
-
204
-
221void* mi_reallocf(void* p, size_t newsize);
-
222
-
223
-
232char* mi_strdup(const char* s);
-
233
-
243char* mi_strndup(const char* s, size_t n);
-
244
-
257char* mi_realpath(const char* fname, char* resolved_name);
-
258
-
272size_t mi_usable_size(void* p);
-
273
-
283size_t mi_good_size(size_t size);
-
284
-
286
-
287
-
288// ------------------------------------------------------
-
289// Aligned allocation
-
290// ------------------------------------------------------
-
291
-
299
-
317void* mi_malloc_aligned(size_t size, size_t alignment);
-
318void* mi_zalloc_aligned(size_t size, size_t alignment);
-
319void* mi_calloc_aligned(size_t count, size_t size, size_t alignment);
-
320void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment);
-
321
-
333void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset);
-
334void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset);
-
335void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset);
-
336void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset);
-
337
-
339
-
340
-
350
-
364#define mi_malloc_tp(tp) ((tp*)mi_malloc_csize(sizeof(tp)))
-
365
-
372#define mi_free_tp(tp,p) (mi_free_csize(p,sizeof(tp)))
-
373
-
375#define mi_zalloc_tp(tp) ((tp*)mi_zalloc_csize(sizeof(tp)))
-
376
-
378#define mi_calloc_tp(tp,count) ((tp*)mi_calloc(count,sizeof(tp)))
-
379
-
381#define mi_mallocn_tp(tp,count) ((tp*)mi_mallocn(count,sizeof(tp)))
+
131void mi_free(void* p);
+
132
+
137void mi_cfree(void* p);
+
138
+
143void* mi_malloc(size_t size);
+
144
+
149void* mi_zalloc(size_t size);
+
150
+
160void* mi_calloc(size_t count, size_t size);
+
161
+
175void* mi_realloc(void* p, size_t newsize);
+
176
+
190void* mi_expand(void* p, size_t newsize);
+
191
+
201void* mi_mallocn(size_t count, size_t size);
+
202
+
212void* mi_reallocn(void* p, size_t count, size_t size);
+
213
+
230void* mi_reallocf(void* p, size_t newsize);
+
231
+
232
+
241char* mi_strdup(const char* s);
+
242
+
252char* mi_strndup(const char* s, size_t n);
+
253
+
266char* mi_realpath(const char* fname, char* resolved_name);
+
267
+
281size_t mi_usable_size(void* p);
+
282
+
292size_t mi_good_size(size_t size);
+
293
+
295
+
296
+
297// ------------------------------------------------------
+
298// Aligned allocation
+
299// ------------------------------------------------------
+
300
+
308
+
326void* mi_malloc_aligned(size_t size, size_t alignment);
+
327void* mi_zalloc_aligned(size_t size, size_t alignment);
+
328void* mi_calloc_aligned(size_t count, size_t size, size_t alignment);
+
329void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment);
+
330
+
342void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset);
+
343void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset);
+
344void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset);
+
345void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset);
+
346
+
348
+
349
+
359
+
373#define mi_malloc_tp(tp) ((tp*)mi_malloc_csize(sizeof(tp)))
+
374
+
381#define mi_free_tp(tp,p) (mi_free_csize(p,sizeof(tp)))
382
-
384#define mi_reallocn_tp(p,tp,count) ((tp*)mi_reallocn(p,count,sizeof(tp)))
+
384#define mi_zalloc_tp(tp) ((tp*)mi_zalloc_csize(sizeof(tp)))
385
-
387#define mi_heap_malloc_tp(tp,hp) ((tp*)mi_heap_malloc(hp,sizeof(tp)))
+
387#define mi_calloc_tp(tp,count) ((tp*)mi_calloc(count,sizeof(tp)))
388
-
390#define mi_heap_zalloc_tp(tp,hp) ((tp*)mi_heap_zalloc(hp,sizeof(tp)))
+
390#define mi_mallocn_tp(tp,count) ((tp*)mi_mallocn(count,sizeof(tp)))
391
-
393#define mi_heap_calloc_tp(tp,hp,count) ((tp*)mi_heap_calloc(hp,count,sizeof(tp)))
+
393#define mi_reallocn_tp(p,tp,count) ((tp*)mi_reallocn(p,count,sizeof(tp)))
394
-
396#define mi_heap_mallocn_tp(tp,hp,count) ((tp*)mi_heap_mallocn(hp,count,sizeof(tp)))
+
396#define mi_heap_malloc_tp(tp,hp) ((tp*)mi_heap_malloc(hp,sizeof(tp)))
397
-
399#define mi_heap_reallocn_tp(tp,hp,p,count) ((tp*)mi_heap_reallocn(p,count,sizeof(tp)))
+
399#define mi_heap_zalloc_tp(tp,hp) ((tp*)mi_heap_zalloc(hp,sizeof(tp)))
400
-
402#define mi_heap_recalloc_tp(tp,hp,p,count) ((tp*)mi_heap_recalloc(p,count,sizeof(tp)))
+
402#define mi_heap_calloc_tp(tp,hp,count) ((tp*)mi_heap_calloc(hp,count,sizeof(tp)))
403
-
405
-
418
-
424void* mi_malloc_csize(size_t size);
-
425
-
426void* mi_zalloc_csize(size_t size);
-
427
-
428void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size);
-
429
-
430void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size);
-
431
-
437void mi_free_csize(void* p, size_t size);
-
438
-
440
-
450
-
461void* mi_recalloc(void* p, size_t count, size_t size);
-
462
-
463void* mi_rezalloc(void* p, size_t newsize);
-
464void* mi_recalloc(void* p, size_t newcount, size_t size) ;
-
465
-
466void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment);
-
467void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset);
-
468void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment);
-
469void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
-
470
-
471void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize);
-
472void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size);
-
473
-
474void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment);
-
475void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset);
-
476void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment);
-
477void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
-
478
-
480
+
405#define mi_heap_mallocn_tp(tp,hp,count) ((tp*)mi_heap_mallocn(hp,count,sizeof(tp)))
+
406
+
408#define mi_heap_reallocn_tp(tp,hp,p,count) ((tp*)mi_heap_reallocn(p,count,sizeof(tp)))
+
409
+
411#define mi_heap_recalloc_tp(tp,hp,p,count) ((tp*)mi_heap_recalloc(p,count,sizeof(tp)))
+
412
+
414
+
428
+
434void* mi_malloc_csize(size_t size);
+
435
+
436void* mi_zalloc_csize(size_t size);
+
437
+
438void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size);
+
439
+
440void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size);
+
441
+
448void mi_free_csize(void* p, size_t size);
+
449
+
456void mi_free_csize_nonnull(void* p, size_t size);
+
457
+
459
+
469
+
480void* mi_recalloc(void* p, size_t count, size_t size);
481
-
482
+
482void* mi_rezalloc(void* p, size_t newsize);
+
483void* mi_recalloc(void* p, size_t newcount, size_t size) ;
+
484
+
485void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment);
+
486void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset);
+
487void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment);
+
488void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
+
489
+
490void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize);
+
491void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size);
+
492
+
493void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment);
+
494void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset);
+
495void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment);
+
496void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
+
497
+
499
500
-
506struct mi_heap_s;
-
507
-
513typedef struct mi_heap_s mi_heap_t;
-
514
-
516mi_heap_t* mi_heap_new();
-
517
-
524void mi_heap_delete(mi_heap_t* heap);
-
525
-
532void mi_heap_destroy(mi_heap_t* heap);
+
501
+
519
+
525struct mi_heap_s;
+
526
+
532typedef struct mi_heap_s mi_heap_t;
533
-
538mi_heap_t* mi_heap_set_default(mi_heap_t* heap);
-
539
-
543mi_heap_t* mi_heap_get_default();
+
535mi_heap_t* mi_heap_new();
+
536
+
543void mi_heap_delete(mi_heap_t* heap);
544
-
551mi_heap_t* mi_heap_get_backing();
+
551void mi_heap_destroy(mi_heap_t* heap);
552
-
556void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node);
-
557
-
563mi_heap_t* mi_heap_main(void);
-
564
-
570mi_heap_t* mi_heap_of(const void* p);
+
557mi_heap_t* mi_heap_set_default(mi_heap_t* heap);
+
558
+
562mi_heap_t* mi_heap_get_default();
+
563
+
570mi_heap_t* mi_heap_get_backing();
571
-
576bool mi_heap_contains(const mi_heap_t* heap, const void* p);
-
577
-
581bool mi_any_heap_contains(const void* p);
-
582
+
575void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node);
+
576
+
582mi_heap_t* mi_heap_main(void);
583
-
589bool mi_is_in_heap_region(const void* p);
+
589mi_heap_t* mi_heap_of(const void* p);
590
-
596bool mi_check_owned(const void* p);
-
597
-
604bool mi_heap_contains_block(mi_heap_t* heap, const void* p);
-
605
-
612bool mi_heap_check_owned(mi_heap_t* heap, const void* p);
-
613
-
615void mi_heap_collect(mi_heap_t* heap, bool force);
+
595bool mi_heap_contains(const mi_heap_t* heap, const void* p);
+
596
+
600bool mi_any_heap_contains(const void* p);
+
601
+
602
+
608bool mi_is_in_heap_region(const void* p);
+
609
+
615bool mi_check_owned(const void* p);
616
-
619void* mi_heap_malloc(mi_heap_t* heap, size_t size);
-
620
-
624void* mi_heap_malloc_small(mi_heap_t* heap, size_t size);
-
625
-
628void* mi_heap_zalloc(mi_heap_t* heap, size_t size);
-
629
-
633void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size);
-
634
-
637void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size);
-
638
-
641void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size);
-
642
-
645char* mi_heap_strdup(mi_heap_t* heap, const char* s);
-
646
-
649char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n);
-
650
-
653char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name);
-
654
-
655void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize);
-
656void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size);
-
657void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize);
-
658
-
659void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment);
-
660void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset);
-
661void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment);
-
662void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset);
-
663void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment);
-
664void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset);
-
665void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment);
-
666void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset);
-
667
+
623bool mi_heap_contains_block(mi_heap_t* heap, const void* p);
+
624
+
631bool mi_heap_check_owned(mi_heap_t* heap, const void* p);
+
632
+
634void mi_heap_collect(mi_heap_t* heap, bool force);
+
635
+
638void* mi_heap_malloc(mi_heap_t* heap, size_t size);
+
639
+
643void* mi_heap_malloc_small(mi_heap_t* heap, size_t size);
+
644
+
647void* mi_heap_zalloc(mi_heap_t* heap, size_t size);
+
648
+
652void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size);
+
653
+
656void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size);
+
657
+
660void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size);
+
661
+
664char* mi_heap_strdup(mi_heap_t* heap, const char* s);
+
665
+
668char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n);
669
-
670
-
671
-
676
+
672char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name);
+
673
+
674void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize);
+
675void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size);
+
676void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize);
677
-
-
680typedef struct mi_heap_area_s {
-
681 void* blocks;
-
682 size_t reserved;
-
683 size_t committed;
-
684 size_t used;
-
685 size_t block_size;
-
686 size_t full_block_size;
-
687 int heap_tag;
-
688} mi_heap_area_t;
-
+
678void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment);
+
679void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset);
+
680void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment);
+
681void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset);
+
682void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment);
+
683void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset);
+
684void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment);
+
685void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset);
+
686
+
688
689
-
697typedef bool (mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);
-
698
-
713bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
-
714
-
730bool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
-
731
+
690
+
695
+
696
+
+
699typedef struct mi_heap_area_s {
+
700 void* blocks;
+
701 size_t reserved;
+
702 size_t committed;
+
703 size_t used;
+
704 size_t block_size;
+
705 size_t full_block_size;
+
706 int heap_tag;
+
707} mi_heap_area_t;
+
+
708
+
716typedef bool (mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);
+
717
+
732bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
733
-
734// ------------------------------------------------------
-
735// Arenas
-
736// ------------------------------------------------------
-
737
-
745
-
747typedef int mi_arena_id_t;
-
748
-
757int mi_reserve_os_memory(size_t size, bool commit, bool allow_large);
-
758
-
766int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id);
+
749bool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
+
750
+
752
+
753// ------------------------------------------------------
+
754// Arenas
+
755// ------------------------------------------------------
+
756
+
764
+
766typedef int mi_arena_id_t;
767
-
780int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs);
-
781
-
794int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs);
-
795
-
803int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id);
-
804
-
807size_t mi_arena_min_alignment(void);
-
808
-
820bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node);
-
821
-
832bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id);
-
833
-
834
-
836void mi_debug_show_arenas(void);
-
837
-
842void* mi_arena_area(mi_arena_id_t arena_id, size_t* size);
-
843
-
844
-
845
-
849mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id);
-
850
-
862mi_heap_t* mi_heap_new_ex(int heap_tag, bool allow_destroy, mi_arena_id_t arena_id);
+
776int mi_reserve_os_memory(size_t size, bool commit, bool allow_large);
+
777
+
785int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id);
+
786
+
799int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs);
+
800
+
813int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs);
+
814
+
822int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id);
+
823
+
826size_t mi_arena_min_alignment(void);
+
827
+
839bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node);
+
840
+
851bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id);
+
852
+
853
+
855void mi_debug_show_arenas(void);
+
856
+
861void* mi_arena_area(mi_arena_id_t arena_id, size_t* size);
+
862
863
-
865
-
866
-
875
-
880typedef void* mi_subproc_id_t;
-
881
-
883mi_subproc_id_t mi_subproc_main(void);
+
864
+
868mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id);
+
869
+
881mi_heap_t* mi_heap_new_ex(int heap_tag, bool allow_destroy, mi_arena_id_t arena_id);
+
882
884
-
886mi_subproc_id_t mi_subproc_current(void);
-
887
-
890mi_subproc_id_t mi_subproc_new(void);
-
891
-
895void mi_subproc_destroy(mi_subproc_id_t subproc);
-
896
-
901void mi_subproc_delete(mi_subproc_id_t subproc);
-
902
-
906void mi_subproc_add_current_thread(mi_subproc_id_t subproc);
-
907
-
911typedef bool (mi_heap_visit_fun)(mi_heap_t* heap, void* arg);
-
912
-
918bool mi_subproc_visit_heaps(mi_subproc_id_t subproc, mi_heap_visit_fun* visitor, void* arg);
-
919
+
885
+
894
+
899typedef void* mi_subproc_id_t;
+
900
+
902mi_subproc_id_t mi_subproc_main(void);
+
903
+
905mi_subproc_id_t mi_subproc_current(void);
+
906
+
909mi_subproc_id_t mi_subproc_new(void);
+
910
+
914void mi_subproc_destroy(mi_subproc_id_t subproc);
+
915
+
920void mi_subproc_delete(mi_subproc_id_t subproc);
921
-
922
-
923// ------------------------------------------------------
-
924// Extended functionality
-
925// ------------------------------------------------------
+
925void mi_subproc_add_current_thread(mi_subproc_id_t subproc);
926
+
930typedef bool (mi_heap_visit_fun)(mi_heap_t* heap, void* arg);
931
-
935#define MI_SMALL_SIZE_MAX (128*sizeof(void*))
-
936
-
940int mi_version(void);
+
937bool mi_subproc_visit_heaps(mi_subproc_id_t subproc, mi_heap_visit_fun* visitor, void* arg);
+
938
+
940
941
-
949void mi_collect(bool force);
+
942// ------------------------------------------------------
+
943// Extended functionality
+
944// ------------------------------------------------------
+
945
950
-
956void mi_thread_set_in_threadpool(void);
-
957
-
962bool mi_is_redirected();
-
963
-
967void mi_thread_init(void);
-
968
-
973void mi_thread_done(void);
-
974
-
981typedef void (mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg);
+
954#define MI_SMALL_SIZE_MAX (128*sizeof(void*))
+
955
+
959int mi_version(void);
+
960
+
968void mi_collect(bool force);
+
969
+
975void mi_thread_set_in_threadpool(void);
+
976
+
981bool mi_is_redirected();
982
-
998void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg);
-
999
-
1005typedef void (mi_output_fun)(const char* msg, void* arg);
-
1006
-
1013void mi_register_output(mi_output_fun* out, void* arg);
-
1014
-
1020typedef void (mi_error_fun)(int err, void* arg);
-
1021
-
1037void mi_register_error(mi_error_fun* errfun, void* arg);
-
1038
-
1046void* mi_malloc_small(size_t size);
-
1047
-
1055void* mi_zalloc_small(size_t size);
-
1056
-
1071void mi_free_small(void* p);
-
1072
-
1074
+
986void mi_thread_init(void);
+
987
+
992void mi_thread_done(void);
+
993
+
1000typedef void (mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg);
+
1001
+
1017void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg);
+
1018
+
1024typedef void (mi_output_fun)(const char* msg, void* arg);
+
1025
+
1032void mi_register_output(mi_output_fun* out, void* arg);
+
1033
+
1039typedef void (mi_error_fun)(int err, void* arg);
+
1040
+
1056void mi_register_error(mi_error_fun* errfun, void* arg);
+
1057
+
1065void* mi_malloc_small(size_t size);
+
1066
+
1074void* mi_zalloc_small(size_t size);
1075
-
1076
-
1077
-
1078
-
1079// ------------------------------------------------------
-
1080// Statistics
-
1081// ------------------------------------------------------
-
1082
-
1086
-
1088#define MI_STAT_VERSION 4
-
1089
-
1099#define mi_stats_t_decl(name)
-
1100
-
-
1102struct mi_stats_s {
-
1104 size_t size;
-
1106 size_t version;
-
1107};
-
-
1108
-
1110typedef struct mi_stats_s mi_stats_t;
+
1091void mi_free_small(void* p);
+
1092
+
1108void mi_free_small_nonnull(void* p);
+
1109
1111
1112
-
1115void mi_process_info_print(void);
-
1116
-
1121void mi_process_info_print_out(mi_output_fun* out, void* arg);
-
1122
-
1136void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults);
+
1113
+
1114
+
1115
+
1116// ------------------------------------------------------
+
1117// Statistics
+
1118// ------------------------------------------------------
+
1119
+
1123
+
1125#define MI_STAT_VERSION 4
+
1126
+
1136#define mi_stats_t_decl(name)
1137
-
1138
-
1143void mi_stats_print(void* out);
-
1144
-
1150void mi_stats_print_out(mi_output_fun* out, void* arg);
-
1151
-
1157bool mi_stats_get(mi_stats_t* stats);
-
1158
-
1164char* mi_stats_get_json(size_t buf_size, char* buf);
-
1165
-
1167size_t mi_stats_get_bin_size(size_t bin);
-
1168
-
1175bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats);
-
1176
-
1183char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf); // use mi_free to free the result if the input buf == NULL
-
1184
-
1189void mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg);
-
1190
-
1194void mi_heap_stats_merge_to_subproc(mi_heap_t* heap);
+
+
1139struct mi_stats_s {
+
1141 size_t size;
+
1143 size_t version;
+
1144};
+
+
1145
+
1147typedef struct mi_stats_s mi_stats_t;
+
1148
+
1149
+
1152void mi_process_info_print(void);
+
1153
+
1158void mi_process_info_print_out(mi_output_fun* out, void* arg);
+
1159
+
1173void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults);
+
1174
+
1175
+
1180void mi_stats_print(void* out);
+
1181
+
1187void mi_stats_print_out(mi_output_fun* out, void* arg);
+
1188
+
1194bool mi_stats_get(mi_stats_t* stats);
1195
-
1202bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats);
-
1203
-
1210char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf); // use mi_free to free the result if the input buf == NULL
-
1211
-
1216void mi_subproc_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg);
-
1217
-
1218
-
1223void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg);
-
1224
-
1231char* mi_stats_as_json( mi_stats_t* stats, size_t buf_size, char* buf);
+
1201char* mi_stats_get_json(size_t buf_size, char* buf);
+
1202
+
1204size_t mi_stats_get_bin_size(size_t bin);
+
1205
+
1212bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats);
+
1213
+
1220char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf); // use mi_free to free the result if the input buf == NULL
+
1221
+
1226void mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg);
+
1227
+
1231void mi_heap_stats_merge_to_subproc(mi_heap_t* heap);
1232
-
1234void mi_stats_reset(void);
-
1235
-
1237void mi_stats_merge(void);
-
1238
+
1239bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats);
1240
-
1241// ------------------------------------------------------
-
1242// Runtime Options
-
1243// ------------------------------------------------------
-
1244
-
1249
-
1252void mi_options_print(void);
-
1253
-
1259void mi_options_print_out(mi_output_fun* out, void* arg);
-
1260
+
1247char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf); // use mi_free to free the result if the input buf == NULL
+
1248
+
1253void mi_subproc_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg);
+
1254
+
1255
+
1260void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg);
1261
-
-
1263typedef enum mi_option_e {
-
1264 // stable options
-
1265 mi_option_show_errors,
-
1266 mi_option_show_stats,
-
1267 mi_option_verbose,
-
1268 mi_option_max_errors,
-
1269 mi_option_max_warnings,
-
1270
-
1271 // advanced options
-
1272 mi_option_reserve_huge_os_pages,
-
1273 mi_option_reserve_huge_os_pages_at,
-
1274 mi_option_reserve_os_memory,
-
1275 mi_option_allow_large_os_pages,
-
1276 mi_option_purge_decommits,
-
1277 mi_option_arena_reserve,
-
1278 mi_option_os_tag,
-
1279 mi_option_retry_on_oom,
-
1280 mi_option_generic_collect,
-
1281 mi_option_allow_thp,
-
1282
-
1283 // guard pages
-
1284 mi_option_guarded_min,
-
1285 mi_option_guarded_max,
-
1286 mi_option_guarded_precise,
-
1287 mi_option_guarded_sample_rate,
-
1288 mi_option_guarded_sample_seed,
-
1289
-
1290 // experimental options
-
1291 mi_option_eager_commit,
-
1292 mi_option_eager_commit_delay,
-
1293 mi_option_arena_eager_commit,
-
1294 mi_option_abandoned_page_purge,
-
1295 mi_option_purge_delay,
-
1296 mi_option_use_numa_nodes,
-
1297 mi_option_disallow_os_alloc,
-
1298 mi_option_max_segment_reclaim,
-
1299 mi_option_destroy_on_exit,
-
1300 mi_option_arena_purge_mult,
-
1301 mi_option_abandoned_reclaim_on_free,
-
1302 mi_option_purge_extend_delay,
-
1303 mi_option_disallow_arena_alloc,
-
1304 mi_option_visit_abandoned,
-
1305 mi_option_target_segments_per_thread,
-
1306
-
1307 // v3 options
-
1308 mi_option_page_reclaim_on_free,
-
1309 mi_option_page_full_retain,
-
1310 mi_option_page_max_candidates,
-
1311 mi_option_max_vabits,
-
1312 mi_option_pagemap_commit,
-
1313 mi_option_page_commit_on_demand,
-
1314 mi_option_page_max_reclaim,
-
1315 mi_option_page_cross_thread_max_reclaim,
-
1316 mi_option_minimal_purge_size,
-
1317 mi_option_arena_max_object_size,
-
1318
-
1319 _mi_option_last
-
1320} mi_option_t;
+
1268char* mi_stats_as_json( mi_stats_t* stats, size_t buf_size, char* buf);
+
1269
+
1271void mi_stats_reset(void);
+
1272
+
1274void mi_stats_merge(void);
+
1275
+
1277
+
1278// ------------------------------------------------------
+
1279// Runtime Options
+
1280// ------------------------------------------------------
+
1281
+
1286
+
1289void mi_options_print(void);
+
1290
+
1296void mi_options_print_out(mi_output_fun* out, void* arg);
+
1297
+
1298
+
+
1300typedef enum mi_option_e {
+
1301 // stable options
+
1302 mi_option_show_errors,
+
1303 mi_option_show_stats,
+
1304 mi_option_verbose,
+
1305 mi_option_max_errors,
+
1306 mi_option_max_warnings,
+
1307
+
1308 // advanced options
+
1309 mi_option_reserve_huge_os_pages,
+
1310 mi_option_reserve_huge_os_pages_at,
+
1311 mi_option_reserve_os_memory,
+
1312 mi_option_allow_large_os_pages,
+
1313 mi_option_purge_decommits,
+
1314 mi_option_arena_reserve,
+
1315 mi_option_os_tag,
+
1316 mi_option_retry_on_oom,
+
1317 mi_option_generic_collect,
+
1318 mi_option_allow_thp,
+
1319
+
1320 // guard pages
+
1321 mi_option_guarded_min,
+
1322 mi_option_guarded_max,
+
1323 mi_option_guarded_precise,
+
1324 mi_option_guarded_sample_rate,
+
1325 mi_option_guarded_sample_seed,
+
1326
+
1327 // experimental options
+
1328 mi_option_eager_commit,
+
1329 mi_option_eager_commit_delay,
+
1330 mi_option_arena_eager_commit,
+
1331 mi_option_abandoned_page_purge,
+
1332 mi_option_purge_delay,
+
1333 mi_option_use_numa_nodes,
+
1334 mi_option_disallow_os_alloc,
+
1335 mi_option_max_segment_reclaim,
+
1336 mi_option_destroy_on_exit,
+
1337 mi_option_arena_purge_mult,
+
1338 mi_option_abandoned_reclaim_on_free,
+
1339 mi_option_purge_extend_delay,
+
1340 mi_option_disallow_arena_alloc,
+
1341 mi_option_visit_abandoned,
+
1342 mi_option_target_segments_per_thread,
+
1343
+
1344 // v3 options
+
1345 mi_option_page_reclaim_on_free,
+
1346 mi_option_page_full_retain,
+
1347 mi_option_page_max_candidates,
+
1348 mi_option_max_vabits,
+
1349 mi_option_pagemap_commit,
+
1350 mi_option_page_commit_on_demand,
+
1351 mi_option_page_max_reclaim,
+
1352 mi_option_page_cross_thread_max_reclaim,
+
1353 mi_option_minimal_purge_size,
+
1354 mi_option_arena_max_object_size,
+
1355
+
1356 _mi_option_last
+
1357} mi_option_t;
-
1321
-
1322
-
1323bool mi_option_is_enabled(mi_option_t option);
-
1324void mi_option_enable(mi_option_t option);
-
1325void mi_option_disable(mi_option_t option);
-
1326void mi_option_set_enabled(mi_option_t option, bool enable);
-
1327void mi_option_set_enabled_default(mi_option_t option, bool enable);
-
1328
-
1329long mi_option_get(mi_option_t option);
-
1330long mi_option_get_clamp(mi_option_t option, long min, long max);
-
1331size_t mi_option_get_size(mi_option_t option);
-
1332
-
1333void mi_option_set(mi_option_t option, long value);
-
1334void mi_option_set_default(mi_option_t option, long value);
-
1335
-
1337
-
1350
-
1352struct mi_theap_s;
-
1353
-
1355typedef struct mi_theap_s mi_theap_t;
-
1356
-
1363mi_theap_t* mi_heap_theap(mi_heap_t* heap);
-
1364
-
1366void mi_theap_collect(mi_theap_t* theap, bool force);
-
1367
-
1372mi_theap_t* mi_theap_set_default(mi_theap_t* theap);
-
1373
-
1376mi_theap_t* mi_theap_get_default();
-
1377
-
1380void* mi_theap_malloc(mi_theap_t* theap, size_t size);
-
1381
-
1385void* mi_theap_malloc_small(mi_theap_t* theap, size_t size);
-
1386
-
1389void* mi_theap_zalloc(mi_theap_t* theap, size_t size);
+
1358
+
1359
+
1360bool mi_option_is_enabled(mi_option_t option);
+
1361void mi_option_enable(mi_option_t option);
+
1362void mi_option_disable(mi_option_t option);
+
1363void mi_option_set_enabled(mi_option_t option, bool enable);
+
1364void mi_option_set_enabled_default(mi_option_t option, bool enable);
+
1365
+
1366long mi_option_get(mi_option_t option);
+
1367long mi_option_get_clamp(mi_option_t option, long min, long max);
+
1368size_t mi_option_get_size(mi_option_t option);
+
1369
+
1370void mi_option_set(mi_option_t option, long value);
+
1371void mi_option_set_default(mi_option_t option, long value);
+
1372
+
1374
+
1387
+
1389struct mi_theap_s;
1390
-
1394void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size);
-
1395
-
1396void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size);
-
1397void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment);
-
1398void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize);
-
1399
+
1392typedef struct mi_theap_s mi_theap_t;
+
1393
+
1400mi_theap_t* mi_heap_theap(mi_heap_t* heap);
1401
-
1402
-
1408
-
1411void mi_cfree(void* p);
-
1412void* mi__expand(void* p, size_t newsize);
-
1413
-
1414size_t mi_malloc_size(const void* p);
-
1415size_t mi_malloc_good_size(size_t size);
-
1416size_t mi_malloc_usable_size(const void *p);
-
1417
-
1418int mi_posix_memalign(void** p, size_t alignment, size_t size);
-
1419int mi__posix_memalign(void** p, size_t alignment, size_t size);
-
1420void* mi_memalign(size_t alignment, size_t size);
-
1421void* mi_valloc(size_t size);
-
1422void* mi_pvalloc(size_t size);
-
1423void* mi_aligned_alloc(size_t alignment, size_t size);
-
1424
-
1425unsigned short* mi_wcsdup(const unsigned short* s);
-
1426unsigned char* mi_mbsdup(const unsigned char* s);
-
1427int mi_dupenv_s(char** buf, size_t* size, const char* name);
-
1428int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name);
-
1429
-
1432void* mi_reallocarray(void* p, size_t count, size_t size);
-
1433
-
1435int mi_reallocarr(void* p, size_t count, size_t size);
+
1403void mi_theap_collect(mi_theap_t* theap, bool force);
+
1404
+
1409mi_theap_t* mi_theap_set_default(mi_theap_t* theap);
+
1410
+
1413mi_theap_t* mi_theap_get_default();
+
1414
+
1417void* mi_theap_malloc(mi_theap_t* theap, size_t size);
+
1418
+
1422void* mi_theap_malloc_small(mi_theap_t* theap, size_t size);
+
1423
+
1426void* mi_theap_zalloc(mi_theap_t* theap, size_t size);
+
1427
+
1431void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size);
+
1432
+
1433void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size);
+
1434void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment);
+
1435void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize);
1436
-
1437void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment);
-
1438void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
+
1438
1439
-
1447void mi_free_size(void* p, size_t size);
-
1448
-
1454void mi_free_size_aligned(void* p, size_t size, size_t alignment);
-
1455
-
1456void mi_free_aligned(void* p, size_t alignment);
-
1457
-
1459
-
1472
-
1474void* mi_new(std::size_t n) noexcept(false);
-
1475
-
1477void* mi_new_n(size_t count, size_t size) noexcept(false);
-
1478
-
1480void* mi_new_aligned(std::size_t n, std::align_val_t alignment) noexcept(false);
-
1481
-
1483void* mi_new_nothrow(size_t n);
-
1484
-
1486void* mi_new_aligned_nothrow(size_t n, size_t alignment);
-
1487
-
1489void* mi_new_realloc(void* p, size_t newsize);
-
1490
-
1492void* mi_new_reallocn(void* p, size_t newcount, size_t size);
-
1493
-
1501template<class T> struct mi_stl_allocator { }
-
1502
-
1504
+
1445
+
1446void* mi__expand(void* p, size_t newsize);
+
1447
+
1448size_t mi_malloc_size(const void* p);
+
1449size_t mi_malloc_good_size(size_t size);
+
1450size_t mi_malloc_usable_size(const void *p);
+
1451
+
1452int mi_posix_memalign(void** p, size_t alignment, size_t size);
+
1453int mi__posix_memalign(void** p, size_t alignment, size_t size);
+
1454void* mi_memalign(size_t alignment, size_t size);
+
1455void* mi_valloc(size_t size);
+
1456void* mi_pvalloc(size_t size);
+
1457void* mi_aligned_alloc(size_t alignment, size_t size);
+
1458
+
1459unsigned short* mi_wcsdup(const unsigned short* s);
+
1460unsigned char* mi_mbsdup(const unsigned char* s);
+
1461int mi_dupenv_s(char** buf, size_t* size, const char* name);
+
1462int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name);
+
1463
+
1466void* mi_reallocarray(void* p, size_t count, size_t size);
+
1467
+
1469int mi_reallocarr(void* p, size_t count, size_t size);
+
1470
+
1471void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment);
+
1472void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
+
1473
+
1482void mi_free_size(void* p, size_t size);
+
1483
+
1491void mi_free_size_nonnull(void* p, size_t size);
+
1492
+
1498void mi_free_size_aligned(void* p, size_t size, size_t alignment);
+
1499
+
1500void mi_free_aligned(void* p, size_t alignment);
+
1501
+
1503
+
1516
+
1518void* mi_new(std::size_t n) noexcept(false);
+
1519
+
1521void* mi_new_n(size_t count, size_t size) noexcept(false);
+
1522
+
1524void* mi_new_aligned(std::size_t n, std::align_val_t alignment) noexcept(false);
+
1525
+
1527void* mi_new_nothrow(size_t n);
+
1528
+
1530void* mi_new_aligned_nothrow(size_t n, size_t alignment);
+
1531
+
1533void* mi_new_realloc(void* p, size_t newsize);
+
1534
+
1536void* mi_new_reallocn(void* p, size_t newcount, size_t size);
+
1537
+
1545template<class T> struct mi_stl_allocator { }
+
1546
+
1548
mi_malloc_aligned_at
void * mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset)
Allocate size bytes aligned by alignment at a specified offset.
mi_calloc_aligned
void * mi_calloc_aligned(size_t count, size_t size, size_t alignment)
mi_realloc_aligned
void * mi_realloc_aligned(void *p, size_t newsize, size_t alignment)
@@ -652,17 +659,17 @@
mi_calloc_aligned_at
void * mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset)
mi_zalloc_aligned
void * mi_zalloc_aligned(size_t size, size_t alignment)
mi_realloc_aligned_at
void * mi_realloc_aligned_at(void *p, size_t newsize, size_t alignment, size_t offset)
-
mi_heap_area_t::heap_tag
int heap_tag
heap tag associated with this area (see mi_heap_new_ex)
Definition mimalloc-doc.h:687
-
mi_heap_area_t::block_size
size_t block_size
size in bytes of one block
Definition mimalloc-doc.h:685
-
mi_heap_area_t::committed
size_t committed
current committed bytes of this area
Definition mimalloc-doc.h:683
-
mi_heap_area_t::full_block_size
size_t full_block_size
size in bytes of a full block including padding and metadata.
Definition mimalloc-doc.h:686
-
mi_heap_area_t::used
size_t used
bytes in use by allocated blocks
Definition mimalloc-doc.h:684
-
mi_heap_area_t::blocks
void * blocks
start of the area containing heap blocks
Definition mimalloc-doc.h:681
-
mi_heap_area_t::reserved
size_t reserved
bytes reserved for this area
Definition mimalloc-doc.h:682
+
mi_heap_area_t::heap_tag
int heap_tag
heap tag associated with this area (see mi_heap_new_ex)
Definition mimalloc-doc.h:706
+
mi_heap_area_t::block_size
size_t block_size
size in bytes of one block
Definition mimalloc-doc.h:704
+
mi_heap_area_t::committed
size_t committed
current committed bytes of this area
Definition mimalloc-doc.h:702
+
mi_heap_area_t::full_block_size
size_t full_block_size
size in bytes of a full block including padding and metadata.
Definition mimalloc-doc.h:705
+
mi_heap_area_t::used
size_t used
bytes in use by allocated blocks
Definition mimalloc-doc.h:703
+
mi_heap_area_t::blocks
void * blocks
start of the area containing heap blocks
Definition mimalloc-doc.h:700
+
mi_heap_area_t::reserved
size_t reserved
bytes reserved for this area
Definition mimalloc-doc.h:701
mi_abandoned_visit_blocks
bool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool visit_blocks, mi_block_visit_fun *visitor, void *arg)
v1,v2: Visit all areas and blocks in abandoned heaps.
-
mi_block_visit_fun
bool mi_block_visit_fun(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *arg)
Visitor function passed to mi_heap_visit_blocks()
Definition mimalloc-doc.h:697
+
mi_block_visit_fun
bool mi_block_visit_fun(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *arg)
Visitor function passed to mi_heap_visit_blocks()
Definition mimalloc-doc.h:716
mi_heap_visit_blocks
bool mi_heap_visit_blocks(const mi_heap_t *heap, bool visit_blocks, mi_block_visit_fun *visitor, void *arg)
Visit all areas and blocks in a heap.
-
mi_heap_area_t
An area of heap space contains blocks of a single size.
Definition mimalloc-doc.h:680
+
mi_heap_area_t
An area of heap space contains blocks of a single size.
Definition mimalloc-doc.h:699
mi_reserve_os_memory
int mi_reserve_os_memory(size_t size, bool commit, bool allow_large)
Reserve OS memory for use by mimalloc.
mi_reserve_huge_os_pages_interleave
int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs)
Reserve pages of huge OS pages (1GiB) evenly divided over numa_nodes nodes, but stops after at most t...
mi_reserve_os_memory_ex
int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t *arena_id)
Reserve OS memory to be managed in an arena.
@@ -672,14 +679,15 @@
mi_reserve_huge_os_pages_at_ex
int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t *arena_id)
Reserve huge OS pages (1GiB) into a single arena.
mi_reserve_huge_os_pages_at
int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs)
Reserve pages of huge OS pages (1GiB) at a specific numa_node, but stops after at most timeout_msecs ...
mi_debug_show_arenas
void mi_debug_show_arenas(void)
Show all current arena's.
-
mi_arena_id_t
int mi_arena_id_t
Each arena has an associated identifier.
Definition mimalloc-doc.h:747
+
mi_arena_id_t
int mi_arena_id_t
Each arena has an associated identifier.
Definition mimalloc-doc.h:766
mi_arena_area
void * mi_arena_area(mi_arena_id_t arena_id, size_t *size)
Return the start and size of an arena.
mi_arena_min_alignment
size_t mi_arena_min_alignment(void)
Return the minimal alignment required for managed OS memory.
mi_heap_new_in_arena
mi_heap_t * mi_heap_new_in_arena(mi_arena_id_t arena_id)
Create a new heap that only allocates in the specified arena.
mi_theap_malloc_csize
void * mi_theap_malloc_csize(mi_theap_t *theap, size_t size)
mi_theap_zalloc_csize
void * mi_theap_zalloc_csize(mi_theap_t *theap, size_t size)
-
mi_free_csize
void mi_free_csize(void *p, size_t size)
Free a pointer with a known constant size.
-
mi_malloc_csize
void * mi_malloc_csize(size_t size)
Allocate a constant size object.
+
mi_free_csize_nonnull
void mi_free_csize_nonnull(void *p, size_t size)
v3: Free a non-null pointer with a known constant size.
+
mi_free_csize
void mi_free_csize(void *p, size_t size)
v3: Free a pointer with a known constant size.
+
mi_malloc_csize
void * mi_malloc_csize(size_t size)
v3: Allocate a constant size object.
mi_zalloc_csize
void * mi_zalloc_csize(size_t size)
mi_new_nothrow
void * mi_new_nothrow(size_t n)
like mi_malloc, but when out of memory, use std::get_new_handler but return NULL on failure.
mi_new
void * mi_new(std::size_t n) noexcept(false)
like mi_malloc(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exception...
@@ -688,19 +696,20 @@
mi_new_aligned_nothrow
void * mi_new_aligned_nothrow(size_t n, size_t alignment)
like mi_malloc_aligned, but when out of memory, use std::get_new_handler but return NULL on failure.
mi_new_reallocn
void * mi_new_reallocn(void *p, size_t newcount, size_t size)
like mi_reallocn(), but when out of memory, use std::get_new_handler and raise std::bad_alloc excepti...
mi_new_n
void * mi_new_n(size_t count, size_t size) noexcept(false)
like mi_mallocn(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exceptio...
-
mi_stl_allocator
std::allocator implementation for mimalloc for use in STL containers. For example:
Definition mimalloc-doc.h:1501
+
mi_stl_allocator
std::allocator implementation for mimalloc for use in STL containers. For example:
Definition mimalloc-doc.h:1545
mi_thread_done
void mi_thread_done(void)
Uninitialize mimalloc on a thread.
-
mi_deferred_free_fun
void mi_deferred_free_fun(bool force, unsigned long long heartbeat, void *arg)
Type of deferred free functions.
Definition mimalloc-doc.h:981
+
mi_free_small_nonnull
void mi_free_small_nonnull(void *p)
v3: Can be used to free an object that was allocated with mi_malloc_small() or any allocation with a ...
+
mi_deferred_free_fun
void mi_deferred_free_fun(bool force, unsigned long long heartbeat, void *arg)
Type of deferred free functions.
Definition mimalloc-doc.h:1000
mi_free_small
void mi_free_small(void *p)
v3: Can be used to free an object that was allocated with mi_malloc_small() or any allocation with a ...
mi_register_deferred_free
void mi_register_deferred_free(mi_deferred_free_fun *deferred_free, void *arg)
Register a deferred free function.
mi_collect
void mi_collect(bool force)
Eagerly free memory.
mi_zalloc_small
void * mi_zalloc_small(size_t size)
Allocate a zero initialized small object.
mi_malloc_small
void * mi_malloc_small(size_t size)
Allocate a small object.
-
mi_error_fun
void mi_error_fun(int err, void *arg)
Type of error callback functions.
Definition mimalloc-doc.h:1020
+
mi_error_fun
void mi_error_fun(int err, void *arg)
Type of error callback functions.
Definition mimalloc-doc.h:1039
mi_register_error
void mi_register_error(mi_error_fun *errfun, void *arg)
Register an error callback function.
mi_is_redirected
bool mi_is_redirected()
Is the C runtime malloc API redirected?
mi_thread_set_in_threadpool
void mi_thread_set_in_threadpool(void)
v3: Communicate that a thread is in a threadpool.
-
mi_output_fun
void mi_output_fun(const char *msg, void *arg)
Type of output functions.
Definition mimalloc-doc.h:1005
+
mi_output_fun
void mi_output_fun(const char *msg, void *arg)
Type of output functions.
Definition mimalloc-doc.h:1024
mi_register_output
void mi_register_output(mi_output_fun *out, void *arg)
Register an output function.
mi_version
int mi_version(void)
Return the mimalloc version.
mi_thread_init
void mi_thread_init(void)
Initialize mimalloc on a thread.
@@ -712,7 +721,7 @@
mi_heap_delete
void mi_heap_delete(mi_heap_t *heap)
Delete a previously allocated heap.
mi_heap_malloc_aligned
void * mi_heap_malloc_aligned(mi_heap_t *heap, size_t size, size_t alignment)
mi_heap_set_default
mi_heap_t * mi_heap_set_default(mi_heap_t *heap)
v1,v2: Set the default heap to use in the current thread for mi_malloc() et al.
-
mi_heap_t
struct mi_heap_s mi_heap_t
Type of first-class heaps.
Definition mimalloc-doc.h:513
+
mi_heap_t
struct mi_heap_s mi_heap_t
Type of first-class heaps.
Definition mimalloc-doc.h:532
mi_heap_of
mi_heap_t * mi_heap_of(const void *p)
v3: return the heap that contains the given pointer.
mi_heap_zalloc_aligned_at
void * mi_heap_zalloc_aligned_at(mi_heap_t *heap, size_t size, size_t alignment, size_t offset)
mi_heap_realpath
char * mi_heap_realpath(mi_heap_t *heap, const char *fname, char *resolved_name)
Resolve a file path name using a specific heap to allocate the result.
@@ -749,6 +758,7 @@
mi_reallocf
void * mi_reallocf(void *p, size_t newsize)
Re-allocate memory to newsize bytes,.
mi_mallocn
void * mi_mallocn(size_t count, size_t size)
Allocate count elements of size bytes.
mi_calloc
void * mi_calloc(size_t count, size_t size)
Allocate zero-initialized count elements of size bytes.
+
mi_cfree
void mi_cfree(void *p)
Checked free: just as mi_free() but always checks if the pointer p belongs to our heap.
mi_reallocn
void * mi_reallocn(void *p, size_t count, size_t size)
Re-allocate memory to count elements of size bytes.
mi_realpath
char * mi_realpath(const char *fname, char *resolved_name)
Resolve a file path name.
mi_good_size
size_t mi_good_size(size_t size)
Return the probable allocation block size for a given required size.
@@ -767,53 +777,53 @@
mi_option_disable
void mi_option_disable(mi_option_t option)
mi_options_print
void mi_options_print(void)
Print out all runtime parameters for mimalloc. Also printed with MIMALLOC_VERBOSE=1 at startup.
mi_option_set
void mi_option_set(mi_option_t option, long value)
-
mi_option_t
mi_option_t
Runtime options.
Definition mimalloc-doc.h:1263
-
mi_option_guarded_min
@ mi_option_guarded_min
only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0)
Definition mimalloc-doc.h:1284
-
mi_option_abandoned_reclaim_on_free
@ mi_option_abandoned_reclaim_on_free
v1,v2: allow to reclaim an abandoned segment on a free (=1)
Definition mimalloc-doc.h:1301
-
mi_option_purge_extend_delay
@ mi_option_purge_extend_delay
v1,v2: extend purge delay on each subsequent delay (=1)
Definition mimalloc-doc.h:1302
-
mi_option_allow_thp
@ mi_option_allow_thp
allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the proces...
Definition mimalloc-doc.h:1281
-
mi_option_show_stats
@ mi_option_show_stats
Print statistics on termination.
Definition mimalloc-doc.h:1266
-
mi_option_use_numa_nodes
@ mi_option_use_numa_nodes
0 = use all available numa nodes, otherwise use at most N nodes.
Definition mimalloc-doc.h:1296
-
mi_option_page_max_reclaim
@ mi_option_page_max_reclaim
v3: don't reclaim pages of the same originating theap if we already own N pages (in that size class) ...
Definition mimalloc-doc.h:1314
-
mi_option_abandoned_page_purge
@ mi_option_abandoned_page_purge
v1,v2: immediately purge delayed purges on thread termination
Definition mimalloc-doc.h:1294
-
mi_option_eager_commit_delay
@ mi_option_eager_commit_delay
v2: the first N segments per thread are not eagerly committed (but per page in the segment on demand)
Definition mimalloc-doc.h:1292
-
mi_option_eager_commit
@ mi_option_eager_commit
v1,v2: eager commit segments? (after eager_commit_delay segments) (enabled by default).
Definition mimalloc-doc.h:1291
-
mi_option_guarded_precise
@ mi_option_guarded_precise
disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard pa...
Definition mimalloc-doc.h:1286
-
mi_option_visit_abandoned
@ mi_option_visit_abandoned
allow visiting heap blocks from abandoned threads (=0)
Definition mimalloc-doc.h:1304
-
mi_option_os_tag
@ mi_option_os_tag
tag used for OS logging (macOS only for now) (=100)
Definition mimalloc-doc.h:1278
-
mi_option_arena_max_object_size
@ mi_option_arena_max_object_size
v3: set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit).
Definition mimalloc-doc.h:1317
-
mi_option_minimal_purge_size
@ mi_option_minimal_purge_size
v3: set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled....
Definition mimalloc-doc.h:1316
-
_mi_option_last
@ _mi_option_last
Definition mimalloc-doc.h:1319
-
mi_option_destroy_on_exit
@ mi_option_destroy_on_exit
if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe
Definition mimalloc-doc.h:1299
-
mi_option_page_cross_thread_max_reclaim
@ mi_option_page_cross_thread_max_reclaim
v3: don't reclaim pages across threads if we already own N pages (in that size class) (=16)
Definition mimalloc-doc.h:1315
-
mi_option_page_commit_on_demand
@ mi_option_page_commit_on_demand
v3: commit page memory on-demand (=0)
Definition mimalloc-doc.h:1313
-
mi_option_guarded_sample_seed
@ mi_option_guarded_sample_seed
can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0)
Definition mimalloc-doc.h:1288
-
mi_option_verbose
@ mi_option_verbose
Print verbose messages.
Definition mimalloc-doc.h:1267
-
mi_option_allow_large_os_pages
@ mi_option_allow_large_os_pages
allow large (2 or 4 MiB) OS pages, implies eager commit.
Definition mimalloc-doc.h:1275
-
mi_option_arena_purge_mult
@ mi_option_arena_purge_mult
multiplier for purge_delay for the purging delay for arenas (=10)
Definition mimalloc-doc.h:1300
-
mi_option_target_segments_per_thread
@ mi_option_target_segments_per_thread
v1,v2: experimental (=0)
Definition mimalloc-doc.h:1305
-
mi_option_generic_collect
@ mi_option_generic_collect
collect heaps every N (=10000) generic allocation calls
Definition mimalloc-doc.h:1280
-
mi_option_retry_on_oom
@ mi_option_retry_on_oom
retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows)
Definition mimalloc-doc.h:1279
-
mi_option_guarded_max
@ mi_option_guarded_max
only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0)
Definition mimalloc-doc.h:1285
-
mi_option_purge_decommits
@ mi_option_purge_decommits
should a memory purge decommit? (=1). Set to 0 to use memory reset on a purge (instead of decommit)
Definition mimalloc-doc.h:1276
-
mi_option_reserve_huge_os_pages_at
@ mi_option_reserve_huge_os_pages_at
Reserve N huge OS pages at a specific NUMA node N.
Definition mimalloc-doc.h:1273
-
mi_option_pagemap_commit
@ mi_option_pagemap_commit
v3: commit the full pagemap (to always catch invalid pointer uses) (=0)
Definition mimalloc-doc.h:1312
-
mi_option_max_segment_reclaim
@ mi_option_max_segment_reclaim
v2: max. percentage of the abandoned segments can be reclaimed per try (=10%)
Definition mimalloc-doc.h:1298
-
mi_option_max_vabits
@ mi_option_max_vabits
v3: max user space virtual address bits to consider (=48)
Definition mimalloc-doc.h:1311
-
mi_option_arena_reserve
@ mi_option_arena_reserve
initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use ...
Definition mimalloc-doc.h:1277
-
mi_option_page_reclaim_on_free
@ mi_option_page_reclaim_on_free
v3: reclaim abandoned pages on a free (=0). -1 disallowr always, 0 allows if the page originated from...
Definition mimalloc-doc.h:1308
-
mi_option_reserve_huge_os_pages
@ mi_option_reserve_huge_os_pages
reserve N huge OS pages (1GiB pages) at startup
Definition mimalloc-doc.h:1272
-
mi_option_page_max_candidates
@ mi_option_page_max_candidates
v3: max candidate pages to consider for allocation (=4)
Definition mimalloc-doc.h:1310
-
mi_option_disallow_os_alloc
@ mi_option_disallow_os_alloc
1 = do not use OS memory for allocation (but only programmatically reserved arenas)
Definition mimalloc-doc.h:1297
-
mi_option_purge_delay
@ mi_option_purge_delay
memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all...
Definition mimalloc-doc.h:1295
-
mi_option_disallow_arena_alloc
@ mi_option_disallow_arena_alloc
1 = do not use arena's for allocation (except if using specific arena id's)
Definition mimalloc-doc.h:1303
-
mi_option_guarded_sample_rate
@ mi_option_guarded_sample_rate
1 out of N allocations in the min/max range will be guarded (=1000)
Definition mimalloc-doc.h:1287
-
mi_option_max_errors
@ mi_option_max_errors
issue at most N error messages
Definition mimalloc-doc.h:1268
-
mi_option_page_full_retain
@ mi_option_page_full_retain
v3: retain N full (small) pages per size class (=2). Use -1 for infinite (as in v1,...
Definition mimalloc-doc.h:1309
-
mi_option_max_warnings
@ mi_option_max_warnings
issue at most N warning messages
Definition mimalloc-doc.h:1269
-
mi_option_show_errors
@ mi_option_show_errors
Print error messages.
Definition mimalloc-doc.h:1265
-
mi_option_reserve_os_memory
@ mi_option_reserve_os_memory
reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use m...
Definition mimalloc-doc.h:1274
-
mi_option_arena_eager_commit
@ mi_option_arena_eager_commit
eager commit arenas? Use 2 to enable just on overcommit systems (=2)
Definition mimalloc-doc.h:1293
+
mi_option_t
mi_option_t
Runtime options.
Definition mimalloc-doc.h:1300
+
mi_option_guarded_min
@ mi_option_guarded_min
only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0)
Definition mimalloc-doc.h:1321
+
mi_option_abandoned_reclaim_on_free
@ mi_option_abandoned_reclaim_on_free
v1,v2: allow to reclaim an abandoned segment on a free (=1)
Definition mimalloc-doc.h:1338
+
mi_option_purge_extend_delay
@ mi_option_purge_extend_delay
v1,v2: extend purge delay on each subsequent delay (=1)
Definition mimalloc-doc.h:1339
+
mi_option_allow_thp
@ mi_option_allow_thp
allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the proces...
Definition mimalloc-doc.h:1318
+
mi_option_show_stats
@ mi_option_show_stats
Print statistics on termination.
Definition mimalloc-doc.h:1303
+
mi_option_use_numa_nodes
@ mi_option_use_numa_nodes
0 = use all available numa nodes, otherwise use at most N nodes.
Definition mimalloc-doc.h:1333
+
mi_option_page_max_reclaim
@ mi_option_page_max_reclaim
v3: don't reclaim pages of the same originating theap if we already own N pages (in that size class) ...
Definition mimalloc-doc.h:1351
+
mi_option_abandoned_page_purge
@ mi_option_abandoned_page_purge
v1,v2: immediately purge delayed purges on thread termination
Definition mimalloc-doc.h:1331
+
mi_option_eager_commit_delay
@ mi_option_eager_commit_delay
v2: the first N segments per thread are not eagerly committed (but per page in the segment on demand)
Definition mimalloc-doc.h:1329
+
mi_option_eager_commit
@ mi_option_eager_commit
v1,v2: eager commit segments? (after eager_commit_delay segments) (enabled by default).
Definition mimalloc-doc.h:1328
+
mi_option_guarded_precise
@ mi_option_guarded_precise
disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard pa...
Definition mimalloc-doc.h:1323
+
mi_option_visit_abandoned
@ mi_option_visit_abandoned
allow visiting heap blocks from abandoned threads (=0)
Definition mimalloc-doc.h:1341
+
mi_option_os_tag
@ mi_option_os_tag
tag used for OS logging (macOS only for now) (=100)
Definition mimalloc-doc.h:1315
+
mi_option_arena_max_object_size
@ mi_option_arena_max_object_size
v3: set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit).
Definition mimalloc-doc.h:1354
+
mi_option_minimal_purge_size
@ mi_option_minimal_purge_size
v3: set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled....
Definition mimalloc-doc.h:1353
+
_mi_option_last
@ _mi_option_last
Definition mimalloc-doc.h:1356
+
mi_option_destroy_on_exit
@ mi_option_destroy_on_exit
if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe
Definition mimalloc-doc.h:1336
+
mi_option_page_cross_thread_max_reclaim
@ mi_option_page_cross_thread_max_reclaim
v3: don't reclaim pages across threads if we already own N pages (in that size class) (=16)
Definition mimalloc-doc.h:1352
+
mi_option_page_commit_on_demand
@ mi_option_page_commit_on_demand
v3: commit page memory on-demand (=0)
Definition mimalloc-doc.h:1350
+
mi_option_guarded_sample_seed
@ mi_option_guarded_sample_seed
can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0)
Definition mimalloc-doc.h:1325
+
mi_option_verbose
@ mi_option_verbose
Print verbose messages.
Definition mimalloc-doc.h:1304
+
mi_option_allow_large_os_pages
@ mi_option_allow_large_os_pages
allow large (2 or 4 MiB) OS pages, implies eager commit.
Definition mimalloc-doc.h:1312
+
mi_option_arena_purge_mult
@ mi_option_arena_purge_mult
multiplier for purge_delay for the purging delay for arenas (=10)
Definition mimalloc-doc.h:1337
+
mi_option_target_segments_per_thread
@ mi_option_target_segments_per_thread
v1,v2: experimental (=0)
Definition mimalloc-doc.h:1342
+
mi_option_generic_collect
@ mi_option_generic_collect
collect heaps every N (=10000) generic allocation calls
Definition mimalloc-doc.h:1317
+
mi_option_retry_on_oom
@ mi_option_retry_on_oom
retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows)
Definition mimalloc-doc.h:1316
+
mi_option_guarded_max
@ mi_option_guarded_max
only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0)
Definition mimalloc-doc.h:1322
+
mi_option_purge_decommits
@ mi_option_purge_decommits
should a memory purge decommit? (=1). Set to 0 to use memory reset on a purge (instead of decommit)
Definition mimalloc-doc.h:1313
+
mi_option_reserve_huge_os_pages_at
@ mi_option_reserve_huge_os_pages_at
Reserve N huge OS pages at a specific NUMA node N.
Definition mimalloc-doc.h:1310
+
mi_option_pagemap_commit
@ mi_option_pagemap_commit
v3: commit the full pagemap (to always catch invalid pointer uses) (=0)
Definition mimalloc-doc.h:1349
+
mi_option_max_segment_reclaim
@ mi_option_max_segment_reclaim
v2: max. percentage of the abandoned segments can be reclaimed per try (=10%)
Definition mimalloc-doc.h:1335
+
mi_option_max_vabits
@ mi_option_max_vabits
v3: max user space virtual address bits to consider (=48)
Definition mimalloc-doc.h:1348
+
mi_option_arena_reserve
@ mi_option_arena_reserve
initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use ...
Definition mimalloc-doc.h:1314
+
mi_option_page_reclaim_on_free
@ mi_option_page_reclaim_on_free
v3: reclaim abandoned pages on a free (=0). -1 disallowr always, 0 allows if the page originated from...
Definition mimalloc-doc.h:1345
+
mi_option_reserve_huge_os_pages
@ mi_option_reserve_huge_os_pages
reserve N huge OS pages (1GiB pages) at startup
Definition mimalloc-doc.h:1309
+
mi_option_page_max_candidates
@ mi_option_page_max_candidates
v3: max candidate pages to consider for allocation (=4)
Definition mimalloc-doc.h:1347
+
mi_option_disallow_os_alloc
@ mi_option_disallow_os_alloc
1 = do not use OS memory for allocation (but only programmatically reserved arenas)
Definition mimalloc-doc.h:1334
+
mi_option_purge_delay
@ mi_option_purge_delay
memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all...
Definition mimalloc-doc.h:1332
+
mi_option_disallow_arena_alloc
@ mi_option_disallow_arena_alloc
1 = do not use arena's for allocation (except if using specific arena id's)
Definition mimalloc-doc.h:1340
+
mi_option_guarded_sample_rate
@ mi_option_guarded_sample_rate
1 out of N allocations in the min/max range will be guarded (=1000)
Definition mimalloc-doc.h:1324
+
mi_option_max_errors
@ mi_option_max_errors
issue at most N error messages
Definition mimalloc-doc.h:1305
+
mi_option_page_full_retain
@ mi_option_page_full_retain
v3: retain N full (small) pages per size class (=2). Use -1 for infinite (as in v1,...
Definition mimalloc-doc.h:1346
+
mi_option_max_warnings
@ mi_option_max_warnings
issue at most N warning messages
Definition mimalloc-doc.h:1306
+
mi_option_show_errors
@ mi_option_show_errors
Print error messages.
Definition mimalloc-doc.h:1302
+
mi_option_reserve_os_memory
@ mi_option_reserve_os_memory
reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use m...
Definition mimalloc-doc.h:1311
+
mi_option_arena_eager_commit
@ mi_option_arena_eager_commit
eager commit arenas? Use 2 to enable just on overcommit systems (=2)
Definition mimalloc-doc.h:1330
mi_malloc_usable_size
size_t mi_malloc_usable_size(const void *p)
mi_free_aligned
void mi_free_aligned(void *p, size_t alignment)
mi_aligned_offset_recalloc
void * mi_aligned_offset_recalloc(void *p, size_t newcount, size_t size, size_t alignment, size_t offset)
@@ -823,9 +833,9 @@
mi_pvalloc
void * mi_pvalloc(size_t size)
mi__expand
void * mi__expand(void *p, size_t newsize)
mi_wdupenv_s
int mi_wdupenv_s(unsigned short **buf, size_t *size, const unsigned short *name)
-
mi_cfree
void mi_cfree(void *p)
Checked free: just as mi_free but always checks if the pointer p belongs to our heap and is safe to u...
mi_memalign
void * mi_memalign(size_t alignment, size_t size)
mi_free_size_aligned
void mi_free_size_aligned(void *p, size_t size, size_t alignment)
Free an object that was allocated aligned.
+
mi_free_size_nonnull
void mi_free_size_nonnull(void *p, size_t size)
Free a non-NULL pointer that was allocated with a known size.
mi_mbsdup
unsigned char * mi_mbsdup(const unsigned char *s)
mi_reallocarr
int mi_reallocarr(void *p, size_t count, size_t size)
Corresponds to reallocarr in NetBSD.
mi_malloc_good_size
size_t mi_malloc_good_size(size_t size)
@@ -834,10 +844,10 @@
mi_posix_memalign
int mi_posix_memalign(void **p, size_t alignment, size_t size)
mi__posix_memalign
int mi__posix_memalign(void **p, size_t alignment, size_t size)
mi_reallocarray
void * mi_reallocarray(void *p, size_t count, size_t size)
Correspond s to reallocarray in FreeBSD.
-
mi_free_size
void mi_free_size(void *p, size_t size)
Free an object that was allocated with a known size.
+
mi_free_size
void mi_free_size(void *p, size_t size)
Free a block that was allocated with a known size.
mi_aligned_recalloc
void * mi_aligned_recalloc(void *p, size_t newcount, size_t size, size_t alignment)
-
mi_stats_s::version
size_t version
initialize with `MI_STAT_VERSION (so linking dynamically with a separately compiled mimalloc is safe)
Definition mimalloc-doc.h:1106
-
mi_stats_s::size
size_t size
initialize with sizeof(mi_stats_t) (so linking dynamically with a separately compiled mimalloc is saf...
Definition mimalloc-doc.h:1104
+
mi_stats_s::version
size_t version
initialize with `MI_STAT_VERSION (so linking dynamically with a separately compiled mimalloc is safe)
Definition mimalloc-doc.h:1143
+
mi_stats_s::size
size_t size
initialize with sizeof(mi_stats_t) (so linking dynamically with a separately compiled mimalloc is saf...
Definition mimalloc-doc.h:1141
mi_process_info_print_out
void mi_process_info_print_out(mi_output_fun *out, void *arg)
Print out all process info.
mi_subproc_heap_stats_print_out
void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun *out, void *arg)
v3: Print statistics for a given subprocess with each heap separately printed.
mi_heap_stats_get
bool mi_heap_stats_get(mi_heap_t *heap, mi_stats_t *stats)
v3: Return statistics for a given heap.
@@ -857,11 +867,11 @@
mi_stats_get_bin_size
size_t mi_stats_get_bin_size(size_t bin)
v3: Return the block size for the given bin.
mi_process_info_print
void mi_process_info_print(void)
Print out all process info.
mi_heap_stats_print_out
void mi_heap_stats_print_out(mi_heap_t *heap, mi_output_fun *out, void *arg)
v3: Show the heap statistics as JSON.
-
mi_stats_s
Statistics. See include/mimalloc-stats.h for the full definition.
Definition mimalloc-doc.h:1102
+
mi_stats_s
Statistics. See include/mimalloc-stats.h for the full definition.
Definition mimalloc-doc.h:1139
mi_subproc_main
mi_subproc_id_t mi_subproc_main(void)
Get the main sub-process identifier.
-
mi_heap_visit_fun
bool mi_heap_visit_fun(mi_heap_t *heap, void *arg)
The type of a heap visitor function.
Definition mimalloc-doc.h:911
+
mi_heap_visit_fun
bool mi_heap_visit_fun(mi_heap_t *heap, void *arg)
The type of a heap visitor function.
Definition mimalloc-doc.h:930
mi_subproc_new
mi_subproc_id_t mi_subproc_new(void)
Create a fresh sub-process (with no associated threads yet).
-
mi_subproc_id_t
void * mi_subproc_id_t
A process can associate threads with sub-processes.
Definition mimalloc-doc.h:880
+
mi_subproc_id_t
void * mi_subproc_id_t
A process can associate threads with sub-processes.
Definition mimalloc-doc.h:899
mi_subproc_delete
void mi_subproc_delete(mi_subproc_id_t subproc)
v1,v2: Delete a previously created sub-process.
mi_subproc_current
mi_subproc_id_t mi_subproc_current(void)
Get the current sub-process identifier of this thread.
mi_subproc_destroy
void mi_subproc_destroy(mi_subproc_id_t subproc)
v3: Destroy a previously created sub-process.
@@ -873,7 +883,7 @@
mi_theap_get_default
mi_theap_t * mi_theap_get_default()
Get the default theap that is used for mi_malloc() et al. for the current thread.
mi_heap_theap
mi_theap_t * mi_heap_theap(mi_heap_t *heap)
Return the thread-local theap for the given heap.
mi_theap_malloc_aligned
void * mi_theap_malloc_aligned(mi_theap_t *theap, size_t size, size_t alignment)
-
mi_theap_t
struct mi_theap_s mi_theap_t
Type of thread-local heaps.
Definition mimalloc-doc.h:1355
+
mi_theap_t
struct mi_theap_s mi_theap_t
Type of thread-local heaps.
Definition mimalloc-doc.h:1392
mi_theap_collect
void mi_theap_collect(mi_theap_t *theap, bool force)
Release outstanding resources in a specific theap.
mi_theap_zalloc
void * mi_theap_zalloc(mi_theap_t *theap, size_t size)
Allocate zero-initialized in a specific heap.
mi_theap_zalloc_small
void * mi_theap_zalloc_small(mi_theap_t *theap, size_t size)
Allocate a small object in a specific theap. size must be smaller or equal to MI_SMALL_SIZE_MAX().
diff --git a/docs/modes.html b/docs/modes.html index a2c488c29..7e29f1c65 100644 --- a/docs/modes.html +++ b/docs/modes.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/navtreedata.js b/docs/navtreedata.js index 91658f29a..dc276b57b 100644 --- a/docs/navtreedata.js +++ b/docs/navtreedata.js @@ -47,7 +47,7 @@ var NAVTREE = var NAVTREEINDEX = [ "annotated.html", -"group__theap.html#ga7905a73865733da8a43339a500533180" +"group__theap.html#ga11943128dc354313c4189e72f4299dbb" ]; var SYNCONMSG = 'click to disable panel synchronisation'; diff --git a/docs/navtreeindex0.js b/docs/navtreeindex0.js index 24e0f8478..fd485b71e 100644 --- a/docs/navtreeindex0.js +++ b/docs/navtreeindex0.js @@ -43,11 +43,12 @@ var NAVTREEINDEX0 = "group__arenas.html#gaa1a59464dbf3bc972d34fca0f86668fc":[7,7,2], "group__arenas.html#gaaf2d9976576d5efd5544be12848af949":[7,7,5], "group__constantsize.html":[7,3], -"group__constantsize.html#ga105bd667da91a15d113fe79be04d625c":[7,3,2], -"group__constantsize.html#ga1addf755829380581dd9c3c9175e376f":[7,3,3], +"group__constantsize.html#ga105bd667da91a15d113fe79be04d625c":[7,3,3], +"group__constantsize.html#ga1addf755829380581dd9c3c9175e376f":[7,3,4], +"group__constantsize.html#ga38a93d373a9dc8edc8bca83edb6af5c5":[7,3,1], "group__constantsize.html#ga7a421e302074e598527c3eb052a6a3c5":[7,3,0], -"group__constantsize.html#ga9cd44d925d34f7180922e12bbf594571":[7,3,1], -"group__constantsize.html#gadca6d340ee15b5c70695ed82d95c6cec":[7,3,4], +"group__constantsize.html#ga9cd44d925d34f7180922e12bbf594571":[7,3,2], +"group__constantsize.html#gadca6d340ee15b5c70695ed82d95c6cec":[7,3,5], "group__cpp.html":[7,14], "group__cpp.html#ga5cb4f120d1f7296074256215aa9a9e54":[7,14,5], "group__cpp.html#ga633d96e3bc7011f960df9f3b2731fc6a":[7,14,1], @@ -58,22 +59,23 @@ var NAVTREEINDEX0 = "group__cpp.html#gadd11b85c15d21d308386844b5233856c":[7,14,4], "group__cpp.html#structmi__stl__allocator":[7,14,0], "group__extended.html":[7,9], -"group__extended.html#ga0ae4581e85453456a0d658b2b98bf7bf":[7,9,11], +"group__extended.html#ga0ae4581e85453456a0d658b2b98bf7bf":[7,9,12], "group__extended.html#ga1ea64283508718d9d645c38efc2f4305":[7,9,0], +"group__extended.html#ga22890f54e5be433837069c8e97fdf379":[7,9,6], "group__extended.html#ga292a45f7dbc7cd23c5352ce1f0002816":[7,9,1], "group__extended.html#ga2a8469764c38dcaf878f36e5f50e3610":[7,9,5], -"group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece":[7,9,8], +"group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece":[7,9,9], "group__extended.html#ga421430e2226d7d468529cec457396756":[7,9,4], -"group__extended.html#ga51c47637e81df0e2f13a2d7a2dec123e":[7,9,15], -"group__extended.html#ga7f050bc6b897da82692174f5fce59cde":[7,9,7], +"group__extended.html#ga51c47637e81df0e2f13a2d7a2dec123e":[7,9,16], +"group__extended.html#ga7f050bc6b897da82692174f5fce59cde":[7,9,8], "group__extended.html#ga83fc6a688b322261e1c2deab000b0591":[7,9,2], -"group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45":[7,9,9], -"group__extended.html#gaad25050b19f30cd79397b227e0157a3f":[7,9,6], -"group__extended.html#gada899104276aac52539eb4de7c020f1f":[7,9,13], +"group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45":[7,9,10], +"group__extended.html#gaad25050b19f30cd79397b227e0157a3f":[7,9,7], +"group__extended.html#gada899104276aac52539eb4de7c020f1f":[7,9,14], "group__extended.html#gadf31cea7d0332a81c8b882cbbdbadb8d":[7,9,3], -"group__extended.html#gae5b17ff027cd2150b43a33040250cf3f":[7,9,10], -"group__extended.html#gae9ddddcf14a58d0224ef3c8778e5b057":[7,9,14], -"group__extended.html#gaf8e73efc2cbca9ebfdfb166983a04c17":[7,9,12], +"group__extended.html#gae5b17ff027cd2150b43a33040250cf3f":[7,9,11], +"group__extended.html#gae9ddddcf14a58d0224ef3c8778e5b057":[7,9,15], +"group__extended.html#gaf8e73efc2cbca9ebfdfb166983a04c17":[7,9,13], "group__heap.html":[7,5], "group__heap.html#ga012c5c8abe22b10043de39ff95909541":[7,5,18], "group__heap.html#ga0994f44cc595f9c1840a9af162d98597":[7,5,35], @@ -113,20 +115,21 @@ var NAVTREEINDEX0 = "group__heap.html#gae7cd171425bee04c683c65a3701f0b4a":[7,5,25], "group__heap.html#gae7ffc045c3996497a7f3a5f6fe7b8aaa":[7,5,17], "group__malloc.html":[7,0], -"group__malloc.html#ga0621af6a5e3aa384e6a1b548958bf583":[7,0,6], -"group__malloc.html#ga089c859d9eddc5f9b4bd946cd53cebee":[7,0,12], -"group__malloc.html#ga19299856216cfbb08e2628593654dfb0":[7,0,1], -"group__malloc.html#ga245ac90ebc2cfdd17de599e5fea59889":[7,0,10], -"group__malloc.html#ga486d0d26b3b3794f6d1cdb41a9aed92d":[7,0,11], -"group__malloc.html#ga4dc3a4067037b151a64629fe8a332641":[7,0,7], -"group__malloc.html#ga61f46bade3db76ca24aaafedc40de7b6":[7,0,5], +"group__malloc.html#ga0621af6a5e3aa384e6a1b548958bf583":[7,0,7], +"group__malloc.html#ga089c859d9eddc5f9b4bd946cd53cebee":[7,0,13], +"group__malloc.html#ga19299856216cfbb08e2628593654dfb0":[7,0,2], +"group__malloc.html#ga245ac90ebc2cfdd17de599e5fea59889":[7,0,11], +"group__malloc.html#ga486d0d26b3b3794f6d1cdb41a9aed92d":[7,0,12], +"group__malloc.html#ga4dc3a4067037b151a64629fe8a332641":[7,0,8], +"group__malloc.html#ga61f46bade3db76ca24aaafedc40de7b6":[7,0,6], "group__malloc.html#ga6686568014b54d1e6c7ac64a076e4f56":[7,0,0], -"group__malloc.html#ga8bddfb4a1270a0854bbcf44cb3980467":[7,0,8], -"group__malloc.html#ga94c3afcc086e85d75a57e9f76b9b71dd":[7,0,9], -"group__malloc.html#gac057927cd06c854b45fe7847e921bd47":[7,0,3], -"group__malloc.html#gae1dd97b542420c87ae085e822b1229e8":[7,0,4], -"group__malloc.html#gae6e38c4403247a7b40d80419e093bfb8":[7,0,13], -"group__malloc.html#gaf2c7b89c327d1f60f59e68b9ea644d95":[7,0,2], +"group__malloc.html#ga705dc7a64bffacfeeb0141501a5c35d7":[7,0,1], +"group__malloc.html#ga8bddfb4a1270a0854bbcf44cb3980467":[7,0,9], +"group__malloc.html#ga94c3afcc086e85d75a57e9f76b9b71dd":[7,0,10], +"group__malloc.html#gac057927cd06c854b45fe7847e921bd47":[7,0,4], +"group__malloc.html#gae1dd97b542420c87ae085e822b1229e8":[7,0,5], +"group__malloc.html#gae6e38c4403247a7b40d80419e093bfb8":[7,0,14], +"group__malloc.html#gaf2c7b89c327d1f60f59e68b9ea644d95":[7,0,3], "group__options.html":[7,11], "group__options.html#ga04180ae41b0d601421dd62ced40ca050":[7,11,2], "group__options.html#ga1dd50c7533baec4cc6c1a2e9adf05dd2":[7,11,12], @@ -189,7 +192,7 @@ var NAVTREEINDEX0 = "group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafd0c5ddbc4b59fd8b5216871728167a5":[7,11,0,22], "group__posix.html":[7,13], "group__posix.html#ga06d07cf357bbac5c73ba5d0c0c421e17":[7,13,12], -"group__posix.html#ga0d28d5cf61e6bfbb18c63092939fe5c9":[7,13,7], +"group__posix.html#ga0d28d5cf61e6bfbb18c63092939fe5c9":[7,13,6], "group__posix.html#ga16570deddd559001b44953eedbad0084":[7,13,3], "group__posix.html#ga430ed1513f0571ff83be00ec58a98ee0":[7,13,2], "group__posix.html#ga4531c9e775bb3ae12db57c1ba8a5d7de":[7,13,11], @@ -197,18 +200,18 @@ var NAVTREEINDEX0 = "group__posix.html#ga644bebccdbb2821542dd8c7e7641f476":[7,13,16], "group__posix.html#ga66bcfeb4faedbb42b796bc680821ef84":[7,13,0], "group__posix.html#ga6ac6a6a8f3c96f1af24bb8d0439cbbd1":[7,13,21], -"group__posix.html#ga705dc7a64bffacfeeb0141501a5c35d7":[7,13,5], "group__posix.html#ga726867f13fd29ca36064954c0285b1d8":[7,13,14], -"group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc":[7,13,9], +"group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc":[7,13,8], +"group__posix.html#ga796602dae2a89797026f72e0c0240f32":[7,13,9], "group__posix.html#ga7b82a44094fdec4d2084eb4288a979b0":[7,13,13], "group__posix.html#ga7e1934d60a3e697950eeb48e042bfad5":[7,13,17], "group__posix.html#ga9d23ac7885fed7413c11d8e0ffa31071":[7,13,10], "group__posix.html#gaa9fd7f25c9ac3a20e89b33bd6e383fcf":[7,13,20], -"group__posix.html#gab41369c1a1da7504013a7a0b1d4dd958":[7,13,6], +"group__posix.html#gab41369c1a1da7504013a7a0b1d4dd958":[7,13,5], "group__posix.html#gacff84f226ba9feb2031b8992e5579447":[7,13,15], "group__posix.html#gad5a69c8fea96aa2b7a7c818c2130090a":[7,13,1], "group__posix.html#gadfeccb72748a2f6305474a37d9d57bce":[7,13,18], -"group__posix.html#gae01389eedab8d67341ff52e2aad80ebb":[7,13,8], +"group__posix.html#gae01389eedab8d67341ff52e2aad80ebb":[7,13,7], "group__posix.html#gaf82cbb4b4f24acf723348628451798d3":[7,13,4], "group__stats.html":[7,10], "group__stats.html#aa74fc9c40dfaef5e06dec4019c19de68":[7,10,0,1], @@ -246,8 +249,5 @@ var NAVTREEINDEX0 = "group__subproc.html#gadbc53414eb68b275588ec001ce1ddc7c":[7,8,2], "group__subproc.html#gafd2136e7f167b5cd02b6ab00bed0ac93":[7,8,8], "group__theap.html":[7,12], -"group__theap.html#ga04420ce52785019412397e4de24a984a":[7,12,9], -"group__theap.html#ga11943128dc354313c4189e72f4299dbb":[7,12,8], -"group__theap.html#ga1fc65302a212daaecb21abda96ff247c":[7,12,7], -"group__theap.html#ga78afbb07c1e0cd333c7539cf7403dfd5":[7,12,4] +"group__theap.html#ga04420ce52785019412397e4de24a984a":[7,12,9] }; diff --git a/docs/navtreeindex1.js b/docs/navtreeindex1.js index 79286c5a6..3b01ba053 100644 --- a/docs/navtreeindex1.js +++ b/docs/navtreeindex1.js @@ -1,5 +1,8 @@ var NAVTREEINDEX1 = { +"group__theap.html#ga11943128dc354313c4189e72f4299dbb":[7,12,8], +"group__theap.html#ga1fc65302a212daaecb21abda96ff247c":[7,12,7], +"group__theap.html#ga78afbb07c1e0cd333c7539cf7403dfd5":[7,12,4], "group__theap.html#ga7905a73865733da8a43339a500533180":[7,12,1], "group__theap.html#ga84503fc2bd83fbfb7f789b7d80d76f36":[7,12,6], "group__theap.html#ga8722f6ac4f638b4692e06ddfddc844f8":[7,12,0], diff --git a/docs/overrides.html b/docs/overrides.html index d5736734f..d7f7e0909 100644 --- a/docs/overrides.html +++ b/docs/overrides.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/pages.html b/docs/pages.html index c1d698676..5b13b6dfd 100644 --- a/docs/pages.html +++ b/docs/pages.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/search/all_9.js b/docs/search/all_9.js index 54d47ba9a..bf716a23b 100644 --- a/docs/search/all_9.js +++ b/docs/search/all_9.js @@ -19,7 +19,7 @@ var searchData= ['mi_5fcalloc_5faligned_16',['mi_calloc_aligned',['../group__aligned.html#ga424ef386fb1f9f8e0a86ab53f16eaaf1',1,'mimalloc-doc.h']]], ['mi_5fcalloc_5faligned_5fat_17',['mi_calloc_aligned_at',['../group__aligned.html#ga977f96bd2c5c141bcd70e6685c90d6c3',1,'mimalloc-doc.h']]], ['mi_5fcalloc_5ftp_18',['mi_calloc_tp',['../group__typed.html#gae80c47c9d4cab10961fff1a8ac98fc07',1,'mimalloc-doc.h']]], - ['mi_5fcfree_19',['mi_cfree',['../group__posix.html#ga705dc7a64bffacfeeb0141501a5c35d7',1,'mimalloc-doc.h']]], + ['mi_5fcfree_19',['mi_cfree',['../group__malloc.html#ga705dc7a64bffacfeeb0141501a5c35d7',1,'mimalloc-doc.h']]], ['mi_5fcheck_5fowned_20',['mi_check_owned',['../group__heap.html#ga628c237489c2679af84a4d0d143b3dd5',1,'mimalloc-doc.h']]], ['mi_5fcollect_21',['mi_collect',['../group__extended.html#ga421430e2226d7d468529cec457396756',1,'mimalloc-doc.h']]], ['mi_5fdebug_5fshow_5farenas_22',['mi_debug_show_arenas',['../group__arenas.html#ga98cda17aa7adeda8b213382af8e7810b',1,'mimalloc-doc.h']]], @@ -30,231 +30,234 @@ var searchData= ['mi_5ffree_27',['mi_free',['../group__malloc.html#gaf2c7b89c327d1f60f59e68b9ea644d95',1,'mimalloc-doc.h']]], ['mi_5ffree_5faligned_28',['mi_free_aligned',['../group__posix.html#ga0d28d5cf61e6bfbb18c63092939fe5c9',1,'mimalloc-doc.h']]], ['mi_5ffree_5fcsize_29',['mi_free_csize',['../group__constantsize.html#ga7a421e302074e598527c3eb052a6a3c5',1,'mimalloc-doc.h']]], - ['mi_5ffree_5fsize_30',['mi_free_size',['../group__posix.html#gae01389eedab8d67341ff52e2aad80ebb',1,'mimalloc-doc.h']]], - ['mi_5ffree_5fsize_5faligned_31',['mi_free_size_aligned',['../group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc',1,'mimalloc-doc.h']]], - ['mi_5ffree_5fsmall_32',['mi_free_small',['../group__extended.html#ga2a8469764c38dcaf878f36e5f50e3610',1,'mimalloc-doc.h']]], - ['mi_5ffree_5ftp_33',['mi_free_tp',['../group__typed.html#gafc80000c300f02bdd4fe695eb53ca70a',1,'mimalloc-doc.h']]], - ['mi_5fgood_5fsize_34',['mi_good_size',['../group__malloc.html#gac057927cd06c854b45fe7847e921bd47',1,'mimalloc-doc.h']]], - ['mi_5fheap_5farea_5ft_35',['mi_heap_area_t',['../group__analysis.html#structmi__heap__area__t',1,'']]], - ['mi_5fheap_5fcalloc_36',['mi_heap_calloc',['../group__heap.html#gac0098aaf231d3e9586c73136d5df95da',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcalloc_5faligned_37',['mi_heap_calloc_aligned',['../group__heap.html#gacafcc26df827c7a7de5e850217566108',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcalloc_5faligned_5fat_38',['mi_heap_calloc_aligned_at',['../group__heap.html#gaa42ec2079989c4374f2c331d9b35f4e4',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcalloc_5ftp_39',['mi_heap_calloc_tp',['../group__typed.html#ga953f22101bb354a8b6de2a6d6fda9b98',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcheck_5fowned_40',['mi_heap_check_owned',['../group__heap.html#ga0d67c1789faaa15ff366c024fcaf6377',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcollect_41',['mi_heap_collect',['../group__heap.html#ga7922f7495cde30b1984d0e6072419298',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcontains_42',['mi_heap_contains',['../group__heap.html#gac988496aea8ffc5c8bf92a8b10165fa0',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcontains_5fblock_43',['mi_heap_contains_block',['../group__heap.html#gaa862aa8ed8d57d84cae41fc1022d71af',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fdelete_44',['mi_heap_delete',['../group__heap.html#ga2ab1af8d438819b55319c7ef51d1e409',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fdestroy_45',['mi_heap_destroy',['../group__heap.html#ga9f9c0844edb9717f4feacd79116b8e0d',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fget_5fbacking_46',['mi_heap_get_backing',['../group__heap.html#gac6ac9f0e7be9ab4ff70acfc8dad1235a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fget_5fdefault_47',['mi_heap_get_default',['../group__heap.html#ga14c667a6e2c5d28762d8cb7d4e057909',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmain_48',['mi_heap_main',['../group__heap.html#ga8fc318a65dc79ee8778c6a0e52d12477',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_49',['mi_heap_malloc',['../group__heap.html#gab374e206c7034e0d899fb934e4f4a863',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5faligned_50',['mi_heap_malloc_aligned',['../group__heap.html#ga33f4f05b7fea7af2113c62a4bf882cc5',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5faligned_5fat_51',['mi_heap_malloc_aligned_at',['../group__heap.html#gae7ffc045c3996497a7f3a5f6fe7b8aaa',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5fsmall_52',['mi_heap_malloc_small',['../group__heap.html#ga012c5c8abe22b10043de39ff95909541',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5ftp_53',['mi_heap_malloc_tp',['../group__typed.html#ga02d3e85bf26aa3120132f2552c408a88',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmallocn_54',['mi_heap_mallocn',['../group__heap.html#gab0f755c0b21c387fe8e9024200faa372',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmallocn_5ftp_55',['mi_heap_mallocn_tp',['../group__typed.html#gaa58690658cc6f5dd54c99a9971d4aa7d',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fnew_56',['mi_heap_new',['../group__heap.html#gaa718bb226ec0546ba6d1b6cb32179f3a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fnew_5fex_57',['mi_heap_new_ex',['../group__arenas.html#ga3ae360583f4351aa5267ee7e43008faf',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fnew_5fin_5farena_58',['mi_heap_new_in_arena',['../group__arenas.html#gaaf2d9976576d5efd5544be12848af949',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fof_59',['mi_heap_of',['../group__heap.html#ga424b69f285109b2515631d59f5eecb1b',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealloc_60',['mi_heap_realloc',['../group__heap.html#gac5252d6a2e510bd349e4fcb452e6a93a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealloc_5faligned_61',['mi_heap_realloc_aligned',['../group__heap.html#gaccf8c249872f30bf1c2493a09197d734',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealloc_5faligned_5fat_62',['mi_heap_realloc_aligned_at',['../group__heap.html#ga6df988a7219d5707f010d5f3eb0dc3f5',1,'mimalloc-doc.h']]], - ['mi_5fheap_5freallocf_63',['mi_heap_reallocf',['../group__heap.html#gae7cd171425bee04c683c65a3701f0b4a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5freallocn_64',['mi_heap_reallocn',['../group__heap.html#gaccf7bfe10ce510a000d3547d9cf7fa29',1,'mimalloc-doc.h']]], - ['mi_5fheap_5freallocn_5ftp_65',['mi_heap_reallocn_tp',['../group__typed.html#ga817e315c24133294caad4d50470ff313',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealpath_66',['mi_heap_realpath',['../group__heap.html#ga55545a3ec6da29c5b4f62e540ecac1e2',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_67',['mi_heap_recalloc',['../group__zeroinit.html#gad1a0d325d930eeb80f25e3fea37aacde',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_5faligned_68',['mi_heap_recalloc_aligned',['../group__zeroinit.html#ga87ddd674bf1c67237d780d0b9e0f0f32',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_5faligned_5fat_69',['mi_heap_recalloc_aligned_at',['../group__zeroinit.html#ga07b5bcbaf00d0d2e598c232982588496',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_5ftp_70',['mi_heap_recalloc_tp',['../group__typed.html#ga7bf06ad296e20d1d74c90d64294eecec',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frezalloc_71',['mi_heap_rezalloc',['../group__zeroinit.html#ga8d8b7ebb24b513cd84d1a696048da60d',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frezalloc_5faligned_72',['mi_heap_rezalloc_aligned',['../group__zeroinit.html#ga5129f6dc46ee1613d918820a8a0533a7',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frezalloc_5faligned_5fat_73',['mi_heap_rezalloc_aligned_at',['../group__zeroinit.html#ga2bafa79c3f98ea74882349d44cffa5d9',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fset_5fdefault_74',['mi_heap_set_default',['../group__heap.html#ga349b677dec7da5eacdbc7a385bd62a4a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fset_5fnuma_5faffinity_75',['mi_heap_set_numa_affinity',['../group__heap.html#gaab93abbc0624f570e2900f95cac18591',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fget_76',['mi_heap_stats_get',['../group__stats.html#ga2654195cef93ed3613e98b59a19a296f',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fget_5fjson_77',['mi_heap_stats_get_json',['../group__stats.html#gacb42dede4c38c38f2014b5450c5fcba8',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fmerge_5fto_5fsubproc_78',['mi_heap_stats_merge_to_subproc',['../group__stats.html#ga5a34328f846c661aa560f4769cfc15f1',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fprint_5fout_79',['mi_heap_stats_print_out',['../group__stats.html#gafea365d7842f5f5bc1f235bbc303d585',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstrdup_80',['mi_heap_strdup',['../group__heap.html#ga5754e09ccc51dd6bc73885bb6ea21b7a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstrndup_81',['mi_heap_strndup',['../group__heap.html#gad224df78f1fbee942df8adf023e12cf3',1,'mimalloc-doc.h']]], - ['mi_5fheap_5ft_82',['mi_heap_t',['../group__heap.html#ga34a47cde5a5b38c29f1aa3c5e76943c2',1,'mimalloc-doc.h']]], - ['mi_5fheap_5ftheap_83',['mi_heap_theap',['../group__theap.html#ga7905a73865733da8a43339a500533180',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fvisit_5fblocks_84',['mi_heap_visit_blocks',['../group__analysis.html#gae69fcff3324f8e419af850e34cf54c46',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fvisit_5ffun_85',['mi_heap_visit_fun',['../group__subproc.html#ga7dcbc5863df9989b45e7051999448a4a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_86',['mi_heap_zalloc',['../group__heap.html#gabebc796399619d964d8db77aa835e8c1',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5faligned_87',['mi_heap_zalloc_aligned',['../group__heap.html#ga6466bde8b5712aa34e081a8317f9f471',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5faligned_5fat_88',['mi_heap_zalloc_aligned_at',['../group__heap.html#ga484e3d01cd174f78c7e53370e5a7c819',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5fsmall_89',['mi_heap_zalloc_small',['../group__heap.html#ga0994f44cc595f9c1840a9af162d98597',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5ftp_90',['mi_heap_zalloc_tp',['../group__typed.html#ga0aaa1aa747d8863e844c37e424ed096c',1,'mimalloc-doc.h']]], - ['mi_5fis_5fin_5fheap_5fregion_91',['mi_is_in_heap_region',['../group__heap.html#ga5f071b10d4df1c3658e04e7fd67a94e6',1,'mimalloc-doc.h']]], - ['mi_5fis_5fredirected_92',['mi_is_redirected',['../group__extended.html#gaad25050b19f30cd79397b227e0157a3f',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_93',['mi_malloc',['../group__malloc.html#gae1dd97b542420c87ae085e822b1229e8',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5faligned_94',['mi_malloc_aligned',['../group__aligned.html#ga69578ff1a98ca16e1dcd02c0995cd65c',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5faligned_5fat_95',['mi_malloc_aligned_at',['../group__aligned.html#ga2022f71b95a7cd6cce1b6e07752ae8ca',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fcsize_96',['mi_malloc_csize',['../group__constantsize.html#ga9cd44d925d34f7180922e12bbf594571',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fgood_5fsize_97',['mi_malloc_good_size',['../group__posix.html#ga9d23ac7885fed7413c11d8e0ffa31071',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fsize_98',['mi_malloc_size',['../group__posix.html#ga4531c9e775bb3ae12db57c1ba8a5d7de',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fsmall_99',['mi_malloc_small',['../group__extended.html#ga7f050bc6b897da82692174f5fce59cde',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5ftp_100',['mi_malloc_tp',['../group__typed.html#ga0619a62c5fd886f1016030abe91f0557',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fusable_5fsize_101',['mi_malloc_usable_size',['../group__posix.html#ga06d07cf357bbac5c73ba5d0c0c421e17',1,'mimalloc-doc.h']]], - ['mi_5fmallocn_102',['mi_mallocn',['../group__malloc.html#ga61f46bade3db76ca24aaafedc40de7b6',1,'mimalloc-doc.h']]], - ['mi_5fmallocn_5ftp_103',['mi_mallocn_tp',['../group__typed.html#gae5cb6e0fafc9f23169c5622e077afe8b',1,'mimalloc-doc.h']]], - ['mi_5fmanage_5fos_5fmemory_104',['mi_manage_os_memory',['../group__arenas.html#ga4c6486a1fdcd7a423b5f25fe4be8e0cf',1,'mimalloc-doc.h']]], - ['mi_5fmanage_5fos_5fmemory_5fex_105',['mi_manage_os_memory_ex',['../group__arenas.html#ga41ce8525d77bbb60f618fa1029994f6e',1,'mimalloc-doc.h']]], - ['mi_5fmbsdup_106',['mi_mbsdup',['../group__posix.html#ga7b82a44094fdec4d2084eb4288a979b0',1,'mimalloc-doc.h']]], - ['mi_5fmemalign_107',['mi_memalign',['../group__posix.html#ga726867f13fd29ca36064954c0285b1d8',1,'mimalloc-doc.h']]], - ['mi_5fnew_108',['mi_new',['../group__cpp.html#ga633d96e3bc7011f960df9f3b2731fc6a',1,'mimalloc-doc.h']]], - ['mi_5fnew_5faligned_109',['mi_new_aligned',['../group__cpp.html#ga79c54da0b4b4ce9fcc11d2f6ef6675f8',1,'mimalloc-doc.h']]], - ['mi_5fnew_5faligned_5fnothrow_110',['mi_new_aligned_nothrow',['../group__cpp.html#ga92ae00b6dd64406c7e64557711ec04b7',1,'mimalloc-doc.h']]], - ['mi_5fnew_5fn_111',['mi_new_n',['../group__cpp.html#gadd11b85c15d21d308386844b5233856c',1,'mimalloc-doc.h']]], - ['mi_5fnew_5fnothrow_112',['mi_new_nothrow',['../group__cpp.html#ga5cb4f120d1f7296074256215aa9a9e54',1,'mimalloc-doc.h']]], - ['mi_5fnew_5frealloc_113',['mi_new_realloc',['../group__cpp.html#ga6867d89baf992728e0cc20a1f47db4d0',1,'mimalloc-doc.h']]], - ['mi_5fnew_5freallocn_114',['mi_new_reallocn',['../group__cpp.html#gaace912ce086682d56f3ce9f7638d9d67',1,'mimalloc-doc.h']]], - ['mi_5foption_5fabandoned_5fpage_5fpurge_115',['mi_option_abandoned_page_purge',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca11e62ed69200a489a5be955582078c0c',1,'mimalloc-doc.h']]], - ['mi_5foption_5fabandoned_5freclaim_5fon_5ffree_116',['mi_option_abandoned_reclaim_on_free',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca009e4b5684922ce664d73d2a8e1698d9',1,'mimalloc-doc.h']]], - ['mi_5foption_5fallow_5flarge_5fos_5fpages_117',['mi_option_allow_large_os_pages',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7cc4804ced69004fa42a9a136a9ba556',1,'mimalloc-doc.h']]], - ['mi_5foption_5fallow_5fthp_118',['mi_option_allow_thp',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca04c45931ce2c82a20d08ca1471e74087',1,'mimalloc-doc.h']]], - ['mi_5foption_5farena_5feager_5fcommit_119',['mi_option_arena_eager_commit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafd0c5ddbc4b59fd8b5216871728167a5',1,'mimalloc-doc.h']]], - ['mi_5foption_5farena_5fmax_5fobject_5fsize_120',['mi_option_arena_max_object_size',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca4dee58d60bc95ce0eeef759c61fd849b',1,'mimalloc-doc.h']]], - ['mi_5foption_5farena_5fpurge_5fmult_121',['mi_option_arena_purge_mult',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8236501f1ab45d26e6fd885d191a2b5e',1,'mimalloc-doc.h']]], - ['mi_5foption_5farena_5freserve_122',['mi_option_arena_reserve',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cab1c88e23ae290bbeec824038a97959de',1,'mimalloc-doc.h']]], - ['mi_5foption_5fdestroy_5fon_5fexit_123',['mi_option_destroy_on_exit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca6364331e305e7d3c0218b058ff3afc88',1,'mimalloc-doc.h']]], - ['mi_5foption_5fdisable_124',['mi_option_disable',['../group__options.html#gaebf6ff707a2e688ebb1a2296ca564054',1,'mimalloc-doc.h']]], - ['mi_5foption_5fdisallow_5farena_5falloc_125',['mi_option_disallow_arena_alloc',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeae1696100e4057ffc4182730cc04e40',1,'mimalloc-doc.h']]], - ['mi_5foption_5fdisallow_5fos_5falloc_126',['mi_option_disallow_os_alloc',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cadcfb5a09580361b1be65901d2d812de6',1,'mimalloc-doc.h']]], - ['mi_5foption_5feager_5fcommit_127',['mi_option_eager_commit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca1e8de72c93da7ff22d91e1e27b52ac2b',1,'mimalloc-doc.h']]], - ['mi_5foption_5feager_5fcommit_5fdelay_128',['mi_option_eager_commit_delay',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca17a190c25be381142d87e0468c4c068c',1,'mimalloc-doc.h']]], - ['mi_5foption_5fenable_129',['mi_option_enable',['../group__options.html#ga04180ae41b0d601421dd62ced40ca050',1,'mimalloc-doc.h']]], - ['mi_5foption_5fgeneric_5fcollect_130',['mi_option_generic_collect',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8b5815a577955fe8125f2b964f41ee7f',1,'mimalloc-doc.h']]], - ['mi_5foption_5fget_131',['mi_option_get',['../group__options.html#ga7e8af195cc81d3fa64ccf2662caa565a',1,'mimalloc-doc.h']]], - ['mi_5foption_5fget_5fclamp_132',['mi_option_get_clamp',['../group__options.html#ga96ad9c406338bd314cfe878cfc9bf723',1,'mimalloc-doc.h']]], - ['mi_5foption_5fget_5fsize_133',['mi_option_get_size',['../group__options.html#ga274db5a6ac87cc24ef0b23e7006ed02c',1,'mimalloc-doc.h']]], - ['mi_5foption_5fguarded_5fmax_134',['mi_option_guarded_max',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca96077da4e5c841edeec0c4d6e45a5ffa',1,'mimalloc-doc.h']]], - ['mi_5foption_5fguarded_5fmin_135',['mi_option_guarded_min',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca00734c528aae52784654a8db996ae62b',1,'mimalloc-doc.h']]], - ['mi_5foption_5fguarded_5fprecise_136',['mi_option_guarded_precise',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca1eaf89bb1698f988cec27b66e546760c',1,'mimalloc-doc.h']]], - ['mi_5foption_5fguarded_5fsample_5frate_137',['mi_option_guarded_sample_rate',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeb3c75b1a838eb42106750979d0888ec',1,'mimalloc-doc.h']]], - ['mi_5foption_5fguarded_5fsample_5fseed_138',['mi_option_guarded_sample_seed',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7b1e41ece69f9abbe5ba81d9ea7a3dc0',1,'mimalloc-doc.h']]], - ['mi_5foption_5fis_5fenabled_139',['mi_option_is_enabled',['../group__options.html#ga459ad98f18b3fc9275474807fe0ca188',1,'mimalloc-doc.h']]], - ['mi_5foption_5fmax_5ferrors_140',['mi_option_max_errors',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caec6ecbe29d46a48205ed8823a8a52a6a',1,'mimalloc-doc.h']]], - ['mi_5foption_5fmax_5fsegment_5freclaim_141',['mi_option_max_segment_reclaim',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa9ad9005d7017c8c30ad2d6ba31db909',1,'mimalloc-doc.h']]], - ['mi_5foption_5fmax_5fvabits_142',['mi_option_max_vabits',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caabf8c9793d421ce28361b6d93c432de9',1,'mimalloc-doc.h']]], - ['mi_5foption_5fmax_5fwarnings_143',['mi_option_max_warnings',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caf9595921087e942602ee079158762665',1,'mimalloc-doc.h']]], - ['mi_5foption_5fminimal_5fpurge_5fsize_144',['mi_option_minimal_purge_size',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca5635112095936356425d9a01910afff7',1,'mimalloc-doc.h']]], - ['mi_5foption_5fos_5ftag_145',['mi_option_os_tag',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca4b74ae2a69e445de6c2361b73c1d14bf',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpage_5fcommit_5fon_5fdemand_146',['mi_option_page_commit_on_demand',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca679d9ee91221bffba4d5aafe488a8d7f',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpage_5fcross_5fthread_5fmax_5freclaim_147',['mi_option_page_cross_thread_max_reclaim',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca671e806447c15a53c7d593afdd8d45c5',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpage_5ffull_5fretain_148',['mi_option_page_full_retain',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeede7d0652e391940357e1c054d29829',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpage_5fmax_5fcandidates_149',['mi_option_page_max_candidates',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cad13382757101aa4fbc4984fa0f917133',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpage_5fmax_5freclaim_150',['mi_option_page_max_reclaim',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca11a13739488294318ffc03eea378f5a4',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpage_5freclaim_5fon_5ffree_151',['mi_option_page_reclaim_on_free',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cac16e3f838e861338fb2357b430f6974c',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpagemap_5fcommit_152',['mi_option_pagemap_commit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa88f56e3bccce85542dc2b3297024236',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpurge_5fdecommits_153',['mi_option_purge_decommits',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca9d15c5e3d2115eef681c17e4dd5ab9a4',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpurge_5fdelay_154',['mi_option_purge_delay',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cadd351e615acd8563529c20a347be7290',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpurge_5fextend_5fdelay_155',['mi_option_purge_extend_delay',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca02005f164bdf03f5f00c5be726adf487',1,'mimalloc-doc.h']]], - ['mi_5foption_5freserve_5fhuge_5fos_5fpages_156',['mi_option_reserve_huge_os_pages',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caca7ed041be3b0b9d0b82432c7bf41af2',1,'mimalloc-doc.h']]], - ['mi_5foption_5freserve_5fhuge_5fos_5fpages_5fat_157',['mi_option_reserve_huge_os_pages_at',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa13e7926d4339d2aa6fbf61d4473fd5c',1,'mimalloc-doc.h']]], - ['mi_5foption_5freserve_5fos_5fmemory_158',['mi_option_reserve_os_memory',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafbf4999c828cf79a0fb2de65d23f7333',1,'mimalloc-doc.h']]], - ['mi_5foption_5fretry_5fon_5foom_159',['mi_option_retry_on_oom',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8f51df355bf6651db899e6085b54865e',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_160',['mi_option_set',['../group__options.html#gaf84921c32375e25754dc2ee6a911fa60',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_5fdefault_161',['mi_option_set_default',['../group__options.html#ga7ef623e440e6e5545cb08c94e71e4b90',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_5fenabled_162',['mi_option_set_enabled',['../group__options.html#ga9a13d05fcb77489cb06d4d017ebd8bed',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_5fenabled_5fdefault_163',['mi_option_set_enabled_default',['../group__options.html#ga65518b69ec5d32336b50e07f74b3f629',1,'mimalloc-doc.h']]], - ['mi_5foption_5fshow_5ferrors_164',['mi_option_show_errors',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafbf4822e5c00732c5984b32a032837f0',1,'mimalloc-doc.h']]], - ['mi_5foption_5fshow_5fstats_165',['mi_option_show_stats',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca0957ef73b2550764b4840edf48422fda',1,'mimalloc-doc.h']]], - ['mi_5foption_5ft_166',['mi_option_t',['../group__options.html#gafebf7ed116adb38ae5218bc3ce06884c',1,'mimalloc-doc.h']]], - ['mi_5foption_5ftarget_5fsegments_5fper_5fthread_167',['mi_option_target_segments_per_thread',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca874f36a7a1e51f7baee4a78b2830b643',1,'mimalloc-doc.h']]], - ['mi_5foption_5fuse_5fnuma_5fnodes_168',['mi_option_use_numa_nodes',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca0ac33a18f6b659fcfaf44efb0bab1b74',1,'mimalloc-doc.h']]], - ['mi_5foption_5fverbose_169',['mi_option_verbose',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7c8b7bf5281c581bad64f5daa6442777',1,'mimalloc-doc.h']]], - ['mi_5foption_5fvisit_5fabandoned_170',['mi_option_visit_abandoned',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca38c67733a3956a1f4eeaca89fab9e78e',1,'mimalloc-doc.h']]], - ['mi_5foptions_5fprint_171',['mi_options_print',['../group__options.html#gaecfcd3ed90af99932670d881de1fa8f1',1,'mimalloc-doc.h']]], - ['mi_5foptions_5fprint_5fout_172',['mi_options_print_out',['../group__options.html#ga1dd50c7533baec4cc6c1a2e9adf05dd2',1,'mimalloc-doc.h']]], - ['mi_5foutput_5ffun_173',['mi_output_fun',['../group__extended.html#gadf31cea7d0332a81c8b882cbbdbadb8d',1,'mimalloc-doc.h']]], - ['mi_5fposix_5fmemalign_174',['mi_posix_memalign',['../group__posix.html#gacff84f226ba9feb2031b8992e5579447',1,'mimalloc-doc.h']]], - ['mi_5fprocess_5finfo_175',['mi_process_info',['../group__stats.html#ga7d862c2affd5790381da14eb102a364d',1,'mimalloc-doc.h']]], - ['mi_5fprocess_5finfo_5fprint_176',['mi_process_info_print',['../group__stats.html#gaf887e7aa75f13242c4e7ab95d8390536',1,'mimalloc-doc.h']]], - ['mi_5fprocess_5finfo_5fprint_5fout_177',['mi_process_info_print_out',['../group__stats.html#ga03ac88d3efee2ae7c34f5340502f4732',1,'mimalloc-doc.h']]], - ['mi_5fpvalloc_178',['mi_pvalloc',['../group__posix.html#ga644bebccdbb2821542dd8c7e7641f476',1,'mimalloc-doc.h']]], - ['mi_5frealloc_179',['mi_realloc',['../group__malloc.html#ga0621af6a5e3aa384e6a1b548958bf583',1,'mimalloc-doc.h']]], - ['mi_5frealloc_5faligned_180',['mi_realloc_aligned',['../group__aligned.html#ga5d7a46d054b4d7abe9d8d2474add2edf',1,'mimalloc-doc.h']]], - ['mi_5frealloc_5faligned_5fat_181',['mi_realloc_aligned_at',['../group__aligned.html#gad06dcf2bb8faadb2c8ea61ee5d24bbf6',1,'mimalloc-doc.h']]], - ['mi_5freallocarr_182',['mi_reallocarr',['../group__posix.html#ga7e1934d60a3e697950eeb48e042bfad5',1,'mimalloc-doc.h']]], - ['mi_5freallocarray_183',['mi_reallocarray',['../group__posix.html#gadfeccb72748a2f6305474a37d9d57bce',1,'mimalloc-doc.h']]], - ['mi_5freallocf_184',['mi_reallocf',['../group__malloc.html#ga4dc3a4067037b151a64629fe8a332641',1,'mimalloc-doc.h']]], - ['mi_5freallocn_185',['mi_reallocn',['../group__malloc.html#ga8bddfb4a1270a0854bbcf44cb3980467',1,'mimalloc-doc.h']]], - ['mi_5freallocn_5ftp_186',['mi_reallocn_tp',['../group__typed.html#ga1158b49a55dfa81f58a4426a7578f523',1,'mimalloc-doc.h']]], - ['mi_5frealpath_187',['mi_realpath',['../group__malloc.html#ga94c3afcc086e85d75a57e9f76b9b71dd',1,'mimalloc-doc.h']]], - ['mi_5frecalloc_188',['mi_recalloc',['../group__zeroinit.html#ga23a0fbb452b5dce8e31fab1a1958cacc',1,'mimalloc-doc.h']]], - ['mi_5frecalloc_5faligned_189',['mi_recalloc_aligned',['../group__zeroinit.html#ga3e2169b48683aa0ab64f813fd68d839e',1,'mimalloc-doc.h']]], - ['mi_5frecalloc_5faligned_5fat_190',['mi_recalloc_aligned_at',['../group__zeroinit.html#gaae25e4ddedd4e0fb61b1a8bd5d452750',1,'mimalloc-doc.h']]], - ['mi_5fregister_5fdeferred_5ffree_191',['mi_register_deferred_free',['../group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece',1,'mimalloc-doc.h']]], - ['mi_5fregister_5ferror_192',['mi_register_error',['../group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45',1,'mimalloc-doc.h']]], - ['mi_5fregister_5foutput_193',['mi_register_output',['../group__extended.html#gae5b17ff027cd2150b43a33040250cf3f',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fhuge_5fos_5fpages_5fat_194',['mi_reserve_huge_os_pages_at',['../group__arenas.html#ga7795a13d20087447281858d2c771cca1',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fhuge_5fos_5fpages_5fat_5fex_195',['mi_reserve_huge_os_pages_at_ex',['../group__arenas.html#ga591aab1c2bc2ca920e33f0f9f9cb5c52',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fhuge_5fos_5fpages_5finterleave_196',['mi_reserve_huge_os_pages_interleave',['../group__arenas.html#ga3132f521fb756fc0e8ec0b74fb58df50',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fos_5fmemory_197',['mi_reserve_os_memory',['../group__arenas.html#ga00ec3324b6b2591c7fe3677baa30a767',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fos_5fmemory_5fex_198',['mi_reserve_os_memory_ex',['../group__arenas.html#ga32f519797fd9a81acb4f52d36e6d751b',1,'mimalloc-doc.h']]], - ['mi_5frezalloc_199',['mi_rezalloc',['../group__zeroinit.html#gadfd34cd7b4f2bbda7ae06367a6360756',1,'mimalloc-doc.h']]], - ['mi_5frezalloc_5faligned_200',['mi_rezalloc_aligned',['../group__zeroinit.html#ga4d02404fe1e7db00beb65f185e012caa',1,'mimalloc-doc.h']]], - ['mi_5frezalloc_5faligned_5fat_201',['mi_rezalloc_aligned_at',['../group__zeroinit.html#ga6843a88285bbfcc3bdfccc60aafd1270',1,'mimalloc-doc.h']]], - ['mi_5fsmall_5fsize_5fmax_202',['MI_SMALL_SIZE_MAX',['../group__extended.html#ga1ea64283508718d9d645c38efc2f4305',1,'mimalloc-doc.h']]], - ['mi_5fstat_5fversion_203',['MI_STAT_VERSION',['../group__stats.html#gac4411cf3eed23719e511b9e1128888e0',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fas_5fjson_204',['mi_stats_as_json',['../group__stats.html#ga6ad266c70a0eabf6c26248aed1af6fa1',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fget_205',['mi_stats_get',['../group__stats.html#gaa26d5060ad21b1331a150bb92f6f7086',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fget_5fbin_5fsize_206',['mi_stats_get_bin_size',['../group__stats.html#gadecd549d1a7216af596322ac27e78b0d',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fget_5fjson_207',['mi_stats_get_json',['../group__stats.html#gadaa3be15899cc82e314fa7d3f8e2afbd',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fmerge_208',['mi_stats_merge',['../group__stats.html#ga854b1de8cb067c7316286c28b2fcd3d1',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fprint_209',['mi_stats_print',['../group__stats.html#ga2d126e5c62d3badc35445e5d84166df2',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fprint_5fout_210',['mi_stats_print_out',['../group__stats.html#ga537f13b299ddf801e49a5a94fde02c79',1,'mimalloc-doc.h']]], - ['mi_5fstats_5freset_211',['mi_stats_reset',['../group__stats.html#ga3bb8468b8cfcc6e2a61d98aee85c5f99',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fs_212',['mi_stats_s',['../group__stats.html#structmi__stats__s',1,'']]], - ['mi_5fstats_5ft_5fdecl_213',['mi_stats_t_decl',['../group__stats.html#ga961d0f9f0327ebc9bd8a0b84f6dcb410',1,'mimalloc-doc.h']]], - ['mi_5fstl_5fallocator_214',['mi_stl_allocator',['../group__cpp.html#structmi__stl__allocator',1,'']]], - ['mi_5fstrdup_215',['mi_strdup',['../group__malloc.html#ga245ac90ebc2cfdd17de599e5fea59889',1,'mimalloc-doc.h']]], - ['mi_5fstrndup_216',['mi_strndup',['../group__malloc.html#ga486d0d26b3b3794f6d1cdb41a9aed92d',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fadd_5fcurrent_5fthread_217',['mi_subproc_add_current_thread',['../group__subproc.html#gadbc53414eb68b275588ec001ce1ddc7c',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fcurrent_218',['mi_subproc_current',['../group__subproc.html#gaabc4db6585bf16099e43e60d64364950',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fdelete_219',['mi_subproc_delete',['../group__subproc.html#gaa7d263e9429bac9ac8345c9d25de610e',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fdestroy_220',['mi_subproc_destroy',['../group__subproc.html#gada12e4d9bd382935c398ec324d0391f9',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fheap_5fstats_5fprint_5fout_221',['mi_subproc_heap_stats_print_out',['../group__stats.html#ga19f1a04bd972e11e4f3b586f25229d5b',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fid_5ft_222',['mi_subproc_id_t',['../group__subproc.html#ga8c0bcd1fee27c7641e9c3c0d991b3b7d',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fmain_223',['mi_subproc_main',['../group__subproc.html#ga2ecba0d7ebdc99e71bb985c4a1609806',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fnew_224',['mi_subproc_new',['../group__subproc.html#ga8068cac328e41fa2170faef707315243',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fstats_5fget_225',['mi_subproc_stats_get',['../group__stats.html#ga5ec3f9058eec804924711ab38ba64b51',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fstats_5fget_5fjson_226',['mi_subproc_stats_get_json',['../group__stats.html#gad1cf918f08b6506e65ae134423477a0b',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fstats_5fprint_5fout_227',['mi_subproc_stats_print_out',['../group__stats.html#ga59ec89969837db414a6bce8b58915624',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fvisit_5fheaps_228',['mi_subproc_visit_heaps',['../group__subproc.html#gafd2136e7f167b5cd02b6ab00bed0ac93',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fcalloc_229',['mi_theap_calloc',['../group__theap.html#gadbf8f27bfe9edf7834e19d32cab63edc',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fcollect_230',['mi_theap_collect',['../group__theap.html#ga9377a43095b8dd14996613ceb73d9a32',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fget_5fdefault_231',['mi_theap_get_default',['../group__theap.html#ga78afbb07c1e0cd333c7539cf7403dfd5',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_232',['mi_theap_malloc',['../group__theap.html#gac7308e029c59e6675dd5ab5ec54f304a',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_5faligned_233',['mi_theap_malloc_aligned',['../group__theap.html#ga84503fc2bd83fbfb7f789b7d80d76f36',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_5fcsize_234',['mi_theap_malloc_csize',['../group__constantsize.html#ga105bd667da91a15d113fe79be04d625c',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_5fsmall_235',['mi_theap_malloc_small',['../group__theap.html#ga1fc65302a212daaecb21abda96ff247c',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5frealloc_236',['mi_theap_realloc',['../group__theap.html#ga11943128dc354313c4189e72f4299dbb',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fset_5fdefault_237',['mi_theap_set_default',['../group__theap.html#ga04420ce52785019412397e4de24a984a',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5ft_238',['mi_theap_t',['../group__theap.html#ga8722f6ac4f638b4692e06ddfddc844f8',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fzalloc_239',['mi_theap_zalloc',['../group__theap.html#gaa7fd973dcd8c75eb05707bdeffbc06f7',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fzalloc_5fcsize_240',['mi_theap_zalloc_csize',['../group__constantsize.html#ga1addf755829380581dd9c3c9175e376f',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fzalloc_5fsmall_241',['mi_theap_zalloc_small',['../group__theap.html#gaa85fccf9d40af3e826c5932aded1d534',1,'mimalloc-doc.h']]], - ['mi_5fthread_5fdone_242',['mi_thread_done',['../group__extended.html#ga0ae4581e85453456a0d658b2b98bf7bf',1,'mimalloc-doc.h']]], - ['mi_5fthread_5finit_243',['mi_thread_init',['../group__extended.html#gaf8e73efc2cbca9ebfdfb166983a04c17',1,'mimalloc-doc.h']]], - ['mi_5fthread_5fset_5fin_5fthreadpool_244',['mi_thread_set_in_threadpool',['../group__extended.html#gada899104276aac52539eb4de7c020f1f',1,'mimalloc-doc.h']]], - ['mi_5fusable_5fsize_245',['mi_usable_size',['../group__malloc.html#ga089c859d9eddc5f9b4bd946cd53cebee',1,'mimalloc-doc.h']]], - ['mi_5fvalloc_246',['mi_valloc',['../group__posix.html#ga50cafb9722020402f065de93799f64ca',1,'mimalloc-doc.h']]], - ['mi_5fversion_247',['mi_version',['../group__extended.html#gae9ddddcf14a58d0224ef3c8778e5b057',1,'mimalloc-doc.h']]], - ['mi_5fwcsdup_248',['mi_wcsdup',['../group__posix.html#gaa9fd7f25c9ac3a20e89b33bd6e383fcf',1,'mimalloc-doc.h']]], - ['mi_5fwdupenv_5fs_249',['mi_wdupenv_s',['../group__posix.html#ga6ac6a6a8f3c96f1af24bb8d0439cbbd1',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_250',['mi_zalloc',['../group__malloc.html#gae6e38c4403247a7b40d80419e093bfb8',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5faligned_251',['mi_zalloc_aligned',['../group__aligned.html#gaac7d0beb782f9b9ac31f47492b130f82',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5faligned_5fat_252',['mi_zalloc_aligned_at',['../group__aligned.html#ga7c1778805ce50ebbf02ccbd5e39d5dba',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5fcsize_253',['mi_zalloc_csize',['../group__constantsize.html#gadca6d340ee15b5c70695ed82d95c6cec',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5fsmall_254',['mi_zalloc_small',['../group__extended.html#ga51c47637e81df0e2f13a2d7a2dec123e',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5ftp_255',['mi_zalloc_tp',['../group__typed.html#gac77a61bdaf680a803785fe307820b48c',1,'mimalloc-doc.h']]], - ['modes_256',['Build Modes',['../modes.html',1,'']]] + ['mi_5ffree_5fcsize_5fnonnull_30',['mi_free_csize_nonnull',['../group__constantsize.html#ga38a93d373a9dc8edc8bca83edb6af5c5',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsize_31',['mi_free_size',['../group__posix.html#gae01389eedab8d67341ff52e2aad80ebb',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsize_5faligned_32',['mi_free_size_aligned',['../group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsize_5fnonnull_33',['mi_free_size_nonnull',['../group__posix.html#ga796602dae2a89797026f72e0c0240f32',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsmall_34',['mi_free_small',['../group__extended.html#ga2a8469764c38dcaf878f36e5f50e3610',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsmall_5fnonnull_35',['mi_free_small_nonnull',['../group__extended.html#ga22890f54e5be433837069c8e97fdf379',1,'mimalloc-doc.h']]], + ['mi_5ffree_5ftp_36',['mi_free_tp',['../group__typed.html#gafc80000c300f02bdd4fe695eb53ca70a',1,'mimalloc-doc.h']]], + ['mi_5fgood_5fsize_37',['mi_good_size',['../group__malloc.html#gac057927cd06c854b45fe7847e921bd47',1,'mimalloc-doc.h']]], + ['mi_5fheap_5farea_5ft_38',['mi_heap_area_t',['../group__analysis.html#structmi__heap__area__t',1,'']]], + ['mi_5fheap_5fcalloc_39',['mi_heap_calloc',['../group__heap.html#gac0098aaf231d3e9586c73136d5df95da',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcalloc_5faligned_40',['mi_heap_calloc_aligned',['../group__heap.html#gacafcc26df827c7a7de5e850217566108',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcalloc_5faligned_5fat_41',['mi_heap_calloc_aligned_at',['../group__heap.html#gaa42ec2079989c4374f2c331d9b35f4e4',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcalloc_5ftp_42',['mi_heap_calloc_tp',['../group__typed.html#ga953f22101bb354a8b6de2a6d6fda9b98',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcheck_5fowned_43',['mi_heap_check_owned',['../group__heap.html#ga0d67c1789faaa15ff366c024fcaf6377',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcollect_44',['mi_heap_collect',['../group__heap.html#ga7922f7495cde30b1984d0e6072419298',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcontains_45',['mi_heap_contains',['../group__heap.html#gac988496aea8ffc5c8bf92a8b10165fa0',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcontains_5fblock_46',['mi_heap_contains_block',['../group__heap.html#gaa862aa8ed8d57d84cae41fc1022d71af',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fdelete_47',['mi_heap_delete',['../group__heap.html#ga2ab1af8d438819b55319c7ef51d1e409',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fdestroy_48',['mi_heap_destroy',['../group__heap.html#ga9f9c0844edb9717f4feacd79116b8e0d',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fget_5fbacking_49',['mi_heap_get_backing',['../group__heap.html#gac6ac9f0e7be9ab4ff70acfc8dad1235a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fget_5fdefault_50',['mi_heap_get_default',['../group__heap.html#ga14c667a6e2c5d28762d8cb7d4e057909',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmain_51',['mi_heap_main',['../group__heap.html#ga8fc318a65dc79ee8778c6a0e52d12477',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_52',['mi_heap_malloc',['../group__heap.html#gab374e206c7034e0d899fb934e4f4a863',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5faligned_53',['mi_heap_malloc_aligned',['../group__heap.html#ga33f4f05b7fea7af2113c62a4bf882cc5',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5faligned_5fat_54',['mi_heap_malloc_aligned_at',['../group__heap.html#gae7ffc045c3996497a7f3a5f6fe7b8aaa',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5fsmall_55',['mi_heap_malloc_small',['../group__heap.html#ga012c5c8abe22b10043de39ff95909541',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5ftp_56',['mi_heap_malloc_tp',['../group__typed.html#ga02d3e85bf26aa3120132f2552c408a88',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmallocn_57',['mi_heap_mallocn',['../group__heap.html#gab0f755c0b21c387fe8e9024200faa372',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmallocn_5ftp_58',['mi_heap_mallocn_tp',['../group__typed.html#gaa58690658cc6f5dd54c99a9971d4aa7d',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fnew_59',['mi_heap_new',['../group__heap.html#gaa718bb226ec0546ba6d1b6cb32179f3a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fnew_5fex_60',['mi_heap_new_ex',['../group__arenas.html#ga3ae360583f4351aa5267ee7e43008faf',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fnew_5fin_5farena_61',['mi_heap_new_in_arena',['../group__arenas.html#gaaf2d9976576d5efd5544be12848af949',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fof_62',['mi_heap_of',['../group__heap.html#ga424b69f285109b2515631d59f5eecb1b',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealloc_63',['mi_heap_realloc',['../group__heap.html#gac5252d6a2e510bd349e4fcb452e6a93a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealloc_5faligned_64',['mi_heap_realloc_aligned',['../group__heap.html#gaccf8c249872f30bf1c2493a09197d734',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealloc_5faligned_5fat_65',['mi_heap_realloc_aligned_at',['../group__heap.html#ga6df988a7219d5707f010d5f3eb0dc3f5',1,'mimalloc-doc.h']]], + ['mi_5fheap_5freallocf_66',['mi_heap_reallocf',['../group__heap.html#gae7cd171425bee04c683c65a3701f0b4a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5freallocn_67',['mi_heap_reallocn',['../group__heap.html#gaccf7bfe10ce510a000d3547d9cf7fa29',1,'mimalloc-doc.h']]], + ['mi_5fheap_5freallocn_5ftp_68',['mi_heap_reallocn_tp',['../group__typed.html#ga817e315c24133294caad4d50470ff313',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealpath_69',['mi_heap_realpath',['../group__heap.html#ga55545a3ec6da29c5b4f62e540ecac1e2',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_70',['mi_heap_recalloc',['../group__zeroinit.html#gad1a0d325d930eeb80f25e3fea37aacde',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_5faligned_71',['mi_heap_recalloc_aligned',['../group__zeroinit.html#ga87ddd674bf1c67237d780d0b9e0f0f32',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_5faligned_5fat_72',['mi_heap_recalloc_aligned_at',['../group__zeroinit.html#ga07b5bcbaf00d0d2e598c232982588496',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_5ftp_73',['mi_heap_recalloc_tp',['../group__typed.html#ga7bf06ad296e20d1d74c90d64294eecec',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frezalloc_74',['mi_heap_rezalloc',['../group__zeroinit.html#ga8d8b7ebb24b513cd84d1a696048da60d',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frezalloc_5faligned_75',['mi_heap_rezalloc_aligned',['../group__zeroinit.html#ga5129f6dc46ee1613d918820a8a0533a7',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frezalloc_5faligned_5fat_76',['mi_heap_rezalloc_aligned_at',['../group__zeroinit.html#ga2bafa79c3f98ea74882349d44cffa5d9',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fset_5fdefault_77',['mi_heap_set_default',['../group__heap.html#ga349b677dec7da5eacdbc7a385bd62a4a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fset_5fnuma_5faffinity_78',['mi_heap_set_numa_affinity',['../group__heap.html#gaab93abbc0624f570e2900f95cac18591',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fget_79',['mi_heap_stats_get',['../group__stats.html#ga2654195cef93ed3613e98b59a19a296f',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fget_5fjson_80',['mi_heap_stats_get_json',['../group__stats.html#gacb42dede4c38c38f2014b5450c5fcba8',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fmerge_5fto_5fsubproc_81',['mi_heap_stats_merge_to_subproc',['../group__stats.html#ga5a34328f846c661aa560f4769cfc15f1',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fprint_5fout_82',['mi_heap_stats_print_out',['../group__stats.html#gafea365d7842f5f5bc1f235bbc303d585',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstrdup_83',['mi_heap_strdup',['../group__heap.html#ga5754e09ccc51dd6bc73885bb6ea21b7a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstrndup_84',['mi_heap_strndup',['../group__heap.html#gad224df78f1fbee942df8adf023e12cf3',1,'mimalloc-doc.h']]], + ['mi_5fheap_5ft_85',['mi_heap_t',['../group__heap.html#ga34a47cde5a5b38c29f1aa3c5e76943c2',1,'mimalloc-doc.h']]], + ['mi_5fheap_5ftheap_86',['mi_heap_theap',['../group__theap.html#ga7905a73865733da8a43339a500533180',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fvisit_5fblocks_87',['mi_heap_visit_blocks',['../group__analysis.html#gae69fcff3324f8e419af850e34cf54c46',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fvisit_5ffun_88',['mi_heap_visit_fun',['../group__subproc.html#ga7dcbc5863df9989b45e7051999448a4a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_89',['mi_heap_zalloc',['../group__heap.html#gabebc796399619d964d8db77aa835e8c1',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5faligned_90',['mi_heap_zalloc_aligned',['../group__heap.html#ga6466bde8b5712aa34e081a8317f9f471',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5faligned_5fat_91',['mi_heap_zalloc_aligned_at',['../group__heap.html#ga484e3d01cd174f78c7e53370e5a7c819',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5fsmall_92',['mi_heap_zalloc_small',['../group__heap.html#ga0994f44cc595f9c1840a9af162d98597',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5ftp_93',['mi_heap_zalloc_tp',['../group__typed.html#ga0aaa1aa747d8863e844c37e424ed096c',1,'mimalloc-doc.h']]], + ['mi_5fis_5fin_5fheap_5fregion_94',['mi_is_in_heap_region',['../group__heap.html#ga5f071b10d4df1c3658e04e7fd67a94e6',1,'mimalloc-doc.h']]], + ['mi_5fis_5fredirected_95',['mi_is_redirected',['../group__extended.html#gaad25050b19f30cd79397b227e0157a3f',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_96',['mi_malloc',['../group__malloc.html#gae1dd97b542420c87ae085e822b1229e8',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5faligned_97',['mi_malloc_aligned',['../group__aligned.html#ga69578ff1a98ca16e1dcd02c0995cd65c',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5faligned_5fat_98',['mi_malloc_aligned_at',['../group__aligned.html#ga2022f71b95a7cd6cce1b6e07752ae8ca',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fcsize_99',['mi_malloc_csize',['../group__constantsize.html#ga9cd44d925d34f7180922e12bbf594571',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fgood_5fsize_100',['mi_malloc_good_size',['../group__posix.html#ga9d23ac7885fed7413c11d8e0ffa31071',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fsize_101',['mi_malloc_size',['../group__posix.html#ga4531c9e775bb3ae12db57c1ba8a5d7de',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fsmall_102',['mi_malloc_small',['../group__extended.html#ga7f050bc6b897da82692174f5fce59cde',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5ftp_103',['mi_malloc_tp',['../group__typed.html#ga0619a62c5fd886f1016030abe91f0557',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fusable_5fsize_104',['mi_malloc_usable_size',['../group__posix.html#ga06d07cf357bbac5c73ba5d0c0c421e17',1,'mimalloc-doc.h']]], + ['mi_5fmallocn_105',['mi_mallocn',['../group__malloc.html#ga61f46bade3db76ca24aaafedc40de7b6',1,'mimalloc-doc.h']]], + ['mi_5fmallocn_5ftp_106',['mi_mallocn_tp',['../group__typed.html#gae5cb6e0fafc9f23169c5622e077afe8b',1,'mimalloc-doc.h']]], + ['mi_5fmanage_5fos_5fmemory_107',['mi_manage_os_memory',['../group__arenas.html#ga4c6486a1fdcd7a423b5f25fe4be8e0cf',1,'mimalloc-doc.h']]], + ['mi_5fmanage_5fos_5fmemory_5fex_108',['mi_manage_os_memory_ex',['../group__arenas.html#ga41ce8525d77bbb60f618fa1029994f6e',1,'mimalloc-doc.h']]], + ['mi_5fmbsdup_109',['mi_mbsdup',['../group__posix.html#ga7b82a44094fdec4d2084eb4288a979b0',1,'mimalloc-doc.h']]], + ['mi_5fmemalign_110',['mi_memalign',['../group__posix.html#ga726867f13fd29ca36064954c0285b1d8',1,'mimalloc-doc.h']]], + ['mi_5fnew_111',['mi_new',['../group__cpp.html#ga633d96e3bc7011f960df9f3b2731fc6a',1,'mimalloc-doc.h']]], + ['mi_5fnew_5faligned_112',['mi_new_aligned',['../group__cpp.html#ga79c54da0b4b4ce9fcc11d2f6ef6675f8',1,'mimalloc-doc.h']]], + ['mi_5fnew_5faligned_5fnothrow_113',['mi_new_aligned_nothrow',['../group__cpp.html#ga92ae00b6dd64406c7e64557711ec04b7',1,'mimalloc-doc.h']]], + ['mi_5fnew_5fn_114',['mi_new_n',['../group__cpp.html#gadd11b85c15d21d308386844b5233856c',1,'mimalloc-doc.h']]], + ['mi_5fnew_5fnothrow_115',['mi_new_nothrow',['../group__cpp.html#ga5cb4f120d1f7296074256215aa9a9e54',1,'mimalloc-doc.h']]], + ['mi_5fnew_5frealloc_116',['mi_new_realloc',['../group__cpp.html#ga6867d89baf992728e0cc20a1f47db4d0',1,'mimalloc-doc.h']]], + ['mi_5fnew_5freallocn_117',['mi_new_reallocn',['../group__cpp.html#gaace912ce086682d56f3ce9f7638d9d67',1,'mimalloc-doc.h']]], + ['mi_5foption_5fabandoned_5fpage_5fpurge_118',['mi_option_abandoned_page_purge',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca11e62ed69200a489a5be955582078c0c',1,'mimalloc-doc.h']]], + ['mi_5foption_5fabandoned_5freclaim_5fon_5ffree_119',['mi_option_abandoned_reclaim_on_free',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca009e4b5684922ce664d73d2a8e1698d9',1,'mimalloc-doc.h']]], + ['mi_5foption_5fallow_5flarge_5fos_5fpages_120',['mi_option_allow_large_os_pages',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7cc4804ced69004fa42a9a136a9ba556',1,'mimalloc-doc.h']]], + ['mi_5foption_5fallow_5fthp_121',['mi_option_allow_thp',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca04c45931ce2c82a20d08ca1471e74087',1,'mimalloc-doc.h']]], + ['mi_5foption_5farena_5feager_5fcommit_122',['mi_option_arena_eager_commit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafd0c5ddbc4b59fd8b5216871728167a5',1,'mimalloc-doc.h']]], + ['mi_5foption_5farena_5fmax_5fobject_5fsize_123',['mi_option_arena_max_object_size',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca4dee58d60bc95ce0eeef759c61fd849b',1,'mimalloc-doc.h']]], + ['mi_5foption_5farena_5fpurge_5fmult_124',['mi_option_arena_purge_mult',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8236501f1ab45d26e6fd885d191a2b5e',1,'mimalloc-doc.h']]], + ['mi_5foption_5farena_5freserve_125',['mi_option_arena_reserve',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cab1c88e23ae290bbeec824038a97959de',1,'mimalloc-doc.h']]], + ['mi_5foption_5fdestroy_5fon_5fexit_126',['mi_option_destroy_on_exit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca6364331e305e7d3c0218b058ff3afc88',1,'mimalloc-doc.h']]], + ['mi_5foption_5fdisable_127',['mi_option_disable',['../group__options.html#gaebf6ff707a2e688ebb1a2296ca564054',1,'mimalloc-doc.h']]], + ['mi_5foption_5fdisallow_5farena_5falloc_128',['mi_option_disallow_arena_alloc',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeae1696100e4057ffc4182730cc04e40',1,'mimalloc-doc.h']]], + ['mi_5foption_5fdisallow_5fos_5falloc_129',['mi_option_disallow_os_alloc',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cadcfb5a09580361b1be65901d2d812de6',1,'mimalloc-doc.h']]], + ['mi_5foption_5feager_5fcommit_130',['mi_option_eager_commit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca1e8de72c93da7ff22d91e1e27b52ac2b',1,'mimalloc-doc.h']]], + ['mi_5foption_5feager_5fcommit_5fdelay_131',['mi_option_eager_commit_delay',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca17a190c25be381142d87e0468c4c068c',1,'mimalloc-doc.h']]], + ['mi_5foption_5fenable_132',['mi_option_enable',['../group__options.html#ga04180ae41b0d601421dd62ced40ca050',1,'mimalloc-doc.h']]], + ['mi_5foption_5fgeneric_5fcollect_133',['mi_option_generic_collect',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8b5815a577955fe8125f2b964f41ee7f',1,'mimalloc-doc.h']]], + ['mi_5foption_5fget_134',['mi_option_get',['../group__options.html#ga7e8af195cc81d3fa64ccf2662caa565a',1,'mimalloc-doc.h']]], + ['mi_5foption_5fget_5fclamp_135',['mi_option_get_clamp',['../group__options.html#ga96ad9c406338bd314cfe878cfc9bf723',1,'mimalloc-doc.h']]], + ['mi_5foption_5fget_5fsize_136',['mi_option_get_size',['../group__options.html#ga274db5a6ac87cc24ef0b23e7006ed02c',1,'mimalloc-doc.h']]], + ['mi_5foption_5fguarded_5fmax_137',['mi_option_guarded_max',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca96077da4e5c841edeec0c4d6e45a5ffa',1,'mimalloc-doc.h']]], + ['mi_5foption_5fguarded_5fmin_138',['mi_option_guarded_min',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca00734c528aae52784654a8db996ae62b',1,'mimalloc-doc.h']]], + ['mi_5foption_5fguarded_5fprecise_139',['mi_option_guarded_precise',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca1eaf89bb1698f988cec27b66e546760c',1,'mimalloc-doc.h']]], + ['mi_5foption_5fguarded_5fsample_5frate_140',['mi_option_guarded_sample_rate',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeb3c75b1a838eb42106750979d0888ec',1,'mimalloc-doc.h']]], + ['mi_5foption_5fguarded_5fsample_5fseed_141',['mi_option_guarded_sample_seed',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7b1e41ece69f9abbe5ba81d9ea7a3dc0',1,'mimalloc-doc.h']]], + ['mi_5foption_5fis_5fenabled_142',['mi_option_is_enabled',['../group__options.html#ga459ad98f18b3fc9275474807fe0ca188',1,'mimalloc-doc.h']]], + ['mi_5foption_5fmax_5ferrors_143',['mi_option_max_errors',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caec6ecbe29d46a48205ed8823a8a52a6a',1,'mimalloc-doc.h']]], + ['mi_5foption_5fmax_5fsegment_5freclaim_144',['mi_option_max_segment_reclaim',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa9ad9005d7017c8c30ad2d6ba31db909',1,'mimalloc-doc.h']]], + ['mi_5foption_5fmax_5fvabits_145',['mi_option_max_vabits',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caabf8c9793d421ce28361b6d93c432de9',1,'mimalloc-doc.h']]], + ['mi_5foption_5fmax_5fwarnings_146',['mi_option_max_warnings',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caf9595921087e942602ee079158762665',1,'mimalloc-doc.h']]], + ['mi_5foption_5fminimal_5fpurge_5fsize_147',['mi_option_minimal_purge_size',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca5635112095936356425d9a01910afff7',1,'mimalloc-doc.h']]], + ['mi_5foption_5fos_5ftag_148',['mi_option_os_tag',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca4b74ae2a69e445de6c2361b73c1d14bf',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpage_5fcommit_5fon_5fdemand_149',['mi_option_page_commit_on_demand',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca679d9ee91221bffba4d5aafe488a8d7f',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpage_5fcross_5fthread_5fmax_5freclaim_150',['mi_option_page_cross_thread_max_reclaim',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca671e806447c15a53c7d593afdd8d45c5',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpage_5ffull_5fretain_151',['mi_option_page_full_retain',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeede7d0652e391940357e1c054d29829',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpage_5fmax_5fcandidates_152',['mi_option_page_max_candidates',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cad13382757101aa4fbc4984fa0f917133',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpage_5fmax_5freclaim_153',['mi_option_page_max_reclaim',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca11a13739488294318ffc03eea378f5a4',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpage_5freclaim_5fon_5ffree_154',['mi_option_page_reclaim_on_free',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cac16e3f838e861338fb2357b430f6974c',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpagemap_5fcommit_155',['mi_option_pagemap_commit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa88f56e3bccce85542dc2b3297024236',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpurge_5fdecommits_156',['mi_option_purge_decommits',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca9d15c5e3d2115eef681c17e4dd5ab9a4',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpurge_5fdelay_157',['mi_option_purge_delay',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cadd351e615acd8563529c20a347be7290',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpurge_5fextend_5fdelay_158',['mi_option_purge_extend_delay',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca02005f164bdf03f5f00c5be726adf487',1,'mimalloc-doc.h']]], + ['mi_5foption_5freserve_5fhuge_5fos_5fpages_159',['mi_option_reserve_huge_os_pages',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caca7ed041be3b0b9d0b82432c7bf41af2',1,'mimalloc-doc.h']]], + ['mi_5foption_5freserve_5fhuge_5fos_5fpages_5fat_160',['mi_option_reserve_huge_os_pages_at',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa13e7926d4339d2aa6fbf61d4473fd5c',1,'mimalloc-doc.h']]], + ['mi_5foption_5freserve_5fos_5fmemory_161',['mi_option_reserve_os_memory',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafbf4999c828cf79a0fb2de65d23f7333',1,'mimalloc-doc.h']]], + ['mi_5foption_5fretry_5fon_5foom_162',['mi_option_retry_on_oom',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8f51df355bf6651db899e6085b54865e',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_163',['mi_option_set',['../group__options.html#gaf84921c32375e25754dc2ee6a911fa60',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_5fdefault_164',['mi_option_set_default',['../group__options.html#ga7ef623e440e6e5545cb08c94e71e4b90',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_5fenabled_165',['mi_option_set_enabled',['../group__options.html#ga9a13d05fcb77489cb06d4d017ebd8bed',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_5fenabled_5fdefault_166',['mi_option_set_enabled_default',['../group__options.html#ga65518b69ec5d32336b50e07f74b3f629',1,'mimalloc-doc.h']]], + ['mi_5foption_5fshow_5ferrors_167',['mi_option_show_errors',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafbf4822e5c00732c5984b32a032837f0',1,'mimalloc-doc.h']]], + ['mi_5foption_5fshow_5fstats_168',['mi_option_show_stats',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca0957ef73b2550764b4840edf48422fda',1,'mimalloc-doc.h']]], + ['mi_5foption_5ft_169',['mi_option_t',['../group__options.html#gafebf7ed116adb38ae5218bc3ce06884c',1,'mimalloc-doc.h']]], + ['mi_5foption_5ftarget_5fsegments_5fper_5fthread_170',['mi_option_target_segments_per_thread',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca874f36a7a1e51f7baee4a78b2830b643',1,'mimalloc-doc.h']]], + ['mi_5foption_5fuse_5fnuma_5fnodes_171',['mi_option_use_numa_nodes',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca0ac33a18f6b659fcfaf44efb0bab1b74',1,'mimalloc-doc.h']]], + ['mi_5foption_5fverbose_172',['mi_option_verbose',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7c8b7bf5281c581bad64f5daa6442777',1,'mimalloc-doc.h']]], + ['mi_5foption_5fvisit_5fabandoned_173',['mi_option_visit_abandoned',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca38c67733a3956a1f4eeaca89fab9e78e',1,'mimalloc-doc.h']]], + ['mi_5foptions_5fprint_174',['mi_options_print',['../group__options.html#gaecfcd3ed90af99932670d881de1fa8f1',1,'mimalloc-doc.h']]], + ['mi_5foptions_5fprint_5fout_175',['mi_options_print_out',['../group__options.html#ga1dd50c7533baec4cc6c1a2e9adf05dd2',1,'mimalloc-doc.h']]], + ['mi_5foutput_5ffun_176',['mi_output_fun',['../group__extended.html#gadf31cea7d0332a81c8b882cbbdbadb8d',1,'mimalloc-doc.h']]], + ['mi_5fposix_5fmemalign_177',['mi_posix_memalign',['../group__posix.html#gacff84f226ba9feb2031b8992e5579447',1,'mimalloc-doc.h']]], + ['mi_5fprocess_5finfo_178',['mi_process_info',['../group__stats.html#ga7d862c2affd5790381da14eb102a364d',1,'mimalloc-doc.h']]], + ['mi_5fprocess_5finfo_5fprint_179',['mi_process_info_print',['../group__stats.html#gaf887e7aa75f13242c4e7ab95d8390536',1,'mimalloc-doc.h']]], + ['mi_5fprocess_5finfo_5fprint_5fout_180',['mi_process_info_print_out',['../group__stats.html#ga03ac88d3efee2ae7c34f5340502f4732',1,'mimalloc-doc.h']]], + ['mi_5fpvalloc_181',['mi_pvalloc',['../group__posix.html#ga644bebccdbb2821542dd8c7e7641f476',1,'mimalloc-doc.h']]], + ['mi_5frealloc_182',['mi_realloc',['../group__malloc.html#ga0621af6a5e3aa384e6a1b548958bf583',1,'mimalloc-doc.h']]], + ['mi_5frealloc_5faligned_183',['mi_realloc_aligned',['../group__aligned.html#ga5d7a46d054b4d7abe9d8d2474add2edf',1,'mimalloc-doc.h']]], + ['mi_5frealloc_5faligned_5fat_184',['mi_realloc_aligned_at',['../group__aligned.html#gad06dcf2bb8faadb2c8ea61ee5d24bbf6',1,'mimalloc-doc.h']]], + ['mi_5freallocarr_185',['mi_reallocarr',['../group__posix.html#ga7e1934d60a3e697950eeb48e042bfad5',1,'mimalloc-doc.h']]], + ['mi_5freallocarray_186',['mi_reallocarray',['../group__posix.html#gadfeccb72748a2f6305474a37d9d57bce',1,'mimalloc-doc.h']]], + ['mi_5freallocf_187',['mi_reallocf',['../group__malloc.html#ga4dc3a4067037b151a64629fe8a332641',1,'mimalloc-doc.h']]], + ['mi_5freallocn_188',['mi_reallocn',['../group__malloc.html#ga8bddfb4a1270a0854bbcf44cb3980467',1,'mimalloc-doc.h']]], + ['mi_5freallocn_5ftp_189',['mi_reallocn_tp',['../group__typed.html#ga1158b49a55dfa81f58a4426a7578f523',1,'mimalloc-doc.h']]], + ['mi_5frealpath_190',['mi_realpath',['../group__malloc.html#ga94c3afcc086e85d75a57e9f76b9b71dd',1,'mimalloc-doc.h']]], + ['mi_5frecalloc_191',['mi_recalloc',['../group__zeroinit.html#ga23a0fbb452b5dce8e31fab1a1958cacc',1,'mimalloc-doc.h']]], + ['mi_5frecalloc_5faligned_192',['mi_recalloc_aligned',['../group__zeroinit.html#ga3e2169b48683aa0ab64f813fd68d839e',1,'mimalloc-doc.h']]], + ['mi_5frecalloc_5faligned_5fat_193',['mi_recalloc_aligned_at',['../group__zeroinit.html#gaae25e4ddedd4e0fb61b1a8bd5d452750',1,'mimalloc-doc.h']]], + ['mi_5fregister_5fdeferred_5ffree_194',['mi_register_deferred_free',['../group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece',1,'mimalloc-doc.h']]], + ['mi_5fregister_5ferror_195',['mi_register_error',['../group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45',1,'mimalloc-doc.h']]], + ['mi_5fregister_5foutput_196',['mi_register_output',['../group__extended.html#gae5b17ff027cd2150b43a33040250cf3f',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fhuge_5fos_5fpages_5fat_197',['mi_reserve_huge_os_pages_at',['../group__arenas.html#ga7795a13d20087447281858d2c771cca1',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fhuge_5fos_5fpages_5fat_5fex_198',['mi_reserve_huge_os_pages_at_ex',['../group__arenas.html#ga591aab1c2bc2ca920e33f0f9f9cb5c52',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fhuge_5fos_5fpages_5finterleave_199',['mi_reserve_huge_os_pages_interleave',['../group__arenas.html#ga3132f521fb756fc0e8ec0b74fb58df50',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fos_5fmemory_200',['mi_reserve_os_memory',['../group__arenas.html#ga00ec3324b6b2591c7fe3677baa30a767',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fos_5fmemory_5fex_201',['mi_reserve_os_memory_ex',['../group__arenas.html#ga32f519797fd9a81acb4f52d36e6d751b',1,'mimalloc-doc.h']]], + ['mi_5frezalloc_202',['mi_rezalloc',['../group__zeroinit.html#gadfd34cd7b4f2bbda7ae06367a6360756',1,'mimalloc-doc.h']]], + ['mi_5frezalloc_5faligned_203',['mi_rezalloc_aligned',['../group__zeroinit.html#ga4d02404fe1e7db00beb65f185e012caa',1,'mimalloc-doc.h']]], + ['mi_5frezalloc_5faligned_5fat_204',['mi_rezalloc_aligned_at',['../group__zeroinit.html#ga6843a88285bbfcc3bdfccc60aafd1270',1,'mimalloc-doc.h']]], + ['mi_5fsmall_5fsize_5fmax_205',['MI_SMALL_SIZE_MAX',['../group__extended.html#ga1ea64283508718d9d645c38efc2f4305',1,'mimalloc-doc.h']]], + ['mi_5fstat_5fversion_206',['MI_STAT_VERSION',['../group__stats.html#gac4411cf3eed23719e511b9e1128888e0',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fas_5fjson_207',['mi_stats_as_json',['../group__stats.html#ga6ad266c70a0eabf6c26248aed1af6fa1',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fget_208',['mi_stats_get',['../group__stats.html#gaa26d5060ad21b1331a150bb92f6f7086',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fget_5fbin_5fsize_209',['mi_stats_get_bin_size',['../group__stats.html#gadecd549d1a7216af596322ac27e78b0d',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fget_5fjson_210',['mi_stats_get_json',['../group__stats.html#gadaa3be15899cc82e314fa7d3f8e2afbd',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fmerge_211',['mi_stats_merge',['../group__stats.html#ga854b1de8cb067c7316286c28b2fcd3d1',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fprint_212',['mi_stats_print',['../group__stats.html#ga2d126e5c62d3badc35445e5d84166df2',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fprint_5fout_213',['mi_stats_print_out',['../group__stats.html#ga537f13b299ddf801e49a5a94fde02c79',1,'mimalloc-doc.h']]], + ['mi_5fstats_5freset_214',['mi_stats_reset',['../group__stats.html#ga3bb8468b8cfcc6e2a61d98aee85c5f99',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fs_215',['mi_stats_s',['../group__stats.html#structmi__stats__s',1,'']]], + ['mi_5fstats_5ft_5fdecl_216',['mi_stats_t_decl',['../group__stats.html#ga961d0f9f0327ebc9bd8a0b84f6dcb410',1,'mimalloc-doc.h']]], + ['mi_5fstl_5fallocator_217',['mi_stl_allocator',['../group__cpp.html#structmi__stl__allocator',1,'']]], + ['mi_5fstrdup_218',['mi_strdup',['../group__malloc.html#ga245ac90ebc2cfdd17de599e5fea59889',1,'mimalloc-doc.h']]], + ['mi_5fstrndup_219',['mi_strndup',['../group__malloc.html#ga486d0d26b3b3794f6d1cdb41a9aed92d',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fadd_5fcurrent_5fthread_220',['mi_subproc_add_current_thread',['../group__subproc.html#gadbc53414eb68b275588ec001ce1ddc7c',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fcurrent_221',['mi_subproc_current',['../group__subproc.html#gaabc4db6585bf16099e43e60d64364950',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fdelete_222',['mi_subproc_delete',['../group__subproc.html#gaa7d263e9429bac9ac8345c9d25de610e',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fdestroy_223',['mi_subproc_destroy',['../group__subproc.html#gada12e4d9bd382935c398ec324d0391f9',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fheap_5fstats_5fprint_5fout_224',['mi_subproc_heap_stats_print_out',['../group__stats.html#ga19f1a04bd972e11e4f3b586f25229d5b',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fid_5ft_225',['mi_subproc_id_t',['../group__subproc.html#ga8c0bcd1fee27c7641e9c3c0d991b3b7d',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fmain_226',['mi_subproc_main',['../group__subproc.html#ga2ecba0d7ebdc99e71bb985c4a1609806',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fnew_227',['mi_subproc_new',['../group__subproc.html#ga8068cac328e41fa2170faef707315243',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fstats_5fget_228',['mi_subproc_stats_get',['../group__stats.html#ga5ec3f9058eec804924711ab38ba64b51',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fstats_5fget_5fjson_229',['mi_subproc_stats_get_json',['../group__stats.html#gad1cf918f08b6506e65ae134423477a0b',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fstats_5fprint_5fout_230',['mi_subproc_stats_print_out',['../group__stats.html#ga59ec89969837db414a6bce8b58915624',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fvisit_5fheaps_231',['mi_subproc_visit_heaps',['../group__subproc.html#gafd2136e7f167b5cd02b6ab00bed0ac93',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fcalloc_232',['mi_theap_calloc',['../group__theap.html#gadbf8f27bfe9edf7834e19d32cab63edc',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fcollect_233',['mi_theap_collect',['../group__theap.html#ga9377a43095b8dd14996613ceb73d9a32',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fget_5fdefault_234',['mi_theap_get_default',['../group__theap.html#ga78afbb07c1e0cd333c7539cf7403dfd5',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_235',['mi_theap_malloc',['../group__theap.html#gac7308e029c59e6675dd5ab5ec54f304a',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_5faligned_236',['mi_theap_malloc_aligned',['../group__theap.html#ga84503fc2bd83fbfb7f789b7d80d76f36',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_5fcsize_237',['mi_theap_malloc_csize',['../group__constantsize.html#ga105bd667da91a15d113fe79be04d625c',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_5fsmall_238',['mi_theap_malloc_small',['../group__theap.html#ga1fc65302a212daaecb21abda96ff247c',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5frealloc_239',['mi_theap_realloc',['../group__theap.html#ga11943128dc354313c4189e72f4299dbb',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fset_5fdefault_240',['mi_theap_set_default',['../group__theap.html#ga04420ce52785019412397e4de24a984a',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5ft_241',['mi_theap_t',['../group__theap.html#ga8722f6ac4f638b4692e06ddfddc844f8',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fzalloc_242',['mi_theap_zalloc',['../group__theap.html#gaa7fd973dcd8c75eb05707bdeffbc06f7',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fzalloc_5fcsize_243',['mi_theap_zalloc_csize',['../group__constantsize.html#ga1addf755829380581dd9c3c9175e376f',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fzalloc_5fsmall_244',['mi_theap_zalloc_small',['../group__theap.html#gaa85fccf9d40af3e826c5932aded1d534',1,'mimalloc-doc.h']]], + ['mi_5fthread_5fdone_245',['mi_thread_done',['../group__extended.html#ga0ae4581e85453456a0d658b2b98bf7bf',1,'mimalloc-doc.h']]], + ['mi_5fthread_5finit_246',['mi_thread_init',['../group__extended.html#gaf8e73efc2cbca9ebfdfb166983a04c17',1,'mimalloc-doc.h']]], + ['mi_5fthread_5fset_5fin_5fthreadpool_247',['mi_thread_set_in_threadpool',['../group__extended.html#gada899104276aac52539eb4de7c020f1f',1,'mimalloc-doc.h']]], + ['mi_5fusable_5fsize_248',['mi_usable_size',['../group__malloc.html#ga089c859d9eddc5f9b4bd946cd53cebee',1,'mimalloc-doc.h']]], + ['mi_5fvalloc_249',['mi_valloc',['../group__posix.html#ga50cafb9722020402f065de93799f64ca',1,'mimalloc-doc.h']]], + ['mi_5fversion_250',['mi_version',['../group__extended.html#gae9ddddcf14a58d0224ef3c8778e5b057',1,'mimalloc-doc.h']]], + ['mi_5fwcsdup_251',['mi_wcsdup',['../group__posix.html#gaa9fd7f25c9ac3a20e89b33bd6e383fcf',1,'mimalloc-doc.h']]], + ['mi_5fwdupenv_5fs_252',['mi_wdupenv_s',['../group__posix.html#ga6ac6a6a8f3c96f1af24bb8d0439cbbd1',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_253',['mi_zalloc',['../group__malloc.html#gae6e38c4403247a7b40d80419e093bfb8',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5faligned_254',['mi_zalloc_aligned',['../group__aligned.html#gaac7d0beb782f9b9ac31f47492b130f82',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5faligned_5fat_255',['mi_zalloc_aligned_at',['../group__aligned.html#ga7c1778805ce50ebbf02ccbd5e39d5dba',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5fcsize_256',['mi_zalloc_csize',['../group__constantsize.html#gadca6d340ee15b5c70695ed82d95c6cec',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5fsmall_257',['mi_zalloc_small',['../group__extended.html#ga51c47637e81df0e2f13a2d7a2dec123e',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5ftp_258',['mi_zalloc_tp',['../group__typed.html#gac77a61bdaf680a803785fe307820b48c',1,'mimalloc-doc.h']]], + ['modes_259',['Build Modes',['../modes.html',1,'']]] ]; diff --git a/docs/search/functions_0.js b/docs/search/functions_0.js index d16b716ee..6d7a877dc 100644 --- a/docs/search/functions_0.js +++ b/docs/search/functions_0.js @@ -12,7 +12,7 @@ var searchData= ['mi_5fcalloc_9',['mi_calloc',['../group__malloc.html#ga6686568014b54d1e6c7ac64a076e4f56',1,'mimalloc-doc.h']]], ['mi_5fcalloc_5faligned_10',['mi_calloc_aligned',['../group__aligned.html#ga424ef386fb1f9f8e0a86ab53f16eaaf1',1,'mimalloc-doc.h']]], ['mi_5fcalloc_5faligned_5fat_11',['mi_calloc_aligned_at',['../group__aligned.html#ga977f96bd2c5c141bcd70e6685c90d6c3',1,'mimalloc-doc.h']]], - ['mi_5fcfree_12',['mi_cfree',['../group__posix.html#ga705dc7a64bffacfeeb0141501a5c35d7',1,'mimalloc-doc.h']]], + ['mi_5fcfree_12',['mi_cfree',['../group__malloc.html#ga705dc7a64bffacfeeb0141501a5c35d7',1,'mimalloc-doc.h']]], ['mi_5fcheck_5fowned_13',['mi_check_owned',['../group__heap.html#ga628c237489c2679af84a4d0d143b3dd5',1,'mimalloc-doc.h']]], ['mi_5fcollect_14',['mi_collect',['../group__extended.html#ga421430e2226d7d468529cec457396756',1,'mimalloc-doc.h']]], ['mi_5fdebug_5fshow_5farenas_15',['mi_debug_show_arenas',['../group__arenas.html#ga98cda17aa7adeda8b213382af8e7810b',1,'mimalloc-doc.h']]], @@ -21,162 +21,165 @@ var searchData= ['mi_5ffree_18',['mi_free',['../group__malloc.html#gaf2c7b89c327d1f60f59e68b9ea644d95',1,'mimalloc-doc.h']]], ['mi_5ffree_5faligned_19',['mi_free_aligned',['../group__posix.html#ga0d28d5cf61e6bfbb18c63092939fe5c9',1,'mimalloc-doc.h']]], ['mi_5ffree_5fcsize_20',['mi_free_csize',['../group__constantsize.html#ga7a421e302074e598527c3eb052a6a3c5',1,'mimalloc-doc.h']]], - ['mi_5ffree_5fsize_21',['mi_free_size',['../group__posix.html#gae01389eedab8d67341ff52e2aad80ebb',1,'mimalloc-doc.h']]], - ['mi_5ffree_5fsize_5faligned_22',['mi_free_size_aligned',['../group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc',1,'mimalloc-doc.h']]], - ['mi_5ffree_5fsmall_23',['mi_free_small',['../group__extended.html#ga2a8469764c38dcaf878f36e5f50e3610',1,'mimalloc-doc.h']]], - ['mi_5fgood_5fsize_24',['mi_good_size',['../group__malloc.html#gac057927cd06c854b45fe7847e921bd47',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcalloc_25',['mi_heap_calloc',['../group__heap.html#gac0098aaf231d3e9586c73136d5df95da',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcalloc_5faligned_26',['mi_heap_calloc_aligned',['../group__heap.html#gacafcc26df827c7a7de5e850217566108',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcalloc_5faligned_5fat_27',['mi_heap_calloc_aligned_at',['../group__heap.html#gaa42ec2079989c4374f2c331d9b35f4e4',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcheck_5fowned_28',['mi_heap_check_owned',['../group__heap.html#ga0d67c1789faaa15ff366c024fcaf6377',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcollect_29',['mi_heap_collect',['../group__heap.html#ga7922f7495cde30b1984d0e6072419298',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcontains_30',['mi_heap_contains',['../group__heap.html#gac988496aea8ffc5c8bf92a8b10165fa0',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcontains_5fblock_31',['mi_heap_contains_block',['../group__heap.html#gaa862aa8ed8d57d84cae41fc1022d71af',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fdelete_32',['mi_heap_delete',['../group__heap.html#ga2ab1af8d438819b55319c7ef51d1e409',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fdestroy_33',['mi_heap_destroy',['../group__heap.html#ga9f9c0844edb9717f4feacd79116b8e0d',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fget_5fbacking_34',['mi_heap_get_backing',['../group__heap.html#gac6ac9f0e7be9ab4ff70acfc8dad1235a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fget_5fdefault_35',['mi_heap_get_default',['../group__heap.html#ga14c667a6e2c5d28762d8cb7d4e057909',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmain_36',['mi_heap_main',['../group__heap.html#ga8fc318a65dc79ee8778c6a0e52d12477',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_37',['mi_heap_malloc',['../group__heap.html#gab374e206c7034e0d899fb934e4f4a863',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5faligned_38',['mi_heap_malloc_aligned',['../group__heap.html#ga33f4f05b7fea7af2113c62a4bf882cc5',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5faligned_5fat_39',['mi_heap_malloc_aligned_at',['../group__heap.html#gae7ffc045c3996497a7f3a5f6fe7b8aaa',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5fsmall_40',['mi_heap_malloc_small',['../group__heap.html#ga012c5c8abe22b10043de39ff95909541',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmallocn_41',['mi_heap_mallocn',['../group__heap.html#gab0f755c0b21c387fe8e9024200faa372',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fnew_42',['mi_heap_new',['../group__heap.html#gaa718bb226ec0546ba6d1b6cb32179f3a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fnew_5fex_43',['mi_heap_new_ex',['../group__arenas.html#ga3ae360583f4351aa5267ee7e43008faf',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fnew_5fin_5farena_44',['mi_heap_new_in_arena',['../group__arenas.html#gaaf2d9976576d5efd5544be12848af949',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fof_45',['mi_heap_of',['../group__heap.html#ga424b69f285109b2515631d59f5eecb1b',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealloc_46',['mi_heap_realloc',['../group__heap.html#gac5252d6a2e510bd349e4fcb452e6a93a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealloc_5faligned_47',['mi_heap_realloc_aligned',['../group__heap.html#gaccf8c249872f30bf1c2493a09197d734',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealloc_5faligned_5fat_48',['mi_heap_realloc_aligned_at',['../group__heap.html#ga6df988a7219d5707f010d5f3eb0dc3f5',1,'mimalloc-doc.h']]], - ['mi_5fheap_5freallocf_49',['mi_heap_reallocf',['../group__heap.html#gae7cd171425bee04c683c65a3701f0b4a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5freallocn_50',['mi_heap_reallocn',['../group__heap.html#gaccf7bfe10ce510a000d3547d9cf7fa29',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealpath_51',['mi_heap_realpath',['../group__heap.html#ga55545a3ec6da29c5b4f62e540ecac1e2',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_52',['mi_heap_recalloc',['../group__zeroinit.html#gad1a0d325d930eeb80f25e3fea37aacde',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_5faligned_53',['mi_heap_recalloc_aligned',['../group__zeroinit.html#ga87ddd674bf1c67237d780d0b9e0f0f32',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_5faligned_5fat_54',['mi_heap_recalloc_aligned_at',['../group__zeroinit.html#ga07b5bcbaf00d0d2e598c232982588496',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frezalloc_55',['mi_heap_rezalloc',['../group__zeroinit.html#ga8d8b7ebb24b513cd84d1a696048da60d',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frezalloc_5faligned_56',['mi_heap_rezalloc_aligned',['../group__zeroinit.html#ga5129f6dc46ee1613d918820a8a0533a7',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frezalloc_5faligned_5fat_57',['mi_heap_rezalloc_aligned_at',['../group__zeroinit.html#ga2bafa79c3f98ea74882349d44cffa5d9',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fset_5fdefault_58',['mi_heap_set_default',['../group__heap.html#ga349b677dec7da5eacdbc7a385bd62a4a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fset_5fnuma_5faffinity_59',['mi_heap_set_numa_affinity',['../group__heap.html#gaab93abbc0624f570e2900f95cac18591',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fget_60',['mi_heap_stats_get',['../group__stats.html#ga2654195cef93ed3613e98b59a19a296f',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fget_5fjson_61',['mi_heap_stats_get_json',['../group__stats.html#gacb42dede4c38c38f2014b5450c5fcba8',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fmerge_5fto_5fsubproc_62',['mi_heap_stats_merge_to_subproc',['../group__stats.html#ga5a34328f846c661aa560f4769cfc15f1',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fprint_5fout_63',['mi_heap_stats_print_out',['../group__stats.html#gafea365d7842f5f5bc1f235bbc303d585',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstrdup_64',['mi_heap_strdup',['../group__heap.html#ga5754e09ccc51dd6bc73885bb6ea21b7a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstrndup_65',['mi_heap_strndup',['../group__heap.html#gad224df78f1fbee942df8adf023e12cf3',1,'mimalloc-doc.h']]], - ['mi_5fheap_5ftheap_66',['mi_heap_theap',['../group__theap.html#ga7905a73865733da8a43339a500533180',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fvisit_5fblocks_67',['mi_heap_visit_blocks',['../group__analysis.html#gae69fcff3324f8e419af850e34cf54c46',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_68',['mi_heap_zalloc',['../group__heap.html#gabebc796399619d964d8db77aa835e8c1',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5faligned_69',['mi_heap_zalloc_aligned',['../group__heap.html#ga6466bde8b5712aa34e081a8317f9f471',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5faligned_5fat_70',['mi_heap_zalloc_aligned_at',['../group__heap.html#ga484e3d01cd174f78c7e53370e5a7c819',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5fsmall_71',['mi_heap_zalloc_small',['../group__heap.html#ga0994f44cc595f9c1840a9af162d98597',1,'mimalloc-doc.h']]], - ['mi_5fis_5fin_5fheap_5fregion_72',['mi_is_in_heap_region',['../group__heap.html#ga5f071b10d4df1c3658e04e7fd67a94e6',1,'mimalloc-doc.h']]], - ['mi_5fis_5fredirected_73',['mi_is_redirected',['../group__extended.html#gaad25050b19f30cd79397b227e0157a3f',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_74',['mi_malloc',['../group__malloc.html#gae1dd97b542420c87ae085e822b1229e8',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5faligned_75',['mi_malloc_aligned',['../group__aligned.html#ga69578ff1a98ca16e1dcd02c0995cd65c',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5faligned_5fat_76',['mi_malloc_aligned_at',['../group__aligned.html#ga2022f71b95a7cd6cce1b6e07752ae8ca',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fcsize_77',['mi_malloc_csize',['../group__constantsize.html#ga9cd44d925d34f7180922e12bbf594571',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fgood_5fsize_78',['mi_malloc_good_size',['../group__posix.html#ga9d23ac7885fed7413c11d8e0ffa31071',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fsize_79',['mi_malloc_size',['../group__posix.html#ga4531c9e775bb3ae12db57c1ba8a5d7de',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fsmall_80',['mi_malloc_small',['../group__extended.html#ga7f050bc6b897da82692174f5fce59cde',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fusable_5fsize_81',['mi_malloc_usable_size',['../group__posix.html#ga06d07cf357bbac5c73ba5d0c0c421e17',1,'mimalloc-doc.h']]], - ['mi_5fmallocn_82',['mi_mallocn',['../group__malloc.html#ga61f46bade3db76ca24aaafedc40de7b6',1,'mimalloc-doc.h']]], - ['mi_5fmanage_5fos_5fmemory_83',['mi_manage_os_memory',['../group__arenas.html#ga4c6486a1fdcd7a423b5f25fe4be8e0cf',1,'mimalloc-doc.h']]], - ['mi_5fmanage_5fos_5fmemory_5fex_84',['mi_manage_os_memory_ex',['../group__arenas.html#ga41ce8525d77bbb60f618fa1029994f6e',1,'mimalloc-doc.h']]], - ['mi_5fmbsdup_85',['mi_mbsdup',['../group__posix.html#ga7b82a44094fdec4d2084eb4288a979b0',1,'mimalloc-doc.h']]], - ['mi_5fmemalign_86',['mi_memalign',['../group__posix.html#ga726867f13fd29ca36064954c0285b1d8',1,'mimalloc-doc.h']]], - ['mi_5fnew_87',['mi_new',['../group__cpp.html#ga633d96e3bc7011f960df9f3b2731fc6a',1,'mimalloc-doc.h']]], - ['mi_5fnew_5faligned_88',['mi_new_aligned',['../group__cpp.html#ga79c54da0b4b4ce9fcc11d2f6ef6675f8',1,'mimalloc-doc.h']]], - ['mi_5fnew_5faligned_5fnothrow_89',['mi_new_aligned_nothrow',['../group__cpp.html#ga92ae00b6dd64406c7e64557711ec04b7',1,'mimalloc-doc.h']]], - ['mi_5fnew_5fn_90',['mi_new_n',['../group__cpp.html#gadd11b85c15d21d308386844b5233856c',1,'mimalloc-doc.h']]], - ['mi_5fnew_5fnothrow_91',['mi_new_nothrow',['../group__cpp.html#ga5cb4f120d1f7296074256215aa9a9e54',1,'mimalloc-doc.h']]], - ['mi_5fnew_5frealloc_92',['mi_new_realloc',['../group__cpp.html#ga6867d89baf992728e0cc20a1f47db4d0',1,'mimalloc-doc.h']]], - ['mi_5fnew_5freallocn_93',['mi_new_reallocn',['../group__cpp.html#gaace912ce086682d56f3ce9f7638d9d67',1,'mimalloc-doc.h']]], - ['mi_5foption_5fdisable_94',['mi_option_disable',['../group__options.html#gaebf6ff707a2e688ebb1a2296ca564054',1,'mimalloc-doc.h']]], - ['mi_5foption_5fenable_95',['mi_option_enable',['../group__options.html#ga04180ae41b0d601421dd62ced40ca050',1,'mimalloc-doc.h']]], - ['mi_5foption_5fget_96',['mi_option_get',['../group__options.html#ga7e8af195cc81d3fa64ccf2662caa565a',1,'mimalloc-doc.h']]], - ['mi_5foption_5fget_5fclamp_97',['mi_option_get_clamp',['../group__options.html#ga96ad9c406338bd314cfe878cfc9bf723',1,'mimalloc-doc.h']]], - ['mi_5foption_5fget_5fsize_98',['mi_option_get_size',['../group__options.html#ga274db5a6ac87cc24ef0b23e7006ed02c',1,'mimalloc-doc.h']]], - ['mi_5foption_5fis_5fenabled_99',['mi_option_is_enabled',['../group__options.html#ga459ad98f18b3fc9275474807fe0ca188',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_100',['mi_option_set',['../group__options.html#gaf84921c32375e25754dc2ee6a911fa60',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_5fdefault_101',['mi_option_set_default',['../group__options.html#ga7ef623e440e6e5545cb08c94e71e4b90',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_5fenabled_102',['mi_option_set_enabled',['../group__options.html#ga9a13d05fcb77489cb06d4d017ebd8bed',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_5fenabled_5fdefault_103',['mi_option_set_enabled_default',['../group__options.html#ga65518b69ec5d32336b50e07f74b3f629',1,'mimalloc-doc.h']]], - ['mi_5foptions_5fprint_104',['mi_options_print',['../group__options.html#gaecfcd3ed90af99932670d881de1fa8f1',1,'mimalloc-doc.h']]], - ['mi_5foptions_5fprint_5fout_105',['mi_options_print_out',['../group__options.html#ga1dd50c7533baec4cc6c1a2e9adf05dd2',1,'mimalloc-doc.h']]], - ['mi_5fposix_5fmemalign_106',['mi_posix_memalign',['../group__posix.html#gacff84f226ba9feb2031b8992e5579447',1,'mimalloc-doc.h']]], - ['mi_5fprocess_5finfo_107',['mi_process_info',['../group__stats.html#ga7d862c2affd5790381da14eb102a364d',1,'mimalloc-doc.h']]], - ['mi_5fprocess_5finfo_5fprint_108',['mi_process_info_print',['../group__stats.html#gaf887e7aa75f13242c4e7ab95d8390536',1,'mimalloc-doc.h']]], - ['mi_5fprocess_5finfo_5fprint_5fout_109',['mi_process_info_print_out',['../group__stats.html#ga03ac88d3efee2ae7c34f5340502f4732',1,'mimalloc-doc.h']]], - ['mi_5fpvalloc_110',['mi_pvalloc',['../group__posix.html#ga644bebccdbb2821542dd8c7e7641f476',1,'mimalloc-doc.h']]], - ['mi_5frealloc_111',['mi_realloc',['../group__malloc.html#ga0621af6a5e3aa384e6a1b548958bf583',1,'mimalloc-doc.h']]], - ['mi_5frealloc_5faligned_112',['mi_realloc_aligned',['../group__aligned.html#ga5d7a46d054b4d7abe9d8d2474add2edf',1,'mimalloc-doc.h']]], - ['mi_5frealloc_5faligned_5fat_113',['mi_realloc_aligned_at',['../group__aligned.html#gad06dcf2bb8faadb2c8ea61ee5d24bbf6',1,'mimalloc-doc.h']]], - ['mi_5freallocarr_114',['mi_reallocarr',['../group__posix.html#ga7e1934d60a3e697950eeb48e042bfad5',1,'mimalloc-doc.h']]], - ['mi_5freallocarray_115',['mi_reallocarray',['../group__posix.html#gadfeccb72748a2f6305474a37d9d57bce',1,'mimalloc-doc.h']]], - ['mi_5freallocf_116',['mi_reallocf',['../group__malloc.html#ga4dc3a4067037b151a64629fe8a332641',1,'mimalloc-doc.h']]], - ['mi_5freallocn_117',['mi_reallocn',['../group__malloc.html#ga8bddfb4a1270a0854bbcf44cb3980467',1,'mimalloc-doc.h']]], - ['mi_5frealpath_118',['mi_realpath',['../group__malloc.html#ga94c3afcc086e85d75a57e9f76b9b71dd',1,'mimalloc-doc.h']]], - ['mi_5frecalloc_119',['mi_recalloc',['../group__zeroinit.html#ga23a0fbb452b5dce8e31fab1a1958cacc',1,'mimalloc-doc.h']]], - ['mi_5frecalloc_5faligned_120',['mi_recalloc_aligned',['../group__zeroinit.html#ga3e2169b48683aa0ab64f813fd68d839e',1,'mimalloc-doc.h']]], - ['mi_5frecalloc_5faligned_5fat_121',['mi_recalloc_aligned_at',['../group__zeroinit.html#gaae25e4ddedd4e0fb61b1a8bd5d452750',1,'mimalloc-doc.h']]], - ['mi_5fregister_5fdeferred_5ffree_122',['mi_register_deferred_free',['../group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece',1,'mimalloc-doc.h']]], - ['mi_5fregister_5ferror_123',['mi_register_error',['../group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45',1,'mimalloc-doc.h']]], - ['mi_5fregister_5foutput_124',['mi_register_output',['../group__extended.html#gae5b17ff027cd2150b43a33040250cf3f',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fhuge_5fos_5fpages_5fat_125',['mi_reserve_huge_os_pages_at',['../group__arenas.html#ga7795a13d20087447281858d2c771cca1',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fhuge_5fos_5fpages_5fat_5fex_126',['mi_reserve_huge_os_pages_at_ex',['../group__arenas.html#ga591aab1c2bc2ca920e33f0f9f9cb5c52',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fhuge_5fos_5fpages_5finterleave_127',['mi_reserve_huge_os_pages_interleave',['../group__arenas.html#ga3132f521fb756fc0e8ec0b74fb58df50',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fos_5fmemory_128',['mi_reserve_os_memory',['../group__arenas.html#ga00ec3324b6b2591c7fe3677baa30a767',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fos_5fmemory_5fex_129',['mi_reserve_os_memory_ex',['../group__arenas.html#ga32f519797fd9a81acb4f52d36e6d751b',1,'mimalloc-doc.h']]], - ['mi_5frezalloc_130',['mi_rezalloc',['../group__zeroinit.html#gadfd34cd7b4f2bbda7ae06367a6360756',1,'mimalloc-doc.h']]], - ['mi_5frezalloc_5faligned_131',['mi_rezalloc_aligned',['../group__zeroinit.html#ga4d02404fe1e7db00beb65f185e012caa',1,'mimalloc-doc.h']]], - ['mi_5frezalloc_5faligned_5fat_132',['mi_rezalloc_aligned_at',['../group__zeroinit.html#ga6843a88285bbfcc3bdfccc60aafd1270',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fas_5fjson_133',['mi_stats_as_json',['../group__stats.html#ga6ad266c70a0eabf6c26248aed1af6fa1',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fget_134',['mi_stats_get',['../group__stats.html#gaa26d5060ad21b1331a150bb92f6f7086',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fget_5fbin_5fsize_135',['mi_stats_get_bin_size',['../group__stats.html#gadecd549d1a7216af596322ac27e78b0d',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fget_5fjson_136',['mi_stats_get_json',['../group__stats.html#gadaa3be15899cc82e314fa7d3f8e2afbd',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fmerge_137',['mi_stats_merge',['../group__stats.html#ga854b1de8cb067c7316286c28b2fcd3d1',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fprint_138',['mi_stats_print',['../group__stats.html#ga2d126e5c62d3badc35445e5d84166df2',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fprint_5fout_139',['mi_stats_print_out',['../group__stats.html#ga537f13b299ddf801e49a5a94fde02c79',1,'mimalloc-doc.h']]], - ['mi_5fstats_5freset_140',['mi_stats_reset',['../group__stats.html#ga3bb8468b8cfcc6e2a61d98aee85c5f99',1,'mimalloc-doc.h']]], - ['mi_5fstrdup_141',['mi_strdup',['../group__malloc.html#ga245ac90ebc2cfdd17de599e5fea59889',1,'mimalloc-doc.h']]], - ['mi_5fstrndup_142',['mi_strndup',['../group__malloc.html#ga486d0d26b3b3794f6d1cdb41a9aed92d',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fadd_5fcurrent_5fthread_143',['mi_subproc_add_current_thread',['../group__subproc.html#gadbc53414eb68b275588ec001ce1ddc7c',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fcurrent_144',['mi_subproc_current',['../group__subproc.html#gaabc4db6585bf16099e43e60d64364950',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fdelete_145',['mi_subproc_delete',['../group__subproc.html#gaa7d263e9429bac9ac8345c9d25de610e',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fdestroy_146',['mi_subproc_destroy',['../group__subproc.html#gada12e4d9bd382935c398ec324d0391f9',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fheap_5fstats_5fprint_5fout_147',['mi_subproc_heap_stats_print_out',['../group__stats.html#ga19f1a04bd972e11e4f3b586f25229d5b',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fmain_148',['mi_subproc_main',['../group__subproc.html#ga2ecba0d7ebdc99e71bb985c4a1609806',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fnew_149',['mi_subproc_new',['../group__subproc.html#ga8068cac328e41fa2170faef707315243',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fstats_5fget_150',['mi_subproc_stats_get',['../group__stats.html#ga5ec3f9058eec804924711ab38ba64b51',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fstats_5fget_5fjson_151',['mi_subproc_stats_get_json',['../group__stats.html#gad1cf918f08b6506e65ae134423477a0b',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fstats_5fprint_5fout_152',['mi_subproc_stats_print_out',['../group__stats.html#ga59ec89969837db414a6bce8b58915624',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fvisit_5fheaps_153',['mi_subproc_visit_heaps',['../group__subproc.html#gafd2136e7f167b5cd02b6ab00bed0ac93',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fcalloc_154',['mi_theap_calloc',['../group__theap.html#gadbf8f27bfe9edf7834e19d32cab63edc',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fcollect_155',['mi_theap_collect',['../group__theap.html#ga9377a43095b8dd14996613ceb73d9a32',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fget_5fdefault_156',['mi_theap_get_default',['../group__theap.html#ga78afbb07c1e0cd333c7539cf7403dfd5',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_157',['mi_theap_malloc',['../group__theap.html#gac7308e029c59e6675dd5ab5ec54f304a',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_5faligned_158',['mi_theap_malloc_aligned',['../group__theap.html#ga84503fc2bd83fbfb7f789b7d80d76f36',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_5fcsize_159',['mi_theap_malloc_csize',['../group__constantsize.html#ga105bd667da91a15d113fe79be04d625c',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_5fsmall_160',['mi_theap_malloc_small',['../group__theap.html#ga1fc65302a212daaecb21abda96ff247c',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5frealloc_161',['mi_theap_realloc',['../group__theap.html#ga11943128dc354313c4189e72f4299dbb',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fset_5fdefault_162',['mi_theap_set_default',['../group__theap.html#ga04420ce52785019412397e4de24a984a',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fzalloc_163',['mi_theap_zalloc',['../group__theap.html#gaa7fd973dcd8c75eb05707bdeffbc06f7',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fzalloc_5fcsize_164',['mi_theap_zalloc_csize',['../group__constantsize.html#ga1addf755829380581dd9c3c9175e376f',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fzalloc_5fsmall_165',['mi_theap_zalloc_small',['../group__theap.html#gaa85fccf9d40af3e826c5932aded1d534',1,'mimalloc-doc.h']]], - ['mi_5fthread_5fdone_166',['mi_thread_done',['../group__extended.html#ga0ae4581e85453456a0d658b2b98bf7bf',1,'mimalloc-doc.h']]], - ['mi_5fthread_5finit_167',['mi_thread_init',['../group__extended.html#gaf8e73efc2cbca9ebfdfb166983a04c17',1,'mimalloc-doc.h']]], - ['mi_5fthread_5fset_5fin_5fthreadpool_168',['mi_thread_set_in_threadpool',['../group__extended.html#gada899104276aac52539eb4de7c020f1f',1,'mimalloc-doc.h']]], - ['mi_5fusable_5fsize_169',['mi_usable_size',['../group__malloc.html#ga089c859d9eddc5f9b4bd946cd53cebee',1,'mimalloc-doc.h']]], - ['mi_5fvalloc_170',['mi_valloc',['../group__posix.html#ga50cafb9722020402f065de93799f64ca',1,'mimalloc-doc.h']]], - ['mi_5fversion_171',['mi_version',['../group__extended.html#gae9ddddcf14a58d0224ef3c8778e5b057',1,'mimalloc-doc.h']]], - ['mi_5fwcsdup_172',['mi_wcsdup',['../group__posix.html#gaa9fd7f25c9ac3a20e89b33bd6e383fcf',1,'mimalloc-doc.h']]], - ['mi_5fwdupenv_5fs_173',['mi_wdupenv_s',['../group__posix.html#ga6ac6a6a8f3c96f1af24bb8d0439cbbd1',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_174',['mi_zalloc',['../group__malloc.html#gae6e38c4403247a7b40d80419e093bfb8',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5faligned_175',['mi_zalloc_aligned',['../group__aligned.html#gaac7d0beb782f9b9ac31f47492b130f82',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5faligned_5fat_176',['mi_zalloc_aligned_at',['../group__aligned.html#ga7c1778805ce50ebbf02ccbd5e39d5dba',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5fcsize_177',['mi_zalloc_csize',['../group__constantsize.html#gadca6d340ee15b5c70695ed82d95c6cec',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5fsmall_178',['mi_zalloc_small',['../group__extended.html#ga51c47637e81df0e2f13a2d7a2dec123e',1,'mimalloc-doc.h']]] + ['mi_5ffree_5fcsize_5fnonnull_21',['mi_free_csize_nonnull',['../group__constantsize.html#ga38a93d373a9dc8edc8bca83edb6af5c5',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsize_22',['mi_free_size',['../group__posix.html#gae01389eedab8d67341ff52e2aad80ebb',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsize_5faligned_23',['mi_free_size_aligned',['../group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsize_5fnonnull_24',['mi_free_size_nonnull',['../group__posix.html#ga796602dae2a89797026f72e0c0240f32',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsmall_25',['mi_free_small',['../group__extended.html#ga2a8469764c38dcaf878f36e5f50e3610',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsmall_5fnonnull_26',['mi_free_small_nonnull',['../group__extended.html#ga22890f54e5be433837069c8e97fdf379',1,'mimalloc-doc.h']]], + ['mi_5fgood_5fsize_27',['mi_good_size',['../group__malloc.html#gac057927cd06c854b45fe7847e921bd47',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcalloc_28',['mi_heap_calloc',['../group__heap.html#gac0098aaf231d3e9586c73136d5df95da',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcalloc_5faligned_29',['mi_heap_calloc_aligned',['../group__heap.html#gacafcc26df827c7a7de5e850217566108',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcalloc_5faligned_5fat_30',['mi_heap_calloc_aligned_at',['../group__heap.html#gaa42ec2079989c4374f2c331d9b35f4e4',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcheck_5fowned_31',['mi_heap_check_owned',['../group__heap.html#ga0d67c1789faaa15ff366c024fcaf6377',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcollect_32',['mi_heap_collect',['../group__heap.html#ga7922f7495cde30b1984d0e6072419298',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcontains_33',['mi_heap_contains',['../group__heap.html#gac988496aea8ffc5c8bf92a8b10165fa0',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcontains_5fblock_34',['mi_heap_contains_block',['../group__heap.html#gaa862aa8ed8d57d84cae41fc1022d71af',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fdelete_35',['mi_heap_delete',['../group__heap.html#ga2ab1af8d438819b55319c7ef51d1e409',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fdestroy_36',['mi_heap_destroy',['../group__heap.html#ga9f9c0844edb9717f4feacd79116b8e0d',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fget_5fbacking_37',['mi_heap_get_backing',['../group__heap.html#gac6ac9f0e7be9ab4ff70acfc8dad1235a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fget_5fdefault_38',['mi_heap_get_default',['../group__heap.html#ga14c667a6e2c5d28762d8cb7d4e057909',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmain_39',['mi_heap_main',['../group__heap.html#ga8fc318a65dc79ee8778c6a0e52d12477',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_40',['mi_heap_malloc',['../group__heap.html#gab374e206c7034e0d899fb934e4f4a863',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5faligned_41',['mi_heap_malloc_aligned',['../group__heap.html#ga33f4f05b7fea7af2113c62a4bf882cc5',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5faligned_5fat_42',['mi_heap_malloc_aligned_at',['../group__heap.html#gae7ffc045c3996497a7f3a5f6fe7b8aaa',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5fsmall_43',['mi_heap_malloc_small',['../group__heap.html#ga012c5c8abe22b10043de39ff95909541',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmallocn_44',['mi_heap_mallocn',['../group__heap.html#gab0f755c0b21c387fe8e9024200faa372',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fnew_45',['mi_heap_new',['../group__heap.html#gaa718bb226ec0546ba6d1b6cb32179f3a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fnew_5fex_46',['mi_heap_new_ex',['../group__arenas.html#ga3ae360583f4351aa5267ee7e43008faf',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fnew_5fin_5farena_47',['mi_heap_new_in_arena',['../group__arenas.html#gaaf2d9976576d5efd5544be12848af949',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fof_48',['mi_heap_of',['../group__heap.html#ga424b69f285109b2515631d59f5eecb1b',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealloc_49',['mi_heap_realloc',['../group__heap.html#gac5252d6a2e510bd349e4fcb452e6a93a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealloc_5faligned_50',['mi_heap_realloc_aligned',['../group__heap.html#gaccf8c249872f30bf1c2493a09197d734',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealloc_5faligned_5fat_51',['mi_heap_realloc_aligned_at',['../group__heap.html#ga6df988a7219d5707f010d5f3eb0dc3f5',1,'mimalloc-doc.h']]], + ['mi_5fheap_5freallocf_52',['mi_heap_reallocf',['../group__heap.html#gae7cd171425bee04c683c65a3701f0b4a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5freallocn_53',['mi_heap_reallocn',['../group__heap.html#gaccf7bfe10ce510a000d3547d9cf7fa29',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealpath_54',['mi_heap_realpath',['../group__heap.html#ga55545a3ec6da29c5b4f62e540ecac1e2',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_55',['mi_heap_recalloc',['../group__zeroinit.html#gad1a0d325d930eeb80f25e3fea37aacde',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_5faligned_56',['mi_heap_recalloc_aligned',['../group__zeroinit.html#ga87ddd674bf1c67237d780d0b9e0f0f32',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_5faligned_5fat_57',['mi_heap_recalloc_aligned_at',['../group__zeroinit.html#ga07b5bcbaf00d0d2e598c232982588496',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frezalloc_58',['mi_heap_rezalloc',['../group__zeroinit.html#ga8d8b7ebb24b513cd84d1a696048da60d',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frezalloc_5faligned_59',['mi_heap_rezalloc_aligned',['../group__zeroinit.html#ga5129f6dc46ee1613d918820a8a0533a7',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frezalloc_5faligned_5fat_60',['mi_heap_rezalloc_aligned_at',['../group__zeroinit.html#ga2bafa79c3f98ea74882349d44cffa5d9',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fset_5fdefault_61',['mi_heap_set_default',['../group__heap.html#ga349b677dec7da5eacdbc7a385bd62a4a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fset_5fnuma_5faffinity_62',['mi_heap_set_numa_affinity',['../group__heap.html#gaab93abbc0624f570e2900f95cac18591',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fget_63',['mi_heap_stats_get',['../group__stats.html#ga2654195cef93ed3613e98b59a19a296f',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fget_5fjson_64',['mi_heap_stats_get_json',['../group__stats.html#gacb42dede4c38c38f2014b5450c5fcba8',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fmerge_5fto_5fsubproc_65',['mi_heap_stats_merge_to_subproc',['../group__stats.html#ga5a34328f846c661aa560f4769cfc15f1',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fprint_5fout_66',['mi_heap_stats_print_out',['../group__stats.html#gafea365d7842f5f5bc1f235bbc303d585',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstrdup_67',['mi_heap_strdup',['../group__heap.html#ga5754e09ccc51dd6bc73885bb6ea21b7a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstrndup_68',['mi_heap_strndup',['../group__heap.html#gad224df78f1fbee942df8adf023e12cf3',1,'mimalloc-doc.h']]], + ['mi_5fheap_5ftheap_69',['mi_heap_theap',['../group__theap.html#ga7905a73865733da8a43339a500533180',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fvisit_5fblocks_70',['mi_heap_visit_blocks',['../group__analysis.html#gae69fcff3324f8e419af850e34cf54c46',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_71',['mi_heap_zalloc',['../group__heap.html#gabebc796399619d964d8db77aa835e8c1',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5faligned_72',['mi_heap_zalloc_aligned',['../group__heap.html#ga6466bde8b5712aa34e081a8317f9f471',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5faligned_5fat_73',['mi_heap_zalloc_aligned_at',['../group__heap.html#ga484e3d01cd174f78c7e53370e5a7c819',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5fsmall_74',['mi_heap_zalloc_small',['../group__heap.html#ga0994f44cc595f9c1840a9af162d98597',1,'mimalloc-doc.h']]], + ['mi_5fis_5fin_5fheap_5fregion_75',['mi_is_in_heap_region',['../group__heap.html#ga5f071b10d4df1c3658e04e7fd67a94e6',1,'mimalloc-doc.h']]], + ['mi_5fis_5fredirected_76',['mi_is_redirected',['../group__extended.html#gaad25050b19f30cd79397b227e0157a3f',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_77',['mi_malloc',['../group__malloc.html#gae1dd97b542420c87ae085e822b1229e8',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5faligned_78',['mi_malloc_aligned',['../group__aligned.html#ga69578ff1a98ca16e1dcd02c0995cd65c',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5faligned_5fat_79',['mi_malloc_aligned_at',['../group__aligned.html#ga2022f71b95a7cd6cce1b6e07752ae8ca',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fcsize_80',['mi_malloc_csize',['../group__constantsize.html#ga9cd44d925d34f7180922e12bbf594571',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fgood_5fsize_81',['mi_malloc_good_size',['../group__posix.html#ga9d23ac7885fed7413c11d8e0ffa31071',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fsize_82',['mi_malloc_size',['../group__posix.html#ga4531c9e775bb3ae12db57c1ba8a5d7de',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fsmall_83',['mi_malloc_small',['../group__extended.html#ga7f050bc6b897da82692174f5fce59cde',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fusable_5fsize_84',['mi_malloc_usable_size',['../group__posix.html#ga06d07cf357bbac5c73ba5d0c0c421e17',1,'mimalloc-doc.h']]], + ['mi_5fmallocn_85',['mi_mallocn',['../group__malloc.html#ga61f46bade3db76ca24aaafedc40de7b6',1,'mimalloc-doc.h']]], + ['mi_5fmanage_5fos_5fmemory_86',['mi_manage_os_memory',['../group__arenas.html#ga4c6486a1fdcd7a423b5f25fe4be8e0cf',1,'mimalloc-doc.h']]], + ['mi_5fmanage_5fos_5fmemory_5fex_87',['mi_manage_os_memory_ex',['../group__arenas.html#ga41ce8525d77bbb60f618fa1029994f6e',1,'mimalloc-doc.h']]], + ['mi_5fmbsdup_88',['mi_mbsdup',['../group__posix.html#ga7b82a44094fdec4d2084eb4288a979b0',1,'mimalloc-doc.h']]], + ['mi_5fmemalign_89',['mi_memalign',['../group__posix.html#ga726867f13fd29ca36064954c0285b1d8',1,'mimalloc-doc.h']]], + ['mi_5fnew_90',['mi_new',['../group__cpp.html#ga633d96e3bc7011f960df9f3b2731fc6a',1,'mimalloc-doc.h']]], + ['mi_5fnew_5faligned_91',['mi_new_aligned',['../group__cpp.html#ga79c54da0b4b4ce9fcc11d2f6ef6675f8',1,'mimalloc-doc.h']]], + ['mi_5fnew_5faligned_5fnothrow_92',['mi_new_aligned_nothrow',['../group__cpp.html#ga92ae00b6dd64406c7e64557711ec04b7',1,'mimalloc-doc.h']]], + ['mi_5fnew_5fn_93',['mi_new_n',['../group__cpp.html#gadd11b85c15d21d308386844b5233856c',1,'mimalloc-doc.h']]], + ['mi_5fnew_5fnothrow_94',['mi_new_nothrow',['../group__cpp.html#ga5cb4f120d1f7296074256215aa9a9e54',1,'mimalloc-doc.h']]], + ['mi_5fnew_5frealloc_95',['mi_new_realloc',['../group__cpp.html#ga6867d89baf992728e0cc20a1f47db4d0',1,'mimalloc-doc.h']]], + ['mi_5fnew_5freallocn_96',['mi_new_reallocn',['../group__cpp.html#gaace912ce086682d56f3ce9f7638d9d67',1,'mimalloc-doc.h']]], + ['mi_5foption_5fdisable_97',['mi_option_disable',['../group__options.html#gaebf6ff707a2e688ebb1a2296ca564054',1,'mimalloc-doc.h']]], + ['mi_5foption_5fenable_98',['mi_option_enable',['../group__options.html#ga04180ae41b0d601421dd62ced40ca050',1,'mimalloc-doc.h']]], + ['mi_5foption_5fget_99',['mi_option_get',['../group__options.html#ga7e8af195cc81d3fa64ccf2662caa565a',1,'mimalloc-doc.h']]], + ['mi_5foption_5fget_5fclamp_100',['mi_option_get_clamp',['../group__options.html#ga96ad9c406338bd314cfe878cfc9bf723',1,'mimalloc-doc.h']]], + ['mi_5foption_5fget_5fsize_101',['mi_option_get_size',['../group__options.html#ga274db5a6ac87cc24ef0b23e7006ed02c',1,'mimalloc-doc.h']]], + ['mi_5foption_5fis_5fenabled_102',['mi_option_is_enabled',['../group__options.html#ga459ad98f18b3fc9275474807fe0ca188',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_103',['mi_option_set',['../group__options.html#gaf84921c32375e25754dc2ee6a911fa60',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_5fdefault_104',['mi_option_set_default',['../group__options.html#ga7ef623e440e6e5545cb08c94e71e4b90',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_5fenabled_105',['mi_option_set_enabled',['../group__options.html#ga9a13d05fcb77489cb06d4d017ebd8bed',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_5fenabled_5fdefault_106',['mi_option_set_enabled_default',['../group__options.html#ga65518b69ec5d32336b50e07f74b3f629',1,'mimalloc-doc.h']]], + ['mi_5foptions_5fprint_107',['mi_options_print',['../group__options.html#gaecfcd3ed90af99932670d881de1fa8f1',1,'mimalloc-doc.h']]], + ['mi_5foptions_5fprint_5fout_108',['mi_options_print_out',['../group__options.html#ga1dd50c7533baec4cc6c1a2e9adf05dd2',1,'mimalloc-doc.h']]], + ['mi_5fposix_5fmemalign_109',['mi_posix_memalign',['../group__posix.html#gacff84f226ba9feb2031b8992e5579447',1,'mimalloc-doc.h']]], + ['mi_5fprocess_5finfo_110',['mi_process_info',['../group__stats.html#ga7d862c2affd5790381da14eb102a364d',1,'mimalloc-doc.h']]], + ['mi_5fprocess_5finfo_5fprint_111',['mi_process_info_print',['../group__stats.html#gaf887e7aa75f13242c4e7ab95d8390536',1,'mimalloc-doc.h']]], + ['mi_5fprocess_5finfo_5fprint_5fout_112',['mi_process_info_print_out',['../group__stats.html#ga03ac88d3efee2ae7c34f5340502f4732',1,'mimalloc-doc.h']]], + ['mi_5fpvalloc_113',['mi_pvalloc',['../group__posix.html#ga644bebccdbb2821542dd8c7e7641f476',1,'mimalloc-doc.h']]], + ['mi_5frealloc_114',['mi_realloc',['../group__malloc.html#ga0621af6a5e3aa384e6a1b548958bf583',1,'mimalloc-doc.h']]], + ['mi_5frealloc_5faligned_115',['mi_realloc_aligned',['../group__aligned.html#ga5d7a46d054b4d7abe9d8d2474add2edf',1,'mimalloc-doc.h']]], + ['mi_5frealloc_5faligned_5fat_116',['mi_realloc_aligned_at',['../group__aligned.html#gad06dcf2bb8faadb2c8ea61ee5d24bbf6',1,'mimalloc-doc.h']]], + ['mi_5freallocarr_117',['mi_reallocarr',['../group__posix.html#ga7e1934d60a3e697950eeb48e042bfad5',1,'mimalloc-doc.h']]], + ['mi_5freallocarray_118',['mi_reallocarray',['../group__posix.html#gadfeccb72748a2f6305474a37d9d57bce',1,'mimalloc-doc.h']]], + ['mi_5freallocf_119',['mi_reallocf',['../group__malloc.html#ga4dc3a4067037b151a64629fe8a332641',1,'mimalloc-doc.h']]], + ['mi_5freallocn_120',['mi_reallocn',['../group__malloc.html#ga8bddfb4a1270a0854bbcf44cb3980467',1,'mimalloc-doc.h']]], + ['mi_5frealpath_121',['mi_realpath',['../group__malloc.html#ga94c3afcc086e85d75a57e9f76b9b71dd',1,'mimalloc-doc.h']]], + ['mi_5frecalloc_122',['mi_recalloc',['../group__zeroinit.html#ga23a0fbb452b5dce8e31fab1a1958cacc',1,'mimalloc-doc.h']]], + ['mi_5frecalloc_5faligned_123',['mi_recalloc_aligned',['../group__zeroinit.html#ga3e2169b48683aa0ab64f813fd68d839e',1,'mimalloc-doc.h']]], + ['mi_5frecalloc_5faligned_5fat_124',['mi_recalloc_aligned_at',['../group__zeroinit.html#gaae25e4ddedd4e0fb61b1a8bd5d452750',1,'mimalloc-doc.h']]], + ['mi_5fregister_5fdeferred_5ffree_125',['mi_register_deferred_free',['../group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece',1,'mimalloc-doc.h']]], + ['mi_5fregister_5ferror_126',['mi_register_error',['../group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45',1,'mimalloc-doc.h']]], + ['mi_5fregister_5foutput_127',['mi_register_output',['../group__extended.html#gae5b17ff027cd2150b43a33040250cf3f',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fhuge_5fos_5fpages_5fat_128',['mi_reserve_huge_os_pages_at',['../group__arenas.html#ga7795a13d20087447281858d2c771cca1',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fhuge_5fos_5fpages_5fat_5fex_129',['mi_reserve_huge_os_pages_at_ex',['../group__arenas.html#ga591aab1c2bc2ca920e33f0f9f9cb5c52',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fhuge_5fos_5fpages_5finterleave_130',['mi_reserve_huge_os_pages_interleave',['../group__arenas.html#ga3132f521fb756fc0e8ec0b74fb58df50',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fos_5fmemory_131',['mi_reserve_os_memory',['../group__arenas.html#ga00ec3324b6b2591c7fe3677baa30a767',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fos_5fmemory_5fex_132',['mi_reserve_os_memory_ex',['../group__arenas.html#ga32f519797fd9a81acb4f52d36e6d751b',1,'mimalloc-doc.h']]], + ['mi_5frezalloc_133',['mi_rezalloc',['../group__zeroinit.html#gadfd34cd7b4f2bbda7ae06367a6360756',1,'mimalloc-doc.h']]], + ['mi_5frezalloc_5faligned_134',['mi_rezalloc_aligned',['../group__zeroinit.html#ga4d02404fe1e7db00beb65f185e012caa',1,'mimalloc-doc.h']]], + ['mi_5frezalloc_5faligned_5fat_135',['mi_rezalloc_aligned_at',['../group__zeroinit.html#ga6843a88285bbfcc3bdfccc60aafd1270',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fas_5fjson_136',['mi_stats_as_json',['../group__stats.html#ga6ad266c70a0eabf6c26248aed1af6fa1',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fget_137',['mi_stats_get',['../group__stats.html#gaa26d5060ad21b1331a150bb92f6f7086',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fget_5fbin_5fsize_138',['mi_stats_get_bin_size',['../group__stats.html#gadecd549d1a7216af596322ac27e78b0d',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fget_5fjson_139',['mi_stats_get_json',['../group__stats.html#gadaa3be15899cc82e314fa7d3f8e2afbd',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fmerge_140',['mi_stats_merge',['../group__stats.html#ga854b1de8cb067c7316286c28b2fcd3d1',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fprint_141',['mi_stats_print',['../group__stats.html#ga2d126e5c62d3badc35445e5d84166df2',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fprint_5fout_142',['mi_stats_print_out',['../group__stats.html#ga537f13b299ddf801e49a5a94fde02c79',1,'mimalloc-doc.h']]], + ['mi_5fstats_5freset_143',['mi_stats_reset',['../group__stats.html#ga3bb8468b8cfcc6e2a61d98aee85c5f99',1,'mimalloc-doc.h']]], + ['mi_5fstrdup_144',['mi_strdup',['../group__malloc.html#ga245ac90ebc2cfdd17de599e5fea59889',1,'mimalloc-doc.h']]], + ['mi_5fstrndup_145',['mi_strndup',['../group__malloc.html#ga486d0d26b3b3794f6d1cdb41a9aed92d',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fadd_5fcurrent_5fthread_146',['mi_subproc_add_current_thread',['../group__subproc.html#gadbc53414eb68b275588ec001ce1ddc7c',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fcurrent_147',['mi_subproc_current',['../group__subproc.html#gaabc4db6585bf16099e43e60d64364950',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fdelete_148',['mi_subproc_delete',['../group__subproc.html#gaa7d263e9429bac9ac8345c9d25de610e',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fdestroy_149',['mi_subproc_destroy',['../group__subproc.html#gada12e4d9bd382935c398ec324d0391f9',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fheap_5fstats_5fprint_5fout_150',['mi_subproc_heap_stats_print_out',['../group__stats.html#ga19f1a04bd972e11e4f3b586f25229d5b',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fmain_151',['mi_subproc_main',['../group__subproc.html#ga2ecba0d7ebdc99e71bb985c4a1609806',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fnew_152',['mi_subproc_new',['../group__subproc.html#ga8068cac328e41fa2170faef707315243',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fstats_5fget_153',['mi_subproc_stats_get',['../group__stats.html#ga5ec3f9058eec804924711ab38ba64b51',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fstats_5fget_5fjson_154',['mi_subproc_stats_get_json',['../group__stats.html#gad1cf918f08b6506e65ae134423477a0b',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fstats_5fprint_5fout_155',['mi_subproc_stats_print_out',['../group__stats.html#ga59ec89969837db414a6bce8b58915624',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fvisit_5fheaps_156',['mi_subproc_visit_heaps',['../group__subproc.html#gafd2136e7f167b5cd02b6ab00bed0ac93',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fcalloc_157',['mi_theap_calloc',['../group__theap.html#gadbf8f27bfe9edf7834e19d32cab63edc',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fcollect_158',['mi_theap_collect',['../group__theap.html#ga9377a43095b8dd14996613ceb73d9a32',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fget_5fdefault_159',['mi_theap_get_default',['../group__theap.html#ga78afbb07c1e0cd333c7539cf7403dfd5',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_160',['mi_theap_malloc',['../group__theap.html#gac7308e029c59e6675dd5ab5ec54f304a',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_5faligned_161',['mi_theap_malloc_aligned',['../group__theap.html#ga84503fc2bd83fbfb7f789b7d80d76f36',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_5fcsize_162',['mi_theap_malloc_csize',['../group__constantsize.html#ga105bd667da91a15d113fe79be04d625c',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_5fsmall_163',['mi_theap_malloc_small',['../group__theap.html#ga1fc65302a212daaecb21abda96ff247c',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5frealloc_164',['mi_theap_realloc',['../group__theap.html#ga11943128dc354313c4189e72f4299dbb',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fset_5fdefault_165',['mi_theap_set_default',['../group__theap.html#ga04420ce52785019412397e4de24a984a',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fzalloc_166',['mi_theap_zalloc',['../group__theap.html#gaa7fd973dcd8c75eb05707bdeffbc06f7',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fzalloc_5fcsize_167',['mi_theap_zalloc_csize',['../group__constantsize.html#ga1addf755829380581dd9c3c9175e376f',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fzalloc_5fsmall_168',['mi_theap_zalloc_small',['../group__theap.html#gaa85fccf9d40af3e826c5932aded1d534',1,'mimalloc-doc.h']]], + ['mi_5fthread_5fdone_169',['mi_thread_done',['../group__extended.html#ga0ae4581e85453456a0d658b2b98bf7bf',1,'mimalloc-doc.h']]], + ['mi_5fthread_5finit_170',['mi_thread_init',['../group__extended.html#gaf8e73efc2cbca9ebfdfb166983a04c17',1,'mimalloc-doc.h']]], + ['mi_5fthread_5fset_5fin_5fthreadpool_171',['mi_thread_set_in_threadpool',['../group__extended.html#gada899104276aac52539eb4de7c020f1f',1,'mimalloc-doc.h']]], + ['mi_5fusable_5fsize_172',['mi_usable_size',['../group__malloc.html#ga089c859d9eddc5f9b4bd946cd53cebee',1,'mimalloc-doc.h']]], + ['mi_5fvalloc_173',['mi_valloc',['../group__posix.html#ga50cafb9722020402f065de93799f64ca',1,'mimalloc-doc.h']]], + ['mi_5fversion_174',['mi_version',['../group__extended.html#gae9ddddcf14a58d0224ef3c8778e5b057',1,'mimalloc-doc.h']]], + ['mi_5fwcsdup_175',['mi_wcsdup',['../group__posix.html#gaa9fd7f25c9ac3a20e89b33bd6e383fcf',1,'mimalloc-doc.h']]], + ['mi_5fwdupenv_5fs_176',['mi_wdupenv_s',['../group__posix.html#ga6ac6a6a8f3c96f1af24bb8d0439cbbd1',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_177',['mi_zalloc',['../group__malloc.html#gae6e38c4403247a7b40d80419e093bfb8',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5faligned_178',['mi_zalloc_aligned',['../group__aligned.html#gaac7d0beb782f9b9ac31f47492b130f82',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5faligned_5fat_179',['mi_zalloc_aligned_at',['../group__aligned.html#ga7c1778805ce50ebbf02ccbd5e39d5dba',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5fcsize_180',['mi_zalloc_csize',['../group__constantsize.html#gadca6d340ee15b5c70695ed82d95c6cec',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5fsmall_181',['mi_zalloc_small',['../group__extended.html#ga51c47637e81df0e2f13a2d7a2dec123e',1,'mimalloc-doc.h']]] ]; diff --git a/docs/tools.html b/docs/tools.html index fa662b9fa..d969be621 100644 --- a/docs/tools.html +++ b/docs/tools.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/topics.html b/docs/topics.html index bbbae22f2..efbd5b033 100644 --- a/docs/topics.html +++ b/docs/topics.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
@@ -112,7 +112,7 @@  Basic AllocationThe basic allocation interface  Aligned AllocationAllocating aligned memory blocks  Typed MacrosTyped allocation macros - Constant size allocationConstant size allocation and freeing For example: + Constant size allocationInlined definitions for onstant size allocation and freeing  Zero initialized re-allocationRe-allocation of zero initialized blocks  HeapsFirst-class heaps  Heap IntrospectionWalk areas and blocks of the heap at runtime diff --git a/docs/using.html b/docs/using.html index 87f479dce..ee8b7f7e6 100644 --- a/docs/using.html +++ b/docs/using.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/ide/vs2022/mimalloc-lib.vcxproj b/ide/vs2022/mimalloc-lib.vcxproj index 216f418c0..9a17c0ae3 100644 --- a/ide/vs2022/mimalloc-lib.vcxproj +++ b/ide/vs2022/mimalloc-lib.vcxproj @@ -174,7 +174,7 @@ true Default ../../include - MI_DEBUG=3;%(PreprocessorDefinitions); + MI_OPT_FREE=1;MI_DEBUG=3;%(PreprocessorDefinitions); CompileAsCpp false stdcpp20 @@ -194,7 +194,7 @@ true Default ../../include - MI_DEBUG=3;%(PreprocessorDefinitions); + MI_OPT_FREE=1;MI_DEBUG=3;%(PreprocessorDefinitions); CompileAsCpp false stdcpp20 @@ -222,7 +222,7 @@ true Default ../../include - MI_DEBUG=3;%(PreprocessorDefinitions); + MI_OPT_FREE=1;MI_DEBUG=3;%(PreprocessorDefinitions); CompileAsCpp false stdcpp20 @@ -250,7 +250,7 @@ true Default ../../include - MI_DEBUG=3;MI_GUARDED=0;%(PreprocessorDefinitions); + MI_OPT_FREE=1;MI_DEBUG=3;MI_GUARDED=0;%(PreprocessorDefinitions); CompileAsCpp false stdcpp20 @@ -278,7 +278,7 @@ true Default ../../include - %(PreprocessorDefinitions);NDEBUG + MI_OPT_FREE=1;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) false @@ -307,7 +307,7 @@ true Default ../../include - %(PreprocessorDefinitions);NDEBUG + MI_OPT_FREE=1;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) false @@ -343,7 +343,7 @@ true Default ../../include - %(PreprocessorDefinitions);NDEBUG + MI_OPT_FREE=1;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) false @@ -380,7 +380,7 @@ true Default ../../include - %(PreprocessorDefinitions);NDEBUG + MI_OPT_FREE=1;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) false @@ -451,6 +451,8 @@ + + @@ -488,6 +490,7 @@ + diff --git a/ide/vs2022/mimalloc-lib.vcxproj.filters b/ide/vs2022/mimalloc-lib.vcxproj.filters index 0c23b2834..c140f17cf 100644 --- a/ide/vs2022/mimalloc-lib.vcxproj.filters +++ b/ide/vs2022/mimalloc-lib.vcxproj.filters @@ -70,6 +70,12 @@ Sources + + Sources + + + Sources + @@ -108,6 +114,9 @@ Headers + + Headers + diff --git a/ide/vs2022/mimalloc-override-dll.vcxproj b/ide/vs2022/mimalloc-override-dll.vcxproj index 18fce6938..c2338bb2e 100644 --- a/ide/vs2022/mimalloc-override-dll.vcxproj +++ b/ide/vs2022/mimalloc-override-dll.vcxproj @@ -170,7 +170,7 @@ true true ../../include - MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); + MI_OPT_FREE=1;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false CompileAsCpp @@ -201,7 +201,7 @@ true true ../../include - MI_DEBUG=4;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); + MI_OPT_FREE=1;MI_DEBUG=4;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false CompileAsCpp @@ -232,7 +232,7 @@ true true ../../include - MI_DEBUG=4;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); + MI_OPT_FREE=1;MI_DEBUG=4;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false CompileAsCpp @@ -263,7 +263,7 @@ true true ../../include - MI_DEBUG=4;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); + MI_OPT_FREE=1;MI_DEBUG=4;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false CompileAsCpp @@ -295,7 +295,7 @@ true true ../../include - MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG + MI_OPT_FREE=1;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) false @@ -330,7 +330,7 @@ true true ../../include - MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG + MI_OPT_FREE=1;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) false @@ -365,7 +365,7 @@ true true ../../include - MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG + MI_OPT_FREE=1;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) false @@ -401,7 +401,7 @@ true true ../../include - MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG + MI_OPT_FREE=1;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) false @@ -433,6 +433,7 @@ + @@ -480,6 +481,8 @@ + + diff --git a/ide/vs2022/mimalloc-override-dll.vcxproj.filters b/ide/vs2022/mimalloc-override-dll.vcxproj.filters index f57933c17..8bced90ae 100644 --- a/ide/vs2022/mimalloc-override-dll.vcxproj.filters +++ b/ide/vs2022/mimalloc-override-dll.vcxproj.filters @@ -70,6 +70,12 @@ Sources + + Sources + + + Sources + @@ -108,6 +114,9 @@ Headers + + Headers + diff --git a/include/mimalloc-profile.h b/include/mimalloc-profile.h new file mode 100644 index 000000000..5a1327ff7 --- /dev/null +++ b/include/mimalloc-profile.h @@ -0,0 +1,79 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2024-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_PROFILE_H +#define MIMALLOC_PROFILE_H + +#include +#include // bool +#include // uint64_t + +//--------------------------------------------------------------------------- +// Initial support for profiling hooks; still experimental +//--------------------------------------------------------------------------- + +#define MI_PROFILE_SAMPLE_DATA_MAX_SIZE (1024) + +// User defined profiler; see `test_profile.c` for an example. +typedef struct mi_profiler_s mi_profiler_t; + +// Profiler data is stored together with each sampled allocation (unless the `on_free` field in the profiler is NULL.) +typedef struct mi_profiler_sample_data_s { + size_t user_data_size; // size of the user_data (should be the `mi_profiler_t.sample_data_size`) + void* user_data[1]; // default, but can be less or more (up to 1KiB), depending on `sample_data_size` +} mi_profiler_sample_data_t; + +// Profiling callback invoked on each sampled allocation. +// If `profiler_data!=NULL` (i.e. when `on_free` is not NULL), then `profiler_data->requested_size == requested_size`. +typedef size_t (mi_cdecl mi_profiler_on_alloc_fun )(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, size_t requested_size, size_t bytes_sample_rate, uint64_t bytes_since_last_sample, const mi_heap_t* heap); + +// Profiling callback invoked on each sampled in-place re-allocation. +typedef size_t (mi_cdecl mi_profiler_on_realloc_inplace_fun)(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, size_t old_size, const mi_heap_t* heap); + +// Profiling callback invoked on a previously sampled allocation. +typedef void (mi_cdecl mi_profiler_on_free_fun )(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, const mi_heap_t* heap); + +// A profiler +// All fields are considered immutable such that they can be copied and accessed concurrently. All fields can be NULL/0. +struct mi_profiler_s { + void* reserved; // opaque; reserved by mimalloc + size_t sample_data_size; // size of required profiler data for each sampled allocation (or zero for no data) + size_t initial_sample_rate;// initial sample rate in bytes (set to at least 1 or higher) (can be adjusted by `on_alloc`) + mi_profiler_on_alloc_fun* on_alloc; // called on a sampled allocation (may be called concurrently) + mi_profiler_on_free_fun* on_free; // called on when previous sampled allocation is freed (may be called concurrently) + mi_profiler_on_realloc_inplace_fun* on_realloc_inplace; // (as yet unused) called on in-place reallocation of a previous sampled allocation (may be called concurrently) + // ... more user fields allowed +}; + +// Exported definitions +#ifdef __cplusplus +extern "C" { +#endif + +// attach a profiler to a particular heap only. +mi_decl_export bool mi_heap_profile(mi_heap_t* heap, mi_profiler_t* profiler); + +// disable profiling for a particular heap; useful for a heap that the profiler uses itself for metadata. +mi_decl_export void mi_heap_profile_disable(mi_heap_t* heap); + +// attach a profiler to any (current and future) heaps in a sub-process (unless those heaps disabled profiling) +mi_decl_export bool mi_subproc_profile(mi_subproc_id_t subproc_id, mi_profiler_t* profiler); + +// attach a profiler to any heaps in the main sub-process +mi_decl_export bool mi_profile(mi_profiler_t* profiler); + +// start sampling +mi_decl_export bool mi_profiler_start(mi_profiler_t* profiler); + +// end sampling +mi_decl_export bool mi_profiler_stop(mi_profiler_t* profiler); + +#ifdef __cplusplus +} +#endif + +#endif // MIMALLOC_PROFILE_H diff --git a/include/mimalloc-stats.h b/include/mimalloc-stats.h index 853ff9c68..7472be806 100644 --- a/include/mimalloc-stats.h +++ b/include/mimalloc-stats.h @@ -49,7 +49,7 @@ typedef struct mi_stat_counter_s { MI_STAT_COUNT(threads) /* number of threads */ \ MI_STAT_COUNT(malloc_normal) /* allocated bytes <= MI_LARGE_OBJ_SIZE_MAX */ \ MI_STAT_COUNT(malloc_huge) /* allocated bytes in huge pages */ \ - MI_STAT_COUNT(malloc_requested) /* malloc requested bytes */ \ + MI_STAT_COUNTER(malloc_requested) /* malloc requested bytes */ \ \ MI_STAT_COUNTER(mmap_calls) \ MI_STAT_COUNTER(commit_calls) \ @@ -75,13 +75,19 @@ typedef struct mi_stat_counter_s { /* only on v3 */ \ MI_STAT_COUNT(heaps) \ MI_STAT_COUNT(theaps) \ + MI_STAT_COUNT(pages_os_abandoned) /* pages in the os abandoned list (bad) */ \ + MI_STAT_COUNT(pages_os_allocated) /* pages allocated outside arenas (bad) */ \ MI_STAT_COUNTER(pages_reclaim_on_alloc) \ MI_STAT_COUNTER(pages_reclaim_on_free) \ MI_STAT_COUNTER(pages_reabandon_full) \ MI_STAT_COUNTER(pages_unabandon_busy_wait) \ - MI_STAT_COUNTER(heaps_delete_wait) + MI_STAT_COUNTER(heaps_delete_wait) \ + MI_STAT_COUNTER(pages_stat_updates) /* calls to successful page_stat_update */ \ + MI_STAT_COUNTER(pages_stat_update_count) /* total free/allocs */ \ + MI_STAT_COUNTER(profile_samples) /* total sampled profiled allocations */ \ + // note: the hole-purging counters are NOT in here -- `mi_stats_t` is embedded in `mi_theap_t`, -// whose size is already at `MI_META_MAX_SIZE` (8KB) and cannot grow. See `mi_purge_holes_stats_t`. +// whose size is bounded by `MI_META_MAX_SIZE` (8KB). See `mi_purge_holes_stats_t`. // Size bins for chunks typedef enum mi_chunkbin_e { @@ -167,6 +173,12 @@ mi_decl_export size_t mi_stats_get_bin_size(size_t bin) mi_attr_noexcept; mi_decl_export char* mi_heap_dump_json(bool include_blocks, bool hash_addresses) mi_attr_noexcept; mi_decl_export size_t mi_heap_get_seq(mi_heap_t* heap) mi_attr_noexcept; +// get theap stats (only thread safe on theaps belonging to the calling thread) +mi_decl_export bool mi_theap_stats_get(mi_theap_t* theap, mi_stats_t* stats) mi_attr_noexcept; + +// add the theap stats to the stats of the parent heap and clear the theap stats +mi_decl_export void mi_theap_stats_merge_to_heap(mi_theap_t* theap) mi_attr_noexcept; + #ifdef __cplusplus } #endif diff --git a/include/mimalloc.h b/include/mimalloc.h index e9c3b5845..bf8744651 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 30500 // major + 2 digits minor + 2 digits patch +#define MI_MALLOC_VERSION 30502 // major + 2 digits minor + 2 digits patch // ------------------------------------------------------ // Compiler specific attributes @@ -97,6 +97,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // size_t, wchar_t #include // bool +#include // inline assertions #ifdef __cplusplus extern "C" { @@ -119,8 +120,8 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* // ------------------------------------------------------ // Extended allocation functions // ------------------------------------------------------ -#define MI_SMALL_WSIZE_MAX (128) -#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*)) +#define MI_SMALL_WSIZE_MAX 128 +#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(size_t)) // mimalloc considers `sizeof(size_t)` as the machine word size mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); @@ -133,13 +134,17 @@ mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize) mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept; -// `mi_free_size` can be more efficient (as at calls mi_free_small internally). +// `mi_free_size` can be more efficient (as it calls mi_free_small internally). mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; // `mi_free_small` is for special applications like language runtimes. -// it should only be used to free objects from `mi_(heap_)(m|z)alloc_small` and is potentially a tiny bit faster than `mi_free` +// it should only be used to free objects from `mi_*alloc_small` and is potentially a tiny bit faster than `mi_free` mi_decl_export void mi_free_small(void* p) mi_attr_noexcept; +// As `mi_free_small` but `p` should not be a NULL pointer. +mi_decl_export void mi_free_small_nonnull(void* p) mi_attr_noexcept; + + // ------------------------------------------------------------------------------------- // Aligned allocation // Note that `alignment` always follows `size` for consistency with unaligned @@ -392,16 +397,9 @@ mi_decl_export void mi_arenas_print(void) mi_attr_noexcept; mi_decl_export int mi_heap_snapshot(int fd, unsigned flags) mi_attr_noexcept; mi_decl_export int mi_heap_snapshot_to_file(const char* path, unsigned flags) mi_attr_noexcept; -// Sampling heap profiler (pprof-compatible). Zero overhead on the malloc/free -// fast paths when disabled (sample_rate_bytes == 0). Output is a profile.proto -// readable by `go tool pprof`. -mi_decl_export void mi_prof_enable(size_t sample_rate_bytes) mi_attr_noexcept; -mi_decl_export void mi_prof_reset(void) mi_attr_noexcept; -mi_decl_export int mi_prof_dump(int fd) mi_attr_noexcept; -mi_decl_export size_t mi_prof_dump_buf(void* buf, size_t cap) mi_attr_noexcept; // returns total size; buf=NULL to query -mi_decl_export int mi_prof_dump_to_file(const char* path) mi_attr_noexcept; mi_decl_export size_t mi_arena_min_alignment(void); mi_decl_export size_t mi_arena_min_size(void); +mi_decl_export size_t mi_arena_max_object_size(void); typedef void* mi_arena_id_t; mi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size); @@ -463,21 +461,38 @@ mi_decl_nodiscard mi_decl_export void* mi_theap_rezalloc(mi_the // Fast constant size allocations. // ------------------------------------------------------ +// Machine word size allocation. `wsize` is the allocation size in machine words (`sizeof(size_t)`) +mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_restrict void* mi_wmalloc_small(size_t wsize) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_restrict void* mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept; + +// get the machine word size from a byte size. +static inline size_t mi_wsize_from_size(size_t size) { + return ((size + sizeof(size_t) - 1) / sizeof(size_t)); +} + static inline mi_decl_restrict void* mi_malloc_csize(size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { return mi_malloc_small(size); } else { return mi_malloc(size); } + if (size <= MI_SMALL_SIZE_MAX) { return mi_wmalloc_small(mi_wsize_from_size(size)); } else { return mi_malloc(size); } } static inline mi_decl_restrict void* mi_zalloc_csize(size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { return mi_zalloc_small(size); } else { return mi_zalloc(size); } + if (size <= MI_SMALL_SIZE_MAX) { return mi_wzalloc_small(mi_wsize_from_size(size)); } else { return mi_zalloc(size); } } -static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_malloc_small(theap,size); } else { return mi_theap_malloc(theap,size); } +static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { + assert(theap!=NULL); + if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wmalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_malloc(theap,size); } } static inline mi_decl_restrict void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_zalloc_small(theap,size); } else { return mi_theap_zalloc(theap,size); } + assert(theap!=NULL); + if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wzalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_zalloc(theap,size); } } static inline void mi_free_csize(void* p, size_t size) mi_attr_noexcept { if (size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); } } +static inline void mi_free_csize_nonnull(void* p, size_t size) mi_attr_noexcept { + assert(p!=NULL); + if (size <= MI_SMALL_SIZE_MAX) { mi_free_small_nonnull(p); } else { mi_free(p); } +} // ------------------------------------------------------ // Experimental @@ -566,20 +581,21 @@ typedef enum mi_option_e { mi_option_guarded_sample_rate, // 1 out of N allocations in the min/max range will be guarded (=1000) mi_option_guarded_sample_seed, // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0) mi_option_generic_collect, // collect theaps every N (=10000) generic allocation calls - mi_option_page_reclaim_on_free, // reclaim abandoned pages on a free (=0). -1 disallowr always, 0 allows if the page originated from the current theap, 1 allow always + mi_option_page_reclaim_on_free, // reclaim abandoned pages on a free (=0). -1 disallow always, 0 allows if the page originated from the current theap, 1 allow always mi_option_page_full_retain, // retain N full (small) pages per size class (=2) mi_option_page_max_candidates, // max candidate pages to consider for allocation (=4) mi_option_max_vabits, // max user space virtual address bits to consider (=48) mi_option_pagemap_commit, // commit the full pagemap (to always catch invalid pointer uses) (=0) mi_option_page_commit_on_demand, // commit page memory on-demand mi_option_page_max_reclaim, // don't reclaim pages of the same originating theap if we already own N pages (in that size class) (=-1 (unlimited)) - mi_option_page_cross_thread_max_reclaim, // don't reclaim pages across threads if we already own N pages (in that size class) (=16) + mi_option_page_cross_thread_max_reclaim, // don't reclaim pages across threads if we already own N pages (in that size class) (=32) mi_option_allow_thp, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to opt the memory mimalloc maps out of THP. mi_option_minimal_purge_size, // set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled. mi_option_arena_max_object_size, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). - mi_option_arena_is_numa_local, // experimental + mi_option_arena_is_numa_local, // experimental: associate local numa node with an initial arena allocation + mi_option_collect_merges_stats, // on each theap collection, the theap stats are merged automatically with the parent heap + // options of this fork (keep these after the upstream options so those keep their upstream numbering) mi_option_snapshot_on_exit, // write a heap snapshot on process exit (=0). 1=on, 2=on with per-block freemaps. Path from MIMALLOC_SNAPSHOT_PATH or "mimalloc-snapshot..bin". - mi_option_prof_sample_rate, // bytes per heap-profile sample (=0, off). Typical: 524288. Dumps profile.proto on exit to MIMALLOC_PROF_PATH. mi_option_scavenger, // run a background scavenger thread that purges freed arena memory when due (=1) mi_option_purge_holes, // discard the memory of free blocks inside a still-used page (=1) mi_option_purge_holes_eager_zero, // zero a hole before discarding it, so that an over-discard destroys data even on an OS that reclaims lazily (=0; for testing -- always on when MI_DEBUG>1) @@ -614,7 +630,7 @@ mi_decl_export void mi_option_set_default(mi_option_t option, long value); // note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our theap before free-ing. // ------------------------------------------------------------------------------------------------------- -mi_decl_export void mi_cfree(void* p) mi_attr_noexcept; +mi_decl_export bool mi_cfree(void* p) mi_attr_noexcept; mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept; diff --git a/include/mimalloc/atomic.h b/include/mimalloc/atomic.h index d9868d88c..33d2d6230 100644 --- a/include/mimalloc/atomic.h +++ b/include/mimalloc/atomic.h @@ -161,6 +161,11 @@ static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) { // Deprecated: MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics. // It is recommended to always compile as C++ when using MSVC. +// In this fork it is required: the wrapper only models word sized atomics, and the park and scavenger +// state (`types.h`: `park_state`, `scavenger_wake`, ...) are 32-bit atomics. Define MI_ALLOW_MSVC_C_ATOMICS to try anyway. +#if !defined(MI_ALLOW_MSVC_C_ATOMICS) +#error "compile mimalloc as C++ with MSVC, clang-cl, or clang targeting the MSVC ABI (cmake: -DMI_USE_CXX=ON)" +#endif #include #ifdef _WIN64 @@ -227,9 +232,11 @@ static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_ if (mo == mi_memory_order_relaxed) { return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p); } + #if !defined(__clang__) // work around __ldar missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689 else if (mo <= mi_memory_order_acquire) { return MI_MSC_XX(__ldar)((volatile const uintptr_t*)p); } + #endif else { const uintptr_t u = (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p); __dmb(15); // _ARM(64)_BARRIER_SY @@ -249,9 +256,11 @@ static inline void mi_atomic_store_explicit(_Atomic(uintptr_t)*p, uintptr_t x, m if (mo == mi_memory_order_relaxed) { MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x); } + #if !defined(__clang__) // work around __stlr missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689 else if (mo <= mi_memory_order_release) { MI_MSC_XX(__stlr)((volatile uintptr_t*)p,x); } + #endif else { mi_atomic_exchange_explicit(p, x, mo); } @@ -269,7 +278,7 @@ static inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_o if (mo == mi_memory_order_relaxed) { return __iso_volatile_load64((volatile const int64_t*)p); } - #if defined(_M_ARM64) + #if defined(_M_ARM64) && !defined(__clang__) // work around __ldar64 missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689 else if (mo <= mi_memory_order_acquire) { return __ldar64((volatile const uintptr_t*)p); } @@ -294,7 +303,7 @@ static inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi if (mo == mi_memory_order_relaxed) { __iso_volatile_store64((volatile int64_t*)p,x); } - #if defined(_M_ARM64) + #if defined(_M_ARM64) && !defined(__clang__) // work around __stlr64 missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689 else if (mo == mi_memory_order_release) { __stlr64((volatile uint64_t*)p, (uint64_t)x); } diff --git a/include/mimalloc/bits.h b/include/mimalloc/bits.h index 8c3e4710f..5aa0d8375 100644 --- a/include/mimalloc/bits.h +++ b/include/mimalloc/bits.h @@ -18,6 +18,14 @@ terms of the MIT license. A copy of the license can be found in the file #include // bool #include // LONG_MAX +#if defined(__cplusplus) +#define mi_decl_externc extern "C" +#define mi_init_struct_zero { } +#else +#define mi_decl_externc +#define mi_init_struct_zero { 0 } +#endif + // ------------------------------------------------------ // Size of a pointer. // We assume that `sizeof(void*)==sizeof(intptr_t)` @@ -127,10 +135,10 @@ typedef int32_t mi_ssize_t; #define MI_MAX_VABITS (32) #endif -// the MI_MIN_VABITS determine how many bits of the address are always mapped in the page_map +// the MI_MIN_VABITS determine how many bits of the address space are always committed in the page_map #if MI_MAX_VABITS <= 32 #define MI_MIN_VABITS (32) -#elif MI_MAX_VABITS <= 43 +#elif MI_MAX_VABITS <= 42 #define MI_MIN_VABITS MI_MAX_VABITS #else #define MI_MIN_VABITS (43) /* 8 TiB */ @@ -138,7 +146,7 @@ typedef int32_t mi_ssize_t; // use a flat page-map or a 2-level one #ifndef MI_PAGE_MAP_FLAT -#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && MI_SECURE==0 // && !MI_PAGE_META_IS_SEPARATED +#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && !MI_SECURE && !MI_FREE_IS_CHECKED && MI_FREE_USE_PAGEMAP #define MI_PAGE_MAP_FLAT 1 #else #define MI_PAGE_MAP_FLAT 0 diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 29433a8e7..5e961dcb0 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -40,6 +40,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_weak #define mi_decl_hidden #define mi_decl_cold +#define mi_assume_aligned(p,sz) (p) #elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc #if !MI_TRACK_ASAN #define mi_decl_forceinline __attribute__((always_inline)) inline @@ -55,10 +56,12 @@ terms of the MIT license. A copy of the license can be found in the file #else #define mi_decl_hidden __attribute__((visibility("hidden"))) #endif -#if (__GNUC__ >= 4) || defined(__clang__) +#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) #define mi_decl_cold __attribute__((cold)) +#define mi_assume_aligned(p,sz) __builtin_assume_aligned(p,sz) #else #define mi_decl_cold +#define mi_assume_aligned(p,sz) (p) #endif #elif __cplusplus >= 201103L // c++11 #define mi_decl_forceinline inline @@ -68,6 +71,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_weak #define mi_decl_hidden #define mi_decl_cold +#define mi_assume_aligned(p,sz) (p) #else #define mi_decl_forceinline inline #define mi_decl_noinline @@ -76,6 +80,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_weak #define mi_decl_hidden #define mi_decl_cold +#define mi_assume_aligned(p,sz) (p) #endif #if defined(__GNUC__) || defined(__clang__) @@ -89,7 +94,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_likely(x) (x) #endif -#if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__) // includes clang and icc +#if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__) #define mi_decl_maybe_unused __attribute__((unused)) #elif __cplusplus >= 201703L // c++17 #define mi_decl_maybe_unused [[maybe_unused]] @@ -101,15 +106,6 @@ terms of the MIT license. A copy of the license can be found in the file #define __has_builtin(x) 0 #endif -#if defined(__cplusplus) -#define mi_decl_externc extern "C" -#define mi_init_struct_zero { } -#else -#define mi_decl_externc -#define mi_init_struct_zero { 0 } -#endif - - #if defined(__EMSCRIPTEN__) && !defined(__wasi__) #define __wasi__ #endif @@ -153,10 +149,10 @@ void _mi_random_init(mi_random_ctx_t* ctx); void _mi_random_init_weak(mi_random_ctx_t* ctx); void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx); void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx); -uintptr_t _mi_random_next(mi_random_ctx_t* ctx); -uintptr_t _mi_theap_random_next(mi_theap_t* theap); -uintptr_t _mi_os_random_weak(uintptr_t extra_seed); -static inline uintptr_t _mi_random_shuffle(uintptr_t x); +size_t _mi_random_next(mi_random_ctx_t* ctx); +size_t _mi_theap_random_next(mi_theap_t* theap); +size_t _mi_os_random_weak(size_t extra_seed); +static inline size_t _mi_random_shuffle(size_t x); // prim-tls.c void _mi_tls_slots_init(void); @@ -215,6 +211,7 @@ size_t _mi_os_guard_page_size(void); size_t _mi_os_good_alloc_size(size_t size); bool _mi_os_has_overcommit(void); bool _mi_os_has_virtual_reserve(void); +bool _mi_os_canuse_thp(void); size_t _mi_os_virtual_address_bits(void); size_t _mi_os_minimal_purge_size(void); @@ -222,7 +219,7 @@ bool _mi_os_reset(mi_subproc_t* subproc, void* addr, size_t size); bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size); void _mi_os_reuse(mi_subproc_t* subproc, void* p, size_t size); mi_decl_nodiscard bool _mi_os_commit(mi_subproc_t* subproc, void* p, size_t size, bool* is_zero); -mi_decl_nodiscard bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_size); +mi_decl_nodiscard bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_already_committed); mi_decl_nodiscard bool _mi_os_protect(void* addr, size_t size); bool _mi_os_unprotect(void* addr, size_t size); bool _mi_os_purge(mi_subproc_t* subproc, void* p, size_t size); @@ -269,7 +266,8 @@ void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_me void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld); void _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld); bool _mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq); // false: dropped, another thread is purging -bool _mi_arenas_purge_guard_reset(void); // true: a purge pass was cut off (by a fork, or by the end of the process) +void _mi_arenas_purge_guard_acquire(void); // fork: wait for the purge pass in progress, and keep the next one out +void _mi_arenas_purge_guard_release(void); // (also on behalf of a holder that is gone: process exit on Windows) void _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc); mi_page_t* _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t page_alignment); @@ -285,12 +283,6 @@ uint8_t* mi_arena_slice_start(mi_arena_t* arena, size_t slice_index); // heap-snapshot.c void _mi_heap_snapshot_on_exit(void); -// prof.c -void _mi_prof_sample(mi_theap_t* theap, mi_page_t* page, void* p, size_t req_size); -void _mi_prof_free(const void* p); -void _mi_prof_init(void); -void _mi_prof_theap_init(mi_theap_t* theap); - // scavenger.c void _mi_scavenger_start(void); void _mi_scavenger_forked_child(void); @@ -299,9 +291,6 @@ void _mi_scavenger_stop(void); void _mi_scavenger_wake(mi_subproc_t* subproc); bool _mi_scavenger_is_running(void); void _mi_arenas_purge_now(mi_subproc_t* subproc); -void _mi_prof_theap_lazy_enable(mi_theap_t* theap); -size_t _mi_prof_rate(void); -void _mi_prof_on_exit(void); // "page-map.c" bool _mi_page_map_init(void); @@ -312,7 +301,8 @@ mi_page_t* _mi_safe_ptr_page(const void* p); void _mi_page_map_unsafe_destroy(void); // "page.c" -void* _mi_malloc_generic(mi_theap_t* heap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks void _mi_page_unfull(mi_page_t* page); @@ -321,9 +311,11 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abando void _mi_deferred_free(mi_theap_t* theap, bool force); void _mi_page_free_collect(mi_page_t* page, bool force); void _mi_page_free_collect_no_unpurge(mi_page_t* page, bool force); // for read-only heap inspection: never un-purges a hole -void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head); +mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head); mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page); bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq); +void _mi_page_update_stats(mi_page_t* page); +void _mi_page_update_stats_for(mi_page_t* page, mi_theap_t* theapx /* can be NULL */); size_t _mi_page_stats_bin(const mi_page_t* page); // for stats size_t _mi_bin_size(size_t bin); // for stats @@ -389,15 +381,13 @@ mi_msecs_t _mi_clock_start(void); // "alloc.c" void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` -void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; -void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` +void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept; mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p); void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); + // "free.c" -void _mi_free_in_page(void* p, mi_page_t* page) mi_attr_noexcept; -void _mi_free_subproc_safe_in_page(void* p, mi_page_t* page) mi_attr_noexcept; void _mi_free_subproc_safe(void* p) mi_attr_noexcept; void _mi_page_unguard_all(mi_page_t* page); size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept; @@ -406,6 +396,20 @@ size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr bool _mi_page_is_valid(mi_page_t* page); #endif +// "sample-guarded.c" +mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; +void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p); +void _mi_page_unguard_all(mi_page_t* page); +void _mi_theap_guarded_init(mi_theap_t* theap); + +// "sample-profile.c" +mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept; +size_t _mi_theap_update_sample_rate(mi_theap_t* theap); + +mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept; +void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p); +size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate); + // ------------------------------------------------------ // Assertions @@ -437,20 +441,32 @@ mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const ----------------------------------------------------------- */ // add to stat keeping track of the peak -void __mi_stat_increase(mi_stat_count_t* stat, size_t amount); -void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount); -void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount); -void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount); +void __mi_stat_increase(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_decrease(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_increase_mt(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_decrease_mt(mi_stat_count_t* stat, uint64_t amount); // adjust stat in special cases to compensate for double counting (and does not adjust peak values and can decrease the total) -void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount); -void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount); -void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount); -void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount); +void __mi_stat_adjust_increase(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_adjust_decrease(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, uint64_t amount); // counters can just be increased -void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); -void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount); +static inline void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); + mi_atomic_addi64_relaxed(&stat->total, (int64_t)amount); +} + +static inline void __mi_stat_counter_increase(mi_stat_counter_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); + stat->total += (int64_t)amount; +} + +static inline void __mi_stat_counter_decrease(mi_stat_counter_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); + stat->total -= (int64_t)amount; +} #define mi_heap_stat_counter_increase(heap,stat,amount) __mi_stat_counter_increase_mt( &(heap)->stats.stat, amount) #define mi_heap_stat_increase(heap,stat,amount) __mi_stat_increase_mt( &(heap)->stats.stat, amount) @@ -465,15 +481,17 @@ void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount); #define mi_subproc_stat_adjust_decrease(subproc,stat,amount) __mi_stat_adjust_decrease_mt( &(subproc)->stats.stat, amount) #define mi_theap_stat_counter_increase(theap,stat,amount) __mi_stat_counter_increase( &(theap)->stats.stat, amount) +#define mi_theap_stat_counter_decrease(theap,stat,amount) __mi_stat_counter_decrease( &(theap)->stats.stat, amount) #define mi_theap_stat_increase(theap,stat,amount) __mi_stat_increase( &(theap)->stats.stat, amount) #define mi_theap_stat_decrease(theap,stat,amount) __mi_stat_decrease( &(theap)->stats.stat, amount) #define mi_theap_stat_adjust_increase(theap,stat,amnt) __mi_stat_adjust_increase( &(theap)->stats.stat, amnt) #define mi_theap_stat_adjust_decrease(theap,stat,amnt) __mi_stat_adjust_decrease( &(theap)->stats.stat, amnt) #define mi_theapx_stat_counter_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_counter_increase(theap,stat,amount); } else { mi_heap_stat_counter_increase(heap,stat,amount); } +#define mi_theapx_stat_adjust_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_adjust_decrease(theap,stat,amount); } else { mi_heap_stat_adjust_decrease(heap,stat,amount); } #define mi_theapx_stat_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_increase(theap,stat,amount); } else { mi_heap_stat_increase(heap,stat,amount); } #define mi_theapx_stat_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_decrease(theap,stat,amount); } else { mi_heap_stat_decrease(heap,stat,amount); } -#define mi_theapx_stat_adjust_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_adjust_decrease(theap,stat,amount); } else { mi_heap_stat_adjust_decrease(heap,stat,amount); } + /* ----------------------------------------------------------- pthread thread locals @@ -562,6 +580,12 @@ typedef struct mi_option_desc_s { // initialize a local variable to zero; use memset as compilers optimize constant sized memset's #define _mi_memzero_var(x) memset(&x,0,sizeof(x)) +// minimum +static inline size_t mi_min(size_t x, size_t y) { return (x <= y ? x : y); } + +// maximum +static inline size_t mi_max(size_t x, size_t y) { return (x >= y ? x : y); } + // Is `x` a power of two? (0 is considered a power of two) static inline bool _mi_is_power_of_two(uintptr_t x) { return ((x & (x - 1)) == 0); @@ -633,13 +657,6 @@ static inline bool mi_mem_is_zero(const void* p, size_t size) { return true; } -// Align a byte size to a size in _machine words_, -// i.e. byte size == `wsize*sizeof(void*)`. -static inline size_t _mi_wsize_from_size(size_t size) { - mi_assert_internal(size <= SIZE_MAX - sizeof(uintptr_t)); - return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t); -} - // Overflow detecting multiply #if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5)) #include // UINT_MAX, ULONG_MAX @@ -714,9 +731,9 @@ static inline mi_subproc_t* _mi_theap_subproc(const mi_theap_t* theap) { return subproc; } -static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t size) { - mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE)); - const size_t idx = _mi_wsize_from_size(size); +static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t xsize, bool is_wsize) { + mi_assert_internal(is_wsize ? xsize <= (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE) : xsize <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE)); + const size_t idx = (is_wsize ? xsize : mi_wsize_from_size(xsize)); mi_assert_internal(idx < MI_PAGES_DIRECT); return theap->pages_free_direct[idx]; } @@ -736,7 +753,7 @@ static inline bool _mi_theap_can_touch(const mi_theap_t* theap) { static inline bool mi_theap_matches_thread(mi_theap_t* theap) { const mi_threadid_t tid = _mi_thread_id(); - return (theap==NULL || theap->tld->thread_id == tid || mi_theap_is_detached(theap)); + return (theap==NULL || theap->tld==NULL || theap->tld->thread_id == tid || mi_theap_is_detached(theap)); } /* ----------------------------------------------------------- @@ -800,7 +817,7 @@ typedef struct mi_page_map_s { size_t reserved_size; // full reserved size (mi_page_map_t + submaps) mi_memid_t memid; // provenance mi_lock_t lock; // used when allocating new submaps - mi_decl_cache_align _Atomic(mi_submap_t) submaps[1]; + _Atomic(mi_submap_t) submaps[1]; } mi_page_map_t; extern mi_decl_hidden _Atomic(mi_page_map_t*) __mi_page_map; @@ -816,7 +833,7 @@ static inline mi_page_map_t* _mi_page_map(void) { } static inline mi_submap_t _mi_page_map_at(const mi_page_map_t* pmap, size_t idx) { - return mi_atomic_load_ptr_relaxed(mi_page_t*, &pmap->submaps[idx]); + return mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]); } static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { @@ -828,10 +845,16 @@ static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { static inline mi_page_t* _mi_checked_ptr_page(const void* p) { const mi_page_map_t* pmap = _mi_page_map(); - const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); size_t sub_idx; const size_t idx = _mi_page_map_index(p, &sub_idx); - if mi_unlikely(idx > committed_count) return NULL; + const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); + if mi_unlikely(idx >= committed_count) return NULL; + // #if MI_MIN_VABITS < MI_INTPTR_BITS // is still invalid if free is called before the pagemap is initialized + // if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { + // const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); + // if mi_unlikely(idx >= committed_count) return NULL; + // } + // #endif mi_submap_t const sub = _mi_page_map_at(pmap,idx); if mi_unlikely(sub == NULL) return NULL; return sub[sub_idx]; @@ -839,12 +862,47 @@ static inline mi_page_t* _mi_checked_ptr_page(const void* p) { #endif +#if MI_PAGE_META_IS_ALIGNED +// if the page meta data is aligned in front of pages we can find it efficiently +// without needing to go through the page map (for valid pointers). +static inline mi_page_t* _mi_aligned_ptr_page0(const void* p) { + mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGNMENT); + // const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE; + const uintptr_t page_idx = ((uintptr_t)p / MI_ARENA_SLICE_SIZE) % (MI_PAGE_META_ALIGNMENT / MI_ARENA_SLICE_SIZE); + mi_assert_internal(page_idx <= MI_PAGE_META_ALIGNED_COUNT); + #if MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_RISCV // better code on x64/x86/riscv64 + mi_page_t* const page = (mi_page_t*)((uintptr_t)page_metas | (page_idx * sizeof(mi_page_t))); + #else + mi_page_t* const page = &page_metas[page_idx]; + #endif + return page; + +} + +static inline mi_page_t* _mi_aligned_ptr_page(const void* p) { + mi_page_t* const page = _mi_aligned_ptr_page0(p); + if mi_unlikely(page==NULL) return NULL; + #if MI_DEBUG + mi_page_t* const cpage = _mi_checked_ptr_page(p); + if mi_unlikely(cpage==NULL) { + _mi_error_message(EINVAL, "_mi_aligned_ptr_page: invalid pointer: %p\n", p); + return NULL; + } + #endif + return mi_atomic_load_ptr_acquire(mi_page_t, &page->self); +} +#endif + static inline mi_page_t* _mi_ptr_page(const void* p) { mi_assert_internal(p==NULL || mi_is_in_heap_region(p)); - #if MI_DEBUG || MI_SECURE || MI_FREE_IS_CHECKED - return _mi_checked_ptr_page(p); - #else - return _mi_unchecked_ptr_page(p); + #if MI_SECURE || MI_FREE_IS_CHECKED + return _mi_checked_ptr_page(p); + #elif MI_PAGE_META_IS_ALIGNED + return _mi_aligned_ptr_page(p); + #elif MI_DEBUG + return _mi_checked_ptr_page(p); + #else + return _mi_unchecked_ptr_page(p); #endif } @@ -857,8 +915,8 @@ static inline size_t mi_page_block_size(const mi_page_t* page) { // Page start static inline uint8_t* mi_page_start(const mi_page_t* page) { - // multiplication must be done in `size_t`; in a 32-bit multiplication the offset wraps for pages whose blocks start 4 GiB or more after the page meta info - return (uint8_t*)page + (((size_t)page->page_ma_offset) * MI_MAX_ALIGN_SIZE); + // multiplication must be done in `size_t` + return (uint8_t*)page + page->page_offset; } static inline size_t mi_page_size(const mi_page_t* page) { @@ -895,8 +953,16 @@ static inline size_t mi_page_usable_block_size(const mi_page_t* page) { } static inline bool mi_page_meta_is_separated(const mi_page_t* page) { - #if MI_PAGE_META_IS_SEPARATED - // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_ALIGNED_FREE_SMALL + #if MI_PAGE_META_IS_ALIGNED + #if MI_PAGE_META_SMALL_IS_ALIGNED + return (page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); + #else + MI_UNUSED_RELEASE(page); + mi_assert_internal(page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); + return true; + #endif + #elif MI_PAGE_META_IS_SEPARATED + // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_SMALL_IS_ALIGNED return (page->memid.memkind == MI_MEM_ARENA && page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); #else MI_UNUSED(page); @@ -931,11 +997,41 @@ static inline size_t mi_page_committed(const mi_page_t* page) { return (slice_committed == 0 ? mi_page_size(page) : slice_committed - mi_page_slice_offset_of(page,0)); } +static inline size_t mi_page_used(const mi_page_t* page) { + mi_assert_internal(page != NULL); + return mi_xused_used_count(page->xused); +} + +static inline void mi_page_used_reset(mi_page_t* page) { + page->xused = mi_xused_used_reset(page->xused); +} + +static inline size_t mi_page_alloc_count(const mi_page_t* page) { + return mi_xused_alloc_count(page->xused); +} + +static inline size_t mi_page_last_used(const mi_page_t* page) { + #if MI_SIZE_SIZE >= 8 + return (page->xused.used_alloc >> 32) & 0xFFFF; + #else + return page->xlast_used; + #endif +} + +static inline size_t mi_page_last_alloc(const mi_page_t* page) { + #if MI_SIZE_SIZE >= 8 + return (page->xused.used_alloc >> 48) & 0xFFFF; + #else + return page->xlast_alloc; + #endif +} + + // are all blocks in a page freed? // note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`. static inline bool mi_page_all_free(const mi_page_t* page) { mi_assert_internal(page != NULL); - return (page->used == 0); + return (mi_page_used(page)==0); } // ------------------------------------------------------ @@ -1109,7 +1205,7 @@ static inline bool mi_page_is_expandable(const mi_page_t* page) { static inline bool mi_page_is_full(const mi_page_t* page) { - const bool full = (page->reserved == page->used); + const bool full = (page->reserved == mi_page_used(page)); mi_assert_internal(!full || page->free == NULL); return full; } @@ -1118,14 +1214,14 @@ static inline bool mi_page_is_full(const mi_page_t* page) { static inline bool mi_page_is_mostly_used(const mi_page_t* page) { if (page==NULL) return true; uint16_t frac = page->reserved / 8U; - return (page->reserved - page->used <= frac); + return (page->reserved - mi_page_used(page) <= frac); } // is more than (n-1)/n'th of a page in use? static inline bool mi_page_is_used_at_frac(const mi_page_t* page, uint16_t n) { if (page==NULL) return true; uint16_t frac = page->reserved / n; - return (page->reserved - page->used <= frac); + return (page->reserved - mi_page_used(page) <= frac); } @@ -1304,7 +1400,7 @@ static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) { // are there any available blocks? static inline bool mi_page_has_any_available(const mi_page_t* page) { mi_assert_internal(page != NULL && page->reserved > 0); - return (page->used < page->reserved || (mi_page_thread_free(page) != NULL)); + return (mi_page_used(page) < page->reserved || (mi_page_thread_free(page) != NULL)); } // Owned? @@ -1320,14 +1416,26 @@ static inline bool mi_page_claim_ownership(mi_page_t* page) { /* ------------------------------------------------------------------- - Guarded objects + Guarded and profiled objects ------------------------------------------------------------------- */ -#if MI_GUARDED + +#define MI_SAMPLE_RATE_MAX (SIZE_MAX/4) +#define MI_SAMPLE_COUNTDOWN_MAX (MI_MAX_ALLOC_SIZE) + +static inline bool mi_theap_should_sample(mi_theap_t* theap, size_t req_size) { + // note: this should return `true` on an empty theap so we initialize it's countdown to `-1`. + mi_assert_internal(req_size <= SIZE_MAX/2); + // const size_t sample_countdown = theap->sample_countdown - req_size; + // return ((mi_ssize_t)sample_countdown < 0); + return ((mi_ssize_t)theap->sample_countdown < (mi_ssize_t)req_size); +} + // we always align guarded pointers in a block at an offset // the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones #define MI_BLOCK_TAG_ALIGNED ((mi_encoded_t)(0)) +#define MI_BLOCK_TAG_PROFILED ((mi_encoded_t)(1)) #define MI_BLOCK_TAG_GUARDED (~MI_BLOCK_TAG_ALIGNED) -#endif + static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* p) { #if MI_GUARDED @@ -1339,36 +1447,26 @@ static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* #endif } -#if MI_GUARDED -static inline bool mi_theap_malloc_use_guarded(mi_theap_t* theap, size_t size) { - // this code is written to result in fast assembly as it is on the hot path for allocation - const size_t count = theap->guarded_sample_count - 1; // if the rate was 0, this will underflow and count for a long time.. - if mi_likely(count != 0) { - // no sample - theap->guarded_sample_count = count; - return false; - } - else { - // count == 0 - const size_t rate = theap->guarded_sample_rate; - if (rate == 0) { - return false; // don't write to an empty theap - } - else if (size >= theap->guarded_size_min && size <= theap->guarded_size_max) { - // use guarded allocation - theap->guarded_sample_count = rate; // reset - return true; - } - else { - // failed size criteria, rewind count - theap->guarded_sample_count = 1; - return false; - } - } +static inline bool mi_block_ptr_is_sampled(const mi_block_t* block, const void* p) { +#if MI_GUARDED || MI_PROFILE + const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block; + return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next != MI_BLOCK_TAG_ALIGNED); +#else + MI_UNUSED(block); MI_UNUSED(p); + return false; +#endif +} + +static inline bool mi_profiler_is_enabled(const mi_profiler_t* prof) { + _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved; + return (mi_atomic_load_acquire(penabled) != 0); +} + +static inline bool mi_profiler_set_enabled(mi_profiler_t* prof, bool enable) { + _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved; + return (mi_atomic_exchange_release(penabled, (enable ? 1 : 0)) != 0); } -mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; -#endif /* ------------------------------------------------------------------- Encoding/Decoding the free list next pointers @@ -1402,25 +1500,46 @@ static inline bool mi_is_in_same_page(const void* p, const void* q) { } static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) { - void* p = (void*)(mi_rotr(x - keys[0], keys[0]) ^ keys[1]); + const uintptr_t k1 = keys[0]; + #if MI_PAGE_KEY_COUNT==2 + const uintptr_t k2 = keys[1]; + #else + const uintptr_t k2 = mi_rotr(k1,13); + #endif + void* p = (void*)(mi_rotr(x - k1, k1) ^ k2); return (p==null ? NULL : p); } static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const uintptr_t* keys) { - uintptr_t x = (uintptr_t)(p==NULL ? null : p); - return mi_rotl(x ^ keys[1], keys[0]) + keys[0]; + const uintptr_t k1 = keys[0]; + #if MI_PAGE_KEY_COUNT==2 + const uintptr_t k2 = keys[1]; + #else + const uintptr_t k2 = mi_rotr(k1,13); + #endif + const uintptr_t x = (uintptr_t)(p==NULL ? null : p); + return mi_rotl(x ^ k2, k1) + k1; } static inline uint32_t mi_ptr_encode_canary(const void* null, const void* p, const uintptr_t* keys) { const uint32_t x = (uint32_t)(mi_ptr_encode(null,p,keys)); // make the lowest byte 0 to prevent spurious read overflows which could be a security issue (issue #951) + // also clear bit 9 which we set only when a block is freed. #if MI_BIG_ENDIAN - return (x & 0x00FFFFFF); + return (x & 0x00FFFEFF); #else - return (x & 0xFFFFFF00); + return (x & 0xFFFFFE00); #endif } +static inline uint32_t mi_ptr_encode_canary_freed(void) { + return (0x00DEAD00); // set bit 9 so it is different from any valid canary +} + +static inline bool mi_ptr_decode_canary_is_freed(uint32_t canary) { + return (canary == mi_ptr_encode_canary_freed()); +} + static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) { mi_track_mem_defined(block,sizeof(mi_block_t)); mi_block_t* next; @@ -1445,14 +1564,15 @@ static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_track_mem_noaccess(block,sizeof(mi_block_t)); } +mi_block_t* _mi_block_next_is_corrupted(const mi_page_t* page, const mi_block_t* block, const mi_block_t* next); // in options.c + static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) { #if MI_ENCODE_FREELIST mi_block_t* next = mi_block_nextx(page,block,page->keys); // check for free list corruption: is `next` at least in the same page? // todo: check if `next` is `page->block_size` aligned? - if mi_unlikely(next!=NULL && !mi_is_in_same_page(block, next)) { - _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next); - next = NULL; + if mi_unlikely(next!=NULL && !mi_page_contains_address(page,next)) { + return _mi_block_next_is_corrupted(page,block,next); // returns NULL } return next; #else @@ -1551,16 +1671,16 @@ static inline size_t _mi_memid_size(mi_memid_t memid) { // Fast "random" shuffle // ------------------------------------------------------------------- -static inline uintptr_t _mi_random_shuffle(uintptr_t x) { +static inline size_t _mi_random_shuffle(size_t x) { if (x==0) { x = 17; } // ensure we don't get stuck in generating zeros -#if (MI_INTPTR_SIZE>=8) +#if (MI_SIZE_SIZE>=8) // by Sebastiano Vigna, see: x ^= x >> 30; x *= 0xbf58476d1ce4e5b9UL; x ^= x >> 27; x *= 0x94d049bb133111ebUL; x ^= x >> 31; -#elif (MI_INTPTR_SIZE==4) +#elif (MI_SIZE_SIZE==4) // by Chris Wellons, see: x ^= x >> 16; x *= 0x7feb352dUL; @@ -1574,108 +1694,82 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x) { // --------------------------------------------------------------------------------- // Provide our own `_mi_memcpy/set` for potential performance optimizations. -// -// For now, only on x64/x86 we optimize to `rep movsb/stosb`. -// Generally, we check for "fast short rep movsb/stosb" (FSRM/FSRS) or "fast enhanced rep movsb" (ERMS) support -// (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017). See also issue #201 and pr #253. -// Todo: we see improvements on win32 but less with glibc; we might want to only enable this on windows. // --------------------------------------------------------------------------------- -#if !MI_TRACK_ENABLED && (MI_ARCH_X64 || MI_ARCH_X86) && (defined(_WIN32) || defined(__GNUC__)) - -extern mi_decl_hidden size_t _mi_cpu_movsb_max; // in init.c -extern mi_decl_hidden size_t _mi_cpu_stosb_max; - -static inline void mi_rep_movsb(void* dst, const void* src, size_t n) { - #if defined(__GNUC__) - __asm volatile("rep movsb" : "+D"(dst), "+c"(n), "+S"(src) : : "memory"); - #else - __movsb((unsigned char*)dst, (const unsigned char*)src, n); - #endif -} - -static inline void mi_rep_stosb(void* dst, uint8_t val, size_t n) { - #if defined(__GNUC__) - __asm volatile("rep stosb" : "+D"(dst), "+c"(n) : "a"(val) : "memory"); - #else - __stosb((unsigned char*)dst, val, n); - #endif -} - -static inline void _mi_memcpy(void* dst, const void* src, size_t n) { - if mi_likely(n <= _mi_cpu_movsb_max) { // has fsrm && n <= 127 (todo: and maybe has erms?) - mi_rep_movsb(dst, src, n); - } - else { - memcpy(dst, src, n); - } +static inline void* _mi_memcpy(void* dst, const void* src, size_t n) { + return memcpy(dst, src, n); } -static inline void _mi_memset(void* dst, int val, size_t n) { - if mi_likely(n <= _mi_cpu_stosb_max) { // has fsrs && n <= 127 - mi_rep_stosb(dst, (uint8_t)val, n); - } - else { - memset(dst, val, n); - } -} - -#else - -static inline void _mi_memcpy(void* dst, const void* src, size_t n) { - memcpy(dst, src, n); +static inline void* _mi_memset(void* dst, int val, size_t n) { + return memset(dst, val, n); } -static inline void _mi_memset(void* dst, int val, size_t n) { - memset(dst, val, n); +static inline void* _mi_memset_backward(void* dst, int val, size_t n) { + memset((uint8_t*)dst - n, val, n); + return dst; } -#endif - -// ------------------------------------------------------------------------------- -// The `_mi_memcpy_aligned` can be used if the pointers are machine-word aligned -// This is used for example in `mi_realloc`. -// ------------------------------------------------------------------------------- - -#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) - -// On GCC/CLang we provide a hint that the pointers are word aligned. -static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { - mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); - void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); - const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE); - _mi_memcpy(adst, asrc, n); +static inline void* _mi_memzero(void* dst, size_t n) { + return _mi_memset(dst, 0, n); } -static inline void _mi_memset_aligned(void* dst, int val, size_t n) { - mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); - void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); - _mi_memset(adst, val, n); +static inline void* _mi_memzero_backward(void* dst, size_t n) { + return _mi_memset_backward(dst, 0, n); } -#else - -// Default fallback on `_mi_memcpy` -static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { - mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); - _mi_memcpy(dst, src, n); +static inline void* _mi_memcpy_aligned(void* dst, const void* src, size_t n) { + // on gcc/clang we can provide a hint that the pointers are word aligned. + mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE) && _mi_is_aligned(src,MI_SIZE_SIZE)); + void* adst = mi_assume_aligned(dst, MI_SIZE_SIZE); + const void* asrc = mi_assume_aligned(src, MI_SIZE_SIZE); + return _mi_memcpy(adst, asrc, n); } -static inline void _mi_memset_aligned(void* dst, int val, size_t n) { - mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); - _mi_memset(dst, val, n); +static inline void* _mi_memset_aligned(void* dst, int val, size_t n) { + mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE)); + void* adst = mi_assume_aligned(dst, MI_SIZE_SIZE); + return _mi_memset(adst, val, n); } -#endif - -static inline void _mi_memzero(void* dst, size_t n) { - _mi_memset(dst, 0, n); +static inline void* _mi_memzero_aligned(void* dst, size_t n) { + return _mi_memset_aligned(dst, 0, n); } -static inline void _mi_memzero_aligned(void* dst, size_t n) { - _mi_memset_aligned(dst, 0, n); +// Zero a block: blocks are always aligned with a positive bsize in machine-word bytes. +static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize) { + mi_assert_internal(bsize%MI_SIZE_SIZE == 0); + mi_assert_internal(bsize > 0); + mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE)); + mi_assert_internal(bsize < MI_MAX_ALIGN_SIZE || _mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); + + // fast memzero for small sizes based on overlapping writes (and assuming non-zero size_t-multiple size, and size_t aligned) + // assumes constant memset(p,0,N) gets optimized to fast simd stores by the compiler + // (compile with -DMI_USE_MEMZERO16X=0 to disable this) + #if !defined(MI_USE_MEMZERO16X) || (MI_USE_MEMZERO16X != 0) // 16x MI_SIZE_SIZE (128 bytes on 64-bit) + if mi_unlikely(bsize < 2*MI_SIZE_SIZE) { // bsize < 16 (8) + *((size_t*)dst) = 0; + return dst; + } + mi_assert_internal(_mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); + uint8_t* const start = (uint8_t*)mi_assume_aligned(dst, MI_MAX_ALIGN_SIZE); + uint8_t* const end = start + bsize; // note: if bsize is always a multiple of 16 then end is always aligned as well (but due to padding this does not hold) + if mi_likely(bsize < 8*MI_SIZE_SIZE) { // bsize < 64 (32) + const size_t ofs = (bsize>>1)&(2*MI_SIZE_SIZE); mi_assert_internal(bsize < 4*MI_SIZE_SIZE ? ofs==0 : ofs==2*MI_SIZE_SIZE); // ofs == 16 (8) + _mi_memzero(start, 2*MI_SIZE_SIZE); + _mi_memzero(start+ofs, 2*MI_SIZE_SIZE); + _mi_memzero_backward(end-ofs, 2*MI_SIZE_SIZE); + _mi_memzero_backward(end, 2*MI_SIZE_SIZE); + return dst; + } + if mi_likely(bsize <= 16*MI_SIZE_SIZE) { // bsize < 128 (64) + _mi_memzero(start, 8*MI_SIZE_SIZE); + _mi_memzero_backward(end, 8*MI_SIZE_SIZE); + return dst; + } + #endif + // fallback to regular memset for larger sizes + void* const wdst = mi_assume_aligned(dst,MI_SIZE_SIZE); + return _mi_memzero_aligned(wdst, bsize); } - - #endif // MI_INTERNAL_H diff --git a/include/mimalloc/prim-tls.h b/include/mimalloc/prim-tls.h index e11cfb098..c947be046 100644 --- a/include/mimalloc/prim-tls.h +++ b/include/mimalloc/prim-tls.h @@ -35,7 +35,7 @@ static inline mi_theap_t* _mi_theap_cached(void); // static inline bool _mi_thread_is_initialized(void); // a thread is initialized if it has a default theap static inline mi_theap_t* _mi_heap_theap(mi_heap_t* heap); // get the thread local theap belonging to a heap static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap); // get the theap but don't update _mi_theap_cached -static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page); // get the theap associated with a page (used in `mi_free_collect_mt`) +static inline mi_theap_t* _mi_page_associated_theap_peek(const mi_page_t* page); // get the theap associated with a page (used in `mi_free_collect_mt`) // Default TLS model @@ -68,7 +68,7 @@ static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page); // #if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \ || (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__riscv)) \ || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \ - || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__))) + || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__) || defined(__riscv))) #define MI_USE_BUILTIN_THREAD_POINTER 1 #endif #endif @@ -91,39 +91,39 @@ static inline void** mi_prim_thread_pointer(void) { static inline void** mi_prim_thread_pointer(void) { void** tcb; #if defined(__APPLE__) // M1, issue rgb(62, 76, 62) - __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); + __asm__ ("mrs %0, tpidrro_el0\n\tbic %0, %0, #7" : "=r" (tcb)); #else - __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); + __asm__ ("mrs %0, tpidr_el0" : "=r" (tcb)); #endif return tcb; } #elif defined(__riscv) static inline void** mi_prim_thread_pointer(void) { void** tcb; - __asm__ volatile ("mv %0, tp" : "=r" (tcb)); + __asm__ ("mv %0, tp" : "=r" (tcb)); return tcb; } #elif defined(__arm__) static inline void** mi_prim_thread_pointer(void) { void** tcb; - __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); + __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\n\tbic %0, %0, #3" : "=r" (tcb)); return tcb; } #elif defined(__i386__) static inline void** mi_prim_thread_pointer(void) { void** tcb; - __asm__("movl %%gs:0, %0" : "=r" (tcb) : : ); // x86 32-bit always uses GS + __asm__ ("movl %%gs:0, %0" : "=r" (tcb) : : ); // x86 32-bit always uses GS return tcb; } #elif defined(__x86_64__) static inline void** mi_prim_thread_pointer(void) { void** tcb; #if defined(__APPLE__) - __asm__("movq %%gs:0, %0" : "=r" (tcb) : : ); // x86_64 macOSX uses GS - #elif (MI_INTPTR_SIZE==4) - __asm__("movl %%fs:0, %0" : "=r" (tcb) : : ); // x32 ABI + __asm__ ("movq %%gs:0, %0" : "=r" (tcb) : : ); // x86_64 macOSX uses GS + #elif (MI_SIZE_SIZE==4) + __asm__ ("movl %%fs:0, %0" : "=r" (tcb) : : ); // x32 ABI #else - __asm__("movq %%fs:0, %0" : "=r" (tcb) : : ); // x86_64 Linux, BSD uses FS + __asm__ ("movq %%fs:0, %0" : "=r" (tcb) : : ); // x86_64 Linux, BSD uses FS #endif return tcb; } @@ -313,9 +313,9 @@ static inline mi_theap_t* _mi_theap_cached(void) { // We try to use direct slots (64 available), but can also use the expansion slots (upto 1024 extra available) // See for the offsets. #if MI_SIZE_SIZE==4 -#define MI_TLS_EXPANSION_SLOT (0x0F94 / MI_INTPTR_SIZE) +#define MI_TLS_EXPANSION_SLOT (0x0F94 / MI_SIZE_SIZE) #else -#define MI_TLS_EXPANSION_SLOT (0x1780 / MI_INTPTR_SIZE) +#define MI_TLS_EXPANSION_SLOT (0x1780 / MI_SIZE_SIZE) #endif extern mi_decl_hidden _Atomic(size_t) _mi_theap_default_slot; @@ -414,6 +414,16 @@ static inline mi_theap_t* _mi_heap_theap(mi_heap_t* heap) { return _mi_heap_theap_get_or_init(heap); } +static inline mi_theap_t* _mi_heap_theap_cached(mi_heap_t* heap) { + mi_theap_t* theap = _mi_theap_cached(); + #if MI_THEAP_INITASNULL + if mi_likely(theap!=NULL && _mi_theap_heap_peek(theap)==heap) return theap; + #else + if mi_likely(_mi_theap_heap_peek(theap)==heap) return theap; + #endif + return NULL; +} + // Get the theap belonging to a heap without creating it if it is not yet initialized. static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) { mi_theap_t* theap = _mi_theap_cached(); @@ -432,7 +442,7 @@ static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) { // Find the associated theap or NULL if it does not exist (during shutdown) // Should be fast as it is called in `free.c:mi_free_try_collect`. -static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page) { +static inline mi_theap_t* _mi_page_associated_theap_peek(const mi_page_t* page) { mi_heap_t* const heap = mi_page_heap(page); mi_theap_t* const theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); if (theap==NULL) return NULL; diff --git a/include/mimalloc/track.h b/include/mimalloc/track.h index 753363366..e6a57c8ec 100644 --- a/include/mimalloc/track.h +++ b/include/mimalloc/track.h @@ -133,7 +133,7 @@ defined, undefined, or not accessible at all: #endif -#if MI_PADDING +#if MI_PADDING_CHECK_BYTES && !MI_GUARDED #define mi_track_malloc(p,reqsize,zero) \ do { if ((p)!=NULL) { \ mi_assert_internal(mi_usable_size(p)==(reqsize)); \ diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index a5f41c26b..b0308d4bb 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -24,12 +24,13 @@ terms of the MIT license. A copy of the license can be found in the file #include +#include #include // ptrdiff_t #include // uintptr_t, uint16_t, etc #include // bool #include // SIZE_MAX etc. #include // error codes -#include "bits.h" // size defines (MI_INTPTR_SIZE etc), bit operations +#include "bits.h" // size defines (MI_SIZE_SIZE etc), bit operations #include "atomic.h" // _Atomic primitives // Minimal alignment necessary. On most platforms 16 bytes are needed @@ -38,6 +39,9 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t) #endif +#if MI_MAX_ALIGN_SIZE < MI_INTPTR_SIZE +#error MI_MAX_ALIGN_SIZE must be at least MI_INTPTR_SIZE +#endif // ------------------------------------------------------ // Variants @@ -51,15 +55,15 @@ terms of the MIT license. A copy of the license can be found in the file // #define MI_TRACK_ASAN 1 // #define MI_TRACK_ETW 1 -// Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). -// #define MI_STAT 1 +// Define MI_STATS as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). +// #define MI_STATS 1 // Define MI_SECURE to enable security mitigations -// #define MI_SECURE 1 // guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption -// #define MI_SECURE 2 // randomize relative allocation addresses (within mimalloc pages) -// #define MI_SECURE 3 // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free) -// #define MI_SECURE 4 // checks for double free (may be more expensive) (`-DMI_SECURE=ON`) -// #define MI_SECURE 5 // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`) +// #define MI_SECURE 1 // check invalid pointer free, guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption +// #define MI_SECURE 2 // randomize relative allocation addresses within mimalloc pages +// #define MI_SECURE 3 // check buffer overflow, check double free, encode free lists (and detect corrupted free lists) +// #define MI_SECURE 4 // same as level 3 for now (`-DMI_SECURE=ON`) +// #define MI_SECURE 5 // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`), and byte-precise buffer overflow checks. #if !defined(MI_SECURE) #define MI_SECURE 0 @@ -77,13 +81,18 @@ terms of the MIT license. A copy of the license can be found in the file #endif #endif -// Statistics (0=only essential, 1=normal, 2=more fine-grained (expensive) tracking) -#ifndef MI_STAT +// Statistics (0=only essential, 1=detailed (fast, can be enabled always), 2=slightly more expensive tracking of precise requested bytes) +#ifndef MI_STATS #if (MI_DEBUG>0) -#define MI_STAT 2 +#define MI_STATS 2 #else -#define MI_STAT 0 +#define MI_STATS 1 +#endif #endif + +// Enable profiling support (0=off, 1=fast, can be enabled always, 2=allow fine-grained sample rates(<64 KiB), a tad more expensive) +#ifndef MI_PROFILE +#define MI_PROFILE 1 #endif // Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options) @@ -91,15 +100,30 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_GUARDED 1 #endif +// For profiling or guarded pages we need to sample every once in a while. (0=no sampling, 1=fast, 2=allow fine grained sample rates(<64 KiB), a tad more expensive) +#ifndef MI_SAMPLE +#if MI_PROFILE>1 || MI_GUARDED>1 + #define MI_SAMPLE 2 +#elif MI_PROFILE || MI_GUARDED + #define MI_SAMPLE 1 +#else + #define MI_SAMPLE 0 +#endif +#endif + // Reserve extra padding at the end of each block to be more resilient against theap block overflows. -// The padding can detect buffer overflow on free. -#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW)) +// The padding can detect heap-block overflow on free, and provides byte-precise `mi_usable_size`. +#if !defined(MI_PADDING) +#if (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW)) #define MI_PADDING 1 +#else +#define MI_PADDING 0 +#endif #endif -// Check padding bytes; allows byte-precise buffer overflow detection -#if !defined(MI_PADDING_CHECK) && MI_PADDING && (MI_SECURE>=3 || MI_DEBUG>=1) -#define MI_PADDING_CHECK 1 +// Check for byte-precise buffer overflow? +#if !defined(MI_PADDING_CHECK_BYTES) && MI_PADDING && (MI_SECURE>=5 || MI_DEBUG>=1) +#define MI_PADDING_CHECK_BYTES 1 #endif @@ -109,8 +133,15 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_ENCODE_FREELIST 1 #endif -#if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0)) -#define MI_CHECK_DOUBLE_FREE 1 +// Deprecated (double-free is now checked with padding) +// #if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0)) +// #define MI_CHECK_DOUBLE_FREE 1 +// #endif + +#if MI_SECURE>=4 || MI_PADDING || (MI_FREE_IS_CHECKED && MI_FREE_USE_PAGEMAP) +#define MI_PAGE_KEY_COUNT 2 +#else +#define MI_PAGE_KEY_COUNT 1 #endif // Enable large pages for objects between 64KiB and 512KiB. @@ -123,7 +154,6 @@ terms of the MIT license. A copy of the license can be found in the file // Place page meta info at the start of the page area or keep it separate? // Separate keeps the page info at the arena start (default) which is more secure // and reduces wasted space due to alignment and block sizes. -// (but also reserves more memory up front (about 2MiB per GiB)) #if !defined(MI_PAGE_META_IS_SEPARATED) #if MI_PAGE_MAP_FLAT #define MI_PAGE_META_IS_SEPARATED 0 @@ -132,21 +162,38 @@ terms of the MIT license. A copy of the license can be found in the file #endif #endif +// We can choose to page meta info aligned at the start of every MI_PAGE_META_ALIGNED_CHUNKS +// This can be used to have a faster `mi_free(_small)` as we can avoid a page_map lookup. +// This only works if valid pointers are passed to `mi_free` though. However, checked +// free `mi_cfree` can still use the page map to validate pointers. +#if !MI_FREE_IS_CHECKED && !MI_FREE_USE_PAGEMAP +#if MI_PAGE_META_IS_SEPARATED +#define MI_PAGE_META_IS_ALIGNED 1 +#define MI_PAGE_META_ALIGNED_CHUNKS MI_SIZE_SIZE +#else +#warning "cannot optimize free with alignment since the page meta data is not separated (due to MI_PAGE_MAP_FLAT?)" +#endif +#endif + // We can choose to only put page info of small pages at the start of the page area. // This can be used to have a slightly faster `mi_free_small` function for specialized // cases (like language runtime systems). -#if MI_OPT_FREE_SMALL && !defined(MI_PAGE_META_ALIGNED_FREE_SMALL) -#define MI_PAGE_META_ALIGNED_FREE_SMALL 1 -#elif !defined(MI_PAGE_META_ALIGNED_FREE_SMALL) -#define MI_PAGE_META_ALIGNED_FREE_SMALL 0 +#if !defined(MI_PAGE_META_SMALL_IS_ALIGNED) +#if defined(MI_OPT_FREE_SMALL) && MI_OPT_FREE_SMALL==0 +#define MI_PAGE_META_SMALL_IS_ALIGNED 0 +#elif (MI_OPT_FREE_SMALL || MI_PAGE_META_IS_ALIGNED) && !MI_SECURE && !MI_GUARDED // guarded can put small allocations into big blocks in a medium page (which would make `mi_free_small` fail) +#define MI_PAGE_META_SMALL_IS_ALIGNED 1 +#else +#define MI_PAGE_META_SMALL_IS_ALIGNED 0 +#endif #endif // Configuration checks #if !MI_PAGE_META_IS_SEPARATED && MI_SECURE #error "secure mode should use separated page infos" #endif -#if MI_PAGE_META_ALIGNED_FREE_SMALL && MI_SECURE -#error "secure mode cannot use MI_OPT_FREE_SMALL (MI_PAGE_META_ALIGNED_FREE_SMALL)" +#if MI_PAGE_META_SMALL_IS_ALIGNED && MI_SECURE +#error "secure mode cannot use MI_OPT_FREE_SMALL (MI_PAGE_META_SMALL_IS_ALIGNED)" #endif #if MI_PAGE_META_IS_SEPARATED && MI_PAGE_MAP_FLAT #error "cannot have a flat page map with separated page infos" @@ -155,7 +202,6 @@ terms of the MIT license. A copy of the license can be found in the file #warning "mimalloc assertions enabled in a release build" #endif - // -------------------------------------------------------------- // Sizes of internal data-structures // (comments specify sizes on 64-bit, usually 32-bit is halved) @@ -188,6 +234,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_BCHUNK_BITS (1 << MI_BCHUNK_BITS_SHIFT) // sub-bitmaps in arena's are "bchunks" of 512 bits #define MI_ARENA_SLICE_SIZE (MI_ZU(1) << MI_ARENA_SLICE_SHIFT) // arena's allocate in slices of 64 KiB #define MI_ARENA_SLICE_ALIGN (MI_ARENA_SLICE_SIZE) +#define MI_ARENA_CHUNK_SIZE (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE) #define MI_ARENA_MIN_OBJ_SLICES (1) #define MI_ARENA_MAX_CHUNK_OBJ_SLICES (MI_BCHUNK_BITS) // 32 MiB (or 8 MiB on 32-bit) @@ -222,11 +269,18 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_BIN_COUNT (MI_BIN_FULL+1) // We never allocate more than PTRDIFF_MAX (see also ) -#define MI_MAX_ALLOC_SIZE PTRDIFF_MAX +#define MI_MAX_ALLOC_SIZE PTRDIFF_MAX // Minimal commit for a page on-demand commit -#define MI_PAGE_MIN_COMMIT_SIZE (16*MI_KiB) /* MI_ARENA_SLICE_SIZE */ +#define MI_PAGE_MIN_COMMIT_SIZE (16*MI_KiB) /* MI_ARENA_SLICE_SIZE */ +#if MI_PAGE_META_IS_ALIGNED +#define MI_PAGE_META_ALIGNED_COUNT (MI_PAGE_META_ALIGNED_CHUNKS * MI_BCHUNK_BITS) +#define MI_PAGE_META_ALIGNMENT (MI_PAGE_META_ALIGNED_COUNT * MI_ARENA_SLICE_SIZE) // 256 MiB (32 MiB on 32-bit) +#define MI_ARENA_ALIGNMENT MI_PAGE_META_ALIGNMENT +#else +#define MI_ARENA_ALIGNMENT MI_ARENA_SLICE_ALIGN +#endif // ------------------------------------------------------ // Arena's are large reserved areas of memory allocated from @@ -351,9 +405,8 @@ typedef struct mi_block_s { // `in_full_queue` is true if the page is full and resides in the full queue (so we move it to a regular queue on free-ing) #define MI_PAGE_IN_FULL_QUEUE MI_ZU(0x01) #define MI_PAGE_HAS_INTERIOR_POINTERS MI_ZU(0x02) -#define MI_PAGE_HAS_PROF_SAMPLES MI_ZU(0x04) // page contains at least one profiled allocation; routes free to the generic path -#define MI_PAGE_FLAG_MASK MI_ZU(0x07) -#define MI_PAGE_FLAG_BITS (3) +#define MI_PAGE_FLAG_MASK MI_ZU(0x03) +#define MI_PAGE_FLAG_BITS (2) typedef size_t mi_page_flags_t; // There are two special threadid's: 0 for pages that are abandoned (and not in a theap queue), @@ -371,6 +424,37 @@ typedef size_t mi_page_flags_t; // This way we can push a block on the thread free list and try to claim ownership atomically in `free.c:mi_free_block_mt`. typedef uintptr_t mi_thread_free_t; +// We store the currently used block count together with the total malloc call count as 16-bit numbers. +// This is done for better codegen `mi_malloc/mi_free` (where we can increment both at once as `used_alloc += 0x10001` for example). +// The `last_used` is the `used` count since the statistics are last updated; on 32-bit platforms this +// is a separate field in `mi_page_t` but on 64-bit we use the upper 32-bits to store it. +// We need the `alloc_count` and `last_used` to efficiently calculate allocation and free statistics even +// in a release build; this way we can update the stats in the slow path (`_mi_page_update_stats`). +typedef union mi_used_s { + size_t used_alloc; // used + alloc_count + // the same in parts, for a debugger and for `mi_xused_dec_used` + struct { + uint16_t used_count; + uint16_t alloc_count; + #if MI_SIZE_SIZE >= 8 + uint16_t last_used; + uint16_t last_alloc; + #endif + } debug_le; +} mi_used_t; + +static inline size_t mi_xused_used_count(mi_used_t xused) { return (xused.used_alloc & 0xFFFF); } +static inline size_t mi_xused_alloc_count(mi_used_t xused) { return ((xused.used_alloc>>16) & 0xFFFF); } +static inline mi_used_t mi_xused_used_reset(mi_used_t xused) { xused.used_alloc = xused.used_alloc & ~0xFFFF; return xused; } + +// Decrement the used count in place and return the new count (`mi_free`). Where its 16 bits can be addressed by themselves +// that is one `decw` on memory whose flags feed the branch; on the whole word it is a load, a decrement, a store and a test. +#if !MI_BIG_ENDIAN && ((defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) || defined(_WIN32)) +static inline size_t mi_xused_dec_used(mi_used_t* xused) { return --xused->debug_le.used_count; } +#else +static inline size_t mi_xused_dec_used(mi_used_t* xused) { xused->used_alloc--; return mi_xused_used_count(*xused); } +#endif + // A page contains blocks of one specific size (`block_size`). // Each page has three list of free blocks: // `free` for blocks that can be allocated, @@ -396,35 +480,40 @@ typedef uintptr_t mi_thread_free_t; // that case the `xthreadid` is 0 or 4 (4 is for abandoned pages that // are in the `pages_abandoned` lists of an arena, these are called "mapped" abandoned pages). // - page flags are in the bottom 3 bits of `xthread_id` for the fast path in `mi_free`. -// - The layout is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc` -// - Using `uint16_t` does not seem to slow things down +// - The layout below is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc` -typedef struct mi_page_s { +typedef struct mi_page_s { + #if (MI_PAGE_META_IS_ALIGNED) + _Atomic(struct mi_page_s*) self; // points to the actual page info (for pages that span multiple slices) + #endif _Atomic(mi_threadid_t) xthread_id; // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`) - mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) - uint32_t used; // number of blocks in use (including blocks in `thread_free`) + mi_used_t xused; // number of blocks in use (including blocks in `thread_free`) (and the allocated count for statistics) + #if MI_SIZE_SIZE < 8 + uint16_t xlast_used; // for statistics; on 64-bit platforms it is in bits 32..47 of xused. + uint16_t xlast_alloc; // for sampling; on 64-bit platforms it is in bits 48..63 of xused. + #endif + mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) + + size_t block_size; // const: size available in each block (always `>0`) + size_t page_offset; // const: relative offset from the page to the start of the blocks uint16_t capacity; // number of blocks committed + uint16_t reserved; // number of blocks reserved in memory + uint16_t slice_pcommitted; // committed size in OS page sizes relative to the first arena slice of the page data (or 0 if the page is fully committed already) uint8_t retire_expire; // expiration count for retired blocks bool free_is_zero; // `true` if the blocks in the free list are zero initialized - - mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) + + // next cache line _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads (= `mi_block_t* | (1 if owned)`) - - size_t block_size; // const: size available in each block (always `>0`) - uint32_t page_ma_offset; // const: offset relative to the page (in MI_MAX_ALIGN_SIZE parts) to the start of the blocks - uint16_t slice_pcommitted; // committed size in OS page sizes relative to the first arena slice of the page data (or 0 if the page is fully committed already) - uint16_t reserved; // number of blocks reserved in memory - mi_theap_t* theap; // the theap owning this page (may not be valid or NULL for abandoned pages) mi_heap_t* heap; // the heap owning this page (`mi_heap_delete` moves it to the main heap; only dereference while owning the page, see `mi_page_heap`) struct mi_page_s* next; // next page owned by the theap with the same `block_size` struct mi_page_s* prev; // previous page owned by the theap with the same `block_size` mi_memid_t memid; // const: provenance of the page memory - + #if (MI_ENCODE_FREELIST || MI_PADDING) - uintptr_t keys[2]; // const: two random keys to encode the free lists (see `_mi_block_next`) or padding canary + uintptr_t keys[MI_PAGE_KEY_COUNT]; // const: one or two random keys to encode the free lists (see `_mi_block_next`) or padding canary #endif // The OS pages inside this page whose memory was discarded to the OS. A block is @@ -444,7 +533,7 @@ typedef struct mi_page_s { uint32_t unformed_purged_lo; uint32_t unformed_purged_hi; - // The `(capacity,used)` the last hole sweep left this page in, packed as `(capacity<<16)|used`. + // The `(capacity,used)` the last hole sweep left this page in, packed as `(capacity<<32)|used`. // A sweep that finds it unchanged skips the page without walking its free list at all: nothing // was allocated or freed in it since, so the sweep has nothing new to discard (see // `_mi_page_purge_holes`). `MI_PAGE_SWEPT_NONE` means "unknown". Cold, like `purged` above. @@ -454,12 +543,11 @@ typedef struct mi_page_s { // An impossible `(capacity,used)` (`used > capacity` never holds): "this page has no sweep state". #define MI_PAGE_SWEPT_NONE (~(uint64_t)0) // capacity is 16-bit, so a state with all upper bits set cannot occur - // ------------------------------------------------------ // Object sizes // ------------------------------------------------------ -#define MI_PAGE_ALIGN MI_ARENA_SLICE_ALIGN // pages must be aligned on this for the page map. +#define MI_PAGE_ALIGN MI_ARENA_SLICE_ALIGN // page area's must be aligned on this for the page map. #define MI_PAGE_MIN_START_BLOCK_ALIGN MI_MAX_ALIGN_SIZE // minimal block alignment for the first block in a page (16b) #define MI_PAGE_MAX_START_BLOCK_ALIGN2 (16*MI_KiB) // maximal block alignment for "power of 2"-sized blocks (such that we guarantee natural alignment) #define MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4*MI_KiB) // also aligns any multiple of this size to avoid TLB misses. @@ -547,7 +635,7 @@ typedef struct mi_padding_s { uint32_t delta; // padding bytes before the block. (mi_full_usable_size(p) - delta == exact allocated bytes) } mi_padding_t; #define MI_PADDING_SIZE (sizeof(mi_padding_t)) -#define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE) +#define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_SIZE_SIZE - 1) / MI_SIZE_SIZE) #else #define MI_PADDING_SIZE 0 #define MI_PADDING_WSIZE 0 @@ -558,13 +646,34 @@ typedef struct mi_padding_s { // A thread-local heap ("theap") owns a set of thread-local pages. struct mi_theap_s { + // put in front for fast small allocations + mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. + + // less frequently accessed fields mi_tld_t* tld; // thread-local data _Atomic(mi_heap_t*) heap; // the heap this theap belongs to. _Atomic(mi_subproc_t*)subproc; // subproc this belongs too (always `subproc == heap->subproc` but needed for safe destruction) - _Atomic(size_t) refcount; // reference count + _Atomic(size_t) refcount; // reference count (needed for safe heap destroy) + + // config + long page_full_retain; // how many full pages can be retained per queue (before abandoning them) + bool allow_page_reclaim; // `true` if this theap can reclaim abandoned pages + bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint + bool is_detached; // `true` if `tld->thread_id == MI_THREADID_DETACHED` + + // sampling + size_t sample_countdown; // sample countdown in requested bytes (don't change the field order; see `internal.h:_mi_theap_get_free_small_page`) + size_t sample_rate; // current sampling rate in requested bytes (or 0 to disable) (for profiling and guarded mode) + uint64_t sample_requested; // total allocated/requested bytes since the last sample + size_t profile_sample_rate; // sampling rate in requested bytes for profiling + size_t profile_sample_countdown; // countdown in requested bytes for profiling + size_t guarded_sample_rate; // sampling rate in requested bytes for guarded objects + size_t guarded_sample_countdown; // countdown in requested bytes for guarded objects + size_t guarded_size_min; // minimal size for guarded objects + size_t guarded_size_max; // maximal size for guarded objects + // stats unsigned long long heartbeat; // monotonic heartbeat count - uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`) mi_random_ctx_t random; // random number context used for secure allocation size_t page_count; // total number of pages in the `pages` queues. size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues) @@ -573,24 +682,13 @@ struct mi_theap_s { long generic_count; // how often is `_mi_malloc_generic` called? long generic_collect_count; // how often is `_mi_malloc_generic` called without collecting? + // theaps belong to heaps and threads mi_theap_t* tnext; // list of theaps in this thread mi_theap_t* tprev; mi_theap_t* hnext; // list of theaps of the owning `heap` mi_theap_t* hprev; - long page_full_retain; // how many full pages can be retained per queue (before abandoning them) - bool allow_page_reclaim; // `true` if this theap can reclaim abandoned pages - bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint - bool is_detached; // `true` if `tld->thread_id == MI_THREADID_DETACHED` - bool prof_force_slow; // if profiling is enabled: keep `pages_free_direct` poisoned so every malloc routes through `_mi_malloc_generic` - intptr_t prof_countdown; // bytes until next profiling sample (only consulted in `_mi_malloc_generic`; 0 if profiling is off) - #if MI_GUARDED - size_t guarded_size_min; // minimal size for guarded objects - size_t guarded_size_max; // maximal size for guarded objects - size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages) - size_t guarded_sample_count; // current sample count (counting down to 0) - #endif - mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. + // page queues mi_page_queue_t pages[MI_BIN_COUNT]; // queue of pages for each size class (or "bin") mi_memid_t memid; // provenance of the theap struct itself (meta or os) mi_stats_t stats; // thread-local statistics @@ -627,6 +725,8 @@ typedef struct mi_heap_s { mi_theap_t* theaps; // list of all thread-local theaps belonging to this heap (using the `hnext`/`hprev` fields) mi_lock_t theaps_lock; // lock for the theaps list operations + _Atomic(mi_profiler_t*) profiler; + _Atomic(size_t) abandoned_count[MI_BIN_COUNT]; // total count of abandoned pages in this heap _Atomic(uintptr_t) releasing; // set when `mi_heap_delete`/`mi_heap_destroy` starts: its pages are abandoned unmapped mi_page_t* os_abandoned_pages; // list of pages that are OS allocated and not in an arena @@ -661,7 +761,7 @@ struct mi_subproc_s { _Atomic(int64_t) purge_expire; // expiration is set if any arenas can be purged _Atomic(uint32_t) scavenger_wake; // futex word signalled when a purge is scheduled (scavenger thread waits on this) - _Atomic(mi_heap_t*) heap_main; // main heap for this sub process + _Atomic(mi_heap_t*) heap_main; // main heap for this sub process mi_heap_t* heaps; // heaps belonging to this sub-process mi_lock_t heaps_lock; @@ -677,6 +777,8 @@ struct mi_subproc_s { _Atomic(size_t) heap_count; // current heaps in this sub-process (== |heaps|) _Atomic(size_t) heap_total_count; // total created heaps in this sub-process + _Atomic(mi_profiler_t*) profiler; + mi_memid_t memid; // provenance of this memory block (meta or static) mi_subproc_t* parent; // subproc in which this one was allocated mi_decl_align(8) // needed on some 32-bit platforms @@ -732,7 +834,6 @@ struct mi_tld_s { bool holes_sweep_full; // this sweep ignores `page->swept_state` (see `_mi_page_purge_holes`) size_t holes_sweep_skipped; // per-sweep counters, folded into the process-wide ones in `_mi_page_purge_holes_end` size_t holes_sweep_visited; - bool prof_sampling; // inside `_mi_prof_sample`: an allocation made while taking the backtrace is not sampled }; @@ -772,6 +873,7 @@ typedef struct mi_arena_s { mi_memid_t memid; // provenance of the memory area mi_subproc_t* subproc; // subprocess this arena belongs to (`this 'element-of' this->subproc->arenas`) size_t arena_idx; // index in the arenas array + void* start; // actual start of the arena area (the arena_s info may come later due to guard pages etc.) size_t slice_count; // total size of the area in arena slices (of `MI_ARENA_SLICE_SIZE`) size_t info_slices; // initial slices reserved for the arena bitmaps diff --git a/readme.md b/readme.md index 560dabe89..67e6be9b6 100644 --- a/readme.md +++ b/readme.md @@ -15,9 +15,9 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the runtime systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release : `v3.4.5` (2026-08-05) recommended. -Latest v2 release: `v2.4.5` (2026-08-05) stable. -Latest v1 release: `v1.9.15` (2026-08-05) legacy. +Latest release : `v3.5.2` (2026-09-12) recommended. +Latest v2 release: `v2.5.2` (2026-09-12) stable, legacy. +Latest v1 release: `v1.15.2` (2026-09-12) legacy. mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: @@ -84,12 +84,30 @@ New development is mostly on v3, while v1 and v2 are maintained with security an (much) less memory. Also supports true first-class heaps (that can allocate from any thread) and has more efficient heap-walking (for the CPython GC for example). (release tags: `v3.x`, development branch `dev3`). -- __v2__: stable mimalloc version. Uses thread-local segments to reduce fragmentation. (release tags: `v2.x`, development branch `dev2` and `main`) -- __v1__: legacy version: initial design of mimalloc (release tags: `v1.9.x`, development branch `dev`). +- __v2__: stable legacy mimalloc version. Uses thread-local segments to reduce fragmentation. (release tags: `v2.x`, development branch `dev2` and `main`) +- __v1__: legacy version: initial design of mimalloc (release tags: `v1.x`, development branch `dev`). __Send PR's against this version if possible.__ ### Releases +* 2026-09-12, `v3.5.2`, `v2.5.2`, `v1.15.2`: (v3) Reduced cache contention, improved `mi_malloc_csize`, + improved zero'ing of small blocks, `mi_wmalloc` variants for runtime systems, always enable detailed + statistics. Experimental support for profiling hooks (`mimalloc-profile.h`). +* 2026-09-01: Added a readme section on [getting the best performance](#getting-the-best-performance). +* 2026-09-01, `v3.5.1`, `v2.5.1`, `v1.15.1`: (v3) Yet better performance for `free` calls. + (v3) Added `mi_free_small(_nonnull)` for fast small allocations, and `mi_free_csize(_nonnull)` for inlined constant size free-ing. + (v3) Improved MI_OPT_ARCH options; on arm64, defaults to armv8.3 on Apple and armv8.1 otherwise. + (v3) Default to `-DMI_ALLOW_THP=FULL` on Linux, where we never split transparent huge pages (THP) during purging (#1282). This + may increase rss but can also improve performance. Use `-DMI_ALLOW_THP=OFF` to disable the use of THP. + (v3) Improved cache behavior for small allocations. Improved riscV support. Various small build fixes. +* 2026-08-18, `v1.15.0`, `v2.5.0`, `v3.5.0`: (v3) slightly better performance for `free` + using aligned chunks, cleanup cmake options, require armv8.3 (with MI_OPT_ARCH) for + faster load-acquire, increase retired page count from 1 to 3, faster double free + detection in secure mode, use faster atomics for the arena bitmaps (thank you Alan + Andrade, #1346), faster pagemap lookup. + Other: fix numa sparse node count detection (#1365), add theap stats retrieval, + use __builtin_thread_pointer on riscV (#1363), fix C mode compilation on x86 with + msvc (#1361), improved mingw-ucrt64 support. * 2026-08-05, `v1.9.15`, `v2.4.5`, `v3.4.5`: Fix compilation with xmalloc (#1353), make the build deterministic (#1355), mi_zalloc_aligned fix (#763), improve support for mingw (ucrt64), fix fputs fallback on windows (#1354), (v3): use proper lock backoff for first-class heap deletion, improved riscv64 codegen. @@ -110,27 +128,6 @@ New development is mostly on v3, while v1 and v2 are maintained with security an * 2026-04-20, `v1.9.9`, `v2.3.1`, `v3.3.1`: various bug and security fixes. Special thanks to @jinpzhanAMD, @res2k, and @GoldJohnKing for their help in improving Windows finalization, and @Zoxc for his help in finding various issues. -* 2026-04-15, `v1.9.8`, `v2.3.0`, `v3.3.0`: initial support for github (binary) releases, - fix visiting of full pages during collection (performance), - fix THP alignment (performance), fix arm64 cross-compilation on Windows, enable guard pages in debug mode, - always use uncommitted areas between arenas (security), enable static overloading of `malloc` etc. on Windows with the - static CRT (by @Noxybot), fix TLS slot leak on Windows (v3), enable clean DLL load/unload with statically linked - mimalloc (v3), fix race in `mi_heap_destroy` (v3), by default put page meta info separate from allocated - objects (v3,security), fix C++ overrides for emscripten. Various bugs found by DeepTest include: - fix offset for `mi_heap_realloc_aligned`, fix `mi_(w)dupenv_s` buffer size, fix potential overflow in size options, - and error codes for `mi_reallocarr(ay)`. -* 2026-02-03, `v3.2.8` (rc3): Fix thread reinitialize issue on macOS. Fix SIMD codegen bug on older - GCC versions. Extend Windows TLS slot limit from 64 to 1088. Report commit statistics more precise. - Fixes issue in free-page search in arenas. -* 2026-01-15, `v1.9.7`, `v2.2.7`, `v3.2.7` (rc2): Fix zero initializing blocks that were OS allocated. - For v3 various bug and performance fixes. Fix Debian 32-bit compilation. -* 2026-01-08, `v1.9.6`, `v2.2.6`, `v3.2.6` (rc1): Important bug fixes. Many improvements to v3 including - true first-class heaps where one can allocate in heap from any thread, and track statistics per heap as well. - Added `MIMALLOC_ALLOW_THP` option. This is by default enabled except on Android. When THP is detected on v3, - mimalloc will set the `MIMALLOC_MINIMAL_PURGE_SIZE` to 2MiB to avoid breaking up potential THP huge pages. - v3 uses faster TLS access on Windows, and has improved performance for `mi_calloc` and aligned allocations. - Fixed rare race condition on older v3, fixed potential buffer overflow in debug statistics, add API for returning - allocated sizes on allocation and free. * [Older release notes](#older-release-notes) @@ -361,16 +358,16 @@ Advanced options: - `MIMALLOC_PURGE_DECOMMITS=1`: By default "purging" memory means unused memory is decommitted (`MEM_DECOMMIT` on Windows, `MADV_DONTNEED` (which decresease rss immediately) on `mmap` systems). Set this to 0 to instead "reset" unused memory on a purge (`MEM_RESET` on Windows, generally `MADV_FREE` (which does not decrease rss immediately) on `mmap` systems). - Mimalloc generally does not "free" OS memory but only "purges" OS memory, in other words, it tries to keep virtual - address ranges and decommits within those ranges (to make the underlying physical memory available to other processes). + Mimalloc generally does not "free" OS memory but only "purges" OS memory, in other words, it will keep virtual + address ranges, but decommits within those ranges (to make the underlying physical memory available to other processes). Further options for large workloads and services: -- `MIMALLOC_ALLOW_THP=1`: By default always allow transparent huge pages (THP) on Linux systems. On Android only this is - by default off. When set to `0`, mimalloc opts the memory it maps itself out of - THP (per mapping, so this does not affect programs spawned by the process, nor memory handed to mimalloc via - `mi_manage_os_memory`). If enabled, mimalloc also sets - the `MIMALLOC_MINIMAL_PURGE_SIZE` in v3 to 2MiB to avoid potentially breaking up transparent huge pages when purging memory. +- `MIMALLOC_ALLOW_THP=[0,1,2]`: When set to `0`, mimalloc opts the memory it maps itself out of transparent huge pages (THP) + (per mapping, so this does not affect programs spawned by the process, nor memory handed to mimalloc via + `mi_manage_os_memory`), and otherwise THP is used whenever it is [enabled in the OS][linux-thp] (`[always]` or `[madvise]`). If set to 2 (default on Linux), + mimalloc also sets the `MIMALLOC_MINIMAL_PURGE_SIZE` in v3 to 2MiB to avoid potentially breaking up transparent huge pages when + purging memory which is better for performance (but may increase the rss!). On Android the default setting is 0 to reduce rss. - `MIMALLOC_USE_NUMA_NODES=N`: pretend there are at most `N` NUMA nodes. If not set, the actual NUMA nodes are detected at runtime. Setting `N` to 1 may avoid problems in some virtual environments. Also, setting it to a lower number than the actual NUMA nodes is fine and will only cause threads to potentially allocate more memory across actual NUMA @@ -397,6 +394,7 @@ Use caution when using `fork` in combination with either large or huge OS pages: for all pages in the original process including the huge OS pages. When any memory is now written in that area, the OS will copy the entire 1GiB huge page (or 2MiB large page) which can cause the memory usage to grow in large increments. +[linux-thp]: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/performance_tuning_guide/sect-red_hat_enterprise_linux-performance_tuning_guide-configuring_transparent_huge_pages [linux-huge]: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/5/html/tuning_and_optimizing_red_hat_enterprise_linux_for_oracle_9i_and_10g_databases/sect-oracle_9i_and_10g_tuning_guide-large_memory_optimization_big_pages_and_huge_pages-configuring_huge_pages_in_red_hat_enterprise_linux_4_or_5 [windows-huge]: https://docs.microsoft.com/en-us/sql/database-engine/configure-windows/enable-the-lock-pages-in-memory-option-windows?view=sql-server-2017 @@ -413,7 +411,7 @@ to make mimalloc more robust against exploits. In particular: - The free lists are initialized in a random order and allocation randomly chooses between extension and reuse within a page to mitigate against attacks that rely on a predicable allocation order. Similarly, the larger heap blocks allocated by mimalloc from the OS are also address randomized. -- If enabling `-DMI_SECURE_FULL=ON` there will also be guard pages at the end of each (64KiB) mimalloc page (thus interleaving +- If using `-DMI_SECURE=FULL` there will also be guard pages at the end of each (64KiB) mimalloc page (thus interleaving valid block data with inaccessible gaps). This setting is not recommended in general as it is more expensive and can lead to reaching the maximum VMA limit on Linux systems if the heap gets too large. @@ -421,7 +419,7 @@ As always, evaluate with care as part of an overall security strategy as all of ## Debug Mode -When _mimalloc_ is built using debug mode, (`-DCMAKE_BUILD_TYPE=Debug`), +When _mimalloc_ is built using debug mode, (`-DCMAKE_BUILD_TYPE=Debug`) (or when using `-DMI_DEBUG=ON` explicitly), various checks are done at runtime to catch development errors. - Statistics are maintained in detail for each object size. They can be shown using `MIMALLOC_SHOW_STATS=1` at runtime. @@ -460,6 +458,33 @@ many system calls. Therefore, there are various environment variables (and optio Overriding the standard `malloc` (and `new`) can be done either _dynamically_ or _statically_. + +## Static override + +On Unix like systems, you can statically link with _mimalloc_ to override the standard +malloc interface. The recommended way is to link the final program with the +_mimalloc_ single object file (`mimalloc.o`). We use +an object file instead of a library file as linkers give preference to +that over archives to resolve symbols. To ensure that the standard +malloc interface resolves to the _mimalloc_ library, link it as the first +object file. For example: + +``` +> gcc -o myprogram mimalloc.o myfile1.c ... +``` + +Another way to override statically that works on all platforms, is to +link statically to mimalloc (as shown in the introduction) and include a +header file in each source file that re-defines `malloc` etc. to `mi_malloc`. +This is provided by [`mimalloc-override.h`](include/mimalloc-override.h). This only works +reliably though if all sources are +under your control or otherwise mixing of pointers from different heaps may occur! + +Note: recently we also enabled static overloading on Windows. In that case you need +to link with the static CRT _release_ runtime (`/MT`) and link with the static +`mimalloc(-debug).obj` (to take precendence over the definitions in the CRT library). + + ## Dynamic override This is the recommended way to override the standard malloc interface. @@ -537,30 +562,55 @@ first in the import table. In such cases the [`minject`](bin) tool can be used to patch the executable's import tables. -## Static override +# Getting the best performance -On Unix-like systems, you can also statically link with _mimalloc_ to override the standard -malloc interface. The recommended way is to link the final program with the -_mimalloc_ single object file (`mimalloc.o`). We use -an object file instead of a library file as linkers give preference to -that over archives to resolve symbols. To ensure that the standard -malloc interface resolves to the _mimalloc_ library, link it as the first -object file. For example: +Consider the following build options (v3): -``` -> gcc -o myprogram mimalloc.o myfile1.c ... -``` +- `-DMI_OPT_ARCH=ON` (by default ON on arm64): enables architecture specific optimizations. + This uses `-march=haswell;-march=avx2` on x64 (available since 2013), `-march=armv8.1` on arm64 (2016), + and `-march=rv64gcb_zacas` on riscV (2023). + Each benefits mimalloc with better bit operations and/or native atomic operations (but also, if your + emulator or hardware does not support these extensions the program will fail with an illegal instruction). + On Apple arm64 the default is `-march=armv8.3` (which is supported on the M1 and better) which includes + efficient load-acquire instructions which mimalloc uses. For riscV the corresponding `zalasr` extension + can be enabled as `-DMI_OPT_ARCH=rv64gcb_zacas_zalasr` when your hardware supports it. -Another way to override statically that works on all platforms, is to -link statically to mimalloc (as shown in the introduction) and include a -header file in each source file that re-defines `malloc` etc. to `mi_malloc`. -This is provided by [`mimalloc-override.h`](include/mimalloc-override.h). This only works -reliably though if all sources are -under your control or otherwise mixing of pointers from different heaps may occur! +- `-DMI_ALLOW_THP=OFF`: disabling THP can reduce the rss of certain programs significantly. + +- `-DMI_ALLOW_THP=FULL`: enabling THP can increase performance of certain programs significantly. This setting + still disables THP if the OS THP setting is configured as `[never]`. + +- `-DMI_FREE_IS_CHECKED=ON` (by default ON in secure mode): slightly slower performance but checks validity + of every pointer passed to `mi_free`. Sometimes needed when overriding malloc/free process wide. + Note also that `mi_cfree` (checked free) always checks the validity of pointers passed to it which can + also be used if appropiate without needing to check all pointers process wide. + +In the source code: + +- Use `mi_zalloc` for zero initialized allocations. + +- Use the typed macros `mi_malloc_tp` etc. as these are both more safe and also use `mi_malloc_csize` internally. + +- Use `mi_malloc_csize`/`mi_free_csize` etc. when the allocated or free'd size is a constant as these are inlined + and statically redirect to `mi_malloc_small`/`mi_free_small` when possible. + These are also used by the C++ `mimalloc-new-delete.h` sized overrides. + You can use `mi_free_csize_nonnull` when the pointer is guaranteed to be not `NULL` as well. + +- Use `mi_malloc_small` when the allocation size is guaranteed to be less than `MI_SMALL_SIZE_MAX` (1KiB) for faster allocations. + +- Use `mi_free_small` when the pointer was returned from `mi_malloc_small` (and guaranteed to have a + size of less than `MI_SMALL_SIZE_MAX`). Use `mi_free_small_nonnull` when the pointer is guaranteed to be not `NULL` as well. + +For run-time systems and compilers (like Koka, Lean, etc.), we can do slightly better still. + +- If the allocation size is statically known, use `mi_malloc_csize` and `mi_free_csize` etc. when possible. + (or directly `mi_malloc_small`/`mi_free_small`). + +- If the runtime already carries thread local state, it may be faster to get the default `theap` for each thread + up-front (`mi_theap_get_default()`) and use the very fast `mi_theap_malloc(_small)` etc. passing the theap + pointer directly. This avoids having mimalloc look up the thread local pointer all the time. Whether this is + faster depends a bit on the OS implementation of thread local variables. -Note: recently we also enabled static overloading on Windows. In that case you need -to link with the static CRT _release_ runtime (`/MT`) and link with the static -`mimalloc(-debug).obj` (to take precendence over the definitions in the CRT library). # Tools @@ -656,9 +706,11 @@ use a tool like [TraceControl] that is specialized for analyzing mimalloc traces [TraceControl]: https://github.com/xinglonghe/TraceControl + # Performance -Last update: 2021-01-30 +Last update: 2021-01-30. +(Note 2026-09-01: this is a while ago now, many of these allocators have improved over time as well. YMMV.) We tested _mimalloc_ against many other top allocators over a wide range of benchmarks, ranging from various real world programs to @@ -916,6 +968,27 @@ provided by the bot. You will only need to do this once across all repos using o # Older Release Notes +* 2026-04-15, `v1.9.8`, `v2.3.0`, `v3.3.0`: initial support for github (binary) releases, + fix visiting of full pages during collection (performance), + fix THP alignment (performance), fix arm64 cross-compilation on Windows, enable guard pages in debug mode, + always use uncommitted areas between arenas (security), enable static overloading of `malloc` etc. on Windows with the + static CRT (by @Noxybot), fix TLS slot leak on Windows (v3), enable clean DLL load/unload with statically linked + mimalloc (v3), fix race in `mi_heap_destroy` (v3), by default put page meta info separate from allocated + objects (v3,security), fix C++ overrides for emscripten. Various bugs found by DeepTest include: + fix offset for `mi_heap_realloc_aligned`, fix `mi_(w)dupenv_s` buffer size, fix potential overflow in size options, + and error codes for `mi_reallocarr(ay)`. +* 2026-02-03, `v3.2.8` (rc3): Fix thread reinitialize issue on macOS. Fix SIMD codegen bug on older + GCC versions. Extend Windows TLS slot limit from 64 to 1088. Report commit statistics more precise. + Fixes issue in free-page search in arenas. +* 2026-01-15, `v1.9.7`, `v2.2.7`, `v3.2.7` (rc2): Fix zero initializing blocks that were OS allocated. + For v3 various bug and performance fixes. Fix Debian 32-bit compilation. +* 2026-01-08, `v1.9.6`, `v2.2.6`, `v3.2.6` (rc1): Important bug fixes. Many improvements to v3 including + true first-class heaps where one can allocate in heap from any thread, and track statistics per heap as well. + Added `MIMALLOC_ALLOW_THP` option. This is by default enabled except on Android. When THP is detected on v3, + mimalloc will set the `MIMALLOC_MINIMAL_PURGE_SIZE` to 2MiB to avoid breaking up potential THP huge pages. + v3 uses faster TLS access on Windows, and has improved performance for `mi_calloc` and aligned allocations. + Fixed rare race condition on older v3, fixed potential buffer overflow in debug statistics, add API for returning + allocated sizes on allocation and free. * 2025-06-09, `v1.9.4`, `v2.2.4`, `v3.1.4` (beta) : Some important bug fixes, including a case where OS memory was not always fully released. Improved v3 performance, build on XBox, fix build on Android, support interpose for older macOS versions, use MADV_FREE_REUSABLE on macOS, always check commit success, better support for Windows diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 111d63a8b..3a4383e0b 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -27,7 +27,7 @@ static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) { return ok; } -#if MI_GUARDED +#if 0 static mi_decl_noinline mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, size_t size, size_t alignment, bool zero, mi_page_t** ppage) mi_attr_noexcept { // use over allocation for guarded blocksl #if MI_THEAP_INITASNULL @@ -47,21 +47,22 @@ static mi_decl_noinline mi_decl_restrict void* mi_theap_malloc_guarded_aligned(m mi_assert_internal(_mi_is_aligned(p, alignment)); return p; } - +#endif +#if MI_GUARDED static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) { - #if MI_THEAP_INITASNULL - if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); } - #endif - const size_t rate = theap->guarded_sample_rate; + // #if MI_THEAP_INITASNULL + // if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); } + // #endif + // const size_t rate = theap->guarded_sample_rate; // only write if `rate!=0` so we don't write to the constant `_mi_theap_empty` - if (rate != 0) { theap->guarded_sample_rate = 0; } - void* p = _mi_theap_malloc_zero(theap, size, zero, ppage); - if (rate != 0) { theap->guarded_sample_rate = rate; } + // if (rate != 0) { theap->guarded_sample_rate = 0; } + void* p = _mi_theap_malloc_zero(theap, size, zero, 0, ppage); + // if (rate != 0) { theap->guarded_sample_rate = rate; } return p; } #else static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) { - return _mi_theap_malloc_zero(theap, size, zero, ppage); + return _mi_theap_malloc_zero(theap, size, zero, 0, ppage); } #endif @@ -85,7 +86,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap } oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size); // note: no guarded as alignment > 0 - p = _mi_theap_malloc_zero_ex(theap, oversize, zero, alignment, &page); // the page block size should be large enough to align in the single huge page block + p = _mi_theap_malloc_zero(theap, oversize, zero, alignment, &page); // the page block size should be large enough to align in the single huge page block if (p == NULL) return NULL; } else { @@ -112,19 +113,18 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap // (we can access the page though since the just allocated pointer keeps it alive) if (aligned_p != p) { mi_page_set_has_interior_pointers(page, true); - // if (usable!=NULL) { - // mi_assert_internal(*usable > adjust); - // if (*usable > adjust) { *usable = *usable - adjust; } - // mi_assert_internal(*usable >= size); - // } - #if MI_GUARDED + + #if MI_GUARDED || MI_PROFILE // set tag to aligned so mi_usable_size works with guard pages + // note: we could have gotten a profiled or guarded block but that is still ok + // and in such case the MI_BLOCK_TAG_ALIGNED is just not at the start of the real heap block if (adjust >= sizeof(mi_block_t)) { mi_block_t* const block = (mi_block_t*)p; block->next = MI_BLOCK_TAG_ALIGNED; } #endif - _mi_padding_shrink(page, (mi_block_t*)p, adjust + size); + mi_block_t* const base = _mi_page_ptr_unalign(page, p); + _mi_padding_shrink(page, base, adjust + size); } // todo: expand padding if overallocated ? @@ -134,22 +134,13 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust); #if MI_DEBUG > 1 mi_page_t* const apage = _mi_ptr_page(aligned_p); - void* unalign_p = _mi_page_ptr_unalign(apage, aligned_p); - mi_assert_internal(p == unalign_p); + mi_block_t* unalign_p = _mi_page_ptr_unalign(apage, aligned_p); + mi_assert_internal(p == (void*)unalign_p || mi_block_ptr_is_sampled(unalign_p, aligned_p)); #endif - // now zero the block if needed - //if (alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) { - // // for the tracker, on huge aligned allocations only from the start of the large block is defined - // mi_track_mem_undefined(aligned_p, size); - // if (zero) { - // _mi_memzero_(aligned_p, mi_usable_size(aligned_p)); - // } - //} - if (p != aligned_p) { mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p)); - #if MI_GUARDED + #if MI_GUARDED || MI_PROFILE mi_track_mem_defined(p, sizeof(mi_block_t)); #endif } @@ -169,17 +160,30 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t // use regular allocation if it is guaranteed to fit the alignment constraints. // this is important to try as the fast path in `mi_theap_malloc_zero_aligned` only works when there exist // a page with the right block size, and if we always use the over-alloc fallback that would never happen. - if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { - void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, ppage); - mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0); - const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; - if mi_likely(is_aligned_or_null) { - return p; - } - else { - // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct.. - mi_assert(false); - mi_free(p); + #if MI_THEAP_INITASNULL + if mi_likely(theap!=NULL) + #endif + { + #if MI_SAMPLE // only try if we would not take a sample + if mi_likely(!mi_theap_should_sample(theap,size)) + #endif + { + if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) + { + mi_page_t* page = NULL; + void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, &page); + if (ppage!=NULL) { *ppage = page; } + const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; + if mi_likely(is_aligned_or_null) { + return p; + } + else { + // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct.. + // (but it can happen if this allocation just so happens to be sampled but we checked for that) + mi_assert_internal(false); // p==NULL || mi_block_ptr_is_profiled_or_guarded(_mi_page_ptr_unalign(page,p),p)); + mi_free(p); + } + } } } @@ -201,36 +205,33 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, con return mi_error_bad_alignment(size, alignment, offset); } - #if MI_GUARDED - #if MI_THEAP_INITASNULL - if mi_likely(theap!=NULL) - #endif - if (offset==0 && alignment < MI_PAGE_MAX_OVERALLOC_ALIGN && mi_theap_malloc_use_guarded(theap,size)) { - return mi_theap_malloc_guarded_aligned(theap, size, alignment, zero, ppage); - } - #endif - // try first if there happens to be a small block available with just the right alignment // since most small power-of-2 blocks (under MI_PAGE_MAX_BLOCK_START_ALIGN2) are already // naturally aligned this can be often the case. - #if MI_THEAP_INITASNULL - if mi_likely(theap!=NULL) - #endif + if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) { - if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) { - const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` - const size_t padsize = size + MI_PADDING_SIZE; - mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize); - if mi_likely(page->free != NULL) { - const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0; - if mi_likely(is_aligned) - { - if (ppage!=NULL) { *ppage = page; } - void* p = _mi_page_malloc_zero(theap, page, padsize, zero); - mi_assert_internal(p != NULL); - mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); - mi_track_malloc(p, size, zero); - return p; + #if MI_THEAP_INITASNULL + if mi_likely(theap!=NULL) + #endif + { + #if MI_SAMPLE // check if we shouldn't take a sample + if mi_likely(!mi_theap_should_sample(theap,size)) + #endif + { + const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` + const size_t padsize = size + MI_PADDING_SIZE; + mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize, false); + if mi_likely(page->free != NULL) { + const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0; + if mi_likely(is_aligned) + { + if (ppage!=NULL) { *ppage = page; } + void* p = _mi_page_malloc_zero(theap, page, padsize, zero); + mi_assert_internal(p != NULL); + mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); + mi_track_malloc(p, size, zero); + return p; + } } } } diff --git a/src/alloc-override.c b/src/alloc-override.c index 04da5f293..bcb96b52d 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -293,7 +293,7 @@ typedef void* mi_nothrow_t; void _ZdlPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete(void*, std::align_val_t, std::nothrow_t const&) void _ZdaPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete[](void*, std::align_val_t, std::nothrow_t const&) - #if (MI_INTPTR_SIZE==8) || (MI_INTPTR_SIZE==4 && defined(__EMSCRIPTEN__)) // pr #1257 + #if (MI_SIZE_SIZE==8) || (MI_SIZE_SIZE==4 && defined(__EMSCRIPTEN__)) // pr #1257 void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } @@ -302,7 +302,7 @@ typedef void* mi_nothrow_t; void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } - #elif (MI_INTPTR_SIZE==4) + #elif (MI_SIZE_SIZE==4) void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } diff --git a/src/alloc.c b/src/alloc.c index 11072fe1c..f3b5717bf 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -26,31 +26,71 @@ terms of the MIT license. A copy of the license can be found in the file // Allocation // ------------------------------------------------------ +#if MI_PADDING +static mi_decl_noinline void mi_page_block_setup_padding(mi_page_t* page, mi_block_t* block, size_t size) mi_attr_noexcept { + const size_t bsize = mi_page_usable_block_size(page); + mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + bsize); + const ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE)); + mi_assert_internal(delta >= 0 && bsize >= (size - MI_PADDING_SIZE + delta)); + mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess + padding->canary = mi_ptr_encode_canary(page,block,page->keys); + padding->delta = (uint32_t)(delta); + #if MI_PADDING_CHECK_BYTES + if (!mi_page_is_huge(page)) { + uint8_t* fill = (uint8_t*)padding - delta; + const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes + for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; } + } + #endif +} +#endif + // Fast allocation in a page: just pop from the free list. // Fall back to generic allocation only if the list is empty. // Note: in release mode the (inlined) routine is about 7 instructions with a single test. -static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept +static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, size_t sample_countdown, bool zero, mi_page_t** ppage) mi_attr_noexcept { if (page->block_size != 0) { // not the empty theap mi_assert_internal(mi_page_block_size(page) >= size); mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); } // check the free list mi_block_t* const block = page->free; - if mi_unlikely(block == NULL) { + if (block == NULL) { return _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); } mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); if (ppage != NULL) { *ppage = page; }; // pop from the free list - page->free = mi_block_next(page, block); - page->used++; + mi_block_t* next = mi_block_next(page,block); + mi_track_mem_undefined(block,sizeof(*block)); + + #if MI_SECURE + if (!zero) block->next = 0; // don't leak internal data + #endif + + page->free = next; + page->xused.used_alloc += 0x10001; // increment both (16-bit) used count and alloc count: one add on memory mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE)); + #if MI_SAMPLE==2 + const size_t req_size = size - MI_PADDING_SIZE; + mi_assert_internal(theap->sample_countdown == sample_countdown); // we pass it to improve codegen + mi_assert_internal(theap->sample_countdown >= req_size); + theap->sample_countdown = sample_countdown - req_size; + #else + MI_UNUSED(sample_countdown); + #endif + + #if MI_STATS>=2 + mi_theap_stat_counter_increase(theap,malloc_requested,size - MI_PADDING_SIZE); + #endif + #if MI_DEBUG>3 if (page->free_is_zero && size > sizeof(*block)) { mi_assert_expensive(mi_mem_is_zero(block+1,size - sizeof(*block))); @@ -63,85 +103,65 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ const size_t bsize = mi_page_usable_block_size(page); mi_track_mem_undefined(block, bsize); - #if (MI_STAT>0) - if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { - mi_theap_stat_increase(theap, malloc_normal, bsize); - #if (MI_STAT>1) - mi_theap_stat_counter_increase(theap, malloc_normal_count, 1); - const size_t bin = _mi_bin(bsize); - mi_theap_stat_increase(theap, malloc_bins[bin], 1); - mi_theap_stat_increase(theap, malloc_requested, size - MI_PADDING_SIZE); - #endif - } + // track per-block statistics + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + + // in debug mode initialize with 0xD0 + #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN + if mi_likely(!zero && !mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); } #endif // zero the block? note: we need to zero the full block size (issue #63) - if mi_likely(!zero) { - // #if MI_SECURE - block->next = 0; // don't leak internal data - // #endif - #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN - if (!mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); } - #endif - } - else { + if mi_unlikely(zero) { if (!page->free_is_zero) { - _mi_memzero_aligned(block, bsize); + mi_assert_internal(bsize%MI_SIZE_SIZE == 0); + #if !MI_PADDING + return // use tail-call + #endif + _mi_memzero_block(block,bsize); } else { - block->next = 0; - mi_track_mem_defined(block, bsize); - } + block->next = 0; + mi_track_mem_defined(block, bsize); + } } - + + // setup padding (must come after zero'ing) #if MI_PADDING // && !MI_TRACK_ENABLED - mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + bsize); - ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE)); - #if (MI_DEBUG>=2) - mi_assert_internal(delta >= 0 && bsize >= (size - MI_PADDING_SIZE + delta)); - #endif - mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess - padding->canary = mi_ptr_encode_canary(page,block,page->keys); - padding->delta = (uint32_t)(delta); - #if MI_PADDING_CHECK - if (!mi_page_is_huge(page)) { - uint8_t* fill = (uint8_t*)padding - delta; - const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes - for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; } - } - #endif + mi_page_block_setup_padding(page,block,size); #endif - return block; + return block; } // extra entries for improved efficiency in `alloc-aligned.c` (and in `page.c:mi_malloc_generic`. extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept { - return mi_page_malloc_zero(theap, page, size, zero, NULL); + return mi_page_malloc_zero(theap, page, size, theap->sample_countdown, zero, NULL); } -// main allocation primitives for small and generic allocation - // internal small size allocation -static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept +static mi_decl_forceinline mi_decl_restrict void* mi_theap_xmalloc_small_zero_nonnull(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { + size_t size = (is_wsize ? xsize * MI_SIZE_SIZE : xsize); mi_assert(theap != NULL); mi_assert(size <= MI_SMALL_SIZE_MAX); #if MI_DEBUG mi_assert(mi_theap_matches_thread(theap)); // theaps are thread local #endif - #if (MI_PADDING || MI_GUARDED) - if mi_unlikely(size == 0) { size = sizeof(void*); } + #if (MI_PADDING) // || MI_GUARDED) + if mi_unlikely(xsize == 0) { xsize = (is_wsize ? 1 : MI_SIZE_SIZE); size += MI_SIZE_SIZE; } #endif - #if MI_GUARDED - if mi_unlikely(mi_theap_malloc_use_guarded(theap,size)) { - return _mi_theap_malloc_guarded(theap, size, zero, ppage); - } + + // we only sample if fine-grained sampling is enabled (otherwise we sample in mi_malloc_generic) + #if MI_SAMPLE==2 + if mi_unlikely(mi_theap_should_sample(theap,size)) { return _mi_theap_malloc_sampled(theap,size,zero,ppage); } #endif + + // get page in constant time + mi_page_t* page = _mi_theap_get_free_small_page(theap, xsize + (is_wsize ? MI_PADDING_WSIZE : MI_PADDING_SIZE), is_wsize); - // get page in constant time, and allocate from it - mi_page_t* page = _mi_theap_get_free_small_page(theap, size + MI_PADDING_SIZE); - void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, zero, ppage); + // and allocate + void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, theap->sample_countdown, zero, ppage); mi_track_malloc(p,size,zero); #if MI_DEBUG>3 @@ -155,14 +175,6 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_non // internal generic allocation static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { - #if MI_GUARDED - #if MI_THEAP_INITASNULL - if (theap!=NULL) - #endif - if (huge_alignment==0 && mi_theap_malloc_use_guarded(theap, size)) { - return _mi_theap_malloc_guarded(theap, size, zero, ppage); - } - #endif #if !MI_THEAP_INITASNULL mi_assert(theap!=NULL); #endif @@ -180,46 +192,64 @@ static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size } // internal small allocation -static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { +static mi_decl_forceinline mi_decl_restrict void* mi_theap_xmalloc_small_zero(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { #if MI_THEAP_INITASNULL if (theap!=NULL) { - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); + return mi_theap_xmalloc_small_zero_nonnull(theap, xsize, is_wsize, zero, ppage); } else { - return mi_theap_malloc_generic(theap, size, zero, 0, ppage); // tailcall + return mi_theap_malloc_generic(theap, (is_wsize ? xsize * MI_SIZE_SIZE : xsize), zero, 0, ppage); // tailcall } #else - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); + return mi_theap_xmalloc_small_zero_nonnull(theap, xsize, is_wsize, zero, ppage); #endif } - // allocate a small block mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero(theap, size, false, NULL); + mi_assert(theap!=NULL); + return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, false, NULL); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { + mi_assert(theap!=NULL); + return mi_theap_xmalloc_small_zero_nonnull(theap, wsize, true, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept { - return mi_theap_malloc_small(_mi_theap_default(), size); + return mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, false, NULL); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_wmalloc_small(size_t wsize) mi_attr_noexcept { + return mi_theap_xmalloc_small_zero(_mi_theap_default(), wsize, true, false, NULL ); +} + +mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_theap_xmalloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, false, NULL); + // we could also use: mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(theap), size, false, NULL); } + // but the following prevents using a stack frame. We use this to optimize some select functions only. + mi_theap_t* const theap = _mi_heap_theap_cached(heap); + if mi_likely(theap!=NULL) { return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, false, NULL); } + else { return mi_heap_init_malloc_small(heap, size); } } // The main internal allocation functions static mi_decl_forceinline void* mi_theap_malloc_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { + mi_assert(theap!=NULL); // fast path for small objects if mi_likely(size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(huge_alignment == 0); - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); + return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, zero, ppage); } else { return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); } } -extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { +extern mi_decl_forceinline void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { // fast path for small objects #if MI_THEAP_INITASNULL if mi_likely(theap!=NULL && size <= MI_SMALL_SIZE_MAX) @@ -228,57 +258,83 @@ extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, siz #endif { mi_assert_internal(huge_alignment == 0); - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); + return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, zero, ppage); } else { return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); } } -inline void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { - return _mi_theap_malloc_zero_ex(theap, size, zero, 0, ppage); -} // Main allocation functions mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept { - return _mi_theap_malloc_zero_ex(theap, size, false, 0, NULL); + mi_assert(theap!=NULL); + return mi_theap_malloc_zero_nonnull(theap, size, false, 0, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept { - return mi_theap_malloc(_mi_theap_default(), size); + return _mi_theap_malloc_zero(_mi_theap_default(), size, false, 0, NULL); } +mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_theap_malloc_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, 0, NULL); +} mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, false, 0, NULL); + // return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, false, 0, NULL); + mi_theap_t* const theap = _mi_heap_theap_cached(heap); + if mi_likely(theap!=NULL) { return mi_theap_malloc_zero_nonnull(theap, size, false, 0, NULL); } + else { return mi_heap_init_malloc(heap, size); } + } // zero initialized small block mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero(_mi_theap_default(), size, true, NULL); + return mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, true, NULL); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept { + return mi_theap_xmalloc_small_zero(_mi_theap_default(), wsize, true, true, NULL); } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero(theap, size, true, NULL); + mi_assert(theap!=NULL); + return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, true, NULL); } -mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL); +mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { + mi_assert(theap!=NULL); + return mi_theap_xmalloc_small_zero_nonnull(theap, wsize, true, true, NULL); } +mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_theap_xmalloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, true, NULL); +} +mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { + // optimize: return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL); + mi_theap_t* const theap = _mi_heap_theap_cached(heap); + if mi_likely(theap!=NULL) { return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, true, NULL); } + else { return mi_heap_init_zalloc_small(heap, size); } +} mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept { - return _mi_theap_malloc_zero(theap, size, true, NULL); + return _mi_theap_malloc_zero(theap, size, true, 0, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept { - return _mi_theap_malloc_zero(_mi_theap_default(), size, true, NULL); + return _mi_theap_malloc_zero(_mi_theap_default(), size, true, 0, NULL); } +mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_theap_malloc_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, true, 0, NULL); +} mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL); + // optimize: return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL); + mi_theap_t* const theap = _mi_heap_theap_cached(heap); + if mi_likely(theap!=NULL) { return mi_theap_malloc_zero_nonnull(theap, size, true, 0, NULL); } + else { return mi_heap_init_zalloc(heap, size); } } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept { @@ -308,25 +364,25 @@ static void* mi_ublock_size( void* p, mi_page_t* page, size_t* pblock_size ) { // Return usable size mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = mi_theap_malloc_small_zero(_mi_theap_default(), size, false, &page); + void* p = mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, false, &page); return mi_ublock_size(p,page,pblock_size); } mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = mi_theap_malloc_small_zero(_mi_theap_default(), size, true, &page); + void* p = mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, true, &page); return mi_ublock_size(p,page,pblock_size); } mi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, false, &page); + void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, false, 0, &page); return mi_ublock_size(p,page,pblock_size); } mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, true, &page); + void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, true, 0, &page); return mi_ublock_size(p,page,pblock_size); } @@ -362,7 +418,7 @@ void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { return NULL; #else if (p == NULL) return NULL; - const mi_page_t* const page = mi_validate_ptr_page(p,"mi_expand"); + const mi_page_t* const page = mi_ptr_page_validate(p,"mi_expand"); const size_t size = _mi_page_usable_size(page,p); if (newsize > size) return NULL; return p; // it fits @@ -381,7 +437,7 @@ static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, voi if (pblock_size_pre!=NULL) { *pblock_size_pre = 0; } } else { - page = mi_validate_ptr_page(p,"mi_realloc"); + page = mi_ptr_page_validate(p,"mi_realloc"); if mi_unlikely(page==NULL) { // invalid pointer if (pblock_size_pre!=NULL) { *pblock_size_pre = 0; } if (pblock_size_post!=NULL) { *pblock_size_post = 0; } @@ -409,7 +465,7 @@ static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, voi } // note: we don't zero allocate upfront so we only zero initialize the expanded part mi_page_t* newpage; // use block_size for zero-ing, issue #763 - void* const newp = _mi_theap_malloc_zero(theap,newsize,false /* no zero */,&newpage); + void* const newp = _mi_theap_malloc_zero(theap,newsize,false /* no zero */,0,&newpage); if mi_likely(newp != NULL) { if (pblock_size_post!=NULL) { *pblock_size_post = mi_page_block_size(newpage); } const size_t copy_size = (newsize > size ? size : newsize); @@ -857,95 +913,6 @@ mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { } } -#if MI_GUARDED -// We always allocate a guarded allocation at an offset (`mi_page_has_interior_pointers` will be true). -// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`) -// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size` - -static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size, size_t* usable_size) { - // todo: we can still make padding work by moving it out of the guard page area - mi_page_t* const page = _mi_ptr_page(block); - mi_page_set_has_interior_pointers(page, true); - block->next = MI_BLOCK_TAG_GUARDED; - - // set guard page at the end of the block - const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local` - const size_t os_page_size = _mi_os_page_size(); - mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t)); - if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) { - // should never happen - mi_free(block); - return NULL; - } - uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size; - // note: the alignment of the guard page relies on blocks being os_page_size aligned which - // is ensured in `mi_arena_page_alloc_fresh`. - mi_assert_internal(_mi_is_aligned(block, os_page_size)); - mi_assert_internal(_mi_is_aligned(guard_page, os_page_size)); - if (!page->memid.is_pinned && _mi_is_aligned(guard_page, os_page_size)) { - const bool ok = _mi_os_protect(guard_page, os_page_size); - if mi_unlikely(!ok) { - _mi_warning_message("failed to set a guard page behind an object (object %p of size %zu)\n", block, block_size); - } - } - else { - _mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size); - } - - // align pointer just in front of the guard page - size_t offset = block_size - os_page_size - obj_size; - mi_assert_internal(offset > sizeof(mi_block_t)); - if (offset > MI_PAGE_MAX_OVERALLOC_ALIGN) { - // give up to place it right in front of the guard page if the offset is too large for unalignment - offset = MI_PAGE_MAX_OVERALLOC_ALIGN; - } - uint8_t* const p = (uint8_t*)block + offset; - mi_assert_internal(p == guard_page - obj_size || offset >= MI_PAGE_MAX_OVERALLOC_ALIGN); - if (usable_size != NULL) { *usable_size = (guard_page - p); mi_assert_internal(mi_usable_size(p)==*usable_size); } - mi_track_align(block, p, offset, obj_size); - mi_track_mem_defined(block, sizeof(mi_block_t)); - return p; -} - -mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept -{ - // allocate multiple of page size ending in a guard page - // ensure minimal alignment requirement? - if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow - _mi_error_message(EOVERFLOW, "(guarded) allocation request is too large (%zu bytes)\n", size); - return NULL; - } - const size_t os_page_size = _mi_os_page_size(); - const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE)); - const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); - const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); - mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(theap, req_size, 0 /* don't zero */, ppage); - if (block==NULL) return NULL; - size_t usable_size = 0; - void* const p = mi_block_ptr_set_guarded(block, obj_size, &usable_size); - if (p == NULL) return NULL; - if (zero) { - _mi_memzero(p,obj_size); // we have to zero afterwards as padding might have written inside the block (if the `blocksize > reqsize + os_page_size`) - } - - // stats - mi_track_malloc(p, usable_size, zero); - if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); } - mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1); - #if MI_STAT>1 - // adjust stats to only count the allocated size of the block (and not the guard page) - mi_theap_stat_adjust_decrease(theap, malloc_requested, req_size); - mi_theap_stat_increase(theap, malloc_requested, size); - #endif - #if MI_DEBUG>3 - if (zero) { - mi_assert_expensive(mi_mem_is_zero(p, size)); - } - #endif - return p; -} -#endif - // ------------------------------------------------------ // ensure explicit external inline definitions are emitted! // ------------------------------------------------------ @@ -954,7 +921,6 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, void* _mi_externs[] = { (void*)&_mi_page_malloc_zero, (void*)&_mi_theap_malloc_zero, - (void*)&_mi_theap_malloc_zero_ex, (void*)&mi_theap_malloc, (void*)&mi_theap_zalloc, (void*)&mi_theap_malloc_small, diff --git a/src/arena.c b/src/arena.c index 5dc14ffad..2b844cd9d 100644 --- a/src/arena.c +++ b/src/arena.c @@ -25,9 +25,6 @@ The arena allocation needs to be thread safe and we use an atomic bitmap to allo #include "mimalloc/prim-tls.h" #include "bitmap.h" -#if (MI_ARENA_MAX_SIZE > MI_MAX_ALIGN_SIZE*UINT32_MAX) -#error "The page_t.page_ma_offset field is not large enough to cover a full arena" -#endif static mi_bitmap_t* mi_arena_pages_abandoned(mi_arena_pages_t* arena_pages, size_t bin); static mi_bitmap_t* mi_arena_pages_abandoned_ensure(mi_arena_t* arena, mi_arena_pages_t* arena_pages, size_t bin); @@ -94,34 +91,53 @@ static bool mi_heap_has_page(mi_heap_t* heap, mi_arena_t* arena, mi_page_t* page #endif size_t mi_arena_min_alignment(void) { - return MI_ARENA_SLICE_ALIGN; + return MI_ARENA_ALIGNMENT; } size_t mi_arena_min_size(void) { return MI_ARENA_MIN_SIZE; } -static size_t mi_arena_max_object_size(void) { +// slices reserved for page meta info at the start of aligned chunks +static size_t mi_arena_page_meta_aligned_slice_count(void) { + #if MI_PAGE_META_IS_ALIGNED + return _mi_divide_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE); + #else + return 0; + #endif +} + +// fixed limit for the maximum object size in an arena +static size_t mi_arena_max_fixed_object_size(void) { + #if MI_PAGE_META_IS_ALIGNED + return (MI_PAGE_META_ALIGNMENT - _mi_align_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE)); + #else + return (MI_ARENA_MAX_SIZE - MI_ARENA_CHUNK_SIZE); // minus an initial chunk to accommodate meta info + #endif +} + +// Maximum object size allowed to be allocated in an arena +size_t mi_arena_max_object_size(void) { size_t max_size = mi_option_get_size(mi_option_arena_max_object_size); max_size = _mi_align_up(max_size, MI_ARENA_SLICE_SIZE); if (max_size <= MI_ARENA_MIN_OBJ_SIZE) { return MI_ARENA_MIN_OBJ_SIZE; } - else if (max_size >= MI_ARENA_MAX_SIZE - (MI_BCHUNK_BITS*MI_ARENA_SLICE_SIZE)) { // minus an initial chunk to accommodate meta info - return (MI_ARENA_MAX_SIZE - (MI_BCHUNK_BITS*MI_ARENA_SLICE_SIZE)); + else if (max_size >= mi_arena_max_fixed_object_size()) { + return mi_arena_max_fixed_object_size(); } else { return max_size; } } -mi_decl_nodiscard static bool mi_arena_commit(mi_subproc_t* subproc, mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t already_committed) { +mi_decl_nodiscard static bool mi_arena_commit(mi_subproc_t* subproc, mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t stat_already_committed) { mi_assert_internal(subproc!=NULL); if (arena != NULL && arena->commit_fun != NULL) { return (*arena->commit_fun)(true, start, size, is_zero, arena->commit_fun_arg); } - else if (already_committed > 0) { - return _mi_os_commit_ex(subproc, start, size, is_zero, already_committed); + else if (stat_already_committed > 0) { + return _mi_os_commit_ex(subproc, start, size, is_zero, stat_already_committed); } else { return _mi_os_commit(subproc, start, size, is_zero); @@ -142,7 +158,7 @@ static size_t mi_arena_size(mi_arena_t* arena) { // Start of the arena memory area static uint8_t* mi_arena_start(mi_arena_t* arena) { - return ((uint8_t*)arena); + return ((uint8_t*)arena->start); } // Start of a slice @@ -153,17 +169,22 @@ uint8_t* mi_arena_slice_start(mi_arena_t* arena, size_t slice_index) { mi_page_t* mi_arena_page_at_slice(mi_arena_t* arena, size_t slice_index) { mi_assert_internal(slice_index < arena->slice_count); + #if MI_PAGE_META_IS_ALIGNED + mi_page_t* const page = _mi_aligned_ptr_page(mi_arena_slice_start(arena,slice_index)); // todo: optimize? + return page; + #else if (arena->pages_meta != NULL) { mi_page_t* const page = &arena->pages_meta[slice_index]; - #if MI_PAGE_META_ALIGNED_FREE_SMALL + #if MI_PAGE_META_SMALL_IS_ALIGNED // pages with small blocks still have the page at the start of the slice (and set the `block_size` in pages_meta to 0) if (page->block_size>0) return page; #else return page; #endif - } - // fall through (for MI_PAGE_META_ALIGNED_FREE_SMALL) + } + // fall through (for MI_PAGE_META_SMALL_IS_ALIGNED) return (mi_page_t*)mi_arena_slice_start(arena,slice_index); + #endif } // Arena area @@ -216,7 +237,6 @@ static size_t mi_page_full_size(mi_page_t* page) { } } - /* ----------------------------------------------------------- Arena Allocation ----------------------------------------------------------- */ @@ -254,7 +274,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at( if (already_committed < slice_count) { // not all committed, try to commit now bool commit_zero = false; - if (!mi_arena_commit(arena->subproc, arena, p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(slice_count - already_committed))) { + if (!mi_arena_commit(arena->subproc, arena, p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(already_committed))) { // if the commit fails, release ownership, and return NULL; // note: this does not roll back dirty bits but that is ok. mi_bbitmap_setN(arena->slices_free, slice_index, slice_count); @@ -527,7 +547,7 @@ static mi_decl_noinline void* mi_arenas_try_alloc( if (req_arena != NULL) return NULL; // don't create arena's while preloading (todo: or should we?) - if (_mi_preloading()) return NULL; + // if (_mi_preloading()) return NULL; // don't create arena's if OS allocation is disallowed if (mi_option_is_enabled(mi_option_disallow_os_alloc)) return NULL; @@ -760,8 +780,8 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t sl return NULL; } -static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool os_align, bool commit, mi_memid_t* memid, mi_arena_pages_t** parena_pages ) { - MI_UNUSED_RELEASE(block_size); +static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t max_page_meta_count, size_t block_alignment, bool os_align, bool commit, mi_memid_t* memid, mi_arena_pages_t** parena_pages ) { + MI_UNUSED(max_page_meta_count); mi_assert_internal(parena_pages!=NULL); *parena_pages = NULL; @@ -800,23 +820,62 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_ // otherwise fall back to the OS if (start == NULL) { + #if MI_PAGE_META_IS_ALIGNED + size_t page_offset; // offset in the block for the page area + uint8_t* os_start; + if (block_alignment < MI_PAGE_META_ALIGNMENT) { + page_offset = (block_alignment < MI_PAGE_ALIGN ? MI_PAGE_ALIGN : block_alignment); + os_start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, MI_PAGE_META_ALIGNMENT, 0 /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); + } + else { + // if we allow alignment >= MI_PAGE_META_ALIGNMENT we need to substract 1 from a pointer + // in _mi_aligned_ptr_page (and test for (intptr_t)p < 0 instead of NULL). We avoid this by limiting the max alignment. + _mi_warning_message("requested alignment is too large (%zu KiB)\n", block_alignment / MI_KiB); + errno = EINVAL; + return NULL; + // page_offset = MI_PAGE_META_ALIGNMENT; + // os_start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, block_alignment, MI_PAGE_META_ALIGNMENT /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); + } + if (os_start==NULL) return NULL; + // commit page info and page + const size_t min_page_count = _mi_divide_up(page_offset + MI_ARENA_SLICE_SIZE,MI_ARENA_SLICE_SIZE) + max_page_meta_count; + const size_t min_page_meta = min_page_count * sizeof(mi_page_t); + mi_assert_internal(min_page_meta < page_offset); + bool is_zero; + bool ok = mi_arena_commit(heap->subproc,req_arena,os_start,min_page_meta,&is_zero,min_page_meta /* don't count in stats? */); + if (ok && commit) { + ok = mi_arena_commit(heap->subproc,req_arena,os_start + page_offset,alloc_size,NULL,0); + } + if (!ok) { _mi_os_free(heap->subproc,os_start,alloc_size+page_offset,*memid); return NULL; } + if (!is_zero && !memid->initially_zero) { + _mi_memzero_aligned(os_start,min_page_meta); + } + start = os_start + page_offset; + mi_assert_internal(_mi_is_aligned(start,block_alignment)); + mi_assert_internal(_mi_is_aligned(os_start,MI_PAGE_META_ALIGNMENT)); + mi_assert_internal(_mi_align_down_ptr(start-1,MI_PAGE_META_ALIGNMENT) == os_start); + mi_assert_internal((uint8_t*)_mi_aligned_ptr_page0(start) < os_start + min_page_meta); + memid->initially_committed = true; // so we don't commit again + #else if (os_align) { // note: slice_count already includes the page - mi_assert_internal(slice_count >= mi_slice_count_of_size(block_size) + mi_slice_count_of_size(page_alignment)); start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size, block_alignment, page_alignment /* align offset */, commit, allow_large, req_arena, memid); + mi_assert_internal(_mi_is_aligned(start + page_alignment, block_alignment)); } else { start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size, page_alignment, 0 /* align offset */, commit, allow_large, req_arena, memid); } + #endif + if (start!=NULL) { mi_heap_stat_increase(heap,pages_os_allocated,1); } } if (start == NULL) return NULL; mi_assert_internal(_mi_is_aligned(start, MI_PAGE_ALIGN)); - mi_assert_internal(!os_align || _mi_is_aligned(start + page_alignment, block_alignment)); return start; } -static size_t mi_page_block_start(size_t block_size, bool os_align) +// Only used for non-separate pages +mi_decl_maybe_unused static size_t mi_page_block_start(size_t block_size, bool os_align) { size_t offset; #if MI_GUARDED @@ -835,6 +894,7 @@ static size_t mi_page_block_start(size_t block_size, bool os_align) else if (_mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) { // naturally align power-of-2 blocks up to MI_PAGE_MAX_START_BLOCK_ALIGN2 size (4KiB) offset = _mi_align_up(mi_page_info_size(), block_size); + if (block_size < 64) { offset += 3*block_size; } } else if (block_size != 0 && (block_size % MI_PAGE_OSPAGE_BLOCK_ALIGN2) == 0) { // also align large pages that are a multiple of MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4KiB) @@ -847,17 +907,60 @@ static size_t mi_page_block_start(size_t block_size, bool os_align) return _mi_align_up(offset,MI_MAX_ALIGN_SIZE); } + // Free a page without modifying page_bin stats static void mi_arenas_page_free_prim(mi_page_t* page, mi_subproc_t* subproc, mi_arena_pages_t* arena_pages); +static mi_page_t* mi_arena_page_meta(mi_memid_t memid_slice, const void* slice_start) { + #if MI_PAGE_META_IS_ALIGNED + if (memid_slice.memkind == MI_MEM_ARENA || mi_memid_is_os(memid_slice)) { + // ensure the meta data is committed + if (memid_slice.memkind == MI_MEM_ARENA) { + mi_arena_t* const arena = memid_slice.mem.arena.arena; + uint8_t* const meta_slices = (uint8_t*)_mi_align_down_ptr(slice_start,MI_PAGE_META_ALIGNMENT); + mi_assert_internal(meta_slices >= mi_arena_start(arena)); + const size_t meta_slice_index = (meta_slices - mi_arena_start(arena)) / MI_ARENA_SLICE_SIZE; + if mi_unlikely(mi_bitmap_is_clear(arena->slices_committed, meta_slice_index)) { + // try to commit all page meta slices now + const size_t meta_slice_count = mi_arena_page_meta_aligned_slice_count(); + // the following assertion does not hold in a concurrent setting.. + // mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, meta_slice_index, meta_slice_count)); + const size_t commit_size = meta_slice_count * MI_ARENA_SLICE_SIZE; + if (!mi_arena_commit(arena->subproc, arena, meta_slices, commit_size, NULL, 0)) { + // if the commit fails return NULL + return NULL; + } + // set the commit bits + mi_bitmap_setN(arena->slices_committed, meta_slice_index, meta_slice_count, NULL); + } + } + mi_page_t* const page_meta = _mi_aligned_ptr_page0(slice_start); + return page_meta; + } + #else + if (memid_slice.memkind == MI_MEM_ARENA) { + MI_UNUSED(slice_start); + mi_arena_t* const arena = memid_slice.mem.arena.arena; + if (arena->pages_meta != NULL) { + mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0); + return &arena->pages_meta[memid_slice.mem.arena.slice_index]; + } + } + #endif + return NULL; +} + // Allocate a fresh page static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool commit) { const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); + const bool singleton = (os_align || block_size > MI_LARGE_MAX_OBJ_SIZE); + const size_t max_page_meta_count = (singleton && slice_count > 2 ? 2 : slice_count); + const size_t alloc_size = mi_size_of_slices(slice_count); mi_memid_t memid = _mi_memid_none(); mi_arena_pages_t* arena_pages = NULL; - uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,block_size,block_alignment,os_align,commit,&memid,&arena_pages); + uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,max_page_meta_count,block_alignment,os_align,commit,&memid,&arena_pages); if (!slice_start) return NULL; // guard page at the end of mimalloc page? @@ -874,40 +977,58 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou size_t block_start = 0; // allocate page meta info at the arena start? - if (memid.memkind == MI_MEM_ARENA) { - mi_arena_t* const arena = memid.mem.arena.arena; - if (arena->pages_meta != NULL) { - mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0); - mi_page_t* const page_meta = &arena->pages_meta[memid.mem.arena.slice_index]; - #if MI_PAGE_META_ALIGNED_FREE_SMALL - // Only pre-zeroed in this configuration; see the `pages_meta` note in - // `mi_arena_initialize`. Otherwise an unclaimed entry may hold anything - // (external memory we were not told was zero) until the memzero below. - mi_assert_internal(page_meta->block_size == 0); - // if `block_size <= MI_SMALL_SIZE_MAX` we put the page info in front of the slice, - // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`) - if (block_size > MI_SMALL_SIZE_MAX) + mi_page_t* const page_meta = mi_arena_page_meta(memid,slice_start); + if (page_meta!=NULL) { + #if MI_PAGE_META_IS_ALIGNED || MI_PAGE_META_SMALL_IS_ALIGNED + // Only pre-zeroed in these configurations; see the `pages_meta` note in + // `mi_arena_initialize`. Otherwise an unclaimed entry may hold anything + // (external memory we were not told was zero) until the memzero below. + mi_assert_internal(page_meta->block_size == 0); + #endif + #if MI_PAGE_META_SMALL_IS_ALIGNED + // if `block_size <= MI_SMALL_MAX_OBJ_SIZE` we put the page info in front of the slice, + // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`) + if (!os_align && block_size <= MI_SMALL_MAX_OBJ_SIZE) { + // put page info in front of the slice + page = (mi_page_t*)slice_start; + block_start = mi_page_block_start(block_size, os_align); + } + else + #endif + { + page_meta_is_separate = true; + page = page_meta; + block_start = 0; + #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET) + #define MI_PAGE_BLOCK_START_MAX_OFFSET (8*MI_INTPTR_BITS) /* 512 */ #endif + if (block_size >= MI_SIZE_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && + _mi_is_power_of_two(block_size)) { - page = page_meta; - page_meta_is_separate = true; - block_start = 0; - #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET) - #define MI_PAGE_BLOCK_START_MAX_OFFSET (8*MI_INTPTR_BITS) /* 512 */ - #endif - if (block_size >= MI_INTPTR_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && _mi_is_power_of_two(block_size)) { - block_start += block_size; - } - _mi_memzero_aligned(page, sizeof(*page)); + block_start = _mi_align_up(mi_page_info_size(), block_size); // to maintain natural alignment + if (block_size < 64) { block_start += 3*block_size; } } + #if MI_PAGE_META_IS_ALIGNED || MI_PAGE_META_SMALL_IS_ALIGNED + mi_assert_internal(page->block_size == 0); + #endif + _mi_memzero_aligned(page, sizeof(*page)); } } if (page == NULL) { + #if MI_PAGE_META_IS_ALIGNED + // can only happen on failing to commit the page meta info + _mi_arenas_free(_mi_theap_subproc(theap),slice_start,alloc_size,memid); + return NULL; + #else // put page meta info in front of the slice page = (mi_page_t*)slice_start; block_start = mi_page_block_start(block_size, os_align); + #endif + } + mi_assert_internal(block_size < MI_MAX_ALIGN_SIZE || block_start % MI_MAX_ALIGN_SIZE == 0); + if (_mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) { + mi_assert_internal(block_start % block_size == 0); // natural alignment (see also alloc_aligned.c) } - mi_assert_internal(block_start % MI_MAX_ALIGN_SIZE == 0); // commit first block? size_t commit_size = 0; @@ -956,10 +1077,8 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou // initialize the page start uint8_t* const start = slice_start + block_start; mi_assert_internal(start > (uint8_t*)page); - const size_t offset = start - (uint8_t*)page; - mi_assert_internal((offset % MI_MAX_ALIGN_SIZE) == 0 && (offset / MI_MAX_ALIGN_SIZE) <= UINT32_MAX); - page->page_ma_offset = (uint32_t)(offset / MI_MAX_ALIGN_SIZE); - + page->page_offset = start - (uint8_t*)page; + // initialize page meta-data page->reserved = (uint16_t)reserved; page->block_size = block_size; @@ -971,12 +1090,33 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou page->heap = _mi_theap_heap(theap); mi_page_set_theap(page,theap); - // mi_assert_internal(mi_page_theap(page) == _mi_heap_theap_peek(page->heap)); + // mi_assert_internal(mi_page_theap(page) == _mi_heap_theap_peek(page->heap)) + + #if MI_PAGE_META_IS_ALIGNED + mi_assert_internal(page_meta!=NULL); + mi_atomic_store_ptr_release(mi_page_t,&page_meta->self,page); + if (slice_count > 1) { + // at least two for large singleton blocks as guard pages can have a large offset beyond a single slice + for(size_t i = 1; i < max_page_meta_count; i++) { + mi_assert_internal(page_meta[i].block_size == 0); + mi_atomic_store_ptr_release(mi_page_t,&page_meta[i].self,page); + } + } + #if MI_DEBUG>1 + mi_page_t* pstart = _mi_aligned_ptr_page0(slice_start); + mi_assert_internal(mi_atomic_load_ptr_acquire(mi_page_t,&pstart->self)==page); + if (reserved>1) { + mi_page_t* pend = _mi_aligned_ptr_page0(slice_start + (slice_count*MI_ARENA_SLICE_SIZE) - 1); + mi_assert_internal(mi_atomic_load_ptr_acquire(mi_page_t,&pend->self)==page); + } + #endif + #endif mi_assert_internal(page->free==NULL); mi_assert_internal(page_meta_is_separate == mi_page_meta_is_separated(page)); mi_assert_internal(mi_page_slice_start(page) == slice_start); mi_assert_internal(mi_page_size(page) <= page_noguard_size); + // own it mi_page_claim_ownership(page); @@ -989,6 +1129,7 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou // register in the page map if mi_unlikely(!_mi_page_map_register(page)) { + if (arena_pages==NULL) { mi_heap_stat_decrease(page->heap, pages_os_allocated, 1); } mi_arenas_page_free_prim(page, _mi_theap_subproc(theap), arena_pages); return NULL; } @@ -1017,7 +1158,7 @@ static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_c // 2. find a free block, potentially allocating a new arena const long commit_on_demand = mi_option_get(mi_option_page_commit_on_demand); - const bool commit = (slice_count <= mi_slice_count_of_size(mi_page_min_commit_size()) || // always commit small pages + const bool commit = (mi_page_min_commit_size() >= slice_count * MI_ARENA_SLICE_SIZE || // always commit small pages (slice_count >= mi_slice_count_of_size(UINT16_MAX * _mi_os_page_size())) || // always commit pages too large to hold a 32-bit slice_committed (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0)); page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, 1, commit); @@ -1035,8 +1176,12 @@ static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_c // Allocate a page containing one block (very large, or with large alignment) static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment) { + #if MI_PAGE_META_IS_ALIGNED + const size_t info_size = 0; + #else const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); - const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size()); + const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size()); + #endif #if MI_SECURE < 2 const size_t slice_count = mi_slice_count_of_size(info_size + block_size); #else @@ -1178,8 +1323,11 @@ static void mi_arenas_page_free_ex(mi_page_t* page, mi_theap_t* current_theapx, mi_subproc_t* const subproc = heap->subproc; mi_arena_pages_t* arena_pages = NULL; if (page->memid.memkind == MI_MEM_ARENA) { mi_page_arena_pages(page, NULL, NULL, &arena_pages); } + else { mi_heap_stat_decrease(heap, pages_os_allocated, 1); } // (here and not in `mi_arenas_page_free_prim`, which cannot read the heap) mi_theapx_stat_decrease(heap, current_theapx, page_bins[_mi_page_stats_bin(page)], 1); mi_theapx_stat_decrease(heap, current_theapx, pages, 1); + _mi_page_free_collect(page,false); // update used count for cross-thread free's (nothing is purged anymore, so this cannot un-purge) + _mi_page_update_stats_for(page,current_theapx); // and update the stats if (unabandon) { _mi_arenas_page_unabandon(page, current_theapx); // for an OS page this is where it is unpublished } @@ -1210,6 +1358,11 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) { mi_assert_internal(_mi_theap_can_touch(current_theapx)); // mi_assert_internal(current_theap == _mi_page_associated_theap(page)); + // note: somewhat expensive to update here, but might be good as then we attribute + // the current allocations/frees to the current thread/theap. Otherwise it might be + // reclaimed later in another thread/theap and those allocations/frees get attributed there... + _mi_page_update_stats_for(page, current_theapx); + // add to abandoned? // (not for a heap that is being released: its teardown claims every page through `pages`, see `mi_heap_release_pages`) mi_heap_t* heap = mi_page_heap(page); @@ -1254,6 +1407,7 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) { if (page->next != NULL) { page->next->prev = page; } heap->os_abandoned_pages = page; } + mi_theapx_stat_increase(heap, current_theapx, pages_os_abandoned, 1); } mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1); mi_abandoned_page_unown(page, current_theapx); @@ -1322,6 +1476,7 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) { // if not an arena page, remove it from the heap os pages list. That unpublishes it from the heap: when we // are about to free the page this is our last access to the heap (see `mi_arenas_page_free_prim`). if (page->memid.memkind != MI_MEM_ARENA) { + mi_theapx_stat_decrease(heap, current_theapx, pages_os_abandoned, 1); // (before the unlink: that is the last access to the heap) mi_lock(&heap->os_abandoned_pages_lock) { if (page->prev != NULL) { page->prev->next = page->next; } if (page->next != NULL) { page->next->prev = page->prev; } @@ -1681,13 +1836,17 @@ static size_t mi_arena_pages_size(size_t slice_count, size_t* bitmap_base) { return size; } +static mi_arena_t* mi_arena_info(void* area) { + return (mi_arena_t*)((uint8_t*)area + mi_size_of_slices(mi_arena_page_meta_aligned_slice_count())); +} + static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_base, size_t* pages_meta_offset) { if (slice_count == 0) slice_count = MI_BCHUNK_BITS; mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0); - const size_t base_size = _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE); + const size_t base_size = mi_size_of_slices(mi_arena_page_meta_aligned_slice_count()) + _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE); const size_t bitmaps_count = 4; // commit, dirty, purge, and pages (the abandoned bitmaps are allocated on demand) const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL) + mi_bbitmap_size(slice_count, NULL); // + free - #if MI_PAGE_META_IS_SEPARATED + #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNED const size_t pages_size = slice_count * sizeof(mi_page_t); #else const size_t pages_size = 0; @@ -1786,7 +1945,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, int numa_node, bool exclusive, mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) { - mi_assert_internal(_mi_is_aligned(start,MI_ARENA_SLICE_ALIGN)); + mi_assert_internal(_mi_is_aligned(start,MI_ARENA_ALIGNMENT)); mi_assert_internal(mi_size_of_slices(slice_count)>=MI_ARENA_MIN_SIZE); if (slice_count > MI_BITMAP_MAX_BIT_COUNT) { // 16 GiB for now @@ -1806,9 +1965,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, // _mi_warning_message("cannot use OS memory since it is too large with respect to the maximum object size (size %zu MiB, meta-info slices %zu, maximum object slices are %zu)", mi_size_of_slices(slice_count)/MI_MiB, info_slices, MI_ARENA_MAX_CHUNK_OBJ_SLICES); // return NULL; // } - - mi_arena_t* arena = (mi_arena_t*)start; - + // commit & zero if needed if (!memid.initially_committed) { size_t commit_size = mi_size_of_slices(info_slices); @@ -1816,10 +1973,10 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, if (!memid.is_pinned) { commit_size -= _mi_os_secure_guard_page_size(); } bool ok = false; if (commit_fun != NULL) { - ok = (*commit_fun)(true /* commit */, arena, commit_size, NULL, commit_fun_arg); + ok = (*commit_fun)(true /* commit */, start, commit_size, NULL, commit_fun_arg); } else { - ok = _mi_os_commit(subproc, arena, commit_size, NULL); + ok = _mi_os_commit(subproc, start, commit_size, NULL); } if (!ok) { _mi_warning_message("unable to commit meta-data for OS memory"); @@ -1829,25 +1986,28 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, else if (!memid.is_pinned) { // if MI_SECURE, set a guard page at the end of the arena info // todo: this does not respect the commit_fun as the memid is of external memory - _mi_os_secure_guard_page_set_before(subproc, (uint8_t*)arena + mi_size_of_slices(info_slices), memid); + _mi_os_secure_guard_page_set_before(subproc, (uint8_t*)start + mi_size_of_slices(info_slices), memid); } if (!memid.initially_zero) { size_t zero_size = mi_size_of_slices(info_slices) - _mi_os_secure_guard_page_size(); - #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_ALIGNED_FREE_SMALL + #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNED && !MI_PAGE_META_SMALL_IS_ALIGNED // Only the header and the bitmaps have to start out zeroed. `pages_meta` is one // mi_page_t per slice -- ~2.5 MiB per GiB of arena -- and every entry is zeroed // when its slice is claimed (`mi_arenas_page_alloc_fresh`); no one reads an entry // for a slice that was never claimed. Zeroing it here would fault in megabytes of - // meta-info the arena may never use. (Under MI_PAGE_META_ALIGNED_FREE_SMALL, + // meta-info the arena may never use. (Under MI_PAGE_META_SMALL_IS_ALIGNED, // `mi_arena_page_at_slice` does read `block_size` of unclaimed entries, so the - // array must be zeroed up front and this shortcut does not apply.) + // array must be zeroed up front and this shortcut does not apply; with + // MI_PAGE_META_IS_ALIGNED there is no `pages_meta` array in the arena info at all.) mi_assert_internal(pages_meta_offset <= zero_size); if (pages_meta_offset <= zero_size) { zero_size = pages_meta_offset; } #endif - _mi_memzero(arena, zero_size); + _mi_memzero(start, zero_size); } // init + mi_arena_t* arena = mi_arena_info(start); + arena->start = start; arena->subproc = subproc; arena->memid = memid; arena->is_exclusive = exclusive; @@ -1875,7 +2035,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { mi_atomic_store_ptr_relaxed(mi_bitmap_t, &arena->pages_main.pages_abandoned[i], NULL); // allocated on first abandon } - #if MI_PAGE_META_IS_SEPARATED + #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNED arena->pages_meta = (mi_page_t*)base; base += (slice_count * sizeof(mi_page_t)); #else @@ -1884,7 +2044,23 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, mi_assert_internal(mi_size_of_slices(info_slices) >= (size_t)(base - mi_arena_start(arena))); // reserve our meta info (and reserve slices outside the memory area) + #if MI_PAGE_META_IS_ALIGNED + for(size_t i = 0; i < arena->slice_count; i += MI_PAGE_META_ALIGNED_COUNT) { + // set all free slices (and skip the slices reserved for the page meta info) + const size_t meta_slices = (i==0 ? info_slices : mi_arena_page_meta_aligned_slice_count()); + const size_t start_idx = (i + meta_slices); + size_t count = MI_PAGE_META_ALIGNED_COUNT - meta_slices; + if (start_idx < arena->slice_count) { + if (count + start_idx > arena->slice_count) { + count = arena->slice_count - start_idx; + mi_assert_internal(count > 0); + } + mi_bbitmap_unsafe_setN(arena->slices_free, start_idx, count); + } + } + #else mi_bbitmap_unsafe_setN(arena->slices_free, info_slices /* start */, arena->slice_count - info_slices); + #endif if (memid.initially_committed) { mi_bitmap_unsafe_setN(arena->slices_committed, 0, arena->slice_count); } @@ -1903,11 +2079,11 @@ static bool mi_manage_os_memory_ex2(mi_subproc_t* subproc, void* start, size_t s mi_assert(start!=NULL); if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); } if (start==NULL) return false; - if (!_mi_is_aligned(start, MI_ARENA_SLICE_SIZE)) { + if (!_mi_is_aligned(start, MI_ARENA_ALIGNMENT)) { // we can align the start since the memid tracks the real base of the memory. - void* const aligned_start = _mi_align_up_ptr(start, MI_ARENA_SLICE_SIZE); + void* const aligned_start = _mi_align_up_ptr(start, MI_ARENA_ALIGNMENT); const size_t diff = (uint8_t*)aligned_start - (uint8_t*)start; - if (diff >= size || (size - diff) < MI_ARENA_SLICE_SIZE) { + if (diff >= size || (size - diff) < MI_ARENA_ALIGNMENT) { _mi_warning_message("after alignment, the size of the arena becomes too small (memory at %p with size %zu)\n", start, size); return false; } @@ -1998,8 +2174,8 @@ static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool com return ENOMEM; } mi_memid_t memid; - void* start = _mi_os_alloc_aligned(subproc, size, MI_ARENA_SLICE_ALIGN, commit, allow_large, &memid); - if (start == NULL) return ENOMEM; + void* start = _mi_os_alloc_aligned(subproc, size, MI_ARENA_ALIGNMENT, commit, allow_large, &memid); + if (start == NULL) return ENOMEM; if (!mi_manage_os_memory_ex2(subproc, start, size, -1 /* numa node */, exclusive, memid, NULL, NULL, arena_id)) { _mi_os_free_ex(subproc, start, size, commit, memid); _mi_verbose_message("failed to reserve %zu KiB memory\n", _mi_divide_up(size, 1024)); @@ -2078,7 +2254,7 @@ static void mi_debug_color(char* buf, size_t* k, mi_ansi_color_t color) { static int mi_page_commit_usage(mi_page_t* page) { const size_t committed_size = mi_page_committed(page); - const size_t used_size = page->used * mi_page_block_size(page); + const size_t used_size = mi_page_used(page) * mi_page_block_size(page); return (int)(used_size * 100 / committed_size); } @@ -2113,6 +2289,11 @@ static size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena, if (bit_of_page > 0) { c = '-'; } // else if (_mi_meta_is_meta_page(arena->subproc,start)) { c = 'm'; color = MI_GRAY; } else if (slice_index + bit < arena->info_slices) { c = 'i'; color = MI_GRAY; } + #if MI_PAGE_META_IS_ALIGNED + else if ((slice_index % MI_PAGE_META_ALIGNED_COUNT) == 0 && (size_t)bit <= mi_arena_page_meta_aligned_slice_count()) { + { c = 'i'; color = MI_GRAY; } + } + #endif // else if (mi_bitmap_is_setN(arena->pages_purge, slice_index + bit, NULL)) { c = '*'; } else if (mi_bbitmap_is_setN(arena->slices_free, slice_index+bit,1)) { if (mi_bitmap_is_set(arena->slices_purge, slice_index + bit)) { c = '~'; color = MI_ORANGE; } @@ -2500,10 +2681,8 @@ static int mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force) // expired yet? mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire); - if (!force) { - if (expire==0) return -1; - if (expire > now) return 0; - } + if (expire==0) return -1; + if (!force && expire > now) return 0; // reset expire mi_atomic_storei64_release(&arena->purge_expire, (mi_msecs_t)0); @@ -2556,12 +2735,24 @@ void _mi_arenas_purge_now(mi_subproc_t* subproc) { // allow only one thread to purge at a time (todo: allow concurrent purging?) static mi_atomic_guard_t mi_arenas_purge_guard; -// Release the guard for a thread that is gone: the one that held it across fork() is not in the child, and at process exit -// on Windows every other thread is terminated before the detach callback runs. Nothing else would ever release it, and a -// forced purge waits for it. Returns true if it was held: that pass had reset the `purge_expire` of a sub-process, and -// it is not there to put back what was still pending (see `_mi_arenas_try_purge`). -bool _mi_arenas_purge_guard_reset(void) { - return (mi_atomic_exchange_acq_rel(&mi_arenas_purge_guard, (uintptr_t)0) != 0); +// No purge pass runs across a fork(): the prepare handler takes the guard, the parent and the child release it. A pass +// changes the protection of arena memory (a decommit is `mprotect(PROT_NONE)` in a debug or secure build) and the bits that +// describe that memory in separate steps, and a fork in between gives the child the one without the other. +// Taken last in the prepare handler, under its locks: a pass takes none of them and waits for no one, so it ends. +void _mi_arenas_purge_guard_acquire(void) { + size_t spin = 0; + uintptr_t expected = 0; + while (!mi_atomic_cas_weak_acq_rel(&mi_arenas_purge_guard, &expected, (uintptr_t)1)) { + expected = 0; + if (spin < 256) { mi_atomic_pause(); spin++; } + else { _mi_prim_thread_yield(); } + } +} + +// Also for a holder that is gone: at process exit on Windows every other thread is terminated before the detach callback +// runs. Nothing else would ever release it then, and a forced purge waits for it. +void _mi_arenas_purge_guard_release(void) { + mi_atomic_store_release(&mi_arenas_purge_guard, (uintptr_t)0); } // Make a purge pass over the arenas of `subproc` due at `expire`, unless one is due before that already. The scavenger @@ -2697,7 +2888,16 @@ static mi_page_t* mi_heap_visit_page_claim(mi_heap_visit_info_t* vinfo, mi_arena for (;;) { if (!mi_bitmap_clear(pages, slice_index)) return NULL; // freed by a concurrent `mi_free` // pinned: now it is a page of this heap and stays one while we hold the bit - page = mi_arena_page_at_slice(arena, slice_index); + #if MI_PAGE_META_IS_ALIGNED + if mi_unlikely(_mi_process_is_forked_child) { + // not through `mi_arena_page_at_slice`: in a debug build that checks the page map, whose entry may be the very thing that is missing (see below) + page = mi_atomic_load_ptr_acquire(mi_page_t, &_mi_aligned_ptr_page0(mi_arena_slice_start(arena, slice_index))->self); + } + else + #endif + { + page = mi_arena_page_at_slice(arena, slice_index); + } #if MI_DEBUG > 0 if (mi_atomic_load_acquire(&mi_debug_stall_in_heap_delete_claim) == 1) { mi_atomic_store_release(&mi_debug_stall_in_heap_delete_claim, (uintptr_t)2); // signal: pinned, not yet claimed @@ -2708,7 +2908,7 @@ static mi_page_t* mi_heap_visit_page_claim(mi_heap_visit_info_t* vinfo, mi_arena // After a multi-threaded fork() the child may inherit a torn snapshot of a page that another // thread was allocating or freeing: the bit propagated but the page-map entry or the owned bit // did not, and that thread is gone. Re-derive what we can and take the page. - if (mi_page_start(page) == NULL) return NULL; // the page struct never made it across: leave it unpublished + if (page == NULL || page->block_size == 0 || mi_page_start(page) == NULL) return NULL; // the page struct never made it across: leave it unpublished if (_mi_safe_ptr_page(mi_page_start(page)) != page && !_mi_page_map_register(page)) return NULL; mi_page_claim_ownership(page); // ours now, whether or not the dead thread held it mi_bitmap_set(pages, slice_index); @@ -2847,7 +3047,12 @@ static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* are mi_assert_internal(mi_page_is_abandoned(page)); _mi_arenas_page_unabandon(page,theap); - if (page->used==0) { + // Fold in the blocks that other threads freed: an abandoned page keeps at least the last of them on its + // thread-free list (`_mi_page_free_collect_partly` never collects the head), and they still count as used. + // (no un-purging: nothing is allocated from this page here) + _mi_page_free_collect_no_unpurge(page, false); + + if (mi_page_used(page)==0) { // free the page _mi_arenas_page_free(page, theap); } @@ -2856,7 +3061,10 @@ static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* are _mi_page_unguard_all(page); // remove potential interior guard pages #endif // destroy the page - page->used=0; // note: invariant `|local_free| + |free| == reserved - used` does not hold in this case + // Drop what is (still) on the thread-free list first: `_mi_arenas_page_free` collects it, and with the used + // count reset that reads as a corrupted list (more blocks freed than were in use). + mi_atomic_store_release(&page->xthread_free, mi_tf_create(NULL, true /* owned */)); + mi_page_used_reset(page); // note: invariant `|local_free| + |free| == reserved - used` does not hold in this case _mi_arenas_page_free(page, theap); } else if (page->memid.memkind != MI_MEM_ARENA) { diff --git a/src/bitmap.h b/src/bitmap.h index f4d7a2954..dff08c153 100644 --- a/src/bitmap.h +++ b/src/bitmap.h @@ -71,7 +71,7 @@ typedef size_t mi_bfield_t; #define MI_BCHUNK_FIELDS (MI_BCHUNK_BITS / MI_BFIELD_BITS) // 8 on both 64- and 32-bit -// some compiler (msvc in C mode) cannot have expressions in the alignment attribute +// some compilers (msvc in C mode) cannot have expressions in the alignment attribute #if MI_BCHUNK_SIZE==64 #define mi_decl_bchunk_align mi_decl_align(64) #elif MI_BCHUNK_SIZE==32 @@ -95,11 +95,11 @@ typedef mi_bchunk_t mi_bchunkmap_t; #define MI_BITMAP_MAX_CHUNK_COUNT (MI_BCHUNKMAP_BITS) #define MI_BITMAP_MIN_CHUNK_COUNT (1) -#if MI_SIZE_BITS > 32 -#define MI_BITMAP_DEFAULT_CHUNK_COUNT (64) // 2 GiB on 64-bit -- this is for the page map -#else +// #if MI_SIZE_BITS > 32 +// #define MI_BITMAP_DEFAULT_CHUNK_COUNT (128) // 4 GiB on 64-bit -- this is for the page map +// #else #define MI_BITMAP_DEFAULT_CHUNK_COUNT (1) -#endif +// #endif #define MI_BITMAP_MAX_BIT_COUNT (MI_BITMAP_MAX_CHUNK_COUNT * MI_BCHUNK_BITS) // 16 GiB arena #define MI_BITMAP_MIN_BIT_COUNT (MI_BITMAP_MIN_CHUNK_COUNT * MI_BCHUNK_BITS) // 32 MiB arena #define MI_BITMAP_DEFAULT_BIT_COUNT (MI_BITMAP_DEFAULT_CHUNK_COUNT * MI_BCHUNK_BITS) // 2 GiB arena diff --git a/src/free.c b/src/free.c index 295ed7138..56f5d3266 100644 --- a/src/free.c +++ b/src/free.c @@ -13,11 +13,25 @@ terms of the MIT license. A copy of the license can be found in the file #endif // forward declarations -static void mi_check_padding(const mi_page_t* page, const mi_block_t* block); -static bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block); +mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size); +mi_decl_nodiscard static bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block); static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool was_guarded); -static void mi_stat_free(const mi_page_t* page, const mi_block_t* block); +// A purged block (see the "Page hole purging" section in `page.c`) is free but held off every free list, and its +// memory is discarded: it reads back as zeros, so the padding canary cannot tell that it was freed before (a zero +// canary reads as a buffer overflow) and a walk over the free lists does not find it. Only checked where double frees +// are checked at all, and only on the owning thread (the `purged` bitmap is not atomic). +static inline bool mi_check_purged_double_free(const mi_page_t* page, const mi_block_t* block) { + #if MI_PADDING || MI_SECURE>=3 + if mi_unlikely(mi_page_block_is_purged(page, block)) { + _mi_error_message(EAGAIN, "double free detected of block %p with size %zu\n", block, mi_page_block_size(page)); + return false; + } + #else + MI_UNUSED(page); MI_UNUSED(block); + #endif + return true; +} // ------------------------------------------------------ // Free @@ -25,29 +39,28 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block); // regular free of a (thread local) block pointer // fast path written carefully to prevent spilling on the stack -static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool track_stats, bool check_full) +static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool check_full) { - MI_UNUSED(was_guarded); // checks - if mi_unlikely(mi_check_is_double_free(page, block)) return; - if (!was_guarded) { mi_check_padding(page, block); } - if (track_stats) { mi_stat_free(page, block); } + size_t usable_size; + if mi_unlikely(!mi_check_purged_double_free(page, block)) return; // before the padding check, which would read discarded memory + if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; + if mi_unlikely(!mi_check_double_free(page,block)) return; // usually checked with padding + + // mi_stat_free(page, block); + mi_track_free_size(block, usable_size); + #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN - size_t dbgsize = mi_page_block_size(page); - if (dbgsize > 1*MI_MiB) { dbgsize = 1*MI_MiB; } + const size_t dbgsize = (usable_size > MI_MiB ? MI_MiB : usable_size); _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize); #endif - if (track_stats) { mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded)); } // faster then mi_usable_size as we already know the page and that p is unaligned - + // actual free: push on the local free list mi_block_set_next(page, block, page->local_free); page->local_free = block; - #if defined(__clang__) && defined(__aarch64__) - if mi_unlikely(page->used-- == 1) // better code on arm64 than using `--page->used == 0` - #else - if mi_unlikely(--page->used == 0) - #endif - { + const size_t used = mi_xused_dec_used(&page->xused); + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + if mi_unlikely(used == 0) { if (page->retire_expire==0) { // no need to re-retire retired pages (happens when we alloc/free one block repeatedly in an empty page) _mi_page_retire(page); } @@ -58,22 +71,22 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool } // Forward declaration for multi-threaded collect -static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept; +static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim) mi_attr_noexcept; // Free a block multi-threaded -static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool allow_collect) mi_attr_noexcept +static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool allow_reclaim) mi_attr_noexcept { - // todo: we cannot safely check for double free in _mt -- should check when collecting the thread_free list - if (!was_guarded) { mi_check_padding(page, block); } // checking padding is safe for mt + size_t usable_size; + if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; // checking padding is safe for mt + // adjust stats (after padding check ) - mi_stat_free(page, block); // stat_free may access the padding - mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded)); + // mi_stat_free(page, block); // stat_free may access the padding + mi_track_free_size(block, usable_size); // _mi_padding_shrink(page, block, sizeof(mi_block_t)); #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading if (!was_guarded) { - size_t dbgsize = mi_usable_size(block); - if (dbgsize > 1*MI_MiB) { dbgsize = 1*MI_MiB; } + const size_t dbgsize = (usable_size > MI_MiB ? MI_MiB : usable_size); _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize); } #endif @@ -82,19 +95,16 @@ static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was mi_thread_free_t tf_new; mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free); do { - mi_block_set_next(page, block, mi_tf_block(tf_old)); - const bool new_owned = (allow_collect ? true : mi_tf_is_owned(tf_old)); // if allow collection then always try to claim it if the page is abandoned - tf_new = mi_tf_create(block, new_owned); + mi_block_set_next(page, block, mi_tf_block(tf_old)); + tf_new = mi_tf_create(block, true /* try to own it */ ); } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); // todo: release is enough? // and atomically try to collect the page if it was abandoned - if (allow_collect) { - const bool is_owned_now = !mi_tf_is_owned(tf_old); - if (is_owned_now) { - mi_assert_internal(mi_page_is_abandoned(page)); - mi_free_try_collect_mt(page,block); - } - } + const bool is_owned_now = !mi_tf_is_owned(tf_old); + if (is_owned_now) { + mi_assert_internal(mi_page_is_abandoned(page)); + mi_free_try_collect_mt(page,block,allow_reclaim); + } } @@ -102,15 +112,23 @@ static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was // note: this can be called from `mi_free_generic_mt` where a non-owning thread accesses the // `page_woffset` and `block_size` fields; however these are constant and the page won't be // deallocated (as the block we are freeing keeps it alive) and thus safe to read concurrently. -mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) { +static inline mi_block_t* mi_page_ptr_unalign_ex(const mi_page_t* page, const void* p, size_t* poffset) { mi_assert_internal(page!=NULL && p!=NULL); const size_t diff = (uint8_t*)p - mi_page_start(page); const size_t block_size = mi_page_block_size(page); - const size_t adjust = (_mi_is_power_of_two(block_size) ? diff & (block_size - 1) : diff % block_size); + size_t adjust = diff & (block_size - 1); + if mi_unlikely(!_mi_is_power_of_two(block_size)) { + adjust = diff % block_size; + } + if (poffset!=NULL) { *poffset = adjust; } return (mi_block_t*)((uintptr_t)p - adjust); } +mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) { + return mi_page_ptr_unalign_ex(page,p,NULL); +} + static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, const void* p ) { mi_assert(_mi_page_ptr_unalign(page,p) == (mi_block_t*)p); // should never be an interior pointer #if MI_SECURE > 0 @@ -122,57 +140,56 @@ static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, con #endif } -// forward declaration for a MI_GUARDED build -#if MI_GUARDED -static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration -static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) { - if (mi_block_ptr_is_guarded(block, p)) { - mi_block_unguard(page, block, p); - return true; +static inline mi_block_t* mi_page_ptr_block_check(mi_page_t* page, void* p, bool* was_guarded) mi_attr_noexcept { + MI_UNUSED(was_guarded); + if mi_likely(!mi_page_has_interior_pointers(page)) { + return mi_validate_block_from_ptr(page,p); } else { - return false; + size_t offset; + mi_block_t* const block = mi_page_ptr_unalign_ex(page,p,&offset); + #if MI_GUARDED || MI_PROFILE + if (offset >= sizeof(mi_block_t)) { + #if MI_PROFILE + if (block->next == MI_BLOCK_TAG_PROFILED) { + _mi_page_profile_on_free(page,block,p); + } + else + #endif + #if MI_GUARDED + if (block->next == MI_BLOCK_TAG_GUARDED) { + _mi_page_block_unguard(page, block, p); + *was_guarded = true; + } + else + #endif + { /* nothing */ } + } + #endif + return block; } } -#else -static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) { - MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p); - return false; -} -#endif - - -// outlined slow paths for pages with profiled samples; tail-called so the -// common generic_local/generic_mt remain leaf functions (no frame, no spills). -static mi_decl_noinline void mi_free_generic_local_prof(mi_page_t* page, void* p) mi_attr_noexcept { - mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); - _mi_prof_free(block); - const bool was_guarded = mi_block_check_unguard(page, block, p); - mi_free_block_local(page, block, was_guarded, true, true); -} -static mi_decl_noinline void mi_free_generic_mt_prof(mi_page_t* page, void* p, bool allow_collect) mi_attr_noexcept { - mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); - _mi_prof_free(block); - const bool was_guarded = mi_block_check_unguard(page, block, p); - mi_free_block_mt(page, block, was_guarded, allow_collect); -} // free a local pointer (page parameter comes first for better codegen) static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_attr_noexcept { mi_assert_internal(p!=NULL && page != NULL); - if mi_unlikely(mi_page_flags(page) & MI_PAGE_HAS_PROF_SAMPLES) { mi_free_generic_local_prof(page, p); return; } - mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); - const bool was_guarded = mi_block_check_unguard(page, block, p); - mi_free_block_local(page, block, was_guarded, true /* track stats */, true /* check for a full page */); + bool was_guarded = false; + mi_block_t* block = mi_page_ptr_block_check(page,p,&was_guarded); + // mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); + // mi_block_check_profiled(page,block,p); + // const bool was_guarded = mi_block_check_unguard(page, block, p); + mi_free_block_local(page, block, was_guarded, true /* check for a full page */); } // free a pointer owned by another thread (page parameter comes first for better codegen) -static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p, bool allow_collect) mi_attr_noexcept { +static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p, bool allow_reclaim) mi_attr_noexcept { mi_assert_internal(p!=NULL && page != NULL); - if mi_unlikely(mi_page_flags(page) & MI_PAGE_HAS_PROF_SAMPLES) { mi_free_generic_mt_prof(page, p, allow_collect); return; } - mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); - const bool was_guarded = mi_block_check_unguard(page, block, p); - mi_free_block_mt(page, block, was_guarded, allow_collect); + bool was_guarded = false; + mi_block_t* block = mi_page_ptr_block_check(page,p,&was_guarded); + // mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); + // mi_block_check_profiled(page,block,p); + // const bool was_guarded = mi_block_check_unguard(page, block, p); + mi_free_block_mt(page, block, was_guarded, allow_reclaim); } // generic free (for runtime integration) @@ -184,40 +201,86 @@ void mi_decl_noinline _mi_free_generic(mi_page_t* page, bool is_local, void* p) // Get the page belonging to a pointer // Does further checks in debug mode to see if this was a valid pointer. -static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg) +static mi_decl_forceinline bool mi_ptr_page_is_valid_ex(const void* p, const char* msg, bool free_small, bool check_p_for_null, mi_page_t** ppage) { - MI_UNUSED_RELEASE(msg); + MI_UNUSED_RELEASE(msg); MI_UNUSED(free_small); #if MI_DEBUG - if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0 && !mi_option_is_enabled(mi_option_guarded_precise)) { + if mi_unlikely(!_mi_is_aligned(p,MI_SIZE_SIZE) && !mi_option_is_enabled(mi_option_guarded_precise)) { _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p); - return NULL; - } - mi_page_t* page = _mi_safe_ptr_page(p); - if (p != NULL && page == NULL) { - _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); + return false; } - return page; - #else - return _mi_ptr_page(p); #endif + + mi_page_t* page; + #if MI_PAGE_META_SMALL_IS_ALIGNED + if (free_small) { + const uintptr_t up = _mi_align_down((uintptr_t)p,MI_SMALL_PAGE_SIZE); // like this for codegen on gcc + if mi_unlikely(up==0 && check_p_for_null) { return false; } + page = (mi_page_t*)up; + } + else + #endif + { + #if MI_PAGE_META_IS_ALIGNED + { page = _mi_aligned_ptr_page0(p); } + #else + { page = _mi_ptr_page(p); } + #endif + if mi_unlikely(page==NULL && check_p_for_null) { + #if MI_DEBUG + if (p!=NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } + #endif + return false; + } + } + #if MI_DEBUG + mi_page_t* const cpage = _mi_checked_ptr_page(p); + if mi_unlikely(cpage==NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } + #endif + + #if MI_PAGE_META_IS_ALIGNED + #if MI_PAGE_META_SMALL_IS_ALIGNED + if (free_small) { mi_assert_internal(NULL == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); } + else + #elif MI_SMALL_PAGE_SIZE == MI_ARENA_SLICE_SIZE && !MI_GUARDED + // for mi_free_small we can avoid a load-acquire (but not when guarded as that may still allocate large blocks) + if (free_small) { mi_assert_internal(page == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); } + else + #endif + { page = mi_atomic_load_ptr_acquire(mi_page_t,&page->self); } + #endif + + mi_assert_internal(page!=NULL); + mi_assert(cpage==page /* page_map lookup should be the same as aligned lookup */ ); + #if !MI_GUARDED + if (free_small) { mi_assert_internal(page->block_size <= mi_good_size(MI_SMALL_SIZE_MAX) /* free small should only be called on small pages */); } + #endif + *ppage = page; + return true; +} + +static mi_decl_forceinline bool mi_ptr_page_is_valid(const void* p, const char* msg, mi_page_t** ppage) { + return mi_ptr_page_is_valid_ex(p,msg,false /* free_small? */, true /* check_p_for_null */, ppage ); +} + +static mi_decl_forceinline mi_page_t* mi_ptr_page_validate(const void* p, const char* msg) { + mi_page_t* page; + return (mi_ptr_page_is_valid(p,msg,&page) ? page : NULL); } // Free a block // Fast path written carefully to prevent register spilling on the stack -static mi_decl_forceinline void mi_free_ex(void* p, mi_page_t* page, size_t* pblock_size, bool allow_collect) +static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t* pblock_size, bool allow_reclaim) { - if mi_unlikely(page==NULL) { // page will be NULL if p==NULL - if (pblock_size!=NULL) { *pblock_size = 0; } - return; - } mi_assert_internal(p!=NULL && page!=NULL); if (pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } - const mi_threadid_t xtid = (_mi_prim_thread_id() ^ mi_page_xthread_id(page)); + const mi_threadid_t ptid = mi_page_xthread_id(page); + const mi_threadid_t xtid = (_mi_prim_thread_id() ^ ptid); if mi_likely(xtid == 0) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0` // thread-local, aligned, and not a full page mi_block_t* const block = mi_validate_block_from_ptr(page,p); - mi_free_block_local(page, block, false /* was guarded */, true /* track stats */, false /* no need to check if the page is full */); + mi_free_block_local(page, block, false /* was guarded */, false /* no need to check if the page is full */); } else if (xtid <= MI_PAGE_FLAG_MASK) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0` // page is local, but is full or contains (inner) aligned blocks; use generic path @@ -227,53 +290,44 @@ static mi_decl_forceinline void mi_free_ex(void* p, mi_page_t* page, size_t* pbl else if ((xtid & MI_PAGE_FLAG_MASK) == 0) { // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0` // blocks are aligned (and not a full page); push on the thread_free list mi_block_t* const block = mi_validate_block_from_ptr(page,p); - mi_free_block_mt(page,block,false /* was_guarded */, allow_collect); + mi_free_block_mt(page,block,false /* was_guarded */, allow_reclaim); } else { // page is full or contains (inner) aligned blocks; use generic multi-thread path - mi_free_generic_mt(page, p, allow_collect); + mi_free_generic_mt(page, p, allow_reclaim); } } -void _mi_free_in_page(void* p, mi_page_t* page) mi_attr_noexcept { - mi_free_ex(p, page, NULL, true); -} - -void mi_free(void* p) mi_attr_noexcept { - mi_page_t* const page = mi_validate_ptr_page(p,"mi_free"); - mi_free_ex(p, page, NULL, true); +void mi_free(void* p) mi_attr_noexcept { + mi_page_t* page; + if mi_likely(mi_ptr_page_is_valid(p,"mi_free",&page)) { + mi_free_nonnull(p, page, NULL, true /* allow reclaim? */); + } } void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept { - mi_page_t* const page = mi_validate_ptr_page(p,"mi_ufree"); - mi_free_ex(p, page, pblock_size, true); + mi_page_t* page; + if mi_likely(mi_ptr_page_is_valid(p,"mi_ufree",&page)) { + mi_free_nonnull(p, page, pblock_size, true /* allow reclaim? */); + } + else { + if (pblock_size!=NULL) { *pblock_size = 0; } + } } void mi_free_small(void* p) mi_attr_noexcept { - // We can only call `mi_free_small` for pointers allocated with `mi_(heap_)malloc_small`. - // If we keep page info in front of the page area for small objects, we can find the info - // just by aligning down the pointer instead of looking it up in the page map. - #if MI_PAGE_META_ALIGNED_FREE_SMALL - #if MI_GUARDED - #warning "MI_OPT_FREE_SMALL (MI_PAGE_META_ALIGNED_FREE_SMALL) ignored as MI_GUARDED is defined" - mi_free(p); - #elif MI_ARENA_SLICE_ALIGN < MI_SMALL_PAGE_SIZE - #warning "MI_OPT_FREE_SMALL (MI_PAGE_META_ALIGNED_FREE_SMALL) ignored as the MI_ARENA_SLICE_ALIGN is less than the small page size" - mi_free(p); - #else - mi_page_t* const page = (mi_page_t*)_mi_align_down_ptr(p,MI_SMALL_PAGE_SIZE); - mi_assert(page == mi_validate_ptr_page(p,"mi_free_small")); - mi_assert((void*)page == _mi_align_down_ptr(mi_page_start(page),MI_SMALL_PAGE_SIZE)); - mi_assert(page->block_size <= mi_good_size(MI_SMALL_SIZE_MAX)); // note: not `MI_SMALL_MAX_OBJ_SIZE` as we need to match `mi_(heap_)malloc_small` - mi_free_ex(p, page, NULL, true); - #endif - #else - mi_free(p); - #endif + mi_page_t* page; + if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small",true /* is_small? */,true /*check p for null*/, &page)) { + mi_free_nonnull(p, page, NULL, true /* allow reclaim? */); + } } -void _mi_free_subproc_safe_in_page(void* p, mi_page_t* page) mi_attr_noexcept { - mi_free_ex(p, page, NULL, false); +void mi_free_small_nonnull(void* p) mi_attr_noexcept { + mi_assert(p!=NULL); + mi_page_t* page; + if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small_nonnull",true /* is_small? */,false /*check p for null*/, &page)) { + mi_free_nonnull(p, page, NULL, true /* allow reclaim? */); + } } // The sub-process of a page we do not own. On a multi-threaded free we cannot go through `page->heap` @@ -290,15 +344,79 @@ static mi_subproc_t* mi_page_subproc_unowned(const mi_page_t* page) { } // Free a pointer that is potentially allocated in a different sub-process. -// Collecting an abandoned page (free it, reclaim it, or re-map it) must not cross sub-processes, -// but inside our own it must still happen: a full page is abandoned and unmapped, and a block -// freed into it without collecting is never found again and pins the page (`mi_heap_free` frees -// every `mi_heap_t` and `mi_arena_pages_t` through here). +// An abandoned page is always collected (freed, or re-mapped so it can be found again: `mi_heap_free` +// frees every `mi_heap_t` and `mi_arena_pages_t` through here), but it is only reclaimed inside our own sub-process. void _mi_free_subproc_safe(void* p) mi_attr_noexcept { - mi_page_t* const page = mi_validate_ptr_page(p,"_mi_free_subproc_safe"); - if mi_unlikely(page==NULL) return; - const bool allow_collect = (mi_page_subproc_unowned(page) == _mi_subproc()); - mi_free_ex(p, page, NULL, allow_collect); + mi_page_t* page; + if mi_likely(mi_ptr_page_is_valid(p,"_mi_free_subproc_safe",&page)) { + const bool allow_reclaim = (_mi_subproc() == mi_page_subproc_unowned(page)); + mi_free_nonnull(p, page, NULL, allow_reclaim); + } +} + +// ------------------------------------------------------ +// Free variants +// ------------------------------------------------------ + +void mi_free_size(void* p, size_t size) mi_attr_noexcept { + MI_UNUSED_RELEASE(size); + #if MI_DEBUG + const mi_page_t* const page = mi_ptr_page_validate(p,"mi_free_size"); + if (page==NULL) return; + mi_assert(p!=NULL); + const size_t usable = _mi_page_usable_size(page,p); + if mi_unlikely(size > usable) { + const mi_block_t* block = _mi_page_ptr_unalign(page, p); + const bool is_guarded = mi_block_ptr_is_guarded(block,p); + if (!is_guarded) { + _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the size %zu is greater than the usable size %zu\n", p, size, usable); + mi_free(p); + return; + } + } + if mi_unlikely(size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size(MI_SMALL_SIZE_MAX)) { + const mi_block_t* block = _mi_page_ptr_unalign(page, p); + const bool is_guarded = mi_block_ptr_is_guarded(block,p); + if (!is_guarded) { + _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n (maybe a `new[]` was matched with `delete` instead of `delete[]`?)\n", p, size, mi_page_block_size(page)); + mi_free(p); + return; + } + } + #endif + #if MI_PAGE_META_SMALL_IS_ALIGNED || MI_PAGE_META_IS_ALIGNED + if mi_likely(size <= MI_SMALL_SIZE_MAX) { + mi_free_small(p); + } + else + #endif + { + mi_free(p); + } +} + +void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept { + MI_UNUSED_RELEASE(alignment); + mi_assert(((uintptr_t)p % alignment) == 0); + mi_free_size(p,size); +} + +void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { + MI_UNUSED_RELEASE(alignment); + mi_assert(((uintptr_t)p % alignment) == 0); + mi_free(p); +} + +// checked free +bool mi_cfree(void* p) mi_attr_noexcept { + mi_page_t* const page = _mi_checked_ptr_page(p); + if mi_likely(page!=NULL) { + mi_free_nonnull(p, page, NULL, true /* allow reclaim? */); + return true; + } + else { + return false; + } } @@ -417,12 +535,12 @@ static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long // We freed a block in an abandoned page (that was not owned). Try to collect -static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept +static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim) mi_attr_noexcept { mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(mt_free != NULL); - // mi_assert_internal(_mi_subproc() == mi_page_subproc(page)); // never collect across subprocesses + // mi_assert_internal(!allow_reclaim || _mi_subproc() == mi_page_subproc(page)); // never reclaim across subprocesses // we own the page now, and it is safe to collect the thread atomic free list if (page->block_size <= MI_SMALL_SIZE_MAX) { @@ -430,14 +548,14 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* // (after this the `used` count might be too high (as some blocks may have been concurrently added to the thread free list and are yet uncounted). // however, if the page became completely free, the used count is guaranteed to be 0.) mi_assert_internal(page->reserved>=16); // below this even one freed block goes from full to no longer mostly used. - _mi_page_free_collect_partly(page, mt_free); + mt_free = _mi_page_free_collect_partly(page, mt_free); } else { // for larger blocks we use the regular collect _mi_page_free_collect(page,false /* no force */); mt_free = NULL; // expected page->xthread_free value after collection } - const long reclaim_on_free = _mi_option_get_fast(mi_option_page_reclaim_on_free); + const long reclaim_on_free = (allow_reclaim ? _mi_option_get_fast(mi_option_page_reclaim_on_free) : -1); // (fork: upstream has `allow_reclaim && option`, which yields 0/1) #if MI_DEBUG > 1 if (mi_page_is_singleton(page)) { mi_assert_internal(mi_page_all_free(page)); } if (mi_page_is_full(page)) { mi_assert(mi_page_is_mostly_used(page)); } @@ -460,8 +578,8 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* // ------------------------------------------------------ // Bytes available in a block -static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p) mi_attr_noexcept { - const mi_block_t* block = _mi_page_ptr_unalign(page, p); +static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p, const mi_block_t* block) mi_attr_noexcept { + // const mi_block_t* block = _mi_page_ptr_unalign(page, p); const bool is_guarded = mi_block_ptr_is_guarded(block,p); const size_t size = mi_page_usable_size_of(page, block, is_guarded); mi_assert_internal((void*)p >= (void*)block); @@ -473,86 +591,31 @@ static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* p size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept { if mi_unlikely(page==NULL) return 0; - mi_assert_internal(mi_validate_ptr_page(p,"_mi_page_usable_size") == page); - if mi_likely(!mi_page_has_interior_pointers(page)) { - const mi_block_t* block = mi_validate_block_from_ptr(page,p); + mi_assert_internal(mi_ptr_page_validate(p,"_mi_page_usable_size") == page); + const mi_block_t* block = _mi_page_ptr_unalign(page, p); // for safety, always unalign + if mi_likely((const void*)block==p) { return mi_page_usable_size_of(page, block, false /* is guarded */); } else { // split out to separate routine for improved code generation - return mi_page_usable_aligned_size_of(page, p); + return mi_page_usable_aligned_size_of(page, p, block); } } mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept { - const mi_page_t* const page = mi_validate_ptr_page(p,"mi_usable_size"); + const mi_page_t* const page = mi_ptr_page_validate(p,"mi_usable_size"); return _mi_page_usable_size(page,p); } // ------------------------------------------------------ -// Free variants -// ------------------------------------------------------ - -void mi_free_size(void* p, size_t size) mi_attr_noexcept { - MI_UNUSED_RELEASE(size); - #if MI_DEBUG - const mi_page_t* const page = mi_validate_ptr_page(p,"mi_free_size"); - if (page==NULL) return; - mi_assert(p!=NULL); - const size_t usable = _mi_page_usable_size(page,p); - if (size > usable) { - _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the size %zu is greater than the usable size %zu\n", p, size, usable); - mi_free(p); - return; - } - else if (size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size(MI_SMALL_SIZE_MAX)) { - _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n", p, size, mi_page_block_size(page)); - mi_free(p); - return; - } - #endif - #if MI_OPT_FREE_SMALL - if mi_likely(size <= MI_SMALL_SIZE_MAX) { - mi_free_small(p); - } - else - #endif - { - mi_free(p); - } -} - -void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept { - MI_UNUSED_RELEASE(alignment); - mi_assert(((uintptr_t)p % alignment) == 0); - mi_free_size(p,size); -} - -void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { - MI_UNUSED_RELEASE(alignment); - mi_assert(((uintptr_t)p % alignment) == 0); - mi_free(p); -} - -// checked free -void mi_cfree(void* p) mi_attr_noexcept { - mi_page_t* const page = _mi_checked_ptr_page(p); - if mi_likely(page!=NULL) { - _mi_free_in_page(p,page); - } - // if (mi_is_in_heap_region(p)) { - // mi_free(p); - // } -} - - -// ------------------------------------------------------ +// Deprecated: double free is usually checked with padding now +// as that is faster and works better for cross-thread free'ing. // Check for double free in secure and debug mode // This is somewhat expensive so only enabled for secure mode 4 // ------------------------------------------------------ -#if MI_CHECK_DOUBLE_FREE +#if MI_SECURE>=3 && !MI_PADDING // linear check if the free list contains a specific element static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem, const char* list_kind) { const size_t max_count = page->capacity; // can never hold more blocks than the capacity @@ -568,7 +631,7 @@ static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, cons return false; } -static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, const mi_block_t* block) { +static mi_decl_noinline bool mi_check_double_freex(const mi_page_t* page, const mi_block_t* block) { // The decoded value is in the same page (or NULL). // Walk the free lists to verify positively if it is already freed if (mi_list_contains(page, page->free, block, "free") || @@ -576,39 +639,33 @@ static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, con mi_list_contains(page, mi_page_thread_free(page), block, "thread free")) { _mi_error_message(EAGAIN, "double free detected of block %p with size %zu\n", block, mi_page_block_size(page)); - return true; + return false; } - return false; + return true; } // Used for double free checking to avoid checking free lists too frequently static inline bool mi_block_could_be_double_free(const mi_page_t* page, const mi_block_t* block) { mi_block_t* n = mi_block_nextx(page,block,page->keys); - return (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer? - (n==NULL || mi_is_in_same_page(block,n))); // quick check: in the same page or NULL? + return (_mi_is_aligned(block,MI_SIZE_SIZE) && // quick check: aligned pointer? + (n==NULL || mi_page_contains_address(page,n))); // quick check: in the same page or NULL? } // check if `block` was free'd before -static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { - // A purged block is free but held off every free list and its memory is discarded, so - // neither the decoded-`next` heuristic nor the free-list walk below can recognize it. - if mi_unlikely(mi_page_block_is_purged(page, block)) { - _mi_error_message(EAGAIN, "double free detected of block %p with size %zu\n", block, mi_page_block_size(page)); - return true; - } +static inline bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block) { if mi_unlikely(mi_block_could_be_double_free(page,block)) // quick check: next field is aligned in the same page or NULL? { // Suspicious: decoded value a in block is in the same page (or NULL) -- maybe a double free? // (continue in separate function to improve code generation) - return mi_check_is_double_freex(page, block); + return mi_check_double_freex(page, block); } - else return false; + else return true; } #else -static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { +static inline bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block) { MI_UNUSED(page); MI_UNUSED(block); - return false; + return true; } #endif @@ -618,16 +675,17 @@ static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block // --------------------------------------------------------------------------- #if MI_PADDING // && !MI_TRACK_ENABLED -static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize) { +static inline bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize, bool* double_free) { *bsize = mi_page_usable_block_size(page); - const mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize); + mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize); mi_track_mem_defined(padding,sizeof(mi_padding_t)); *delta = padding->delta; - uint32_t canary = padding->canary; - uintptr_t keys[2]; - keys[0] = page->keys[0]; - keys[1] = page->keys[1]; - bool ok = (mi_ptr_encode_canary(page,block,keys) == canary && *delta <= *bsize); + const uint32_t canary = padding->canary; + const bool ok = (mi_ptr_encode_canary(page,block,page->keys) == canary && *delta <= *bsize); + if (double_free!=NULL) { + if mi_unlikely(!ok) { *double_free = mi_ptr_decode_canary_is_freed(canary); } // double free? + else { padding->canary = mi_ptr_encode_canary_freed(); } // mark as freed + } mi_track_mem_noaccess(padding,sizeof(mi_padding_t)); return ok; } @@ -641,7 +699,7 @@ static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* bl else { size_t bsize; size_t delta; - bool ok = mi_page_decode_padding(page, block, &delta, &bsize); + bool ok = mi_page_decode_padding(page, block, &delta, &bsize, NULL); mi_assert_internal(ok); mi_assert_internal(delta <= bsize); return (ok ? bsize - delta : 0); } @@ -652,9 +710,10 @@ static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* bl // contain the pointer for the delayed list, then shrink the padding (by decreasing delta) // so it will later not trigger an overflow error in `mi_free_block`. void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) { + if (mi_block_ptr_is_guarded(block,block + 1)) return; size_t bsize; size_t delta; - bool ok = mi_page_decode_padding(page, block, &delta, &bsize); + bool ok = mi_page_decode_padding(page, block, &delta, &bsize, NULL); mi_assert_internal(ok); if (!ok || (bsize - delta) >= min_size) return; // usually already enough space mi_assert_internal(bsize >= min_size); @@ -683,16 +742,17 @@ void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const si } #endif -#if MI_PADDING && MI_PADDING_CHECK +#if MI_PADDING -static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong) { +static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong, bool* is_double_free) { size_t bsize; size_t delta; - bool ok = mi_page_decode_padding(page, block, &delta, &bsize); + bool ok = mi_page_decode_padding(page, block, &delta, &bsize, is_double_free ); *size = *wrong = bsize; if (!ok) return false; mi_assert_internal(bsize >= delta); *size = bsize - delta; + #if MI_PADDING_CHECK_BYTES if (!mi_page_is_huge(page)) { uint8_t* fill = (uint8_t*)block + bsize - delta; const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // check at most the first N padding bytes @@ -706,50 +766,65 @@ static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, si } mi_track_mem_noaccess(fill, maxpad); } + #endif return ok; } -static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) { - size_t size; - size_t wrong; - if (!mi_verify_padding(page,block,&size,&wrong)) { - _mi_error_message(EFAULT, "buffer overflow in theap block %p of size %zu: write after %zu bytes\n", block, size, wrong ); +mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size) { + if mi_unlikely(is_guarded) { + const size_t bsize = mi_page_block_size(page); + *usable_size = (bsize - _mi_os_page_size()); + return true; + } + else { + size_t wrong; + bool is_double_free = false; // stays unset when the canary is fine and only the padding bytes are not + if mi_unlikely(!mi_verify_padding(page,block,usable_size,&wrong,&is_double_free)) { + if (is_double_free) { + _mi_error_message(EAGAIN, "double free detected of heap block %p with size %zu\n", block, *usable_size); + } + else { + _mi_error_message(EFAULT, "buffer overflow in heap block %p of size %zu: write after %zu bytes\n", block, *usable_size, wrong ); + } + return false; + } + return true; } } #else -static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) { - MI_UNUSED(page); - MI_UNUSED(block); +mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size) { + *usable_size = mi_page_usable_size_of(page,block,is_guarded); + return true; } #endif // only maintain stats for smaller objects if requested -#if (MI_STAT>0) +#if 0 && (MI_STATS>0) static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { MI_UNUSED(block); mi_theap_t* theap = _mi_theap_default(); mi_lock_t* lock = NULL; - if mi_unlikely(!mi_theap_is_initialized(theap) || // can happen if free'd after thread_done was called (usually a thread cleanup call by the OS) - mi_page_thread_id(page) == MI_THREADID_DETACHED) { // page->theap == subproc->theap_meta .. but we cannot read `theap` if we don't own the page (only the meta theap has the detached thread id) - // account on the subproc of the page (we cannot use our own theap here) - mi_subproc_t* const subproc = mi_page_subproc_unowned(page); - if (subproc->theap_meta == NULL) return; // give up (the subproc is being destroyed) - theap = subproc->theap_meta; + mi_subproc_t* const subproc = mi_page_subproc(page); + mi_theap_t* const theap_meta = subproc->theap_meta; + if mi_unlikely(!mi_theap_is_initialized(theap) || // can happen if free'd after thread_done was called (usually a thread cleanup call by the OS) + // page->theap == subproc->theap_meta .. but we cannot read `theap` if we don't own the page + (theap_meta != NULL && mi_page_thread_id(page) == theap_meta->tld->thread_id)) { + theap = theap_meta; lock = &subproc->theap_meta_lock; mi_lock_acquire(lock); } const size_t bsize = mi_page_usable_block_size(page); - // #if (MI_STAT>1) + // #if (MI_STATS>=2) // const size_t usize = mi_page_usable_size_of(page, block); // mi_theap_stat_decrease(theap, malloc_requested, usize); // #endif if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { mi_theap_stat_decrease(theap, malloc_normal, bsize); - #if (MI_STAT > 1) + #if (MI_STATS > 1) mi_theap_stat_decrease(theap, malloc_bins[_mi_bin(bsize)], 1); #endif } @@ -769,34 +844,3 @@ void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { #endif -// Remove guard page when building with MI_GUARDED -#if MI_GUARDED -static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { - MI_UNUSED(p); - mi_assert_internal(mi_block_ptr_is_guarded(block, p)); - mi_assert_internal(mi_page_has_interior_pointers(page)); - mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t)); - mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED); - - const size_t bsize = mi_page_block_size(page); - const size_t psize = _mi_os_page_size(); - mi_assert_internal(bsize > psize); - mi_assert_internal(!page->memid.is_pinned); - void* gpage = (uint8_t*)block + bsize - psize; - mi_assert_internal(_mi_is_aligned(gpage, psize)); - _mi_os_unprotect(gpage, psize); -} - -// unguard a whole page (called from `mi_heap_destroy`) -void _mi_page_unguard_all(mi_page_t* page) { - if mi_likely(!mi_page_has_interior_pointers(page)) return; - uint8_t* const start = mi_page_start(page); - const size_t psize = mi_page_committed(page); - _mi_os_unprotect(start,psize); // unprotect all at once as we cannot know which blocks are guarded -} -#else -void _mi_page_unguard_all(mi_page_t* page) { - MI_UNUSED(page); - // nothing to do -} -#endif diff --git a/src/heap-snapshot.c b/src/heap-snapshot.c index a4f23ec88..48471841d 100644 --- a/src/heap-snapshot.c +++ b/src/heap-snapshot.c @@ -223,7 +223,7 @@ static void mi_snap_emit_page(mi_snap_out_t* out, mi_page_t* page, int32_t arena mi_snap_u64(out, (uint64_t)mi_page_block_size(page)); mi_snap_u32(out, (uint32_t)page->reserved); mi_snap_u32(out, (uint32_t)page->capacity); - mi_snap_u32(out, (uint32_t)page->used); + mi_snap_u32(out, (uint32_t)mi_page_used(page)); mi_snap_u64(out, (uint64_t)mi_page_committed(page)); mi_snap_u64(out, (uint64_t)tid); mi_snap_u64(out, (uint64_t)(page->heap != NULL ? page->heap->heap_seq : 0)); @@ -256,7 +256,7 @@ typedef struct mi_snap_ctx_s { static void mi_snap_emit_arena_header(mi_snap_out_t* out, mi_arena_t* arena, size_t idx) { mi_snap_u32(out, MI_SNAP_SEC_ARENA); mi_snap_u32(out, (uint32_t)idx); - mi_snap_u64(out, (uint64_t)(uintptr_t)arena); // base address + mi_snap_u64(out, (uint64_t)(uintptr_t)mi_arena_slice_start(arena,0)); // base address (the arena info itself comes after the aligned page meta slices) mi_snap_u64(out, (uint64_t)mi_size_of_slices(arena->slice_count)); mi_snap_u32(out, (uint32_t)arena->slice_count); mi_snap_u32(out, (uint32_t)arena->info_slices); diff --git a/src/heap.c b/src/heap.c index f1a049c85..1b118e519 100644 --- a/src/heap.c +++ b/src/heap.c @@ -88,7 +88,7 @@ static mi_decl_noinline mi_theap_t* mi_heap_init_theap(const mi_heap_t* const_he // get (and possibly create) the theap belonging to a heap -mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap) +mi_decl_cold mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap) { mi_assert_internal(heap->theap != 0); mi_theap_t* theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); @@ -108,6 +108,7 @@ void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap_slot, mi_subproc_t* heap->heap_seq = mi_atomic_increment_relaxed(&subproc->heap_total_count); heap->exclusive_arena = _mi_arena_from_id(exclusive_arena_id); heap->numa_node = -1; // no initial affinity + heap->profiler = mi_atomic_load_ptr_acquire(mi_profiler_t,&subproc->profiler); mi_stats_header_init(&heap->stats); mi_lock_init(&heap->theaps_lock); mi_lock_init(&heap->os_abandoned_pages_lock); @@ -218,6 +219,13 @@ static void mi_heap_free_theaps(mi_heap_t* heap) { _mi_theap_decref(theap); // a thread's `_mi_theap_cached` can still reference (but no longer use) it theap = next; } + } + + // set the theap thread local to NULL (so _mi_page_associated_theap does not read from a freed theap (through delete pages -> page_update_stats)) + // (in this fork the theaps are detached before, and freed after, the pages leave the heap, so this only releases the slot) + if (!_mi_is_process_heap_main(heap)) { + _mi_thread_local_free(heap->theap); + heap->theap = 0; } } @@ -258,7 +266,7 @@ static void mi_heap_free(mi_heap_t* heap, bool acquire_heaps_lock) { mi_lock_done(&heap->os_abandoned_pages_lock); mi_lock_done(&heap->arena_pages_lock); if (!_mi_is_process_heap_main(heap)) { - _mi_thread_local_free(heap->theap); + // _mi_thread_local_free(heap->theap); _mi_free_subproc_safe(heap); } } @@ -324,7 +332,7 @@ bool mi_check_owned(const void* p) { bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept { if (p==NULL) return false; const mi_page_t* const page = _mi_safe_ptr_page(p); // Get the page containing this pointer - if (page==NULL || page->used==page->capacity || page->capacity < page->reserved) return false; + if (page==NULL || mi_page_used(page)==page->capacity || page->capacity < page->reserved) return false; // If the page is the head of the queue, it is currently being used for // allocations; we skip it to avoid immediate thrashing. if (page->prev == NULL) return false; @@ -337,5 +345,5 @@ bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc // check utilization if (page->capacity==0) return false; if (perc_threshold>=100) return true; - return (perc_threshold >= ((100UL*page->used) / page->capacity)); + return (perc_threshold >= ((100UL*mi_page_used(page)) / page->capacity)); } diff --git a/src/init.c b/src/init.c index 8ab8dcd8e..a6e8f8db4 100644 --- a/src/init.c +++ b/src/init.c @@ -19,24 +19,35 @@ static void mi_tld_register(mi_tld_t* tld); // fork: tld registry for the scav // Empty page used to initialize the small free pages array static const mi_page_t mi_page_empty = { + #if MI_PAGE_META_IS_ALIGNED + MI_ATOMIC_VAR_INIT(NULL), // self + #endif MI_ATOMIC_VAR_INIT(0), // xthread_id NULL, // free - 0, // used + {0}, // xused + #if MI_SIZE_SIZE < 8 + 0, // xlast_used + 0, // xlast_alloc + #endif + NULL, // local_free + 0, // block_size + 0, // page_offset 0, // capacity + 0, // reserved capacity + 0, // slice_pcommitted 0, // retire_expire false, // is_zero - NULL, // local_free MI_ATOMIC_VAR_INIT(0), // xthread_free - 0, // block_size - 0, // page_ma_offset - 0, // slice_pcommitted - 0, // reserved capacity NULL, // theap NULL, // heap NULL, NULL, // next, prev MI_MEMID_STATIC, // memid #if (MI_PADDING || MI_ENCODE_FREELIST) + #if MI_PAGE_KEY_COUNT==2 { 0, 0 }, // keys + #else + { 0 }, // key + #endif #endif { 0 }, // purged (fork) 0, 0, // unformed_purged_lo/hi (fork) @@ -45,12 +56,20 @@ static const mi_page_t mi_page_empty = { #define MI_PAGE_EMPTY() ((mi_page_t*)&mi_page_empty) -#if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8) -#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } +#if MI_SMALL_WSIZE_MAX == 128 +#define MI_INIT_PAGES_DIRECT(p) MI_INIT128(p) +#elif MI_SMALL_WSIZE_MAX == 256 +#define MI_INIT_PAGES_DIRECT(p) MI_INIT128(p), MI_INIT128(p) +#else +#error define initializer for direct pages +#endif + +#if (MI_PADDING>0) && (MI_SIZE_SIZE >= 8) +#define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } #elif (MI_PADDING>0) -#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } +#define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } #else -#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() } +#define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY() } #endif @@ -112,17 +131,24 @@ static mi_decl_cache_align mi_tld_t mi_tld_detached = { MI_ATOMIC_VAR_INIT(0), // park_swept NULL, // subproc_next 0, 0, // holes_sweep_seq / _last - false, false, 0, 0, // holes_sweeping / _full / _skipped / _visited - false // prof_sampling + false, false, 0, 0 // holes_sweeping / _full / _skipped / _visited }; mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { + MI_SMALL_PAGES_EMPTY, // direct small pages &mi_tld_detached, // tld MI_ATOMIC_VAR_INIT(NULL), // heap MI_ATOMIC_VAR_INIT(NULL), // subproc MI_ATOMIC_VAR_INIT(1), // refcount + 0, // full page retain + false, // allow reclaim + true, // allow abandon + true, // is_detached + ~MI_ZU(0), // sample countdown: "-1" (with a sample rate of 0, so we won't write to the empty theap with MI_SAMPLE==2) + 0, 0, // sample rate, requested + 0, 0, // profile rate, countdown + 0, 0, 0, 0, // guarded rate, countdown, min, max 0, // heartbeat - 0, // cookie { {0}, {0}, 0, true }, // random 0, // page count MI_BIN_FULL, 0, // page retired min/max @@ -130,15 +156,6 @@ mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { 0, 0, // generic count NULL, NULL, // tnext, tprev NULL, NULL, // hnext, hprev - 0, // full page retain - false, // allow reclaim - true, // allow abandon - true, // is_detached - false, 0, // prof_force_slow, prof_countdown (fork) - #if MI_GUARDED - 0, 0, 0, 1, // rate is 0 and count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`) - #endif - MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY, MI_MEMID_STATIC, MI_STATS_NULL, // stats @@ -202,6 +219,8 @@ static void mi_heap_main_init_once(void) { _mi_theap_init(&mi_process_theap_meta,&mi_process_heap_main,&mi_tld_detached); mi_process_theap_meta.allow_page_abandon = false; // for security, don't share with other threads mi_process_theap_meta.page_full_retain = 2; + mi_process_theap_meta.sample_rate = 0; // no sampling for meta data + mi_process_theap_meta.sample_countdown = 0; subproc_main->theap_meta = &mi_process_theap_meta; // mi_heap_theap_set(&mi_process_heap_main,&mi_process_theap_main); // set in `mi_thread_init(_theap_default)` @@ -489,6 +508,11 @@ static void mi_thread_theaps_done(mi_tld_t* tld) static void mi_process_setup_auto_thread_done(void) { mi_atomic_do_once { _mi_prim_thread_init_auto_done(); + mi_theap_t* theap = _mi_theap_default(); + mi_assert_internal(mi_theap_is_initialized(theap)); + if (mi_theap_is_initialized(theap)) { + _mi_theap_default_set(theap); + } } } @@ -497,7 +521,7 @@ void mi_thread_done(void) mi_attr_noexcept { } void _mi_thread_done(mi_theap_t* _theap_main) -{ +{ // NULL can be passed on some platforms if (_theap_main==NULL) { _theap_main = _mi_theap_default(); @@ -511,8 +535,7 @@ void _mi_thread_done(mi_theap_t* _theap_main) // get the current tld mi_tld_t* const tld = _theap_main->tld; - // release dynamic thread_local's - _mi_thread_locals_thread_done(); + // (the dynamic thread_local's are released below, after the park leave) // adjust stats mi_subproc_stat_decrease(tld->subproc, threads, 1); // todo: or `_theap_main->heap`? @@ -567,7 +590,6 @@ void _mi_auto_process_init(void) { os_preloading = false; mi_process_init(); - mi_process_setup_auto_thread_done(); _mi_options_post_init(); // now we can print to stderr if (_mi_is_redirected()) _mi_verbose_message("malloc is redirected.\n"); @@ -612,6 +634,7 @@ static void mi_process_init_once(void) { // the following can potentially allocate (on freeBSD for pthread keys) _mi_tls_slots_init(); // pthread key create _mi_thread_locals_init(); // pthread key create + mi_process_setup_auto_thread_done(); // after the above mi_thread_init so it can add the current theap _mi_process_is_initialized = true; #if !defined(_WIN32) && !defined(__wasi__) @@ -627,7 +650,6 @@ static void mi_process_init_once(void) { // mi_stats_reset(); // only call stat reset *after* thread init (or the theap tld == NULL) mi_track_init(); - _mi_prof_init(); if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { size_t pages = mi_option_get_clamp(mi_option_reserve_huge_os_pages, 0, 128*1024); int reserve_at = (int)mi_option_get_clamp(mi_option_reserve_huge_os_pages_at, -1, INT_MAX); @@ -669,10 +691,9 @@ static void mi_process_done_once(void) { #if defined(_WIN32) // This runs in the process detach callback, after the system terminated every other thread. One that was in a purge // pass then (the scavenger, as a rule) holds the purge guard for good, and the forced collect below would wait for it. - _mi_arenas_purge_guard_reset(); + _mi_arenas_purge_guard_release(); #endif - _mi_heap_snapshot_on_exit(); // fork: heap snapshot / profile at exit, before anything is torn down - _mi_prof_on_exit(); + _mi_heap_snapshot_on_exit(); // fork: heap snapshot at exit, before anything is torn down // only shutdown if we were initialized if (!_mi_process_is_initialized) return; // ensure we are called once @@ -696,7 +717,7 @@ static void mi_process_done_once(void) { #endif // done with tracking tools - mi_track_done() + mi_track_done(); // Forcefully release all retained memory; this can be dangerous in general if overriding regular malloc/free // since after process_done there might still be other code running that calls `free` (like at_exit routines, @@ -711,10 +732,11 @@ static void mi_process_done_once(void) { _mi_thread_locals_done(); if (subproc_main->heap_main != NULL) { if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { + mi_theap_collect(subproc_main->theap_meta, false /* force */); // update stats of all pages in theap_meta _mi_theap_merge_stats(subproc_main->theap_meta); _mi_theap_merge_stats(_mi_theap_default()); // _mi_thread_locals_done can free mi_heap_stats_merge_to_subproc(subproc_main->heap_main); - mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL); + mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL); // note: can try to access (the now freed) thread_locals in mi_heap_theap_peek } } } @@ -722,7 +744,7 @@ static void mi_process_done_once(void) { _mi_tls_slots_done(); _mi_subproc_main_done(); _mi_allocator_done(); - _mi_verbose_message("process done\n"); // : 0x%zx\n", mi_process_tld_main.thread_id); + _mi_verbose_message("process done %zu\n", sizeof(mi_page_t)); // : 0x%zx\n", mi_process_tld_main.thread_id); os_preloading = true; // don't call the C runtime anymore } diff --git a/src/libc.c b/src/libc.c index 17dd1d1dc..8f211230d 100644 --- a/src/libc.c +++ b/src/libc.c @@ -160,8 +160,6 @@ mi_decl_noinline bool _mi_pthread_key_create(pthread_key_t* pkey, void (*destruc // -------------------------------------------------------- // Detect CPU features // -------------------------------------------------------- -mi_decl_cache_align size_t _mi_cpu_movsb_max = 0; // for size <= max, rep movsb is fast -mi_decl_cache_align size_t _mi_cpu_stosb_max = 0; // for size <= max, rep stosb is fast mi_decl_cache_align bool _mi_cpu_has_popcnt = false; #if (MI_ARCH_X64 || MI_ARCH_X86) @@ -192,39 +190,15 @@ static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) { #endif void _mi_detect_cpu_features(void) { - // FSRM for fast short rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017)) - // EMRS for fast enhanced rep movsb/stosb support (not used at the moment, memcpy always seems faster?) - // FSRS for fast short rep stosb - bool amd = false; - bool fsrm = false; - // bool erms = false; - bool fsrs = false; uint32_t cpu_info[4]; - if (mi_cpuid(cpu_info, 0, 0)) { - amd = (cpu_info[2]==0x444d4163); // (Auth enti cAMD) - } - if (mi_cpuid(cpu_info, 7, 0)) { - fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see - // erms = ((cpu_info[1] & (1 << 9)) != 0); // bit 9 of EBX : see - } - if (mi_cpuid(cpu_info, 7, 1)) { - fsrs = ((cpu_info[1] & (1 << 11)) != 0); // bit 11 of EBX: see - } if (mi_cpuid(cpu_info, 1, 0)) { _mi_cpu_has_popcnt = ((cpu_info[2] & (1 << 23)) != 0); // bit 23 of ECX : see } - - if (fsrm) { - _mi_cpu_movsb_max = 127; - } - if (fsrs || (amd && fsrm)) { // fsrm on amd implies fsrs, see: https://marc.info/?l=git-commits-head&m=168186277717803 - _mi_cpu_stosb_max = 127; - } } #else void _mi_detect_cpu_features(void) { - #if MI_ARCH_ARM64 + #if MI_ARCH_ARM64 || defined(__riscv_zbb) || defined(__riscv_b) _mi_cpu_has_popcnt = true; #endif } diff --git a/src/options.c b/src/options.c index bc5e12fa6..792c720a2 100644 --- a/src/options.c +++ b/src/options.c @@ -49,7 +49,7 @@ int mi_version(void) { // in KiB #ifndef MI_DEFAULT_ARENA_RESERVE - #if (MI_INTPTR_SIZE>4) + #if (MI_SIZE_SIZE>4) #define MI_DEFAULT_ARENA_RESERVE 1024L*1024L #else #define MI_DEFAULT_ARENA_RESERVE 128L*1024L @@ -77,10 +77,10 @@ int mi_version(void) { #endif #ifndef MI_DEFAULT_GUARDED_SAMPLE_RATE -#if MI_GUARDED && !MI_DEBUG -#define MI_DEFAULT_GUARDED_SAMPLE_RATE 4000 +#if MI_GUARDED && MI_DEBUG +#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0 /* MI_MiB */ #else -#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0 +#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0 #endif #endif @@ -104,10 +104,15 @@ int mi_version(void) { #if defined(__ANDROID__) #define MI_DEFAULT_ALLOW_THP 0 #else -#define MI_DEFAULT_ALLOW_THP 1 +// #define MI_DEFAULT_ALLOW_THP 1 // allow THP but purging may split up THP pages +#define MI_DEFAULT_ALLOW_THP 2 // allow THP and set the minimal purge size to 2MiB to avoid breaking them up #endif #endif +#ifndef MI_DEFAULT_COLLECT_MERGES_STATS +#define MI_DEFAULT_COLLECT_MERGES_STATS 1 +#endif + // Static options static mi_option_desc_t mi_options[_mi_option_last] = { @@ -146,7 +151,7 @@ static mi_option_desc_t mi_options[_mi_option_last] = { 10, MI_OPTION_UNINIT, MI_OPTION(deprecated_max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try. { 0, MI_OPTION_UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees! { MI_DEFAULT_ARENA_RESERVE, MI_OPTION_UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`) - { 1, MI_OPTION_UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's + { 1, MI_OPTION_UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's (fork: 1, upstream uses 4 with a 1000ms purge_delay; the scavenger thread purges here so 100ms x 1 is the measured setting) { 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(deprecated_purge_extend_delay, decommit_extend_delay) }, { MI_DEFAULT_DISALLOW_ARENA_ALLOC, MI_OPTION_UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) { 400, MI_OPTION_UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. @@ -171,12 +176,13 @@ static mi_option_desc_t mi_options[_mi_option_last] = MI_OPTION_UNINIT, MI_OPTION(page_cross_thread_max_reclaim) }, // don't reclaim (small) pages across threads if we already own N pages in that size class { MI_DEFAULT_ALLOW_THP, MI_OPTION_UNINIT, MI_OPTION(allow_thp) }, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to opt the memory mimalloc maps out of THP. - { 0, MI_OPTION_UNINIT, MI_OPTION(minimal_purge_size) }, // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if `mi_option_allow_thp==2`). + { 0, MI_OPTION_UNINIT, MI_OPTION(minimal_purge_size) }, // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if THP is enabled). { MI_DEFAULT_ARENA_MAX_OBJECT_SIZE, MI_OPTION_UNINIT, MI_OPTION(arena_max_object_size) }, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). { 0, MI_OPTION_UNINIT, MI_OPTION(arena_is_numa_local) }, // associate local numa node with an initial arena allocation + { MI_DEFAULT_COLLECT_MERGES_STATS, + MI_OPTION_UNINIT, MI_OPTION(collect_merges_stats) }, // on each theap collect, stats are merged with the parent heap { 0, MI_OPTION_UNINIT, MI_OPTION(snapshot_on_exit) }, // write a heap snapshot on process exit (=0). 1=on, 2=on+blocks. - { 0, MI_OPTION_UNINIT, MI_OPTION(prof_sample_rate) }, // bytes per heap-profile sample (=0, off). Typical: 524288. { 1, MI_OPTION_UNINIT, MI_OPTION(scavenger) }, // run a background scavenger thread that purges freed arena memory when due (=1) { 1, MI_OPTION_UNINIT, MI_OPTION(purge_holes) }, // discard free blocks inside a still-used page (=1) { 0, MI_OPTION_UNINIT, MI_OPTION(purge_holes_eager_zero) }, // zero a hole before discarding it (=0; for testing) @@ -246,16 +252,34 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_ _mi_fprintf(out, arg, "option '%s': %ld %s\n", desc->name, desc->value, (mi_option_has_size_in_kib(option) ? "KiB" : "")); } - // show build configuration - _mi_fprintf(out, arg, "debug level : %d\n", MI_DEBUG ); - _mi_fprintf(out, arg, "secure level: %d\n", MI_SECURE ); - _mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL); + // show build configuration + #if MI_PAGE_META_IS_ALIGNED && MI_PAGE_META_SMALL_IS_ALIGNED + _mi_fprintf(out, arg, "free mode : (small) aligned, page size: %zu\n", sizeof(mi_page_t)); + #elif MI_PAGE_META_IS_ALIGNED + _mi_fprintf(out, arg, "free mode : aligned, page size: %zu\n", sizeof(mi_page_t)); + #elif MI_PAGE_META_SMALL_IS_ALIGNED + _mi_fprintf(out, arg, "free mode : small aligned + pagemap, page size: %zu\n", sizeof(mi_page_t)); + #elif MI_FREE_IS_CHECKED + _mi_fprintf(out, arg, "free mode : checked, page size: %zu\n", sizeof(mi_page_t)); + #else + _mi_fprintf(out, arg, "free mode : pagemap, page size: %zu\n", sizeof(mi_page_t)); + #endif + #if MI_ENCODE_FREELIST + _mi_fprintf(out, arg, "free lists : encoded with %d key(s)\n", MI_PAGE_KEY_COUNT); + #endif #if MI_GUARDED - _mi_fprintf(out, arg, "guarded build: %s\n", mi_option_get(mi_option_guarded_sample_rate) != 0 ? "enabled" : "disabled"); + _mi_fprintf(out, arg, "guarded mode: %s, rate=%ld\n", MI_SAMPLE==2 ? "fine-grained" : "enabled", mi_option_get(mi_option_guarded_sample_rate)); + #endif + #if MI_PROFILE + _mi_fprintf(out, arg, "profiling : %s\n", MI_SAMPLE==2 ? "fine-grained" : "enabled"); #endif #if MI_TSAN _mi_fprintf(out, arg, "thread santizer enabled\n"); #endif + _mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL); + _mi_fprintf(out, arg, "debug level : %d\n", MI_DEBUG ); + _mi_fprintf(out, arg, "secure level: %d\n", MI_SECURE ); + _mi_fprintf(out, arg, "padding : %s\n", (MI_PADDING==0 ? "none" : (MI_PADDING==1 ? "enabled" : "byte precise"))); } mi_decl_export void mi_options_print(void) mi_attr_noexcept { @@ -440,25 +464,12 @@ static void mi_add_stderr_output(void) { static _Atomic(size_t) error_count; // = 0; // when >= max_error_count stop emitting errors static _Atomic(size_t) warning_count; // = 0; // when >= max_warning_count stop emitting warnings -// When overriding malloc, we may recurse into mi_vfprintf if an allocation -// inside the C runtime causes another message. -// In some cases (like on macOS) the loader already allocates which -// calls into mimalloc; if we then access thread locals (like `recurse`) -// this may crash as the access may call _tlv_bootstrap that tries to -// (recursively) invoke malloc again to allocate space for the thread local -// variables on demand. This is why we use a _mi_preloading test on such -// platforms. However, C code generator may move the initial thread local address -// load before the `if` and we therefore split it out in a separate function. -static mi_decl_thread bool recurse = false; - static mi_decl_noinline bool mi_recurse_enter_prim(void) { - if (recurse) return false; - recurse = true; return true; } static mi_decl_noinline void mi_recurse_exit_prim(void) { - recurse = false; + /* nothing */ } static bool mi_recurse_enter(void) { @@ -628,6 +639,11 @@ void _mi_error_message(int err, const char* fmt, ...) { } } +mi_decl_noinline mi_block_t* _mi_block_next_is_corrupted(const mi_page_t* page, const mi_block_t* block, const mi_block_t* next) { + _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next); + return NULL; +} + // -------------------------------------------------------- // Initialize options by checking the environment // -------------------------------------------------------- diff --git a/src/os.c b/src/os.c index 3afb16821..ca37f1786 100644 --- a/src/os.c +++ b/src/os.c @@ -14,7 +14,7 @@ terms of the MIT license. A copy of the license can be found in the file Initialization. ----------------------------------------------------------- */ #ifndef MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB -#if MI_INTPTR_SIZE < 8 +#if MI_SIZE_SIZE < 8 #define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 4*MI_MiB // 4 GiB #else #define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 32*MI_MiB // 32 GiB @@ -41,6 +41,9 @@ bool _mi_os_has_virtual_reserve(void) { return mi_os_mem_config.has_virtual_reserve; } +bool _mi_os_canuse_thp(void) { + return mi_os_mem_config.has_transparent_huge_pages; +} // OS (small) page size size_t _mi_os_page_size(void) { @@ -55,13 +58,19 @@ size_t _mi_os_large_page_size(void) { // minimal purge size. Can be larger than the page size if transparent huge pages are enabled. size_t _mi_os_minimal_purge_size(void) { size_t minsize = mi_option_get_size(mi_option_minimal_purge_size); - if (minsize != 0) { + if (minsize != 0) { + // set by user return _mi_align_up(minsize, _mi_os_page_size()); } else if (mi_os_mem_config.has_transparent_huge_pages && mi_option_get(mi_option_allow_thp) == 2) { + // don't break up THP pages; + // we don't do this by default as that can lead to increased memory usage (see issue #1282). + // on the other hand, breaking up THP pages can lead to performance degradation and perhaps + // allow_thp should be 2 by default (and the user can set the OS THP setting to [never]). return _mi_os_large_page_size(); } else { + // OS page size by default return _mi_os_page_size(); } } @@ -114,49 +123,51 @@ bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero // On systems with enough virtual address bits, we can do efficient aligned allocation by using // the 2TiB to 30TiB area to allocate those. If we have at least 46 bits of virtual address // space (64TiB) we use this technique. (but see issue #939) -#if (MI_INTPTR_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) // && !defined(WIN32) && !defined(ANDROID) +#if (MI_SIZE_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) // && !defined(WIN32) && !defined(ANDROID) -// Return a MI_HINT_ALIGN (4MiB) aligned address that is probably available. +// Return a `try_alignment` aligned address that is probably available. // If this returns NULL, the OS will determine the address but on some OS's that may not be // properly aligned which can be more costly as it needs to be adjusted afterwards. // In secure mode, for a size > 16GiB this always returns NULL in order to guarantee good ASLR randomization; // (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses // in the middle of the 2TiB - 6TiB address range (see issue #372)) -#define MI_HINT_ALIGN ((uintptr_t)4 << 20) // 4MiB alignment #define MI_HINT_BASE ((uintptr_t)2 << 40) // 2TiB start #define MI_HINT_AREA ((uintptr_t)4 << 40) // upto (2+4) 6TiB (since before win8 there is "only" 8TiB available to processes) #define MI_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages) -void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) +void* _mi_os_get_aligned_hint(size_t try_alignment, size_t sze) { static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // = 0 // todo: perhaps only do alignment hints if THP is enabled? - if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > MI_HINT_ALIGN) return NULL; + if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > 16*MI_GiB) return NULL; if (mi_os_mem_config.virtual_address_bits < 46) return NULL; // < 64TiB virtual address space - size = _mi_align_up(size, MI_HINT_ALIGN); + + size_t req_size = sze + _mi_os_page_size(); // always reserve a bit more to create virtual gaps between hinted blocks. + req_size += (try_alignment - 1); // ensure we can align in the requested size + req_size = _mi_align_up(req_size, _mi_os_large_page_size()); #if (MI_SECURE>=1) - if (size > 16*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<34) = 1/256 + if (req_size > 32*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<34) = 1/256 #endif - size += MI_HINT_ALIGN; // put in virtual gaps between hinted blocks; this splits VLA's but increases guarded areas. - - uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size); + + uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, req_size); if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize uintptr_t init = MI_HINT_BASE; - #if (MI_SECURE>=1 || defined(NDEBUG)) // security: randomize start of aligned allocations unless in debug mode + #if (MI_SECURE>=1 || !MI_DEBUG) // security: randomize start of aligned allocations unless in debug mode mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267) if (!mi_theap_is_initialized(theap)) return NULL; // no hint as we lack randomness at this point const uintptr_t r = _mi_theap_random_next(theap); - init = init + ((MI_HINT_ALIGN * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB + init = init + ((MI_MiB * ((r>>17) & 0x3FFFFF)) % MI_HINT_AREA); // (randomly 22 bits)* 1MiB == 0 to 4TiB #endif - uintptr_t expected = hint + size; + uintptr_t expected = hint + req_size; mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init); - hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all + hint = mi_atomic_add_acq_rel(&aligned_base, req_size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all + if (hint==0) return NULL; } - mi_assert_internal(hint%MI_HINT_ALIGN == 0); - if (hint%try_alignment != 0) return NULL; - return (void*)hint; + const uintptr_t hint_align = _mi_align_up(hint,try_alignment); + mi_assert_internal(hint_align + sze < hint + req_size); + return (void*)hint_align; } #else void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { @@ -239,7 +250,7 @@ bool _mi_os_secure_guard_page_reset_before(mi_subproc_t* subproc, void* addr, mi static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size); -static void mi_os_prim_free(mi_subproc_t* subproc, void* addr, size_t size, size_t commit_size) { +static void mi_os_prim_free(mi_subproc_t* subproc, void* addr, size_t size, size_t commit_size, bool adjust) { mi_assert_internal(subproc!=NULL); mi_assert_internal((size % _mi_os_page_size()) == 0); if (addr == NULL) return; // || _mi_os_is_huge_reserved(addr) @@ -247,10 +258,14 @@ static void mi_os_prim_free(mi_subproc_t* subproc, void* addr, size_t size, size if (err != 0) { _mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr); } - if (commit_size > 0) { - mi_subproc_stat_decrease(subproc, committed, commit_size); + if (adjust) { + if (commit_size>0) { mi_subproc_stat_adjust_decrease(subproc, committed, commit_size); } + mi_subproc_stat_adjust_decrease(subproc, reserved, size); + } + else { + if (commit_size>0) { mi_subproc_stat_decrease(subproc, committed, commit_size); } + mi_subproc_stat_decrease(subproc, reserved, size); } - mi_subproc_stat_decrease(subproc, reserved, size); } void _mi_os_free_ex(mi_subproc_t* subproc, void* addr, size_t size, bool still_committed, mi_memid_t memid) { @@ -278,7 +293,7 @@ void _mi_os_free_ex(mi_subproc_t* subproc, void* addr, size_t size, bool still_c mi_os_free_huge_os_pages(subproc, base, csize); } else { - mi_os_prim_free(subproc, base, csize, (still_committed ? commit_size : 0)); + mi_os_prim_free(subproc, base, csize, (still_committed ? commit_size : 0), false); } } else { @@ -350,8 +365,12 @@ static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL; size = _mi_align_up(size, _mi_os_page_size()); + #if MI_SIZE_SIZE >= 8 + const bool try_direct_alloc = true; + #else // try a direct allocation if the alignment is below the default, or less than or equal to 1/4 fraction of the size. const bool try_direct_alloc = (alignment <= mi_os_mem_config.alloc_granularity || alignment <= size/4); + #endif bool os_is_large = false; bool os_is_zero = false; @@ -373,7 +392,7 @@ static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit); } #endif - if (p != NULL) { mi_os_prim_free(subproc, p, size, (commit ? size : 0)); } + if (p != NULL) { mi_os_prim_free(subproc, p, size, (commit ? size : 0), true /* adjust so we "forget" the previous reservation */); } if (size >= (SIZE_MAX - alignment)) return NULL; // overflow const size_t over_size = size + alignment; @@ -392,14 +411,15 @@ static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t // explicitly commit only the aligned part if (commit) { if (!_mi_os_commit(subproc, p, size, NULL)) { - mi_os_prim_free(subproc, os_base, over_size, 0); + mi_os_prim_free(subproc, os_base, over_size, 0, true); return NULL; } } } else { // mmap can free inside an allocation // overallocate... - p = mi_os_prim_alloc(subproc, over_size, 1, commit, false, &os_is_large, &os_is_zero); + // note: we keep `allow_large` but this only works if we can partially free any large OS pages later on. This seems to be the case though. + p = mi_os_prim_alloc(subproc, over_size, 1, commit, allow_large /* or false? */, &os_is_large, &os_is_zero); if (p == NULL) return NULL; // and selectively unmap parts around the over-allocated area. @@ -408,8 +428,8 @@ static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t const size_t mid_size = _mi_align_up(size, _mi_os_page_size()); const size_t post_size = over_size - pre_size - mid_size; mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size); - if (pre_size > 0) { mi_os_prim_free(subproc, p, pre_size, (commit ? pre_size : 0)); } - if (post_size > 0) { mi_os_prim_free(subproc, (uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0)); } + if (pre_size > 0) { mi_os_prim_free(subproc, p, pre_size, (commit ? pre_size : 0), true /* adjust */); } + if (post_size > 0) { mi_os_prim_free(subproc, (uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0), true /* adjust */); } // we can return the aligned pointer on `mmap` systems p = aligned_p; os_base = aligned_p; // since we freed the pre part, `*base == p`. @@ -548,7 +568,8 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* return mi_os_page_align_areax(true, addr, size, newsize); } -bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_size) { +bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_already_committed) { + mi_assert_internal(size >= stat_already_committed); if (is_zero != NULL) { *is_zero = false; } mi_subproc_stat_counter_increase(subproc, commit_calls, 1); @@ -581,12 +602,12 @@ bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_z if (os_is_zero) { mi_track_mem_defined(start,csize); } else { mi_track_mem_undefined(start,csize); } #endif - mi_subproc_stat_increase(subproc, committed, stat_size); // use size for precise commit vs. decommit + mi_subproc_stat_increase(subproc, committed, size - stat_already_committed); // for precise commit vs. decommit return true; } bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero) { - return _mi_os_commit_ex(subproc, addr, size, is_zero, size); + return _mi_os_commit_ex(subproc, addr, size, is_zero, 0); } static bool mi_os_decommit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* needs_recommit, size_t stat_size) { @@ -792,7 +813,7 @@ and possibly associated with a specific NUMA node. (use `numa_node>=0`) #define MI_HUGE_OS_PAGE_SIZE (MI_GiB) -#if (MI_INTPTR_SIZE >= 8) +#if (MI_SIZE_SIZE >= 8) // To ensure proper alignment, use our own area for huge OS pages static mi_decl_cache_align _Atomic(uintptr_t) mi_huge_start; // = 0 @@ -870,7 +891,7 @@ void* _mi_os_alloc_huge_os_pages(mi_subproc_t* subproc, size_t pages, int numa_n // no success, issue a warning and break if (p != NULL) { _mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr); - mi_os_prim_free(subproc, p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE); + mi_os_prim_free(subproc, p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, true /* adjust */); } break; } @@ -917,7 +938,7 @@ static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size if (p==NULL || size==0) return; uint8_t* base = (uint8_t*)p; while (size >= MI_HUGE_OS_PAGE_SIZE) { - mi_os_prim_free(subproc, base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE); + mi_os_prim_free(subproc, base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, false /* adjust? */); size -= MI_HUGE_OS_PAGE_SIZE; base += MI_HUGE_OS_PAGE_SIZE; } diff --git a/src/page-map.c b/src/page-map.c index 4aa99fd6b..989611285 100644 --- a/src/page-map.c +++ b/src/page-map.c @@ -293,17 +293,14 @@ static bool mi_page_map_init_once(void) { // Allocate the page map and commit bits mi_assert(MI_MAX_VABITS >= vbits); - const size_t reserve_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); + mi_assert(MI_MIN_VABITS <= vbits); + const size_t reserve_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); const size_t os_page_size = _mi_os_page_size(); const size_t reserve_size = _mi_align_up( sizeof(mi_page_map_t) + ((reserve_count - 1) * sizeof(mi_submap_t)), os_page_size); const size_t submap_size = MI_PAGE_MAP_SUB_SIZE; const size_t extra_reserve_size = reserve_size + submap_size; - #if MI_SECURE - const bool commit = true; // the whole page map is valid and we can reliably check any pointer - #else - const bool commit = false; // reserve_size <= 256*MI_KiB || // 40 virtual address bits - // mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit(); - #endif + const bool commit = (vbits == MI_MIN_VABITS) || (reserve_size <= 64*MI_KiB) || // 42 virtual address bits + mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit(); mi_subproc_t* const subproc = _mi_subproc_main(); mi_memid_t memid; mi_page_map_t* const pmap = (mi_page_map_t*)_mi_os_alloc_aligned(subproc, extra_reserve_size, 1, commit, true /* allow large */, &memid); @@ -323,15 +320,19 @@ static bool mi_page_map_init_once(void) { commit_count = mi_page_map_count_of_size(reserve_size); } else { - // commit first page + // commit first entries up to MI_MIN_VABITS entries + const size_t min_commit_count = (MI_ZU(1) << (MI_MIN_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); + const size_t min_commit_size = _mi_align_up( sizeof(mi_page_map_t) + ((min_commit_count-1) * sizeof(mi_submap_t)), os_page_size); + mi_assert_internal(min_commit_size <= reserve_size); bool is_zero; - if (!_mi_os_commit(subproc,pmap,os_page_size,&is_zero)) { + if (!_mi_os_commit(subproc,pmap,min_commit_size,&is_zero)) { mi_page_map_cannot_commit(); _mi_os_free(subproc,pmap,extra_reserve_size,memid); return false; }; mi_assert_internal(is_zero || memid.initially_zero); - commit_count = mi_page_map_count_of_size(os_page_size); + commit_count = mi_page_map_count_of_size(min_commit_size); + mi_assert_internal(commit_count >= min_commit_count); } // ensure there is a submap for the NULL address diff --git a/src/page-queue.c b/src/page-queue.c index 29860078c..c8e9b5425 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -21,11 +21,11 @@ terms of the MIT license. A copy of the license can be found in the file Minimal alignment in machine words (i.e. `sizeof(void*)`) ----------------------------------------------------------- */ -#if (MI_MAX_ALIGN_SIZE > 4*MI_INTPTR_SIZE) +#if (MI_MAX_ALIGN_SIZE > 4*MI_SIZE_SIZE) #error "define alignment for more than 4x word size for this platform" -#elif (MI_MAX_ALIGN_SIZE > 2*MI_INTPTR_SIZE) +#elif (MI_MAX_ALIGN_SIZE > 2*MI_SIZE_SIZE) #define MI_ALIGN4W // 4 machine words minimal alignment -#elif (MI_MAX_ALIGN_SIZE > MI_INTPTR_SIZE) +#elif (MI_MAX_ALIGN_SIZE > MI_SIZE_SIZE) #define MI_ALIGN2W // 2 machine words minimal alignment #else // ok, default alignment is 1 word @@ -62,7 +62,7 @@ static inline size_t mi_page_queue_count(const mi_page_queue_t* pq) { // We use `wsize` for the size in "machine word sizes", // i.e. byte size == `wsize*sizeof(void*)`. static size_t mi_bin(size_t size) { - size_t wsize = _mi_wsize_from_size(size); + size_t wsize = mi_wsize_from_size(size); #if defined(MI_ALIGN4W) if mi_likely(wsize <= 4) { return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes @@ -152,10 +152,10 @@ bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq) { for (mi_page_t* page = pq->first; page != NULL; page = page->next) { mi_assert_internal(page->prev == prev); if (mi_page_is_in_full(page)) { - mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 2); + mi_assert_internal(mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 2); } else if (mi_page_is_huge(page)) { - mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 1); + mi_assert_internal(mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 1); } else { mi_assert_internal(mi_page_block_size(page) == pq->block_size); @@ -213,15 +213,9 @@ static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_ mi_page_t* page = pq->first; if (pq->first == NULL) page = _mi_page_empty_get(); - if mi_unlikely(theap->prof_force_slow) { - // profiling: route all allocs through _mi_malloc_generic; lazy-disable here - // (cold path) if the global rate has since gone to 0. - if (_mi_prof_rate() == 0) { theap->prof_force_slow = false; theap->prof_countdown = 0; } - else { page = _mi_page_empty_get(); } - } // find index in the right direct page array - const size_t idx = _mi_wsize_from_size(size); + const size_t idx = mi_wsize_from_size(size); mi_page_t** const pages_free = theap->pages_free_direct; if (pages_free[idx] == page) return; // already set @@ -238,7 +232,7 @@ static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_ while( bin == mi_bin(prev->block_size) && prev > &theap->pages[0]) { prev--; } - start = 1 + _mi_wsize_from_size(prev->block_size); + start = 1 + mi_wsize_from_size(prev->block_size); if (start > idx) start = idx; } @@ -347,6 +341,15 @@ static void mi_page_queue_move_to_front(mi_theap_t* theap, mi_page_queue_t* queu mi_assert_internal(queue->first == page); } +static void mi_page_queue_move_to_back(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) { + mi_assert_internal(mi_page_theap(page) == theap); + mi_assert_internal(mi_page_queue_contains(queue, page)); + if (queue->last == page) return; + mi_page_queue_remove(queue, page); + mi_page_queue_push_at_end(theap, queue, page); + mi_assert_internal(queue->last == page); +} + static void mi_page_queue_enqueue_from_ex(mi_page_queue_t* to, mi_page_queue_t* from, bool enqueue_at_end, mi_page_t* page) { mi_assert_internal(page != NULL); mi_assert_internal(from->count >= 1); diff --git a/src/page.c b/src/page.c index ac0281fe1..bc0bed042 100644 --- a/src/page.c +++ b/src/page.c @@ -48,9 +48,9 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page); ----------------------------------------------------------- */ #if !MI_PADDING && !MI_ENCODE_FREELIST -// The fields up to and including `theap` are read by `mi_free` and `mi_page_alloc`; +// The fields before `xthread_free` are read by `mi_free` and `mi_page_alloc`; // they must all sit in the first cache line (this is why `purged` is at the end). -typedef char mi_page_hot_fields_first_cacheline[(offsetof(mi_page_t,theap) + sizeof(mi_theap_t*) <= 64) ? 1 : -1]; +typedef char mi_page_hot_fields_first_cacheline[(offsetof(mi_page_t,xthread_free) <= 64 && offsetof(mi_page_t,purged) > offsetof(mi_page_t,memid)) ? 1 : -1]; #endif void _mi_page_purged_reset(mi_page_t* page) { @@ -62,12 +62,12 @@ void _mi_page_purged_reset(mi_page_t* page) { // The block state of the page, as the sweep sees it (see `_mi_page_purge_holes`): `(capacity << 32) | used`. Lossless // while `capacity` fits 16 bits and `used` 32, and then `MI_PAGE_SWEPT_NONE` (all ones) is impossible: -typedef char mi_page_sweep_state_fits[(sizeof(((mi_page_t*)0)->capacity) <= 2 && - sizeof(((mi_page_t*)0)->used) <= 4) ? 1 : -1]; +// (`mi_page_used` is a 16-bit count) +typedef char mi_page_sweep_state_fits[(sizeof(((mi_page_t*)0)->capacity) <= 2) ? 1 : -1]; static inline uint64_t mi_page_sweep_state(const mi_page_t* page) { - mi_assert_internal(page->used <= page->capacity); - return (((uint64_t)page->capacity) << 32) | (uint64_t)page->used; + mi_assert_internal(mi_page_used(page) <= page->capacity); + return (((uint64_t)page->capacity) << 32) | (uint64_t)mi_page_used(page); } // Anything that changes which blocks are free *without* changing `(capacity,used)` must say so, @@ -170,7 +170,7 @@ static bool mi_page_list_is_valid(mi_page_t* page, mi_block_t* p) { static bool mi_page_is_valid_init(mi_page_t* page) { mi_assert_internal(mi_page_block_size(page) > 0); - mi_assert_internal(page->used <= page->capacity); + mi_assert_internal(mi_page_used(page) <= page->capacity); mi_assert_internal(page->capacity <= page->reserved); mi_assert_internal(page->heap!=NULL); @@ -203,8 +203,9 @@ static bool mi_page_is_valid_init(mi_page_t* page) { // blocks are conserved: used (incl. not-yet-collected thread frees) + free-listed + purged == capacity. // purged blocks are free but held OFF every free list (their memory is discarded). size_t free_count = mi_page_list_count(page, page->free) + mi_page_list_count(page, page->local_free); - mi_assert_internal(page->used + free_count + _mi_page_purged_count(page) == page->capacity); + mi_assert_internal(mi_page_used(page) + free_count + _mi_page_purged_count(page) == page->capacity); + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); return true; } @@ -247,10 +248,193 @@ bool _mi_page_is_valid(mi_page_t* page) { // mi_assert_internal(mi_theap_contains_queue(mi_page_theap(page),pq)); } } + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page)); return true; } #endif +#if MI_STATS || MI_SAMPLE +// Gets the theap belonging to a page. +static mi_theap_t* mi_theap_of_page(mi_page_t* page) { + mi_theap_t* theap = page->theap; + if mi_unlikely(mi_page_thread_id(page) != _mi_prim_thread_id()) { + // An abandoned page belongs to no theap: account on its heap (atomically). In particular not on the theap this + // thread may have for that heap: we get here from a `mi_free` of a block of another thread, and a concurrent + // `mi_heap_delete` may be detaching that theap and merging its (non-atomic) statistics (see `test-heap-mt.c`). + // Where the abandoning thread is known, it is passed explicitly (`_mi_page_update_stats_for`). + if (mi_page_is_abandoned(page)) return NULL; + theap = _mi_page_associated_theap_peek(page); + } + return theap; +} +#endif + +#if MI_SAMPLE==1 /* for ==2, the countdown is already done at every `alloc.c:mi_page_alloc_zero` */ +static void mi_theap_adjust_sample_countdown(mi_theap_t* theap, mi_page_t* page, size_t alloc_count) +{ + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + mi_assert_internal(theap!=NULL); + if (theap->sample_rate==0) return; + + const size_t last_alloc = mi_page_last_alloc(page); + if (alloc_count <= last_alloc) return; + + // update countdown + const size_t bsize = mi_page_usable_block_size(page); mi_assert_internal(bsize >= MI_PADDING_SIZE); + const size_t alloc_diff = alloc_count - last_alloc; + const uint64_t requested = (uint64_t)alloc_diff * (uint64_t)(bsize - MI_PADDING_SIZE); + if (requested <= SIZE_MAX && theap->sample_countdown >= (size_t)requested) { + theap->sample_countdown -= (size_t)requested; + } + else { + theap->sample_requested += (requested - theap->sample_countdown); + theap->sample_countdown = 0; + } +} +#endif + +#if MI_STATS +// Merge stats from the page into the corresponding theap or heap. +static void mi_theap_page_merge_stats(mi_theap_t* theap, const mi_page_t* page, size_t alloc_count, size_t free_count ) { + // get heap (as the theap might be NULL) + mi_heap_t* const heap = mi_page_heap(page); + mi_assert_internal(_mi_theap_can_touch(theap)); // (fork: also a theap that `mi_heap_delete` detached, or that a parked thread handed to the scavenger) + mi_theapx_stat_counter_increase(heap,theap,pages_stat_updates,1); + mi_theapx_stat_counter_increase(heap,theap,pages_stat_update_count, alloc_count + free_count); + + // allocation sizes + const size_t bsize = mi_page_usable_block_size(page); + const uint64_t allocated = (uint64_t)alloc_count * (uint64_t)bsize; + const uint64_t freed = (uint64_t)free_count * (uint64_t)bsize; + #if MI_STATS==1 + const uint64_t requested = allocated - ((uint64_t)alloc_count * MI_PADDING_SIZE); + #endif + mi_assert_internal(allocated <= INT64_MAX); // safe to cast to int64_t for stats + mi_assert_internal(freed <= INT64_MAX); + + // adjust stats + if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { + const size_t bin = _mi_bin(bsize); + // allocations + if (alloc_count > 0) { + mi_theapx_stat_counter_increase(heap, theap, malloc_normal_count, alloc_count); + mi_theapx_stat_increase(heap, theap, malloc_normal, allocated); + mi_theapx_stat_increase(heap, theap, malloc_bins[bin], alloc_count); + #if MI_STATS==1 + // use coarse total requested bytes + mi_theapx_stat_counter_increase(heap, theap, malloc_requested, requested); + #endif + } + // frees + if (free_count > 0) { + mi_theapx_stat_decrease(heap, theap, malloc_normal, freed); + mi_theapx_stat_decrease(heap, theap, malloc_bins[bin], free_count); + } + } + else { + // allocations + mi_assert_internal(alloc_count<=1); + mi_assert_internal(free_count<=1); + if (alloc_count > 0) { + mi_theapx_stat_counter_increase(heap, theap, malloc_huge_count, alloc_count); + mi_theapx_stat_increase(heap, theap, malloc_huge, allocated); + #if MI_STATS==1 + // use coarse total requested bytes + mi_theapx_stat_counter_increase(heap, theap, malloc_requested, requested); + #endif + } + // frees + if (free_count > 0) { + mi_theapx_stat_decrease(heap, theap, malloc_huge, freed); + } + } +} + +// Update stats for a page +static void mi_theap_page_update_stats(mi_theap_t* theap, mi_page_t* page) { + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page)); + + // get stat counts + const size_t used = mi_page_used(page); + const size_t alloc_count = mi_page_alloc_count(page); + const size_t last_used = mi_page_last_used(page); + mi_assert_internal(last_used + alloc_count >= used); + mi_assert_internal(used <= UINT16_MAX); + const size_t free_count = last_used + alloc_count - used; + + #if MI_SAMPLE==1 // for ==2 it is already counted in every `alloc.c:mi_page_alloc_zero` + if (theap!=NULL) { mi_theap_adjust_sample_countdown(theap,page,alloc_count); } + #endif + + // update stats? + if (alloc_count + free_count == 0) { + mi_assert_internal(last_used == used); + return; + } + + // reset the `alloc_count` (and `last_alloc`), and set `last_used` to `used` + #if MI_SIZE_SIZE >= 8 + page->xused.used_alloc = (used << 32) | used; + #else + page->xused.used_alloc = used; + page->xlast_used = (uint16_t)used; + page->xlast_alloc = 0; + #endif + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page)); + + mi_theap_page_merge_stats(theap, page, alloc_count, free_count); +} + +void _mi_page_update_stats(mi_page_t* page) { // called on abandoned pages etc. + mi_theap_page_update_stats(mi_theap_of_page(page),page); +} + +// `theapx`: a theap the calling thread may touch (`_mi_theap_can_touch`), or NULL to account on the heap of the page +void _mi_page_update_stats_for(mi_page_t* page, mi_theap_t* theapx) { + mi_theap_page_update_stats(theapx,page); +} + +#else +void _mi_page_update_stats(mi_page_t* page) { + MI_UNUSED(page); +} +void _mi_page_update_stats_for(mi_page_t* page, mi_theap_t* theapx) { + MI_UNUSED(page); MI_UNUSED(theapx); +} +#endif + +static void mi_page_update_sample_countdown(mi_page_t* page) +{ + // adjust count down + const size_t alloc_count = mi_page_alloc_count(page); + if (alloc_count==0) { + return; + } + else if mi_unlikely(alloc_count>=0x8000) { // if the count could overflow, update stats so the counter is reset + _mi_page_update_stats(page); + } + #if MI_SAMPLE==1 // if ==2 the countdown is already always counted in `alloc.c:mi_page_alloc_zero_ex` + else { + mi_theap_t* theap = mi_theap_of_page(page); + if (theap==NULL) return; + mi_theap_adjust_sample_countdown(theap,page,alloc_count); + // update last_alloc to alloc_count + mi_assert_internal(alloc_count <= UINT16_MAX); + #if MI_SIZE_SIZE >= 8 + page->xused.used_alloc = (alloc_count << 48) | (page->xused.used_alloc & (~MI_ZU(0) >> 16)); + #else + page->xlast_alloc = (uint16_t)alloc_count; + #endif + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page)); + + } + #endif +} + /* ----------------------------------------------------------- Page collect the `local_free` and `thread_free` lists @@ -276,7 +460,7 @@ static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head) return; // the thread-free items cannot be freed } // if `count > page->used` there was another kind memory corruption (either in the page meta-data or in the linked list) - else if mi_unlikely(count > page->used) { + else if mi_unlikely(count > mi_page_used(page)) { _mi_error_message(EFAULT, "corrupted meta-data in thread-free list\n"); return; // the thread-free items cannot be freed } @@ -287,8 +471,8 @@ static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head) // update counts now mi_assert_internal(count <= UINT16_MAX); - mi_assert_internal(page->used >= (uint16_t)count); - page->used = page->used - (uint16_t)count; + mi_assert_internal(mi_page_used(page) >= count); + page->xused.used_alloc -= count; // page->used = page->used - (uint16_t)count; } // Collect the local `thread_free` list using an atomic exchange. @@ -309,6 +493,7 @@ static void mi_page_thread_free_collect(mi_page_t* page) mi_page_thread_collect_to_local(page, head); } + // returns `true` if after collection `mi_page_immediate_available` is true. static inline bool mi_page_free_quick_collect(mi_page_t* page) { if mi_likely(page->free != NULL) return true; @@ -316,7 +501,8 @@ static inline bool mi_page_free_quick_collect(mi_page_t* page) { // move local_free to free page->free = page->local_free; page->local_free = NULL; - page->free_is_zero = false; + page->free_is_zero = false; + mi_page_update_sample_countdown(page); return true; } @@ -448,7 +634,7 @@ void _mi_page_holes_count_ineligible(const mi_page_t* page) { // Eligibility is fixed for a page's lifetime (page size, block size, memid), so an ineligible // page never carries a purged block and the last term is 0 -- O(1), on every page of every sweep. mi_assert_internal(!mi_page_has_purged(page)); - const size_t nfree = (size_t)(page->capacity - page->used); + const size_t nfree = (size_t)(page->capacity - mi_page_used(page)); mi_atomic_addi64_relaxed(&mi_holes_inelig_pages, 1); mi_atomic_addi64_relaxed(&mi_holes_inelig_bytes, (int64_t)mi_page_size(page)); mi_atomic_addi64_relaxed(&mi_holes_inelig_free_bytes, (int64_t)(nfree * page->block_size)); @@ -1214,8 +1400,8 @@ static void mi_page_free_collect_ex(mi_page_t* page, bool force, bool allow_unpu page->local_free = NULL; page->free_is_zero = false; } - } - + mi_page_update_sample_countdown(page); + } mi_assert_internal(!force || page->local_free == NULL); // Free list empty but this page has discarded holes: bring a whole run of them @@ -1238,12 +1424,13 @@ void _mi_page_free_collect_no_unpurge(mi_page_t* page, bool force) { // Collect elements in the thread-free list starting at `head`. This is an optimized // version of `_mi_page_free_collect` to be used from `free.c:_mi_free_collect_mt` that avoids atomic access to `xthread_free`. +// returns a possibly updated expected value for the thread_free pointer. // // `head` must be in the `xthread_free` list. It will not collect `head` itself // so the `used` count is not fully updated in general. However, if the `head` is // the last remaining element, it will be collected and the used count will become `0` (so `mi_page_all_free` becomes true). -void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { - if (head == NULL) return; +mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { + if (head == NULL) return NULL; mi_block_t* next = mi_block_next(page,head); // we cannot collect the head element itself as `page->thread_free` may point to it (and we want to avoid atomic ops) if (next != NULL) { mi_block_set_next(page, head, NULL); @@ -1252,13 +1439,18 @@ void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { page->free = page->local_free; page->local_free = NULL; page->free_is_zero = false; - } + mi_page_update_sample_countdown(page); + } } - if (page->used == 1) { + if (mi_page_used(page) == 1) { // all elements are free'd since we skipped the `head` element itself mi_assert_internal(mi_tf_block(mi_atomic_load_relaxed(&page->xthread_free)) == head); mi_assert_internal(mi_block_next(page,head) == NULL); _mi_page_free_collect(page, false); // collect the final element + return NULL; + } + else { + return head; } } @@ -1277,6 +1469,7 @@ void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page) mi_page_set_theap(page,theap); _mi_page_free_collect(page, false); // ensure used count is up to date + mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page); mi_page_queue_push_at_end(theap, pq, page); mi_assert_expensive(_mi_page_is_valid(page)); @@ -1381,6 +1574,7 @@ static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) { // put full pages in a theap local queue (this is for theaps that cannot abandon, for example, if the theap can be destroyed) mi_page_queue_enqueue_from(&mi_page_theap(page)->pages[MI_BIN_FULL], pq, page); _mi_page_free_collect(page, false); // try to collect right away in case another thread freed just before MI_USE_DELAYED_FREE was set + _mi_page_update_stats(page); // we must update stats here as mi_theap_collect does not normally visit full pages } } @@ -1411,7 +1605,8 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) { // _mi_arenas_collect(false, false, theap->tld); // allow purging } -#define MI_RETIRE_CYCLES (16) +#define MI_RETIRE_CYCLES (16) /* keep a retired page around for about 16 "admin cycles" before free'ing it */ +#define MI_RETIRE_MAX_PAGES (3) /* keep at most N pages per size bin as retired */ // Retire a page with no more used blocks // Important to not retire too quickly though as new @@ -1437,12 +1632,10 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { mi_page_queue_t* pq = mi_page_queue_of(page); #if MI_RETIRE_CYCLES > 0 const size_t bsize = mi_page_block_size(page); - if mi_likely( /* bsize < MI_MAX_RETIRE_SIZE && */ !mi_page_queue_is_special(pq)) { // not full or huge queue? - if (pq->last==page && pq->first==page) { // the only page in the queue? + if mi_likely( pq->count <= MI_RETIRE_MAX_PAGES && !mi_page_queue_is_special(pq)) { // not full or huge queue? + if (pq->count==1 || bsize < MI_SMALL_SIZE_MAX) { mi_theap_t* theap = mi_page_theap(page); - #if MI_STAT>0 mi_theap_stat_counter_increase(theap, pages_retire, 1); - #endif page->retire_expire = (bsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4); mi_assert_internal(pq >= theap->pages); const size_t index = pq - theap->pages; @@ -1451,7 +1644,7 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { if (index > theap->page_retired_max) theap->page_retired_max = index; mi_assert_internal(mi_page_all_free(page)); return; // don't free after all - } + } } #endif _mi_page_free(page, pq); @@ -1479,29 +1672,36 @@ static void mi_theap_collect_full_pages(mi_theap_t* theap) { } } +static void mi_page_try_retire(mi_page_queue_t* pq, mi_page_t* page, size_t bin, bool force, size_t* min, size_t* max) { + mi_assert_internal(page!=NULL && page->retire_expire!=0); + if (mi_page_all_free(page)) { + page->retire_expire--; + if (page->retire_expire == 0 || force) { + _mi_page_free(page, pq); + } + else { + // keep retired, update min/max + if (bin < *min) *min = bin; + if (bin > *max) *max = bin; + } + } + else { + page->retire_expire = 0; + } +} + // free retired pages: we don't need to look at the entire queues // since we only retire pages that are at the head position in a queue. void _mi_theap_collect_retired(mi_theap_t* theap, bool force) { size_t min = MI_BIN_FULL; size_t max = 0; for(size_t bin = theap->page_retired_min; bin <= theap->page_retired_max; bin++) { - mi_page_queue_t* pq = &theap->pages[bin]; - mi_page_t* page = pq->first; - if (page != NULL && page->retire_expire != 0) { - if (mi_page_all_free(page)) { - page->retire_expire--; - if (page->retire_expire == 0 || force) { - _mi_page_free(page, pq); - } - else { - // keep retired, update min/max - if (bin < min) min = bin; - if (bin > max) max = bin; - } - } - else { - page->retire_expire = 0; - } + mi_page_queue_t* pq = &theap->pages[bin]; + mi_page_t* page = pq->first; + for (int i = 0; iretire_expire!=0; i++) { + mi_page_t* next = page->next; + mi_page_try_retire(pq,page,bin,force,&min,&max); + page = next; } } theap->page_retired_min = min; @@ -1552,15 +1752,15 @@ static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t* // and initialize the free list by randomly threading through them // set up first element - const uintptr_t r = _mi_theap_random_next(theap); + const size_t r = _mi_theap_random_next(theap); size_t current = r % slice_count; counts[current]--; mi_block_t* const free_start = blocks[current]; // and iterate through the rest; use `random_shuffle` for performance - uintptr_t rnd = _mi_random_shuffle(r|1); // ensure not 0 + size_t rnd = _mi_random_shuffle(r|1); // ensure not 0 for (size_t i = 1; i < extend; i++) { - // call random_shuffle only every INTPTR_SIZE rounds - const size_t round = i%MI_INTPTR_SIZE; + // call random_shuffle only every SIZE_SIZE rounds + const size_t round = i%MI_SIZE_SIZE; if (round == 0) rnd = _mi_random_shuffle(rnd); // select a random next slice index size_t next = ((rnd >> 8*round) & (slice_count-1)); @@ -1609,7 +1809,7 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co Page initialize and extend the capacity ----------------------------------------------------------- */ -#define MI_MAX_EXTEND_SIZE (4*1024) // heuristic, one OS page seems to work well. +#define MI_MAX_EXTEND_SIZE (8*1024) // heuristic, one or two OS pages seems to work well. #if (MI_SECURE>=2) #define MI_MIN_EXTEND (8*MI_SECURE) // extend at least by this many #else @@ -1633,10 +1833,8 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { size_t page_size; //uint8_t* page_start = mi_page_area(page, &page_size); - #if MI_STAT>0 mi_theap_stat_counter_increase(theap, pages_extended, 1); - #endif - + // calculate the extend count const size_t bsize = mi_page_block_size(page); size_t extend = (size_t)page->reserved - page->capacity; @@ -1697,9 +1895,7 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { } // enable the new free list page->capacity += (uint16_t)extend; - #if MI_STAT>0 mi_theap_stat_increase(theap, page_committed, extend * bsize); - #endif mi_assert_expensive(mi_page_is_valid_init(page)); return true; } @@ -1720,8 +1916,10 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) { mi_assert_internal(page->reserved > 0); #if (MI_PADDING || MI_ENCODE_FREELIST) page->keys[0] = _mi_theap_random_next(theap); + #if MI_PAGE_KEY_COUNT==2 page->keys[1] = _mi_theap_random_next(theap); #endif + #endif #if MI_DEBUG>2 if (page->memid.initially_zero) { mi_track_mem_defined(mi_page_start(page), mi_page_committed(page)); @@ -1735,7 +1933,8 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) { mi_assert_internal(page->theap == mi_page_theap(page)); mi_assert_internal(page->capacity == 0); mi_assert_internal(page->free == NULL); - mi_assert_internal(page->used == 0); + mi_assert_internal(mi_page_used(page) == 0); + mi_assert_internal(page->xused.used_alloc == 0); mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(page->xthread_free == 1); mi_assert_internal(page->next == NULL); @@ -1744,8 +1943,10 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) { mi_assert_internal(!mi_page_has_interior_pointers(page)); #if (MI_PADDING || MI_ENCODE_FREELIST) mi_assert_internal(page->keys[0] != 0); + #if MI_PAGE_KEY_COUNT==2 mi_assert_internal(page->keys[1] != 0); #endif + #endif mi_assert_expensive(mi_page_is_valid_init(page)); // initialize an initial free list @@ -1768,8 +1969,9 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, long page_full_retain = (pq->block_size > MI_SMALL_MAX_OBJ_SIZE ? 0 : theap->page_full_retain); // only retain small pages mi_page_t* page_candidate = NULL; // a page with free space mi_page_t* page = pq->first; + mi_page_t* const last = pq->last; - while (page != NULL) + while (page!=NULL) { mi_page_t* next = page->next; // remember next (as this page can move to another queue) count++; @@ -1793,6 +1995,10 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page)); mi_page_to_full(page, pq); } + else if (page!=last && pq->last!=page) { + // avoid revisiting this page for a while + mi_page_queue_move_to_back(theap, pq, page); + } } else { // the page has free space, make it a candidate @@ -1806,7 +2012,7 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, page_candidate = page; } // prefer to reuse fuller pages (in the hope the less used page gets freed) - else if (page->used >= page_candidate->used && !mi_page_is_mostly_used(page)) { // && !mi_page_is_expandable(page)) { + else if (mi_page_used(page) >= mi_page_used(page_candidate) && !mi_page_is_mostly_used(page)) { // && !mi_page_is_expandable(page)) { page_candidate = page; } // if we find a non-expandable candidate, or searched for N pages, return with the best candidate @@ -1828,8 +2034,8 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page)); mi_page_to_full(page, pq); #endif - - page = next; + if (page==last) { page=NULL; } // don't revisit earlier pages that moved to the back + else { page = next; } } // for each page mi_theap_stat_counter_increase(theap, page_searches, count); @@ -1936,8 +2142,8 @@ static mi_page_t* mi_huge_page_alloc(mi_theap_t* theap, size_t size, size_t page mi_assert_internal(mi_page_is_abandoned(page)); mi_page_set_theap(page, NULL); #endif - mi_theap_stat_increase(theap, malloc_huge, mi_page_block_size(page)); - mi_theap_stat_counter_increase(theap, malloc_huge_count, 1); + // mi_theap_stat_increase(theap, malloc_huge, mi_page_block_size(page)); + // mi_theap_stat_counter_increase(theap, malloc_huge_count, 1); } return page; } @@ -1946,7 +2152,7 @@ static mi_page_t* mi_huge_page_alloc(mi_theap_t* theap, size_t size, size_t page // Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. static mi_page_t* mi_find_page(mi_theap_t* theap, size_t size, size_t huge_alignment) mi_attr_noexcept { const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` - if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE) { + if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE) { // TODO: remove as we check in generic_fallback ? _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size); return NULL; } @@ -2006,35 +2212,54 @@ void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noex /* ----------------------------------------------------------- Admin ----------------------------------------------------------- */ +static mi_theap_t* mi_theap_init(mi_theap_t* theap) { + if mi_likely(mi_theap_is_initialized(theap)) return theap; + if (theap==&_mi_theap_empty_wrong) { + // we were unable to allocate a theap for a first-class heap + return NULL; + } + // otherwise we initialize the thread and its default theap + theap = _mi_thread_init(); + if mi_unlikely(!mi_theap_is_initialized(theap)) { return NULL; } + mi_assert_internal(mi_theap_is_initialized(theap)); + return theap; +} static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) { - if mi_unlikely(!mi_theap_is_initialized(theap)) { - if (theap==&_mi_theap_empty_wrong) { - // we were unable to allocate a theap for a first-class heap - return NULL; - } - // otherwise we initialize the thread and its default theap - theap = _mi_thread_init(); - if mi_unlikely(!mi_theap_is_initialized(theap)) { return NULL; } - } + theap = mi_theap_init(theap); + if (theap==NULL) return NULL; mi_assert_internal(mi_theap_is_initialized(theap)); // do administrative tasks every N generic mallocs if mi_unlikely(theap->generic_count >= 1000) { theap->generic_collect_count += theap->generic_count; theap->generic_count = 0; - + + // check if the profiler is enabled + mi_heap_t* const heap = _mi_theap_heap(theap); + mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler); + const bool prof_enabled = (prof!=NULL && mi_profiler_is_enabled(prof)); + if (prof_enabled && theap->profile_sample_rate==0) { + _mi_theap_set_profile_sample_rate(theap,mi_max(1,prof->initial_sample_rate)); // start profiling + } + else if (!prof_enabled && theap->profile_sample_rate!=0) { + _mi_theap_set_profile_sample_rate(theap,0); // stop profiling + } + // do a full theap collect every once in a while (10000 by default) const long generic_collect = mi_option_get_clamp(mi_option_generic_collect, 1, 1000000L); if (theap->generic_collect_count >= generic_collect) { theap->generic_collect_count = 0; mi_theap_collect(theap, false /* force? */); } + // else if (theap->sample_rate != 0) { // update stats more aggressively if we are sampling + // mi_theap_collect(theap, false /* force? */); // TODO: make specialized mi_theap_collect_update_stats ? + // } else { // otherwise we do a mini-collect _mi_deferred_free(theap, false); // call potential deferred free routines - _mi_theap_collect_retired(theap, false); // free retired pages + _mi_theap_collect_retired(theap, false); // free retired pages } } return theap; @@ -2044,25 +2269,41 @@ static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) Generic allocation ----------------------------------------------------------- */ -// Heap profiling (fork): while profiling is on, the direct pages are poisoned so every allocation comes through the generic -// path, and this is checked at each of its exits; zero overhead on the real fast path. Gated on the global rate so a thread -// picks up profiling that another thread enabled. -static inline void mi_prof_note_generic_alloc(mi_theap_t* theap, mi_page_t* page, void* p) { - if mi_unlikely(p != NULL && _mi_prof_rate() != 0) { - if (theap->prof_countdown == 0) { _mi_prof_theap_lazy_enable(theap); } - const size_t bsize = mi_page_block_size(page); - if ((theap->prof_countdown -= (intptr_t)bsize) <= 0) { - _mi_prof_sample(theap, page, p, bsize); - } - } -} - static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) { // initialize if necessary theap = mi_malloc_generic_admin(theap); if (theap==NULL) return NULL; + // check allocation size + const size_t req_size = size - MI_PADDING_SIZE; + if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { + _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size); + return NULL; + } + + // take a sample? + bool sample_countdown_is_adjusted = false; + if mi_unlikely(mi_theap_should_sample(theap,req_size)) { + if (huge_alignment==0 && theap->sample_rate!=0) { + return _mi_theap_malloc_sampled(theap,req_size,zero,ppage); + } + mi_assert_internal(!_mi_is_empty_theap(theap)); // cannot write to the empty theap + mi_assert_internal(req_size <= MI_SAMPLE_COUNTDOWN_MAX); + if (theap->sample_rate==0) { + theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // reset the countdown counter + } + #if MI_SAMPLE==2 + else { + // huge_alignment!=0 so we need to adjust the countdown + mi_assert_internal(huge_alignment!=0); + mi_assert_internal(theap->sample_countdown <= SIZE_MAX - req_size); + sample_countdown_is_adjusted = true; + theap->sample_countdown += req_size; // adjust such that after `_mi_page_malloc_zero` the countdown is correct again + } + #endif + } + // find (or allocate) a page of the right size mi_page_t* page = mi_find_page(theap, size, huge_alignment); if mi_unlikely(page == NULL) { // first time out of memory, try to collect and retry the allocation once more @@ -2071,9 +2312,9 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size } if mi_unlikely(page == NULL) { // out of memory - const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` + if (sample_countdown_is_adjusted) { theap->sample_countdown -= req_size; } // unadjust again if we failed to find a page _mi_error_message(ENOMEM, "unable to allocate memory (%zu bytes)\n", req_size); - return NULL; + return NULL; } mi_assert_internal(mi_page_immediate_available(page)); @@ -2081,17 +2322,19 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); - // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc) + // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc_zero) if (ppage!=NULL) { *ppage = page; } void* const p = _mi_page_malloc_zero(theap,page,size,zero); mi_assert_internal(p != NULL); - - mi_prof_note_generic_alloc(theap, page, p); - + mi_page_update_sample_countdown(page); + // move full pages to the full queue - if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE && mi_page_is_full(page)) { - mi_page_to_full(page, mi_page_queue_of(page)); - } + // this will also call _mi_page_update_stats for huge pages + if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE) { + if (mi_page_is_full(page)) { + mi_page_to_full(page, mi_page_queue_of(page)); + } + } return p; } @@ -2113,21 +2356,38 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm // fast path objects that fit in a small page if mi_likely(mi_theap_is_initialized(theap) && ++theap->generic_count < 1000 && huge_alignment==0) { - const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` - if (req_size < MI_SMALL_MAX_OBJ_SIZE) { - mi_page_queue_t* pq = mi_page_queue(theap, size); - mi_assert_internal(pq!=NULL && !mi_page_queue_is_huge(pq)); - page = mi_page_queue_find_free(theap,pq); - // mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); - if (page!=NULL) { - if (ppage!=NULL) { *ppage = page; } - mi_assert_internal(mi_page_immediate_available(page)); - void* const p = _mi_page_malloc_zero(theap,page,size,zero); - mi_prof_note_generic_alloc(theap, page, p); - return p; + const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` + if (req_size < MI_SMALL_MAX_OBJ_SIZE) + { + #if MI_SAMPLE + if mi_likely(!mi_theap_should_sample(theap,req_size)) // ensure we don't need to take a sample + #endif + { + mi_page_queue_t* pq = mi_page_queue(theap, size); + mi_assert_internal(pq!=NULL && !mi_page_queue_is_huge(pq)); + page = mi_page_queue_find_free(theap,pq); + // mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); + if (page!=NULL) { + if (ppage!=NULL) { *ppage = page; } + mi_assert_internal(mi_page_immediate_available(page)); // we should never recurse in _mi_page_malloc_zero + return _mi_page_malloc_zero(theap,page,size,zero); + } } } } // otherwise fallback - return mi_malloc_generic_fallback(theap,size,zero, huge_alignment,ppage); + return mi_malloc_generic_fallback(theap,size,zero,huge_alignment,ppage); +} + +void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { + theap = mi_theap_init(theap); + if (theap==NULL) return NULL; + const size_t sample_rate = theap->sample_rate; + const size_t sample_countdown = theap->sample_countdown; + theap->sample_rate = 0; // prevent a recursive call to mi_theap_malloc_sampled from _mi_malloc_generic + theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; + void* p = _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); + theap->sample_rate = sample_rate; + theap->sample_countdown = sample_countdown; + return p; } diff --git a/src/prim/osx/alloc-override-zone.c b/src/prim/osx/alloc-override-zone.c index 201c15860..77f65503b 100644 --- a/src/prim/osx/alloc-override-zone.c +++ b/src/prim/osx/alloc-override-zone.c @@ -75,20 +75,12 @@ static void* zone_valloc(malloc_zone_t* zone, size_t size) { } static void zone_free(malloc_zone_t* zone, void* p) { - mi_page_t* const page = _mi_checked_ptr_page(p); - if mi_likely(page!=NULL) { - mi_assert_internal(_mi_thread_is_initialized()); - _mi_free_in_page(p,page); - // if mi_likely(_mi_thread_is_initialized()) { - // _mi_free_in_page(p,page); - // } - // else { - // // during thread shutdown `_pthread_tsd_cleanup` may call `zone_free` on a pointer that was allocated in another subproc. - // _mi_free_subproc_safe_in_page(p,page); - // } - } - else if (!is_mimalloc_zone(zone)) { // can happen due to interpose - zone->free(zone,p); + // during C++ thread shutdown `_pthread_tsd_cleanup` may call `zone_free` + // after mimalloc mi_thread_done, and also on a pointer that was allocated in another subproc. + if mi_unlikely(!mi_cfree(p)) { + if (!is_mimalloc_zone(zone)) { // can happen due to interpose + zone->free(zone,p); + } } } @@ -228,8 +220,8 @@ static kern_return_t mi_zone_enum_remote_page(task_t task, memory_reader_t reade const size_t ubsize = bsize - MI_PADDING_SIZE; // `lpage` is a local copy of a page that lives in the TARGET process, so the block start // must be rebuilt from the remote page address -- mi_page_start(&lpage) would offset from - // our own copy. (The start is stored as an offset from the page struct, in `MI_MAX_ALIGN_SIZE` units.) - const vm_address_t pstart = (vm_address_t)(rpage + ((size_t)lpage.page_ma_offset * MI_MAX_ALIGN_SIZE)); + // our own copy. (The start is stored as a byte offset from the page struct.) + const vm_address_t pstart = (vm_address_t)(rpage + lpage.page_offset); if (e->type_mask & MALLOC_PTR_REGION_RANGE_TYPE) { mi_zone_enum_flush(e, MALLOC_PTR_IN_USE_RANGE_TYPE); @@ -237,7 +229,7 @@ static kern_return_t mi_zone_enum_remote_page(task_t task, memory_reader_t reade mi_zone_enum_flush(e, MALLOC_PTR_REGION_RANGE_TYPE); } if (!(e->type_mask & MALLOC_PTR_IN_USE_RANGE_TYPE)) return KERN_SUCCESS; - if (lpage.used == 0) return KERN_SUCCESS; + if (mi_page_used(&lpage) == 0) return KERN_SUCCESS; mi_block_t* xtf = (mi_block_t*)((uintptr_t)lpage.xthread_free & ~(uintptr_t)1); if (lpage.free == NULL && lpage.local_free == NULL && xtf == NULL) { @@ -270,8 +262,9 @@ static kern_return_t mi_zone_enum_remote_page(task_t task, memory_reader_t reade return KERN_SUCCESS; } +// `rarena_start`: the start of the arena memory area (`arena->start`; the `mi_arena_t` itself comes after the aligned page meta slices) static kern_return_t mi_zone_enum_remote_bitmap(task_t task, memory_reader_t reader, - mi_zone_enum_t* e, vm_address_t rarena, + mi_zone_enum_t* e, vm_address_t rarena_start, vm_address_t rpages_meta, vm_address_t rbitmap) { size_t chunk_count; @@ -284,16 +277,29 @@ static kern_return_t mi_zone_enum_remote_bitmap(task_t task, memory_reader_t rea while (b != 0) { size_t bit = mi_ctz(b); size_t slice_index = c*MI_BCHUNK_BITS + f*MI_BFIELD_BITS + bit; + vm_address_t rpage = rarena_start + slice_index * MI_ARENA_SLICE_SIZE; + #if MI_PAGE_META_IS_ALIGNED + // mirror `mi_arena_page_at_slice`: the aligned page meta entry of the slice points to the actual page + // struct through `self` (also for small-block pages that keep it at the slice start). + // (`_mi_aligned_ptr_page0` is address arithmetic only, so it is fine on a remote address) + MI_UNUSED(rpages_meta); + const vm_address_t rmeta = (vm_address_t)_mi_aligned_ptr_page0((const void*)rpage); + vm_address_t rself = 0; + MI_ZR(&rself, rmeta + offsetof(mi_page_t, self), sizeof(rself)); + rpage = rself; + #else // mirror `mi_arena_page_at_slice`: with separated metadata, small-block // pages still keep the page struct at the slice start (block_size==0 in // pages_meta marks that case). - vm_address_t rpage = rarena + slice_index * MI_ARENA_SLICE_SIZE; if (rpages_meta != 0) { vm_address_t rmeta = rpages_meta + slice_index * sizeof(mi_page_t); size_t bs; MI_ZR(&bs, rmeta + offsetof(mi_page_t, block_size), sizeof(bs)); if (bs > 0) rpage = rmeta; } - if (mi_zone_enum_remote_page(task, reader, e, rpage) != KERN_SUCCESS) return KERN_FAILURE; + #endif + if (rpage != 0) { + if (mi_zone_enum_remote_page(task, reader, e, rpage) != KERN_SUCCESS) return KERN_FAILURE; + } b &= b - 1; } } @@ -323,7 +329,7 @@ static kern_return_t intro_enumerator(task_t task, void* context, mi_arena_t* ra = lsubproc.arenas[i]; if (ra == NULL) continue; mi_arena_t la; MI_ZR(&la, ra, sizeof(la)); - mi_zone_enum_push(&e, MALLOC_ADMIN_REGION_RANGE_TYPE, ra, la.slice_count * MI_ARENA_SLICE_SIZE); + mi_zone_enum_push(&e, MALLOC_ADMIN_REGION_RANGE_TYPE, la.start, la.slice_count * MI_ARENA_SLICE_SIZE); } mi_zone_enum_flush(&e, MALLOC_ADMIN_REGION_RANGE_TYPE); } @@ -343,7 +349,7 @@ static kern_return_t intro_enumerator(task_t task, void* context, mi_arena_t la; MI_ZR(&la, ra, sizeof(la)); mi_arena_pages_t lap; MI_ZR(&lap, rap, sizeof(lap)); kern_return_t kr = mi_zone_enum_remote_bitmap(task, reader, &e, - (vm_address_t)ra, (vm_address_t)la.pages_meta, (vm_address_t)lap.pages); + (vm_address_t)la.start, (vm_address_t)la.pages_meta, (vm_address_t)lap.pages); if (kr != KERN_SUCCESS) return kr; } // OS-allocated abandoned pages (not in any arena bitmap) diff --git a/src/prim/prim-tls.c b/src/prim/prim-tls.c index 8c8b0c71f..1b79321a0 100644 --- a/src/prim/prim-tls.c +++ b/src/prim/prim-tls.c @@ -46,9 +46,9 @@ mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { // If we can, we use one of the 64 direct TLS slots (but fall back to expansion slots if needed) // See for the offsets. #if MI_SIZE_SIZE==4 -#define MI_TLS_DIRECT_FIRST (0x0E10 / MI_INTPTR_SIZE) +#define MI_TLS_DIRECT_FIRST (0x0E10 / MI_SIZE_SIZE) #else -#define MI_TLS_DIRECT_FIRST (0x1480 / MI_INTPTR_SIZE) +#define MI_TLS_DIRECT_FIRST (0x1480 / MI_SIZE_SIZE) #endif #define MI_TLS_DIRECT_SLOTS (64) #define MI_TLS_EXPANSION_SLOTS (1024) diff --git a/src/prim/prim.c b/src/prim/prim.c index d7912dae7..afc026ae2 100644 --- a/src/prim/prim.c +++ b/src/prim/prim.c @@ -29,9 +29,8 @@ terms of the MIT license. A copy of the license can be found in the file // Generic process initialization #if !defined(MI_PRIM_HAS_PROCESS_ATTACH) #if defined(__GNUC__) || defined(__clang__) - // gcc,clang: use the constructor/destructor attribute - // which for both seem to run before regular constructors/destructors - #if defined(__clang__) + // gcc,clang: use the constructor/destructor attribute with the highest user priority (<=100 is reserved for the compiler itself) + #if defined(__clang__) || (__GNUC__ >= 5 && !defined(__APPLE__)) // gcc has support since 4.2 but not on macOS (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=119435) #define mi_attr_constructor __attribute__((constructor(101))) #define mi_attr_destructor __attribute__((destructor(101))) #else diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 7040fede4..37c4e7cae 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -347,7 +347,7 @@ static void* unix_mmap_prim_aligned(void* addr, size_t size, size_t try_alignmen // fall back to regular mmap } #endif - #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED) + #if (MI_SIZE_SIZE >= 8) && !defined(MAP_ALIGNED) // on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations if (addr == NULL) { void* hint = _mi_os_get_aligned_hint(try_alignment, size); @@ -471,15 +471,17 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec #if !defined(MI_NO_THP) if (p != NULL && allow_large && mi_option_is_enabled(mi_option_allow_thp) && _mi_os_canuse_large_page(size, try_alignment)) { #if defined(MADV_HUGEPAGE) - // Many Linux systems don't allow MAP_HUGETLB but they support instead - // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE - // though since properly aligned allocations will already use large pages if available - // in that case -- in particular for our large regions (in `memory.c`). - // However, some systems only allow THP if called with explicit `madvise`, so - // when large OS pages are enabled for mimalloc, we call `madvise` anyways. - if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) { - // *is_large = true; // possibly - }; + if (_mi_os_canuse_thp()) { + // Many Linux systems don't allow MAP_HUGETLB but they support instead + // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE + // though since properly aligned allocations will already use large pages if available + // in that case -- in particular for our large regions (in `memory.c`). + // However, some systems only allow THP if called with explicit `madvise`, so + // when large OS pages are enabled for mimalloc, we call `madvise` anyways. + if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) { + // *is_large = true; // possibly + }; + } #elif defined(__sun) struct memcntl_mha cmd = {0}; cmd.mha_pagesize = _mi_os_large_page_size(); @@ -519,7 +521,7 @@ int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool comm //--------------------------------------------- static void unix_mprotect_hint(int err) { - #if defined(__linux__) && (MI_SECURE>=5) // guard page around every mimalloc page + #if defined(__linux__) && (MI_SECURE>=5 || MI_GUARDED) // guard page around every mimalloc page if (err == ENOMEM) { _mi_warning_message("The next warning may be caused by a low memory map limit.\n" " On Linux this is controlled by the vm.max_map_count -- maybe increase it?\n" @@ -703,7 +705,7 @@ int _mi_prim_protect(void* start, size_t size, bool protect) { // Huge page allocation //--------------------------------------------- -#if (MI_INTPTR_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__) +#if (MI_SIZE_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__) #ifndef MPOL_PREFERRED #define MPOL_PREFERRED 1 @@ -724,12 +726,12 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo bool is_large = true; *is_zero = true; *addr = unix_mmap(hint_addr, size, MI_ARENA_SLICE_ALIGN, PROT_READ | PROT_WRITE, true, true, &is_large); - if (*addr != NULL && numa_node >= 0 && numa_node < (8*MI_INTPTR_SIZE - 1)) { // at most 63 nodes + if (*addr != NULL && numa_node >= 0 && numa_node < (8*MI_SIZE_SIZE - 1)) { // at most 63 nodes unsigned long numa_mask = (1UL << numa_node); // todo: does `mbind` work correctly for huge OS pages? should we // use `set_mempolicy` before calling mmap instead? // see: - long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); + long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_SIZE_SIZE, 0); if (err != 0) { err = errno; _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %ld (0x%lx))\n", numa_node, err, err); diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index f46f8d544..a95915061 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -281,7 +281,7 @@ int _mi_prim_free(void* addr, size_t size ) { //--------------------------------------------- static void* win_virtual_alloc_prim_once(void* addr, size_t size, size_t try_alignment, DWORD flags) { - #if (MI_INTPTR_SIZE >= 8) + #if (MI_SIZE_SIZE >= 8) // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations if (addr == NULL) { void* hint = _mi_os_get_aligned_hint(try_alignment,size); diff --git a/src/prof.c b/src/prof.c deleted file mode 100644 index b6beaade9..000000000 --- a/src/prof.c +++ /dev/null @@ -1,720 +0,0 @@ -/* ---------------------------------------------------------------------------- -Copyright (c) 2026, Microsoft Research, Daan Leijen -This is free software; you can redistribute it and/or modify it under the -terms of the MIT license. A copy of the license can be found in the file -"LICENSE" at the root of this distribution. ------------------------------------------------------------------------------*/ - -// Sampling heap profiler that emits pprof's profile.proto format so the -// output can be read directly by `go tool pprof`, Polar Signals, etc. -// -// Design: zero overhead on the malloc/free fast paths when profiling is -// disabled at runtime. When enabled: -// - `theap->prof_force_slow` poisons `pages_free_direct` so every malloc -// routes through `_mi_malloc_generic`, where the byte countdown lives. -// - sampled blocks set `MI_PAGE_HAS_PROF_SAMPLES` on their page so frees -// route through the existing generic-free path, which calls `_mi_prof_free`. -// -// Samples are stored in a flat array (so allocation totals survive frees) plus -// an open-addressed hash map from block address -> sample index for inuse -// tracking. Backtraces are deduplicated at dump time. - -#if defined(__linux__) && !defined(_GNU_SOURCE) -#define _GNU_SOURCE // dl_iterate_phdr / struct dl_phdr_info (must precede the first libc header) -#endif -#include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/atomic.h" -#include "mimalloc/prim.h" -#include "mimalloc/prim-tls.h" - -#if defined(_WIN32) - #include - #include - #include - #define mi_prof_write(fd,buf,n) _write(fd,buf,(unsigned)(n)) - #define mi_prof_open(p) _open(p, _O_WRONLY|_O_CREAT|_O_TRUNC|_O_BINARY, 0644) - #define mi_prof_close(fd) _close(fd) -#else - #include - #include - #if defined(__linux__) || defined(__FreeBSD__) - #include - #include - #include - #endif - #if defined(__APPLE__) - #include - #include - #endif - #if defined(__GLIBC__) || defined(__APPLE__) - #include // backtrace() fallback; musl/bionic don't ship this (FP walk is primary) - #endif - #define mi_prof_write(fd,buf,n) write(fd,buf,n) - #define mi_prof_open(p) open(p, O_WRONLY|O_CREAT|O_TRUNC, 0644) - #define mi_prof_close(fd) close(fd) -#endif - -#define MI_PROF_MAX_FRAMES 32 -#define MI_PROF_SKIP_FRAMES 1 // skip backtrace() itself; mimalloc frames are filtered by pprof via mapping - -typedef struct mi_prof_sample_s { - uintptr_t addr; // block address (0 once freed; kept for alloc totals) - size_t size; // requested size - uint8_t nframes; - uintptr_t frames[MI_PROF_MAX_FRAMES]; -} mi_prof_sample_t; - -typedef struct mi_prof_state_s { - mi_lock_t lock; - _Atomic(size_t) rate; // bytes per sample (0 = disabled); read in _mi_malloc_generic - // samples (grow-only) - mi_prof_sample_t* samples; - size_t sample_count; - size_t sample_cap; - // open-addressed hash: addr -> sample index+1 (0 = empty, tombstone = ~0) - uintptr_t* ht_keys; - uint32_t* ht_vals; - size_t ht_cap; // power of two - size_t ht_used; -} mi_prof_state_t; - -static mi_prof_state_t mi_prof; - -// --------------------------------------------------------------------------- -// Hash table (addr -> sample index) -// --------------------------------------------------------------------------- - -static inline size_t mi_prof_hash(uintptr_t key, size_t cap) { - uint64_t k = (uint64_t)key; // 64-bit even on 32-bit targets, where `>> 33` on a uintptr_t is undefined - k ^= k >> 33; k *= 0xff51afd7ed558ccdull; k ^= k >> 33; - return (size_t)(k & (cap - 1)); -} - -static void mi_prof_ht_grow(size_t want); - -static void mi_prof_ht_put(uintptr_t key, uint32_t val) { - if (mi_prof.ht_used * 4 >= mi_prof.ht_cap * 3) mi_prof_ht_grow(mi_prof.ht_cap * 2); - size_t i = mi_prof_hash(key, mi_prof.ht_cap); - while (mi_prof.ht_keys[i] != 0 && mi_prof.ht_keys[i] != key) { - i = (i + 1) & (mi_prof.ht_cap - 1); - } - if (mi_prof.ht_keys[i] == 0) mi_prof.ht_used++; - mi_prof.ht_keys[i] = key; - mi_prof.ht_vals[i] = val; -} - -static void mi_prof_ht_grow(size_t want) { - size_t old_cap = mi_prof.ht_cap; - uintptr_t* old_keys = mi_prof.ht_keys; - uint32_t* old_vals = mi_prof.ht_vals; - size_t cap = (want < 1024 ? 1024 : want); - // use OS memory directly so we don't recurse into mimalloc - mi_memid_t memid; - uintptr_t* nk = (uintptr_t*)_mi_os_zalloc(_mi_subproc_main(), cap * (sizeof(uintptr_t) + sizeof(uint32_t)), &memid); - if (nk == NULL) return; - mi_prof.ht_keys = nk; - mi_prof.ht_vals = (uint32_t*)(nk + cap); - mi_prof.ht_cap = cap; - mi_prof.ht_used = 0; - for (size_t i = 0; i < old_cap; i++) { - if (old_keys[i] != 0 && old_keys[i] != (uintptr_t)~0ull) { - mi_prof_ht_put(old_keys[i], old_vals[i]); - } - } - // leak old table (process-lifetime; freed at exit by OS) - MI_UNUSED(old_keys); MI_UNUSED(old_vals); -} - -static bool mi_prof_ht_remove(uintptr_t key, uint32_t* out_val) { - if (mi_prof.ht_cap == 0) return false; - size_t i = mi_prof_hash(key, mi_prof.ht_cap); - while (mi_prof.ht_keys[i] != 0) { - if (mi_prof.ht_keys[i] == key) { - if (out_val) *out_val = mi_prof.ht_vals[i]; - mi_prof.ht_keys[i] = (uintptr_t)~0ull; // tombstone - return true; - } - i = (i + 1) & (mi_prof.ht_cap - 1); - } - return false; -} - -// --------------------------------------------------------------------------- -// Sample storage -// --------------------------------------------------------------------------- - -static mi_prof_sample_t* mi_prof_samples_push(void) { - if (mi_prof.sample_count == mi_prof.sample_cap) { - size_t ncap = (mi_prof.sample_cap == 0 ? 1024 : mi_prof.sample_cap * 2); - mi_memid_t memid; - mi_prof_sample_t* ns = (mi_prof_sample_t*)_mi_os_zalloc(_mi_subproc_main(), ncap * sizeof(mi_prof_sample_t), &memid); - if (ns == NULL) return NULL; - if (mi_prof.samples != NULL) { - _mi_memcpy(ns, mi_prof.samples, mi_prof.sample_count * sizeof(mi_prof_sample_t)); - } - mi_prof.samples = ns; - mi_prof.sample_cap = ncap; - } - return &mi_prof.samples[mi_prof.sample_count++]; -} - -// --------------------------------------------------------------------------- -// Backtrace -// --------------------------------------------------------------------------- - -static uint8_t mi_prof_backtrace(uintptr_t* frames) { - #if defined(_WIN32) - void* stack[MI_PROF_MAX_FRAMES + MI_PROF_SKIP_FRAMES]; - USHORT n = RtlCaptureStackBackTrace(MI_PROF_SKIP_FRAMES, MI_PROF_MAX_FRAMES, stack, NULL); - for (USHORT i = 0; i < n; i++) frames[i] = (uintptr_t)stack[i]; - return (uint8_t)n; - #else - // Frame-pointer walk. Works on stripped binaries without .eh_frame as long - // as the program is built with -fno-omit-frame-pointer (Bun does; macOS arm64 - // mandates it by ABI). ~10x faster than _Unwind_Backtrace. - // Layout (SysV x86_64 / AAPCS64): fp[0] = prev_fp, fp[1] = return_addr. - uintptr_t* fp = (uintptr_t*)__builtin_frame_address(0); - uintptr_t lo = (uintptr_t)fp; - uintptr_t hi = lo + (8 * 1024 * 1024); // conservative upper bound on remaining stack - uint8_t n = 0, skip = MI_PROF_SKIP_FRAMES; - while (fp != NULL && (uintptr_t)fp >= lo && (uintptr_t)fp < hi - && ((uintptr_t)fp & (sizeof(void*)-1)) == 0 - && n < MI_PROF_MAX_FRAMES) - { - uintptr_t ret = fp[1]; - uintptr_t* prev = (uintptr_t*)fp[0]; - if (ret == 0) break; - if (skip > 0) { skip--; } - else { frames[n++] = ret; } - if (prev <= fp) break; // not strictly ascending -> end of chain or corrupt - fp = prev; - } - #if defined(__GLIBC__) || defined(__APPLE__) - // Fallback if FP chain yielded nothing (caller compiled with -fomit-frame-pointer): - // try DWARF/compact-unwind. Harmless if .eh_frame is stripped (returns ~0). - if (n == 0) { - void* stack[MI_PROF_MAX_FRAMES + MI_PROF_SKIP_FRAMES]; - int bn = backtrace(stack, MI_PROF_MAX_FRAMES + MI_PROF_SKIP_FRAMES); - int sk = (bn > MI_PROF_SKIP_FRAMES ? MI_PROF_SKIP_FRAMES : 0); - for (int i = sk; i < bn && n < MI_PROF_MAX_FRAMES; i++) frames[n++] = (uintptr_t)stack[i]; - } - #endif - return n; - #endif -} - -// --------------------------------------------------------------------------- -// Geometric next-sample interval (so sampling is bytes-unbiased). -// Approximation of -ln(U)*rate using integer math; close enough for profiling. -// --------------------------------------------------------------------------- - -static intptr_t mi_prof_next_countdown(mi_theap_t* theap) { - MI_UNUSED(theap); - if (mi_prof.rate == 0) return 0; - // Fixed rate. (Go/jemalloc use a geometric draw to avoid bias with periodic - // allocation patterns; can be added later. Fixed rate is unbiased for - // typical workloads and keeps the math simple.) - return (intptr_t)mi_prof.rate; -} - -// --------------------------------------------------------------------------- -// Hooks called from page.c / free.c -// --------------------------------------------------------------------------- - -void _mi_prof_sample(mi_theap_t* theap, mi_page_t* page, void* p, size_t req_size) { - theap->prof_countdown = mi_prof_next_countdown(theap); - if (p == NULL || mi_prof.rate == 0) return; - // Taking the backtrace can allocate (glibc's `backtrace` dlopen's libgcc_s on first use, and that dlopen - // calls calloc before it is re-entrant); with a rate of 1 that allocation would sample again, without end. - mi_tld_t* const tld = theap->tld; - if (tld == NULL || tld->prof_sampling) return; - tld->prof_sampling = true; - - uintptr_t frames[MI_PROF_MAX_FRAMES]; - uint8_t n = mi_prof_backtrace(frames); - tld->prof_sampling = false; - - mi_lock(&mi_prof.lock) { - mi_prof_sample_t* s = mi_prof_samples_push(); - if (s == NULL) return; - s->addr = (uintptr_t)p; - s->size = req_size; - s->nframes = n; - for (uint8_t i = 0; i < n; i++) s->frames[i] = frames[i]; - mi_prof_ht_put((uintptr_t)p, (uint32_t)(mi_prof.sample_count - 1) + 1); - } - // mark the page so frees on it route through the generic path - mi_atomic_or_relaxed(&page->xthread_id, (mi_threadid_t)MI_PAGE_HAS_PROF_SAMPLES); -} - -void _mi_prof_free(const void* p) { - uint32_t idx1; - mi_lock(&mi_prof.lock) { - if (mi_prof_ht_remove((uintptr_t)p, &idx1)) { - mi_prof.samples[idx1 - 1].addr = 0; // mark not-inuse; keep for alloc totals - } - } -} - -// --------------------------------------------------------------------------- -// Enable / theap init -// --------------------------------------------------------------------------- - -static void mi_prof_set_theap(mi_theap_t* theap, bool on) { - theap->prof_force_slow = on; - theap->prof_countdown = (on ? mi_prof_next_countdown(theap) : 0); - // poison/restore pages_free_direct - for (size_t i = 0; i < MI_PAGES_DIRECT; i++) { - theap->pages_free_direct[i] = _mi_page_empty_get(); - } - // (when turning off, the slots refill lazily via mi_theap_queue_first_update) -} - -void _mi_prof_theap_init(mi_theap_t* theap) { - if (mi_atomic_load_relaxed(&mi_prof.rate) > 0) mi_prof_set_theap(theap, true); -} - -// Called from _mi_malloc_generic when the global rate is on but this theap -// hasn't been enabled yet (e.g., another thread called mi_prof_enable). -void _mi_prof_theap_lazy_enable(mi_theap_t* theap) { - if (theap->prof_countdown == 0 && mi_atomic_load_relaxed(&mi_prof.rate) > 0) { - mi_prof_set_theap(theap, true); - } -} - -// Read by _mi_malloc_generic's hot-path gate. -size_t _mi_prof_rate(void) { - return mi_atomic_load_relaxed(&mi_prof.rate); -} - -// Walk every theap in the process (all subprocs -> heaps -> theaps) and toggle. -// Writes to other threads' theap fields are aligned word stores; the worst case -// is a few fast-path allocs slip through before the target thread observes the -// poisoned slot, after which queue_first_update keeps it poisoned. -static void mi_prof_set_all_theaps(bool on) { - mi_subproc_t* sp = _mi_subproc_main(); - for (; sp != NULL; sp = sp->next) { - mi_lock(&sp->heaps_lock) { - for (mi_heap_t* h = sp->heaps; h != NULL; h = h->next) { - mi_lock(&h->theaps_lock) { - for (mi_theap_t* t = h->theaps; t != NULL; t = t->hnext) { - mi_prof_set_theap(t, on); - } - } - } - } - } -} - -void _mi_prof_init(void) { - long rate = mi_option_get(mi_option_prof_sample_rate); - if (rate <= 0) return; - mi_lock_init(&mi_prof.lock); - mi_atomic_store_release(&mi_prof.rate, (size_t)rate); - mi_prof_ht_grow(1024); - mi_prof_set_all_theaps(true); -} - -void mi_prof_reset(void) mi_attr_noexcept { - if (mi_prof.ht_cap == 0) return; // never initialized - mi_lock(&mi_prof.lock) { - mi_prof.sample_count = 0; - _mi_memzero(mi_prof.ht_keys, mi_prof.ht_cap * sizeof(uintptr_t)); - mi_prof.ht_used = 0; - // note: pages with MI_PAGE_HAS_PROF_SAMPLES keep the flag (frees still - // route to slow path) but the hash lookup misses harmlessly. The flag - // clears naturally on page re-init. - } -} - -void mi_prof_enable(size_t sample_rate_bytes) mi_attr_noexcept { - if (sample_rate_bytes == 0) { - mi_atomic_store_release(&mi_prof.rate, 0); - mi_prof_set_all_theaps(false); - return; - } - if (mi_atomic_load_relaxed(&mi_prof.rate) == 0) mi_lock_init(&mi_prof.lock); - mi_atomic_store_release(&mi_prof.rate, sample_rate_bytes); - if (mi_prof.ht_cap == 0) mi_prof_ht_grow(1024); - mi_prof_set_all_theaps(true); -} - -// --------------------------------------------------------------------------- -// profile.proto writer -// (https://github.com/google/pprof/blob/main/proto/profile.proto) -// All integers are varint; submessages are length-delimited (wire type 2). -// --------------------------------------------------------------------------- - -typedef struct mi_pb_s { - int fd; // -1 -> write to out_buf instead - bool err; - uint8_t* out_buf; // user buffer (NULL -> count only) - size_t out_cap; - size_t total; // total bytes produced (regardless of cap) - size_t pos; - uint8_t buf[16*1024]; -} mi_pb_t; - -static void pb_flush(mi_pb_t* w) { - if (w->err) { w->pos = 0; return; } // discard, so pb_raw keeps making progress and the dump returns -1 - if (w->pos == 0) return; - if (w->fd >= 0) { - if (mi_prof_write(w->fd, w->buf, w->pos) != (long)w->pos) w->err = true; - } - else if (w->out_buf != NULL) { - if (w->total < w->out_cap) { - size_t take = w->out_cap - w->total; if (take > w->pos) take = w->pos; - _mi_memcpy(w->out_buf + w->total, w->buf, take); - } - } - w->total += w->pos; - w->pos = 0; -} -static void pb_raw(mi_pb_t* w, const void* p, size_t n) { - const uint8_t* s = (const uint8_t*)p; - while (n > 0) { - if (w->pos == sizeof(w->buf)) pb_flush(w); - size_t take = sizeof(w->buf) - w->pos; if (take > n) take = n; - _mi_memcpy(w->buf + w->pos, s, take); w->pos += take; s += take; n -= take; - } -} -static void pb_varint(mi_pb_t* w, uint64_t v) { - uint8_t tmp[10]; size_t i = 0; - while (v >= 0x80) { tmp[i++] = (uint8_t)(v | 0x80); v >>= 7; } - tmp[i++] = (uint8_t)v; - pb_raw(w, tmp, i); -} -static size_t pb_varint_len(uint64_t v) { size_t n=1; while(v>=0x80){n++;v>>=7;} return n; } -static void pb_tag(mi_pb_t* w, uint32_t field, uint32_t wt) { pb_varint(w, (uint64_t)((field<<3)|wt)); } - -// ValueType { type=1, unit=2 } -static void pb_value_type(mi_pb_t* w, uint32_t field, int64_t type_str, int64_t unit_str) { - size_t len = pb_varint_len((1<<3)|0)+pb_varint_len((uint64_t)type_str) - + pb_varint_len((2<<3)|0)+pb_varint_len((uint64_t)unit_str); - pb_tag(w, field, 2); pb_varint(w, len); - pb_tag(w, 1, 0); pb_varint(w, (uint64_t)type_str); - pb_tag(w, 2, 0); pb_varint(w, (uint64_t)unit_str); -} - -// Location { id=1, mapping_id=2, address=3 } -static void pb_location(mi_pb_t* w, uint64_t id, uint64_t mapping_id, uint64_t addr) { - size_t len = pb_varint_len((1<<3))+pb_varint_len(id) - + pb_varint_len((2<<3))+pb_varint_len(mapping_id) - + pb_varint_len((3<<3))+pb_varint_len(addr); - pb_tag(w, 4, 2); pb_varint(w, len); - pb_tag(w, 1, 0); pb_varint(w, id); - pb_tag(w, 2, 0); pb_varint(w, mapping_id); - pb_tag(w, 3, 0); pb_varint(w, addr); -} - -// Mapping { id=1, memory_start=2, memory_limit=3, file_offset=4, filename=5, build_id=6 } -static void pb_mapping(mi_pb_t* w, uint64_t id, uint64_t start, uint64_t end, uint64_t off, int64_t name_str, int64_t bid_str) { - size_t len = pb_varint_len(1<<3)+pb_varint_len(id) - + pb_varint_len(2<<3)+pb_varint_len(start) - + pb_varint_len(3<<3)+pb_varint_len(end) - + pb_varint_len(4<<3)+pb_varint_len(off) - + pb_varint_len(5<<3)+pb_varint_len((uint64_t)name_str) - + pb_varint_len(6<<3)+pb_varint_len((uint64_t)bid_str); - pb_tag(w, 3, 2); pb_varint(w, len); - pb_tag(w, 1, 0); pb_varint(w, id); - pb_tag(w, 2, 0); pb_varint(w, start); - pb_tag(w, 3, 0); pb_varint(w, end); - pb_tag(w, 4, 0); pb_varint(w, off); - pb_tag(w, 5, 0); pb_varint(w, (uint64_t)name_str); - pb_tag(w, 6, 0); pb_varint(w, (uint64_t)bid_str); -} - -// String table entry (field 6) -static void pb_string(mi_pb_t* w, const char* s) { - size_t n = _mi_strlen(s); - pb_tag(w, 6, 2); pb_varint(w, n); pb_raw(w, s, n); -} - -// --------------------------------------------------------------------------- -// Sample emission: aggregate by (stack, size_bucket) for compactness. -// We emit each raw sample as its own Sample with values scaled by `rate/size` -// (the inverse sampling probability) so pprof shows correct totals. -// --------------------------------------------------------------------------- - -typedef struct mi_prof_loc_s { uintptr_t addr; uint64_t id; uint64_t mapping_id; } mi_prof_loc_t; -typedef struct mi_prof_map_s { uint64_t start, end, off; const char* name; char build_id[48]; } mi_prof_map_t; - -static void mi_prof_hex(char* dst, const uint8_t* src, size_t n) { - static const char hx[] = "0123456789abcdef"; - for (size_t i = 0; i < n; i++) { dst[2*i] = hx[src[i]>>4]; dst[2*i+1] = hx[src[i]&0xf]; } - dst[2*n] = 0; -} - -static inline size_t mi_prof_loc_find(mi_prof_loc_t* locs, size_t cap, uintptr_t a) { - size_t h = (size_t)(a * 0x9E3779B97F4A7C15ull) & (cap - 1); - while (locs[h].addr != 0 && locs[h].addr != a) h = (h + 1) & (cap - 1); // callers keep the table at most half full - return h; -} - -// Double the location table (kept at most half full so `mi_prof_loc_find` always terminates). Returns false on OOM. -static bool mi_prof_loc_grow(mi_prof_loc_t** plocs, size_t* pcap, mi_memid_t* pmemid) { - const size_t old_cap = *pcap; - const size_t new_cap = 2 * old_cap; - mi_memid_t nm; - mi_prof_loc_t* nlocs = (mi_prof_loc_t*)_mi_os_zalloc(_mi_subproc_main(), new_cap * sizeof(mi_prof_loc_t), &nm); - if (nlocs == NULL) return false; - mi_prof_loc_t* olocs = *plocs; - for (size_t i = 0; i < old_cap; i++) { - if (olocs[i].addr != 0) { nlocs[mi_prof_loc_find(nlocs, new_cap, olocs[i].addr)] = olocs[i]; } - } - _mi_os_free(_mi_subproc_main(), olocs, old_cap * sizeof(mi_prof_loc_t), *pmemid); - *plocs = nlocs; *pcap = new_cap; *pmemid = nm; - return true; -} - -#if defined(__linux__) || defined(__FreeBSD__) -typedef struct { mi_prof_map_t* maps; size_t* n; size_t cap; char* strbuf; size_t strcap; size_t soff; } mi_prof_dlctx_t; - -static int mi_prof_dl_cb(struct dl_phdr_info* info, size_t sz, void* arg) { - MI_UNUSED(sz); - mi_prof_dlctx_t* c = (mi_prof_dlctx_t*)arg; - if (*c->n >= c->cap) return 0; - uint64_t start = 0, end = 0, off = 0; - const uint8_t* bid = NULL; size_t bid_len = 0; - for (int i = 0; i < info->dlpi_phnum; i++) { - const ElfW(Phdr)* ph = &info->dlpi_phdr[i]; - if (ph->p_type == PT_LOAD && (ph->p_flags & PF_X)) { - uint64_t s = info->dlpi_addr + ph->p_vaddr; - if (start == 0) { start = s; off = ph->p_offset; } - if (s + ph->p_memsz > end) end = s + ph->p_memsz; - } - else if (ph->p_type == PT_NOTE) { - const uint8_t* p = (const uint8_t*)(info->dlpi_addr + ph->p_vaddr); - const uint8_t* e = p + ph->p_memsz; - while (p + 12 <= e) { - uint32_t namesz = *(const uint32_t*)p, descsz = *(const uint32_t*)(p+4), type = *(const uint32_t*)(p+8); - const uint8_t* name = p + 12; const uint8_t* desc = name + ((namesz+3)&~3u); - if (type == 3 /*NT_GNU_BUILD_ID*/ && namesz == 4 && name[0]=='G'&&name[1]=='N'&&name[2]=='U') { - bid = desc; bid_len = descsz; break; - } - p = desc + ((descsz+3)&~3u); - } - } - } - if (start == 0) return 0; - mi_prof_map_t* m = &c->maps[(*c->n)++]; - m->start = start; m->end = end; m->off = off; - m->build_id[0] = 0; - if (bid != NULL && bid_len > 0 && bid_len <= 20) mi_prof_hex(m->build_id, bid, bid_len); - const char* nm = (info->dlpi_name && info->dlpi_name[0]) ? info->dlpi_name : ""; - size_t plen = _mi_strlen(nm)+1; - if (c->soff + plen <= c->strcap) { _mi_memcpy(c->strbuf+c->soff, nm, plen); m->name = c->strbuf+c->soff; c->soff += plen; } - else m->name = ""; - return 0; -} -#endif - -static void mi_prof_collect_mappings(mi_prof_map_t* maps, size_t* nmaps, size_t cap, char* strbuf, size_t strbuf_cap) { - size_t n = 0; - size_t soff = 0; - #if defined(__linux__) || defined(__FreeBSD__) - // dl_iterate_phdr gives us PT_LOAD + PT_NOTE for every loaded object, - // including the main executable, so we get build-id without opening files. - mi_prof_dlctx_t ctx = { maps, &n, cap, strbuf, strbuf_cap, 0 }; - dl_iterate_phdr(mi_prof_dl_cb, &ctx); - soff = ctx.soff; - // dl_iterate_phdr reports the main exe with empty name; fill from /proc/self/exe - if (n > 0 && maps[0].name[0] == 0) { - ssize_t r = readlink("/proc/self/exe", strbuf + soff, strbuf_cap - soff - 1); - if (r > 0) { strbuf[soff+r] = 0; maps[0].name = strbuf+soff; soff += (size_t)r+1; } - } - #elif defined(__APPLE__) - uint32_t cnt = _dyld_image_count(); - for (uint32_t i = 0; i < cnt && n < cap; i++) { - const struct mach_header_64* mh = (const struct mach_header_64*)_dyld_get_image_header(i); - if (mh == NULL || mh->magic != MH_MAGIC_64) continue; - // walk load commands to find __TEXT extent and LC_UUID - uint64_t start = (uint64_t)(uintptr_t)mh, end = start; - uint64_t text_vmaddr = 0; // unslid vmaddr of the executable segment - maps[n].build_id[0] = 0; - const struct load_command* lc = (const struct load_command*)((const uint8_t*)mh + sizeof(*mh)); - for (uint32_t c = 0; c < mh->ncmds; c++) { - if (lc->cmd == LC_SEGMENT_64) { - const struct segment_command_64* sc = (const struct segment_command_64*)lc; - intptr_t slide = _dyld_get_image_vmaddr_slide(i); - uint64_t s = sc->vmaddr + (uint64_t)slide; - if (sc->initprot & 0x4 /* VM_PROT_EXECUTE */) { - if (end == start) { start = s; text_vmaddr = sc->vmaddr; } - if (s + sc->vmsize > end) end = s + sc->vmsize; - } - } - else if (lc->cmd == LC_UUID) { - const struct uuid_command* uc = (const struct uuid_command*)lc; - mi_prof_hex(maps[n].build_id, uc->uuid, 16); - } - lc = (const struct load_command*)((const uint8_t*)lc + lc->cmdsize); - } - if (end <= start) end = start + 0x1000; - // pprof reconstructs a file address as `addr - memory_start + file_offset`. - // Mach-O symbols are recorded at their (unslid) vmaddr, not at an offset - // from __TEXT, so file_offset must be that vmaddr or nothing symbolizes. - maps[n].start = start; maps[n].end = end; maps[n].off = text_vmaddr; - const char* nm = _dyld_get_image_name(i); - size_t plen = _mi_strlen(nm)+1; - if (soff + plen <= strbuf_cap) { _mi_memcpy(strbuf+soff, nm, plen); maps[n].name = strbuf+soff; soff += plen; } - else maps[n].name = nm; - n++; - } - #else - MI_UNUSED(strbuf); MI_UNUSED(strbuf_cap); MI_UNUSED(soff); - #endif - *nmaps = n; -} - -static uint64_t mi_prof_mapping_for(mi_prof_map_t* maps, size_t nmaps, uintptr_t addr) { - for (size_t i = 0; i < nmaps; i++) { - if (addr >= maps[i].start && addr < maps[i].end) return (uint64_t)(i + 1); - } - return 0; -} - -static int mi_prof_dump_pb(mi_pb_t* wp) { - mi_pb_t w = *wp; - - // string table (index 0 must be "") - // 0:"" 1:alloc_objects 2:count 3:alloc_space 4:bytes 5:inuse_objects 6:inuse_space 7:space - // 8.. : mapping filenames - static const char* fixed_strs[] = { "", "alloc_objects","count","alloc_space","bytes","inuse_objects","inuse_space","space" }; - enum { STR_EMPTY=0, STR_ALLOC_OBJ, STR_COUNT, STR_ALLOC_SPACE, STR_BYTES, STR_INUSE_OBJ, STR_INUSE_SPACE, STR_SPACE, STR_FIXED_N }; - - // collect mappings (executable regions) for symbolization - mi_prof_map_t maps[256]; size_t nmaps = 0; - static char map_strbuf[32*1024]; - mi_prof_collect_mappings(maps, &nmaps, 256, map_strbuf, sizeof(map_strbuf)); - - // sample_type: alloc_objects/count, alloc_space/bytes, inuse_objects/count, inuse_space/bytes - pb_value_type(&w, 1, STR_ALLOC_OBJ, STR_COUNT); - pb_value_type(&w, 1, STR_ALLOC_SPACE, STR_BYTES); - pb_value_type(&w, 1, STR_INUSE_OBJ, STR_COUNT); - pb_value_type(&w, 1, STR_INUSE_SPACE, STR_BYTES); - - // We emit Locations on the fly as we see new addresses; dedup via a small - // open-addressed table (locations are typically few — call sites). - // To keep this simple, do two passes: pass 1 collect unique frame addrs, - // pass 2 emit samples referencing location ids. - // Unique-address collection: - size_t loc_cap = 4096; - mi_memid_t lm; - mi_prof_loc_t* locs = (mi_prof_loc_t*)_mi_os_zalloc(_mi_subproc_main(), loc_cap * sizeof(mi_prof_loc_t), &lm); - if (locs == NULL) { *wp = w; return -1; } - size_t nlocs = 0; - bool oom = false; - - mi_lock(&mi_prof.lock) { - for (size_t i = 0; i < mi_prof.sample_count && !oom; i++) { - mi_prof_sample_t* s = &mi_prof.samples[i]; - for (uint8_t f = 0; f < s->nframes; f++) { - uintptr_t a = s->frames[f]; if (a==0) continue; - size_t h = mi_prof_loc_find(locs, loc_cap, a); - if (locs[h].addr == 0) { - if (2 * (nlocs + 1) > loc_cap) { - if (!mi_prof_loc_grow(&locs, &loc_cap, &lm)) { oom = true; break; } - h = mi_prof_loc_find(locs, loc_cap, a); - } - locs[h].addr = a; - locs[h].id = ++nlocs; - locs[h].mapping_id = mi_prof_mapping_for(maps, nmaps, a); - } - } - } - if (oom) { w.err = true; } - - // emit samples (field 2): location_id[] (packed), value[] (packed) - for (size_t i = 0; i < mi_prof.sample_count && !oom; i++) { - mi_prof_sample_t* s = &mi_prof.samples[i]; - // scale: a sample triggers after `rate` bytes of countdown, decremented by - // block_size per alloc. So a sample of size s represents ~max(rate, s) bytes: - // s >= rate -> every such alloc triggers (1 sample = 1 alloc = s bytes) - // s < rate -> ~rate/s allocs per sample (1 sample = rate bytes) - uint64_t scale_bytes = (mi_prof.rate > (size_t)s->size ? (uint64_t)mi_prof.rate : (uint64_t)s->size); - uint64_t scale_objs = (s->size > 0 ? (scale_bytes + s->size/2) / s->size : 1); - if (scale_objs == 0) scale_objs = 1; - uint64_t inuse = (s->addr != 0 ? 1 : 0); - uint64_t v[4] = { scale_objs, scale_bytes, inuse*scale_objs, inuse*scale_bytes }; - - // packed location ids - size_t loc_len = 0; - for (uint8_t f = 0; f < s->nframes; f++) { - size_t h = mi_prof_loc_find(locs, loc_cap, s->frames[f]); loc_len += pb_varint_len(locs[h].id); - } - // packed values - size_t val_len = 0; for (int k=0;k<4;k++) val_len += pb_varint_len(v[k]); - size_t body = pb_varint_len((1<<3)|2)+pb_varint_len(loc_len)+loc_len - + pb_varint_len((2<<3)|2)+pb_varint_len(val_len)+val_len; - pb_tag(&w, 2, 2); pb_varint(&w, body); - pb_tag(&w, 1, 2); pb_varint(&w, loc_len); - for (uint8_t f = 0; f < s->nframes; f++) { size_t h = mi_prof_loc_find(locs, loc_cap, s->frames[f]); pb_varint(&w, locs[h].id); } - pb_tag(&w, 2, 2); pb_varint(&w, val_len); - for (int k=0;k<4;k++) pb_varint(&w, v[k]); - } - } // unlock - - // mappings (field 3) — string indices start after fixed strings - for (size_t i = 0; i < nmaps; i++) { - int64_t bid_str = (maps[i].build_id[0] != 0 ? (int64_t)(STR_FIXED_N + nmaps + i) : 0); - pb_mapping(&w, (uint64_t)(i+1), maps[i].start, maps[i].end, maps[i].off, (int64_t)(STR_FIXED_N + i), bid_str); - } - - // locations (field 4) - for (size_t i = 0; i < loc_cap; i++) { - if (locs[i].addr != 0) pb_location(&w, locs[i].id, locs[i].mapping_id, (uint64_t)locs[i].addr); - } - - // string_table (field 6) - for (size_t i = 0; i < STR_FIXED_N; i++) pb_string(&w, fixed_strs[i]); - for (size_t i = 0; i < nmaps; i++) pb_string(&w, maps[i].name); - for (size_t i = 0; i < nmaps; i++) pb_string(&w, maps[i].build_id); - - // period_type (field 11), period (field 12), default_sample_type (field 14) - pb_value_type(&w, 11, STR_SPACE, STR_BYTES); - pb_tag(&w, 12, 0); pb_varint(&w, (uint64_t)mi_prof.rate); - pb_tag(&w, 14, 0); pb_varint(&w, STR_INUSE_SPACE); // string-table index of the default sample type name - - pb_flush(&w); - *wp = w; - _mi_os_free(_mi_subproc_main(), locs, loc_cap * sizeof(mi_prof_loc_t), lm); - return (w.err ? -1 : 0); -} - -int mi_prof_dump(int fd) mi_attr_noexcept { - if (fd < 0) return -1; - mi_pb_t w; _mi_memzero(&w, sizeof(w)); w.fd = fd; - return mi_prof_dump_pb(&w); -} - -// Write profile.proto into `buf` (up to `cap` bytes). Returns the total -// encoded size; if the return value > cap the output is truncated (call again -// with a larger buffer). Pass buf=NULL, cap=0 to query the size. -size_t mi_prof_dump_buf(void* buf, size_t cap) mi_attr_noexcept { - mi_pb_t w; _mi_memzero(&w, sizeof(w)); - w.fd = -1; w.out_buf = (uint8_t*)buf; w.out_cap = cap; - mi_prof_dump_pb(&w); - return w.total; -} - -int mi_prof_dump_to_file(const char* path) mi_attr_noexcept { - if (path == NULL) return -1; - int fd = mi_prof_open(path); - if (fd < 0) return -1; - int rc = mi_prof_dump(fd); - mi_prof_close(fd); - return rc; -} - -void _mi_prof_on_exit(void) { - if (mi_prof.rate == 0 || mi_prof.sample_count == 0) return; - char path[512]; - if (!_mi_getenv("MIMALLOC_PROF_PATH", path, sizeof(path))) { - _mi_snprintf(path, sizeof(path), "mimalloc-prof.%lu.pb", (unsigned long)_mi_prim_thread_id()); - } - if (mi_prof_dump_to_file(path) == 0) { - _mi_message("heap profile written to %s (%zu samples; view with: go tool pprof %s)\n", - path, mi_prof.sample_count, path); - } -} diff --git a/src/random.c b/src/random.c index 464627acf..3c012adf4 100644 --- a/src/random.c +++ b/src/random.c @@ -139,14 +139,14 @@ void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* ctx_new) { chacha_split(ctx, (uintptr_t)ctx_new ^ nonce_rnd /*nonce*/, ctx_new); } -uintptr_t _mi_random_next(mi_random_ctx_t* ctx) { +size_t _mi_random_next(mi_random_ctx_t* ctx) { mi_assert_internal(mi_random_is_initialized(ctx)); - uintptr_t r; + size_t r; do { - #if MI_INTPTR_SIZE <= 4 + #if MI_SIZE_SIZE <= 4 r = chacha_next32(ctx); - #elif MI_INTPTR_SIZE == 8 - r = (((uintptr_t)chacha_next32(ctx) << 32) | chacha_next32(ctx)); + #elif MI_SIZE_SIZE == 8 + r = (((size_t)chacha_next32(ctx) << 32) | chacha_next32(ctx)); #else # error "define mi_random_next for this platform" #endif @@ -160,12 +160,12 @@ To initialize a fresh random context. If we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR. -----------------------------------------------------------------------------*/ -uintptr_t _mi_os_random_weak(uintptr_t extra_seed) { - uintptr_t x = (uintptr_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random +size_t _mi_os_random_weak(size_t extra_seed) { + size_t x = (size_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random x ^= _mi_prim_clock_now(); // and do a few randomization steps - uintptr_t max = ((x ^ (x >> 17)) & 0x0F) + 1; - for (uintptr_t i = 0; i < max || x==0; i++, x++) { + size_t max = ((x ^ (x >> 17)) & 0x0F) + 1; + for (size_t i = 0; i < max || x==0; i++, x++) { x = _mi_random_shuffle(x); } mi_assert_internal(x != 0); @@ -180,7 +180,7 @@ static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) { #if !defined(__wasi__) if (!use_weak) { _mi_warning_message("unable to use secure randomness\n"); } #endif - uintptr_t x = _mi_os_random_weak(0); + size_t x = _mi_os_random_weak(0); for (size_t i = 0; i < 32; i+=4, x++) { x = _mi_random_shuffle(x); key[i] = (uint8_t)(x); diff --git a/src/sample-guarded.c b/src/sample-guarded.c new file mode 100644 index 000000000..f7c1fe81e --- /dev/null +++ b/src/sample-guarded.c @@ -0,0 +1,173 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim-tls.h" + +mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { + theap->guarded_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate); + if (theap->guarded_sample_rate > 1) { + if (seed == 0) { + seed = _mi_theap_random_next(theap); + } + theap->guarded_sample_countdown = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate` + } + else if (theap->guarded_sample_countdown > theap->guarded_sample_rate) { + theap->guarded_sample_countdown = theap->guarded_sample_rate; + } + _mi_theap_update_sample_rate(theap); +} + +mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { + theap->guarded_size_min = min; + theap->guarded_size_max = (min > max ? min : max); +} + +void _mi_theap_guarded_init(mi_theap_t* theap) { + #if MI_GUARDED + mi_theap_guarded_set_sample_rate(theap, + (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX), + (size_t)mi_option_get(mi_option_guarded_sample_seed)); + mi_theap_guarded_set_size_bound(theap, + (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX), + (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) ); + #else + mi_theap_guarded_set_sample_rate(theap,0,0); + #endif +} + + +// We always allocate a guarded allocation at an offset (`mi_page_has_interior_pointers` will be true). +// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`) +// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size` +static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size, size_t* usable_size) { + // todo: we can still make padding work by moving it out of the guard page area + mi_page_t* const page = _mi_ptr_page(block); + mi_page_set_has_interior_pointers(page, true); + block->next = MI_BLOCK_TAG_GUARDED; + + // set guard page at the end of the block + const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local` + const size_t os_page_size = _mi_os_page_size(); + mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t)); + if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) { + // should never happen + mi_free(block); + return NULL; + } + uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size; + // note: the alignment of the guard page relies on blocks being os_page_size aligned which + // is ensured in `mi_arena_page_alloc_fresh`. + mi_assert_internal(_mi_is_aligned(block, os_page_size)); + mi_assert_internal(_mi_is_aligned(guard_page, os_page_size)); + if (!page->memid.is_pinned && _mi_is_aligned(guard_page, os_page_size)) { + const bool ok = _mi_os_protect(guard_page, os_page_size); + if mi_unlikely(!ok) { + _mi_warning_message("failed to set a guard page behind an object (object %p of size %zu)\n", block, block_size); + } + } + else { + _mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size); + } + + // align pointer just in front of the guard page + size_t offset = block_size - os_page_size - obj_size; + mi_assert_internal(offset > sizeof(mi_block_t)); + if (offset > MI_PAGE_MAX_OVERALLOC_ALIGN) { + // give up to place it right in front of the guard page if the offset is too large for unalignment + offset = MI_PAGE_MAX_OVERALLOC_ALIGN; + } + uint8_t* const p = (uint8_t*)block + offset; + mi_assert_internal(p == guard_page - obj_size || offset >= MI_PAGE_MAX_OVERALLOC_ALIGN); + if (usable_size != NULL) { *usable_size = (guard_page - p); mi_assert_internal(mi_usable_size(p)==*usable_size); } + mi_track_align(block, p, offset, obj_size); + mi_track_mem_defined(block, sizeof(mi_block_t)); + return p; +} + +// Allocate a block with a guard page behind it. +mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept +{ + // allocate multiple of page size ending in a guard page + // ensure minimal alignment requirement? + if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow + _mi_error_message(EOVERFLOW, "(guarded) allocation request is too large (%zu bytes)\n", size); + return NULL; + } + const size_t os_page_size = _mi_os_page_size(); + const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE)); + const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); + const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); + // const size_t threshold = mi_theap_disable_profiler(theap); + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap, req_size, false /* don't zero */, ppage); + // mi_theap_enable_profiler(theap,threshold); + if (block==NULL) return NULL; + size_t usable_size = 0; + void* const p = mi_block_ptr_set_guarded(block, obj_size, &usable_size); + if (p == NULL) return NULL; + if (zero) { + _mi_memzero(p,obj_size); // we have to zero afterwards as padding might have written inside the block (if the `blocksize > reqsize + os_page_size`) + } + + // stats + mi_track_malloc(p, usable_size, zero); + if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); } + mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1); + #if MI_STATS + // adjust request stats to only count the allocated size of the block (and not the guard page) + mi_theap_stat_counter_decrease(theap, malloc_requested, req_size); + mi_theap_stat_counter_increase(theap, malloc_requested, size); + #endif + #if MI_DEBUG>3 + if (zero) { + mi_assert_expensive(mi_mem_is_zero(p, size)); + } + #endif + #if MI_PAGE_META_SMALL_IS_ALIGNED && MI_DEBUG>=2 + // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail. + if (size <= MI_SMALL_SIZE_MAX) { + mi_page_t* const page = _mi_ptr_page(p); + mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); + } + #endif + return p; +} + + +// Remove guard page when building with MI_GUARDED +#if MI_GUARDED +void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { + // if (!mi_block_ptr_is_guarded(block,p)) return; + mi_assert_internal(mi_block_ptr_is_guarded(block, p)); + mi_assert_internal(mi_page_has_interior_pointers(page)); + mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t)); + mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED); + + const size_t bsize = mi_page_block_size(page); + const size_t psize = _mi_os_page_size(); + mi_assert_internal(bsize > psize); + mi_assert_internal(!page->memid.is_pinned); + void* gpage = (uint8_t*)block + bsize - psize; + mi_assert_internal(_mi_is_aligned(gpage, psize)); + _mi_os_unprotect(gpage, psize); +} + +// unguard a whole page (called from `mi_heap_destroy`) +void _mi_page_unguard_all(mi_page_t* page) { + if mi_likely(!mi_page_has_interior_pointers(page)) return; + uint8_t* const start = mi_page_start(page); + const size_t psize = mi_page_committed(page); + _mi_os_unprotect(start,psize); // unprotect all at once as we cannot know which blocks are guarded +} +#else +void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { + MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p); +} +void _mi_page_unguard_all(mi_page_t* page) { + MI_UNUSED(page); +} +#endif diff --git a/src/sample-profile.c b/src/sample-profile.c new file mode 100644 index 000000000..f1b5db491 --- /dev/null +++ b/src/sample-profile.c @@ -0,0 +1,264 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim-tls.h" + +//---------------------------------------------------------------------------- +// General sampled allocation (called from `page.c:mi_malloc_generic_fallback`) +// This is for both guarded and profiled sampling +//---------------------------------------------------------------------------- + +size_t _mi_theap_update_sample_rate(mi_theap_t* theap) { + const size_t old_sample_rate = theap->sample_rate; + theap->sample_rate = theap->profile_sample_rate; + if (theap->sample_rate == 0 || (theap->guarded_sample_rate!=0 && theap->sample_rate > theap->guarded_sample_rate)) { + theap->sample_rate = theap->guarded_sample_rate; + } + if (theap->sample_countdown > theap->sample_rate) { + theap->sample_countdown = theap->sample_rate; // todo: adjust difference? + } + return old_sample_rate; +} + +// Called when the theap sample countdown <= req_size; should call guarded or profiled allocation. +mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept +{ + // the size has not yet been counted against the countdown (and does not include MI_PADDING_SIZE) + mi_assert_internal(req_size <= MI_MAX_ALLOC_SIZE); + mi_assert_internal((theap->sample_countdown==SIZE_MAX && _mi_is_empty_theap(theap)) || + (theap->sample_countdown <= MI_SAMPLE_COUNTDOWN_MAX && theap->sample_countdown < req_size)); + const size_t size = req_size + MI_PADDING_SIZE; + + // handle empty theap and disabled sampling + if (theap->sample_rate==0) { + if (!_mi_is_empty_theap(theap)) { // avoid writing to the initial empty theap + theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // avoid the sampling path for a long time + } + return _mi_malloc_generic_no_sample(theap,size,zero,ppage); + } + + // update countdown and total accummulated requested bytes since the last sample + mi_assert_internal(!_mi_is_empty_theap(theap)); + mi_assert_internal(req_size <= SIZE_MAX/2); + mi_assert_internal(theap->sample_rate > 0); + mi_assert_internal(theap->sample_rate <= SIZE_MAX/2); + mi_assert_internal(theap->sample_rate >= theap->sample_countdown); + + const uint64_t requested = theap->sample_requested = (uint64_t)theap->sample_rate + (uint64_t)(req_size - theap->sample_countdown) + theap->sample_requested; + mi_assert_internal(requested > 0); + mi_assert_internal(requested >= req_size); + theap->sample_countdown = theap->sample_rate; // reset sampling + + // update derived countdowns + mi_assert_internal(theap->profile_sample_rate!=0 || theap->profile_sample_countdown==0); + mi_assert_internal(theap->guarded_sample_rate!=0 || theap->guarded_sample_countdown==0); + bool sample_profile = false; + bool sample_guarded = false; + if (theap->profile_sample_rate!=0) { + if (theap->profile_sample_countdown >= requested) { theap->profile_sample_countdown -= (size_t)requested; } else { sample_profile = true; } + } + if (theap->guarded_sample_rate!=0) { + if (theap->guarded_sample_countdown >= requested) { theap->guarded_sample_countdown -= (size_t)requested; } else { sample_guarded = true; } + } + + // invoke callback? + if (sample_profile) { + theap->profile_sample_countdown = theap->profile_sample_rate; // reset countdown + theap->sample_requested = 0; // reset requested as we pass it to malloc_profiled + #if MI_PROFILE + return _mi_theap_malloc_profiled(theap,size,requested,zero,ppage); + #endif + } + else if (sample_guarded) { + if (req_size >= theap->guarded_size_min && req_size <= theap->guarded_size_max) { + // use guarded allocation + theap->guarded_sample_countdown = theap->guarded_sample_rate; // reset countdown + #if MI_GUARDED + return _mi_theap_malloc_guarded(theap,size,zero,ppage); + #endif + } + #if MI_GUARDED + else { + // failed size criteria, rewind the sample countdown so we sample asap again + // todo: can we do better here as this will cause many samples until it fits the size.. + theap->sample_countdown = 0; + } + #endif + } + // take generic path + return _mi_malloc_generic_no_sample(theap,size,zero,ppage); +} + + + +//---------------------------------------------------------------------------- +// Util +//---------------------------------------------------------------------------- + +static mi_profiler_t* mi_heap_profiler(const mi_heap_t* heap) { + return mi_atomic_load_ptr_acquire(mi_profiler_t,&heap->profiler); +} + +static mi_profiler_t* mi_theap_get_enabled_profiler(const mi_theap_t* theap) { + mi_heap_t* const heap = _mi_theap_heap(theap); + mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler); + if (prof!=NULL && mi_profiler_is_enabled(prof) && prof->on_alloc != NULL) { + return prof; + } + else { + return NULL; + } +} + +//---------------------------------------------------------------------------- +// Profile an allocation +//----------------------------------------------------------------------------- + +mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept +{ + mi_assert_internal(theap!=NULL); + mi_assert_internal(size>=MI_PADDING_SIZE); + const size_t req_size = size - MI_PADDING_SIZE; + mi_assert_internal(req_size<=requested_since_last_sample); + mi_profiler_t* const prof = mi_theap_get_enabled_profiler(theap); + if (prof == NULL) { return _mi_malloc_generic_no_sample(theap,size,zero,ppage); } + mi_assert_internal(prof!=NULL && prof->on_alloc!=NULL && mi_profiler_is_enabled(prof)); + + void* p = NULL; + size_t new_sample_rate = 0; + if (prof->on_free==NULL || prof->sample_data_size==0) { + // just allocate without profiler data + p = _mi_malloc_generic_no_sample(theap,size,zero,ppage); + if (p==NULL) { return p; } + if (prof->on_alloc!=NULL) { + // we are just allocation profiling (not heap profiling as on_free == NULL) + new_sample_rate = (*prof->on_alloc)(prof, NULL /* no data */, p, req_size, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap) ); + } + } + else { + // overallocate a larger block to store the profiler data + // [MI_BLOCK_TAG_PROFILE] [usable size] [ ... profile data ... ] [... user data ...] + const size_t sample_data_offset = sizeof(mi_block_t); + const size_t sample_user_data_size = _mi_align_up(prof->sample_data_size > MI_PROFILE_SAMPLE_DATA_MAX_SIZE ? MI_PROFILE_SAMPLE_DATA_MAX_SIZE : prof->sample_data_size, sizeof(void*)); + const size_t sample_data_size = sizeof(mi_profiler_sample_data_t) + sample_user_data_size; // one void* too many just in case + const size_t user_offset = _mi_align_up(sample_data_offset + sample_data_size, MI_MAX_ALIGN_SIZE); + const size_t oversize = user_offset + size; + mi_page_t* page = NULL; + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap,oversize,zero,&page); + if (block==NULL) return NULL; + mi_assert_internal(page!=NULL); + if (ppage!=NULL) { *ppage = page; } + #if MI_PAGE_META_SMALL_IS_ALIGNED + // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail. + // (that is why we need to limit the profiler_data_size as well) + // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail. + if (size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); } + #endif + + // Set up the profiled block as an interior pointer so the interior "slow" path is taken on mi_free (where we catch it to call on_free) + mi_page_set_has_interior_pointers(page, true); + block->next = MI_BLOCK_TAG_PROFILED; + p = (uint8_t*)block + user_offset; + mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sample_data_offset); + sample_data->user_data_size = sample_user_data_size; + + // and call the profiler on_alloc + if (prof->on_alloc!=NULL) { + new_sample_rate = (*prof->on_alloc)(prof, sample_data, p, req_size, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap) ); + } + } + if (new_sample_rate!=0 && new_sample_rate != (size_t)theap->profile_sample_rate) { + _mi_theap_set_profile_sample_rate(theap,new_sample_rate); + } + mi_theap_stat_counter_increase(theap,profile_samples,1); + return p; +} + +void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p) { + mi_assert_internal(mi_block_ptr_is_sampled(block,p)); + + // get the heap and profiler + mi_heap_t* const heap = mi_page_heap(page); + if (heap==NULL) return; + mi_profiler_t* prof = mi_heap_profiler(heap); + if (prof==NULL || !mi_profiler_is_enabled(prof) || prof->on_free==NULL) return; + + // call the on_free callback + mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sizeof(mi_block_t)); + (*prof->on_free)(prof, sample_data, p, heap); +} + + +//---------------------------------------------------------------------------- +// Profiling API +//-----------------------------------------------------------------------------* + +size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate) { + const size_t old_sample_rate = theap->profile_sample_rate; + theap->profile_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate); + if (theap->profile_sample_countdown > theap->profile_sample_rate) { + theap->profile_sample_countdown = theap->profile_sample_rate; // todo: adjust difference? + } + _mi_theap_update_sample_rate(theap); + return old_sample_rate; +} + +static bool mi_heap_set_profiler(mi_heap_t* heap, mi_profiler_t* profiler) { + mi_profiler_t* previous = (profiler==NULL ? mi_atomic_load_ptr_acquire(mi_profiler_t,&heap->profiler) : NULL); // don't overwrite unless it is NULL + return mi_atomic_cas_ptr_strong_acq_rel(mi_profiler_t, &heap->profiler, &previous, profiler); +} + +mi_decl_export bool mi_heap_profile(mi_heap_t* heap, mi_profiler_t* profiler) { + mi_profiler_stop(profiler); + return mi_heap_set_profiler(heap,profiler); +} + +mi_decl_export void mi_heap_profile_disable(mi_heap_t* heap) { + static mi_profiler_t mi_nosample_profiler = { NULL, 0, MI_SAMPLE_RATE_MAX, NULL, NULL, NULL }; + mi_heap_set_profiler(heap,NULL); // first clear any previous one + mi_heap_set_profiler(heap,&mi_nosample_profiler); // and then install the nosample one +} + +mi_decl_export bool mi_subproc_profile(mi_subproc_id_t subproc_id, mi_profiler_t* profiler) { + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + if (subproc==NULL) return false; + mi_profiler_stop(profiler); + mi_profiler_t* previous = (profiler==NULL ? mi_atomic_load_ptr_acquire(mi_profiler_t,&subproc->profiler) : NULL); // don't overwrite unless it is NULL + if (!mi_atomic_cas_ptr_strong_acq_rel(mi_profiler_t,&subproc->profiler, &previous, profiler)) { return false; } + mi_lock(&subproc->heaps_lock) { + for (mi_heap_t* heap = subproc->heaps; heap!=NULL; heap = heap->next) { + mi_heap_set_profiler(heap,profiler); + } + } + return true; +} + +mi_decl_export bool mi_profile( mi_profiler_t* profiler) { + return mi_subproc_profile(mi_subproc_main(),profiler); +} + +bool mi_profiler_start(mi_profiler_t* profiler ) { + if (profiler==NULL) return false; + const bool was_running = mi_profiler_set_enabled(profiler,true); + if (was_running) return true; + // for the main heap, if this is the profiler, start the theap more aggressively + // otherwise it will be picked up when theaps take the slow generic malloc path. + mi_heap_t* heap = mi_heap_main(); + if (mi_heap_profiler(heap)==profiler) { + mi_theap_t* theap = _mi_heap_theap_peek(heap); + if (theap!=NULL) { + _mi_theap_set_profile_sample_rate(theap,1); + } + } + return false; +} + +bool mi_profiler_stop(mi_profiler_t* profiler) { + if (profiler==NULL) return true; + return mi_profiler_set_enabled(profiler,false); +} diff --git a/src/static.c b/src/static.c index 1052b5f20..0a29331dc 100644 --- a/src/static.c +++ b/src/static.c @@ -6,9 +6,6 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ #ifndef _DEFAULT_SOURCE #define _DEFAULT_SOURCE -#if defined(__linux__) && !defined(_GNU_SOURCE) -#define _GNU_SOURCE // prof.c: dl_iterate_phdr / struct dl_phdr_info -#endif #endif #if defined(__sun) // same remarks as os.c for the static's context. @@ -30,21 +27,22 @@ terms of the MIT license. A copy of the license can be found in the file #include "bitmap.c" #include "heap.c" #include "heap-snapshot.c" -#include "prof.c" #include "init.c" #include "libc.c" #include "options.c" #include "os.c" +#include "prim/prim.c" // includes platform specific prim//prim.c +#include "prim/prim-tls.c" +#if MI_OSX_ZONE +#include "prim/osx/alloc-override-zone.c" +#endif #include "page.c" // includes page-queue.c #include "page-map.c" #include "random.c" +#include "sample-guarded.c" +#include "sample-profile.c" #include "scavenger.c" #include "stats.c" #include "subproc.c" #include "theap.c" #include "threadlocal.c" -#include "prim/prim.c" -#include "prim/prim-tls.c" -#if MI_OSX_ZONE -#include "prim/osx/alloc-override-zone.c" -#endif diff --git a/src/stats.c b/src/stats.c index fd823f2f7..885a491fc 100644 --- a/src/stats.c +++ b/src/stats.c @@ -39,26 +39,21 @@ static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) { if (amount > 0) { stat->total += amount; } } - -void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount) { - mi_atomic_addi64_relaxed(&stat->total, (int64_t)amount); -} - -void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) { - stat->total += amount; -} - -void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_increase_mt(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_update_mt(stat, (int64_t)amount); } -void __mi_stat_increase(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_increase(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_update(stat, (int64_t)amount); } -void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_decrease_mt(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_update_mt(stat, -((int64_t)amount)); } -void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_decrease(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_update(stat, -((int64_t)amount)); } @@ -69,26 +64,33 @@ void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount) { static void mi_stat_adjust_mt(mi_stat_count_t* stat, int64_t amount) { if (amount == 0) return; // adjust atomically + const int64_t peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&stat->peak); mi_atomic_addi64_relaxed(&stat->current, amount); - mi_atomic_addi64_relaxed(&stat->total, amount); + const int64_t prev_total = mi_atomic_addi64_relaxed(&stat->total, amount); + if (prev_total == peak) { mi_atomic_addi64_relaxed(&stat->peak, amount); } } static void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount) { if (amount == 0) return; stat->current += amount; - stat->total += amount; + if (stat->total==stat->peak) { stat->peak += amount; } + stat->total += amount; } -void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_adjust_mt(stat, (int64_t)amount); } -void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_adjust_increase(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_adjust(stat, (int64_t)amount); } -void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_adjust_mt(stat, -((int64_t)amount)); } -void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_adjust_decrease(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_adjust(stat, -((int64_t)amount)); } @@ -102,7 +104,7 @@ static void mi_stat_count_add_mt(mi_stat_count_t* stat, const mi_stat_count_t* s const int64_t prev_current = mi_atomic_addi64_relaxed(&stat->current, src_current); // Global current plus thread peak approximates new global peak - // note: peak scores do really not work across threads. + // note: peak scores do not really work across threads. // we used to just add them together but that often overestimates in practice. // similarly, max does not seem to work well. The current approach // by Artem Kharytoniuk (@artem-lunarg) seems to work better, see PR#1112 @@ -125,7 +127,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { // copy all fields MI_STAT_FIELDS() - #if MI_STAT>1 + #if MI_STATS for (size_t i = 0; i <= MI_BIN_HUGE; i++) { mi_stat_count_add_mt(&stats->malloc_bins[i], &src->malloc_bins[i]); } @@ -192,28 +194,27 @@ static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* ar } static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg, const char* notok ) { - _mi_fprintf(out, arg," %-10s:", msg); + _mi_fprintf(out, arg," %-12s:", msg); if (unit != 0) { - if (unit > 0) { + if (unit > 0) { // as KiB (power of two) mi_print_amount(stat->peak, unit, out, arg); mi_print_amount(stat->total, unit, out, arg); - // mi_print_amount(stat->freed, unit, out, arg); mi_print_amount(stat->current, unit, out, arg); mi_print_amount(unit, 1, out, arg); mi_print_count(stat->total, unit, out, arg); } - else { + else if (unit==-1) { // as K (decimal) mi_print_amount(stat->peak, -1, out, arg); mi_print_amount(stat->total, -1, out, arg); - // mi_print_amount(stat->freed, -1, out, arg); mi_print_amount(stat->current, -1, out, arg); - if (unit == -1) { - _mi_fprintf(out, arg, "%24s", ""); - } - else { - mi_print_amount(-unit, 1, out, arg); - mi_print_count((stat->total / -unit), 0, out, arg); - } + _mi_fprintf(out, arg, "%24s", ""); + } + else { // as KiB , unit is final count + mi_print_amount(stat->peak, 1, out, arg); + mi_print_amount(stat->total, 1, out, arg); + mi_print_amount(stat->current, -1, out, arg); + _mi_fprintf(out, arg, "%12s", ""); + mi_print_count(-unit, 0, out, arg); } if (stat->current != 0) { _mi_fprintf(out, arg, " "); @@ -236,23 +237,23 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t mi_stat_print_ex(stat, msg, unit, out, arg, NULL); } -#if MI_STAT>1 -static void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) { - _mi_fprintf(out, arg, " %-10s:", msg); +#if MI_STATS +static void mi_stat_total_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg) { + _mi_fprintf(out, arg, " %-12s:", msg); _mi_fprintf(out, arg, "%12s", " "); // no peak - mi_print_amount(stat->total, unit, out, arg); + mi_print_amount(stat->total, 1, out, arg); _mi_fprintf(out, arg, "\n"); } #endif static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) { - _mi_fprintf(out, arg, " %-10s:", msg); + _mi_fprintf(out, arg, " %-12s:", msg); mi_print_amount(stat->total, 0, out, arg); _mi_fprintf(out, arg, "\n"); } static void mi_stat_counter_print_size(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) { - _mi_fprintf(out, arg, " %-10s:", msg); + _mi_fprintf(out, arg, " %-12s:", msg); mi_print_amount(stat->total, 1, out, arg); _mi_fprintf(out, arg, "\n"); } @@ -261,16 +262,16 @@ static void mi_stat_average_print(int64_t count, int64_t total, const char* msg, const int64_t avg_tens = (count == 0 ? 0 : (total*10 / count)); const int64_t avg_whole = avg_tens/10; const int64_t avg_frac1 = avg_tens%10; - _mi_fprintf(out, arg, " %-10s: %5lld.%lld avg\n", msg, avg_whole, avg_frac1); + _mi_fprintf(out, arg, " %-12s: %5lld.%lld avg\n", msg, avg_whole, avg_frac1); } static void mi_print_header(const char* name,mi_output_fun* out, void* arg ) { - _mi_fprintf(out, arg, " %-11s %11s %11s %11s %11s %11s\n", + _mi_fprintf(out, arg, " %-13s %11s %11s %11s %11s %11s\n", name, "peak ", "total ", "current ", "block ", "total# "); } -#if MI_STAT>1 +#if MI_STATS static bool mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, mi_output_fun* out, void* arg) { bool found = false; char buf[64]; @@ -339,8 +340,8 @@ mi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_ size_t peak_commit; size_t page_faults; mi_process_info(&elapsed, &user_time, &sys_time, ¤t_rss, &peak_rss, ¤t_commit, &peak_commit, &page_faults); - _mi_fprintf(out, arg, " %-10s: %5zu.%03zu s\n", "elapsed", elapsed/1000, elapsed%1000); - _mi_fprintf(out, arg, " %-10s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: ", "process", + _mi_fprintf(out, arg, " %-12s: %5zu.%03zu s\n", "elapsed", elapsed/1000, elapsed%1000); + _mi_fprintf(out, arg, " %-12s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: ", "process", user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, page_faults); mi_printf_amount((int64_t)peak_rss, 1, out, arg, false); if (peak_commit > 0) { @@ -371,19 +372,21 @@ static void mi_stats_print_copy(const char* name, size_t id, const mi_stats_t* s _mi_fprintf(out, arg, "%s %zu\n", name, id); if (stats->malloc_normal.total + stats->malloc_huge.total != 0) { - #if MI_STAT>1 + #if MI_STATS mi_print_header("blocks", out, arg); mi_stats_print_bins(stats->malloc_bins, MI_BIN_HUGE, out, arg); #endif - #if MI_STAT - mi_stat_print(&stats->malloc_normal, "binned", (stats->malloc_normal_count.total == 0 ? -1 : 1), out, arg); - mi_stat_print(&stats->malloc_huge, "huge", (stats->malloc_huge_count.total == 0 ? -1 : 1), out, arg); + #if MI_STATS + mi_stat_print(&stats->malloc_normal, "binned", -stats->malloc_normal_count.total, out, arg); + mi_stat_print(&stats->malloc_huge, "huge", -stats->malloc_huge_count.total, out, arg); mi_stat_count_t total = { 0,0,0 }; mi_stat_count_add_mt(&total, &stats->malloc_normal); mi_stat_count_add_mt(&total, &stats->malloc_huge); - mi_stat_print_ex(&total, "total", 1, out, arg, ""); - #if MI_STAT>1 - mi_stat_total_print(&stats->malloc_requested, "malloc req", 1, out, arg); + mi_stat_print_ex(&total, "total", -(stats->malloc_normal_count.total + stats->malloc_huge_count.total), out, arg, ""); + #if MI_STATS>=2 + mi_stat_total_print(&stats->malloc_requested, "malloc req", out, arg); + #else + mi_stat_total_print(&stats->malloc_requested, "malloc req~", out, arg); #endif _mi_fprintf(out, arg, "\n"); #endif @@ -397,13 +400,18 @@ static void mi_stats_print_copy(const char* name, size_t id, const mi_stats_t* s // mi_stat_print(&stats->segments_cache, "-cached", -1, out, arg); mi_stat_print(&stats->pages, "pages", 0, out, arg); mi_stat_print(&stats->pages_abandoned, "abandoned", 0, out, arg); + mi_stat_print(&stats->pages_os_abandoned, "os abandoned", 0, out, arg); + mi_stat_print(&stats->pages_os_allocated, "os allocated", 0, out, arg); mi_stat_counter_print(&stats->pages_reclaim_on_alloc, "reclaima", out, arg); mi_stat_counter_print(&stats->pages_reclaim_on_free, "reclaimf", out, arg); mi_stat_counter_print(&stats->pages_reabandon_full, "reabandon", out, arg); mi_stat_counter_print(&stats->pages_unabandon_busy_wait, "waits", out, arg); mi_stat_counter_print(&stats->pages_extended, "extended", out, arg); mi_stat_counter_print(&stats->pages_retire, "retire", out, arg); + mi_stat_counter_print(&stats->pages_stat_updates, "stat updates", out, arg); + mi_stat_average_print(stats->pages_stat_updates.total, stats->pages_stat_update_count.total, "stat avg", out, arg); mi_stat_average_print(stats->page_searches_count.total, stats->page_searches.total, "searches", out, arg); + mi_stat_counter_print(&stats->profile_samples, "prof samples", out, arg); _mi_fprintf(out, arg, "\n"); } @@ -425,12 +433,13 @@ static void mi_stats_print_copy(const char* name, size_t id, const mi_stats_t* s mi_stat_print_ex(&stats->heaps, "heaps", 0, out, arg, ""); mi_stat_counter_print(&stats->heaps_delete_wait, "heap waits", out, arg); _mi_fprintf(out, arg, "\n"); - - mi_print_header("process", out, arg); - mi_stat_print_ex(&stats->threads, "threads", 0, out, arg, ""); - _mi_fprintf(out, arg, " %-10s: %5i\n", "numa nodes", _mi_os_numa_node_count()); - mi_process_info_print_out(out, arg); } + + mi_print_header("process", out, arg); + mi_stat_print_ex(&stats->threads, "threads", 0, out, arg, ""); + _mi_fprintf(out, arg, " %-12s: %5i\n", "numa nodes", _mi_os_numa_node_count()); + mi_process_info_print_out(out, arg); + _mi_fprintf(out, arg, "\n"); } @@ -469,6 +478,10 @@ static const mi_stats_t* mi_heap_get_stats(mi_heap_t* heap) { else return mi_stats_merge_theap_to_heap(theap); } +static const mi_stats_t* mi_theap_get_stats(mi_theap_t* theap) { + return &theap->stats; +} + // deprecated void mi_stats_reset(void) mi_attr_noexcept { if (!mi_theap_is_initialized(_mi_theap_default())) return; @@ -641,6 +654,14 @@ bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_attr_noexcept { return mi_stats_copy(stats, mi_heap_get_stats(heap)); } +bool mi_theap_stats_get(mi_theap_t* theap, mi_stats_t* stats) mi_attr_noexcept { + return mi_stats_copy(stats, mi_theap_get_stats(theap)); +} + +void mi_theap_stats_merge_to_heap(mi_theap_t* theap) mi_attr_noexcept { + if (theap == NULL) return; + _mi_theap_merge_stats(theap); +} static bool mi_cdecl mi_heap_aggregate_visitor(mi_heap_t* heap, void* arg) { mi_stats_t* stats = (mi_stats_t*)arg; diff --git a/src/subproc.c b/src/subproc.c index 8fa23f3f0..351e10f37 100644 --- a/src/subproc.c +++ b/src/subproc.c @@ -102,7 +102,7 @@ bool _mi_subproc_is_main(mi_subproc_t* subproc) { mi_subproc_t* _mi_subproc(void) { mi_theap_t* theap = _mi_theap_default(); - if (theap == NULL || theap->tld == NULL) { // see issue #1289 + if (theap == NULL || theap->tld == NULL || theap->tld->subproc == NULL) { // see issue #1289 and #1391 return _mi_subproc_main(); } else { @@ -189,6 +189,13 @@ mi_subproc_id_t mi_subproc_new(void) { mi_assert_internal(parent->theap_meta->tld!=NULL); mi_assert_internal(parent->theap_meta->tld->thread_id == MI_THREADID_DETACHED); _mi_theap_init(theap_meta,heap_main,parent->theap_meta->tld /* detached tld */); + // as for the meta theap of the main sub-process (`init.c:mi_heap_main_init_once`): it is not shared with other threads. + // (abandoning one of its pages would also allocate the per-bin abandoned bitmap (`arena.c:mi_arena_pages_abandoned_ensure`) + // from the meta theap itself, under the `theap_meta_lock` that the allocation that filled the page still holds) + theap_meta->allow_page_abandon = false; + theap_meta->page_full_retain = 2; + theap_meta->sample_rate = 0; // no sampling for meta data + theap_meta->sample_countdown = 0; subproc->theap_meta = theap_meta; return _mi_subproc_to_id(subproc); @@ -229,8 +236,8 @@ static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subpro } } - subproc->theap_meta = NULL; // theap meta stats are merged during heap_destroy of the main heap - + subproc->theap_meta = NULL; // theap meta stats are merged during heap_destroy of the main heap + if (!_mi_subproc_is_main(subproc)) { // merge stats back into the main subproc _mi_stats_merge_into(&mi_process_subproc_main.stats, &subproc->stats); @@ -387,11 +394,13 @@ void _mi_process_fork_prepare(void) { mi_lock_acquire(&mi_subprocs_lock); _mi_thread_locals_fork_prepare(); for (mi_subproc_t* sp = mi_subprocs; sp != NULL; sp = sp->next) { mi_subproc_fork_prepare(sp); } + _mi_arenas_purge_guard_acquire(); // last: a purge pass takes no lock, so the one in progress ends (see `arena.c`) } void _mi_process_fork_parent(void) { if (!_mi_process_is_initialized) return; if (mi_atomic_decrement_acq_rel(&mi_fork_depth) != 1) return; + _mi_arenas_purge_guard_release(); // release in reverse: last sub-process first (the registry is a stack: newest first, so walk it into a reversed order) size_t n = 0; for (mi_subproc_t* sp = mi_subprocs; sp != NULL; sp = sp->next) { n++; } @@ -410,11 +419,9 @@ void _mi_process_fork_child(void) { if (mi_atomic_exchange_acq_rel(&mi_fork_depth, 0) == 0) return; _mi_process_is_forked_child = true; _mi_scavenger_forked_child(); // the scavenger thread did not survive the fork; clear the state that says it did - const bool purge_cut_off = _mi_arenas_purge_guard_reset(); // nor did a thread that was purging; release the guard it held + _mi_arenas_purge_guard_release(); // taken by the prepare handler: no purge pass was in progress, so each `purge_expire` is as a finished pass left it mi_lock_init(&mi_subprocs_lock); for (mi_subproc_t* sp = mi_subprocs; sp != NULL; sp = sp->next) { - // the purge pass that was cut off may have been over this one: make a pass due, for the arenas whose expire is still set - if (purge_cut_off) { mi_atomic_storei64_relaxed(&sp->purge_expire, (mi_msecs_t)1); } mi_lock_init(&sp->arena_reserve_lock); mi_lock_init(&sp->heaps_lock); mi_lock_init(&sp->tlds_lock); @@ -426,6 +433,7 @@ void _mi_process_fork_child(void) { mi_atomic_store_relaxed(&t->park_state, (uint32_t)MI_PARK_RUNNING); mi_atomic_store_relaxed(&t->park_reclaim, (uint32_t)0); mi_atomic_store_relaxed(&t->park_swept, (uint32_t)0); + t->holes_sweeping = false; // a sweep that was running at the fork (by the scavenger, or by a thread that is gone) does not continue here } for (mi_heap_t* h = sp->heaps; h != NULL; h = h->next) { mi_lock_init(&h->theaps_lock); diff --git a/src/theap.c b/src/theap.c index e37164a87..a35a001d2 100644 --- a/src/theap.c +++ b/src/theap.c @@ -98,7 +98,6 @@ typedef enum mi_collect_e { static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) { MI_UNUSED(arg2); - MI_UNUSED(theap); mi_assert_expensive(mi_theap_page_is_valid(theap, pq, page, NULL, NULL)); mi_collect_t collect = *((mi_collect_t*)arg_collect); // A collect has no allocation to serve, so it must not un-purge: the allocation path @@ -106,7 +105,8 @@ static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_pag // `mi_on_thread_idle`, which collects before it sweeps, un-purges a run of every page whose free // blocks are all discarded and then re-discards it: two syscalls per page per park, forever. if (theap->tld != NULL && mi_atomic_load_relaxed(&theap->tld->park_reclaim) != 0) return false; - _mi_page_free_collect_no_unpurge(page, collect >= MI_FORCE); + _mi_page_free_collect_no_unpurge(page, collect >= MI_FORCE); // update used count + _mi_page_update_stats_for(page,theap); // (on `theap`, which is not the calling thread's when the scavenger collects for a parked thread) if (mi_page_all_free(page)) { // no more used blocks, possibly free the page. if (collect >= MI_FORCE || page->retire_expire == 0) { // either forced/abandon, or not already retired @@ -122,7 +122,7 @@ static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_pag } void _mi_theap_merge_stats(mi_theap_t* theap) { - mi_assert_internal(mi_theap_is_initialized(theap)); + mi_assert_internal(mi_theap_is_initialized(theap)); mi_heap_t* const heap = _mi_theap_heap(theap); _mi_stats_merge_into(&heap->stats, &theap->stats); } @@ -150,12 +150,14 @@ static void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect) // wait would become an unbounded subproc-wide madvise pass. The sweep's caller runs the arena // purge itself as a reclaim-gated phase (`_mi_arenas_purge_now`). //mi_atomic_storei64_release(&theap->tld->subproc->purge_expire, 1); - if (theap->tld == NULL || mi_atomic_load_relaxed(&theap->tld->park_state) != MI_PARK_SWEEPING) { + if (collect != MI_ABANDON && (theap->tld == NULL || mi_atomic_load_relaxed(&theap->tld->park_state) != MI_PARK_SWEEPING)) { _mi_arenas_collect(collect == MI_FORCE /* force purge? */, collect >= MI_FORCE /* visit all? */, theap->tld); } // merge statistics - _mi_theap_merge_stats(theap); + if (mi_option_is_enabled(mi_option_collect_merges_stats)) { + _mi_theap_merge_stats(theap); + } } void _mi_theap_collect_abandon(mi_theap_t* theap) { @@ -486,49 +488,11 @@ mi_theap_t* mi_theap_set_default(mi_theap_t* theap) { return previous; } -#if MI_GUARDED -mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { - theap->guarded_sample_rate = sample_rate; - theap->guarded_sample_count = sample_rate; // count down samples - if (theap->guarded_sample_rate > 1) { - if (seed == 0) { - seed = _mi_theap_random_next(theap); - } - theap->guarded_sample_count = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate` - } -} - -mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { - theap->guarded_size_min = min; - theap->guarded_size_max = (min > max ? min : max); -} - -static void mi_theap_guarded_init(mi_theap_t* theap) { - mi_theap_guarded_set_sample_rate(theap, - (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX), - (size_t)mi_option_get(mi_option_guarded_sample_seed)); - mi_theap_guarded_set_size_bound(theap, - (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX), - (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) ); -} -#else -mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { - MI_UNUSED(theap); MI_UNUSED(sample_rate); MI_UNUSED(seed); -} - -mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { - MI_UNUSED(theap); MI_UNUSED(min); MI_UNUSED(max); -} -static void mi_theap_guarded_init(mi_theap_t* theap) { - MI_UNUSED(theap); -} -#endif static void mi_theap_options_init(mi_theap_t* theap) { theap->allow_page_reclaim = (mi_option_get(mi_option_page_reclaim_on_free) >= 0); theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0); theap->page_full_retain = mi_option_get_clamp(mi_option_page_full_retain, -1, 32); - _mi_prof_theap_init(theap); // fork: sampling state theap->is_detached = (theap->tld->thread_id == MI_THREADID_DETACHED); } @@ -586,8 +550,8 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) else { _mi_random_split(&head_random, &theap->random); // &theap->random is used as nonce so it is ok if threads capture the same head->random } - theap->cookie = _mi_theap_random_next(theap) | 1; - mi_theap_guarded_init(theap); // needs theap->random + // theap->cookie = _mi_theap_random_next(theap) | 1; + _mi_theap_guarded_init(theap); // needs theap->random if (!theap->is_detached) { mi_subproc_stat_increase(_mi_theap_subproc(theap),theaps,1); // on subproc to match theap_free_mem } @@ -851,7 +815,7 @@ void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) { area->reserved = page->reserved * bsize; area->committed = page->capacity * bsize; area->blocks = mi_page_start(page); - area->used = page->used; // number of blocks in use (#553) + area->used = mi_page_used(page); // number of blocks in use (#553) area->block_size = ubsize; area->full_block_size = bsize; area->reserved1 = page; @@ -877,7 +841,7 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi _mi_page_free_collect_no_unpurge(page,true); // collect both thread_delayed and local_free; visiting must not un-purge a hole mi_assert_internal(page->local_free == NULL); - if (page->used == 0) return true; + if (mi_page_used(page) == 0) return true; size_t psize; uint8_t* const pstart = mi_page_area(page, &psize); @@ -887,13 +851,13 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi // optimize page with one block if (page->capacity == 1) { - mi_assert_internal(page->used == 1 && page->free == NULL); + mi_assert_internal(mi_page_used(page) == 1 && page->free == NULL); return visitor(heap, area, pstart, ubsize, arg); } mi_assert(bsize <= UINT32_MAX); // optimize full pages - if (page->used == page->capacity) { + if (mi_page_used(page) == page->capacity) { uint8_t* block = pstart; for (size_t i = 0; i < page->capacity; i++) { if (!visitor(heap, area, block, ubsize, arg)) return false; @@ -953,7 +917,7 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi free_map[bitidx] |= ((uintptr_t)1 << bit); } } - mi_assert_internal(page->capacity == (free_count + purged_count + page->used)); + mi_assert_internal(page->capacity == (free_count + purged_count + mi_page_used(page))); // walk through all blocks skipping the free ones #if MI_DEBUG>1 @@ -985,7 +949,7 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi block += bsize * MI_INTPTR_BITS; } } - mi_assert_internal(page->used == used_count); + mi_assert_internal(mi_page_used(page) == used_count); return true; } diff --git a/src/threadlocal.c b/src/threadlocal.c index fc93b0890..765b6f04b 100644 --- a/src/threadlocal.c +++ b/src/threadlocal.c @@ -186,7 +186,6 @@ bool _mi_thread_local_set( mi_thread_local_t key, void* val ) { // get a tls slot value static mi_decl_noinline void* mi_thread_local_get_regular( mi_thread_local_t key ) { - mi_assert_internal(key!=0); const mi_thread_locals_t* const tls = mi_thread_locals_peek(); if mi_unlikely(tls==NULL) { // this can happen if a thread local is accessed after the thread local has been freed @@ -205,12 +204,13 @@ static mi_decl_noinline void* mi_thread_local_get_regular( mi_thread_local_t key // get a thread local value void* _mi_thread_local_get( mi_thread_local_t key ) { - mi_assert_internal(key!=0); if mi_likely(key == mi_thread_local_key_fast) { return mi_slot_fast_get(); } else { - return mi_thread_local_get_regular(key); + void* p = mi_thread_local_get_regular(key); + if (key==0) { mi_assert_internal(p==NULL); } + return p; } } diff --git a/test/main-override-static.c b/test/main-override-static.c index c3230e1af..368710c3c 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -50,11 +50,11 @@ int main() { // double_free1(); // double_free2(); // corrupt_free(); + // invalid_free(); // block_overflow1(); // block_overflow2(); test_canary_leak(); // test_aslr(); - // invalid_free(); // test_reserved(); // negative_stat(); // test_theap_walk(); @@ -349,7 +349,7 @@ static void test_large_pages(void) { #include #include -#define MI_LARGE_WSIZE_MAX (4*1024*1024 / MI_INTPTR_SIZE) +#define MI_LARGE_WSIZE_MAX (4*1024*1024 / MI_SIZE_SIZE) #define MI_BIN_HUGE 100 //#define MI_ALIGN2W @@ -390,10 +390,10 @@ static inline uint8_t mi_bsr32(uint32_t x) { // Bit scan reverse: return the index of the highest bit. uint8_t _mi_bsr(uintptr_t x) { if (x == 0) return 0; - #if MI_INTPTR_SIZE==8 + #if MI_SIZE_SIZE==8 uint32_t hi = (x >> 32); return (hi == 0 ? mi_bsr32((uint32_t)x) : 32 + mi_bsr32(hi)); - #elif MI_INTPTR_SIZE==4 + #elif MI_SIZE_SIZE==4 return mi_bsr32(x); #else # error "define bsr for non-32 or 64-bit platforms" diff --git a/test/test-abandoned-lazy.c b/test/test-abandoned-lazy.c index 87ce326ab..45db04bf9 100644 --- a/test/test-abandoned-lazy.c +++ b/test/test-abandoned-lazy.c @@ -39,7 +39,7 @@ terms of the MIT license. #include "mimalloc.h" -#if defined(MI_TSAN) || defined(MI_UBSAN) || defined(MI_GUARDED) +#if defined(MI_TSAN) || defined(MI_UBSAN) || (defined(MI_GUARDED) && MI_GUARDED>0) static int ITER = 20; #else static int ITER = 100; diff --git a/test/test-api.c b/test/test-api.c index 2d2b988de..70bba60b1 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -110,12 +110,12 @@ int main(void) { }; #if MI_INTPTR_BITS > 32 CHECK_BODY("malloc-free-invalid-low") { - mi_free((void*)(MI_ZU(0x0000000003990080))); // issue #1087 + mi_cfree((void*)(MI_ZU(0x0000000003990080))); // issue #1087 }; #endif CHECK_BODY("calloc-overflow") { // use (size_t)&mi_calloc to get some number without triggering compiler warnings - result = (mi_calloc((size_t)&mi_calloc,SIZE_MAX/1000) == NULL); + result = (mi_calloc((size_t)&mi_calloc,SIZE_MAX/1000) == NULL); }; CHECK_BODY("malloc-large") { // see PR #544. void* p = mi_malloc(67108872); @@ -132,7 +132,7 @@ int main(void) { CHECK_BODY("mi_urealloc_invalid") { void* p = mi_malloc(64); size_t pre, post; - void* q = mi_urealloc((char*)p + 3, 32, &pre, &post); + void* q = mi_urealloc((char*)p + 3, 40, &pre, &post); mi_free(p); result = (q==NULL || q==(uint8_t*)p+3); } @@ -240,7 +240,7 @@ int main(void) { void* p[8]; const int max_align_shift = #if SIZE_MAX > UINT32_MAX - 28 + 28 /* up to 64 MiB alignment */ #else 20 #endif @@ -330,6 +330,24 @@ int main(void) { } result = ok; }; + CHECK_BODY("zalloc-csize-large") { + // mi_theap_zalloc_csize must zero its large branch too: it used to + // delegate sizes above MI_SMALL_SIZE_MAX to the plain malloc path. + mi_theap_t* theap = mi_theap_get_default(); + size_t size = MI_SMALL_SIZE_MAX + 64; + bool ok = true; + for (int round = 0; round < 16 && ok; round++) { + uint8_t* junk = (uint8_t*)mi_theap_malloc(theap, size); + memset(junk, 0xAB, size); + mi_free(junk); + uint8_t* z = (uint8_t*)mi_theap_zalloc_csize(theap, size); + for (size_t i = 0; i < size; i++) { + if (z[i] != 0) { ok = false; break; } + } + mi_free(z); + } + result = ok; + }; CHECK_BODY("zalloc-aligned-small1") { size_t zalloc_size = MI_SMALL_SIZE_MAX / 2; uint8_t* p = (uint8_t*)mi_zalloc_aligned(zalloc_size, MI_MAX_ALIGN_SIZE * 2); @@ -359,6 +377,7 @@ int main(void) { ptr = mi_rezalloc_aligned(ptr, n, alignment); assert(mem_has_vals((uint8_t*)ptr,n/2,123)); result = mem_is_zero((uint8_t*)ptr + n/2, n/2); + mi_free(ptr); } // --------------------------------------------------- @@ -396,6 +415,7 @@ int main(void) { shared_ptr = mi_realloc(shared_ptr, i * 64); } } + mi_free(shared_ptr); } // --------------------------------------------------- @@ -420,7 +440,23 @@ int main(void) { } } + CHECK_BODY("free_small3") { + for(size_t n = 1; n < MI_SMALL_SIZE_MAX/sizeof(int); n *=2) { + const size_t size = n*sizeof(int); + int* p = (int*)mi_zalloc_small(size); + p[n-1] = 42; + mi_free_small(p); + } + } + CHECK_BODY("free_small4") { + for(size_t n = 1; n < MI_SMALL_SIZE_MAX/sizeof(int); n *=2) { + const size_t size = n*sizeof(int); + int* p = (int*)mi_zalloc_small(size); + p[n-1] = 42; + mi_free(p); + } + } // --------------------------------------------------- // Returned block sizes // --------------------------------------------------- @@ -440,7 +476,7 @@ int main(void) { } } - #if (MI_INTPTR_SIZE > 4) + #if (MI_SIZE_SIZE > 4) CHECK_BODY("arena_reserve") { result = (0==mi_reserve_os_memory(16*MI_GiB,false,true)); } diff --git a/test/test-forced-purge.c b/test/test-forced-purge.c index e2555ce24..63960d211 100644 --- a/test/test-forced-purge.c +++ b/test/test-forced-purge.c @@ -48,6 +48,10 @@ int main(void) { fprintf(stderr, "test-forced-purge: skipped (no scavenger, or no purge delay)\n"); return 0; } + // The rounds count bytes. With transparent huge pages left whole (`MI_ALLOW_THP=FULL`) a pass purges aligned 2 MiB + // units only and leaves the edges of each freed run for later, so purge by the slice here (the size is rounded up to an + // OS page, which is never more than a slice). + mi_option_set(mi_option_minimal_purge_size, 4 /* KiB */); // the scavenger starts when a second thread initializes or a thread first parks void* warm = mi_malloc(64); mi_free(warm); if (mi_on_thread_idle_start()) { mi_on_thread_idle_end(); } diff --git a/test/test-heap-mt.c b/test/test-heap-mt.c index 4657b5e76..c6cc0b60c 100644 --- a/test/test-heap-mt.c +++ b/test/test-heap-mt.c @@ -35,7 +35,7 @@ terms of the MIT license. // argument defaults #if defined(MI_TSAN) static int ITER = 200; -#elif defined(MI_UBSAN) || defined(MI_GUARDED) +#elif defined(MI_UBSAN) || (defined(MI_GUARDED) && MI_GUARDED>0) static int ITER = 200; #else static int ITER = 1000; // (a 3-core CI VM runs ~10 iterations/s; pass a count on the command line for more) diff --git a/test/test-heap-release-mt.c b/test/test-heap-release-mt.c index 67cc4e0be..053aa809f 100644 --- a/test/test-heap-release-mt.c +++ b/test/test-heap-release-mt.c @@ -7,8 +7,13 @@ terms of the MIT license. /* Release heaps on many threads at once while other threads free into them. - NCHURN threads loop: `mi_heap_new`, allocate mixed sizes, hand a third of the blocks to a shared - ring, free a third, leak a third, then `mi_heap_delete` (or `mi_heap_destroy` when nothing was - handed out). Some heaps are also published for a while so that short-lived threads allocate from + ring, free a third, keep a third live, then `mi_heap_delete` (or `mi_heap_destroy` when nothing was + handed out). What was kept live is freed after the delete, when it belongs to the main heap: left + in place it pins a page per block for the rest of the run (tens of GiB in a release build, and with + a guard page behind every page, MI_SECURE=FULL, more mappings than `vm.max_map_count` allows). + After a delete by another thread the owner goes on with its next heap while that delete may still + run, and frees those blocks once it has returned. + Some heaps are also published for a while so that short-lived threads allocate from them and exit with their blocks live (the abandon path, and the per-bin abandoned maps). - NFREE threads pop blocks from the ring and `mi_free` them: those frees land before, during and after the delete of the owning heap, so pages are freed and re-abandoned by a foreign thread while @@ -54,6 +59,7 @@ static _Atomic int stop; static _Atomic int park_mode; // second half: park around the release, see the header static _Atomic unsigned long heaps_done, frees_done, exits_done, parks_done, parks_handed_off, remote_deletes; static _Atomic(mi_heap_t*) to_delete[NCHURN]; // park mode: heap i's owner is parked and waits for a freer to delete it +static _Atomic int deleted[NCHURN]; // park mode: the freer that took `to_delete[i]` is done with the delete static const size_t sizes[] = {8,16,48,96,200,512,1024,2048,5000,8192,20000,40000,70000,200000}; #define NS (sizeof(sizes)/sizeof(sizes[0])) @@ -79,9 +85,22 @@ static unsigned rnd(unsigned* s){ *s = *s*1103515245u+12345u; return *s>>8; } static _Atomic(mi_heap_t*) shared[NSHARED]; static _Atomic int users[NSHARED]; +#define MIN_BLOCKS 50 +#define MAX_BLOCKS 450 // a heap gets MIN_BLOCKS up to (excluding) MAX_BLOCKS blocks + +static void free_all(void** blocks, int* count) { + for (int i = 0; i < *count; i++) mi_free(blocks[i]); + *count = 0; +} + static void* churn(void* arg) { const int self = (int)(uintptr_t)arg - 1; unsigned seed = (unsigned)(uintptr_t)arg * 7919u + 1; + void* kept[MAX_BLOCKS]; int nkept = 0; // live in the heap when it is released + // The kept blocks of the heap that we left to a freer. The delete may still be running when we are back from the park, + // and until it returns our own frees into that heap are not safe (they are not atomic, that is what the park is for). + // So go on with the next heap in the meantime and free these once `deleted[self]` says the delete is over. + void* behind[MAX_BLOCKS]; int nbehind = 0; while (!atomic_load(&stop)) { mi_heap_t* h = mi_heap_new(); int slot = -1; @@ -92,7 +111,7 @@ static void* churn(void* arg) { if (!atomic_compare_exchange_strong(&shared[slot], &expect, h)) slot = -1; } int handed = 0; - int n = 50 + rnd(&seed) % 400; + int n = MIN_BLOCKS + rnd(&seed) % (MAX_BLOCKS - MIN_BLOCKS); for (int i = 0; i < n; i++) { size_t sz = sizes[rnd(&seed) % NS]; void* p = mi_heap_malloc(h, sz); @@ -101,7 +120,7 @@ static void* churn(void* arg) { unsigned r = rnd(&seed) % 3; if (r == 0) { push(p); handed++; } else if (r == 1) mi_free(p); - // else leak into the heap; delete moves it to main / destroy frees it + else kept[nkept++] = p; // delete moves it to main / destroy frees it } if (slot >= 0) { // let exiting threads use it for a bit, then unpublish @@ -110,9 +129,15 @@ static void* churn(void* arg) { while (atomic_load(&users[slot]) != 0) usleep(10); // in-flight allocators finish (contract: no alloc during delete) handed = 1; // exiting threads may hold blocks: must delete, not destroy } + bool released = false; if (atomic_load(&park_mode)) { const bool remote = ((rnd(&seed) & 1) != 0); - if (remote) { atomic_store(&to_delete[self], h); } // a freer deletes it while we are parked + if (remote) { + // one remote delete at a time: `deleted[self]` is about to be reused + if (nbehind > 0) { while (!atomic_load(&deleted[self])) usleep(10); free_all(behind, &nbehind); } + atomic_store(&deleted[self], 0); + atomic_store(&to_delete[self], h); // a freer deletes it while we are parked + } const bool parked = mi_on_thread_idle_start(); if (parked) { atomic_fetch_add(&parks_handed_off, 1); } if (remote) { @@ -128,14 +153,20 @@ static void* churn(void* arg) { if (remote) { mi_heap_t* const left = atomic_exchange(&to_delete[self], NULL); // only non-NULL if we are stopping if (left != NULL) { mi_heap_delete(left); } - atomic_fetch_add(&heaps_done, 1); - continue; + else { memcpy(behind, kept, (size_t)nkept * sizeof(void*)); nbehind = nkept; nkept = 0; } // a freer took it + released = true; } } - if (handed == 0 && (rnd(&seed) & 1)) mi_heap_destroy(h); - else mi_heap_delete(h); + if (!released) { + if (handed == 0 && (rnd(&seed) & 1)) { mi_heap_destroy(h); nkept = 0; } + else { mi_heap_delete(h); } + } + // What stayed live now belongs to the main heap. Left there, each block pins its page for the rest of the run. + free_all(kept, &nkept); + if (nbehind > 0 && atomic_load(&deleted[self])) { free_all(behind, &nbehind); } atomic_fetch_add(&heaps_done, 1); } + if (nbehind > 0) { while (!atomic_load(&deleted[self])) usleep(10); free_all(behind, &nbehind); } return NULL; } @@ -144,7 +175,7 @@ static void delete_parked_heaps(void) { for (int i = 0; i < NCHURN; i++) { if (atomic_load(&to_delete[i]) == NULL) continue; mi_heap_t* const h = atomic_exchange(&to_delete[i], NULL); - if (h != NULL) { mi_heap_delete(h); atomic_fetch_add(&remote_deletes, 1); } + if (h != NULL) { mi_heap_delete(h); atomic_store(&deleted[i], 1); atomic_fetch_add(&remote_deletes, 1); } } } diff --git a/test/test-heap-teardown.c b/test/test-heap-teardown.c index dc3b81f7a..81be9db11 100644 --- a/test/test-heap-teardown.c +++ b/test/test-heap-teardown.c @@ -26,6 +26,9 @@ terms of the MIT license. two-deletes two heaps deleted at once while other threads free into both. parked a thread that used the heap is parked (`mi_on_thread_idle_start`) and swept by the scavenger while the heap is deleted. + foreign-frees destroy (and delete) a heap after another thread freed blocks + in its abandoned (full) pages: such a page keeps the last freed + block on its thread-free list, and it still counts as used. > mimalloc-test-heap-teardown [ITER] */ @@ -40,7 +43,7 @@ terms of the MIT license. #if defined(MI_TSAN) static int ITER = 20; -#elif defined(MI_UBSAN) || defined(MI_GUARDED) +#elif defined(MI_UBSAN) || (defined(MI_GUARDED) && MI_GUARDED>0) static int ITER = 20; #elif !defined(NDEBUG) static int ITER = 50; @@ -447,6 +450,55 @@ static void test_parked(void) { } +/* ----------------------------------------------------------- + foreign-frees: a heap fills pages (which are abandoned once full), another + thread frees a block in many of them and exits, and then the heap is destroyed + (or deleted). A free into an abandoned page collects the thread-free list + except for its head, so the page is left with a block on that list that is + still counted as used. The destroy resets the used count of every page it + frees; folding that list in afterwards reads as a corrupted list ("more + blocks freed than in use"), which aborts a debug or secure build. +----------------------------------------------------------- */ + +#define FF_NPTRS 50000 +static void* ff_ptrs[FF_NPTRS]; +static volatile long ff_errors; + +static void ff_error(int err, void* arg) { + (void)err; (void)arg; + atomic_add_long(&ff_errors, 1); +} + +static void ff_freer(intptr_t tid) { + (void)tid; + for (int i = 0; i < FF_NPTRS; i += 97) { mi_free(ff_ptrs[i]); ff_ptrs[i] = NULL; } +} + +static void test_foreign_frees(void) { + mi_register_error(&ff_error, NULL); // count errors instead of aborting on the first + const int iters = (ITER < 8 ? ITER : 8); + for (int n = 0; n < iters; n++) { + for (int destroy = 0; destroy <= 1; destroy++) { + mi_heap_t* heap = mi_heap_new(); + EXPECT("heap_new", heap != NULL); + if (heap == NULL) break; + for (int i = 0; i < FF_NPTRS; i++) { ff_ptrs[i] = mi_heap_malloc(heap, 64); } + run_os_threads(1, &ff_freer); + if (destroy) { + mi_heap_destroy(heap); + } + else { + mi_heap_delete(heap); + for (int i = 0; i < FF_NPTRS; i++) { if (ff_ptrs[i] != NULL) { mi_free(ff_ptrs[i]); } } + } + } + progress(n * 4); + } + mi_register_error(NULL, NULL); + EXPECT("no error is reported", atomic_load_long(&ff_errors) == 0); +} + + /* ----------------------------------------------------------- Main ----------------------------------------------------------- */ @@ -467,6 +519,7 @@ int main(int argc, char** argv) { fprintf(stderr, "test: page-churn... "); test_page_churn(); fprintf(stderr, " %s.\n", failed ? "FAILED" : "ok"); fprintf(stderr, "test: two-deletes... "); test_two_deletes(); fprintf(stderr, " %s.\n", failed ? "FAILED" : "ok"); fprintf(stderr, "test: parked... "); test_parked(); fprintf(stderr, " %s.\n", failed ? "FAILED" : "ok"); + fprintf(stderr, "test: foreign-frees... "); test_foreign_frees(); fprintf(stderr, " %s.\n", failed ? "FAILED" : "ok"); mi_collect(true); mi_stats_print(NULL); diff --git a/test/test-park-handoff.c b/test/test-park-handoff.c index 7a396c7df..bb2e8d002 100644 --- a/test/test-park-handoff.c +++ b/test/test-park-handoff.c @@ -21,6 +21,8 @@ int main(void) { printf("test-park-handoff: skipped on Windows (uses pthreads/fo #include "mimalloc.h" #include +#include +#include #include #include #include @@ -37,7 +39,7 @@ static void check(const char* name, bool ok) { if (!ok) failures++; } -#if defined(MI_GUARDED) +#if (defined(MI_GUARDED) && MI_GUARDED>0) #define LIVE (2000) // every sampled allocation gets a guard page (its own mapping): stay under vm.max_map_count #else #define LIVE (20000) @@ -220,7 +222,14 @@ static void* park_then_cancel(void* arg) { churn(p); free(p); (void)mi_on_thread_idle_start(); + #if defined(MI_TSAN) + // ThreadSanitizer stops modelling the synchronization of a thread that is cancelled inside one of its + // blocking interceptors (`usleep`: the interceptor scope is never closed, and the mutexes and atomics of + // the thread's destructors are then ignored), which reports everything the teardown does as a race. + for (;;) { pthread_testcancel(); sched_yield(); } + #else for (;;) { pthread_testcancel(); usleep(50); } // cancelled mid-park, as at a blocking syscall + #endif } static void test_park_then_exit(void) { @@ -431,6 +440,25 @@ static void test_park_inside_window_gets_swept(void) { // straight out of the churned pages' free lists without meeting a corrupt entry (which aborts) and // without two allocations aliasing. Refilling exactly the freed slots forces exactly that path. // --------------------------------------------------------------------------- +#if defined(__APPLE__) || defined(__GLIBC__) +#include +#define FORK_CHILD_HAS_BACKTRACE 1 +#endif + +// in the forked child: say where a fault happened (there is no core dump to look at on a CI machine) +static void fork_child_fault(int sig, siginfo_t* info, void* ctx) { + (void)ctx; + char buf[128]; + const int n = snprintf(buf, sizeof(buf), "\n forked child: signal %d at address %p (in a mimalloc heap: %d)\n", sig, info->si_addr, (int)mi_is_in_heap_region(info->si_addr)); + if (n > 0) { (void)!write(2, buf, (size_t)n); } + #if FORK_CHILD_HAS_BACKTRACE + void* frames[48]; + const int count = backtrace(frames, 48); + backtrace_symbols_fd(frames, count, 2); + #endif + _exit(64 + sig); +} + static void test_fork_while_parked(void) { enum { ROUNDS = 16 }; bool all_ok = true; @@ -443,6 +471,12 @@ static void test_fork_while_parked(void) { const pid_t pid = fork(); if (pid == 0) { // child: allocate out of the inherited free lists and check for aliasing + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_sigaction = &fork_child_fault; + sa.sa_flags = SA_SIGINFO; + sigaction(SIGSEGV, &sa, NULL); + sigaction(SIGBUS, &sa, NULL); const size_t ke = keep_every(); int bad = 0; for (int i = 0; i < LIVE; i++) { @@ -466,8 +500,16 @@ static void test_fork_while_parked(void) { else { int status = 0; // a corrupt free-list entry aborts the child (a signal), which is a failure too - if (waitpid(pid, &status, 0) < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) { all_ok = false; } - if (first_corrupt_survivor(p) >= 0) { all_ok = false; } // and the parent stays intact + if (waitpid(pid, &status, 0) < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) { + all_ok = false; + if (WIFSIGNALED(status)) { fprintf(stderr, "\n round %d: the child was killed by signal %d\n", r, WTERMSIG(status)); } + else if (WIFEXITED(status)) { fprintf(stderr, "\n round %d: the child failed with %d (1: out of memory, 2: a survivor changed, 3: two allocations alias, 64+n: signal n)\n", r, WEXITSTATUS(status)); } + else { fprintf(stderr, "\n round %d: waitpid failed or the child stopped (status 0x%x)\n", r, (unsigned)status); } + } + if (first_corrupt_survivor(p) >= 0) { // and the parent stays intact + all_ok = false; + fprintf(stderr, "\n round %d: survivor %ld changed in the parent\n", r, first_corrupt_survivor(p)); + } } for (int i = 0; i < LIVE; i++) { if (p[i] != NULL) mi_free(p[i]); } free(p); diff --git a/test/test-prof-adversarial.c b/test/test-prof-adversarial.c deleted file mode 100644 index f5d4518ad..000000000 --- a/test/test-prof-adversarial.c +++ /dev/null @@ -1,387 +0,0 @@ -// Adversarial tests for the sampling heap profiler. -// Each case targets a specific race or edge condition; failures abort. - -#if defined(_WIN32) -#include -int main(void) { printf("test-prof-adversarial: skipped on Windows (uses pthreads/fork)\n"); return 0; } -#else - -#include -#include -#include -#include -#include -#include -#include -#include -#include "mimalloc.h" - -#if defined(__GNUC__) -#define NOINLINE __attribute__((noinline)) -#else -#define NOINLINE -#endif - -static int g_fail = 0; -#define CHECK(cond, msg) do { if (!(cond)) { fprintf(stderr, "FAIL: %s (%s:%d)\n", msg, __FILE__, __LINE__); g_fail = 1; } } while(0) -#define OK(name) fprintf(stderr, " ok: %s\n", name) - -// --------------------------------------------------------------------------- -// Minimal profile.proto validator: checks header structure + counts samples. -// Not a full parser; enough to catch malformed output and verify inuse totals. -// --------------------------------------------------------------------------- - -static size_t pb_varint(const uint8_t** p, const uint8_t* end) { - size_t v = 0, s = 0; - while (*p < end) { uint8_t b = *(*p)++; v |= (size_t)(b & 0x7f) << s; if (!(b & 0x80)) break; s += 7; } - return v; -} - -typedef struct { size_t nsamples, nlocs, nmaps, nstrings; int64_t inuse_bytes, alloc_bytes; } pb_stats_t; - -static int pb_validate(const char* path, pb_stats_t* st) { - memset(st, 0, sizeof(*st)); - int fd = open(path, O_RDONLY); if (fd < 0) return -1; - off_t sz = lseek(fd, 0, SEEK_END); lseek(fd, 0, SEEK_SET); - uint8_t* buf = malloc((size_t)sz); read(fd, buf, (size_t)sz); close(fd); - const uint8_t* p = buf; const uint8_t* end = buf + sz; - while (p < end) { - size_t tag = pb_varint(&p, end); - uint32_t field = (uint32_t)(tag >> 3), wt = (uint32_t)(tag & 7); - if (wt == 0) { pb_varint(&p, end); } - else if (wt == 2) { - size_t len = pb_varint(&p, end); - const uint8_t* sub_end = p + len; - if (sub_end > end) { free(buf); return -1; } // malformed - if (field == 2) { // Sample: extract value[1]=alloc_space, value[3]=inuse_space - st->nsamples++; - const uint8_t* sp = p; - while (sp < sub_end) { - size_t st2 = pb_varint(&sp, sub_end); uint32_t sf = (uint32_t)(st2>>3), swt=(uint32_t)(st2&7); - if (swt == 2) { - size_t slen = pb_varint(&sp, sub_end); - if (sf == 2) { // value[] packed - const uint8_t* vp = sp; const uint8_t* ve = sp + slen; - int64_t v[4] = {0}; int vi = 0; - while (vp < ve && vi < 4) v[vi++] = (int64_t)pb_varint(&vp, ve); - st->alloc_bytes += v[1]; st->inuse_bytes += v[3]; - } - sp += slen; - } else if (swt == 0) { pb_varint(&sp, sub_end); } - else { free(buf); return -1; } - } - } - else if (field == 3) st->nmaps++; - else if (field == 4) st->nlocs++; - else if (field == 6) st->nstrings++; - p = sub_end; - } - else { free(buf); return -1; } // unknown wire type - } - free(buf); - return 0; -} - -// --------------------------------------------------------------------------- -// Case: cross-thread free — alloc on T1, free on T2; inuse must drop. -// Exercises mi_free_generic_mt -> mi_free_generic_mt_prof path. -// --------------------------------------------------------------------------- - -#define XFREE_N 4096 -static void* g_xfree_blocks[XFREE_N]; -static atomic_int g_xfree_ready; - -static NOINLINE void* xfree_alloc_thread(void* _) { - for (int i = 0; i < XFREE_N; i++) g_xfree_blocks[i] = mi_malloc(512); - atomic_store(&g_xfree_ready, 1); - return NULL; -} -static NOINLINE void* xfree_free_thread(void* _) { - while (!atomic_load(&g_xfree_ready)) sched_yield(); - for (int i = 0; i < XFREE_N; i++) mi_free(g_xfree_blocks[i]); - return NULL; -} - -static void case_cross_thread_free(void) { - mi_prof_reset(); - mi_prof_enable(256); // sample ~every other 512B alloc - atomic_store(&g_xfree_ready, 0); - pthread_t t1, t2; - pthread_create(&t1, NULL, xfree_alloc_thread, NULL); - pthread_create(&t2, NULL, xfree_free_thread, NULL); - pthread_join(t1, NULL); pthread_join(t2, NULL); - mi_prof_dump_to_file("/tmp/prof-xfree.pb"); - pb_stats_t st; CHECK(pb_validate("/tmp/prof-xfree.pb", &st) == 0, "xfree: profile parses"); - CHECK(st.nsamples > 100, "xfree: samples captured"); - // all allocs were freed cross-thread; inuse should be ~0 (small noise from other allocs ok) - CHECK(st.inuse_bytes < st.alloc_bytes / 4, "xfree: cross-thread frees tracked (inuse << alloc)"); - mi_prof_enable(0); - OK("cross-thread free"); -} - -// --------------------------------------------------------------------------- -// Case: page reuse — fill a page (sampled), free everything, page returns to -// arena, allocate again from a fresh page; new page must NOT have stale -// MI_PAGE_HAS_PROF_SAMPLES flag (frees on it would route to slow path forever). -// We can't observe the flag directly, but we can check: with prof DISABLED, -// after page reuse, free should not call _mi_prof_free (would crash on uninit -// lock), and inuse accounting from a re-enabled prof should be clean. -// --------------------------------------------------------------------------- - -static void case_page_flag_reuse(void) { - mi_prof_enable(64); - void* a[2048]; - for (int i = 0; i < 2048; i++) a[i] = mi_malloc(128); - for (int i = 0; i < 2048; i++) mi_free(a[i]); - mi_collect(true); // force page return to arena - mi_prof_enable(0); - // re-allocate same size; pages may be reused. with prof off, frees must be fast-path. - for (int i = 0; i < 2048; i++) a[i] = mi_malloc(128); - for (int i = 0; i < 2048; i++) mi_free(a[i]); // would corrupt if stale flag + uninit prof state - OK("page-flag reuse after collect"); -} - -// --------------------------------------------------------------------------- -// Case: realloc — sampled block realloc'd to larger; old must be marked freed, -// new may be sampled independently; no double-count. -// --------------------------------------------------------------------------- - -static void case_realloc(void) { - mi_prof_reset(); - mi_prof_enable(1); // sample every alloc - void* keep[500]; - for (int i = 0; i < 500; i++) { - void* p = mi_malloc(200); - p = mi_realloc(p, 20000); // free 200B, alloc 20000B (too large to grow in place, even in a guarded block) - keep[i] = p; - } - mi_prof_dump_to_file("/tmp/prof-realloc.pb"); - pb_stats_t st; CHECK(pb_validate("/tmp/prof-realloc.pb", &st) == 0, "realloc: parses"); - // inuse should reflect only the 20000B blocks (old 200B freed); alloc_bytes counts both - double ratio = (double)st.inuse_bytes / (double)st.alloc_bytes; - CHECK(st.inuse_bytes < st.alloc_bytes && ratio > 0.5, "realloc: old block freed, not double-counted"); - for (int i = 0; i < 500; i++) mi_free(keep[i]); - mi_prof_enable(0); - OK("realloc"); -} - -// --------------------------------------------------------------------------- -// Case: aligned alloc — interior pointer; free must resolve to block start -// and remove the sample. -// --------------------------------------------------------------------------- - -static void case_aligned(void) { - mi_prof_reset(); - mi_prof_enable(1); - void* a[200]; - for (int i = 0; i < 200; i++) a[i] = mi_malloc_aligned(300, 256); - for (int i = 0; i < 200; i++) mi_free(a[i]); - mi_prof_dump_to_file("/tmp/prof-aligned.pb"); - pb_stats_t st; CHECK(pb_validate("/tmp/prof-aligned.pb", &st) == 0, "aligned: parses"); - CHECK(st.inuse_bytes < st.alloc_bytes / 4, "aligned: interior-ptr frees tracked"); - mi_prof_enable(0); - OK("aligned alloc"); -} - -// --------------------------------------------------------------------------- -// Case: MT stress — N threads allocating/freeing random sizes concurrently -// while another thread dumps the profile mid-stream. Mutex must serialize. -// --------------------------------------------------------------------------- - -#define MT_THREADS 8 -#define MT_ITERS 50000 -static atomic_int g_mt_stop; - -static NOINLINE void* mt_worker(void* arg) { - uintptr_t seed = (uintptr_t)arg; - void* slot[32] = {0}; - for (int i = 0; i < MT_ITERS; i++) { - seed = seed * 6364136223846793005ull + 1; - int j = (int)(seed % 32); - if (slot[j]) { mi_free(slot[j]); slot[j] = NULL; } - else slot[j] = mi_malloc(16 + (seed % 4096)); - } - for (int j = 0; j < 32; j++) if (slot[j]) mi_free(slot[j]); - return NULL; -} -static void* mt_dumper(void* _) { - while (!atomic_load(&g_mt_stop)) { - mi_prof_dump_to_file("/tmp/prof-mt.pb"); - usleep(1000); - } - return NULL; -} - -static void case_mt_stress(void) { - mi_prof_enable(4096); - atomic_store(&g_mt_stop, 0); - pthread_t th[MT_THREADS], dumper; - pthread_create(&dumper, NULL, mt_dumper, NULL); - for (int i = 0; i < MT_THREADS; i++) pthread_create(&th[i], NULL, mt_worker, (void*)(uintptr_t)(i+1)); - for (int i = 0; i < MT_THREADS; i++) pthread_join(th[i], NULL); - atomic_store(&g_mt_stop, 1); - pthread_join(dumper, NULL); - mi_prof_dump_to_file("/tmp/prof-mt.pb"); - pb_stats_t st; CHECK(pb_validate("/tmp/prof-mt.pb", &st) == 0, "mt: final profile parses"); - CHECK(st.nsamples > 100, "mt: samples captured"); - mi_prof_enable(0); - OK("MT stress + concurrent dump"); -} - -// --------------------------------------------------------------------------- -// Case: rate=1 hammer — sample every single alloc; table grows; no crash/leak -// in the sample-table grow path. -// --------------------------------------------------------------------------- - -static void case_rate1_hammer(void) { - mi_prof_enable(1); - void** a = mi_malloc(20000 * sizeof(void*)); - for (int i = 0; i < 20000; i++) a[i] = mi_malloc(32); - mi_prof_dump_to_file("/tmp/prof-rate1.pb"); - pb_stats_t st; CHECK(pb_validate("/tmp/prof-rate1.pb", &st) == 0, "rate1: parses"); - CHECK(st.nsamples >= 10000, "rate1: many samples captured"); - for (int i = 0; i < 20000; i++) mi_free(a[i]); - mi_free(a); - mi_prof_enable(0); - OK("rate=1 hammer (table growth)"); -} - -// --------------------------------------------------------------------------- -// Case: enable/disable cycling — toggle profiling repeatedly while allocating. -// --------------------------------------------------------------------------- - -static void case_enable_cycle(void) { - for (int c = 0; c < 20; c++) { - mi_prof_enable(c % 2 == 0 ? 256 : 0); - for (int i = 0; i < 1000; i++) { void* p = mi_malloc(100); mi_free(p); } - } - mi_prof_enable(256); - mi_prof_dump_to_file("/tmp/prof-cycle.pb"); - pb_stats_t st; CHECK(pb_validate("/tmp/prof-cycle.pb", &st) == 0, "cycle: parses"); - mi_prof_enable(0); - OK("enable/disable cycling"); -} - -// --------------------------------------------------------------------------- -// Case: empty profile — dump before any samples; file must be valid pprof. -// --------------------------------------------------------------------------- - -static void case_empty(void) { - // fresh state: enable with huge rate so nothing is sampled - mi_prof_enable(1ull << 40); - void* p = mi_malloc(16); mi_free(p); - CHECK(mi_prof_dump_to_file("/tmp/prof-empty.pb") == 0, "empty: dump succeeds"); - pb_stats_t st; CHECK(pb_validate("/tmp/prof-empty.pb", &st) == 0, "empty: parses"); - CHECK(st.nstrings >= 8, "empty: string table present"); - mi_prof_enable(0); - OK("empty profile"); -} - -// --------------------------------------------------------------------------- -// Case: huge allocation — singleton page; flag + free path -// --------------------------------------------------------------------------- - -static void case_huge(void) { - mi_prof_reset(); - mi_prof_enable(1); - void* a[10]; - for (int i = 0; i < 10; i++) a[i] = mi_malloc(2 * 1024 * 1024); - for (int i = 0; i < 10; i++) mi_free(a[i]); - mi_prof_dump_to_file("/tmp/prof-huge.pb"); - pb_stats_t st; CHECK(pb_validate("/tmp/prof-huge.pb", &st) == 0, "huge: parses"); - CHECK(st.inuse_bytes < st.alloc_bytes / 4, "huge: singleton-page frees tracked"); - mi_prof_enable(0); - OK("huge alloc"); -} - -// --------------------------------------------------------------------------- -// Case: invalid fd — dump to bad fd must return -1, not crash. -// --------------------------------------------------------------------------- - -static void case_bad_fd(void) { - mi_prof_enable(256); - void* p = mi_malloc(1000); (void)p; - CHECK(mi_prof_dump(-1) == -1, "bad-fd: returns error"); - CHECK(mi_prof_dump(12345) == -1, "closed-fd: returns error"); - mi_prof_enable(0); - OK("bad fd"); -} - -// --------------------------------------------------------------------------- -// Case: process-wide enable — main calls start(), a worker thread that was -// already running (and had already allocated) must start sampling. -// --------------------------------------------------------------------------- - -static atomic_int g_pw_phase; // 0=warmup, 1=profile, 2=stop -static NOINLINE void* pw_worker(void* _) { - // warmup: allocate before profiling is on so this thread has a populated theap - for (int i = 0; i < 1000; i++) { void* p = mi_malloc(256); mi_free(p); } - atomic_store(&g_pw_phase, 1); - while (atomic_load(&g_pw_phase) == 1) sched_yield(); - // now main has called mi_prof_enable; allocate — these must be sampled - for (int i = 0; i < 5000; i++) { void* p = mi_malloc(256); mi_free(p); } - return NULL; -} - -static void case_process_wide(void) { - mi_prof_reset(); - atomic_store(&g_pw_phase, 0); - pthread_t t; pthread_create(&t, NULL, pw_worker, NULL); - while (atomic_load(&g_pw_phase) == 0) sched_yield(); // wait for worker warmup - mi_prof_enable(128); // ON: must reach the worker's already-existing theap - atomic_store(&g_pw_phase, 2); - pthread_join(t, NULL); - mi_prof_enable(0); - mi_prof_dump_to_file("/tmp/prof-pw.pb"); - pb_stats_t st; CHECK(pb_validate("/tmp/prof-pw.pb", &st) == 0, "process-wide: parses"); - // worker allocated 5000*256 = 1.28MB at rate=128 -> ~10000 samples expected; - // even if a few fast-path allocs slip through before lazy-enable, should be >>100 - CHECK(st.nsamples > 1000, "process-wide: worker thread sampled after main enabled"); - OK("process-wide enable"); -} - -// --------------------------------------------------------------------------- -// Case: dump_buf — two-call size query, then fill; bytes match dump_to_file. -// --------------------------------------------------------------------------- - -static void case_dump_buf(void) { - mi_prof_reset(); - mi_prof_enable(64); - for (int i = 0; i < 1000; i++) { void* p = mi_malloc(500); if (i&1) mi_free(p); } - mi_prof_enable(0); // stop before querying size so output is stable across calls - size_t need = mi_prof_dump_buf(NULL, 0); - CHECK(need > 100, "dump_buf: size query"); - uint8_t* b = mi_malloc(need); - size_t got = mi_prof_dump_buf(b, need); - CHECK(got == need, "dump_buf: stable size between calls"); - // also dump to file and compare bytes - mi_prof_dump_to_file("/tmp/prof-buf.pb"); - int fd = open("/tmp/prof-buf.pb", O_RDONLY); off_t fsz = lseek(fd,0,SEEK_END); lseek(fd,0,SEEK_SET); - uint8_t* fbuf = mi_malloc((size_t)fsz); read(fd, fbuf, (size_t)fsz); close(fd); - CHECK((size_t)fsz == need && memcmp(b, fbuf, need) == 0, "dump_buf: identical to dump_to_file"); - mi_free(b); mi_free(fbuf); - OK("dump_buf"); -} - -// --------------------------------------------------------------------------- - -int main(void) { - fprintf(stderr, "test-prof-adversarial:\n"); - case_empty(); - case_bad_fd(); - case_page_flag_reuse(); - case_realloc(); - case_aligned(); - case_huge(); - case_cross_thread_free(); - case_rate1_hammer(); - case_enable_cycle(); - case_dump_buf(); - case_process_wide(); - case_mt_stress(); - if (g_fail) { fprintf(stderr, "FAILED\n"); return 1; } - fprintf(stderr, "all cases passed\n"); - return 0; -} - -#endif // !_WIN32 diff --git a/test/test-prof.c b/test/test-prof.c deleted file mode 100644 index ad57c09eb..000000000 --- a/test/test-prof.c +++ /dev/null @@ -1,62 +0,0 @@ -// Exercise the sampling heap profiler. -// ./mimalloc-test-prof /tmp/prof.pb -// then: go tool pprof -top /tmp/prof.pb - -#include -#include -#include -#include "mimalloc.h" - -#if defined(__GNUC__) -#define NOINLINE __attribute__((noinline)) -#elif defined(_MSC_VER) -#define NOINLINE __declspec(noinline) -#else -#define NOINLINE -#endif - -static NOINLINE void* leaky_alloc(size_t n) { return mi_malloc(n); } -static NOINLINE void* tidy_alloc(size_t n) { return mi_malloc(n); } - -static NOINLINE void workload(void) { - // ~50 MB through leaky_alloc (kept), ~50 MB through tidy_alloc (freed) - for (int i = 0; i < 50000; i++) { - void* p = leaky_alloc(1000); // leaks - memset(p, 1, 8); - } - for (int i = 0; i < 50000; i++) { - void* p = tidy_alloc(1000); - memset(p, 2, 8); - mi_free(p); // freed - } -} - -// More distinct call sites than the dump's initial location table holds (4096): every expansion of SITE is its own -// return address, and with a sample rate of 1 every allocation is sampled, so the dump sees >5000 unique frames. -#define SITE sites[n++] = leaky_alloc(16); -#define SITE10 SITE SITE SITE SITE SITE SITE SITE SITE SITE SITE -#define SITE100 SITE10 SITE10 SITE10 SITE10 SITE10 SITE10 SITE10 SITE10 SITE10 SITE10 -#define SITE1000 SITE100 SITE100 SITE100 SITE100 SITE100 SITE100 SITE100 SITE100 SITE100 SITE100 -static void* sites[6000]; -static NOINLINE size_t many_call_sites(void) { - size_t n = 0; - SITE1000 SITE1000 SITE1000 SITE1000 SITE1000 - return n; -} - -int main(int argc, char** argv) { - const char* out = (argc > 1 ? argv[1] : "heap-prof.pb"); - mi_prof_enable(64*1024); // 64 KiB sample rate for a small test - workload(); - if (mi_prof_dump_to_file(out) != 0) { fprintf(stderr, "dump failed\n"); return 1; } - printf("wrote %s\n", out); - - mi_prof_reset(); - mi_prof_enable(1); - size_t n = many_call_sites(); - size_t size = mi_prof_dump_buf(NULL, 0); // used to spin forever once the location table filled up - if (size == 0) { fprintf(stderr, "dump of %zu call sites produced nothing\n", n); return 1; } - for (size_t i = 0; i < n; i++) mi_free(sites[i]); - printf("dumped %zu call sites in %zu bytes\n", n, size); - return 0; -} diff --git a/test/test-profile.c b/test/test-profile.c new file mode 100644 index 000000000..41c636622 --- /dev/null +++ b/test/test-profile.c @@ -0,0 +1,189 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2026-2026, Microsoft Research, Daniel Schwartz-Narbonne, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// Tests for the mimalloc heap profiler (src/profile.c). + +#include +#include +#include +#include +#include + +#include "mimalloc.h" +#include "mimalloc-profile.h" +#include "mimalloc/internal.h" +#include "testhelper.h" + +// --------------------------------------------------------------------------- +// Shared callback state (not thread safe!) +// --------------------------------------------------------------------------- + +typedef struct { + mi_profiler_t profiler; + uint64_t alloc_count; + uint64_t free_count; + size_t last_size; + uint64_t last_upscaled; + void* last_ptr; +} my_profiler_t; + +static inline my_profiler_t* downcast( mi_profiler_t* prof ) { + return (my_profiler_t*)prof; +} + +// We store ptr in user_data so on_free can verify the round-trip. + +#define TEST_THRESHOLD (16 * 1024) + +static size_t mi_cdecl on_alloc(mi_profiler_t* profiler, mi_profiler_sample_data_t* data, void* ptr, size_t requested_size, size_t threshold, uint64_t bytes_since_last_sample, const mi_heap_t* heap) { + MI_UNUSED(threshold); MI_UNUSED(heap); MI_UNUSED(requested_size); + my_profiler_t* prof = downcast(profiler); + assert(bytes_since_last_sample >= requested_size); + prof->alloc_count++; + prof->last_ptr = ptr; + prof->last_size = requested_size; + prof->last_upscaled = bytes_since_last_sample; + // store ptr to verify round-trip + assert(data->user_data_size >= sizeof(void*)); + assert(data->user_data_size >= prof->profiler.sample_data_size); + data->user_data[0] = ptr; + return TEST_THRESHOLD; +} + +static void mi_cdecl on_free(mi_profiler_t* profiler, mi_profiler_sample_data_t* data, void* ptr, const mi_heap_t* heap) { + MI_UNUSED(heap); MI_UNUSED_RELEASE(data); MI_UNUSED_RELEASE(ptr); + my_profiler_t* prof = downcast(profiler); + prof->free_count++; + // verify the user_data round-trip + assert(data->user_data[0] == ptr); +} + +static my_profiler_t my_profiler = { + { // profiler_t + NULL, // reserved + sizeof(void*), // needed sample data size + 0, // initial sample rate (default) + &on_alloc, + &on_free, + NULL + }, + 0, 0, 0, 0, NULL +}; + + + + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// Force at least one sample by allocating well over the threshold. +static void allocate_past_threshold(void) { + size_t total = 0; + while (total < TEST_THRESHOLD * 10) { + void* p = mi_malloc(4096); + mi_free(p); + total += 4096; + // mi_collect() + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +bool test_profiler_samples(void) { + CHECK_BODY("profiler: on_alloc called after threshold") { + uint64_t before = my_profiler.alloc_count; + allocate_past_threshold(); + result = (my_profiler.alloc_count > before); + } + return true; +} + +#define MAXLOOP 100000 + +bool test_profiler_record_fields(void) { + CHECK_BODY("profiler: record ptr and size are non-zero") { + uint64_t before = my_profiler.alloc_count; + int count; + for (count = 0; my_profiler.alloc_count == before && count < MAXLOOP; count++) { + void* p = mi_malloc(1024); + mi_free(p); + } + assert(count!=MAXLOOP); + result = (my_profiler.last_ptr != NULL && my_profiler.last_size > 0 && my_profiler.last_upscaled > 0 && count!=MAXLOOP); + } + return true; +} + +bool test_profiler_on_free_called(void) { + CHECK_BODY("profiler: on_free called for sampled allocation") { + uint64_t alloc_before = my_profiler.alloc_count; + uint64_t free_before = my_profiler.free_count; + + // Keep the pointer live until we confirm a sample was taken, then free it. + void* sampled = NULL; + int count; + for (count = 0; my_profiler.alloc_count == alloc_before && count < MAXLOOP; count++) { + if (sampled) { mi_free(sampled); } + sampled = mi_malloc(1024); + } + // At this point my_profiler.last_ptr is the sampled pointer. + // Free it and check on_free fires. + void* expected = my_profiler.last_ptr; + mi_free(expected); + sampled = NULL; + assert(count!=MAXLOOP); + result = (my_profiler.free_count > free_before && count!=MAXLOOP); + } + return true; +} + +bool test_profiler_upscaled_at_least_size(void) { + CHECK_BODY("profiler: upscaled_size >= size") { + uint64_t before = my_profiler.alloc_count; + int count; + for (count = 0; my_profiler.alloc_count == before && count < MAXLOOP; count++) { + void* p = mi_malloc(256); + mi_free(p); + } + assert(count!=MAXLOOP); + result = (my_profiler.last_upscaled >= my_profiler.last_size && count!=MAXLOOP); + } + return true; +} + +bool test_profiler_free_count_le_alloc_count(void) { + CHECK_BODY("profiler: on_free never called more times than on_alloc") { + // Free can only fire for sampled allocations, so free_count <= alloc_count + // must hold at all times. + allocate_past_threshold(); + result = (my_profiler.free_count <= my_profiler.alloc_count); + } + return true; +} + + +// --------------------------------------------------------------------------- +// main +// --------------------------------------------------------------------------- + +int main(void) { + mi_profile(&my_profiler.profiler); + mi_profiler_start(&my_profiler.profiler); + + test_profiler_upscaled_at_least_size(); + test_profiler_samples(); + test_profiler_record_fields(); + test_profiler_on_free_called(); + test_profiler_free_count_le_alloc_count(); + + mi_profiler_stop(&my_profiler.profiler); + + return print_test_summary(); +} diff --git a/test/test-purge-behind-pass.c b/test/test-purge-behind-pass.c index 4c15f6a23..b4696fa01 100644 --- a/test/test-purge-behind-pass.c +++ b/test/test-purge-behind-pass.c @@ -155,6 +155,10 @@ int main(void) { fprintf(stderr, "test-purge-behind-pass: skipped (no scavenger, or no purge delay)\n"); return 0; } + // The rounds count bytes. With transparent huge pages left whole (`MI_ALLOW_THP=FULL`) a pass purges aligned 2 MiB + // units only and leaves the edges of each freed run for later, so purge by the slice here (the size is rounded up to an + // OS page, which is never more than a slice). + mi_option_set(mi_option_minimal_purge_size, 4 /* KiB */); // the scavenger starts when a second thread initializes or a thread first parks void* warm = mi_malloc(64); mi_free(warm); if (mi_on_thread_idle_start()) { mi_on_thread_idle_end(); } diff --git a/test/test-purge-holes.c b/test/test-purge-holes.c index b372b186d..a5d7cd031 100644 --- a/test/test-purge-holes.c +++ b/test/test-purge-holes.c @@ -17,6 +17,7 @@ terms of the MIT license. A copy of the license can be found in the file #include #include #include +#include #include "mimalloc.h" #include "mimalloc-stats.h" @@ -1013,8 +1014,8 @@ static bool test_holes_report(int mode) { } // the ground truth assumes every OTHER block in the page is free, so the page must be // exclusively ours (`used` also counts uncollected thread frees; this test is single-threaded) - if (page->used != nmine) { - fprintf(stderr, "\n page %p is not exclusively ours: used=%zu but we hold %zu\n", (void*)page, (size_t)page->used, nmine); + if (mi_page_used(page) != nmine) { + fprintf(stderr, "\n page %p is not exclusively ours: used=%zu but we hold %zu\n", (void*)page, mi_page_used(page), nmine); free(live); ok = false; goto done; } @@ -1327,8 +1328,8 @@ static bool test_sweep_skip(void) { live[idx / 64] |= ((uint64_t)1 << (idx % 64)); nmine++; } - if (page->used != nmine) { // the oracle below assumes every other block in the page is free - fprintf(stderr, "\n page %p is not exclusively ours: used=%zu but we hold %zu\n", (void*)page, (size_t)page->used, nmine); + if (mi_page_used(page) != nmine) { // the oracle below assumes every other block in the page is free + fprintf(stderr, "\n page %p is not exclusively ours: used=%zu but we hold %zu\n", (void*)page, mi_page_used(page), nmine); ok = false; goto done; } if (nmine < 2) continue; // freeing the victim must not free the page @@ -1504,6 +1505,82 @@ static bool test_sweep_no_unpurge_on_collect(void) { return ok; } +// --------------------------------------------------------------------------- +// a double free of a purged block is reported as a double free +// +// A purged block is free, on no free list, and its memory reads back as zeros. The padding canary +// (which is how a double free is recognized in a debug or secure build) is part of that memory, so +// a zero canary would read as a buffer overflow, and the check itself would fault the discarded OS +// page back in. `mi_free` checks the purged bitmap first. +// --------------------------------------------------------------------------- + +#if (MI_PADDING || MI_SECURE>=3) +static volatile int dfree_eagain; +static volatile int dfree_other; +static void dfree_error(int err, void* arg) { + (void)arg; + if (err == EAGAIN) { dfree_eagain++; } else { dfree_other++; } +} +#endif + +static bool test_double_free_of_purged_block(void) { + #if !(MI_PADDING || MI_SECURE>=3) + return true; // double frees are not checked in this build + #else + if (!purging_enabled) return true; + const size_t os = _mi_os_page_size(); + const size_t n = 256; + void** ptrs = (void**)calloc(n, sizeof(void*)); + void** freed = (void**)calloc(n, sizeof(void*)); + void* victim = NULL; + bool ok = true; + if (ptrs == NULL || freed == NULL) { free(ptrs); free(freed); return false; } + for (size_t i = 0; i < n; i++) { + ptrs[i] = mi_malloc(os); + if (ptrs[i] == NULL) { ok = false; goto done; } + } + for (size_t i = 0; i < n; i++) { + if ((i % 64) != 0) { freed[i] = ptrs[i]; mi_free(ptrs[i]); ptrs[i] = NULL; } // 63 of every 64 blocks: whole OS pages + } + mi_on_thread_idle(); + + // find a freed block that is purged now, and whose last bytes (where the padding canary is) are discarded + for (size_t i = 0; i < n && victim == NULL; i++) { + if (freed[i] == NULL) continue; + mi_page_t* const page = _mi_safe_ptr_page(freed[i]); // (NULL if the page was freed as a whole in the meantime) + if (page == NULL || mi_page_is_abandoned(page) || !mi_page_block_is_purged(page, freed[i])) continue; // (the purged bitmap is only checked on a free by the owning thread) + const uintptr_t last = (uintptr_t)freed[i] + mi_page_block_size(page) - 1; + if (mi_page_os_page_purged(page, (size_t)(last - mi_page_purge_base(page)) / os)) { victim = freed[i]; } + } + if (victim == NULL) { fprintf(stderr, "\n no block was purged, so this proves nothing\n"); ok = false; goto done; } + + { + mi_page_t* const page = _mi_ptr_page(victim); + const size_t used = mi_page_used(page); + const hole_stats_t s1 = hole_stats(); + dfree_eagain = 0; dfree_other = 0; + mi_register_error(&dfree_error, NULL); + mi_free(victim); // double free + mi_register_error(NULL, NULL); + const hole_stats_t s2 = hole_stats(); + if (dfree_eagain != 1 || dfree_other != 0) { + fprintf(stderr, "\n the double free of a purged block raised %d double-free errors and %d other errors (expected 1 and 0)\n", dfree_eagain, dfree_other); + ok = false; goto done; + } + if (mi_page_used(page) != used || !mi_page_block_is_purged(page, victim) || s2.reuses != s1.reuses || s2.bytes_now != s1.bytes_now) { + fprintf(stderr, "\n the double free of a purged block changed the page (used %zu -> %zu, reuses %lld -> %lld)\n", + used, mi_page_used(page), (long long)s1.reuses, (long long)s2.reuses); + ok = false; goto done; + } + } + +done: + for (size_t i = 0; i < n; i++) { if (ptrs[i] != NULL) mi_free(ptrs[i]); } + free(ptrs); free(freed); + return ok; + #endif +} + // --------------------------------------------------------------------------- // the skip cannot lose memory forever // @@ -1550,7 +1627,7 @@ static bool test_sweep_full_every(void) { if (ptrs[i] != NULL && _mi_ptr_page(ptrs[i]) == page) { mi_free(ptrs[i]); ptrs[i] = NULL; break; } } s0 = hole_stats(); - page->swept_state = (((uint64_t)page->capacity) << 32) | (uint64_t)page->used; // must match mi_page_sweep_state() in page.c + page->swept_state = (((uint64_t)page->capacity) << 32) | (uint64_t)mi_page_used(page); // must match mi_page_sweep_state() in page.c // with no periodic full sweep the page is now wedged: no amount of parking finds the hole for (int r = 0; r < BOUND_EVERY * 2; r++) { mi_on_thread_idle(); } @@ -1678,6 +1755,7 @@ int main(void) { CHECK("unformed-tail-freed", test_unformed_tail_freed()); CHECK("sweep-skips-unchanged-pages", test_sweep_skip()); CHECK("sweep-does-not-unpurge-on-collect", test_sweep_no_unpurge_on_collect()); + CHECK("double-free-of-a-purged-block", test_double_free_of_purged_block()); CHECK("sweep-full-every-bounds-a-missed-hole", test_sweep_full_every()); // everything above is freed by now, so every hole must have been handed back diff --git a/test/test-stress.c b/test/test-stress.c index 13f83f1e0..430252ccc 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -42,15 +42,19 @@ terms of the MIT license. // > mimalloc-test-stress [THREADS] [SCALE] [ITER] // // argument defaults -#if defined(MI_TSAN) // with thread-sanitizer reduce the threads to test within the CI limits (240s on 4 cores) +#if defined(MI_TSAN) && MI_TEST_LIGHT // with thread-sanitizer reduce the threads to test within the azure pipeline limits static int THREADS = NTHREADS/4; static int SCALE = 10; static int ITER = 100; +#elif defined(MI_TSAN) // with thread-sanitizer reduce the threads to test within the azure pipeline limits +static int THREADS = NTHREADS/4; +static int SCALE = 25; +static int ITER = 500; #elif defined(MI_UBSAN) // with undefined behavious sanitizer reduce parameters to stay within the azure pipeline limits static int THREADS = NTHREADS/4; static int SCALE = 25; static int ITER = 20; -#elif defined(MI_GUARDED) // with debug guard pages reduce parameters to stay within the azure pipeline limits +#elif (defined(MI_GUARDED) && MI_GUARDED>0) // with debug guard pages reduce parameters to stay within the azure pipeline limits static int THREADS = NTHREADS/4; static int SCALE = 25; static int ITER = 10; @@ -324,7 +328,7 @@ static void test_stress(mi_subproc_id_t subproc) { if ((n + 1) % 10 == 0) { printf("- iterations left: %3d\n", ITER - (n + 1)); #ifndef USE_STD_MALLOC - // mi_debug_show_arenas(); + mi_debug_show_arenas(); #endif //mi_collect(true); //mi_debug_show_arenas(); @@ -421,7 +425,7 @@ int main(int argc, char** argv) { #endif #if !defined(NDEBUG) && !defined(USE_STD_MALLOC) mi_option_set(mi_option_arena_reserve, (long)(mi_arena_min_size()/1024) /* in KiB ! */); - mi_option_set(mi_option_purge_delay,1); + // mi_option_set(mi_option_purge_delay,1); #endif #if defined(NDEBUG) && !defined(USE_STD_MALLOC) // mi_option_set(mi_option_purge_delay,-1); @@ -485,7 +489,7 @@ int main(int argc, char** argv) { // mi_free(json); //} #endif - mi_collect(true); + // mi_collect(true); mi_stats_print(NULL); #endif //bench_end_program(); diff --git a/test/test-wrong.c b/test/test-wrong.c index 666d99f59..4e0590839 100644 --- a/test/test-wrong.c +++ b/test/test-wrong.c @@ -65,6 +65,9 @@ int main(int argc, char** argv) { char* c = (char*)mi(malloc)(3); printf("invalid byte: over: %d, under: %d\n", c[4], c[-1]); mi(free)(c); + + // double free + mi(free)(c); // undefined access long* q = (long*)mi(malloc)(sizeof(long)); diff --git a/tools/README-heapview.md b/tools/README-heapview.md index a458a8a88..1e790993f 100644 --- a/tools/README-heapview.md +++ b/tools/README-heapview.md @@ -1,29 +1,9 @@ # Heap introspection -mimalloc ships two complementary tools for "why is this process using so much -memory": - -| | `mi_heap_snapshot` + `mi-heapview` | `mi_prof_*` (pprof) | -|---|---|---| -| What | Full census of heap structure (arenas, pages, blocks, fragmentation) | Sampled allocation call stacks | -| Answers | "What's in memory? How fragmented?" | "Which code allocated it?" | -| Runtime cost | Zero until called | Zero when off; ~20% when on | -| Output | binary → `mi-heapview` CLI | `profile.proto` → `go tool pprof` | - -## Sampling profiler (pprof-compatible) - -```sh -MIMALLOC_PROF_SAMPLE_RATE=524288 MIMALLOC_PROF_PATH=/tmp/heap.pb ./your-program -go tool pprof -inuse_space ./your-program /tmp/heap.pb -``` - -Or programmatically: `mi_prof_enable(512*1024)` then `mi_prof_dump_to_file(path)`. - -Sample types: `alloc_objects`, `alloc_space`, `inuse_objects`, `inuse_space`. -Freed allocations are tracked (inuse counts are accurate). When the sample rate -is 0, the malloc/free fast paths have **zero added instructions** — profiling -hooks live entirely in the slow path, gated by poisoning `pages_free_direct` -and a third page-flag bit. +`mi_heap_snapshot` + `mi-heapview` give a full census of the heap structure (arenas, pages, +blocks, fragmentation): "what is in memory, and how fragmented is it?". It costs nothing until +it is called. To find out *which code* allocated the memory, use the sampled profiling hooks in +`include/mimalloc-profile.h` (`mi_profiler_t` with `on_alloc`/`on_free`). ## Heap snapshot