charlie12345 · 3rdIteration · Jun 29, 2026 · Jun 29, 2026 · Jul 3, 2026 · Jul 3, 2026
diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml
@@ -0,0 +1,212 @@
+name: Build Packages
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - rocmFPX-cachyllama
+    paths: [
+      '.github/workflows/build-packages.yml',
+      '**/CMakeLists.txt',
+      '**/.cmake',
+      '**/*.h',
+      '**/*.hpp',
+      '**/*.c',
+      '**/*.cpp',
+      '**/*.cu',
+      '**/*.cuh'
+    ]
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths: [
+      '.github/workflows/build-packages.yml',
+      '**/CMakeLists.txt',
+      '**/.cmake',
+      '**/*.h',
+      '**/*.hpp',
+      '**/*.c',
+      '**/*.cpp',
+      '**/*.cu',
+      '**/*.cuh'
+    ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  VULKAN_VERSION: 1.4.313.2
+
+jobs:
+  windows-msvc-vulkan:
+    runs-on: windows-2025
+    name: Windows x64 MSVC Vulkan
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: ccache
+        uses: ggml-org/ccache-action@v1.2.21
+        with:
+          key: windows-x64-msvc-vulkan
+          variant: ccache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' }}
+
+      - name: Install Vulkan SDK
+        id: get_vulkan
+        run: |
+          curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
+          & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
+          Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
+          Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
+
+      - name: Install Ninja
+        id: install_ninja
+        run: |
+          choco install ninja
+
+      - name: Setup MSVC
+        id: setup_msvc
+        uses: ilammy/msvc-dev-cmd@v1
+
+      - name: Build (x64 MSVC Vulkan)
+        id: cmake_build
+        run: |
+          cmake -B build `
+            -G "Ninja Multi-Config" `
+            -DCMAKE_BUILD_TYPE=Release `
+            -DGGML_NATIVE=OFF `
+            -DLLAMA_BUILD_SERVER=ON `
+            -DLLAMA_BUILD_WEBUI=OFF `
+            -DLLAMA_USE_PREBUILT_WEBUI=OFF `
+            -DLLAMA_OPENSSL=OFF `
+            -DGGML_RPC=ON `
+            -DGGML_BACKEND_DL=ON `
+            -DGGML_CPU_ALL_VARIANTS=ON `
+            -DGGML_VULKAN=ON
+          cmake --build build --config Release
+
+      - name: Test
+        id: cmake_test
+        run: |
+          cd build
+          ctest -L main -C Release --verbose --timeout 900
+
+      - name: Create artifact archive
+        run: |
+          cd build
+          mkdir -p artifacts
+          copy bin\Release\*.exe artifacts\ 2>nul || true
+          copy bin\Release\*.dll artifacts\ 2>nul || true
+          7z a llama-windows-msvc-vulkan.7z artifacts\
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v6
+        with:
+          name: llama-windows-x64-msvc-vulkan
+          path: build/llama-windows-msvc-vulkan.7z
+          retention-days: 7
+
+  windows-hip:
+    runs-on: windows-2022
+    name: Windows x64 HIP Radeon
+
+    env:
+      # Make sure this is in sync with build.yml / build-cache.yml
+      HIPSDK_INSTALLER_VERSION: "26.Q1"
+      # RDNA2 / RDNA3 / RDNA3.5 (Strix Halo gfx1151) / RDNA4
+      HIP_GPU_TARGETS: "gfx1030;gfx1031;gfx1032;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1200;gfx1201"
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Grab rocWMMA package
+        id: grab_rocwmma
+        run: |
+          curl -o rocwmma.deb "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocwmma-dev/rocwmma-dev_2.2.0.70201-81~24.04_amd64.deb"
+          7z x rocwmma.deb
+          7z x data.tar
+
+      - name: Use ROCm Installation Cache
+        uses: actions/cache@v5
+        id: cache-rocm
+        with:
+          path: C:\Program Files\AMD\ROCm
+          key: rocm-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ runner.os }}
+
+      - name: Setup ROCm
+        if: steps.cache-rocm.outputs.cache-hit != 'true'
+        uses: ./.github/actions/windows-setup-rocm
+        with:
+          version: ${{ env.HIPSDK_INSTALLER_VERSION }}
+
+      - name: Verify ROCm
+        id: verify
+        run: |
+          $clangPath = Get-ChildItem 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | Select-Object -First 1
+          if (-not $clangPath) {
+            Write-Error "ROCm installation not found"
+            exit 1
+          }
+          & $clangPath.FullName --version
+
+      - name: ccache
+        uses: ggml-org/ccache-action@v1.2.21
+        with:
+          key: windows-x64-hip-radeon
+          variant: ccache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' }}
+
+      - name: Build (x64 HIP)
+        id: cmake_build
+        run: |
+          $env:HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path)
+          $env:CMAKE_PREFIX_PATH="${env:HIP_PATH}"
+          cmake -G "Unix Makefiles" -B build -S . `
+            -DCMAKE_C_COMPILER="${env:HIP_PATH}\bin\clang.exe" `
+            -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" `
+            -DCMAKE_CXX_FLAGS="-I$($PWD.Path.Replace('\', '/'))/opt/rocm-7.2.1/include/ -Wno-ignored-attributes -Wno-nested-anon-types" `
+            -DCMAKE_BUILD_TYPE=Release `
+            -DGGML_NATIVE=OFF `
+            -DLLAMA_BUILD_SERVER=ON `
+            -DLLAMA_BUILD_WEBUI=OFF `
+            -DLLAMA_USE_PREBUILT_WEBUI=OFF `
+            -DLLAMA_OPENSSL=OFF `
+            -DGGML_RPC=ON `
+            -DGGML_BACKEND_DL=ON `
+            -DGGML_CPU_ALL_VARIANTS=ON `
+            -DGGML_HIP=ON `
+            -DGGML_HIP_ROCWMMA_FATTN=ON `
+            -DROCM_DIR="${env:HIP_PATH}" `
+            -DGPU_TARGETS="${env:HIP_GPU_TARGETS}"
+          cmake --build build -j ${env:NUMBER_OF_PROCESSORS}
+
+      - name: Bundle ROCm runtime DLLs
+        run: |
+          $env:HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path)
+          md "build\bin\rocblas\library\"
+          md "build\bin\hipblaslt\library"
+          cp "${env:HIP_PATH}\bin\libhipblas.dll" "build\bin\"
+          cp "${env:HIP_PATH}\bin\libhipblaslt.dll" "build\bin\"
+          cp "${env:HIP_PATH}\bin\rocblas.dll" "build\bin\"
+          cp "${env:HIP_PATH}\bin\rocblas\library\*" "build\bin\rocblas\library\"
+          cp "${env:HIP_PATH}\bin\hipblaslt\library\*" "build\bin\hipblaslt\library\"
+
+      - name: Create artifact archive
+        run: |
+          cd build
+          7z a llama-windows-hip-radeon.7z .\bin\*
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v6
+        with:
+          name: llama-windows-x64-hip-radeon
+          path: build/llama-windows-hip-radeon.7z
+          retention-days: 7
diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt
@@ -90,6 +90,10 @@ add_library(${TARGET}
     ngram-map.h
     ngram-mod.cpp
     ngram-mod.h
+    kv-ssd-cache.cpp
+    kv-ssd-cache.h
+    kv-ssd-system-cache.cpp
+    kv-ssd-system-cache.h
     peg-parser.cpp
     peg-parser.h
     preset.cpp

diff --git a/common/arg.cpp b/common/arg.cpp
@@ -1357,6 +1357,100 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.cache_ram_mib = value;
         }
     ).set_env("LLAMA_ARG_CACHE_RAM").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
+    add_opt(common_arg(
+        {"-ssd", "--cache-ssd"}, "PATH",
+        "enable SSD-backed KV cache with path to storage directory",
+        [](common_params & params, const std::string & value) {
+            params.cache_ssd_path = value;
+        }
+    ).set_env("LLAMA_ARG_CACHE_SSD").set_examples({LLAMA_EXAMPLE_SERVER}));
+    add_opt(common_arg(
+        {"-ssd-cp", "--cache-ssd-checkpoints"}, "N",
+        string_format("max number of SSD-backed checkpoints per slot (default: %d)", params.cache_ssd_max_checkpoints),
+        [](common_params & params, int value) {
+            params.cache_ssd_max_checkpoints = value;
+        }
+    ).set_env("LLAMA_ARG_CACHE_SSD_CHECKPOINTS").set_examples({LLAMA_EXAMPLE_SERVER}));
+    add_opt(common_arg(
+        {"-ssd-hot", "--cache-ssd-hot-window"}, "N",
+        string_format("always-keep window size in tokens for SSD cache (default: %zu)", params.cache_ssd_hot_window_tokens),
+        [](common_params & params, int value) {
+            params.cache_ssd_hot_window_tokens = value;
+        }
+    ).set_env("LLAMA_ARG_CACHE_SSD_HOT_WINDOW").set_examples({LLAMA_EXAMPLE_SERVER}));
+    add_opt(common_arg(
+        {"-ssd-warm", "--cache-ssd-warm-window"}, "N",
+        string_format("keep-in-RAM window size in tokens for SSD cache (default: %zu)", params.cache_ssd_warm_window_tokens),
+        [](common_params & params, int value) {
+            params.cache_ssd_warm_window_tokens = value;
+        }
+    ).set_env("LLAMA_ARG_CACHE_SSD_WARM_WINDOW").set_examples({LLAMA_EXAMPLE_SERVER}));
+    add_opt(common_arg(
+        {"-ssd-mc", "--cache-ssd-max-cold"}, "N",
+        string_format("max cold tier checkpoints before oldest-first eviction (default: %d, 0=unlimited)", params.cache_ssd_max_cold),
+        [](common_params & params, int value) {
+            params.cache_ssd_max_cold = value;
+        }
+    ).set_env("LLAMA_ARG_CACHE_SSD_MAX_COLD").set_examples({LLAMA_EXAMPLE_SERVER}));
+    add_opt(common_arg(
+        {"-ssd-hot-ram", "--cache-ssd-hot-ram"}, "N",
+        string_format("hot tier RAM budget in MiB for SSD cache (default: auto-size, 0=auto)"),
+        [](common_params & params, int value) {
+            params.cache_ssd_hot_ram_mib = value;
+        }
+    ).set_env("LLAMA_ARG_CACHE_SSD_HOT_RAM").set_examples({LLAMA_EXAMPLE_SERVER}));
+    add_opt(common_arg(
+        {"-ssd-warm-ram", "--cache-ssd-warm-ram"}, "N",
+        string_format("warm tier RAM budget in MiB for SSD cache (default: auto-size, 0=auto)"),
+        [](common_params & params, int value) {
+            params.cache_ssd_warm_ram_mib = value;
+        }
+    ).set_env("LLAMA_ARG_CACHE_SSD_WARM_RAM").set_examples({LLAMA_EXAMPLE_SERVER}));
+    add_opt(common_arg(
+        {"--cache-ssd-max-conversations"}, "N",
+        string_format("max conversation directories (default: %d, 0=unlimited)", params.cache_ssd_max_conversations),
+        [](common_params & params, int value) {
+            params.cache_ssd_max_conversations = value;
+        }
+    ).set_env("LLAMA_ARG_CACHE_SSD_MAX_CONVERSATIONS").set_examples({LLAMA_EXAMPLE_SERVER}));
+    add_opt(common_arg(
+        {"--prompt-max"}, "N",
+        string_format("max system prompt cache entries (default: %d, 0=disabled)", params.prompt_cache_max),
+        [](common_params & params, int value) {
+            params.prompt_cache_max = value;
+        }
+    ).set_env("LLAMA_ARG_PROMPT_MAX").set_examples({LLAMA_EXAMPLE_SERVER}));
+    add_opt(common_arg(
+        {"--cache-ssd-system-prompts"}, "N",
+        string_format("max global system prompt entries cached for reuse across conversations (default: %d, 0=disabled)", params.cache_ssd_system_prompts),
+        [](common_params & params, int value) {
+            params.cache_ssd_system_prompts = value;
+        }
+    ).set_env("LLAMA_ARG_CACHE_SSD_SYSTEM_PROMPTS").set_examples({LLAMA_EXAMPLE_SERVER}));
+    add_opt(common_arg(
+        {"--cache-ssd-system-max-days"}, "N",
+        string_format("expire system prompt cache entries unused for N days (default: %d, 0=never)", params.cache_ssd_system_max_days),
+        [](common_params & params, int value) {
+            params.cache_ssd_system_max_days = value;
+        }
+    ).set_env("LLAMA_ARG_CACHE_SSD_SYSTEM_MAX_DAYS").set_examples({LLAMA_EXAMPLE_SERVER}));
+    add_opt(common_arg(
+        {"-ssd-ps", "--cache-ssd-page-size"}, "N",
+        string_format("tokens per page for SSD cache: 512, 1024, 2048 (default: %zu)", params.cache_ssd_page_size_tokens),
+        [](common_params & params, int value) {
+            if (value != 512 && value != 1024 && value != 2048) {
+                throw std::invalid_argument("invalid page size, must be 512, 1024, or 2048");
+            }
+            params.cache_ssd_page_size_tokens = value;
+        }
+    ).set_env("LLAMA_ARG_CACHE_SSD_PAGE_SIZE").set_examples({LLAMA_EXAMPLE_SERVER}));
+    add_opt(common_arg(
+        {"--cache-ssd-no-fsync"},
+        string_format("skip fsync on SSD checkpoint writes (default: %s)", params.cache_ssd_no_fsync ? "enabled" : "disabled"),
+        [](common_params & params) {
+            params.cache_ssd_no_fsync = true;
+        }
+    ).set_examples({LLAMA_EXAMPLE_SERVER}));
     add_opt(common_arg(
         {"-kvu", "--kv-unified"},
         {"-no-kvu", "--no-kv-unified"},
@@ -2189,6 +2283,17 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             }
         ).set_env("LLAMA_ARG_N_PARALLEL"));
     }
+    add_opt(common_arg(
+        {"--max-concurrent-per-user"}, "N",
+        string_format("per-user_id concurrency cap on in-flight slots (default: %d, 0 = unlimited). also applies to the _anonymous bucket.",
+                      params.max_concurrent_per_user),
+        [](common_params & params, int value) {
+            if (value < 0) {
+                throw std::invalid_argument("error: max-concurrent-per-user must be >= 0\n");
+            }
+            params.max_concurrent_per_user = value;
+        }
+    ).set_env("LLAMA_ARG_MAX_CONCURRENT_PER_USER").set_examples({LLAMA_EXAMPLE_SERVER}));
     add_opt(common_arg(
         {"-ns", "--sequences"}, "N",
         string_format("number of sequences to decode (default: %d)", params.n_sequences),

diff --git a/common/common.cpp b/common/common.cpp
@@ -2068,6 +2068,7 @@ common_prompt_checkpoint::common_prompt_checkpoint(const common_prompt_checkpoin
     pos_max(other.pos_max),
     data_tgt(other.data_tgt),
     data_dft(other.data_dft),
+    data_spec(other.data_spec),
     storage_tgt(llama_state_seq_storage_clone(other.storage_tgt)),
     storage_dft(llama_state_seq_storage_clone(other.storage_dft)) {
 }
@@ -2083,6 +2084,7 @@ common_prompt_checkpoint & common_prompt_checkpoint::operator=(const common_prom
 
     data_tgt = other.data_tgt;
     data_dft = other.data_dft;
+    data_spec = other.data_spec;
 
     llama_state_seq_storage_free(storage_tgt);
     llama_state_seq_storage_free(storage_dft);
@@ -2098,6 +2100,7 @@ common_prompt_checkpoint::common_prompt_checkpoint(common_prompt_checkpoint && o
     pos_max(other.pos_max),
     data_tgt(std::move(other.data_tgt)),
     data_dft(std::move(other.data_dft)),
+    data_spec(std::move(other.data_spec)),
     storage_tgt(other.storage_tgt),
     storage_dft(other.storage_dft) {
     other.storage_tgt = nullptr;
@@ -2118,6 +2121,7 @@ common_prompt_checkpoint & common_prompt_checkpoint::operator=(common_prompt_che
 
     data_tgt = std::move(other.data_tgt);
     data_dft = std::move(other.data_dft);
+    data_spec = std::move(other.data_spec);
 
     storage_tgt = other.storage_tgt;
     storage_dft = other.storage_dft;
@@ -2131,6 +2135,7 @@ common_prompt_checkpoint & common_prompt_checkpoint::operator=(common_prompt_che
 size_t common_prompt_checkpoint::size() const {
     return data_tgt.size() +
            data_dft.size() +
+           data_spec.size() +
            llama_state_seq_storage_size(storage_tgt) +
            llama_state_seq_storage_size(storage_dft);
 }
@@ -2147,6 +2152,7 @@ void common_prompt_checkpoint::clear() {
 
     data_tgt.clear();
     data_dft.clear();
+    data_spec.clear();
 
     llama_state_seq_storage_free(storage_tgt);
     llama_state_seq_storage_free(storage_dft);
@@ -2269,6 +2275,7 @@ void common_prompt_checkpoint::clear_tgt() {
 
 void common_prompt_checkpoint::clear_dft() {
     data_dft.clear();
+    data_spec.clear();
     llama_state_seq_storage_free(storage_dft);
     storage_dft = nullptr;
 }