diff --git a/.agents/adding-backends.md b/.agents/adding-backends.md index a7880a17b400..f0f44b815e12 100644 --- a/.agents/adding-backends.md +++ b/.agents/adding-backends.md @@ -92,7 +92,7 @@ If you have a `prepare.sh` doing the clone, delete it — the recipe belongs in - CUDA 13 builds: Add after other CUDA 13 builds (e.g., after `gpu-nvidia-cuda-13-chatterbox`) **Additional build types you may need:** -- ROCm/HIP: Use `build-type: 'hipblas'` with `base-image: "rocm/dev-ubuntu-24.04:7.2.1"` +- ROCm/HIP: Use `build-type: 'hipblas'` with `base-image: "rocm/dev-ubuntu-24.04:7.14.0-full"` - Intel/SYCL: Use `build-type: 'intel'` or `build-type: 'sycl_f16'`/`sycl_f32` with `base-image: "intel/oneapi-basekit:2025.3.2-0-devel-ubuntu24.04"` - L4T (ARM): Use `build-type: 'l4t'` with `platforms: 'linux/arm64'` and `runs-on: 'ubuntu-24.04-arm'` diff --git a/.agents/building-and-testing.md b/.agents/building-and-testing.md index 021d555ec993..a98eeeb7a906 100644 --- a/.agents/building-and-testing.md +++ b/.agents/building-and-testing.md @@ -11,7 +11,7 @@ Let's say the user wants to build a particular backend for a given platform. For - Use `.github/backend-matrix.yml` as a reference — it's the data-only YAML that lists every backend variant's `build-type`, `base-image`, `platforms`, etc. (`backend.yml` and `backend_pr.yml` consume it via `scripts/changed-backends.js`). - l4t and cublas also require the CUDA major and minor version. - For llama-cpp / ik-llama-cpp / turboquant the matrix also sets `builder-base-image` pointing at a prebuilt `quay.io/go-skynet/ci-cache:base-grpc-*` tag. Local `make backends/` defaults to `BUILDER_TARGET=builder-fromsource` and doesn't need it — the Dockerfile's from-source stage installs everything itself. -- You can pretty print a command like `DOCKER_MAKEFLAGS=-j$(nproc --ignore=1) BUILD_TYPE=hipblas BASE_IMAGE=rocm/dev-ubuntu-24.04:7.2.1 make docker-build-coqui` +- You can pretty print a command like `DOCKER_MAKEFLAGS=-j$(nproc --ignore=1) BUILD_TYPE=hipblas BASE_IMAGE=rocm/dev-ubuntu-24.04:7.14.0-full make docker-build-coqui` - Unless the user specifies that they want you to run the command, then just print it because not all agent frontends handle long running jobs well and the output may overflow your context - The user may say they want to build AMD or ROCM instead of hipblas, or Intel instead of SYCL or NVIDIA insted of l4t or cublas. Ask for confirmation if there is ambiguity. - Sometimes the user may need extra parameters to be added to `docker build` (e.g. `--platform` for cross-platform builds or `--progress` to view the full logs), in which case you can generate the `docker build` command directly. diff --git a/.agents/ci-caching.md b/.agents/ci-caching.md index 6742049e68ff..4cbf3ea73d21 100644 --- a/.agents/ci-caching.md +++ b/.agents/ci-caching.md @@ -51,7 +51,7 @@ The C++ backend Dockerfiles (`Dockerfile.{llama-cpp,ik-llama-cpp,turboquant}`) c | `base-grpc-cuda-13-amd64` | the above + CUDA 13.0 toolkit (Ubuntu 22.04 base) | | `base-grpc-cuda-13-arm64` | the above + CUDA 13.0 sbsa toolkit (Ubuntu 24.04 base) | | `base-grpc-l4t-cuda-12-arm64` | JetPack r36.4.0 base (CUDA preinstalled, `SKIP_DRIVERS=true`) + gRPC | -| `base-grpc-rocm-amd64` | rocm/dev-ubuntu-24.04:7.2.1 base + hipblas/hipblaslt/rocblas + gRPC | +| `base-grpc-rocm-amd64` | rocm/dev-ubuntu-24.04:7.14.0-full base + hipblas/hipblaslt/rocblas + gRPC | | `base-grpc-vulkan-amd64` / `base-grpc-vulkan-arm64` | Ubuntu 24.04 + Vulkan SDK 1.4.335 + gRPC | | `base-grpc-intel-amd64` | intel/oneapi-basekit:2025.3.2 base + gRPC | diff --git a/.docker/install-base-deps.sh b/.docker/install-base-deps.sh index 4331921ca6e3..430085e90536 100755 --- a/.docker/install-base-deps.sh +++ b/.docker/install-base-deps.sh @@ -226,16 +226,18 @@ fi # --- 6. ROCm / HIP build deps (BUILD_TYPE=hipblas) --- if [ "${BUILD_TYPE:-}" = "hipblas" ] && [ "${SKIP_DRIVERS:-false}" = "false" ]; then - apt-get update - apt-get install -y --no-install-recommends \ - hipblas-dev \ - hipblaslt-dev \ - rocblas-dev - apt-get clean - rm -rf /var/lib/apt/lists/* - # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, - # which results in local-ai and others not being able to locate the libraries. - # We run ldconfig ourselves to work around this packaging deficiency. + # ROCm 7.x ("TheRock" packaging): the legacy hipblas-dev / hipblaslt-dev / + # rocblas-dev metapackages were removed — BLAS is consolidated and split per + # GPU arch (amdrocm-blas-gfx*, amdrocm-blas-dev, amdrocm-blas-host). The + # rocm/dev-ubuntu-*:*-full base image already ships those BLAS dev libs and + # headers, so no apt install is needed here. + # TheRock ships the ROCm libs under /opt/rocm/core*/lib (reached via the + # /opt/rocm/lib alternatives symlink) but registers no ld.so.conf.d entry, so + # the dynamic linker can't resolve them. Without this the ldd-based GPU-lib + # packaging (scripts/build/package-gpu-libs.sh) silently skips librocm_kpack, + # rocBLAS, hipBLASLt, the bundled rocm_sysdeps (zlib/zstd/elf) and the LLVM + # runtime, and the built backend fails to load at runtime. + printf '/opt/rocm/lib\n/opt/rocm/lib/rocm_sysdeps/lib\n/opt/rocm/llvm/lib\n' > /etc/ld.so.conf.d/rocm.conf ldconfig # Log which GPU architectures have rocBLAS kernel support echo "rocBLAS library data architectures:" diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index e7bada4b4e91..d2633d139884 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -2223,7 +2223,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-rerankers' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "rerankers" dockerfile: "./backend/Dockerfile.python" @@ -2237,7 +2237,7 @@ include: tag-suffix: '-gpu-rocm-hipblas-llama-cpp' builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-rocm-amd64' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "llama-cpp" dockerfile: "./backend/Dockerfile.llama-cpp" @@ -2251,7 +2251,7 @@ include: tag-suffix: '-gpu-rocm-hipblas-bonsai' builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-rocm-amd64' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "bonsai" dockerfile: "./backend/Dockerfile.bonsai" @@ -2264,7 +2264,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-vllm' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "vllm" dockerfile: "./backend/Dockerfile.python" @@ -2277,7 +2277,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-vllm-omni' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "vllm-omni" dockerfile: "./backend/Dockerfile.python" @@ -2290,7 +2290,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-sglang' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "sglang" dockerfile: "./backend/Dockerfile.python" @@ -2303,7 +2303,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-transformers' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "transformers" dockerfile: "./backend/Dockerfile.python" @@ -2316,7 +2316,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-diffusers' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "diffusers" dockerfile: "./backend/Dockerfile.python" @@ -2329,7 +2329,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-ace-step' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "ace-step" dockerfile: "./backend/Dockerfile.python" @@ -2343,7 +2343,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-kokoro' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "kokoro" dockerfile: "./backend/Dockerfile.python" @@ -2356,7 +2356,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-vibevoice' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "vibevoice" dockerfile: "./backend/Dockerfile.python" @@ -2369,7 +2369,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-liquid-audio' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "liquid-audio" dockerfile: "./backend/Dockerfile.python" @@ -2382,7 +2382,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-qwen-asr' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "qwen-asr" dockerfile: "./backend/Dockerfile.python" @@ -2395,7 +2395,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-nemo' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "nemo" dockerfile: "./backend/Dockerfile.python" @@ -2408,7 +2408,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-qwen-tts' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "qwen-tts" dockerfile: "./backend/Dockerfile.python" @@ -2421,7 +2421,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-fish-speech' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "fish-speech" dockerfile: "./backend/Dockerfile.python" @@ -2434,7 +2434,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-voxcpm' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "voxcpm" dockerfile: "./backend/Dockerfile.python" @@ -2447,7 +2447,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-pocket-tts' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "pocket-tts" dockerfile: "./backend/Dockerfile.python" @@ -2460,7 +2460,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-faster-whisper' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "faster-whisper" dockerfile: "./backend/Dockerfile.python" @@ -2473,7 +2473,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-coqui' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "coqui" dockerfile: "./backend/Dockerfile.python" @@ -4098,7 +4098,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-whisper' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "whisper" @@ -4111,7 +4111,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-crispasr' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "crispasr" @@ -4220,7 +4220,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-parakeet-cpp' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "parakeet-cpp" @@ -4409,7 +4409,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-moss-transcribe-cpp' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "moss-transcribe-cpp" @@ -4557,7 +4557,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-ced' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "ced" @@ -4705,7 +4705,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-voice-detect' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "voice-detect" @@ -4853,7 +4853,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-face-detect' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "face-detect" @@ -4962,7 +4962,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-acestep-cpp' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "acestep-cpp" @@ -5924,7 +5924,7 @@ include: # platforms: 'linux/amd64' # tag-latest: 'auto' # tag-suffix: '-gpu-hipblas-rfdetr' - # base-image: "rocm/dev-ubuntu-24.04:7.2.1" + # base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" # runs-on: 'ubuntu-latest' # skip-drivers: 'false' # backend: "rfdetr" @@ -5996,7 +5996,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-neutts' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "neutts" dockerfile: "./backend/Dockerfile.python" diff --git a/.github/workflows/base-images.yml b/.github/workflows/base-images.yml index 637b603a56fb..87ce9c3fded7 100644 --- a/.github/workflows/base-images.yml +++ b/.github/workflows/base-images.yml @@ -83,7 +83,7 @@ jobs: ubuntu-version: '2404' - tag: 'base-grpc-rocm-amd64' runs-on: 'ubuntu-latest' - base-image: 'rocm/dev-ubuntu-24.04:7.2.1' + base-image: 'rocm/dev-ubuntu-24.04:7.14.0-full' build-type: 'hipblas' cuda-major-version: '' cuda-minor-version: '' diff --git a/.github/workflows/image-pr.yml b/.github/workflows/image-pr.yml index 2605c2541252..76b720c905f1 100644 --- a/.github/workflows/image-pr.yml +++ b/.github/workflows/image-pr.yml @@ -80,7 +80,7 @@ platforms: 'linux/amd64' tag-latest: 'false' tag-suffix: '-hipblas' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' makeflags: "--jobs=3 --output-sync=target" ubuntu-version: '2404' diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index cd90e849b9c2..28a5ff8a6cf5 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -85,7 +85,7 @@ platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-hipblas' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' makeflags: "--jobs=3 --output-sync=target" ubuntu-version: '2404' diff --git a/Dockerfile b/Dockerfile index 4ca9b32791ab..1b5b319666ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -158,16 +158,21 @@ RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ ; fi RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - hipblas-dev \ - hipblaslt-dev \ - rocblas-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ + # ROCm 7.x ("TheRock" packaging): the legacy hipblas-dev / hipblaslt-dev / + # rocblas-dev metapackages were removed — BLAS is consolidated and split + # per GPU arch (amdrocm-blas-gfx*, amdrocm-blas-dev, amdrocm-blas-host). + # The rocm/dev-ubuntu-*:*-full base image already ships those dev libraries + # and headers, so no apt install is needed here anymore. We still set the + # AMD capability marker. echo "amd" > /run/localai/capability && \ - # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, which results in local-ai and others not being able - # to locate the libraries. We run ldconfig ourselves to work around this packaging deficiency + # TheRock ships the ROCm libs under /opt/rocm/core*/lib (reached via the + # /opt/rocm/lib alternatives symlink) but registers no ld.so.conf.d entry, + # so the dynamic linker can't find them. Register the path so ldconfig + # (and the ldd-based backend GPU-lib packaging) resolves librocm_kpack, + # rocBLAS, hipBLASLt, the bundled rocm_sysdeps (zlib/zstd/elf) and the + # LLVM runtime — otherwise the built backend fails at runtime with e.g. + # "librocm_kpack.so.0 / librocm_sysdeps_z.so.1: cannot open shared object". + printf '/opt/rocm/lib\n/opt/rocm/lib/rocm_sysdeps/lib\n/opt/rocm/llvm/lib\n' > /etc/ld.so.conf.d/rocm.conf && \ ldconfig \ ; fi diff --git a/backend/Dockerfile.base-grpc-builder b/backend/Dockerfile.base-grpc-builder index 8843a0a72c29..c7c681dde82b 100644 --- a/backend/Dockerfile.base-grpc-builder +++ b/backend/Dockerfile.base-grpc-builder @@ -26,7 +26,7 @@ # base-grpc-cuda-13-amd64 ubuntu:22.04 + CUDA 13.0 # base-grpc-cuda-13-arm64 ubuntu:24.04 + CUDA 13.0 (sbsa) # base-grpc-l4t-cuda-12-arm64 ubuntu:22.04 + CUDA 12.x (legacy JetPack) -# base-grpc-rocm-amd64 rocm/dev-ubuntu-24.04:7.2.1 + hipblas +# base-grpc-rocm-amd64 rocm/dev-ubuntu-24.04:7.14.0-full + hipblas # base-grpc-vulkan-amd64 ubuntu:24.04 + Vulkan SDK 1.4.335 # base-grpc-vulkan-arm64 ubuntu:24.04 + Vulkan SDK ARM 1.4.335 # base-grpc-intel-amd64 intel/oneapi-basekit:2025.3.2 (sycl) diff --git a/backend/Dockerfile.golang b/backend/Dockerfile.golang index e5ab48957883..6a1db28cccd0 100644 --- a/backend/Dockerfile.golang +++ b/backend/Dockerfile.golang @@ -169,15 +169,12 @@ RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ ; fi RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - hipblas-dev \ - hipblaslt-dev \ - rocblas-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, which results in local-ai and others not being able - # to locate the libraries. We run ldconfig ourselves to work around this packaging deficiency + # ROCm 7.x ("TheRock" packaging): the legacy hipblas-dev / hipblaslt-dev / + # rocblas-dev metapackages were removed (consolidated + arch-split). The + # rocm/dev-ubuntu-*:*-full base image already ships the BLAS dev libs and + # headers, so no apt install is needed. Register TheRock's library paths + # so Go backends using cgo can resolve the ROCm runtime libraries. + printf '/opt/rocm/lib\n/opt/rocm/lib/rocm_sysdeps/lib\n/opt/rocm/llvm/lib\n' > /etc/ld.so.conf.d/rocm.conf && \ ldconfig \ ; fi diff --git a/backend/Dockerfile.python b/backend/Dockerfile.python index 2522a6f56be0..ad363e320343 100644 --- a/backend/Dockerfile.python +++ b/backend/Dockerfile.python @@ -6,6 +6,11 @@ FROM ${BASE_IMAGE} AS builder ARG BACKEND=rerankers ARG BUILD_TYPE ENV BUILD_TYPE=${BUILD_TYPE} +# AMD GPU target arch(es) for ROCm builds (e.g. gfx1151, or a ';'-list). Exposed +# to install.sh so ROCm backends can select AMD's per-GPU torch and target the +# right arch when building from source, instead of hardcoding one. +ARG AMDGPU_TARGETS +ENV AMDGPU_TARGETS=${AMDGPU_TARGETS} ARG CUDA_MAJOR_VERSION ARG CUDA_MINOR_VERSION ARG SKIP_DRIVERS=false @@ -171,15 +176,14 @@ RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ ; fi RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - hipblas-dev \ - hipblaslt-dev \ - rocblas-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, which results in local-ai and others not being able - # to locate the libraries. We run ldconfig ourselves to work around this packaging deficiency + # ROCm 7.x ("TheRock" packaging): the legacy hipblas-dev / hipblaslt-dev / + # rocblas-dev metapackages were removed (consolidated + arch-split). The + # rocm/dev-ubuntu-*:*-full base image already ships the BLAS dev libs and + # headers, so no apt install is needed. TheRock also scatters the ROCm libs + # across /opt/rocm/lib, /opt/rocm/lib/rocm_sysdeps/lib and /opt/rocm/llvm/lib + # with no ld.so.conf.d entry — register them so ldconfig (and Python + # backends that dlopen ROCm libs at runtime) can resolve them. + printf '/opt/rocm/lib\n/opt/rocm/lib/rocm_sysdeps/lib\n/opt/rocm/llvm/lib\n' > /etc/ld.so.conf.d/rocm.conf && \ ldconfig \ ; fi diff --git a/backend/python/diffusers/install.sh b/backend/python/diffusers/install.sh index 83703b1b2853..fc27a53c1a22 100755 --- a/backend/python/diffusers/install.sh +++ b/backend/python/diffusers/install.sh @@ -27,4 +27,21 @@ if [ "x${BUILD_PROFILE}" == "xl4t13" ]; then PY_STANDALONE_TAG="20251120" fi +# ROCm/gfx1151: the community whl/rocm7.0 torch wheels do not enumerate Strix +# Halo (device_count == 0). AMD ships stable ROCm 7.14 builds via its multi-arch +# index, selected per GPU with the torch[device-gfx] extra. Install torch + +# torchvision (the same pinned versions, from that index) in an isolated step: +# uv aborts on that index's 403-for-missing-package responses, so we must not let +# it resolve PyPI-only packages (diffusers, transformers, ...) there. Everything +# else then resolves from PyPI in installRequirements. +if [ "x${BUILD_PROFILE}" == "xhipblas" ]; then + # Arch from the build (AMDGPU_TARGETS); default gfx1151 -- the only arch we + # could validate on real hardware. The torch[device-...] extra takes one arch. + _gpu_arch="${AMDGPU_TARGETS:-gfx1151}"; _gpu_arch="${_gpu_arch%%;*}"; _gpu_arch="${_gpu_arch%% *}" + ensureVenv + uv pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ \ + "torch[device-${_gpu_arch}]==2.10.0+rocm7.14.0" \ + "torchvision==0.25.0+rocm7.14.0" +fi + installRequirements diff --git a/backend/python/diffusers/requirements-hipblas.txt b/backend/python/diffusers/requirements-hipblas.txt index f3666d5f504c..47846fc5c91c 100644 --- a/backend/python/diffusers/requirements-hipblas.txt +++ b/backend/python/diffusers/requirements-hipblas.txt @@ -1,6 +1,8 @@ ---extra-index-url https://download.pytorch.org/whl/rocm7.0 -torch==2.10.0+rocm7.0 -torchvision==0.25.0+rocm7.0 +# torch + torchvision are installed separately from AMD's multi-arch ROCm 7.14 +# index in install.sh (gfx1151 needs torch built against ROCm 7.13+; the +# community whl/rocm7.0 wheel does not enumerate Strix Halo). uv aborts on that +# index's 403-for-missing-pkg, so torch is pulled there; everything below resolves +# from PyPI. diffusers==0.38.0 opencv-python transformers==4.57.6 diff --git a/backend/python/kokoro/install.sh b/backend/python/kokoro/install.sh index 3e1ab8606480..c652ba6a8980 100755 --- a/backend/python/kokoro/install.sh +++ b/backend/python/kokoro/install.sh @@ -20,6 +20,23 @@ if [ "x${BUILD_PROFILE}" == "xl4t12" ]; then USE_PIP=true fi +# ROCm/gfx1151: the community whl/rocm7.0 torch wheels do not enumerate Strix +# Halo (device_count == 0). AMD ships stable ROCm 7.14 builds via its multi-arch +# index, selected per GPU with the torch[device-gfx] extra. Install torch + +# torchaudio (the same pinned versions, from that index) in an isolated step: uv +# aborts on that index's 403-for-missing-package responses, so we must not let it +# resolve PyPI-only packages (kokoro, transformers, ...) there. Everything else +# then resolves from PyPI in installRequirements. +if [ "x${BUILD_PROFILE}" == "xhipblas" ]; then + # Arch from the build (AMDGPU_TARGETS); default gfx1151 -- the only arch we + # could validate on real hardware. The torch[device-...] extra takes one arch. + _gpu_arch="${AMDGPU_TARGETS:-gfx1151}"; _gpu_arch="${_gpu_arch%%;*}"; _gpu_arch="${_gpu_arch%% *}" + ensureVenv + uv pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ \ + "torch[device-${_gpu_arch}]==2.10.0+rocm7.14.0" \ + "torchaudio==2.10.0+rocm7.14.0" +fi + installRequirements # spaCy is a dependency of misaki (used by kokoro for English phonemization). diff --git a/backend/python/kokoro/requirements-hipblas.txt b/backend/python/kokoro/requirements-hipblas.txt index 0ca1352282c9..9edeff371eeb 100644 --- a/backend/python/kokoro/requirements-hipblas.txt +++ b/backend/python/kokoro/requirements-hipblas.txt @@ -1,6 +1,8 @@ ---extra-index-url https://download.pytorch.org/whl/rocm7.0 -torch==2.10.0+rocm7.0 -torchaudio==2.10.0+rocm7.0 +# torch + torchaudio are installed separately from AMD's multi-arch ROCm 7.14 +# index in install.sh (gfx1151 needs torch built against ROCm 7.13+; the +# community whl/rocm7.0 wheel does not enumerate Strix Halo). uv aborts on that +# index's 403-for-missing-pkg, so torch is pulled there; everything below resolves +# from PyPI. transformers accelerate kokoro diff --git a/backend/python/transformers/install.sh b/backend/python/transformers/install.sh index 32befa8e6c03..e2084bb30744 100755 --- a/backend/python/transformers/install.sh +++ b/backend/python/transformers/install.sh @@ -16,4 +16,20 @@ if [ "x${BUILD_PROFILE}" == "xintel" ]; then EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" fi +# ROCm/gfx1151: the community whl/rocm7.0 torch wheel does not enumerate Strix +# Halo (device_count == 0). AMD ships a stable ROCm 7.14 build via its multi-arch +# index, selected per GPU with the torch[device-gfx] extra. Install it (plus +# its rocm/triton deps, which all live on that same index) in an isolated step: +# uv aborts on that index's 403-for-missing-package responses, so we must not let +# it look up PyPI-only packages (accelerate, transformers, ...) there. Everything +# else then resolves from PyPI in installRequirements. +if [ "x${BUILD_PROFILE}" == "xhipblas" ]; then + # Arch from the build (AMDGPU_TARGETS); default gfx1151 -- the only arch we + # could validate on real hardware. The torch[device-...] extra takes one arch. + _gpu_arch="${AMDGPU_TARGETS:-gfx1151}"; _gpu_arch="${_gpu_arch%%;*}"; _gpu_arch="${_gpu_arch%% *}" + ensureVenv + uv pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ \ + "torch[device-${_gpu_arch}]==2.12.0+rocm7.14.0" +fi + installRequirements diff --git a/backend/python/transformers/requirements-hipblas.txt b/backend/python/transformers/requirements-hipblas.txt index f2cd4e075dc0..a2f271495667 100644 --- a/backend/python/transformers/requirements-hipblas.txt +++ b/backend/python/transformers/requirements-hipblas.txt @@ -1,5 +1,7 @@ ---extra-index-url https://download.pytorch.org/whl/rocm7.0 -torch==2.10.0+rocm7.0 +# torch is installed separately from AMD's multi-arch ROCm index in install.sh +# (gfx1151 needs torch built against ROCm 7.13+; the community whl/rocm7.0 wheel +# does not enumerate Strix Halo). uv aborts on that index's 403-for-missing-pkg, +# so torch is pulled with pip there; everything below resolves from PyPI. accelerate transformers>=5.15.0 llvmlite==0.43.0