diff --git a/.github/workflows/consolidated-tests-ci.yml b/.github/workflows/consolidated-tests-ci.yml
deleted file mode 100644
index f8eccc7d18..0000000000
--- a/.github/workflows/consolidated-tests-ci.yml
+++ /dev/null
@@ -1,2289 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# One consolidated CPU-only job that runs every test_* function the existing
-# CI does not already cover from this repo plus the full unsloth_zoo@main
-# CPU test suite plus unsloth_zoo.compiler.test_apply_fused_lm_head.
-#
-# Why a separate workflow:
-# - studio-backend-ci.yml's "Repo tests (CPU)" job already auto-discovers
-# tests/ minus tests/qlora, tests/saving, tests/utils, tests/sh. The 16
-# Bucket-A tests below live inside those --ignore dirs (CPU-runnable but
-# historically excluded with their GPU siblings); pulling them out into
-# a sibling job keeps the existing 760-passed baseline stable while we
-# prove the new pieces are green.
-# - unsloth_zoo has no CI on main today (.github/workflows/ is empty
-# upstream as of HEAD 030e4ba). 106 of its 111 test_* functions are
-# CPU-runnable; the 5 GPU/vLLM ones are deselected here.
-# - test_apply_fused_lm_head lives at unsloth_zoo/compiler.py:1983, not
-# under tests/, so it is not picked up by `pytest tests/`. It is a
-# plain function with no fixtures: pure regex over transformers source
-# strings, ~5-15 s wall, no GPU.
-#
-# Strict mode: every test step is gating (no `continue-on-error`). The
-# upstream patch fixes that previously caused per-cell red have landed:
-# - unslothai/unsloth#5319 (patch_fast_lora import, patch_sft_trainer
-# Union, openenv OSError graceful skip).
-# - unslothai/unsloth-zoo#628 (MoE coverage canary so old transformers
-# skips legitimately while real discovery regressions still fail).
-# After those merges every observed cell failure was one of these two
-# things; if they regress we want a red cell, not a green-with-fail-prints
-# cell.
-
-name: Core
-
-on:
- pull_request:
- paths:
- - 'unsloth/**'
- - 'unsloth_cli/**'
- - 'studio/**'
- - 'tests/**'
- - 'pyproject.toml'
- - '.github/workflows/consolidated-tests-ci.yml'
- push:
- branches: [main, pip]
- workflow_dispatch:
- inputs:
- unsloth_zoo_ref:
- description: 'unsloth_zoo git ref to test against (default main)'
- required: false
- default: 'main'
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- consolidated:
- # Matrix: three (transformers, TRL) combos cover the failure surface the
- # PR cares about:
- # 1. transformers==4.57.6 + TRL latest <1.0.0 (the just-before-5.x line)
- # 2. transformers latest 5.x + TRL latest 1.x (the absolute upstream tip;
- # currently 5.8.0 + 1.3.0, both BEYOND the unsloth/unsloth_zoo
- # <=5.5.0 / <=0.24.0 caps -- the cell exists explicitly to surface
- # drift signal)
- # 3. transformers + TRL pinned by pyproject.toml's dependency entries
- # (resolved dynamically at job time via tomllib)
- # fail-fast: false so each cell runs independently and a transformers /
- # TRL drift signal in one cell does not cancel the others. No
- # job-level or per-step `continue-on-error` -- real test failures now
- # fail the cell. Patches with legitimate CPU-runner preconditions
- # (real CUDA dispatcher, runtime args) are explicitly skipped via
- # NEEDS_PRECONDITION in the runtime check shim below.
- strategy:
- fail-fast: false
- matrix:
- combo:
- - id: t4576-trl0latest
- label: "HF=4.57.6 + TRL<1"
- transformers_spec: "transformers==4.57.6"
- trl_spec: "trl>=0.18.2,<1.0.0"
- - id: tlatest5-trl1latest
- label: "HF=latest + TRL=latest"
- transformers_spec: "transformers>=5,<6"
- trl_spec: "trl>=1,<2"
- - id: pyproject
- label: "HF=default + TRL=default"
- transformers_spec: "__from_pyproject__"
- trl_spec: "__from_pyproject__"
- name: "Core (${{ matrix.combo.label }})"
- runs-on: ubuntu-latest
- timeout-minutes: 35
- # No job-level or per-step `continue-on-error`. Earlier iterations
- # masked real test failures behind green check icons; that lie is
- # gone. A failing test step fails the cell. NEEDS_PRECONDITION in
- # the runtime check shim handles patches that legitimately cannot
- # run on a CPU-only runner (real CUDA dispatcher, runtime args).
- env:
- UNSLOTH_ZOO_REF: ${{ inputs.unsloth_zoo_ref || 'main' }}
- MATRIX_TRANSFORMERS_SPEC: ${{ matrix.combo.transformers_spec }}
- MATRIX_TRL_SPEC: ${{ matrix.combo.trl_spec }}
- MATRIX_COMBO_ID: ${{ matrix.combo.id }}
- # Hoisted to job-level so every step (Sanity, Bucket-A, unsloth_zoo
- # pytest, test_apply_fused_lm_head) inherits it. transformers' bundled
- # *_pb2.py was generated against an older protoc; the C++ protobuf
- # 4+/5+/6 implementation rejects them with "Descriptors cannot be
- # created directly". The pure-Python parser bypasses the check; the
- # speed cost is negligible for these tests.
- PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
- PYTHONPATH: ${{ github.workspace }}/studio
- UNSLOTH_COMPILE_DISABLE: '1'
- # unsloth_zoo/__init__.py:314 raises ImportError unless UNSLOTH_IS_PRESENT
- # is set — normally it is set by unsloth.__init__ when unsloth is imported
- # first. In this job we sometimes import unsloth_zoo.* (e.g.
- # unsloth_zoo.saving_utils, unsloth_zoo.temporary_patches) without going
- # through `import unsloth` first; pin the env var to 1 so unsloth_zoo's
- # bootstrap accepts it. Setting it has no effect on unsloth itself.
- UNSLOTH_IS_PRESENT: '1'
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- # Node 22 unblocks tests/studio/test_chat_preset_builtin_invariants.py's
- # `node --experimental-strip-types` subprocess. Cheap to install; keeps
- # the consolidated job self-sufficient even if studio-backend-ci.yml
- # changes its node setup.
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - name: Install uv (some unsloth_zoo dev tooling expects it on PATH)
- run: pip install uv
-
- - name: Resolve matrix specs (handle __from_pyproject__ sentinel)
- # The pyproject cell uses a sentinel; resolve the real `transformers`
- # and `trl` constraints from the project's pyproject.toml at job time.
- # unsloth's pyproject puts the LLM stack pins in
- # [project.optional-dependencies] under the `huggingfacenotorch`
- # extra (top-level [project.dependencies] is just typer/pydantic/etc.),
- # so we walk every optional extra and pick the first matching spec.
- # Other cells pass their spec through unchanged.
- run: |
- set -euxo pipefail
- python <<'PY' >> "$GITHUB_ENV"
- import os, re, tomllib
- spec_t = os.environ["MATRIX_TRANSFORMERS_SPEC"]
- spec_r = os.environ["MATRIX_TRL_SPEC"]
-
- def _pkg_name(spec: str) -> str:
- m = re.match(r"\s*([A-Za-z0-9_.-]+)", spec)
- return (m.group(1).lower() if m else "")
-
- if spec_t == "__from_pyproject__" or spec_r == "__from_pyproject__":
- with open("pyproject.toml", "rb") as f:
- doc = tomllib.load(f)
- proj = doc.get("project", {})
- # Try top-level deps first, then all optional extras.
- all_deps: list[str] = list(proj.get("dependencies", []))
- for _name, dep_list in proj.get("optional-dependencies", {}).items():
- all_deps.extend(dep_list)
-
- if spec_t == "__from_pyproject__":
- spec_t = next((x for x in all_deps if _pkg_name(x) == "transformers"),
- "transformers")
- if spec_r == "__from_pyproject__":
- spec_r = next((x for x in all_deps if _pkg_name(x) == "trl"),
- "trl")
- print(f"RESOLVED_TRANSFORMERS_SPEC={spec_t}")
- print(f"RESOLVED_TRL_SPEC={spec_r}")
- PY
- # Echo to logs so the matrix cell label maps cleanly to a spec.
- grep RESOLVED_ "$GITHUB_ENV" || true
-
- - name: Install runtime deps (mirrors studio-backend-ci.yml + mlx-ci.yml)
- # The shape matches studio-backend-ci.yml's "Repo tests (CPU)" install
- # so we inherit the same CPU-spoof harness in tests/conftest.py and
- # the same import-chain guarantees, plus the extra deps that the
- # tests/saving + tests/utils Bucket-A files transitively need but
- # which Repo tests (CPU) does not require because it --ignores
- # those directories:
- # - protobuf + sentencepiece: tests/saving/test_fix_sentencepiece_gguf_robustness.py
- # does `from transformers.utils import sentencepiece_model_pb2`,
- # which imports `google.protobuf`. Not pulled by transformers'
- # base install.
- # - triton: unsloth/_gpu_init.py:232 does an unconditional
- # `import triton`. The triton PyPI wheel installs cleanly on
- # Linux x86_64 even without CUDA (the import succeeds; runtime
- # GPU work is what would fail, which we never do here).
- # transformers + trl are matrix-parameterized.
- run: |
- set -euxo pipefail
- python -m pip install --upgrade pip
- pip install -r studio/backend/requirements/studio.txt
- pip install \
- python-multipart aiofiles sqlalchemy cryptography \
- pyyaml jinja2 mammoth unpdf requests typer \
- 'numpy<3' pytest==9.0.3 pytest-asyncio httpx \
- protobuf sentencepiece triton \
- psutil packaging tqdm safetensors datasets \
- 'peft>=0.18,<0.20' 'accelerate>=0.34,<2' \
- ipython
- # torchvision: unsloth_zoo.vision_utils imports it at module scope.
- pip install --index-url https://download.pytorch.org/whl/cpu \
- 'torch>=2.4,<2.11' 'torchvision<0.26'
- # transformers + trl from the matrix combo.
- pip install "$RESOLVED_TRANSFORMERS_SPEC"
- pip install "$RESOLVED_TRL_SPEC"
- # bitsandbytes: hard import in unsloth/models/_utils.py. Recent
- # versions ship a CPU build that imports cleanly on Linux.
- pip install 'bitsandbytes>=0.45'
- # unsloth itself, editable, no-deps so pip does not fight the
- # explicit torch CPU-index install above.
- pip install -e . --no-deps
- echo "::group::Installed transformers + trl + torch + unsloth versions"
- pip show transformers
- pip show trl
- pip show torch
- pip show unsloth
- echo "::endgroup::"
-
- - name: Clone unsloth_zoo @ ${{ env.UNSLOTH_ZOO_REF }}
- # We need the repository tree (the wheel does not ship tests/), so
- # clone shallow then editable-install so unsloth_zoo.* imports
- # resolve to the cloned tree. We use `pip show` for the location
- # check rather than `import unsloth_zoo` because the latter calls
- # device_type.get_device_type() at module load and raises on a
- # GPU-less runner; pytest steps below route through the existing
- # tests/conftest.py spoof which handles that.
- run: |
- set -euxo pipefail
- # github.com occasionally 500s on the git fetch; retry so a
- # single upstream blip does not fail CI.
- for attempt in 1 2 3; do
- rm -rf "$RUNNER_TEMP/unsloth-zoo"
- if git clone --depth=1 --branch="$UNSLOTH_ZOO_REF" \
- https://github.com/unslothai/unsloth-zoo \
- "$RUNNER_TEMP/unsloth-zoo"; then
- break
- fi
- if [ "$attempt" -eq 3 ]; then
- echo "::error::git clone unsloth-zoo failed after 3 attempts"
- exit 1
- fi
- delay=$((5 * attempt))
- echo "::warning::clone failed (attempt $attempt/3), retrying in ${delay}s..."
- sleep "$delay"
- done
- pip install -e "$RUNNER_TEMP/unsloth-zoo" --no-deps
- pip show unsloth_zoo
-
- - name: Sanity — collection only (both repos)
- # Catches import-time breakage before we run the suite. Cheap; bails
- # the job out fast if a transformers/torch resolution went sideways.
- # Inherits PYTHONPATH / UNSLOTH_COMPILE_DISABLE / PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION
- # from the job-level env block.
- run: |
- set -euxo pipefail
- python -m pytest --collect-only -q \
- tests/saving/test_save_shell_injection.py \
- tests/saving/test_patch_saving_none_tokenizer.py \
- tests/saving/test_fix_sentencepiece_gguf_robustness.py \
- tests/utils/test_attention_masks.py \
- tests/utils/test_trunc_normal_patch.py
- python -m pytest --collect-only -q "$RUNNER_TEMP/unsloth-zoo/tests/"
-
- - name: import_fixes drift detectors (18 tests, HARD GATE)
- # One drift detector per fix_* / patch_* function in
- # unsloth/import_fixes.py. The detectors assert the *healthy*
- # upstream shape that the fix expects ABSENT the regression;
- # ANY DRIFT DETECTED -> pytest.fail (NEVER skip) so the
- # matrix cell goes red and the maintainer triages on the
- # next PR, not in a downstream user's crash report.
- #
- # Pathologies covered by the suite (each maps to one fix
- # function with the line range cited in the test docstring):
- # * protobuf MessageFactory GetPrototype / GetMessageClass
- # * datasets 4.4.x recursion range
- # * TRL tuple-vs-bool _*_available caching
- # * transformers PreTrainedModel.enable_input_require_grads
- # source pattern flip
- # * transformers torchcodec / causal_conv1d availability
- # flags
- # * transformers + accelerate is_wandb_available
- # * peft.utils.transformers_weight_conversion importability
- # + build_peft_weight_mapping signature
- # * triton 3.6+ CompiledKernel num_ctas / cluster_dims
- # * torch / torchvision pinned compatibility table
- # * vllm guided_decoding_params / structured_outputs +
- # aimv2 ovis config version
- # * huggingface_hub is_offline_mode / HF_HUB_OFFLINE
- # * torch.nn.init.trunc_normal_ presence (patch site for
- # patch_trunc_normal_precision_issue)
- # * xformers post-num_splits-key fix version
- # HARD GATE: a red cell here is a real upstream regression
- # without a corresponding zoo / unsloth-side workaround.
- run: |
- python -m pytest -v --tb=short tests/test_import_fixes_drift.py
-
- - name: public-api surface drift detectors (9 tests, HARD GATE)
- # Companion to test_import_fixes_drift.py: that file catches
- # third-party drift; this one catches drift in unsloth's OWN
- # public surface (FastLanguageModel / FastVisionModel /
- # FastModel + their classmethods + is_bf16_supported). A
- # rename here would silently break the unslothai/notebooks tree
- # one PR cycle later -- this gate catches it BEFORE the
- # breakage reaches users.
- run: |
- python -m pytest -v --tb=short tests/test_public_api_surface.py
-
- - name: callback signature drift detector (HARD GATE)
- # Catches the MLX-style bug from PR #5498: a producer in
- # unsloth_zoo (or unsloth) grows a callback arg, but a consumer
- # callback def still declares the old arity. The producer's
- # try/except swallows the resulting TypeError and the symptom is
- # "callback never fires" -- usually diagnosed downstream as a
- # confusing assertion several seconds later. This static AST
- # check fails fast at PR time. UNSLOTH_ZOO_SRC points at the
- # freshly cloned main so the detector sees platform-specific
- # submodules (e.g. unsloth_zoo/mlx/) that the released wheel
- # may strip.
- env:
- UNSLOTH_ZOO_SRC: ${{ runner.temp }}/unsloth-zoo
- run: |
- python -m pytest -v --tb=short tests/test_callback_signature_drift.py
-
- - name: unsloth Bucket-A — CPU tests not in Repo tests (CPU)
- # 16 tests across 5 files. They live inside tests/saving/ and
- # tests/utils/, both of which Repo tests (CPU) excludes via --ignore
- # because their sibling files need real GPUs / real HF weights.
- # The five files below are pure-Python + AST/protobuf/regex tests
- # that run cleanly on CPU. Env inherited from the job block.
- run: |
- python -m pytest -q --tb=short \
- tests/saving/test_save_shell_injection.py \
- tests/saving/test_patch_saving_none_tokenizer.py \
- tests/saving/test_fix_sentencepiece_gguf_robustness.py \
- tests/utils/test_attention_masks.py \
- tests/utils/test_trunc_normal_patch.py \
- --deselect 'tests/utils/test_attention_masks.py::test_run_attention_flash_varlen_receives_window_and_softcap'
- # The deselected test monkeypatches flash_attn_varlen_func, which is
- # only bound on the module when `flash_attn` is importable. flash_attn
- # requires CUDA + dev toolchain, which the CPU-only ubuntu-latest
- # runner does not have. The other 15 Bucket-A tests pass cleanly.
-
- - name: unsloth_zoo @ ${{ env.UNSLOTH_ZOO_REF }} — full pytest (CPU)
- # 106 of 111 test_* in unsloth_zoo are CPU-only. The two CUDA-skip
- # cases below auto-skip on a GPU-less runner; deselect them
- # explicitly so the no-CUDA outcome is "deselected", not "skipped",
- # making intent visible in the report. Env inherited from job block.
- #
- # test_get_peft_model_passes_finetune_last_n_layers_through is
- # deselected because unsloth_zoo/mlx/loader.py at line 2972 calls
- # model.trainable_parameters() on the fake-model fixture, which
- # the test never stubbed; this fails on every platform regardless
- # of CUDA. Tracked upstream as an unsloth_zoo bug; deselecting
- # here unblocks unsloth CI until the loader fixture is fixed.
- working-directory: ${{ runner.temp }}/unsloth-zoo
- run: |
- python -m pytest -q --tb=short tests/ \
- --deselect tests/test_unsloth_zoo_lora_merge.py::test_active_merge_device_returns_string_on_cuda_host \
- --deselect tests/test_unsloth_zoo_lora_merge.py::test_merge_lora_moves_cpu_inputs_to_active_device \
- --deselect tests/test_mlx_finetune_last_n_layers.py::test_get_peft_model_passes_finetune_last_n_layers_through
-
- - name: unsloth_zoo — test_apply_fused_lm_head (lives in compiler.py)
- # `test_apply_fused_lm_head` lives at unsloth_zoo/compiler.py:1983,
- # not under tests/, so pytest's default discovery does not pick it up.
- # We route it through pytest by writing a one-shot shim test file
- # inside the unsloth checkout's tests/ — pytest then walks UP and
- # picks up tests/conftest.py, whose GPU-spoof harness (lines 84-141)
- # patches torch.cuda.is_available, torch.cuda.memory.mem_get_info,
- # torch.cuda.get_device_capability, and is_bf16_supported. That full
- # spoof is required because unsloth_zoo/temporary_patches/gpt_oss.py
- # at module load reads torch.cuda.memory.mem_get_info(0), which
- # bare `is_available = True` doesn't cover. Env inherited.
- run: |
- set -euxo pipefail
- cat > tests/_zoo_apply_fused_lm_head_shim.py <<'PY'
- # Auto-generated by .github/workflows/consolidated-tests-ci.yml.
- # Wraps unsloth_zoo.compiler.test_apply_fused_lm_head so that
- # tests/conftest.py's GPU-spoof harness applies before the import.
- # _zoo_aggressive_cuda_spoof extends conftest's harness with deeper
- # patches (see tests/_zoo_aggressive_cuda_spoof.py).
- import sys, pathlib
- sys.path.insert(0, str(pathlib.Path(__file__).parent))
- import _zoo_aggressive_cuda_spoof as _spoof
- _spoof.apply()
- from unsloth_zoo.compiler import test_apply_fused_lm_head as _zoo_test
- def test_zoo_apply_fused_lm_head_runs():
- _zoo_test()
- PY
- python -m pytest -q --tb=short tests/_zoo_apply_fused_lm_head_shim.py
- rm -f tests/_zoo_apply_fused_lm_head_shim.py
-
- - name: Static checks — unsloth/trainer.py + unsloth/models/rl.py against latest pip TRL
- # AST-only sanity: confirm both files parse and that every TRL symbol
- # they reference still exists in the installed `trl`. Catches API
- # drift (renamed / removed TRL classes) without running training.
- # Pre-fetches latest pip transformers in case TRL pinned an older one.
- run: |
- set -euxo pipefail
- # Use the matrix-resolved transformers + trl versions already
- # installed by the runtime-deps step (don't upgrade here; that
- # would defeat the matrix's purpose of testing against the
- # specific (transformers, trl) combination the cell selected).
- python <<'PY'
- import ast, importlib, pathlib, sys
- paths = [pathlib.Path("unsloth/trainer.py"),
- pathlib.Path("unsloth/models/rl.py")]
- for p in paths:
- src = p.read_text()
- tree = ast.parse(src, filename=str(p))
- # Collect every `from trl... import X` and `from trl... import (X, Y)`
- missing = []
- for node in ast.walk(tree):
- if isinstance(node, ast.ImportFrom) and node.module and node.module.startswith("trl"):
- mod = importlib.import_module(node.module)
- for alias in node.names:
- if alias.name == "*":
- continue
- if not hasattr(mod, alias.name):
- missing.append(f"{node.module}.{alias.name}")
- print(f"{p}: TRL symbols referenced and resolved -> {'OK' if not missing else 'MISSING ' + ', '.join(missing)}")
- if missing:
- sys.exit(1)
- PY
-
- - name: Static checks — unsloth_zoo/tiled_mlp.py against latest pip transformers
- # AST parse + transformers symbol-resolution. The user flagged tiled
- # MLP patching as the path that breaks first when transformers ships
- # an MLP class rename; this step is the canary against whatever
- # transformers version the matrix cell selected.
- working-directory: ${{ runner.temp }}/unsloth-zoo
- run: |
- set -euxo pipefail
- python <<'PY'
- import ast, importlib, pathlib, sys
- p = pathlib.Path("unsloth_zoo/tiled_mlp.py")
- src = p.read_text()
- tree = ast.parse(src, filename=str(p))
- missing = []
- for node in ast.walk(tree):
- if isinstance(node, ast.ImportFrom) and node.module and node.module.startswith("transformers"):
- try:
- mod = importlib.import_module(node.module)
- except Exception as e:
- missing.append(f"{node.module} (import failed: {type(e).__name__})")
- continue
- for alias in node.names:
- if alias.name == "*":
- continue
- if not hasattr(mod, alias.name):
- missing.append(f"{node.module}.{alias.name}")
- print(f"{p}: transformers symbols referenced -> {'OK' if not missing else 'MISSING ' + ', '.join(missing)}")
- if missing:
- sys.exit(1)
- PY
-
- - name: Static checks — unsloth_zoo/hf_utils.py syntax + import-graph
- working-directory: ${{ runner.temp }}/unsloth-zoo
- run: |
- set -euxo pipefail
- python <<'PY'
- import ast, pathlib
- p = pathlib.Path("unsloth_zoo/hf_utils.py")
- tree = ast.parse(p.read_text(), filename=str(p))
- # Surface every public function + class so the PR check log shows
- # what's covered, not just OK/FAIL.
- public = []
- for node in tree.body:
- if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and not node.name.startswith("_"):
- public.append(f"{type(node).__name__.replace('Def','').lower()}:{node.name}")
- print(f"hf_utils.py public surface ({len(public)}): " + ", ".join(public))
- PY
-
- - name: Runtime checks — invoke every zero-arg patch_* across both repos (via pytest shim)
- # Routed through pytest so tests/conftest.py's GPU-spoof harness
- # applies before any unsloth_zoo.temporary_patches.* import.
- # Locally validated 50/51 zero-arg patches succeed; the lone failure
- # surfaces a real bug (unsloth.models._utils.patch_fast_lora raises
- # NameError: name 'fast_lora_forward' is not defined). The shim
- # reports the full ledger but only fails when one of the two
- # `required` helpers is absent.
- run: |
- set -euxo pipefail
- cat > tests/_runtime_patch_check_shim.py <<'PY'
- # Auto-generated by .github/workflows/consolidated-tests-ci.yml.
- # Wraps the runtime patch_* validation into a pytest test so the
- # tests/conftest.py GPU-spoof harness applies. continue-on-error
- # at the workflow level catches per-patch failures; this shim only
- # asserts that the two `required` helpers are reachable.
- import sys, pathlib
- sys.path.insert(0, str(pathlib.Path(__file__).parent))
- import _zoo_aggressive_cuda_spoof as _spoof
- _spoof.apply()
- import importlib, inspect
-
- MODULES = [
- "unsloth.models._utils", "unsloth.models.rl", "unsloth.import_fixes",
- "unsloth.kernels.cross_entropy_loss", "unsloth.kernels.rms_layernorm",
- "unsloth.tokenizer_utils", "unsloth.save",
- "unsloth_zoo.patching_utils", "unsloth_zoo.gradient_checkpointing",
- "unsloth_zoo.loss_utils", "unsloth_zoo.tokenizer_utils",
- "unsloth_zoo.tiled_mlp", "unsloth_zoo.dataset_utils",
- "unsloth_zoo.patch_torch_functions",
- "unsloth_zoo.temporary_patches.gemma",
- "unsloth_zoo.temporary_patches.ministral",
- "unsloth_zoo.temporary_patches.pixtral",
- "unsloth_zoo.temporary_patches.deepseek_v3_moe",
- "unsloth_zoo.temporary_patches.qwen3_5_moe",
- "unsloth_zoo.temporary_patches.mxfp4",
- "unsloth_zoo.temporary_patches.bitsandbytes",
- "unsloth_zoo.temporary_patches.flex_attention_bwd",
- ]
- REQUIRED = {
- "patch_unsloth_smart_gradient_checkpointing",
- "patch_gradient_accumulation_fix",
- }
- # Patches whose signature looks zero-arg (`()` or all-defaulted)
- # but which actually require either runtime args or real CUDA.
- # Calling these in isolation is meaningless, so skip the
- # invocation. Symbol presence (REQUIRED above) is still verified.
- # patch_linear_scaling / patch_llama_rope_scaling: defaults are
- # None placeholders; the bodies start with
- # `assert is not None`.
- # patch_unsloth_smart_gradient_checkpointing: legitimately
- # allocates CUDA tensors via aten::empty.memory_format inside
- # initialize_unsloth_gradient_checkpointing(); the
- # torch.cuda.* spoof can't intercept that at the dispatcher
- # level.
- NEEDS_PRECONDITION = {
- "patch_linear_scaling",
- "patch_llama_rope_scaling",
- "patch_unsloth_smart_gradient_checkpointing",
- }
-
- def test_zero_arg_patch_invocations():
- ok, fail, args, skipped, miss_imports = 0, [], [], [], {}
- seen_required = set()
- for mod_name in MODULES:
- try:
- mod = importlib.import_module(mod_name)
- except Exception as e:
- miss_imports[mod_name] = f"{type(e).__name__}: {e}"
- continue
- for name in sorted(dir(mod)):
- if not name.startswith("patch_"): continue
- fn = getattr(mod, name, None)
- if not callable(fn): continue
- if name in REQUIRED: seen_required.add(name)
- try:
- sig = inspect.signature(fn)
- need = [p.name for p in sig.parameters.values()
- if p.default is inspect.Parameter.empty
- and p.kind in (inspect.Parameter.POSITIONAL_OR_KEYWORD,
- inspect.Parameter.POSITIONAL_ONLY)]
- except (TypeError, ValueError):
- need = []
- if need:
- args.append((mod_name, name, need)); continue
- if name in NEEDS_PRECONDITION:
- skipped.append(f"{mod_name}.{name}")
- print(f" SKIP {mod_name}.{name} (needs precondition / CUDA)")
- continue
- try:
- fn()
- ok += 1
- print(f" OK {mod_name}.{name}")
- except Exception as e:
- fail.append((mod_name, name, type(e).__name__, str(e)[:200]))
- print(f" FAIL {mod_name}.{name} -> {type(e).__name__}: {str(e)[:200]}")
- print(f"\nzero-arg patch_*: ok={ok} fail={len(fail)} skipped={len(skipped)}")
- print(f"arg-required patch_* (skipped, listed for review): {len(args)}")
- for m, n, r in args:
- print(f" needs={r}: {m}.{n}")
- if skipped:
- print(f"explicitly skipped (needs precondition / CUDA): {skipped}")
- if miss_imports:
- print("\nmodules failed to import (skipped):")
- for k, v in miss_imports.items():
- print(f" {k}: {v}")
- print(f"required patch_* helpers seen: {sorted(seen_required)}")
- missing = REQUIRED - seen_required
- assert not missing, f"required patch_* helpers MISSING: {sorted(missing)}"
- # Strict: any zero-arg patch that raises is a real
- # regression now that #5319 has landed (the three previously
- # known-broken patches are fixed; legitimate
- # CPU-precondition skips are recorded in NEEDS_PRECONDITION
- # above, not in `fail`). Print all failures and re-raise
- # them as one assertion message.
- if fail:
- raise AssertionError(
- f"zero-arg patch_* invocation failures (ok={ok}, "
- f"fail={len(fail)}, skipped={len(skipped)}):\n "
- + "\n ".join(
- f"{m}.{n} -> {ec}: {msg}" for m, n, ec, msg in fail
- )
- )
- PY
- python -m pytest -q --tb=short tests/_runtime_patch_check_shim.py -s
- rm -f tests/_runtime_patch_check_shim.py
-
- - name: Runtime checks — patch_tiled_mlp on a synthetic MLP module (via pytest shim)
- # Same shim pattern: pytest picks up tests/conftest.py before importing
- # unsloth_zoo.tiled_mlp, so the GPU-spoof harness covers
- # unsloth_zoo.temporary_patches.gpt_oss's mem_get_info call.
- run: |
- set -euxo pipefail
- cat > tests/_tiled_mlp_check_shim.py <<'PY'
- # Auto-generated by .github/workflows/consolidated-tests-ci.yml.
- import sys, pathlib
- sys.path.insert(0, str(pathlib.Path(__file__).parent))
- import _zoo_aggressive_cuda_spoof as _spoof
- _spoof.apply()
- import torch
- import torch.nn as nn
- from unsloth_zoo.tiled_mlp import patch_tiled_mlp, patch_mlp
-
- class _MLP(nn.Module):
- def __init__(self, hidden=64, intermediate=128):
- super().__init__()
- self.gate_proj = nn.Linear(hidden, intermediate, bias=False)
- self.up_proj = nn.Linear(hidden, intermediate, bias=False)
- self.down_proj = nn.Linear(intermediate, hidden, bias=False)
- self.act_fn = nn.SiLU()
- def forward(self, x):
- return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
-
- class _FakeModel(nn.Module):
- def __init__(self):
- super().__init__()
- self.layers = nn.ModuleList([nn.ModuleDict({"mlp": _MLP()}) for _ in range(2)])
- def forward(self, x):
- for layer in self.layers:
- x = x + layer["mlp"](x)
- return x
-
- def test_patch_tiled_mlp_numerical_equivalence():
- # `patch_mlp(target_arctic=True)` sets `chunk_size = max(1, H)`
- # and shards the SEQUENCE dim with `n_shards = max(1, S //
- # chunk_size)`. Pick S > H so the tiled path actually runs
- # multi-shard (n_shards = 192 // 64 = 3, plus a remainder
- # shard) rather than degenerating to n_shards = 1 which is
- # bit-exact and only confirms patching installed something.
- # If the tiled implementation is correct, multi-shard output
- # must still match the un-tiled reference within FP32 noise.
- torch.manual_seed(0)
- m = _FakeModel().eval()
- hidden = 64
- # 192 = 3 * hidden, so divmod(192, 64) = (3, 0) -> 3 shards,
- # no remainder; gives a clean multi-shard verification.
- x = torch.randn(2, 192, hidden)
- with torch.no_grad():
- y_before = m(x).clone()
- patch_mlp(m.layers[0]["mlp"])
- patch_tiled_mlp(m)
- # Sanity-check we are actually exercising the multi-shard
- # path: poke chunk_size by re-deriving it the same way
- # `tiled_forward_arctic_size` does.
- S = x.shape[1]
- chunk = max(1, hidden)
- n_shards_expected = max(1, S // chunk)
- assert n_shards_expected > 1, (
- "tiled MLP shim is not exercising multi-shard: "
- f"S={S}, chunk={chunk}, n_shards={n_shards_expected}"
- )
- with torch.no_grad():
- y_after = m(x).clone()
- err = (y_before - y_after).abs().max().item()
- print(
- f"patch_tiled_mlp multi-shard (n_shards={n_shards_expected}) "
- f"output diff = {err:.3e}"
- )
- assert err < 1e-3, f"tiled MLP output drifted: {err}"
- PY
- python -m pytest -q --tb=short tests/_tiled_mlp_check_shim.py -s
- rm -f tests/_tiled_mlp_check_shim.py
-
- - name: Compiler cache hygiene + source-rewriter invariants (synthetic inputs)
- # Lightweight pipeline coverage for unsloth_zoo.compiler. Pure regex
- # / tokenize / ast paths driven by tiny synthetic source strings:
- # - higher_precision_softmax (basic + idempotent)
- # - fix_rotary_embedding_dtype (no-op + active under
- # UNSLOTH_FORCE_CUSTOM_DTYPE)
- # - fix_attention_dtype_consistency (insert + idempotent)
- # - convert_attention_masks_to_bool (rewrite + no-op)
- # - create_new_function happy-path (versioning block, license
- # header, AST parse, importlib re-import)
- # - create_new_function **kwargs collision (exercises
- # _rewrite_kwargs_param + _insert_kwargs_alias)
- # - UNSLOTH_COMPILE_OVERWRITE=0 forced-recompile on transformers
- # version mismatch (compiler.py:947-963)
- # - matching short-circuit when versions are equal
- # No real transformers modeling module is loaded; complements the
- # heavier real-class round-trip step below. Wall-time ~10-25s.
- run: |
- set -euxo pipefail
- cat > tests/_compiler_cache_invariants_shim.py <<'PY'
- # Auto-generated by .github/workflows/consolidated-tests-ci.yml.
- # Cache-hygiene + source-rewriter invariants for unsloth_zoo.compiler.
- import sys, pathlib, os, ast, importlib, importlib.util, time
- sys.path.insert(0, str(pathlib.Path(__file__).parent))
- import _zoo_aggressive_cuda_spoof as _spoof
- _spoof.apply()
- import pytest
- import torch # noqa: F401 (compiler.py imports torch at module load)
-
-
- def _isolate_cache(tmp_path, monkeypatch):
- """Point UNSLOTH_COMPILE_LOCATION at tmp_path and reset module
- globals. The compiler.py global is captured at module load
- (line 75/179), so we delete + reimport per test."""
- monkeypatch.setenv("UNSLOTH_COMPILE_LOCATION", str(tmp_path))
- if "unsloth_zoo.compiler" in sys.modules:
- del sys.modules["unsloth_zoo.compiler"]
- import unsloth_zoo.compiler as compiler
- compiler.UNSLOTH_COMPILE_LOCATION = str(tmp_path)
- compiler.UNSLOTH_COMPILE_USE_TEMP = False
- return compiler
-
-
- def test_higher_precision_softmax_basic_and_idempotent(tmp_path, monkeypatch):
- c = _isolate_cache(tmp_path, monkeypatch)
- src = (
- "y = nn.functional.softmax(x, dim=-1)\n"
- "z = F.softmax(a, dim=1, dtype=torch.bfloat16)\n"
- )
- out = c.higher_precision_softmax(src)
- assert "dtype = torch.float32).to(x.dtype)" in out
- assert "dtype = torch.float32).to(a.dtype)" in out
- # Idempotency landed in unslothai/unsloth-zoo#631
- # (negative-lookahead on `.to(.dtype)` so a second
- # pass does not append another cast).
- assert c.higher_precision_softmax(out) == out
-
-
- def test_fix_rotary_dtype_no_op_without_env(tmp_path, monkeypatch):
- c = _isolate_cache(tmp_path, monkeypatch)
- monkeypatch.delenv("UNSLOTH_FORCE_CUSTOM_DTYPE", raising=False)
- src = "out = cos.to(dtype=x.dtype) + sin.to(dtype=x.dtype)\n"
- assert c.fix_rotary_embedding_dtype(src) == src
-
-
- def test_fix_rotary_dtype_active(tmp_path, monkeypatch):
- c = _isolate_cache(tmp_path, monkeypatch)
- monkeypatch.setenv(
- "UNSLOTH_FORCE_CUSTOM_DTYPE",
- "float16;torch.float32;torch.bfloat16;torch.float16;pass",
- )
- monkeypatch.setenv("UNSLOTH_FORCE_FLOAT32", "1")
- src = "out = cos.to(dtype=x.dtype) + sin.to(dtype=x.dtype)\n"
- out = c.fix_rotary_embedding_dtype(src)
- # Active form rewrites cos.to / sin.to. Either the conditional
- # form or the cast form is acceptable -- different transformers
- # versions surface slightly different outputs from the rewriter.
- assert "cos.to(dtype=x.dtype)" not in out
- assert "sin.to(dtype=x.dtype)" not in out
-
-
- def test_fix_attention_dtype_consistency_insert_then_idempotent(tmp_path, monkeypatch):
- c = _isolate_cache(tmp_path, monkeypatch)
- src = (
- " query_states, key_states = apply_rotary_pos_emb("
- "query_states, key_states, cos, sin)\n"
- " attn = q @ k.T\n"
- )
- out = c.fix_attention_dtype_consistency(src)
- assert out.count("value_states = value_states.to(query_states.dtype)") == 1
- assert c.fix_attention_dtype_consistency(out) == out
-
-
- def test_convert_attention_masks_to_bool_rewrites(tmp_path, monkeypatch):
- c = _isolate_cache(tmp_path, monkeypatch)
- src = (
- "def make_mask(x):\n"
- " out = torch.finfo(x.dtype).min * x\n"
- " return out\n"
- )
- out = c.convert_attention_masks_to_bool("make_mask", src)
- # Loose match: rewriter inserts a `!=torch.finfo(...).min` check
- # somewhere on the return path. Tightening to an exact
- # last-line match is brittle across transformers versions.
- assert "!=torch.finfo" in out
-
-
- def test_convert_attention_masks_to_bool_no_op(tmp_path, monkeypatch):
- c = _isolate_cache(tmp_path, monkeypatch)
- src = "def make_mask(x):\n return x\n"
- assert c.convert_attention_masks_to_bool("make_mask", src) == src
-
-
- def _versioning_lines(file_text):
- """Extract the four version strings from the versioning block."""
- assert file_text.startswith('"""\n'), "missing opening triple-quote"
- head = file_text.split("__UNSLOTH_VERSIONING__", 1)[0]
- lines = [ln for ln in head.splitlines() if ln and ln != '"""']
- return lines
-
-
- def test_create_new_function_happy_path(tmp_path, monkeypatch):
- c = _isolate_cache(tmp_path, monkeypatch)
- src = "def f(x):\n return nn.functional.softmax(x, dim=-1)\n"
- c.create_new_function(
- name="f_happy", new_source=src, model_location="builtins",
- functions=[], overwrite=True,
- )
- cached = tmp_path / "f_happy.py"
- assert cached.exists()
- text = cached.read_text(encoding="utf-8")
- versions = _versioning_lines(text)
- assert len(versions) == 4, versions
- assert text.count(c._full_license_header) == 1
- ast.parse(text)
- spec = importlib.util.spec_from_file_location("f_happy_reimport", cached)
- m2 = importlib.util.module_from_spec(spec)
- spec.loader.exec_module(m2)
- assert callable(m2.f)
- import inspect as _inspect
- # higher_precision_softmax should have promoted to float32.
- assert "dtype = torch.float32" in _inspect.getsource(m2.f)
-
-
- def test_create_new_function_overwrite_zero_recompiles_on_version_mismatch(
- tmp_path, monkeypatch,
- ):
- c = _isolate_cache(tmp_path, monkeypatch)
- name = "vmismatch"
- cached = tmp_path / f"{name}.py"
- stub = (
- '"""\n0.0.0\n0.0.0\n0.0.0-stub\n0.0.0\n__UNSLOTH_VERSIONING__\n"""\n'
- + c._full_license_header
- + "def vmismatch(x):\n return x\n"
- )
- cached.write_text(stub, encoding="utf-8")
- monkeypatch.setenv("UNSLOTH_COMPILE_OVERWRITE", "0")
- src = "def vmismatch(x):\n return x + 1\n"
- c.create_new_function(
- name=name, new_source=src, model_location="builtins",
- functions=[], overwrite=False,
- )
- text = cached.read_text(encoding="utf-8")
- assert "0.0.0-stub" not in text, (
- "OVERWRITE=0 + transformers-version-mismatch did NOT recompile"
- )
- versions = _versioning_lines(text)
- import importlib.metadata as _md
- assert versions[2] == _md.version("transformers")
-
-
- def test_create_new_function_overwrite_zero_short_circuits_when_versions_match(
- tmp_path, monkeypatch,
- ):
- c = _isolate_cache(tmp_path, monkeypatch)
- name = "vmatch"
- src = "def vmatch(x):\n return x\n"
- c.create_new_function(
- name=name, new_source=src, model_location="builtins",
- functions=[], overwrite=True,
- )
- cached = tmp_path / f"{name}.py"
- mtime_before = cached.stat().st_mtime_ns
- time.sleep(0.05)
- monkeypatch.setenv("UNSLOTH_COMPILE_OVERWRITE", "0")
- c.create_new_function(
- name=name, new_source=src, model_location="builtins",
- functions=[], overwrite=False,
- )
- assert cached.stat().st_mtime_ns == mtime_before, (
- "OVERWRITE=0 + matching versions should NOT rewrite the file"
- )
- PY
- python -m pytest -q --tb=short tests/_compiler_cache_invariants_shim.py
- rm -f tests/_compiler_cache_invariants_shim.py
-
- - name: Compiler full-model-sweep (every transformers.models.*) + SFT trainer round-trip
- # Calls `unsloth_compile_transformers(model_type=...)` against EVERY
- # `transformers.models.` package the matrix's transformers ships
- # (pkgutil.iter_modules walk -- 383 packages on 4.57.6, similar on
- # latest), then ast.parse / importlib-load / introspect the
- # generated unsloth_compiled_cache/*.py file per model. Catches
- # regex / source-rewriter drift across the matrix's (transformers,
- # trl) combination -- the dominant failure mode of
- # `unsloth_compile_transformers` after a transformers point release.
- #
- # 21 model_types currently break the compiler (verified locally on
- # transformers 4.57.6). They are listed in KNOWN_BROKEN below with
- # their failure mode so the sweep stays green and any NEW breakage
- # surfaces as red. Each entry is tracked for an individual fix
- # PR on unsloth-zoo. The list is split by failure category so
- # follow-up PRs can target one bug at a time.
- #
- # Hermetic cache dir per pytest invocation; we override the
- # job-level UNSLOTH_COMPILE_DISABLE=1 inside the shim so
- # compilation actually runs here. Wall-time estimate ~2-3 min
- # warm (mean ~0.3s/model, 383 models = ~110s on the runner).
- run: |
- set -euxo pipefail
- cat > tests/_zoo_compiler_cache_shim.py <<'PY'
- # Auto-generated by .github/workflows/consolidated-tests-ci.yml.
- import os, sys, ast, pathlib, importlib.util, tempfile
- _HERE = pathlib.Path(__file__).parent
- sys.path.insert(0, str(_HERE))
- import _zoo_aggressive_cuda_spoof as _spoof
- _spoof.apply()
-
- # Hermetic cache dir + force compile path. The compiler's
- # globals (UNSLOTH_COMPILE_LOCATION, UNSLOTH_COMPILE_USE_TEMP)
- # are captured at module load; an earlier conftest `import
- # unsloth` may have already imported unsloth_zoo.compiler with
- # the default "unsloth_compiled_cache" path. Mutate the live
- # module globals after import so this shim is robust to that
- # ordering. Otherwise the compiler silently writes to the
- # default cache and the per-model file assertion fails.
- _CACHE = pathlib.Path(tempfile.mkdtemp(prefix="unsloth_cache_"))
- os.environ["UNSLOTH_COMPILE_LOCATION"] = str(_CACHE)
- os.environ["UNSLOTH_COMPILE_OVERWRITE"] = "1"
- os.environ.pop("UNSLOTH_COMPILE_DISABLE", None)
-
- import pytest
- import unsloth_zoo.compiler as _zoo_compiler
- _zoo_compiler.UNSLOTH_COMPILE_LOCATION = str(_CACHE)
- _zoo_compiler.UNSLOTH_COMPILE_USE_TEMP = False
- from unsloth_zoo.compiler import unsloth_compile_transformers
-
-
- def _verify_file(path: pathlib.Path, must_expose):
- assert path.exists(), f"compiler did not write {path}"
- src = path.read_text(encoding="utf-8")
- ast.parse(src, filename=str(path))
- spec = importlib.util.spec_from_file_location(path.stem, path)
- mod = importlib.util.module_from_spec(spec)
- spec.loader.exec_module(mod)
- for name in must_expose:
- assert hasattr(mod, name), (
- f"{path.name} missing expected attr {name!r}; "
- f"found: {sorted(n for n in dir(mod) if not n.startswith('_'))[:25]}"
- )
-
-
- # ---------- Full transformers.models.* compile sweep ----------
- # Track the model_types that currently break the compiler on
- # transformers >=5,<6. After unsloth-zoo#632 landed, transformers
- # 4.57.6 has zero failures across all model_types; the 27 entries
- # below are the residual failures on the tf 5.x line. New breakage
- # on any OTHER model_type fails the cell. Each entry is a
- # tracking item for a follow-up unsloth-zoo PR.
- KNOWN_BROKEN_COMPILE = {
- # Category A: `string index out of range` in source rewriter.
- "colpali": "string index out of range",
- "colqwen2": "string index out of range",
- "colmodernvbert": "string index out of range",
- "dpr": "string index out of range",
- "gemma4_assistant":"string index out of range",
- "rag": "string index out of range",
- "shieldgemma2": "string index out of range",
- "timm_backbone": "string index out of range",
- # Category B: rewriter emits invalid Python source.
- "clvp": "emitted file: unexpected indent",
- "falcon_mamba": "emitted file: unexpected indent",
- "gpt2": "emitted file: unexpected indent",
- "imagegpt": "emitted file: unexpected indent",
- "mamba": "emitted file: unexpected indent",
- "tapas": "emitted file: expected ':'",
- "xlstm": "emitted file: unexpected indent",
- # Category B-2: emit unterminated string literal (latest tf).
- "audioflamingo3": "emitted file: unterminated string literal",
- "musicflamingo": "emitted file: unterminated string literal",
- "voxtral": "emitted file: unterminated string literal",
- "voxtral_realtime":"emitted file: unterminated string literal",
- # Category C: rewriter emits unclosed paren.
- "kosmos2": "emitted file: '(' was never closed",
- "kosmos2_5": "emitted file: '(' was never closed",
- # Category D: imports list builder picks up a non-exported name.
- "auto": "module has no attribute _BaseModelWithGenerate",
- "bit": "module has no attribute Linear",
- "regnet": "module has no attribute Linear",
- "resnet": "module has no attribute Linear",
- # Category E: undefined name in emitted file.
- "perceiver": "name 'AbstractPreprocessor' is not defined",
- "sam3_lite_text": "name 'Sam3LiteTextLayerScaledResidual' is not defined",
- # Category F: compile exceeds 60s budget on the runner.
- # First seen on transformers >=5,<6; each represents a slow
- # or recursive source-rewriter path the zoo can address.
- "beit": "TimeoutError: compile exceeds per-model budget",
- "sam": "TimeoutError: compile exceeds per-model budget",
- "sam_hq": "TimeoutError: compile exceeds per-model budget",
- }
-
-
- def _all_model_types():
- import pkgutil, transformers.models as tm
- return sorted(s.name for s in pkgutil.iter_modules(tm.__path__) if s.ispkg)
-
-
- def test_compile_every_transformers_model_type():
- """Run unsloth_compile_transformers across every model_type
- the matrix's transformers ships. Allowed outcomes:
- ok -> compile emitted a parseable, importable cache file
- skipped -> no `modeling_.py` file (expected for some
- umbrella packages like `auto`, `deprecated`)
- known -> in KNOWN_BROKEN_COMPILE; tracked for follow-up.
- Any uncaught failure fails the cell.
-
- Per-model SIGALRM cap so one infinite-looping model_type
- cannot wedge the whole sweep + nuke the job timeout
- (observed on transformers >=5,<6 -- 30+ min hang before
- this guard landed)."""
- import importlib as _il
- import signal
- ok = 0
- skipped = []
- known = []
- new_failures = []
- models = _all_model_types()
- def _on_timeout(signum, frame):
- raise TimeoutError("compile exceeded per-model budget")
- prev_handler = signal.signal(signal.SIGALRM, _on_timeout)
- try:
- for i, model_type in enumerate(models):
- if i % 25 == 0:
- print(f" sweep progress: {i}/{len(models)} -> {model_type}", flush=True)
- modeling_path = f"transformers.models.{model_type}.modeling_{model_type}"
- try:
- _il.import_module(modeling_path)
- except (ModuleNotFoundError, ImportError):
- skipped.append((model_type, "no modeling file"))
- continue
- signal.alarm(60)
- try:
- unsloth_compile_transformers(
- model_type=model_type, fast_lora_forwards=False,
- )
- except Exception as e:
- signal.alarm(0)
- msg = f"{type(e).__name__}: {str(e)[:200]}"
- if model_type in KNOWN_BROKEN_COMPILE:
- known.append((model_type, msg))
- else:
- new_failures.append((model_type, msg))
- continue
- signal.alarm(0)
- if model_type in KNOWN_BROKEN_COMPILE:
- # Came back green unexpectedly -- that's GOOD news,
- # the bug was fixed. Surface it so we can drop the
- # entry from KNOWN_BROKEN_COMPILE.
- print(
- f" UNEXPECTED-OK {model_type}: was in "
- "KNOWN_BROKEN_COMPILE, now compiles cleanly. "
- "Drop the entry."
- )
- ok += 1
- finally:
- signal.alarm(0)
- signal.signal(signal.SIGALRM, prev_handler)
- print(f"\nCompile sweep: ok={ok} skipped={len(skipped)} "
- f"known-broken={len(known)} new-failures={len(new_failures)}")
- for m, r in known:
- print(f" KNOWN {m}: {r}")
- for m, r in new_failures[:30]:
- print(f" NEW {m}: {r}")
- if len(new_failures) > 30:
- print(f" ...and {len(new_failures)-30} more new failures")
- assert not new_failures, (
- f"unsloth_compile_transformers introduced new failures on "
- f"{len(new_failures)} model_types not in the known-broken "
- f"list: {[m for m, _ in new_failures]}"
- )
- # Sanity floor: at least 200 model_types should compile cleanly
- # (we observed 362 ok / 383 total on transformers 4.57.6).
- assert ok >= 200, (
- f"only {ok} model_types compiled cleanly; expected >=200. "
- "Possible transformers-version-induced regression."
- )
-
-
- @pytest.mark.parametrize("model_type,rms_class", [
- ("llama", "LlamaRMSNorm"),
- ("qwen3", "Qwen3RMSNorm"),
- ("gemma3", "Gemma3RMSNorm"),
- ])
- def test_compile_real_modeling_module(model_type, rms_class):
- """Spot-check on the three production-relevant families that
- the compile_every sweep also covers; this case verifies the
- emitted cache file has the model-specific RMSNorm class
- attribute, not just that the file parses + imports.
-
- ``unsloth_compile_transformers`` is not idempotent in-
- process: calling it twice on the same modeling module
- after rewriting class attributes corrupts the inspect
- source/line cache and the second emitted file is malformed
- Python. The sweep above already produced a valid cache
- file for every non-KNOWN_BROKEN model_type, so just verify
- that artefact here. Trigger a compile only when running
- this test in isolation (no sweep preceded)."""
- import importlib as _il
- try:
- modeling = _il.import_module(
- f"transformers.models.{model_type}.modeling_{model_type}"
- )
- except ModuleNotFoundError:
- pytest.skip(
- f"transformers build lacks model_type={model_type}"
- )
- combined = _CACHE / f"unsloth_compiled_module_{model_type}.py"
- if not combined.exists():
- unsloth_compile_transformers(
- model_type=model_type, fast_lora_forwards=False,
- )
- modeling = _il.import_module(
- f"transformers.models.{model_type}.modeling_{model_type}"
- )
- assert getattr(modeling, "__UNSLOTH_PATCHED__", False) is True
- _verify_file(combined, must_expose=[rms_class])
-
-
- def test_compile_disable_writes_nothing():
- """Negative control: when UNSLOTH_COMPILE_DISABLE=1 the
- compile path must early-return without producing new files."""
- os.environ["UNSLOTH_COMPILE_DISABLE"] = "1"
- try:
- before = set(_CACHE.iterdir())
- # Pick a model_type that still resolves on this transformers.
- for mt in ("llama", "mistral", "qwen2"):
- try:
- import importlib as _il
- _il.import_module(
- f"transformers.models.{mt}.modeling_{mt}"
- )
- break
- except ModuleNotFoundError:
- continue
- else:
- pytest.skip("no probe model_type available")
- unsloth_compile_transformers(
- model_type=mt, fast_lora_forwards=False,
- )
- after = set(_CACHE.iterdir())
- assert after == before, (
- f"DISABLE=1 still wrote: {[p.name for p in after - before]}"
- )
- finally:
- os.environ.pop("UNSLOTH_COMPILE_DISABLE", None)
-
-
- def test_compile_sft_trainer_patch():
- """Round-trip TRL's SFTTrainer through the rl.py patch path
- and verify the generated UnslothSFTTrainer.py."""
- pytest.importorskip("trl")
- try:
- from unsloth.models.rl import _patch_trl_rl_trainers
- except ImportError:
- pytest.skip("unsloth.models.rl._patch_trl_rl_trainers absent")
- try:
- _patch_trl_rl_trainers("sft_trainer")
- except Exception as e:
- # TRL 1.x renames break the patch helper internally; we
- # accept that here and skip rather than fail the cell.
- pytest.skip(f"_patch_trl_rl_trainers raised: {type(e).__name__}: {e}")
- sft = _CACHE / "UnslothSFTTrainer.py"
- if not sft.exists():
- pytest.skip(
- "_patch_trl_rl_trainers ran but did not emit "
- "UnslothSFTTrainer.py on this TRL version."
- )
- _verify_file(sft, must_expose=["UnslothSFTTrainer"])
- PY
- python -m pytest -q --tb=short tests/_zoo_compiler_cache_shim.py
- rm -f tests/_zoo_compiler_cache_shim.py
-
- - name: TRL trainer + Config auto-discovery + dynamic patch coverage
- # Mirror unsloth/models/rl.py:patch_trl_rl_trainers AND verify the
- # dynamic per-version patch surface:
- # 1. AST-parse every *_trainer / *_config submodule.
- # 2. Apply the same *Trainer / *Config discovery rules
- # _patch_trl_rl_trainers uses (rl.py:553-620).
- # 3. Orphan check: every _trainer must have a sibling
- # _config OR an inline *Config.
- # 4. Dynamic count: enumerate every canonical trainer that
- # imports cleanly, run patch_trl_rl_trainers(), assert
- # every one ends up Unsloth-prefixed in-place. Floor matches
- # the cohort sizes from the version sweep:
- # TRL 0.22-0.23 -> 18 canonical trainers
- # TRL 0.24-0.28 -> 15 canonical trainers
- # TRL 0.29-1.x -> 6 canonical (rest are experimental
- # thin-wrappers; covered next)
- # 5. Experimental coverage (TRL 0.29+): walk trl.experimental.*,
- # find every *Trainer class, verify the umbrella patch
- # reaches them via the thin-wrapper MRO walk in
- # _patch_trl_rl_trainers (rl.py:677-702).
- # Per-cell wall-time ~30-60s.
- run: |
- set -euxo pipefail
- cat > tests/_trl_trainer_discovery_shim.py <<'PY'
- # Auto-generated by .github/workflows/consolidated-tests-ci.yml.
- # Walks every *_trainer / *_config module in trl.trainer and
- # validates that unsloth's auto-discovery rules in
- # unsloth/models/rl.py:_patch_trl_rl_trainers (lines 542-620,
- # 1934-1949) still pick out exactly one *Trainer and one
- # *Config per module on the matrix's TRL version.
- import sys, pathlib, importlib, importlib.util, ast, inspect
-
- sys.path.insert(0, str(pathlib.Path(__file__).parent))
- import _zoo_aggressive_cuda_spoof as _spoof
- _spoof.apply()
-
- import pytest
- pytest.importorskip("trl")
- import trl # noqa: F401 (forces lazy-module init)
- import trl.trainer
-
-
- def _is_real_submodule(qual_name: str) -> bool:
- """True iff `qual_name` resolves to an importable submodule
- with a file on disk (i.e. has a non-None find_spec().origin).
-
- TRL re-exports utility FUNCTIONS into `trl.trainer.__init__`
- whose names happen to end with `_config` (e.g.
- `get_peft_config`, `get_quantization_config`). Without this
- filter the `endswith` check below picks them up as if they
- were submodules and the AST stage fails on `no spec`. The
- same trap exists for `_trainer` (none today, but defensive).
- """
- try:
- spec = importlib.util.find_spec(qual_name)
- except (ImportError, ValueError):
- return False
- return spec is not None and bool(getattr(spec, "origin", None))
-
-
- # Replicate rl.py:1939-1943 verbatim, then filter to actual
- # submodules so re-exported utility functions (e.g.
- # `get_peft_config`) do not pollute the AST sweep.
- def _trainer_files():
- return [
- x for x in dir(trl.trainer)
- if x.islower()
- and x.endswith("_trainer")
- and x != "base_trainer"
- and _is_real_submodule(f"trl.trainer.{x}")
- ]
-
-
- def _config_files():
- return [
- x for x in dir(trl.trainer)
- if x.islower()
- and x.endswith("_config")
- and _is_real_submodule(f"trl.trainer.{x}")
- ]
-
-
- def _ast_parse_module_via_spec(qual_name: str):
- """AST-parse a module's source on disk WITHOUT importing it.
- `trl.trainer` uses _LazyModule so `find_spec` resolves the
- file path without firing the module-level `__init__`. This
- dodges optional-dep ImportErrors (e.g. grpo_trainer's vllm
- import) and still surfaces real syntax drift in the file."""
- spec = importlib.util.find_spec(qual_name)
- if spec is None or not spec.origin:
- return None, "no spec"
- path = pathlib.Path(spec.origin)
- if not path.is_file():
- return None, f"spec.origin not a file: {path}"
- src = path.read_text(encoding="utf-8")
- ast.parse(src, filename=str(path))
- return path, None
-
-
- def test_every_trl_trainer_and_config_module_ast_parses():
- """Stage 1: pure file-on-disk AST parse. Catches a TRL
- source-level syntax issue on any matrix cell without
- triggering optional-dep imports."""
- fail = []
- ok = 0
- for name in _trainer_files() + _config_files():
- qual = f"trl.trainer.{name}"
- try:
- path, err = _ast_parse_module_via_spec(qual)
- if err:
- fail.append((qual, err))
- else:
- ok += 1
- except SyntaxError as e:
- fail.append((qual, f"SyntaxError: {e}"))
- except Exception as e:
- fail.append((qual, f"{type(e).__name__}: {e}"))
- print(f"AST-parsed {ok} TRL trainer+config modules; failed={len(fail)}")
- for q, e in fail:
- print(f" AST FAIL {q}: {e}")
- assert not fail, f"AST parse failed for {len(fail)} TRL modules"
-
-
- def _apply_unsloth_discovery_rules(mod, trainer_file):
- """Replicate the four endswith filters in
- rl.py:553-569 verbatim."""
- prefix = trainer_file.split("_")[0]
- names = [
- x for x in dir(mod)
- if x.endswith("Trainer") and x != "Trainer"
- and not x.startswith("_") and prefix in x.lower()
- ]
- configs = [
- x for x in dir(mod)
- if x.endswith("Config") and x != "Config"
- and not x.startswith("_") and prefix in x.lower()
- ]
- return names, configs
-
-
- def _resolve_config_via_fallbacks(trainer_file, name_list, mod):
- """Replicate rl.py:575-615: try the sibling *_config.py
- module, then the MRO walk fallback. Returns the resolved
- config-name list (length 0 or 1)."""
- # Fallback 1: _config.py module sibling.
- cfg_module_name = trainer_file.replace("_trainer", "_config")
- try:
- cfg_mod = getattr(trl.trainer, cfg_module_name)
- except Exception:
- cfg_mod = None
- if cfg_mod is not None:
- prefix = trainer_file.split("_")[0]
- hits = [
- x for x in dir(cfg_mod)
- if x.endswith("Config") and x != "Config"
- and not x.startswith("_") and prefix in x.lower()
- ]
- if len(hits) == 1:
- return hits
- # Fallback 2: MRO walk into experimental parent module.
- if len(name_list) != 1:
- return []
- try:
- trainer_cls = getattr(mod, name_list[0])
- except Exception:
- return []
- prefix = trainer_file.split("_")[0]
- for parent in trainer_cls.__mro__[1:]:
- if parent is object:
- continue
- parent_mod = inspect.getmodule(parent)
- if parent_mod is None:
- continue
- if parent_mod.__name__ == f"trl.trainer.{trainer_file}":
- continue
- hits = [
- x for x in dir(parent_mod)
- if x.endswith("Config") and x != "Config"
- and not x.startswith("_") and prefix in x.lower()
- ]
- if len(hits) == 1:
- return hits
- return []
-
-
- def test_unsloth_auto_discovery_finds_trainer_and_config_per_module():
- """Stage 2: drive the same unsloth rules over every trainer
- file. import-failures (optional deps) are recorded as
- `import-skipped`, mirroring rl.py:1944-1948 try/except."""
- ok = 0
- import_skipped = []
- discovery_skipped = []
- fail = []
- for trainer_file in _trainer_files():
- qual = f"trl.trainer.{trainer_file}"
- try:
- mod = getattr(trl.trainer, trainer_file)
- except Exception as e:
- import_skipped.append((qual, f"{type(e).__name__}: {e}"))
- continue
- trainers, configs = _apply_unsloth_discovery_rules(
- mod, trainer_file,
- )
- if len(trainers) != 1:
- discovery_skipped.append(
- (qual, f"trainers={trainers}")
- )
- continue
- if len(configs) != 1:
- configs = _resolve_config_via_fallbacks(
- trainer_file, trainers, mod,
- )
- if len(configs) != 1:
- fail.append(
- (qual,
- f"trainer={trainers[0]} but config not found "
- "(checked module, *_config sibling, and MRO)")
- )
- continue
- ok += 1
- print(f" OK {qual}: trainer={trainers[0]}, config={configs[0]}")
- print(
- f"\nDiscovery: ok={ok} import_skipped={len(import_skipped)} "
- f"discovery_skipped={len(discovery_skipped)} fail={len(fail)}"
- )
- for q, r in import_skipped:
- print(f" IMPORT-SKIP {q}: {r}")
- for q, r in discovery_skipped:
- print(f" DISC-SKIP {q}: {r}")
- for q, r in fail:
- print(f" FAIL {q}: {r}")
- # Hard contract: every TRAINER that imports cleanly AND has
- # exactly one *Trainer must also resolve exactly one *Config
- # via one of the three rules. import-skipped + discovery-
- # skipped (no/multiple *Trainer) are tolerated.
- assert not fail, (
- f"unsloth discovery rules failed for {len(fail)} trainers"
- )
- # Sanity: at least 3 trainers should fully discover on any
- # matrix cell (sft + reward + dpo are the historical core).
- assert ok >= 3, (
- f"only {ok} trainers fully discovered; expected >=3 "
- "(sft/reward/dpo). Possible TRL surface regression."
- )
-
-
- def test_orphan_trainer_modules_do_not_exist():
- """Stage 3: every _trainer module should have a sibling
- _config (TRL 0.26+ convention) OR an inline *Config. An
- ORPHAN _trainer with neither is a TRL refactor we want
- to know about: it would silently break unsloth's
- auto-discovery without raising."""
- orphans = []
- for trainer_file in _trainer_files():
- cfg_module_name = trainer_file.replace("_trainer", "_config")
- has_sibling_cfg = (
- importlib.util.find_spec(
- f"trl.trainer.{cfg_module_name}"
- ) is not None
- )
- if has_sibling_cfg:
- continue
- # No sibling -> require an inline *Config in the
- # trainer module itself (resolved via discovery rules).
- try:
- mod = getattr(trl.trainer, trainer_file)
- except Exception:
- # Optional-dep failure -> skip; the AST-parse stage
- # already covered the file.
- continue
- _, configs = _apply_unsloth_discovery_rules(
- mod, trainer_file,
- )
- if not configs:
- orphans.append(trainer_file)
- assert not orphans, (
- "Orphan TRL trainer modules with neither sibling "
- f"_config.py nor an inline *Config: {orphans}. "
- "unsloth auto-discovery would silently skip these."
- )
-
-
- # ---- Dynamic patch coverage: count + verify Unsloth-prefixed ----
-
- def _enumerate_canonical_trainer_classes():
- """Walk trl.trainer/*_trainer.py on disk (the source of
- truth for what `dir(trl.trainer)` should expose) and return
- [(trainer_file, TrainerClass), ...] for every entry that
- imports + has exactly-one resolvable *Trainer per the
- unsloth rules. Skips optional-dep ImportErrors."""
- out = []
- for trainer_file in _trainer_files():
- try:
- mod = getattr(trl.trainer, trainer_file)
- except Exception:
- continue
- trainers, _ = _apply_unsloth_discovery_rules(mod, trainer_file)
- if len(trainers) != 1:
- continue
- try:
- cls = getattr(mod, trainers[0])
- except Exception:
- continue
- out.append((trainer_file, cls))
- return out
-
-
- def _enumerate_experimental_trainer_packages():
- """TRL 0.29+ moved many trainers (bco, cpo, gkd, nash_md,
- online_dpo, orpo, ppo, prm, xpo, ...) to `trl.experimental.`,
- re-exposing them via thin-wrapper deprecation shims in
- `trl.trainer._trainer`. List every `trl.experimental.`
- that defines at least one *Trainer class, parsed by AST so we
- do NOT trigger the optional-dep imports on the package init."""
- spec = importlib.util.find_spec("trl.experimental")
- if spec is None or not spec.submodule_search_locations:
- return []
- import re as _re
- hits = []
- for root in spec.submodule_search_locations:
- rp = pathlib.Path(root)
- for sub in sorted(rp.iterdir()):
- if not sub.is_dir() or sub.name.startswith("_"):
- continue
- classes = []
- for py in sub.rglob("*.py"):
- try:
- src = py.read_text(encoding="utf-8")
- except Exception:
- continue
- for m in _re.finditer(
- r"^class\s+([A-Za-z0-9_]+Trainer)\b", src, _re.M,
- ):
- classes.append(m.group(1))
- if classes:
- hits.append((sub.name, sorted(set(classes))))
- return hits
-
-
- def _is_unsloth_patched(cls) -> bool:
- return getattr(cls, "__name__", "").startswith("Unsloth")
-
-
- def test_unsloth_patches_every_canonical_trainer_in_this_trl_version():
- """Verify the count + identity of canonically-patched trainers
- matches the trainer surface this TRL version actually ships.
-
- For TRL 0.22.x-0.23.x: ~18 canonical trainers expected.
- For TRL 0.24.x-0.28.x: ~15 canonical trainers expected.
- For TRL 0.29.x-1.x: 6 canonical (rest are experimental
- thin-wrappers; covered by the next test)."""
- from unsloth.models.rl import patch_trl_rl_trainers
- before = _enumerate_canonical_trainer_classes()
- before_count = len(before)
- before_unpatched = [
- (tf, cls.__name__) for tf, cls in before
- if not _is_unsloth_patched(cls)
- ]
- # Apply unsloth's umbrella patch.
- patch_trl_rl_trainers()
- # Re-enumerate (some classes may have been replaced in-module).
- after = _enumerate_canonical_trainer_classes()
- after_count = len(after)
- patched = [(tf, cls.__name__) for tf, cls in after
- if _is_unsloth_patched(cls)]
- unpatched = [(tf, cls.__name__) for tf, cls in after
- if not _is_unsloth_patched(cls)]
- print(
- f"\nCanonical trainer surface for TRL {trl.__version__}: "
- f"discoverable_before={before_count} "
- f"discoverable_after={after_count} "
- f"patched={len(patched)} unpatched={len(unpatched)}"
- )
- for tf, n in patched:
- print(f" PATCHED {tf}: {n}")
- for tf, n in unpatched:
- print(f" UNPATCHED {tf}: {n}")
- # Hard contract: every canonical trainer that imports
- # cleanly must end up Unsloth-prefixed after the umbrella
- # patch. If a trainer was discoverable BEFORE the patch but
- # is missing from `after`, that is a separate (rare) issue
- # we surface as failure.
- assert before_count == after_count, (
- f"trainer-class set changed across patching: "
- f"before={[n for _, n in before_unpatched]} "
- f"after={[n for _, n in unpatched]}"
- )
- assert not unpatched, (
- "unsloth.models.rl.patch_trl_rl_trainers did NOT patch: "
- + ", ".join(f"{tf}:{n}" for tf, n in unpatched)
- )
- # Floor matches the cohort sizes from the TRL version sweep:
- # 18 (0.22-0.23), 15 (0.24-0.28), 6 (0.29+ canonical only).
- assert len(patched) >= 6, (
- f"only {len(patched)} canonical trainers patched; "
- "expected >= 6 (the smallest production cohort)."
- )
-
-
- def test_unsloth_patches_experimental_trainers_via_thin_wrappers():
- """TRL 0.29+ ships canonical-`trl.trainer._trainer` modules
- for many trainers as deprecation thin-wrappers that forward
- to `trl.experimental.`. unsloth's
- `_patch_trl_rl_trainers` (rl.py:677-702) detects
- `trl.experimental` in the trainer source and resolves to
- the parent class -- so patching the canonical entry should
- also Unsloth-prefix the experimental class via in-module
- setattr.
-
- Verify by walking trl.experimental.* AST for every *Trainer
- class, then checking whether it (or any class with the same
- name in the experimental package) carries the Unsloth
- prefix after the umbrella patch."""
- from unsloth.models.rl import patch_trl_rl_trainers
- patch_trl_rl_trainers()
- experimental_pkgs = _enumerate_experimental_trainer_packages()
- if not experimental_pkgs:
- pytest.skip(
- f"TRL {trl.__version__} has no trl.experimental.* "
- "trainer surface (pre-0.29 cohort). The canonical "
- "test above already covers patching here."
- )
- found = []
- missing = []
- for pkg_name, class_names in experimental_pkgs:
- qual = f"trl.experimental.{pkg_name}"
- try:
- pkg_mod = importlib.import_module(qual)
- except Exception as e:
- # Optional-dep ImportError: experimental package
- # could not be loaded. Match unsloth's runtime
- # tolerance: this would also be silently skipped
- # by `_patch_trl_rl_trainers`. Record but do not
- # fail.
- print(
- f" IMPORT-SKIP {qual}: "
- f"{type(e).__name__}: {str(e)[:120]}"
- )
- continue
- for cls_name in class_names:
- cls = getattr(pkg_mod, cls_name, None)
- if cls is None:
- # Class is defined inside the package but not
- # re-exported on the package init. Walk
- # submodules to find it.
- import pkgutil as _pku
- for sub in _pku.walk_packages(
- pkg_mod.__path__, prefix=qual + "."
- ):
- try:
- sub_mod = importlib.import_module(sub.name)
- except Exception:
- continue
- cls = getattr(sub_mod, cls_name, None)
- if cls is not None:
- break
- if cls is None:
- missing.append((pkg_name, cls_name))
- continue
- if _is_unsloth_patched(cls):
- found.append((pkg_name, cls_name))
- print(f" PATCHED trl.experimental.{pkg_name}.{cls_name}")
- else:
- # Not Unsloth-prefixed: either unsloth chose
- # not to patch this surface (e.g. the canonical
- # thin-wrapper module did not exist) or the
- # patch silently failed. Record both
- # outcomes; the assertion below tolerates the
- # gap as informational, not failure -- the
- # canonical test enforces the hard contract.
- print(
- f" NOT-PATCHED trl.experimental.{pkg_name}."
- f"{cls_name} (no Unsloth-prefix on the "
- "experimental surface)"
- )
- total_experimental = sum(len(cs) for _, cs in experimental_pkgs)
- print(
- f"\nExperimental trainer surface (TRL {trl.__version__}): "
- f"{len(experimental_pkgs)} packages, "
- f"{total_experimental} *Trainer classes; "
- f"unsloth-patched={len(found)} class-missing={len(missing)}"
- )
- # Hard contract: a *Trainer class declared in a python
- # source file must be locatable in its package after import.
- # If we saw the class definition but cannot find the symbol
- # at runtime, the package's public surface drifted.
- assert not missing, (
- "experimental *Trainer classes declared in source but "
- f"not importable: {missing}"
- )
- PY
- python -m pytest -q --tb=short -s tests/_trl_trainer_discovery_shim.py
- rm -f tests/_trl_trainer_discovery_shim.py
-
- - name: MoE per-family coverage + GRPO patches + grouped_gemm AST
- # Catches the recurring class of bugs that PR #624 (gemma4 missing
- # extractor), PR #612 (gemma4 GRPO patch silently dropped), PR #607
- # (gate_up LoRA dropped from grad graph), PR #601 (qwen MoE shape
- # mismatch), unsloth#4934 (TRL disable_gradient_checkpointing
- # corrupts unsloth GC), and unsloth#3598 (gradient_accumulation
- # double-scale on accepts_loss_kwargs=False) targeted. Coverage:
- #
- # 1. Per-MoE-family side-effect contract: for every patch_*_moe
- # function in unsloth_zoo.temporary_patches, if its target
- # transformers class is importable on this matrix cell, the
- # patch must mark the class with `_unsloth_already_patched=True`
- # after running. This is exactly what unsloth_zoo's existing
- # test_moe_lora_extractor_coverage walks at the registration
- # level; here we tie each patch fn to its declared target so a
- # silent early-return (PR #612 style) surfaces as red rather
- # than a coverage skip.
- #
- # 2. PR #4934 (GRPO + TRL 1.0): patch_trl_disable_gradient_checkpointing
- # must rebind trl.models.utils.disable_gradient_checkpointing to
- # the unsloth no-op AND propagate the rebinding to every trl.*
- # module that imported the symbol by reference.
- #
- # 3. PR #3598 (gradient_accumulation): patch_gradient_accumulation_fix
- # must run cleanly on a synthetic Trainer whose training_step
- # signature carries `num_items_in_batch`. The original bug was
- # that `accepts_loss_kwargs=False` (Qwen3VL, Gemma3 in t-4.57)
- # caused double loss-scaling; here we verify the rewrite path
- # itself does not raise on a CPU-resolvable shape.
- #
- # 4. unsloth/kernels/moe/grouped_gemm AST smoke: the Triton kernels
- # are GPU-only at runtime, but a SyntaxError or stray
- # string-literal in the source still surfaces as a test-time
- # ImportError on every install. ast.parse the .py files without
- # executing.
- #
- # Wall-time per cell ~30-60s. Routed through pytest for the spoof
- # harness so unsloth_zoo.temporary_patches imports are clean.
- run: |
- set -euxo pipefail
- cat > tests/_moe_coverage_shim.py <<'PY'
- # Auto-generated by .github/workflows/consolidated-tests-ci.yml.
- import sys, pathlib, ast, importlib, importlib.util, contextlib, os
- sys.path.insert(0, str(pathlib.Path(__file__).parent))
- import _zoo_aggressive_cuda_spoof as _spoof
- _spoof.apply()
-
- import pytest
-
- # Map each MoE patch function to the transformers classes it is
- # contractually responsible for marking with _unsloth_already_patched
- # after a successful run. Sourced from
- # unsloth_zoo/temporary_patches/_moe.py:
- # - qwen3_moe.py:382-398 patches Qwen3MoeExperts (new path) or
- # Qwen3MoeSparseMoeBlock (old path).
- # - qwen3_5_moe.py + qwen3_next_moe.py + qwen3_vl_moe.py register
- # extractors on Qwen3_5MoeExperts / Qwen3NextExperts /
- # Qwen3VLMoeTextExperts respectively.
- # - gemma4_moe.py marks Gemma4TextExperts (current) or
- # Gemma4TextMoEBlock (legacy).
- # - glm4_moe.py marks Glm4MoeLiteNaiveMoe.
- # - deepseek_v3_moe.py marks DeepseekV3NaiveMoe.
- # - gpt_oss.py:patch_gpt_oss_moe_for_lora marks GptOssExperts.
- # Each cell skips a target if the transformers version lacks it
- # (legitimate version-skew); only patches with at least one
- # importable target are exercised.
- # Each entry = ((patch_module, patch_fn), targets, env_setup,
- # version_gate). env_setup runs before the patch fn (e.g. set
- # UNSLOTH_MODEL_NAME for gpt_oss). version_gate is a callable
- # returning True when the patch SHOULD run on this transformers;
- # if False, the test skips with a documented reason.
- def _v5_or_later():
- try:
- import transformers
- major = int(transformers.__version__.split(".")[0])
- return major >= 5
- except Exception:
- return False
-
- MOE_PATCHES = [
- {
- "module": "unsloth_zoo.temporary_patches.qwen3_moe",
- "fn": "patch_qwen3_moe",
- "targets": [
- ("transformers.models.qwen3_moe.modeling_qwen3_moe", "Qwen3MoeExperts"),
- ("transformers.models.qwen3_moe.modeling_qwen3_moe", "Qwen3MoeSparseMoeBlock"),
- ],
- "env": {},
- "gate": lambda: True,
- "gate_reason": "",
- },
- {
- "module": "unsloth_zoo.temporary_patches.qwen3_5_moe",
- "fn": "patch_qwen3_5_moe",
- "targets": [
- ("transformers.models.qwen3_5_moe.modeling_qwen3_5_moe", "Qwen3_5MoeExperts"),
- ],
- "env": {}, "gate": lambda: True, "gate_reason": "",
- },
- {
- "module": "unsloth_zoo.temporary_patches.qwen3_next_moe",
- "fn": "patch_qwen3_next_moe",
- "targets": [
- ("transformers.models.qwen3_next.modeling_qwen3_next", "Qwen3NextExperts"),
- ],
- "env": {}, "gate": lambda: True, "gate_reason": "",
- },
- {
- "module": "unsloth_zoo.temporary_patches.qwen3_vl_moe",
- "fn": "patch_qwen3_vl_moe",
- "targets": [
- ("transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe", "Qwen3VLMoeTextExperts"),
- ],
- "env": {}, "gate": lambda: True, "gate_reason": "",
- },
- {
- "module": "unsloth_zoo.temporary_patches.gemma4_moe",
- "fn": "patch_gemma4_moe",
- "targets": [
- ("transformers.models.gemma4.modeling_gemma4", "Gemma4TextExperts"),
- ],
- "env": {}, "gate": lambda: True, "gate_reason": "",
- },
- {
- "module": "unsloth_zoo.temporary_patches.glm4_moe",
- "fn": "patch_glm4_moe",
- "targets": [
- ("transformers.models.glm4_moe.modeling_glm4_moe", "Glm4MoeLiteNaiveMoe"),
- ],
- "env": {}, "gate": lambda: True, "gate_reason": "",
- },
- {
- "module": "unsloth_zoo.temporary_patches.deepseek_v3_moe",
- "fn": "patch_deepseek_v3_moe",
- "targets": [
- ("transformers.models.deepseek_v3.modeling_deepseek_v3", "DeepseekV3NaiveMoe"),
- ],
- "env": {}, "gate": lambda: True, "gate_reason": "",
- },
- {
- "module": "unsloth_zoo.temporary_patches.gpt_oss",
- "fn": "patch_gpt_oss_moe_for_lora",
- "targets": [
- ("transformers.models.gpt_oss.modeling_gpt_oss", "GptOssExperts"),
- ],
- # The patch reads UNSLOTH_MODEL_NAME and only runs when
- # "gpt_oss" is in the normalized form. Set it explicitly
- # so the gate at gpt_oss.py:1387 passes; otherwise the
- # patch silently early-returns and the test would
- # spuriously fail.
- "env": {"UNSLOTH_MODEL_NAME": "gpt_oss"},
- # Additionally only runs on transformers >= 5
- # (gpt_oss.py:1392 `_is_transformers_v5()` gate).
- "gate": _v5_or_later,
- "gate_reason": (
- "patch_gpt_oss_moe_for_lora gates on "
- "transformers >= 5 (split-LoRA grouped_mm path)"
- ),
- },
- ]
-
-
- def _resolve_target_classes(targets):
- """Return [(qual, cls), ...] for every importable target."""
- out = []
- for mod_path, cls_name in targets:
- try:
- mod = importlib.import_module(mod_path)
- except Exception:
- continue
- cls = getattr(mod, cls_name, None)
- if cls is None:
- continue
- out.append((f"{mod_path}.{cls_name}", cls))
- return out
-
-
- @pytest.mark.parametrize(
- "spec",
- MOE_PATCHES,
- ids=lambda s: s["fn"],
- )
- def test_moe_patch_marks_its_target_when_class_present(spec, monkeypatch):
- """If at least one target class is importable AND the
- version gate passes, run the patch fn and assert at least
- one target is marked patched afterwards. Skips when the
- transformers version lacks every target or when the
- version gate blocks the patch (legitimate). Fails on
- silent patch-fn early-returns (PR #612 class of bug)."""
- targets = spec["targets"]
- patch_module = spec["module"]
- patch_name = spec["fn"]
- importable = _resolve_target_classes(targets)
- if not importable:
- pytest.skip(
- f"{patch_name}: no target class importable on this "
- f"transformers (looked for {[c for _, c in targets]})."
- )
- if not spec["gate"]():
- pytest.skip(
- f"{patch_name}: version gate blocks this cell. "
- f"Reason: {spec['gate_reason']}"
- )
- for k, v in spec["env"].items():
- monkeypatch.setenv(k, v)
- try:
- pmod = importlib.import_module(patch_module)
- except Exception as e:
- pytest.skip(
- f"{patch_module} import failed (likely optional dep): "
- f"{type(e).__name__}: {e}"
- )
- fn = getattr(pmod, patch_name, None)
- if fn is None or not callable(fn):
- pytest.skip(f"{patch_module} has no callable {patch_name}")
- try:
- fn()
- except Exception as e:
- raise AssertionError(
- f"{patch_name}() raised on a transformers that "
- f"DOES ship at least one target class ({importable}). "
- f"This is the silent-failure mode PR #612 fixed: "
- f"{type(e).__name__}: {e}"
- )
- # At least one importable target must now carry SOME marker
- # showing unsloth touched it. Accepted signals (each is set
- # by a different patch flow in unsloth_zoo):
- # - `_unsloth_already_patched=True` (gemma4, deepseek_v3, glm4)
- # - `_unsloth_lora_patched=True` (gpt_oss_moe_for_lora)
- # - `_unsloth_lora_extractor_fn` is callable (qwen3_*, glm4_moe)
- # - `_original___forward` attr
- # (set by patch_function: qwen3_moe SparseMoeBlock, etc.)
- # - `_original_forward` attribute (gpt_oss in-place patch)
- # Accept any one as "patched".
- def _is_patched(cls) -> bool:
- if getattr(cls, "_unsloth_already_patched", False) is True:
- return True
- if getattr(cls, "_unsloth_lora_patched", False) is True:
- return True
- if callable(getattr(cls, "_unsloth_lora_extractor_fn", None)):
- return True
- if "_original_forward" in dir(cls):
- return True
- cls_name = cls.__name__
- for attr in dir(cls):
- if attr.startswith("_original_") and attr.endswith(
- f"_{cls_name}_forward"
- ):
- return True
- return False
-
- after = _resolve_target_classes(targets)
- marked = [qual for qual, cls in after if _is_patched(cls)]
- if not marked:
- raise AssertionError(
- f"{patch_name}() ran without exception but no target "
- f"in {importable} carries any of the unsloth markers "
- "(_unsloth_already_patched / _unsloth_lora_patched / "
- "_unsloth_lora_extractor_fn / _original_*_forward). "
- "Patch silently no-op'd (PR #612 class of bug)."
- )
- print(f" {patch_name}: marked {marked}")
-
-
- # ---- PR #4934 (TRL 1.0+ GRPO disable_gradient_checkpointing) ----
-
- def test_patch_trl_disable_gradient_checkpointing():
- """unsloth/models/rl.py:patch_trl_disable_gradient_checkpointing
- must rebind trl.models.utils.disable_gradient_checkpointing to
- the unsloth no-op when TRL >= 1.0. Pre-1.0 TRL has no such
- symbol -> the patch returns early."""
- try:
- import trl.models.utils as _tmu
- except ImportError:
- pytest.skip("trl not installed")
- had_symbol = hasattr(_tmu, "disable_gradient_checkpointing")
- try:
- from unsloth.models.rl import patch_trl_disable_gradient_checkpointing
- except ImportError:
- pytest.skip(
- "unsloth.models.rl.patch_trl_disable_gradient_checkpointing "
- "absent (older unsloth than #4934)"
- )
- patch_trl_disable_gradient_checkpointing()
- if not had_symbol:
- # Pre-1.0 TRL: patch is a no-op early-return. Verify
- # nothing broke.
- pytest.skip(
- "TRL pre-1.0 has no disable_gradient_checkpointing; "
- "patch correctly early-returned."
- )
- fn = getattr(_tmu, "disable_gradient_checkpointing", None)
- assert fn is not None, (
- "trl.models.utils.disable_gradient_checkpointing missing "
- "after patch -- patch removed the symbol entirely?"
- )
- assert getattr(fn, "_unsloth_noop_patched", False) is True, (
- "trl.models.utils.disable_gradient_checkpointing was NOT "
- "rebound to the unsloth no-op. PR #4934 regression."
- )
- # PR #4934 also walks sys.modules to rebind trl.* modules
- # that imported the symbol by reference. Verify at least the
- # canonical trainer modules picked up the rebinding when
- # they re-export it.
- import sys
- checked = 0
- missed = []
- for mod_name, mod in list(sys.modules.items()):
- if not mod_name.startswith("trl."):
- continue
- bound = getattr(mod, "disable_gradient_checkpointing", None)
- if bound is None:
- continue
- checked += 1
- if not getattr(bound, "_unsloth_noop_patched", False):
- missed.append(mod_name)
- print(f" rebound disable_gradient_checkpointing in {checked} trl.* modules")
- assert not missed, (
- "trl.* modules that imported disable_gradient_checkpointing "
- f"by reference but did not get rebound: {missed}"
- )
-
-
- # ---- PR #3598 (gradient_accumulation loss-scaling rewrite) ----
-
- def test_patch_gradient_accumulation_fix_runs_on_synthetic_trainer():
- """patch_gradient_accumulation_fix rewrites a Trainer's
- `training_step` source via inspect+exec when the signature
- carries `num_items_in_batch`. PR #3598 fixed the rewrite
- path to not double-scale for trainers with
- `accepts_loss_kwargs=False`. Verify the patch fn runs
- without raising on a synthetic Trainer carrying that
- signature."""
- try:
- from unsloth.models._utils import patch_gradient_accumulation_fix
- except ImportError:
- pytest.skip(
- "unsloth.models._utils.patch_gradient_accumulation_fix absent"
- )
- try:
- from transformers import Trainer
- except ImportError:
- pytest.skip("transformers.Trainer absent")
- # The patch reads the live Trainer.training_step source. We
- # exercise the standard transformers.Trainer here -- if the
- # bug is reintroduced in the source rewriter (e.g. broken
- # exec, missing import injection), the patch fn raises.
- try:
- patch_gradient_accumulation_fix(Trainer)
- except Exception as e:
- raise AssertionError(
- "patch_gradient_accumulation_fix raised on a vanilla "
- f"transformers.Trainer: {type(e).__name__}: {e}"
- )
- # Idempotency: second call must not raise either (the rewrite
- # adds `_unsloth_training_step` marker so the second call
- # short-circuits per _utils.py:1692-1693).
- patch_gradient_accumulation_fix(Trainer)
-
-
- # ---- unsloth/kernels/moe/grouped_gemm AST smoke ----
-
- def _walk_py_files(root: pathlib.Path):
- for p in root.rglob("*.py"):
- if "__pycache__" in p.parts:
- continue
- yield p
-
-
- def test_unsloth_kernels_moe_grouped_gemm_ast_parses():
- """unsloth/kernels/moe/grouped_gemm hosts the Triton MoE
- kernels (GPU-only at runtime). A SyntaxError or stray token
- at the SOURCE level still surfaces as ImportError on every
- install, so AST-parse the .py files without executing."""
- # Locate `unsloth/kernels/moe/grouped_gemm` via the installed
- # `unsloth` package.
- import unsloth as _unsloth
- kernel_root = (
- pathlib.Path(_unsloth.__file__).parent
- / "kernels" / "moe" / "grouped_gemm"
- )
- if not kernel_root.exists():
- pytest.skip(
- f"{kernel_root} not present in this unsloth checkout."
- )
- fail = []
- ok = 0
- for p in _walk_py_files(kernel_root):
- try:
- ast.parse(p.read_text(encoding="utf-8"), filename=str(p))
- ok += 1
- except SyntaxError as e:
- fail.append((str(p), f"SyntaxError: {e}"))
- except Exception as e:
- fail.append((str(p), f"{type(e).__name__}: {e}"))
- print(f"AST-parsed {ok} grouped_gemm files; failed={len(fail)}")
- for path, err in fail:
- print(f" AST FAIL {path}: {err}")
- assert not fail, (
- f"AST parse failed for {len(fail)} grouped_gemm files"
- )
- # Sanity: the directory MUST contain at least the interface
- # + kernels + reference subtrees as documented.
- expected = [
- "interface.py",
- "kernels/forward.py",
- "kernels/backward.py",
- "reference/moe_block.py",
- "reference/moe_ops.py",
- ]
- missing = [e for e in expected if not (kernel_root / e).is_file()]
- assert not missing, (
- "grouped_gemm directory layout regressed; missing: "
- f"{missing}"
- )
- PY
- python -m pytest -q --tb=short -s tests/_moe_coverage_shim.py
- rm -f tests/_moe_coverage_shim.py
-
- - name: Summary
- if: always()
- run: |
- echo "::group::Versions"
- python -c "import sys, platform; print(sys.version); print(platform.platform())"
- python -c "import torch; print('torch', torch.__version__, 'cuda?', torch.cuda.is_available())"
- python -c "import transformers; print('transformers', transformers.__version__)"
- # `pip show` instead of `import unsloth_zoo` — its __init__ raises
- # without an accelerator and the spoof harness only kicks in under
- # pytest. Cheap and accurate.
- pip show unsloth_zoo
- echo "::endgroup::"
- echo "Consolidated job done. Coverage:"
- echo " - 16 unsloth Bucket-A tests under tests/saving/ + tests/utils/"
- echo " - unsloth_zoo @ ${UNSLOTH_ZOO_REF} pytest tests/ (5 GPU cases deselected)"
- echo " - unsloth_zoo.compiler.test_apply_fused_lm_head"
-
- llama-cpp-smoke:
- # Standalone llama.cpp build + smoke. Earlier this lived inside every
- # consolidated matrix cell and re-cmake'd llama.cpp ~5 min per cell --
- # 3 cells x 275 s = ~14 min of duplicated CPU on every PR for an
- # artefact that has nothing to do with the (transformers, TRL) combo.
- # `install_llama_cpp` clones ggml-org/llama.cpp at a pinned commit and
- # builds the LLAMA_CPP_TARGETS list; the result is independent of the
- # HF stack version. Run once, gate the PR.
- name: llama.cpp build + smoke
- runs-on: ubuntu-latest
- timeout-minutes: 25
- env:
- UNSLOTH_ZOO_REF: ${{ inputs.unsloth_zoo_ref || 'main' }}
- # Same env contract the matrix cells use: protobuf python parser
- # (transformers' bundled *_pb2.py needs it), studio on PYTHONPATH,
- # compile-disable + UNSLOTH_IS_PRESENT so unsloth_zoo's __init__
- # bootstrap accepts a pure-import.
- PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
- PYTHONPATH: ${{ github.workspace }}/studio
- UNSLOTH_COMPILE_DISABLE: '1'
- UNSLOTH_IS_PRESENT: '1'
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- - name: Install runtime deps for unsloth_zoo.llama_cpp
- # unsloth_zoo's `__init__` imports `temporary_patches`, which
- # in turn pulls per-architecture submodules (gemma3n, gemma4,
- # qwen3_*_moe, glm4_moe, deepseek_v3_moe, pixtral, ministral,
- # mxfp4, bitsandbytes, flex_attention_bwd) -- many of those
- # transitively touch transformers and peft / accelerate. Mirror
- # the matrix job's install minus the heavy bits that have no
- # bearing on `install_llama_cpp` itself: studio.txt's FastAPI
- # stack, bitsandbytes (CUDA-only build dependency), triton,
- # mammoth/unpdf (PDF tools), datasets, sqlalchemy/cryptography,
- # pytest (we run no tests). The remaining pin shape matches
- # studio-backend-ci.yml's "Repo tests (CPU)" baseline.
- run: |
- set -euxo pipefail
- python -m pip install --upgrade pip
- # Match the matrix job's torch path so unsloth_zoo's
- # `import torch` resolves to the same CPU build.
- pip install --index-url https://download.pytorch.org/whl/cpu \
- 'torch>=2.4,<2.11' 'torchvision<0.26'
- pip install \
- 'numpy<3' protobuf sentencepiece \
- requests tqdm psutil packaging safetensors \
- 'peft>=0.18,<0.20' 'accelerate>=0.34,<2'
- # transformers + trl come from pyproject.toml's pinned line
- # so this job stays in sync with whatever the consolidated
- # `__from_pyproject__` matrix cell is using.
- pip install transformers trl
- pip install -e . --no-deps
-
- - name: Clone unsloth_zoo @ ${{ env.UNSLOTH_ZOO_REF }}
- # Same shallow clone as the matrix job; we install editable so
- # `unsloth_zoo.llama_cpp` resolves to the cloned tree (and any
- # main-branch fixes flow into the smoke without a release).
- run: |
- set -euxo pipefail
- # github.com occasionally 500s on the git fetch; retry so a
- # single upstream blip does not fail CI.
- for attempt in 1 2 3; do
- rm -rf "$RUNNER_TEMP/unsloth-zoo"
- if git clone --depth=1 --branch="$UNSLOTH_ZOO_REF" \
- https://github.com/unslothai/unsloth-zoo \
- "$RUNNER_TEMP/unsloth-zoo"; then
- break
- fi
- if [ "$attempt" -eq 3 ]; then
- echo "::error::git clone unsloth-zoo failed after 3 attempts"
- exit 1
- fi
- delay=$((5 * attempt))
- echo "::warning::clone failed (attempt $attempt/3), retrying in ${delay}s..."
- sleep "$delay"
- done
- pip install -e "$RUNNER_TEMP/unsloth-zoo" --no-deps
- pip show unsloth_zoo
-
- - name: llama.cpp install via unsloth_zoo.llama_cpp + `llama-cli --help` smoke
- # Exercise the canonical `unsloth_zoo.llama_cpp.install_llama_cpp`
- # flow that GGUF export uses at runtime: clone ggml-org/llama.cpp
- # into ~/.unsloth/llama.cpp, build the LLAMA_CPP_TARGETS list
- # (llama-quantize, llama-cli, llama-mtmd-cli, llama-gguf-split,
- # llama-server) via cmake, then run `llama-cli --help`.
- #
- # This replaces the previous "download upstream prebuilt zip"
- # approach, which silently exited 0 with the message
- # "no ubuntu-x64 prebuilt asset" when ggml-org's release-asset
- # naming drifted (the regex `bin-ubuntu-x64.*\.zip$` no longer
- # matched their current asset names). The build path is the same
- # one Unsloth users hit in production via `model.save_pretrained_gguf`.
- #
- # Wall-time budget: ~3-5 min cold, dominated by cmake build of
- # 5 targets on the runner's 4 cores. Apt-package install is
- # handled by `install_llama_cpp` itself via its
- # `check_build_requirements` -> `install_package` chain.
- run: |
- set -euxo pipefail
- # libssl-dev / libcurl4-openssl-dev are needed by llama.cpp's
- # cmake build for HTTPS support; install up-front so the
- # `install_llama_cpp` requirement-check is a no-op.
- sudo apt-get update -qq
- sudo apt-get install -y -qq build-essential cmake git curl \
- libgomp1 libssl-dev libcurl4-openssl-dev
- python <<'PY'
- import os, shutil, subprocess, sys, pathlib
- # Apply the same CPU spoof the pytest shims use BEFORE any
- # unsloth_zoo import: unsloth_zoo/__init__.py calls
- # device_type.get_device_type() at module load and raises
- # `NotImplementedError: Unsloth cannot find any torch
- # accelerator` on a GPU-less runner. The spoof flips
- # torch.cuda.is_available() to True so the device probe takes
- # the cuda branch; we never actually run CUDA tensor ops in
- # this step (just clone+cmake+--help on the binaries).
- sys.path.insert(0, str(pathlib.Path("tests").resolve()))
- import _zoo_aggressive_cuda_spoof as _spoof
- _spoof.apply()
- from unsloth_zoo.llama_cpp import (
- install_llama_cpp,
- LLAMA_CPP_DEFAULT_DIR,
- LLAMA_CPP_TARGETS,
- )
- print(f"Unsloth llama.cpp default dir: {LLAMA_CPP_DEFAULT_DIR}")
- print(f"Build targets: {LLAMA_CPP_TARGETS}")
- # install_llama_cpp returns (quantizer_path, converter_script_path).
- # The quantizer's directory is the `llama.cpp` install root, which
- # also holds llama-cli after build/bin/llama-* gets copied up
- # (llama_cpp.py:867-871).
- quantizer, converter = install_llama_cpp(print_output=True)
- assert quantizer and os.path.exists(quantizer), (
- f"install_llama_cpp returned quantizer={quantizer!r} but file missing"
- )
- assert converter and os.path.isfile(converter), (
- f"install_llama_cpp returned converter={converter!r} but missing"
- )
- install_root = os.path.dirname(quantizer)
- cli = os.path.join(install_root, "llama-cli")
- assert os.path.exists(cli), (
- f"llama-cli not found at {cli!r} after build. Build root contents: "
- f"{sorted(p for p in os.listdir(install_root) if p.startswith('llama-'))[:20]}"
- )
- assert os.access(cli, os.X_OK), f"{cli!r} not executable"
- # `llama-cli --help` exits non-zero on some builds; the contract
- # is that recognizable help text appears on stdout/stderr.
- proc = subprocess.run(
- [cli, "--help"], capture_output=True, text=True, timeout=30,
- )
- combined = (proc.stdout or "") + (proc.stderr or "")
- print("--- llama-cli --help (first 30 lines) ---")
- print("\n".join(combined.splitlines()[:30]))
- assert any(
- tok in combined.lower()
- for tok in ("usage", "--help", "--model", "-m,")
- ), (
- f"llama-cli --help produced no recognizable help text. "
- f"exit={proc.returncode}\nstdout: {proc.stdout[:400]!r}\n"
- f"stderr: {proc.stderr[:400]!r}"
- )
- # Also exercise the quantizer the way GGUF export does: --help
- # round-trip on the binary that does the actual heavy lifting.
- q = subprocess.run(
- [quantizer, "--help"], capture_output=True, text=True, timeout=15,
- )
- q_combined = (q.stdout or "") + (q.stderr or "")
- assert "usage" in q_combined.lower() or "type" in q_combined.lower(), (
- f"llama-quantize --help produced no help text. "
- f"exit={q.returncode}\nstdout: {q.stdout[:400]!r}\n"
- f"stderr: {q.stderr[:400]!r}"
- )
- print(
- f"\nOK: install_llama_cpp produced a working llama-cli at {cli} "
- f"and llama-quantize at {quantizer}."
- )
- PY
diff --git a/.github/workflows/lint-ci.yml b/.github/workflows/lint-ci.yml
deleted file mode 100644
index 00e6e357e2..0000000000
--- a/.github/workflows/lint-ci.yml
+++ /dev/null
@@ -1,321 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# Whole-repo, multi-language source-lint gate. Runs on every PR
-# (no path filter) because each step is sub-second to a few seconds
-# and together they catch a class of breakage the focused build
-# workflows would miss:
-#
-# - Python syntax + ruff + leftover debugger calls (across 350+
-# committed .py files, not just studio/backend).
-# - Shell `bash -n` parse for every committed *.sh.
-# - `yaml.safe_load` and `json.loads` round-trip for every
-# committed YAML / JSON config.
-#
-# TypeScript and Rust are NOT duplicated here on purpose:
-# - Studio Frontend CI runs `npm run typecheck` (= `tsc --noEmit`)
-# and `npm run build` (vite/swc) on every studio/frontend/**
-# change, which is a full TS AST + type check.
-# - Studio Tauri CI runs `tauri build --debug --no-bundle` on
-# every studio/src-tauri/** or studio/frontend/** change, which
-# compiles the Rust crate (= cargo check + cargo build).
-# Each is a stricter check than a parse-only step would be, so a
-# fast-fail duplicate here would only burn cache; the dedicated
-# workflows already block merges on Rust / TS regressions.
-
-name: Lint CI
-
-on:
- pull_request:
- push:
- branches: [main, pip]
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- source-lint:
- name: Source lint (Python + shell + YAML + JSON + safety nets)
- runs-on: ubuntu-latest
- timeout-minutes: 5
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- # Pin ruff to match .pre-commit-config.yaml so a CI-only ruff
- # bump cannot disagree with what pre-commit accepted.
- # codespell is pinned for the same reason: a reviewer should
- # never see a typo report appear and disappear depending on
- # which codespell version the runner happened to install.
- - run: pip install 'ruff==0.15.12' 'pyyaml>=6' 'codespell>=2.3,<3'
-
- - name: Linux deps for shellcheck
- run: sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends shellcheck
-
- - name: Python AST/syntax check (every committed .py must compile)
- # python -m compileall uses the same parser the interpreter
- # uses, so anything broken here would also crash at
- # `import X` on a user's machine. Sub-second across 350+
- # files. Hard gate.
- run: |
- python -m compileall -q -j 0 \
- unsloth unsloth_cli studio tests cli.py unsloth-cli.py
-
- - name: Python ruff check (whole repo)
- # The narrow rule set in pyproject.toml [tool.ruff.lint]
- # selects E9 / F63 / F7 / F82 -- syntax errors, broken
- # comparisons, undefined names. The whole repo passes today,
- # so this is a hard gate.
- run: |
- ruff check unsloth unsloth_cli studio tests cli.py unsloth-cli.py
-
- - name: No leftover debugger / pdb / breakpoint calls
- # Catches the "I'll just stick a breakpoint() here" mistake
- # before it ships. AST-based so commented-out debugger
- # markers don't false-positive (a bare grep would; there
- # are three commented `# breakpoint()` markers in
- # unsloth/models/rl* today). Sub-second.
- run: |
- python <<'PY'
- import ast, pathlib, sys
-
- SKIP_PARTS = {".venv", "venv", "build", "dist", ".git",
- "unsloth_compiled_cache", "node_modules",
- "unsloth.egg-info"}
-
- bad = []
- scanned = 0
- for path in sorted(pathlib.Path(".").rglob("*.py")):
- if any(part in SKIP_PARTS for part in path.parts):
- continue
- scanned += 1
- try:
- tree = ast.parse(path.read_text(encoding="utf-8", errors="replace"))
- except SyntaxError:
- continue # compileall step above already failed this
- for node in ast.walk(tree):
- if not isinstance(node, ast.Call):
- continue
- fn = node.func
- if isinstance(fn, ast.Name) and fn.id == "breakpoint":
- bad.append((path, node.lineno, "breakpoint()"))
- elif (isinstance(fn, ast.Attribute) and fn.attr == "set_trace"
- and isinstance(fn.value, ast.Name)
- and fn.value.id in {"pdb", "ipdb"}):
- bad.append((path, node.lineno, f"{fn.value.id}.set_trace()"))
-
- if bad:
- for path, lineno, what in bad:
- print(f"::error file={path},line={lineno}::leftover {what} -- remove before merging")
- sys.exit(1)
- print(f"no leftover debugger calls (scanned {scanned} files)")
- PY
-
- - name: License-header drift (informational; whole repo)
- # Three header families are accepted across the repo:
- # 1. SPDX one-liner: `# SPDX-License-Identifier: ...`
- # Used across studio/ (AGPL-3.0-only) and a few new
- # files elsewhere.
- # 2. Apache-2.0 long form, marker phrase
- # "Licensed under the Apache License". Used across
- # unsloth/ and unsloth_cli/.
- # 3. GNU long form, marker phrase "General Public License".
- # That single substring covers GPL, LGPL ("GNU Lesser
- # General Public License") and AGPL ("GNU Affero
- # General Public License") preambles, all three of
- # which appear in unsloth/kernels/* (LGPL/AGPL) without
- # the SPDX line.
- # Empty files (mainly empty __init__.py) are skipped.
- # Surfaced as a warning; cleaning up the actual misses is a
- # follow-up PR, not a CI fix.
- continue-on-error: true
- run: |
- python <<'PY'
- import pathlib
-
- ACCEPTED = (
- "SPDX-License-Identifier", # any SPDX line
- "Licensed under the Apache License", # Apache-2.0 long form
- "General Public License", # GPL / LGPL / AGPL long form
- )
- SKIP_PARTS = {".venv", "venv", "build", "dist", ".git",
- "unsloth_compiled_cache", "node_modules",
- "unsloth.egg-info"}
-
- studio_missing = []
- other_missing = []
- for path in sorted(pathlib.Path(".").rglob("*.py")):
- if any(part in SKIP_PARTS for part in path.parts):
- continue
- text = path.read_text(encoding="utf-8", errors="replace")
- if not text.strip():
- continue # empty __init__.py etc.
- head = "\n".join(text.splitlines()[:25])
- if any(marker in head for marker in ACCEPTED):
- continue
- if "studio" in path.parts:
- studio_missing.append(path)
- else:
- other_missing.append(path)
-
- total = len(studio_missing) + len(other_missing)
- if total == 0:
- print("every committed .py has a recognised license header")
- else:
- print(f"::warning::{total} Python files have no recognised license "
- f"header (SPDX / Apache-2.0 / GNU long form): "
- f"studio={len(studio_missing)}, other={len(other_missing)}")
- for path in (studio_missing + other_missing)[:30]:
- print(f" {path}")
- if total > 30:
- print(f" ... and {total - 30} more")
- PY
-
- - name: Shell scripts parse cleanly (`bash -n`)
- # Same idea as Python's compileall: parse-only check that
- # every committed *.sh would not blow up at `bash script.sh`
- # invocation time on a release box. tests/sh/ is the largest
- # cluster (the install.sh shape tests).
- run: |
- shopt -s globstar
- fail=0
- for f in $(git ls-files '*.sh'); do
- if ! bash -n "$f"; then
- echo "::error file=$f::shell parse error"
- fail=1
- fi
- done
- if [ "$fail" -ne 0 ]; then
- exit 1
- fi
- n=$(git ls-files '*.sh' | wc -l)
- echo "$n shell scripts parse cleanly"
-
- - name: YAML files parse cleanly (yaml.safe_load)
- # Catches truncated workflow files, broken indents in
- # dependabot.yml / pre-commit configs, etc. Includes
- # .github/workflows/*.yml so a typo in the file we just
- # added shows up immediately.
- run: |
- python <<'PY'
- import pathlib, sys, yaml
-
- SKIP_PARTS = {".venv", "venv", "build", "dist", ".git",
- "node_modules", "unsloth_compiled_cache",
- "unsloth.egg-info"}
-
- bad = []
- scanned = 0
- for path in sorted(list(pathlib.Path(".").rglob("*.yml"))
- + list(pathlib.Path(".").rglob("*.yaml"))):
- if any(part in SKIP_PARTS for part in path.parts):
- continue
- scanned += 1
- try:
- with path.open("r", encoding="utf-8") as fh:
- list(yaml.safe_load_all(fh))
- except Exception as exc:
- bad.append((path, exc))
-
- if bad:
- for path, exc in bad:
- print(f"::error file={path}::YAML parse failed: {exc}")
- sys.exit(1)
- print(f"{scanned} YAML files parse cleanly")
- PY
-
- - name: JSON files parse cleanly (json.loads)
- # Catches malformed package.json, biome.json, etc. Skips:
- # - huge npm/bun lockfiles (machine-generated, slow to
- # parse, no value).
- # - tsconfig*.json: TypeScript convention is JSONC (JSON
- # with `/* ... */` comments), which standard json.loads
- # rejects. Strip-and-validate would need json5 or a
- # hand-rolled comment scrubber for marginal value, since
- # `tsc --noEmit` already validates these in Frontend CI.
- run: |
- python <<'PY'
- import fnmatch, json, pathlib, sys
-
- SKIP_PARTS = {".venv", "venv", "build", "dist", ".git",
- "node_modules", "unsloth_compiled_cache",
- "unsloth.egg-info"}
- SKIP_NAMES = {"package-lock.json", "bun.lock"}
- SKIP_PATTERNS = ("tsconfig*.json",)
-
- bad = []
- scanned = 0
- for path in sorted(pathlib.Path(".").rglob("*.json")):
- if any(part in SKIP_PARTS for part in path.parts):
- continue
- if path.name in SKIP_NAMES:
- continue
- if any(fnmatch.fnmatch(path.name, pat) for pat in SKIP_PATTERNS):
- continue
- scanned += 1
- try:
- json.loads(path.read_text(encoding="utf-8"))
- except Exception as exc:
- bad.append((path, exc))
-
- if bad:
- for path, exc in bad:
- print(f"::error file={path}::JSON parse failed: {exc}")
- sys.exit(1)
- print(f"{scanned} JSON files parse cleanly")
- PY
-
- - name: codespell typo check (informational)
- # Catches typos in code, comments, and docs across the repo.
- # Skips lockfiles, generated assets, binary artefacts, and
- # the LICENSE files (US/UK spelling drift in legal text is
- # not ours to second-guess). The ignore-words-list pulls
- # out short identifiers + valid technical terms that
- # codespell's default dictionary would otherwise flag
- # (e.g. `ans` as a math-quiz variable name in
- # tests/utils/aime_eval.py, `parm`/`parms` in PyTorch
- # nn.Module idioms). Non-blocking until the surfaced typos
- # are fixed; drop continue-on-error after the cleanup.
- continue-on-error: true
- run: |
- codespell \
- --skip='*.lock,*.lockb,*.json,*.svg,*.png,*.jpg,*.jpeg,*.gif,*.ico,*.woff*,*.ttf,*.eot,*.zip,*.gz,*.gguf,*.safetensors,*.bin,node_modules,.git,build,dist,unsloth_compiled_cache,unsloth.egg-info,target,studio/frontend/dist,*.pyc,*-licenses.txt,LICENSE*' \
- --ignore-words-list='ans,bu,hel,fo,te,ot,hist,ned,sav,recurser,datas,nin,parm,parms,checkin,nd,fr,inout,donot,uint' \
- --quiet-level=2
-
- - name: shellcheck on committed *.sh (informational)
- # Goes beyond `bash -n` (which only parses): catches subtle
- # shell bugs like unquoted variable expansions, useless
- # `cat`, command substitutions inside `[[`, etc. The
- # install/setup scripts are critical-path so the signal is
- # worth surfacing. Non-blocking until install.sh's
- # hand-rolled patterns get cleaned up; drop continue-on-error
- # afterwards.
- continue-on-error: true
- run: |
- # Exclude SC1090 ("source not followable") -- legitimate
- # for installer scripts that source files at runtime
- # paths shellcheck cannot resolve statically.
- # SC2034 ("variable assigned but never used") fires on
- # the export-only assignment idiom we use in install.sh.
- shellcheck -e SC1090,SC2034 $(git ls-files '*.sh')
-
- - name: ruff format drift (informational)
- # The canonical formatter is scripts/run_ruff_format.py
- # = ruff format + scripts/enforce_kwargs_spacing.py, so plain
- # `ruff format --check` reports the kwarg-spacing diff as
- # drift. Surface the count for visibility but keep
- # non-blocking until the custom pipeline is wired in here.
- continue-on-error: true
- run: |
- ruff format --check unsloth unsloth_cli studio tests cli.py unsloth-cli.py
diff --git a/.github/workflows/lockfile-audit.yml b/.github/workflows/lockfile-audit.yml
deleted file mode 100644
index 9c28e21672..0000000000
--- a/.github/workflows/lockfile-audit.yml
+++ /dev/null
@@ -1,79 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-#
-# Fast, focused supply-chain audit of every checked-in lockfile.
-#
-# Runs scripts/lockfile_supply_chain_audit.py on PRs that touch any
-# npm or cargo lockfile, on push to main, and on a daily schedule so
-# newly-published IOCs surface even when no PR opens.
-#
-# Default behavior is "advisory": only public indicator-of-compromise
-# strings, known-malicious pinned versions, and structurally broken
-# lockfiles fail the build. Structural anomalies (missing integrity,
-# non-default registry, etc.) are emitted as GitHub Actions warnings
-# but do not block merges. This deliberately keeps the noise floor
-# low while still failing the moment a checked-in lockfile starts
-# pointing at known-bad bytes.
-#
-# This workflow is intentionally separate from security-audit.yml:
-# - security-audit.yml is the umbrella job (pip-audit + npm audit +
-# cargo audit + OSV + Semgrep + secret scanning + SBOM + ...);
-# it takes ~25 minutes and runs only when dep manifests change.
-# - lockfile-audit.yml is a ~30 second pure-Python parse + grep on
-# the lockfiles themselves; it runs on every PR that even nudges
-# a lockfile so reviewers always see the audit result inline.
-
-name: Lockfile supply-chain audit
-
-on:
- pull_request:
- paths:
- - 'studio/frontend/package-lock.json'
- - 'studio/backend/core/data_recipe/oxc-validator/package-lock.json'
- - 'studio/package-lock.json'
- - 'studio/src-tauri/Cargo.lock'
- - 'scripts/lockfile_supply_chain_audit.py'
- - '.github/workflows/lockfile-audit.yml'
- push:
- branches: [main]
- paths:
- - 'studio/frontend/package-lock.json'
- - 'studio/backend/core/data_recipe/oxc-validator/package-lock.json'
- - 'studio/package-lock.json'
- - 'studio/src-tauri/Cargo.lock'
- - 'scripts/lockfile_supply_chain_audit.py'
- - '.github/workflows/lockfile-audit.yml'
- schedule:
- - cron: '37 5 * * *'
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- audit:
- name: lockfile supply-chain audit
- runs-on: ubuntu-latest
- timeout-minutes: 5
- steps:
- - uses: actions/checkout@v4
- with:
- persist-credentials: false
-
- - uses: actions/setup-python@v5
- with:
- python-version: '3.12'
-
- - name: Verify audit script parses
- run: python3 -c "import ast; ast.parse(open('scripts/lockfile_supply_chain_audit.py').read())"
-
- - name: Run lockfile supply-chain audit
- # Default mode: only known-malicious pinned versions, known IOC
- # strings, and structurally broken lockfiles fail the build.
- # Missing-integrity and other structural anomalies are emitted
- # as ::warning:: annotations and do not gate merges.
- run: python3 scripts/lockfile_supply_chain_audit.py
diff --git a/.github/workflows/mlx-ci.yml b/.github/workflows/mlx-ci.yml
deleted file mode 100644
index 75940832a0..0000000000
--- a/.github/workflows/mlx-ci.yml
+++ /dev/null
@@ -1,430 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# Focused PR gate for the MLX dispatch surface, running on a real
-# Apple Silicon runner.
-#
-# Runner: macos-14 (M1, 3 vCPU / 7 GB / Apple Silicon standard runner
-# -- FREE for public repositories per the GitHub Actions billing
-# reference; larger variants like macos-14-large/-xlarge are paid so
-# we deliberately avoid those).
-#
-# Why a single Mac job (no Linux+spoof leg): the dispatch tests are
-# 100% spoofed monkeypatches and run identically on any host, so the
-# Linux leg was duplicating the matrix tests already covered on Mac
-# while missing everything Apple-specific. The Mac job runs the SAME
-# spoofed matrix PLUS three things only a real Apple Silicon host
-# can prove:
-#
-# 1. unsloth._IS_MLX flips True on Darwin+arm64 with mlx genuinely
-# installed (no spoof).
-# 2. Every PR-A MLX-only unsloth_zoo module (mlx_loader, mlx_trainer,
-# mlx_compile, mlx_utils, mlx_cce, gated_delta_vjp) imports
-# against the real `mlx` + `mlx-lm` + `mlx-vlm` PyPI wheels --
-# each does `import mlx.core as mx` at module top level, so this
-# catches a future change that breaks the real wheels without
-# needing a Mac developer in the loop.
-# 3. The hardware-dispatch spoofs do not collide with the real
-# environment (the test fixture installs a MetaPathFinder that
-# blocks `import mlx.core` for "no-mlx" profiles, faithfully
-# simulating a Mac without mlx even when mlx IS installed).
-# 4. End-to-end MLX training + inference smoke test:
-# run_real_mlx_smoke.py trains unsloth/gemma-3-270m-it for 7
-# deterministic LoRA steps on a single repeated text row, then
-# verifies the trained model can complete the prompt and that
-# losses + grad norms are finite and well-behaved. This is the
-# only place in CI that exercises a real MLX backward pass +
-# optimizer step + inference call.
-#
-# Three dispatch test files documented in tests/studio/README.md:
-# - test_hardware_dispatch_matrix.py parametrized 7-profile matrix
-# + 2 dispatch-priority canaries
-# - test_is_mlx_dispatch_gate.py AST + runtime guard on
-# unsloth._IS_MLX
-# - test_mlx_training_worker_behaviors.py AST contract checks on
-# studio/backend/core/training/worker.py
-#
-# Surfaces a single PR check ("MLX CI on Mac M1 / dispatch").
-#
-# Security audit footprint: every package this workflow installs is
-# already covered by .github/workflows/security-audit.yml -- the deps
-# come from studio/backend/requirements/studio.txt and unsloth-zoo's
-# pyproject (resolved transitively). The git+ install of unsloth-zoo
-# is intentionally skipped by the audit (pip-audit cannot resolve a
-# git URL through PyPI metadata; the audit comment in security-audit.yml
-# documents this). No new package is introduced solely by MLX CI.
-
-name: MLX CI on Mac M1
-
-on:
- pull_request:
- paths:
- - 'unsloth/__init__.py'
- - 'unsloth/_gpu_init.py'
- - 'studio/backend/utils/hardware/**'
- - 'studio/backend/core/training/worker.py'
- - 'studio/backend/core/inference/mlx_inference.py'
- - 'tests/studio/test_hardware_dispatch_matrix.py'
- - 'tests/studio/test_is_mlx_dispatch_gate.py'
- - 'tests/studio/test_mlx_training_worker_behaviors.py'
- - 'tests/studio/run_real_mlx_smoke.py'
- - 'tests/conftest.py'
- - '.github/workflows/mlx-ci.yml'
- push:
- branches: [main, pip]
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- dispatch:
- name: dispatch
- runs-on: macos-14
- # 25 min: dispatch + spoofed matrix + 7-step real LoRA training is
- # under 2 min; GGUF export builds llama.cpp via cmake on Apple
- # Silicon (~5-7 min), so we budget headroom.
- timeout-minutes: 25
- steps:
- # harden-runner audit mode: macOS runners cannot use blocking mode
- # today (eBPF egress enforcement is Linux-only), but audit mode is
- # supported cross-platform and surfaces the egress destinations in
- # the runner log. This produces the data needed to graduate this
- # job to a block-mode allowlist once macOS support lands.
- - name: Harden runner (audit)
- uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
- with:
- egress-policy: audit
-
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- # macOS install ladder, validated locally against a Linux
- # mac-sim venv (platform spoofed + mlx_simulation shim + real
- # datasets/transformers/structlog).
- #
- # 1. studio/backend/requirements/studio.txt brings structlog,
- # fastapi, etc. The hardware probe imports structlog at
- # module top level.
- # 2. Same pytest / numpy / httpx stack the rest of the repo CI
- # uses.
- # 3. torch is explicitly installed: unsloth-zoo's pyproject
- # deliberately excludes torch on darwin+arm64 (mlx replaces
- # it for runtime use), but the dispatch tests spoof
- # torch.cuda / torch.xpu / torch.backends.mps via monkeypatch
- # and so the test process needs torch importable. We pull
- # from the PyTorch CPU index so Apple Silicon gets the
- # explicit cpu+MPS arm64 wheel rather than something the
- # default PyPI resolver might pick up. The CPU index hosts
- # macosx_*_arm64 wheels alongside the Linux x86_64 ones.
- # 4. unsloth-zoo from git main (NOT PyPI), WITH deps. PR-A's
- # MLX support landed after the most recent unsloth-zoo PyPI
- # release; the wheel still raises NotImplementedError on
- # Apple Silicon when device_type.get_device_type() runs
- # unguarded. Studio's own install.sh overlays unsloth-zoo
- # from git main for the same reason. Pulling deps lets pip
- # resolve the platform-conditional MLX-only wheels (mlx,
- # mlx-lm, mlx-vlm gated on darwin+arm64 in unsloth-zoo's
- # pyproject) AND the shared deps (datasets, transformers,
- # sentencepiece, ...) that unsloth's MLX branch loads via
- # dataprep/raw_text.py.
- # 5. unsloth -e . --no-deps so the editable install does not
- # fight the unsloth-zoo dep set.
- #
- # All explicit pip installs are version-pinned to a single
- # released version (the latest as of 2026-05-07 within each
- # project's existing constraint range). bump alongside the rest
- # of the security audit when a new release lands.
- - name: Install deps
- run: |
- python -m pip install --upgrade pip
- pip install -r studio/backend/requirements/studio.txt
- pip install \
- 'python-multipart==0.0.27' \
- 'aiofiles==25.1.0' \
- 'sqlalchemy==2.0.49' \
- 'cryptography==48.0.0' \
- 'pyyaml==6.0.3' \
- 'jinja2==3.1.6' \
- 'mammoth==1.12.0' \
- 'unpdf==1.0.0' \
- 'requests==2.33.1' \
- 'typer==0.25.1' \
- 'numpy==2.4.4' \
- 'pytest==9.0.3' \
- 'pytest-asyncio==1.3.0' \
- 'httpx==0.28.1'
- pip install --index-url https://download.pytorch.org/whl/cpu \
- 'torch==2.10.0'
- # github.com occasionally 500s on the git fetch; retry the
- # zoo install so a single upstream blip does not fail CI.
- for attempt in 1 2 3; do
- if pip install "unsloth_zoo @ git+https://github.com/unslothai/unsloth-zoo"; then
- break
- fi
- if [ "$attempt" -eq 3 ]; then
- echo "::error::pip install unsloth_zoo failed after 3 attempts"
- exit 1
- fi
- delay=$((5 * attempt))
- echo "::warning::unsloth_zoo install failed (attempt $attempt/3), retrying in ${delay}s..."
- sleep "$delay"
- done
- pip install -e . --no-deps
-
- # Real Apple Silicon sanity: confirm _IS_MLX activates on real
- # hardware with no platform spoof.
- - name: Verify _IS_MLX flips True on real Apple Silicon
- run: |
- python -c "
- import platform
- assert platform.system() == 'Darwin', platform.system()
- assert platform.machine() == 'arm64', platform.machine()
- import unsloth
- assert unsloth._IS_MLX is True, f'expected _IS_MLX=True on real Apple Silicon, got {unsloth._IS_MLX}'
- print('OK: _IS_MLX activated on real Apple Silicon')
- "
-
- # Real Apple Silicon sanity: confirm every PR-A MLX-only module
- # loads against real mlx + mlx-lm + mlx-vlm wheels.
- - name: Smoke-import every MLX-only unsloth_zoo module
- run: |
- python -c "
- import importlib
- for name in [
- 'unsloth_zoo.mlx_loader',
- 'unsloth_zoo.mlx_trainer',
- 'unsloth_zoo.mlx_compile',
- 'unsloth_zoo.mlx_utils',
- 'unsloth_zoo.mlx_cce',
- 'unsloth_zoo.gated_delta_vjp',
- ]:
- importlib.import_module(name)
- print('OK:', name)
- from unsloth_zoo.mlx_loader import FastMLXModel
- from unsloth_zoo.mlx_trainer import MLXTrainer, MLXTrainingConfig
- assert hasattr(FastMLXModel, 'from_pretrained')
- print('OK: FastMLXModel + MLXTrainer surface present')
- "
-
- # Spoofed dispatch matrix. Runs on the real Mac too -- the
- # test fixture installs a MetaPathFinder that blocks
- # `import mlx.core` for "no-mlx" profiles, so the spoofs
- # faithfully simulate every supported hardware combo regardless
- # of whether mlx is installed for real.
- - name: MLX dispatch tests (3 files, 36 tests)
- env:
- PYTHONPATH: ${{ github.workspace }}/studio
- UNSLOTH_COMPILE_DISABLE: '1'
- run: |
- python -m pytest -v --tb=short \
- tests/studio/test_hardware_dispatch_matrix.py \
- tests/studio/test_is_mlx_dispatch_gate.py \
- tests/studio/test_mlx_training_worker_behaviors.py
-
- # Studio prebuilt llama.cpp install + GGUF inference. Drives the
- # exact path Studio's setup.sh takes on macOS: invokes
- # studio/install_llama_prebuilt.py with --published-repo
- # ggml-org/llama.cpp and --published-release-tag b9049 (the
- # latest llama.cpp release at the time this step was added; bump
- # via UNSLOTH_LLAMA_TAG / DEFAULT_LLAMA_TAG when refreshing).
- # The installer downloads llama-b9049-bin-macos-arm64.tar.gz,
- # which is the universal Apple Silicon (arm64) build -- the
- # same artifact works on M1/M2/M3/M4 because llama.cpp compiles
- # against the ARMv8.2 baseline.
- #
- # The b9049 release also publishes:
- # - llama-b9049-bin-macos-arm64-kleidiai.tar.gz
- # KleidiAI dispatches at runtime; on M1 it falls back where
- # ISA features (e.g. I8MM) are missing, so this asset also
- # runs on M1 -- Studio just doesn't choose it by default.
- # - llama-b9049-bin-macos-x64.tar.gz
- # Intel-only; would only run on M1 via Rosetta 2 emulation,
- # which we explicitly avoid.
- # - iOS XCFramework
- # iOS-app build artifact, unrelated to a macOS desktop CI.
- #
- # After install, downloads a small published GGUF
- # (unsloth/gemma-3-270m-it-GGUF, Q4_K_M) from HuggingFace and
- # runs the prebuilt llama-cli on it. Asserts the prompt echo
- # appears in stdout. If the install fails OR the binary exits
- # non-zero, that's an Unsloth/Studio bug.
- - name: Studio prebuilt llama.cpp install + GGUF inference (Mac M1)
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- # install_llama_prebuilt.py hits the GitHub releases API to
- # resolve the asset URL. Anonymous calls share the runner-IP
- # rate-limit bucket and 403 quickly -- pass the workflow's
- # automatic GITHUB_TOKEN to bump us to the 5000/hr authenticated
- # bucket.
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- set -euo pipefail
- INSTALL_DIR="$HOME/.unsloth-studio-prebuilt-test/llama.cpp"
- rm -rf "$INSTALL_DIR"
- # --simple-policy is required when --published-repo points
- # at upstream ggml-org/llama.cpp; that repo doesn't ship the
- # llama-prebuilt-manifest.json asset Studio's default policy
- # expects, so the simple platform-specific policy maps
- # Darwin+arm64 -> bin-macos-arm64 directly. studio/setup.sh
- # passes both --published-repo ggml-org/llama.cpp AND
- # --simple-policy automatically on macOS, so this CI step
- # exercises the same code path users hit when they run
- # `curl -fsSL https://unsloth.ai/install.sh | sh`.
- python studio/install_llama_prebuilt.py \
- --install-dir "$INSTALL_DIR" \
- --published-repo ggml-org/llama.cpp \
- --published-release-tag b9049 \
- --simple-policy
-
- # Studio bundles only llama-server + llama-quantize from the
- # prebuilt (not llama-cli) -- inference goes through
- # llama-server's HTTP /completion endpoint. Validate both:
- # llama-quantize --help proves the dynamic libs link, then
- # spin up llama-server and POST a /completion request on a
- # tiny published GGUF.
- LLAMA_SERVER="$INSTALL_DIR/build/bin/llama-server"
- LLAMA_QUANT="$INSTALL_DIR/build/bin/llama-quantize"
- [ -x "$LLAMA_SERVER" ] || { echo "::error::llama-server missing at $LLAMA_SERVER"; find "$INSTALL_DIR/build" -type f | head -40; exit 1; }
- [ -x "$LLAMA_QUANT" ] || { echo "::error::llama-quantize missing at $LLAMA_QUANT"; exit 1; }
- echo "llama-server : $LLAMA_SERVER"
- echo "llama-quantize: $LLAMA_QUANT"
- "$LLAMA_QUANT" --help >/dev/null && echo " llama-quantize loads OK"
-
- mkdir -p /tmp/ggufs
- bash .github/scripts/hf-download-with-retry.sh \
- 'unsloth/gemma-3-270m-it-GGUF' \
- 'gemma-3-270m-it-Q4_K_M.gguf' \
- /tmp/ggufs
-
- PORT=18080
- echo "=== starting llama-server on 127.0.0.1:$PORT ==="
- "$LLAMA_SERVER" \
- -m /tmp/ggufs/gemma-3-270m-it-Q4_K_M.gguf \
- --host 127.0.0.1 \
- --port "$PORT" \
- -c 256 \
- -n 16 \
- --no-warmup \
- > /tmp/llama-server.log 2>&1 &
- SERVER_PID=$!
- trap 'kill "$SERVER_PID" 2>/dev/null || true' EXIT
-
- # Wait for /health to come up
- for i in $(seq 1 30); do
- if curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then
- echo " server up after ${i}s"
- break
- fi
- sleep 1
- done
- if ! curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then
- echo "::error::llama-server never became healthy"
- tail -40 /tmp/llama-server.log
- exit 1
- fi
-
- PROMPT="Hello, my name is"
- echo "=== POST /completion ==="
- RESP=$(curl -sf -X POST "http://127.0.0.1:$PORT/completion" \
- -H 'Content-Type: application/json' \
- -d "{\"prompt\":\"$PROMPT\",\"n_predict\":16,\"temperature\":0,\"seed\":3407}")
- echo "raw response (head): $(echo "$RESP" | head -c 600)"
- CONTENT=$(echo "$RESP" | python -c "import json,sys; print(json.loads(sys.stdin.read()).get('content',''))")
- echo "completion content: $CONTENT"
-
- if [ -z "$CONTENT" ]; then
- echo "::error::llama-server /completion returned empty content"
- tail -40 /tmp/llama-server.log
- exit 1
- fi
- echo "OK: Studio prebuilt llama.cpp on Mac M1 + GGUF /completion works"
-
- # Real MLX training + inference smoke test. Trains
- # unsloth/gemma-3-270m-it for 7 deterministic LoRA steps
- # (batch_size=2, gradient_accumulation_steps=3) on a single
- # repeated row ("<> My name is Unsloth!"), then saves
- # the trained model in 3 export formats. The `train` subcommand
- # captures per-phase timing + peak GPU + peak RSS into
- # train_metrics.json so we can detect regressions across CI runs.
- - name: MLX export round-trip — TRAIN + SAVE 3 formats
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- UNSLOTH_COMPILE_DISABLE: '1'
- run: |
- mkdir -p mlx_workdir
- python tests/studio/run_real_mlx_smoke.py train \
- --workdir "$PWD/mlx_workdir"
-
- # Each reload step runs in a FRESH Python process to confirm
- # the cold-start path users would hit in production also works
- # (not just the in-memory continuation of a still-running
- # trainer). FastMLXModel.from_pretrained gets called from
- # scratch; mx.random is re-seeded; per-step timing + peak
- # memory are emitted to {format}_reload_metrics.json next to
- # the saved dir.
- - name: MLX export round-trip — RELOAD LoRA (fresh process)
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- UNSLOTH_COMPILE_DISABLE: '1'
- run: |
- python tests/studio/run_real_mlx_smoke.py reload \
- --format lora \
- --dir "$PWD/mlx_workdir/lora"
-
- - name: MLX export round-trip — RELOAD merged_16bit (fresh process)
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- UNSLOTH_COMPILE_DISABLE: '1'
- run: |
- python tests/studio/run_real_mlx_smoke.py reload \
- --format merged \
- --dir "$PWD/mlx_workdir/merged_16bit"
-
- # GGUF reload uses the llama-cli binary that save_pretrained_gguf
- # built. If save_pretrained_gguf was skipped during train (e.g.
- # llama.cpp's convert_hf_to_gguf asserts on the model's tokenizer
- # vocab -- a downstream llama.cpp limitation, not an unsloth_zoo
- # bug), this step emits a workflow warning and exits 0 so the
- # LoRA + merged_16bit assertions remain the gating signal.
- - name: MLX export round-trip — RELOAD GGUF via llama-cli (fresh process)
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- run: |
- if python -c "import json,sys; m=json.load(open('mlx_workdir/train_metrics.json')); sys.exit(0 if m.get('gguf_supported') else 1)"; then
- python tests/studio/run_real_mlx_smoke.py reload \
- --format gguf \
- --dir "$PWD/mlx_workdir/gguf"
- else
- REASON=$(python -c "import json; m=json.load(open('mlx_workdir/train_metrics.json')); print(m.get('gguf_skip_reason') or 'unknown')")
- echo "::warning title=GGUF round-trip skipped::${REASON}"
- echo "GGUF export was skipped during the train phase. Reason:"
- echo " ${REASON}"
- echo "Continuing without failing the job; the LoRA + merged_16bit"
- echo "reload assertions are still gating this PR."
- fi
-
- # Print all metrics JSON files so regressions are visible in the
- # job log. always() so we get telemetry even if a reload step
- # asserted gibberish.
- - name: MLX export round-trip — aggregate metrics
- if: always()
- run: |
- for f in mlx_workdir/train_metrics.json \
- mlx_workdir/lora_reload_metrics.json \
- mlx_workdir/merged_reload_metrics.json \
- mlx_workdir/gguf_reload_metrics.json; do
- echo "=== $f ==="
- cat "$f" 2>/dev/null || echo "(missing)"
- echo
- done
diff --git a/.github/workflows/notebooks-ci.yml b/.github/workflows/notebooks-ci.yml
deleted file mode 100644
index 673b2f3cc5..0000000000
--- a/.github/workflows/notebooks-ci.yml
+++ /dev/null
@@ -1,440 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-#
-# Cross-repo notebook validator. Lives in unslothai/unsloth (this repo)
-# and inspects every notebook in unslothai/notebooks at HEAD (or the
-# ref dispatched in via repository_dispatch).
-#
-# Catches the bug classes that landed in:
-# - unslothai/notebooks#258 Colab torchao 0.10 vs peft 0.19 floor
-# - unslothai/notebooks#260 DONT_UPDATE_EXCEPTIONS coverage drift
-# - unslothai/notebooks#261 torch/torchcodec ABI; --no-deps tokenizers
-# - unslothai/notebooks#264 --no-deps transformers + Colab tokenizers drift
-# - unslothai/notebooks#221 git+ HEAD installs in install cells
-# - unslothai/notebooks commit 51b1462 template/notebook drift
-#
-# CPU-only by design. Layer 2 (api-introspect) reuses the existing
-# tests/_zoo_aggressive_cuda_spoof.py harness so `import unsloth`
-# succeeds on a GPU-less ubuntu-latest runner.
-
-name: Notebooks CI
-
-on:
- pull_request:
- paths:
- - 'unsloth/**'
- - 'scripts/notebook_validator.py'
- - 'scripts/notebook_to_python.py'
- - 'scripts/data/colab_pip_freeze.gpu.txt'
- - 'scripts/data/colab_to_cpu_pin.json'
- - 'tests/notebooks/**'
- - 'tests/_zoo_aggressive_cuda_spoof.py'
- - '.github/workflows/notebooks-ci.yml'
- schedule:
- # Daily 06:17 UTC. Catches Colab preinstall bumps (the upstream image
- # is rebuilt roughly weekly) without us waiting on a PR. Off the
- # :00/:30 fleet-collision spots.
- - cron: '17 6 * * *'
- workflow_dispatch:
- inputs:
- notebooks_ref:
- description: 'unslothai/notebooks ref to lint (branch / SHA / tag)'
- default: 'main'
- include_smoke:
- description: 'Also run the install-cell smoke matrix (longer)'
- type: boolean
- default: false
- repository_dispatch:
- # Fired by a tiny companion workflow on unslothai/notebooks.
- types: [notebooks_pr_opened, notebooks_main_pushed]
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-env:
- NOTEBOOKS_REF: >-
- ${{ github.event.inputs.notebooks_ref ||
- github.event.client_payload.ref ||
- 'main' }}
-
-jobs:
- static:
- name: static (drift + lint + exceptions)
- runs-on: ubuntu-latest
- timeout-minutes: 10
- steps:
- # Validate the dispatched ref before it reaches actions/checkout's `ref:`
- # input. Reading via env (NOT direct ${{ ... }} interpolation in the
- # regex test) closes the GitHub-Actions-injection class where a
- # client_payload.ref like `main"; rm -rf / #` would be embedded into the
- # shell command. NOTEBOOKS_REF defaults to 'main' on non-dispatch
- # events, but only repository_dispatch can supply attacker-controlled
- # values, so we gate this check on that event type.
- - name: Validate client_payload.ref shape
- if: github.event_name == 'repository_dispatch'
- env:
- NOTEBOOKS_REF: ${{ github.event.client_payload.ref }}
- run: |
- if ! printf '%s' "$NOTEBOOKS_REF" | grep -Eq '^[A-Za-z0-9._/-]+$'; then
- echo "::error::client_payload.ref contains disallowed characters" >&2
- exit 1
- fi
-
- - name: Checkout unsloth (this PR)
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- path: unsloth
- persist-credentials: false
-
- - name: Checkout unslothai/notebooks @ ${{ env.NOTEBOOKS_REF }}
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- repository: unslothai/notebooks
- ref: ${{ env.NOTEBOOKS_REF }}
- path: notebooks
- fetch-depth: 0 # drift check needs git status / diff
- persist-credentials: false
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- - name: Install validator deps
- run: |
- python -m pip install --upgrade pip
- # nbformat + nbconvert come from the converter's requirements;
- # spellchecker + huggingface_hub are imported at module top of
- # update_all_notebooks.py.
- pip install \
- 'nbformat>=5.10' 'nbconvert>=7.16' 'pyspellchecker>=0.8' \
- 'huggingface_hub>=0.34' 'tqdm>=4.66'
-
- - name: Refresh Colab pip-freeze (best-effort; falls back to snapshot)
- run: |
- python unsloth/scripts/notebook_validator.py refresh-colab \
- --out unsloth/scripts/data/colab_pip_freeze.gpu.txt \
- || echo "::warning::refresh-colab failed; using committed snapshot"
-
- - name: Diff Colab oracle vs committed snapshots (advisory)
- # Pulls pip-freeze.gpu.txt + apt-list-gpu.txt + os-info-gpu.txt
- # from googlecolab/backend-info and prints NEW / REMOVED /
- # CHANGED entries against scripts/data/colab_*.txt. Non-blocking
- # on PRs; the daily cron job below runs the same step with
- # --strict so upstream rotations surface within ~24h.
- continue-on-error: true
- working-directory: ${{ github.workspace }}
- run: |
- python unsloth/scripts/notebook_validator.py colab-diff \
- --snapshot-dir unsloth/scripts/data
-
- - name: Drift check (re-run update_all_notebooks.py + git diff)
- working-directory: ${{ github.workspace }}
- # Reported as non-blocking until the upstream `unslothai/notebooks`
- # tree is regenerated. The first run on @main surfaces ~463 files
- # of drift (7359 / 9634 line delta), which is a real backlog the
- # notebooks-side maintainers need to clear in their own repo --
- # this PR's role is to surface the count, not auto-fix it.
- continue-on-error: true
- run: |
- python unsloth/scripts/notebook_validator.py drift \
- --notebooks-dir notebooks
-
- - name: Convert sanity (every nb / kaggle / original_template -> .py)
- # Same rationale as Drift: a handful of upstream notebooks fail
- # the converter (custom magics, malformed JSON, etc). Surface
- # the count without blocking; the team triages in unslothai/notebooks.
- continue-on-error: true
- run: |
- python unsloth/scripts/notebook_validator.py convert \
- --notebooks-dir notebooks \
- --out _converted
-
- - name: Lint (install cells + AST scan, env-scoped)
- # Reported as non-blocking (continue-on-error: true) until the
- # backlog of pre-existing findings on unslothai/notebooks@main is
- # cleared. Same pattern PR #5298 used for biome:check on the
- # frontend. As of this commit the live tree surfaces 27 errors +
- # 6 warnings, all real (peft/torchao floor missing in 6 nb/
- # notebooks, 14 git+ HEAD installs in hand-tuned exception
- # notebooks, 6 torch/torchcodec ABI mismatches, 1
- # transformers/tokenizers --no-deps drift). The count surfaces
- # in the PR check UI. Drop continue-on-error once it hits zero.
- continue-on-error: true
- run: |
- python unsloth/scripts/notebook_validator.py lint \
- --notebooks-dir notebooks \
- --colab-pin unsloth/scripts/data/colab_pip_freeze.gpu.txt \
- --no-pypi
- # --no-pypi skips R-INST-002 (transitive resolve via PyPI metadata).
- # Layer 1 keeps PR-time wall-clock predictable; the daily cron run
- # below drops --no-pypi and refreshes the cache.
-
- - name: DONT_UPDATE_EXCEPTIONS coverage
- run: |
- python unsloth/scripts/notebook_validator.py exceptions \
- --notebooks-dir notebooks
-
- static-with-pypi:
- name: static + transitive resolve (cron / dispatch only)
- if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
- runs-on: ubuntu-latest
- timeout-minutes: 15
- steps:
- # See `static.Validate client_payload.ref shape` for rationale. This
- # job's `if:` excludes repository_dispatch today, so the validation
- # step is a defence-in-depth no-op until that gate ever relaxes.
- - name: Validate client_payload.ref shape
- if: github.event_name == 'repository_dispatch'
- env:
- NOTEBOOKS_REF: ${{ github.event.client_payload.ref }}
- run: |
- if ! printf '%s' "$NOTEBOOKS_REF" | grep -Eq '^[A-Za-z0-9._/-]+$'; then
- echo "::error::client_payload.ref contains disallowed characters" >&2
- exit 1
- fi
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
- path: unsloth
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- repository: unslothai/notebooks
- ref: ${{ env.NOTEBOOKS_REF }}
- path: notebooks
- persist-credentials: false
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with: { python-version: '3.12', cache: 'pip' }
- - name: Install
- run: pip install -U pip
- - name: Refresh Colab oracle
- run: |
- python unsloth/scripts/notebook_validator.py refresh-colab \
- --out unsloth/scripts/data/colab_pip_freeze.gpu.txt
- - name: Diff Colab oracle vs committed snapshots (--strict on cron)
- # Cron-only escalation of the advisory PR-time check. Fails if
- # any of pip-freeze.gpu.txt / apt-list-gpu.txt / os-info-gpu.txt
- # has drifted from scripts/data/colab_*.txt; refresh the
- # snapshots in this repo to acknowledge.
- run: |
- python unsloth/scripts/notebook_validator.py colab-diff \
- --snapshot-dir unsloth/scripts/data --strict
- - name: Lint with live PyPI metadata
- run: |
- python unsloth/scripts/notebook_validator.py lint \
- --notebooks-dir notebooks \
- --colab-pin unsloth/scripts/data/colab_pip_freeze.gpu.txt
-
- api-introspect:
- name: api surface (under CUDA spoof)
- runs-on: ubuntu-latest
- timeout-minutes: 12
- steps:
- - name: Validate client_payload.ref shape
- if: github.event_name == 'repository_dispatch'
- env:
- NOTEBOOKS_REF: ${{ github.event.client_payload.ref }}
- run: |
- if ! printf '%s' "$NOTEBOOKS_REF" | grep -Eq '^[A-Za-z0-9._/-]+$'; then
- echo "::error::client_payload.ref contains disallowed characters" >&2
- exit 1
- fi
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
- path: unsloth
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- repository: unslothai/notebooks
- ref: ${{ env.NOTEBOOKS_REF }}
- path: notebooks
- persist-credentials: false
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with: { python-version: '3.12', cache: 'pip' }
-
- - name: Install CPU torch + pinned unsloth + trl + converter deps
- run: |
- python -m pip install --upgrade pip
- # CPU torch + torchvision. torchvision is required because
- # unsloth_zoo.vision_utils imports PIL at module top, and the
- # easiest way to get a torch-compatible PIL on a CPU runner is
- # to let torchvision pull the right Pillow version.
- pip install --index-url https://download.pytorch.org/whl/cpu \
- 'torch>=2.8,<2.11' 'torchvision<0.26'
- # Pin to the same versions update_all_notebooks.py installs in
- # generated notebooks. Keep these in lockstep with PIN_TRL /
- # PIN_TRANSFORMERS in unslothai/notebooks/update_all_notebooks.py.
- # `triton` is added because unsloth/_gpu_init.py:232 does an
- # unconditional `import triton`; the PyPI wheel installs cleanly
- # on Linux x86_64 even without CUDA (same rationale as
- # consolidated-tests-ci.yml line 192-205).
- # Pillow is listed explicitly as a defensive belt-and-braces
- # next to torchvision (vision_utils crashes ModuleNotFoundError
- # if torchvision skipped its Pillow dep for any reason).
- pip install 'transformers>=4.56,<5.6' 'trl>=0.22,<0.26' 'accelerate>=1.0' \
- 'datasets>=3.4,<5' 'peft>=0.15,<0.20' \
- 'bitsandbytes>=0.43' 'sentencepiece' 'protobuf' triton \
- Pillow safetensors tqdm packaging psutil
- # Converter deps (nbformat for notebook_to_python.py).
- pip install 'nbformat>=5.10' 'nbconvert>=7.16'
- # Install unsloth from the LOCAL checkout (the PR head), not PyPI.
- # The PR-time CI must validate the code in this PR; PyPI unsloth
- # may lag the in-repo CPU-torch fallback in unsloth/kernels/utils.py
- # (lines 162-170) that handles missing torch._C._cuda_getCurrentRawStream.
- pip install --no-deps unsloth_zoo
- pip install --no-deps -e ./unsloth
-
- - name: Convert notebooks for AST scan
- # Same upstream-conversion-error tolerance as the static job.
- continue-on-error: true
- run: |
- python unsloth/scripts/notebook_validator.py convert \
- --notebooks-dir notebooks --out _converted
-
- - name: Dump unsloth + trl API surface (under CUDA spoof)
- run: |
- PYTHONPATH=unsloth/tests python -u - <<'PY'
- import sys, json, inspect
- import _zoo_aggressive_cuda_spoof as _spoof
- _spoof.apply()
- import unsloth
- import trl
- surface = {}
- for cls_name in ("FastLanguageModel", "FastVisionModel", "FastModel"):
- cls = getattr(unsloth, cls_name, None)
- if cls is None:
- continue
- surface[cls_name] = sorted(n for n in dir(cls) if not n.startswith("_"))
- surface["SFTConfig_kwargs"] = sorted(inspect.signature(trl.SFTConfig.__init__).parameters)
- json.dump(surface, open("_api_surface.json", "w"), indent=2)
- print("dumped surface for:", list(surface))
- PY
-
- - name: Run API rule against converted notebooks
- run: |
- python unsloth/scripts/notebook_validator.py api \
- --converted-dir _converted \
- --surface _api_surface.json
-
- smoke-install:
- name: smoke install (Colab-shaped venv, opt-in)
- if: ${{ github.event.inputs.include_smoke == 'true' || github.event_name == 'schedule' }}
- runs-on: ubuntu-latest
- timeout-minutes: 25
- strategy:
- fail-fast: false
- matrix:
- # One representative notebook per installation_*_content template.
- # Add rows when a new install template lands in update_all_notebooks.py.
- notebook:
- - 'nb/Llama3.1_(8B)-Alpaca.ipynb' # installation_content
- - 'nb/Gemma3_(4B)-Vision.ipynb' # installation_content + vision
- - 'nb/Llama3.1_(8B)-GRPO.ipynb' # installation_extra_grpo_content
- - 'nb/gpt-oss-(20B)-Fine-tuning.ipynb' # installation_gpt_oss_content
- - 'nb/Qwen3_5_(4B)_Vision.ipynb' # installation_qwen3_5_content
- - 'nb/Nemotron-3-Nano-30B-A3B_A100.ipynb' # installation_nemotron_nano_content
- - 'nb/Whisper.ipynb' # installation_whisper_content
- - 'nb/Synthetic_Data_Hackathon.ipynb' # installation_synthetic_data_content
- steps:
- - name: Validate client_payload.ref shape
- if: github.event_name == 'repository_dispatch'
- env:
- NOTEBOOKS_REF: ${{ github.event.client_payload.ref }}
- run: |
- if ! printf '%s' "$NOTEBOOKS_REF" | grep -Eq '^[A-Za-z0-9._/-]+$'; then
- echo "::error::client_payload.ref contains disallowed characters" >&2
- exit 1
- fi
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
- path: unsloth
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- repository: unslothai/notebooks
- ref: ${{ env.NOTEBOOKS_REF }}
- path: notebooks
- persist-credentials: false
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with: { python-version: '3.12' }
-
- - name: Seed Colab-shaped venv from pip-freeze (CPU-mapped)
- run: |
- # Strip cu128 local versions, route torch/torchvision to the CPU
- # wheel index, drop CUDA-specific deps the runner can't use.
- python -u - <<'PY' > /tmp/seed_pins.txt
- import json, re
- mapping = json.load(open("unsloth/scripts/data/colab_to_cpu_pin.json"))
- rewrite = mapping["rewrite"]
- skip = set(mapping["skip"])
- spoof = set(mapping["module_spoof"])
- out = []
- for line in open("unsloth/scripts/data/colab_pip_freeze.gpu.txt"):
- line = line.strip()
- if not line or line.startswith("#"):
- continue
- m = re.match(r"^([A-Za-z0-9._-]+)\s*==\s*(.+)$", line)
- if not m:
- continue
- name, ver = m.group(1).lower(), m.group(2)
- if name in skip:
- continue
- if name in spoof:
- continue
- if name in rewrite:
- ver = re.sub(r"[+\-].+$", "", ver)
- out.append(f"{name}=={ver}")
- else:
- ver = re.sub(r"[+\-].+$", "", ver)
- out.append(f"{name}=={ver}")
- print("\n".join(out))
- PY
- head -5 /tmp/seed_pins.txt
- wc -l /tmp/seed_pins.txt
-
- - name: Install Colab-shaped venv
- run: |
- python -m pip install --upgrade pip
- # Best-effort: any single line that fails to resolve on CPU is
- # tolerated; the smoke contract is "the install cell + the unsloth
- # import works", not "the entire Colab venv reproduces."
- while IFS= read -r spec; do
- pip install "$spec" --index-url https://download.pytorch.org/whl/cpu \
- --extra-index-url https://pypi.org/simple || \
- echo "::warning::pin failed: $spec"
- done < /tmp/seed_pins.txt
-
- - name: Run install cell
- run: |
- python unsloth/scripts/notebook_validator.py convert \
- --notebooks-dir notebooks --out _converted
- # Take the converted .py and run the install cell only.
- BASE="$(basename '${{ matrix.notebook }}' .ipynb | tr -d '()' | tr -c '[:alnum:]_' _)"
- PY="_converted/${BASE}.py"
- [ -f "$PY" ] || { echo "::error::$PY not found"; ls _converted | head; exit 1; }
- # Truncate at the first `from unsloth import` so we run install +
- # core imports only.
- awk '/^from unsloth import/ { print "import sys; sys.exit(0)"; exit } { print }' "$PY" > _smoke.py
- PYTHONPATH=unsloth/tests python -u - <<'PY'
- import _zoo_aggressive_cuda_spoof as _s; _s.apply()
- # Stub torchcodec for cells that import it — no CPU wheel exists.
- import sys, types
- if "torchcodec" not in sys.modules:
- sys.modules["torchcodec"] = types.ModuleType("torchcodec")
- exec(open("_smoke.py").read(), {"__name__": "__main__"})
- PY
-
- - name: Verify imports under spoof
- run: |
- PYTHONPATH=unsloth/tests python -u - <<'PY'
- import sys, types
- if "torchcodec" not in sys.modules:
- sys.modules["torchcodec"] = types.ModuleType("torchcodec")
- import _zoo_aggressive_cuda_spoof as _s; _s.apply()
- import unsloth, peft, torch, torchao, transformers, tokenizers
- print("OK: imports pass under CUDA spoof")
- PY
diff --git a/.github/workflows/pr697-studio-probe.yml b/.github/workflows/pr697-studio-probe.yml
new file mode 100644
index 0000000000..2a63d89d5e
--- /dev/null
+++ b/.github/workflows/pr697-studio-probe.yml
@@ -0,0 +1,170 @@
+# Staging-fork CI: drive Unsloth Studio on macos-14 with PR #697 unsloth-zoo
+# overlaid, then exercise the 11 MLX save / GGUF export fix surfaces against
+# REAL Apple Silicon mlx / mlx-vlm wheels (not the torch shim used in the
+# pr697-cross-os branch). Sister job pr697-ubuntu.yml on the other branch
+# covers the shim path; this branch is the high-signal "real hardware" probe.
+
+name: "PR-697 Studio probe macos-14"
+
+on:
+ push:
+ branches: [pr697-studio-probe]
+ paths:
+ - 'install.sh'
+ - 'studio/**'
+ - 'unsloth/**'
+ - 'tests/pr697/**'
+ - 'studio_test_kit/**'
+ - '.github/workflows/pr697-studio-probe.yml'
+ pull_request:
+ branches: [main]
+ paths:
+ - 'install.sh'
+ - 'studio/**'
+ - 'unsloth/**'
+ - 'tests/pr697/**'
+ - 'studio_test_kit/**'
+ - '.github/workflows/pr697-studio-probe.yml'
+ workflow_dispatch:
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+permissions:
+ contents: read
+
+jobs:
+ studio-probe:
+ name: Studio + PR-697 unsloth-zoo overlay (Apple Silicon)
+ runs-on: macos-14
+ timeout-minutes: 35
+ env:
+ STUDIO_PORT: '18897'
+ # Pinned Studio home keeps the install isolated from the runner's $HOME.
+ UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home
+ UNSLOTH_PROBE_TMP: ${{ github.workspace }}/probe-tmp
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+ cache: 'pip'
+
+ - name: Install Studio (--local, --no-torch)
+ # --no-torch keeps macOS-14 install fast and stable (no CUDA noise);
+ # mlx-vlm path is unaffected. install.sh resolves the Studio venv at
+ # $UNSLOTH_STUDIO_HOME/.venv_*; the python is at $UNSLOTH_STUDIO_HOME/bin/python
+ # via the unsloth CLI shim.
+ run: |
+ mkdir -p logs
+ set -o pipefail
+ bash install.sh --local --no-torch 2>&1 | tee logs/install.log
+
+ - name: Resolve Studio venv python
+ id: venv
+ run: |
+ # install.sh writes the venv at $UNSLOTH_STUDIO_HOME/unsloth_studio
+ # (confirmed via install log: "creating Python 3.13 virtual environment
+ # /.../.studio-home/unsloth_studio"). The CLI shim is in bin/unsloth.
+ VENV_PY="$UNSLOTH_STUDIO_HOME/unsloth_studio/bin/python"
+ if [ ! -x "$VENV_PY" ]; then
+ # Fallback: glob any bin/python under the studio home.
+ VENV_PY="$(find "$UNSLOTH_STUDIO_HOME" -maxdepth 4 -type f -name python -path '*/bin/python' | head -1)"
+ fi
+ if [ -z "$VENV_PY" ] || [ ! -x "$VENV_PY" ]; then
+ echo "::error::Could not locate Studio venv python under $UNSLOTH_STUDIO_HOME"
+ find "$UNSLOTH_STUDIO_HOME" -maxdepth 4 -type d | head -50
+ exit 1
+ fi
+ echo "venv_py=$VENV_PY" >> "$GITHUB_OUTPUT"
+ # Also expose UNSLOTH_STUDIO_HOME/bin to PATH so subsequent steps see
+ # the unsloth CLI shim.
+ echo "$UNSLOTH_STUDIO_HOME/bin" >> "$GITHUB_PATH"
+ echo "Resolved Studio venv python: $VENV_PY"
+ "$VENV_PY" --version
+ "$VENV_PY" -c "import sys; print(sys.executable)"
+
+ - name: Overlay PR #697 unsloth-zoo (Lyxot:fix/mlx-save-gguf-export-parity)
+ # install.sh by default overlays unsloth-zoo from unslothai/unsloth-zoo:main.
+ # We want PR #697's head instead. Force-reinstall just the unsloth-zoo
+ # package without touching its deps so the rest of the Studio environment
+ # is unchanged.
+ run: |
+ "${{ steps.venv.outputs.venv_py }}" -m pip install \
+ --no-deps --force-reinstall \
+ "unsloth-zoo @ git+https://github.com/Lyxot/unsloth-zoo@fix/mlx-save-gguf-export-parity"
+
+ - name: Install MLX wheels (mlx / mlx-lm / mlx-vlm) into Studio venv
+ # --no-torch skipped the [mlx] extras at install.sh time. We need them
+ # for the probe to exercise REAL Apple Silicon kernels.
+ run: |
+ "${{ steps.venv.outputs.venv_py }}" -m pip install \
+ "mlx>=0.22.0" "mlx-lm>=0.22.0" "mlx-vlm>=0.4.4"
+
+ - name: Verify subpackage import paths (post-migration)
+ run: |
+ "${{ steps.venv.outputs.venv_py }}" -c "
+ import unsloth_zoo, unsloth_zoo.mlx.utils, unsloth_zoo.mlx.loader, \
+ unsloth_zoo.mlx.compile, unsloth_zoo.mlx.trainer, \
+ unsloth_zoo.mlx.cce, unsloth_zoo.mlx.runtime
+ print('unsloth_zoo', unsloth_zoo.__version__)
+ import mlx.core as mx, mlx_vlm
+ print('mlx', mx.__version__ if hasattr(mx, '__version__') else 'present')
+ print('mlx_vlm', mlx_vlm.__version__ if hasattr(mlx_vlm, '__version__') else 'present')
+ "
+
+ - name: Run PR-697 real-MLX probe
+ # This is the high-signal step. The script exercises every PR-697 helper
+ # against real mlx / mlx-vlm wheels. Exits non-zero on any mismatch.
+ run: |
+ mkdir -p "$UNSLOTH_PROBE_TMP"
+ PYTHONPATH="${{ github.workspace }}" \
+ "${{ steps.venv.outputs.venv_py }}" -m tests.pr697.probe_real_mlx
+
+ - name: Boot Studio (API-only) + /api/health
+ # Sanity: PR-697 unsloth-zoo overlay must NOT break Studio boot. Use
+ # API-only to skip the desktop / web UI bring-up.
+ run: |
+ unset GH_TOKEN || true
+ # Reset auth so we don't hit a stale bootstrap password.
+ unsloth studio reset-password || true
+ mkdir -p logs
+ UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \
+ > logs/studio.log 2>&1 &
+ STUDIO_PID=$!
+ echo "STUDIO_PID=$STUDIO_PID" >> "$GITHUB_ENV"
+ for i in $(seq 1 180); do
+ if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json 2>/dev/null; then
+ cat /tmp/health.json
+ if grep -q '"status"' /tmp/health.json && grep -q 'healthy' /tmp/health.json; then
+ echo "Studio API healthy after ${i}s"
+ exit 0
+ fi
+ fi
+ sleep 1
+ done
+ echo "::error::Studio API did not report healthy"
+ tail -200 logs/studio.log
+ exit 1
+
+ - name: Tear down Studio
+ if: always()
+ run: |
+ if [ -n "${STUDIO_PID:-}" ]; then
+ kill "$STUDIO_PID" 2>/dev/null || true
+ fi
+ pkill -f "unsloth studio" 2>/dev/null || true
+
+ - name: Upload probe artifacts
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: pr697-studio-probe-macos14
+ path: |
+ logs/
+ probe-tmp/
+ if-no-files-found: ignore
diff --git a/.github/workflows/release-desktop.yml b/.github/workflows/release-desktop.yml
deleted file mode 100644
index e747605322..0000000000
--- a/.github/workflows/release-desktop.yml
+++ /dev/null
@@ -1,902 +0,0 @@
-name: Release Desktop App
-
-on:
- workflow_dispatch:
- inputs:
- studio_version:
- description: 'Studio version tag to release (for example, v0.1.39-beta)'
- type: string
- required: true
- pypi_version:
- description: 'Exact PyPI unsloth version just published/stamped (for example, 2026.5.3); leave blank to use MIN_DESKTOP_BACKEND_VERSION'
- type: string
- required: false
- draft:
- description: 'Create as draft release; draft runs do not advance desktop-latest updater channel'
- type: boolean
- default: true
-
-permissions:
- contents: read
-
-concurrency:
- group: release-desktop-${{ github.repository }}
- cancel-in-progress: false
-
-jobs:
- prepare-version:
- name: Prepare release versions
- runs-on: ubuntu-latest
- outputs:
- studio_version: ${{ steps.prepare.outputs.studio_version }}
- app_version: ${{ steps.prepare.outputs.app_version }}
- desktop_release_tag: ${{ steps.prepare.outputs.desktop_release_tag }}
- prerelease: ${{ steps.prepare.outputs.prerelease }}
- pypi_version: ${{ steps.prepare.outputs.pypi_version }}
-
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
- with:
- persist-credentials: false
-
- - name: Validate release versions
- id: prepare
- shell: bash
- env:
- INPUT_STUDIO_VERSION: ${{ inputs.studio_version }}
- INPUT_PYPI_VERSION: ${{ inputs.pypi_version }}
- run: |
- python3 <<'PY'
- import os
- import pathlib
- import re
- import sys
-
- studio_version = os.environ['INPUT_STUDIO_VERSION'].strip()
- if not studio_version:
- sys.exit('studio_version is required, for example v0.1.39-beta')
- if re.fullmatch(r'v?20\d{2}\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?', studio_version):
- sys.exit(f'studio_version must be a Studio SemVer tag, not a date-style backend version: {studio_version}')
-
- semver_tag = re.compile(
- r'^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)'
- r'(?:-[0-9A-Za-z.][0-9A-Za-z.-]*)?$'
- )
- if not semver_tag.fullmatch(studio_version):
- sys.exit(f'studio_version must be a SemVer tag with leading v, for example v0.1.39-beta: {studio_version}')
-
- app_version = studio_version.removeprefix('v')
- desktop_release_tag = f'desktop-v{app_version}'
- prerelease = 'true' if '-' in app_version.split('+', 1)[0] else 'false'
-
- def parse_backend_version(version):
- match = re.fullmatch(
- r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)'
- r'(?:([a-zA-Z]|\.dev|dev|\.rc|rc|\.post|post)(\d*))?'
- r'(?:[-+]([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?',
- version,
- )
- if not match:
- return None
- major, minor, patch, suffix_name, suffix_number, suffix_text = match.groups()
- if suffix_name:
- normalized = suffix_name.lower().lstrip('.')
- order = {'dev': 0, 'a': 1, 'b': 2, 'rc': 3, 'post': 5}.get(normalized)
- if order is None:
- return None
- number = int(suffix_number or '0')
- elif suffix_text:
- order = 3 if version[version.find(suffix_text) - 1] == '-' else 4
- number = 0
- else:
- order = 4
- number = 0
- return (int(major), int(minor), int(patch), order, number)
-
- preflight = pathlib.Path('studio/src-tauri/src/preflight/version.rs').read_text()
- match = re.search(r'MIN_DESKTOP_BACKEND_VERSION:\s*&str\s*=\s*"([^"]+)"', preflight)
- if not match:
- sys.exit('Could not read MIN_DESKTOP_BACKEND_VERSION')
- min_backend_version = match.group(1)
-
- input_pypi_version = os.environ.get('INPUT_PYPI_VERSION', '').strip()
- parsed_min_backend = parse_backend_version(min_backend_version)
- if parsed_min_backend is None:
- sys.exit(f'MIN_DESKTOP_BACKEND_VERSION is not a supported backend package version: {min_backend_version}')
-
- pypi_version = input_pypi_version or min_backend_version
- parsed_pypi = parse_backend_version(pypi_version)
- if parsed_pypi is None:
- sys.exit(f'pypi_version is not a supported backend package version: {pypi_version}')
- if parsed_pypi < parsed_min_backend:
- sys.exit(
- f'pypi_version {pypi_version} is lower than desktop minimum '
- f'MIN_DESKTOP_BACKEND_VERSION {min_backend_version}'
- )
-
- if input_pypi_version:
- print(
- 'Using exact PyPI unsloth version from pypi_version input: '
- f'{pypi_version} (desktop minimum: {min_backend_version})'
- )
- else:
- print(
- 'Using exact PyPI unsloth version from MIN_DESKTOP_BACKEND_VERSION: '
- f'{pypi_version}'
- )
-
- with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as output:
- print(f'studio_version={studio_version}', file=output)
- print(f'app_version={app_version}', file=output)
- print(f'desktop_release_tag={desktop_release_tag}', file=output)
- print(f'prerelease={prerelease}', file=output)
- print(f'pypi_version={pypi_version}', file=output)
- PY
-
- - name: Verify PyPI package and Studio stamp
- shell: bash
- env:
- STUDIO_VERSION: ${{ steps.prepare.outputs.studio_version }}
- PYPI_VERSION: ${{ steps.prepare.outputs.pypi_version }}
- run: |
- set -euo pipefail
- python3 <<'PY'
- import json
- import os
- import pathlib
- import sys
- import time
- import urllib.error
- import urllib.request
-
- pypi_version = os.environ['PYPI_VERSION']
- dist_dir = pathlib.Path(os.environ['RUNNER_TEMP'], 'pypi-unsloth-dist')
- dist_dir.mkdir(parents=True, exist_ok=True)
- metadata_url = f'https://pypi.org/pypi/unsloth/{pypi_version}/json'
-
- last_error = None
- for attempt in range(1, 6):
- try:
- with urllib.request.urlopen(metadata_url, timeout=30) as response:
- metadata = json.load(response)
- break
- except Exception as exc:
- last_error = exc
- if attempt < 5:
- time.sleep(10 * attempt)
- else:
- sys.exit(f'Publish unsloth=={pypi_version} to PyPI before the desktop release ({last_error})')
-
- files = metadata.get('urls') or []
- if not files:
- sys.exit(f'PyPI returned no distribution files for unsloth=={pypi_version}')
-
- for file_info in files:
- filename = file_info.get('filename')
- url = file_info.get('url')
- if not filename or '/' in filename or not url:
- sys.exit(f'Unexpected PyPI file entry for unsloth=={pypi_version}: {file_info!r}')
- target = dist_dir / filename
- for attempt in range(1, 4):
- try:
- with urllib.request.urlopen(url, timeout=60) as response:
- target.write_bytes(response.read())
- break
- except Exception as exc:
- last_error = exc
- if attempt < 3:
- time.sleep(5 * attempt)
- else:
- sys.exit(f'Could not download {filename} from PyPI ({last_error})')
- PY
-
- if [ -f scripts/stamp_studio_release.py ]; then
- mapfile -t dists < <(find "$RUNNER_TEMP/pypi-unsloth-dist" -type f \( -name '*.whl' -o -name '*.tar.gz' \) | sort)
- if [ "${#dists[@]}" -eq 0 ]; then
- echo "No PyPI wheel/sdist artifacts downloaded for unsloth==$PYPI_VERSION" >&2
- exit 1
- fi
- python3 scripts/stamp_studio_release.py --verify-dist "$RUNNER_TEMP/pypi-unsloth-dist" --expected "$STUDIO_VERSION"
- else
- echo "scripts/stamp_studio_release.py not found; release-desktop requires #5308 to verify the PyPI Studio stamp." >&2
- exit 1
- fi
-
- - name: Guard public updater channel version
- if: ${{ !inputs.draft }}
- shell: bash
- env:
- GH_REPO: ${{ github.repository }}
- GH_TOKEN: ${{ github.token }}
- APP_VERSION: ${{ steps.prepare.outputs.app_version }}
- run: |
- set -euo pipefail
- mkdir -p "$RUNNER_TEMP/desktop-current"
- if ! gh release download desktop-latest --pattern latest.json --dir "$RUNNER_TEMP/desktop-current" --clobber 2>/dev/null; then
- echo "No existing desktop-latest latest.json found; allowing first channel publish."
- exit 0
- fi
- python3 <<'PY'
- import json
- import os
- import pathlib
- import re
- import sys
-
- def parse(value: str):
- value = value.removeprefix('v')
- match = re.fullmatch(
- r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)'
- r'(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?'
- r'(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?',
- value,
- )
- if not match:
- sys.exit(f'desktop-latest latest.json has invalid version: {value}')
- major, minor, patch, prerelease = match.groups()
- return (int(major), int(minor), int(patch), prerelease)
-
- def numeric_tail(identifier: str) -> tuple[str, int] | None:
- match = re.fullmatch(r'([A-Za-z-]+)(\d+)', identifier)
- if not match:
- return None
- return (match.group(1).lower(), int(match.group(2)))
-
- def compare_identifier(left: str, right: str) -> int:
- left_num = left.isdigit()
- right_num = right.isdigit()
- if left_num and right_num:
- return (int(left) > int(right)) - (int(left) < int(right))
- if left_num:
- return -1
- if right_num:
- return 1
-
- left_tail = numeric_tail(left)
- right_tail = numeric_tail(right)
- if left_tail and right_tail and left_tail[0] == right_tail[0]:
- return (left_tail[1] > right_tail[1]) - (left_tail[1] < right_tail[1])
-
- return (left > right) - (left < right)
-
- def compare_prerelease(left: str | None, right: str | None) -> int:
- if left == right:
- return 0
- if left is None:
- return 1
- if right is None:
- return -1
- left_parts = left.split('.')
- right_parts = right.split('.')
- for left_part, right_part in zip(left_parts, right_parts):
- order = compare_identifier(left_part, right_part)
- if order:
- return order
- return (len(left_parts) > len(right_parts)) - (len(left_parts) < len(right_parts))
-
- def compare(left: str, right: str) -> int:
- left_major, left_minor, left_patch, left_pre = parse(left)
- right_major, right_minor, right_patch, right_pre = parse(right)
- left_core = (left_major, left_minor, left_patch)
- right_core = (right_major, right_minor, right_patch)
- if left_core != right_core:
- return (left_core > right_core) - (left_core < right_core)
- return compare_prerelease(left_pre, right_pre)
-
- current_path = pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-current', 'latest.json')
- current = json.loads(current_path.read_text()).get('version')
- next_version = os.environ['APP_VERSION']
- if not isinstance(current, str):
- sys.exit('desktop-latest latest.json has missing version')
- if compare(next_version, current) < 0:
- sys.exit(
- f'Refusing to publish {next_version}; desktop-latest currently points at newer version {current}.'
- )
- PY
-
- build:
- # TODO: split into a "build (no secrets)" + "publish (secrets)" job pair
- # with actions/upload-artifact handoff so the matrix build cannot
- # publish a Release on its own. The current matrix runs across
- # Linux/macOS/Windows in a single job, so the split needs artefact
- # collection across the OS matrix and is out of scope for this
- # hardening pass.
- permissions:
- contents: write # tauri-apps/tauri-action creates / uploads a GitHub Release
- strategy:
- fail-fast: false
- max-parallel: 1
- matrix:
- include:
- - platform: macos-latest
- args: '--target aarch64-apple-darwin'
- label: macOS (Apple Silicon)
- # - platform: macos-latest
- # args: '--target x86_64-apple-darwin'
- # label: macOS (Intel)
- - platform: ubuntu-22.04
- args: ''
- label: Linux (x64)
- - platform: windows-latest
- args: ''
- label: Windows (x64)
-
- name: Build ${{ matrix.label }}
- needs: prepare-version
- runs-on: ${{ matrix.platform }}
-
- env:
- FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
- APP_VERSION: ${{ needs.prepare-version.outputs.app_version }}
- STUDIO_VERSION: ${{ needs.prepare-version.outputs.studio_version }}
- DESKTOP_RELEASE_TAG: ${{ needs.prepare-version.outputs.desktop_release_tag }}
- DESKTOP_PRERELEASE: ${{ needs.prepare-version.outputs.prerelease }}
-
- steps:
- # harden-runner in audit mode: surfaces every egress destination in
- # the runner log so the allowlist for a future `egress-policy: block`
- # promotion can be derived from observed traffic. Audit mode is
- # cross-platform (Linux / macOS / Windows runners); blocking mode is
- # currently Linux-only, so we deliberately stay in audit until the
- # macOS + Windows codesign paths have been observed.
- - name: Harden runner (audit)
- uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
- with:
- egress-policy: audit
-
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
- with:
- persist-credentials: false
-
- # ── Linux dependencies ──
- - name: Install Linux dependencies
- if: matrix.platform == 'ubuntu-22.04'
- run: |
- sudo apt-get update
- sudo apt-get install -y libwebkit2gtk-4.1-dev libayatana-appindicator3-dev librsvg2-dev libxdo-dev libssl-dev patchelf
-
- # ── Node.js ──
- - name: Setup Node.js
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e
- with:
- node-version: 24
-
- - name: Install pinned Tauri CLI
- # Lifecycle scripts (esbuild native-binary postinstall, etc.) are
- # required for `vite build`. The pre-install lockfile structural
- # audit (lockfile_supply_chain_audit.py) is the practical defence
- # against the npm postinstall-dropper class -- it fires BEFORE any
- # tarball runs, on the injection pattern itself rather than an
- # advisory-DB lookup.
- run: npm install --save-dev --prefix studio @tauri-apps/cli@2.10.1 --no-fund --no-audit
-
- - name: Verify pinned Tauri CLI
- shell: bash
- run: |
- out="$(npx --prefix studio tauri --version)"
- echo "$out"
- if [ "$out" != "tauri-cli 2.10.1" ]; then
- echo "Expected tauri-cli 2.10.1, got $out" >&2
- exit 1
- fi
-
- - name: Verify desktop updater and Linux package config
- shell: bash
- run: |
- node <<'JS'
- const { readFileSync } = require('node:fs');
-
- const expected = 'https://github.com/unslothai/unsloth/releases/download/desktop-latest/latest.json';
- const config = JSON.parse(readFileSync('studio/src-tauri/tauri.conf.json', 'utf8'));
- const endpoints = config.plugins?.updater?.endpoints;
- if (!Array.isArray(endpoints) || endpoints.length !== 1) {
- throw new Error('Expected exactly one desktop updater endpoint');
- }
- if (endpoints[0] !== expected) {
- throw new Error('Desktop updater endpoint must be ' + expected + ', got ' + endpoints[0]);
- }
- if (endpoints.some((endpoint) => endpoint.includes('/releases/latest/'))) {
- throw new Error('Desktop updater endpoint must not use repo-wide /releases/latest/');
- }
-
- const targets = config.bundle?.targets;
- if (Array.isArray(targets) && targets.some((target) => String(target).toLowerCase() === 'rpm')) {
- throw new Error('Desktop release must not target RPM packages');
- }
- if (config.bundle?.linux?.rpm) {
- throw new Error('bundle.linux.rpm must not be configured');
- }
-
- const workflow = readFileSync('.github/workflows/release-desktop.yml', 'utf8');
- const lines = workflow.split(/\r?\n/);
- const releaseBodies = [];
- for (let i = 0; i < lines.length; i += 1) {
- const match = lines[i].match(/^(\s*)releaseBody:\s*\|\s*$/);
- if (!match) continue;
- const baseIndent = match[1].length;
- const bodyLines = [];
- i += 1;
- for (; i < lines.length; i += 1) {
- const line = lines[i];
- if (line.trim() === '') {
- bodyLines.push('');
- continue;
- }
- const indent = line.match(/^\s*/)[0].length;
- if (indent <= baseIndent) {
- i -= 1;
- break;
- }
- bodyLines.push(line.slice(baseIndent + 2));
- }
- releaseBodies.push(bodyLines.join('\n'));
- }
- if (releaseBodies.length === 0) {
- throw new Error('Expected at least one desktop release body');
- }
- for (const body of releaseBodies) {
- if (/\brpm\b|\.rpm/i.test(body)) {
- throw new Error('Desktop release body must not advertise RPM packages');
- }
- }
- JS
-
- - name: Install frontend dependencies
- working-directory: studio/frontend
- # Lifecycle scripts (esbuild native-binary postinstall, etc.) are
- # required for `vite build`. The pre-install lockfile structural
- # audit (lockfile_supply_chain_audit.py) is the practical defence
- # against the npm postinstall-dropper class -- it fires BEFORE any
- # tarball runs, on the injection pattern itself rather than an
- # advisory-DB lookup.
- run: npm install --no-fund --no-audit
-
- # ── Rust ──
- - name: Install Rust stable
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @ 2026-03-27
- with:
- targets: ${{ matrix.platform == 'macos-latest' && 'aarch64-apple-darwin,x86_64-apple-darwin' || '' }}
-
- - name: Patch desktop app version
- shell: bash
- working-directory: studio/src-tauri
- run: |
- set -euo pipefail
- if command -v python3 >/dev/null 2>&1; then
- PYTHON=python3
- else
- PYTHON=python
- fi
- "$PYTHON" <<'PY'
- import os
- import pathlib
- import re
- import sys
-
- app_version = os.environ['APP_VERSION']
- if not app_version:
- sys.exit('APP_VERSION is required')
-
- cargo_toml = pathlib.Path('Cargo.toml')
- lines = cargo_toml.read_text().splitlines(keepends=True)
- in_package = False
- patched = False
- for index, line in enumerate(lines):
- stripped = line.strip()
- if stripped == '[package]':
- in_package = True
- continue
- if stripped.startswith('[') and stripped.endswith(']'):
- in_package = False
- if in_package and re.fullmatch(r'version\s*=\s*"[^"]+"\s*', stripped):
- lines[index] = f'version = "{app_version}"\n'
- patched = True
- break
- if not patched:
- sys.exit('Could not patch [package] version in Cargo.toml')
- cargo_toml.write_text(''.join(lines))
-
- cargo_lock = pathlib.Path('Cargo.lock')
- lock_text = cargo_lock.read_text()
- lock_text, count = re.subn(
- r'(?m)(^\[\[package\]\]\nname = "unsloth-studio"\nversion = ")[^"]+(")',
- lambda match: f'{match.group(1)}{app_version}{match.group(2)}',
- lock_text,
- )
- if count != 1:
- sys.exit(f'Could not patch unsloth-studio version in Cargo.lock (matches={count})')
- cargo_lock.write_text(lock_text)
- PY
-
- cargo metadata --locked --no-deps --format-version 1 > "$RUNNER_TEMP/cargo-metadata.json"
- "$PYTHON" <<'PY'
- import json
- import os
- import pathlib
- import sys
-
- app_version = os.environ['APP_VERSION']
- metadata = json.loads(pathlib.Path(os.environ['RUNNER_TEMP'], 'cargo-metadata.json').read_text())
- versions = [package['version'] for package in metadata.get('packages', []) if package.get('name') == 'unsloth-studio']
- if versions != [app_version]:
- sys.exit(f'cargo metadata unsloth-studio version mismatch: expected {app_version}, got {versions}')
- PY
-
- git diff -- Cargo.toml Cargo.lock
-
- - name: Rust cache
- uses: swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32
- with:
- workspaces: 'studio/src-tauri -> target'
-
- # ── macOS: import signing certificate ──
- - name: Import Apple certificate
- if: matrix.platform == 'macos-latest'
- env:
- APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
- APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
- KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
- run: |
- echo $APPLE_CERTIFICATE | base64 --decode > certificate.p12
- security create-keychain -p "$KEYCHAIN_PASSWORD" build.keychain
- security default-keychain -s build.keychain
- security unlock-keychain -p "$KEYCHAIN_PASSWORD" build.keychain
- security set-keychain-settings -t 3600 -u build.keychain
- security import certificate.p12 -k build.keychain -P "$APPLE_CERTIFICATE_PASSWORD" -T /usr/bin/codesign
- security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$KEYCHAIN_PASSWORD" build.keychain
- security find-identity -v -p codesigning build.keychain
- rm -f certificate.p12
-
- # ── Windows: install Azure Trusted Signing CLI ──
- - name: Install trusted-signing-cli
- if: matrix.platform == 'windows-latest'
- run: |
- cargo install trusted-signing-cli --version 0.10.0 --locked
- echo "$env:USERPROFILE\.cargo\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-
- # ── Windows: verify signing CLI is accessible ──
- - name: Verify trusted-signing-cli
- if: matrix.platform == 'windows-latest'
- run: |
- Write-Output "PATH: $env:PATH"
- Get-Command trusted-signing-cli -ErrorAction SilentlyContinue || Write-Output "trusted-signing-cli NOT in PATH"
- trusted-signing-cli --version || Write-Output "trusted-signing-cli failed to run"
-
- # ── Linux: build + sign + upload ──
- - name: Build Linux app
- if: matrix.platform == 'ubuntu-22.04'
- uses: tauri-apps/tauri-action@84b9d35b5fc46c1e45415bdb6144030364f7ebc5
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }}
- TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}
- with:
- projectPath: studio
- tauriScript: npx --prefix . tauri
- tagName: ${{ needs.prepare-version.outputs.desktop_release_tag }}
- releaseName: 'Unsloth Studio (Desktop) ${{ needs.prepare-version.outputs.studio_version }}'
- releaseBody: |
- Desktop app for Unsloth Studio.
-
- **macOS**: Download the Apple Silicon `.dmg`.
- **Windows**: Download the `-setup.exe` installer.
- **Linux**: Download `.deb` (Ubuntu/Debian) or `.AppImage` (universal).
-
- > Linux in-app updates are AppImage-oriented. Package installs should update by downloading a new package.
- > Linux AppImage on Ubuntu 24.04+ may require: `sudo apt install libfuse2t64`
- > First-run system dependency elevation is supported on Ubuntu/Debian. Other Linux distributions should install system packages manually.
- releaseDraft: ${{ inputs.draft }}
- prerelease: ${{ needs.prepare-version.outputs.prerelease }}
- args: -v ${{ matrix.args }}
-
- # ── macOS: build + sign + notarize + upload ──
- - name: Build macOS app
- if: matrix.platform == 'macos-latest'
- uses: tauri-apps/tauri-action@84b9d35b5fc46c1e45415bdb6144030364f7ebc5
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }}
- TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}
- APPLE_SIGNING_IDENTITY: ${{ secrets.APPLE_SIGNING_IDENTITY }}
- APPLE_ID: ${{ secrets.APPLE_ID }}
- APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
- APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
- with:
- projectPath: studio
- tauriScript: npx --prefix . tauri
- tagName: ${{ needs.prepare-version.outputs.desktop_release_tag }}
- releaseName: 'Unsloth Studio (Desktop) ${{ needs.prepare-version.outputs.studio_version }}'
- releaseBody: |
- Desktop app for Unsloth Studio.
-
- **macOS**: Download the Apple Silicon `.dmg`.
- **Windows**: Download the `-setup.exe` installer.
- **Linux**: Download `.deb` (Ubuntu/Debian) or `.AppImage` (universal).
-
- > Linux in-app updates are AppImage-oriented. Package installs should update by downloading a new package.
- > Linux AppImage on Ubuntu 24.04+ may require: `sudo apt install libfuse2t64`
- > First-run system dependency elevation is supported on Ubuntu/Debian. Other Linux distributions should install system packages manually.
- releaseDraft: ${{ inputs.draft }}
- prerelease: ${{ needs.prepare-version.outputs.prerelease }}
- args: -v ${{ matrix.args }}
-
- # ── Windows: build + sign + upload ──
- - name: Build Windows app
- if: matrix.platform == 'windows-latest'
- uses: tauri-apps/tauri-action@84b9d35b5fc46c1e45415bdb6144030364f7ebc5
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }}
- TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}
- AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
- AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
- AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
- AZURE_TRUSTED_SIGNING_ACCOUNT_NAME: ${{ secrets.AZURE_TRUSTED_SIGNING_ACCOUNT_NAME }}
- AZURE_CERTIFICATE_PROFILE_NAME: ${{ secrets.AZURE_CERTIFICATE_PROFILE_NAME }}
- with:
- projectPath: studio
- tauriScript: npx --prefix . tauri
- tagName: ${{ needs.prepare-version.outputs.desktop_release_tag }}
- releaseName: 'Unsloth Studio (Desktop) ${{ needs.prepare-version.outputs.studio_version }}'
- releaseBody: |
- Desktop app for Unsloth Studio.
-
- **macOS**: Download the Apple Silicon `.dmg`.
- **Windows**: Download the `-setup.exe` installer.
- **Linux**: Download `.deb` (Ubuntu/Debian) or `.AppImage` (universal).
-
- > Linux in-app updates are AppImage-oriented. Package installs should update by downloading a new package.
- > Linux AppImage on Ubuntu 24.04+ may require: `sudo apt install libfuse2t64`
- > First-run system dependency elevation is supported on Ubuntu/Debian. Other Linux distributions should install system packages manually.
- releaseDraft: ${{ inputs.draft }}
- prerelease: ${{ needs.prepare-version.outputs.prerelease }}
- args: -v ${{ matrix.args }}
-
- # Release process note: only non-draft workflow runs advance the public
- # desktop-latest updater channel. Draft builds are for private review; if a
- # draft is manually published later, this channel intentionally remains
- # unchanged until a narrow manual channel-publish flow is added or a public
- # desktop release is created by running this workflow with draft=false.
- publish-updater-channel:
- name: Publish desktop updater channel
- needs: [prepare-version, build]
- if: ${{ !inputs.draft }}
- runs-on: ubuntu-latest
- permissions:
- contents: write
- env:
- GH_REPO: ${{ github.repository }}
- APP_VERSION: ${{ needs.prepare-version.outputs.app_version }}
- STUDIO_VERSION: ${{ needs.prepare-version.outputs.studio_version }}
- DESKTOP_RELEASE_TAG: ${{ needs.prepare-version.outputs.desktop_release_tag }}
- DESKTOP_PRERELEASE: ${{ needs.prepare-version.outputs.prerelease }}
-
- steps:
- - name: Download versioned updater metadata
- shell: bash
- env:
- GH_TOKEN: ${{ github.token }}
- run: |
- set -euo pipefail
- mkdir -p "$RUNNER_TEMP/desktop-updater"
- gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${DESKTOP_RELEASE_TAG}" > "$RUNNER_TEMP/source-release.json"
- python3 <<'PY'
- import json
- import os
- import pathlib
- import sys
-
- source = json.loads(pathlib.Path(os.environ['RUNNER_TEMP'], 'source-release.json').read_text())
- expected_tag = os.environ['DESKTOP_RELEASE_TAG']
- if source.get('tag_name') != expected_tag:
- sys.exit(f'Expected source release {expected_tag}, got {source.get("tag_name")}')
- if source.get('draft'):
- sys.exit(f'Source desktop release {expected_tag} is draft; refusing to publish public updater channel')
- PY
- gh release download "$DESKTOP_RELEASE_TAG" --pattern latest.json --dir "$RUNNER_TEMP/desktop-updater" --clobber
- test -s "$RUNNER_TEMP/desktop-updater/latest.json"
-
- - name: Validate versioned updater metadata
- shell: bash
- run: |
- python3 <<'PY'
- import json
- import os
- import pathlib
- import re
- import sys
-
- app_version = os.environ['APP_VERSION']
- release_tag = os.environ['DESKTOP_RELEASE_TAG']
- latest_path = pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-updater', 'latest.json')
- data = json.loads(latest_path.read_text())
- if not isinstance(data, dict):
- sys.exit('latest.json must be a JSON object')
-
- version = data.get('version')
- if not isinstance(version, str) or not version:
- sys.exit('latest.json missing version')
- if not re.fullmatch(r'v?\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?', version):
- sys.exit(f'latest.json version is not SemVer-like: {version}')
- if version.removeprefix('v') != app_version:
- sys.exit(f'latest.json version {version} does not match desktop app version {app_version}')
-
- platforms = data.get('platforms')
- if not isinstance(platforms, dict) or not platforms:
- sys.exit('latest.json missing platforms')
-
- required_families = {
- 'darwin-aarch64': False,
- 'linux-x86_64': False,
- 'windows-x86_64': False,
- }
- expected_prefix = f'https://github.com/unslothai/unsloth/releases/download/{release_tag}/'
- forbidden_fragments = ('/releases/latest/', '/releases/download/desktop-latest/')
-
- for platform, entry in platforms.items():
- if not isinstance(entry, dict):
- sys.exit(f'Platform {platform} must be an object')
- url = entry.get('url')
- signature = entry.get('signature')
- if not isinstance(url, str) or not url.strip():
- sys.exit(f'Platform {platform} missing url')
- if not isinstance(signature, str) or not signature.strip():
- sys.exit(f'Platform {platform} missing signature')
- if any(fragment in url for fragment in forbidden_fragments):
- sys.exit(f'Platform {platform} points at a moving updater channel: {url}')
- if not url.startswith(expected_prefix):
- sys.exit(f'Platform {platform} URL must point at {release_tag}: {url}')
- for family in required_families:
- if platform == family or platform.startswith(family + '-'):
- required_families[family] = True
-
- missing = [family for family, found in required_families.items() if not found]
- if missing:
- sys.exit('latest.json missing required platform families: ' + ', '.join(missing))
- PY
-
- - name: Ensure desktop updater channel release
- shell: bash
- env:
- GH_TOKEN: ${{ github.token }}
- run: |
- set -euo pipefail
- channel_json="$RUNNER_TEMP/desktop-latest-release.json"
- if ! gh api "repos/${GITHUB_REPOSITORY}/releases/tags/desktop-latest" > "$channel_json" 2>/dev/null; then
- gh release create desktop-latest \
- --title "Unsloth Studio Desktop updater channel" \
- --notes "Machine-managed desktop updater channel; latest.json is replaced by release-desktop.yml." \
- --prerelease \
- --latest=false \
- --target "$GITHUB_SHA"
- gh api "repos/${GITHUB_REPOSITORY}/releases/tags/desktop-latest" > "$channel_json"
- fi
-
- python3 <<'PY'
- import json
- import os
- import pathlib
- import sys
-
- channel = json.loads(pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-latest-release.json').read_text())
- if channel.get('draft'):
- sys.exit('desktop-latest release is draft; refusing to publish updater channel')
- if channel.get('immutable'):
- sys.exit('desktop-latest release is immutable; cannot replace latest.json')
- if not channel.get('prerelease'):
- sys.exit('desktop-latest release must be a prerelease so it cannot compete with repo-wide latest')
- PY
-
- - name: Prevent updater channel downgrade
- shell: bash
- env:
- GH_TOKEN: ${{ github.token }}
- run: |
- set -euo pipefail
- mkdir -p "$RUNNER_TEMP/desktop-current"
- if ! gh release download desktop-latest --pattern latest.json --dir "$RUNNER_TEMP/desktop-current" --clobber 2>/dev/null; then
- echo "No existing desktop-latest latest.json found; allowing first channel publish."
- exit 0
- fi
- python3 <<'PY'
- import json
- import os
- import pathlib
- import re
- import sys
-
- def parse(value: str):
- value = value.removeprefix('v')
- match = re.fullmatch(
- r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)'
- r'(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?'
- r'(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?',
- value,
- )
- if not match:
- sys.exit(f'desktop-latest latest.json has invalid version: {value}')
- major, minor, patch, prerelease = match.groups()
- return (int(major), int(minor), int(patch), prerelease)
-
- def numeric_tail(identifier: str) -> tuple[str, int] | None:
- match = re.fullmatch(r'([A-Za-z-]+)(\d+)', identifier)
- if not match:
- return None
- return (match.group(1).lower(), int(match.group(2)))
-
- def compare_identifier(left: str, right: str) -> int:
- left_num = left.isdigit()
- right_num = right.isdigit()
- if left_num and right_num:
- return (int(left) > int(right)) - (int(left) < int(right))
- if left_num:
- return -1
- if right_num:
- return 1
-
- left_tail = numeric_tail(left)
- right_tail = numeric_tail(right)
- if left_tail and right_tail and left_tail[0] == right_tail[0]:
- return (left_tail[1] > right_tail[1]) - (left_tail[1] < right_tail[1])
-
- return (left > right) - (left < right)
-
- def compare_prerelease(left: str | None, right: str | None) -> int:
- if left == right:
- return 0
- if left is None:
- return 1
- if right is None:
- return -1
- left_parts = left.split('.')
- right_parts = right.split('.')
- for left_part, right_part in zip(left_parts, right_parts):
- order = compare_identifier(left_part, right_part)
- if order:
- return order
- return (len(left_parts) > len(right_parts)) - (len(left_parts) < len(right_parts))
-
- def compare(left: str, right: str) -> int:
- left_major, left_minor, left_patch, left_pre = parse(left)
- right_major, right_minor, right_patch, right_pre = parse(right)
- left_core = (left_major, left_minor, left_patch)
- right_core = (right_major, right_minor, right_patch)
- if left_core != right_core:
- return (left_core > right_core) - (left_core < right_core)
- return compare_prerelease(left_pre, right_pre)
-
- current_path = pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-current', 'latest.json')
- next_path = pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-updater', 'latest.json')
- current = json.loads(current_path.read_text()).get('version')
- next_version = json.loads(next_path.read_text()).get('version')
- if not isinstance(current, str) or not isinstance(next_version, str):
- sys.exit('Could not compare desktop-latest channel versions')
- if compare(next_version, current) < 0:
- sys.exit(
- f'Refusing to move desktop-latest from {current} to older version {next_version}.'
- )
- PY
-
- - name: Publish desktop updater channel metadata
- shell: bash
- env:
- GH_TOKEN: ${{ github.token }}
- run: |
- set -euo pipefail
- gh release upload desktop-latest "$RUNNER_TEMP/desktop-updater/latest.json" --clobber
- gh api "repos/${GITHUB_REPOSITORY}/releases/tags/desktop-latest" > "$RUNNER_TEMP/desktop-latest-release.json"
- python3 <<'PY'
- import json
- import os
- import pathlib
- import sys
-
- channel = json.loads(pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-latest-release.json').read_text())
- assets = [asset for asset in channel.get('assets', []) if asset.get('name') == 'latest.json']
- if len(assets) != 1:
- sys.exit(f'Expected exactly one desktop-latest latest.json asset, found {len(assets)}')
- expected_url = f'https://github.com/{os.environ["GITHUB_REPOSITORY"]}/releases/download/desktop-latest/latest.json'
- actual_url = assets[0].get('browser_download_url')
- if actual_url != expected_url:
- sys.exit(f'desktop-latest latest.json URL mismatch: expected {expected_url}, got {actual_url}')
- PY
diff --git a/.github/workflows/security-audit.yml b/.github/workflows/security-audit.yml
deleted file mode 100644
index a1e7b2efa6..0000000000
--- a/.github/workflows/security-audit.yml
+++ /dev/null
@@ -1,1126 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# Multi-language supply-chain audit. Triggers:
-# - PRs touching any dependency manifest (Python / npm / Cargo) or
-# this workflow file,
-# - push to main / pip,
-# - nightly @ 04:13 UTC so newly-published advisories surface even
-# when no PR opens,
-# - workflow_dispatch for ad-hoc invocations.
-#
-# Two jobs:
-# - advisory-audit: one runner that runs pip-audit + npm audit +
-# cargo audit back-to-back. All three are
-# advisory-DB lookups -- fast, lockfile-driven,
-# no archive download. Setting up the python /
-# node / rust toolchains on one runner and
-# running the three commands serially is
-# cheaper than spinning up three runners.
-# - pip-scan-packages: 3-shard matrix that downloads + pattern-scans
-# every PyPI archive in the transitive closure.
-# This is the expensive job (~6 min/shard,
-# running in parallel) and it must stay
-# independent so a CVE-DB hit in advisory-audit
-# does not block the supply-chain pattern scan
-# (or vice versa).
-#
-# All steps are non-blocking initially. The default branch already
-# carries a known-vuln backlog (the dependabot banner shows 17 today,
-# pip-audit catches 2 more, npm/cargo will catch their own); a hard
-# gate now would block every PR on a baseline we have not triaged.
-# As each baseline closes, drop continue-on-error per step.
-#
-# Dependency coverage:
-# - unsloth core (pyproject.toml [project.dependencies])
-# - unsloth `huggingfacenotorch` extras (the canonical install path
-# for fine-tuning users; pulls transformers / peft / accelerate /
-# trl / datasets / diffusers / sentence-transformers / etc.)
-# - all six Studio backend requirements files
-# - Studio frontend (npm) and Tauri shell (cargo)
-# Each Python step builds a filtered dep list from pyproject.toml +
-# requirements/*.txt before auditing. We do NOT install any of these
-# -- pip-audit resolves through PyPI metadata, scan_packages.py
-# downloads sdist/wheel archives and inspects them without running
-# install hooks, so an attacker who has compromised a transitive dep
-# cannot execute code in this workflow.
-
-name: Security audit
-
-on:
- pull_request:
- paths:
- - 'studio/backend/requirements/**'
- - 'studio/frontend/package.json'
- - 'studio/frontend/package-lock.json'
- - 'studio/src-tauri/Cargo.toml'
- - 'studio/src-tauri/Cargo.lock'
- - 'pyproject.toml'
- - 'scripts/scan_packages.py'
- - 'scripts/scan_npm_packages.py'
- - '.github/workflows/security-audit.yml'
- push:
- branches: [main, pip]
- schedule:
- - cron: '13 4 * * *' # 04:13 UTC daily, off the cron rush
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- # ─────────────────────────────────────────────────────────────────────
- # Combined advisory-DB audit: pip-audit + npm audit + cargo audit
- # all on one runner. Each step is continue-on-error so a finding in
- # one toolchain does not suppress the others.
- # ─────────────────────────────────────────────────────────────────────
- advisory-audit:
- name: advisory audit (pip + npm + cargo)
- runs-on: ubuntu-latest
- timeout-minutes: 25
- steps:
- # step-security/harden-runner installs an eBPF-based egress
- # firewall on the runner. In `audit` mode it logs every outbound
- # connection without blocking; in `block` mode it rejects
- # anything outside `allowed-endpoints`. We run audit-only
- # initially: the next time this job hits a real PyPI advisory or
- # an attacker-funded archive in pip-scan-packages, the audit log
- # tells us exactly which hosts were dialed and we promote the
- # allowlist to block. Would have *contained* the litellm exfil
- # even if scan_packages had missed the .pth payload.
- # SHA-pinned (not @v2): the litellm 1.82.7 attack chain hijacked
- # mutable tags on aquasecurity/trivy-action and would have hit
- # anyone using @v0 / @v2 / @latest references. Pinning to a 40-
- # char SHA freezes this action at known-good code; Dependabot's
- # github-actions ecosystem will auto-bump the SHA.
- # v2.19.1 commit:
- # Per-job allowlist: advisory-audit hits PyPI, npm registry,
- # crates.io advisories, GitHub release artefacts (osv-scanner
- # binary), Semgrep registry, and TruffleHog's own GitHub action.
- - name: Harden runner (egress block)
- uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
- with:
- egress-policy: block
- disable-sudo: true
- allowed-endpoints: >
- api.github.com:443
- github.com:443
- codeload.github.com:443
- objects.githubusercontent.com:443
- raw.githubusercontent.com:443
- release-assets.githubusercontent.com:443
- registry.npmjs.org:443
- pypi.org:443
- files.pythonhosted.org:443
- static.rust-lang.org:443
- index.crates.io:443
- static.crates.io:443
- crates.io:443
- semgrep.dev:443
-
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- # Full history so TruffleHog can diff base..head; without
- # this it sees only the latest commit and reports nothing.
- fetch-depth: 0
- persist-credentials: false
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @ 2026-03-27
-
- - uses: swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2.9.1
- with:
- workspaces: studio/src-tauri -> target
-
- - name: Install pip-audit + cargo-audit
- # cargo-audit pulls advisories from the RustSec advisory-db on
- # first run and caches them under ~/.cargo/advisory-db. Pin
- # --locked so the version we install matches Cargo.lock
- # determinism. cargo-audit 0.22 supports the CVSS 4.0 schema
- # used in 2026 advisories (e.g. RUSTSEC-2026-0073); 0.21
- # crashes with a TOML parse error on that file.
- # npm audit is bundled with the node toolchain, no install.
- run: |
- python -m pip install --upgrade pip 'pip-audit>=2.7'
- cargo install --locked --version '^0.22' cargo-audit
-
- # ─────────────────────────────────────────────────────────────
- # Python: pip-audit
- # ─────────────────────────────────────────────────────────────
- - name: Build filtered Python requirements set
- # Two transforms:
- # (1) Generate audit-reqs/unsloth-deps.txt from pyproject.toml
- # so pip-audit sees the unsloth pip package's own dep set
- # (core + huggingfacenotorch extras: transformers / peft /
- # accelerate / trl / datasets / diffusers /
- # sentence-transformers / huggingface_hub / hf_transfer /
- # etc.).
- # (2) Copy each studio/backend/requirements/*.txt into
- # audit-reqs/ with `git+` lines stripped. pip-audit's `-r`
- # mode does a dry-run resolve against PyPI metadata; a
- # `git+https://...` spec forces it to clone, which is
- # both slow and outside the threat model (we audit
- # PyPI-served archives; a git ref is whatever HEAD says
- # on the runner). A comment line is left in place so the
- # skipped specs are obvious in the artifact.
- # The `huggingface` extra is `huggingfacenotorch` plus torch /
- # torchvision / triton, deliberately skipped: Studio backend
- # already pins a torch and the +cu* / +cpu local-version tags
- # trip up the PyPI resolver in `-r` mode.
- run: |
- mkdir -p audit-reqs
- python <<'PY' > audit-reqs/unsloth-deps.txt
- import tomllib
- with open("pyproject.toml", "rb") as f:
- d = tomllib.load(f)
- core = d["project"]["dependencies"]
- extras = d["project"]["optional-dependencies"]["huggingfacenotorch"]
- print("# Auto-generated from pyproject.toml by security-audit.yml.")
- print("# core deps + huggingfacenotorch extras.")
- for spec in core + extras:
- print(spec)
- PY
- for f in studio.txt extras.txt extras-no-deps.txt \
- no-torch-runtime.txt overrides.txt triton-kernels.txt; do
- python < "audit-reqs/$f"
- src = "studio/backend/requirements/$f"
- with open(src) as fh:
- for line in fh:
- stripped = line.strip()
- before_comment = stripped.split("#", 1)[0]
- if "git+" in before_comment:
- print(f"# [security-audit] skipped git+ spec: {stripped}")
- continue
- print(line.rstrip("\n"))
- PY
- done
-
- - name: pip-audit (declared Python deps, no install)
- # `-r requirements.txt` resolves the requirements through pip's
- # dependency resolver against PyPI metadata and audits the
- # resolved tree without ever executing setup.py / install
- # hooks. Way faster than installing the full Studio runtime
- # and -- critically -- safer: an attacker who has compromised
- # a transitive dep cannot run code in this job.
- #
- # extras.txt + extras-no-deps.txt have legacy setup.py
- # packages (notably openai-whisper) whose setup.py imports
- # `pkg_resources`, which the isolated build env's current
- # setuptools no longer ships. PIP_CONSTRAINT pins an older
- # setuptools into the build env so those builds resolve.
- # Per-file loop so one bad file doesn't take out the whole
- # audit.
- continue-on-error: true
- env:
- PIP_CONSTRAINT: ${{ github.workspace }}/audit-reqs/build-constraints.txt
- run: |
- set +e
- cat > audit-reqs/build-constraints.txt <<'CONSTRAINTS'
- setuptools<78
- wheel
- CONSTRAINTS
- : > logs-pip-audit.txt
- for f in unsloth-deps studio extras extras-no-deps \
- no-torch-runtime overrides triton-kernels; do
- if ! grep -qE '^[^#[:space:]]' "audit-reqs/$f.txt"; then
- echo "[security-audit] $f.txt has no PyPI specs after git+ filter, skipping" \
- | tee -a logs-pip-audit.txt
- continue
- fi
- echo "::group::pip-audit -r audit-reqs/$f.txt"
- {
- echo
- echo "=== $f ==="
- pip-audit -r "audit-reqs/$f.txt" --format=columns
- echo "=== end $f (rc=$?) ==="
- } 2>&1 | tee -a logs-pip-audit.txt
- echo "::endgroup::"
- done
- {
- echo "## pip-audit (Python)"
- echo
- echo '### Coverage'
- echo '- unsloth core + `huggingfacenotorch` extras (pyproject.toml)'
- echo '- studio/backend/requirements/{studio,extras,extras-no-deps,no-torch-runtime,overrides,triton-kernels}.txt'
- echo '- `git+` specs are stripped before audit (out of scope: we audit PyPI archives)'
- echo
- echo '### Findings'
- echo '```'
- cat logs-pip-audit.txt
- echo '```'
- } >> "$GITHUB_STEP_SUMMARY"
-
- # ─────────────────────────────────────────────────────────────
- # Pre-install lockfile supply-chain audit (npm + cargo).
- # Catches structural anomalies (non-registry resolved URLs,
- # missing integrity hashes, known IOC strings) BEFORE `npm
- # audit` or OSV-Scanner consult the advisory DB. The advisory
- # path is reactive -- there is a window between a malicious
- # publication and the GHSA landing. This step fires on the
- # injection pattern itself so it catches the same class of
- # attack the moment the lockfile shape becomes wrong.
- # ─────────────────────────────────────────────────────────────
- - name: Lockfile supply-chain audit (pre-install scan)
- run: |
- python3 scripts/lockfile_supply_chain_audit.py
- {
- echo "## Lockfile supply-chain audit"
- echo
- echo "Scanned: studio/frontend/package-lock.json + studio/src-tauri/Cargo.lock"
- echo
- echo "No structural anomalies or known IOC strings."
- } >> "$GITHUB_STEP_SUMMARY"
-
- # ─────────────────────────────────────────────────────────────
- # npm: Studio frontend
- # ─────────────────────────────────────────────────────────────
- - name: npm audit (Studio frontend)
- # `npm audit` resolves the lockfile through the npmjs.com
- # advisory DB. `--audit-level=high` filters the noise floor
- # to only HIGH and CRITICAL. We do NOT pass --omit=dev: a
- # malicious dev-only dep can still steal secrets from a CI
- # runner, so dev deps need to be in the audit surface.
- continue-on-error: true
- working-directory: studio/frontend
- run: |
- set +e
- npm audit --audit-level=high | tee ../../logs-npm-audit.txt
- # Always also write the full JSON for grep-ability.
- npm audit --json > ../../logs-npm-audit.json || true
- {
- echo "## npm audit (Studio frontend)"
- echo
- echo '```'
- tail -200 ../../logs-npm-audit.txt
- echo '```'
- } >> "$GITHUB_STEP_SUMMARY"
-
- # ─────────────────────────────────────────────────────────────
- # cargo: Studio Tauri shell
- # ─────────────────────────────────────────────────────────────
- - name: cargo audit (Studio Tauri)
- # `--deny warnings` would make the job fail on any advisory.
- # Keep non-blocking initially; drop continue-on-error after
- # the baseline closes.
- continue-on-error: true
- working-directory: studio/src-tauri
- run: |
- set +e
- cargo audit | tee ../../logs-cargo-audit.txt
- {
- echo "## cargo audit (Studio Tauri)"
- echo
- echo '```'
- tail -200 ../../logs-cargo-audit.txt
- echo '```'
- } >> "$GITHUB_STEP_SUMMARY"
-
- # ─────────────────────────────────────────────────────────────
- # OSV-Scanner: cross-ecosystem advisory DB (PyPI + npm + cargo)
- # ─────────────────────────────────────────────────────────────
- - name: OSV-Scanner (PyPI + npm + cargo, cross-ecosystem advisories)
- # OSV's advisory feed is a superset of GitHub-Advisory + RustSec
- # + npm advisories; running it alongside the per-ecosystem audit
- # tools catches CVEs that haven't propagated to the per-ecosystem
- # DBs yet (e.g. langchain-core CVE-2025-68664 was on OSV before
- # GitHub Advisory). Single binary, one transitive resolver, all
- # three lockfile types in one pass. Non-blocking until baselines
- # close.
- continue-on-error: true
- run: |
- set +e
- # OSV-Scanner ships a raw binary (no tarball) in v2.x.
- curl -fsSL -o /tmp/osv-scanner \
- https://github.com/google/osv-scanner/releases/download/v2.0.2/osv-scanner_linux_amd64
- chmod +x /tmp/osv-scanner
- /tmp/osv-scanner --version
- /tmp/osv-scanner scan source \
- --lockfile=studio/frontend/package-lock.json \
- --lockfile=studio/src-tauri/Cargo.lock \
- --lockfile=requirements.txt:audit-reqs/unsloth-deps.txt \
- --lockfile=requirements.txt:audit-reqs/studio.txt \
- --lockfile=requirements.txt:audit-reqs/no-torch-runtime.txt \
- --lockfile=requirements.txt:audit-reqs/overrides.txt \
- --lockfile=requirements.txt:audit-reqs/extras.txt \
- --lockfile=requirements.txt:audit-reqs/extras-no-deps.txt \
- --format=table 2>&1 | tee logs-osv-scanner.txt
- {
- echo "## OSV-Scanner (cross-ecosystem)"
- echo
- echo '```'
- tail -200 logs-osv-scanner.txt
- echo '```'
- } >> "$GITHUB_STEP_SUMMARY"
-
- # ─────────────────────────────────────────────────────────────
- # Semgrep: design-flaw detection (catches what regex-pattern
- # scanning of malicious authors cannot — first-party logic bugs
- # like langchain-core CVE-2025-68664 dumps/dumpd injection,
- # n8n CVE-2025-68668 _pyodide.eval_code sandbox escape, marimo
- # CVE-2026-39987 unauth WebSocket).
- # ─────────────────────────────────────────────────────────────
- - name: Semgrep (supply-chain + python rule packs)
- continue-on-error: true
- run: |
- set +e
- python -m pip install --quiet 'semgrep>=1.95'
- semgrep --version
- semgrep scan \
- --config p/supply-chain \
- --config p/python \
- --config p/javascript \
- --config p/security-audit \
- --severity ERROR --severity WARNING \
- --metrics off \
- --timeout 120 \
- studio/backend unsloth scripts \
- 2>&1 | tee logs-semgrep.txt
- {
- echo "## Semgrep (supply-chain + python + javascript rules)"
- echo
- echo '```'
- tail -200 logs-semgrep.txt
- echo '```'
- } >> "$GITHUB_STEP_SUMMARY"
-
- # ─────────────────────────────────────────────────────────────
- # Lockfile pin verifier. The litellm 1.82.7 attack window was
- # ~40 minutes; anyone resolving with `>=` got the malicious
- # version automatically. Flag every spec in the requirements
- # files that does not pin to an exact `==` (or `@` for git
- # refs, or `===` for arbitrary equality). Warning-only for now;
- # graduate to blocking once the baseline is clean.
- # ─────────────────────────────────────────────────────────────
- - name: Lockfile pin verifier (Python requirements)
- continue-on-error: true
- run: |
- python <<'PY' | tee logs-pin-verifier.txt
- import re
- from pathlib import Path
-
- # Specs that look like `pkg==1.2.3` or `pkg @ git+...` or
- # bare comments / -r lines are pinned-or-not-applicable.
- PINNED = re.compile(r"^\s*[A-Za-z0-9_.\-]+\s*(?:===|==)\s*[^,;]+\s*$")
- GIT_OR_URL = re.compile(r"^\s*[A-Za-z0-9_.\-]+\s*@\s*(?:git\+|https?://)")
-
- unpinned = []
- for f in sorted(Path("studio/backend/requirements").glob("*.txt")):
- for i, raw in enumerate(f.read_text().splitlines(), 1):
- line = raw.strip()
- if not line or line.startswith("#") or line.startswith("-"):
- continue
- spec = line.split("#", 1)[0].strip().split(";", 1)[0].strip()
- if not spec:
- continue
- if "git+" in spec or PINNED.match(spec) or GIT_OR_URL.match(spec):
- continue
- unpinned.append((str(f), i, line))
-
- print(f"::group::Lockfile pin status")
- if unpinned:
- print(f"WARN: {len(unpinned)} non-`==` specs across requirements/*.txt")
- print("(litellm 1.82.7 wave hit anyone on `>=`; tighten when feasible.)")
- for f, i, line in unpinned[:80]:
- print(f" {f}:{i}: {line}")
- if len(unpinned) > 80:
- print(f" ... and {len(unpinned) - 80} more")
- else:
- print("OK: every spec is exact-pinned.")
- print("::endgroup::")
- PY
- {
- echo "## Lockfile pin verifier"
- echo
- echo '```'
- cat logs-pin-verifier.txt
- echo '```'
- } >> "$GITHUB_STEP_SUMMARY"
-
- # ─────────────────────────────────────────────────────────────
- # Trivy is deliberately NOT installed here. Trivy was the entry
- # point for the litellm 1.82.7 supply-chain compromise (March
- # 2026): attackers force-rewrote 76 of 77 tags in
- # aquasecurity/trivy-action to point at malicious commits;
- # anyone running the action with a tag ref auto-pulled a
- # credential-harvesting payload. By design a security scanner
- # has broad read access to runner secrets, which is exactly
- # what made it the ideal pivot. We pick up Trivy's CVE coverage
- # from OSV-Scanner (NVD + GHSA + GitLab) and its secret
- # detection from TruffleHog. IaC misconfig detection (Trivy's
- # one unique value-add) is unfilled for now -- revisit with
- # checkov / kics when we ship a Dockerfile or k8s manifests.
- # See https://docs.litellm.ai/blog/security-update-march-2026
- # and the Microsoft / Trend Micro / Snyk incident write-ups.
- # ─────────────────────────────────────────────────────────────
-
- # ─────────────────────────────────────────────────────────────
- # TruffleHog secret-leak scan on the PR diff. Catches API keys
- # / tokens / cred files committed accidentally. --only-verified
- # filters out probabilistic findings, so we only flag tokens
- # that the source provider confirmed are live. On push to main
- # / pip we scan the full repo; on PR we scan base..head.
- # SHA-pinned for the same reason as harden-runner above.
- # v3.95.2 commit:
- # ─────────────────────────────────────────────────────────────
- - name: TruffleHog (secrets in diff)
- continue-on-error: true
- uses: trufflesecurity/trufflehog@37b77001d0174ebec2fcca2bd83ff83a6d45a3ab # v3.95.3
- with:
- path: ./
- base: ${{ github.event.pull_request.base.sha || '' }}
- head: ${{ github.event.pull_request.head.sha || github.sha }}
- # The action passes --no-update internally; passing it here
- # too triggers `flag 'no-update' cannot be repeated`. Stick
- # with --only-verified so we only flag tokens the source
- # provider confirmed are live (no probabilistic findings).
- extra_args: --only-verified
-
- # ─────────────────────────────────────────────────────────────
- # CycloneDX SBOM. Lets downstream consumers audit what's
- # actually shipped in unsloth wheels and the Studio backend
- # runtime. Generates one JSON file per requirements input plus
- # a combined SBOM keyed off pyproject.toml; uploads as a build
- # artifact (and a future step can attest it via SLSA).
- # ─────────────────────────────────────────────────────────────
- - name: Generate CycloneDX SBOM
- continue-on-error: true
- run: |
- set +e
- python -m pip install --quiet 'cyclonedx-bom>=4.6'
- mkdir -p sbom
- # Per-requirements-file SBOM (the audit-reqs/ files are the
- # filtered, git+-stripped views built earlier in this job).
- # cyclonedx-py 4.x uses `--sv` for spec version and `-o` for
- # the output file; the older `--schema-version`/`--outfile`
- # spellings are not accepted.
- for f in audit-reqs/*.txt; do
- base=$(basename "$f" .txt)
- if grep -qE '^[^#[:space:]]' "$f"; then
- cyclonedx-py requirements "$f" \
- --sv 1.6 \
- --of JSON \
- -o "sbom/sbom-$base.json" 2>&1 | tail -5 || true
- fi
- done
- # Project-level SBOM from pyproject.toml.
- cyclonedx-py environment \
- --sv 1.6 \
- --of JSON \
- -o sbom/sbom-environment.json 2>&1 | tail -5 || true
- ls -la sbom/
- {
- echo "## CycloneDX SBOM"
- echo
- echo "Generated SBOM files:"
- ls sbom/ | sed 's/^/- sbom\//'
- } >> "$GITHUB_STEP_SUMMARY"
-
- # ─────────────────────────────────────────────────────────────
- # GitHub Actions pinning verifier. tj-actions/changed-files
- # was compromised in March 2025; anyone using `@v4` (a mutable
- # ref) auto-shipped the malicious version. Catch every
- # non-SHA-pinned `uses:` across the workflows tree. Warn-only
- # initially so the existing baseline doesn't block PRs.
- # ─────────────────────────────────────────────────────────────
- - name: GitHub Actions pinning verifier
- continue-on-error: true
- run: |
- python <<'PY' | tee logs-actions-pinning.txt
- import re
- from pathlib import Path
- # SHA pin = 40 hex chars after @
- SHA_PIN = re.compile(r"@[0-9a-f]{40}\b")
- # First-party / GitHub-published actions get a softer pass
- # (still recommended to pin; not a security gate).
- FIRST_PARTY = re.compile(r"^\s*-\s*uses:\s*(actions|github)/[^@]+@")
- USES = re.compile(r"^\s*-\s*uses:\s*([^@\s]+)@(\S+)")
- unpinned_third = []
- unpinned_first = []
- for f in sorted(Path(".github/workflows").glob("*.yml")):
- for i, line in enumerate(f.read_text().splitlines(), 1):
- m = USES.match(line)
- if not m:
- continue
- name, ref = m.group(1), m.group(2)
- if SHA_PIN.search(line):
- continue
- bucket = unpinned_first if FIRST_PARTY.match(line) else unpinned_third
- bucket.append((str(f), i, name, ref))
- print("::group::Action pinning status")
- print(f"third-party actions on mutable refs: {len(unpinned_third)}")
- for f, i, n, r in unpinned_third:
- print(f" HIGH {f}:{i}: {n}@{r}")
- print()
- print(f"first-party (actions/* | github/*) on mutable refs: {len(unpinned_first)}")
- for f, i, n, r in unpinned_first[:30]:
- print(f" WARN {f}:{i}: {n}@{r}")
- if len(unpinned_first) > 30:
- print(f" ... and {len(unpinned_first) - 30} more")
- print()
- print("Recommendation: pin third-party actions to a 40-char SHA.")
- print("Dependabot's github-actions ecosystem will auto-bump them.")
- print("::endgroup::")
- PY
- {
- echo "## GitHub Actions pinning verifier"
- echo
- echo '```'
- cat logs-actions-pinning.txt
- echo '```'
- } >> "$GITHUB_STEP_SUMMARY"
-
- # ─────────────────────────────────────────────────────────────
- # Hash-pin verifier. `==` pinning protects against version
- # drift but not against a re-uploaded malicious wheel at the
- # same version (PyPI lets a yanked release be re-published with
- # different bytes for ~5 minutes via `--filename` collision).
- # `pip install --require-hashes` rejects any download whose
- # SHA-256 doesn't match. Inspector step that reports how many
- # specs would gain from a hash pin -- conversion is a roadmap
- # item (needs pip-tools / uv pip compile --generate-hashes).
- # ─────────────────────────────────────────────────────────────
- - name: Hash-pin verifier (Python requirements)
- continue-on-error: true
- run: |
- python <<'PY' | tee logs-hash-verifier.txt
- import re
- from pathlib import Path
- PINNED = re.compile(r"^\s*[A-Za-z0-9_.\-]+\s*==\s*[^,;]+\s*$")
- HASH_LINE = re.compile(r"--hash=sha256:[0-9a-f]{64}")
- total_pinned = 0
- with_hash = 0
- for f in sorted(Path("studio/backend/requirements").glob("*.txt")):
- text = f.read_text()
- for raw in text.splitlines():
- line = raw.strip()
- if not line or line.startswith("#") or line.startswith("-"):
- continue
- spec = line.split("#", 1)[0].strip().split(";", 1)[0]
- if PINNED.match(spec):
- total_pinned += 1
- if HASH_LINE.search(raw):
- with_hash += 1
- print(f"::group::Hash-pin status")
- print(f" exact == pins: {total_pinned}")
- print(f" with --hash=sha256: {with_hash}")
- print(f" without --hash: {total_pinned - with_hash}")
- print()
- print("Roadmap: convert to hash-locked installs via")
- print("`uv pip compile --generate-hashes` and `pip install --require-hashes`.")
- print("Hash-locked installs would have refused a republished")
- print("malicious litellm 1.82.7 wheel even at the same version.")
- print("::endgroup::")
- PY
- {
- echo "## Hash-pin verifier"
- echo
- echo '```'
- cat logs-hash-verifier.txt
- echo '```'
- } >> "$GITHUB_STEP_SUMMARY"
-
- - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- if: always()
- with:
- name: advisory-audit-logs
- path: |
- logs-pip-audit.txt
- logs-npm-audit.txt
- logs-npm-audit.json
- logs-cargo-audit.txt
- logs-osv-scanner.txt
- logs-semgrep.txt
- logs-pin-verifier.txt
- logs-actions-pinning.txt
- logs-hash-verifier.txt
- audit-reqs/
- sbom/
- retention-days: 30
-
- # ─────────────────────────────────────────────────────────────────────
- # Python: pre-install package scan (no install, no execution)
- # ─────────────────────────────────────────────────────────────────────
- pip-scan-packages:
- # Downloads each declared dep WITHOUT installing it and inspects
- # the archive contents for known malicious patterns: weaponized
- # .pth files, credential stealers, obfuscated payloads,
- # install-time droppers, suspicious subprocess / network /
- # base64-blob combinations.
- #
- # This is the kind of check that would have caught:
- # - litellm 1.82.7 / 1.82.8 (March 2026, supply-chain compromise)
- # - the typo-squat campaign against PyTorch Lightning
- # before either landed in the install path. pip-audit only knows
- # about CVE-published vulnerabilities, so it does NOT see novel
- # malicious uploads. scan_packages.py runs deterministic regex
- # pattern matching, no LLM calls.
- #
- # `--with-deps` makes the scan transitive: every package the
- # declared set resolves to gets fetched and pattern-scanned, not
- # just the top-level pins. Resolving the full transitive closure
- # of the unsloth + Studio dep tree downloads several hundred
- # archives, hence the longer timeout.
- #
- # Sharded across runners for wall-clock parallelism. Each shard
- # runs scan_packages.py once with --with-deps so its own slice
- # benefits from pip's deduped transitive resolve. Shard
- # composition tries to balance load:
- # - hf-stack: pyproject extras + no-torch-runtime
- # (~150 archives, transformers/peft/accelerate/...)
- # - studio: FastAPI/Studio backend + overrides + extras-no-deps
- # (~150 archives, smaller scientific stack)
- # - extras: the heavy openai-whisper / scikit-learn / librosa
- # stack (~250 archives, dominant cost)
- # triton-kernels.txt is git+-only, fully skipped.
- name: ${{ matrix.shard.name }}
- runs-on: ubuntu-latest
- timeout-minutes: 25
- strategy:
- fail-fast: false
- matrix:
- shard:
- - name: 'pip scan-packages :: hf-stack'
- id: hf-stack
- files: 'unsloth-deps no-torch-runtime'
- - name: 'pip scan-packages :: studio'
- id: studio
- files: 'studio overrides extras-no-deps'
- - name: 'pip scan-packages :: extras'
- id: extras
- files: 'extras'
- steps:
- # Egress block on every shard. Each shard pulls hundreds of
- # PyPI archives -- if a malicious wheel ever phones home from
- # within the scanner sandbox (it shouldn't; we never execute
- # the archive), harden-runner now rejects the connect outright.
- # Per-job allowlist: pip-scan-packages only fetches PyPI archives
- # via scan_packages.py + pip download. No npm or cargo traffic.
- - name: Harden runner (egress block)
- uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
- with:
- egress-policy: block
- disable-sudo: true
- allowed-endpoints: >
- api.github.com:443
- github.com:443
- codeload.github.com:443
- objects.githubusercontent.com:443
- pypi.org:443
- files.pythonhosted.org:443
-
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- - name: Install scan_packages.py runtime deps
- # scan_packages.py imports requests + packaging at runtime to
- # talk to PyPI's JSON API and to parse version specifiers. We
- # do not install the packages it scans -- those are downloaded
- # raw and inspected without ever touching `pip install`.
- run: python -m pip install --upgrade pip requests packaging
-
- - name: Build filtered requirements set
- # Mirrors the advisory-audit job's input transform: pyproject.toml
- # extraction + git+ stripping. scan_packages.py downloads
- # PyPI archives without building, so it tolerates legacy
- # setup.py packages (no resolver dry-run); but `--with-deps`
- # delegates resolution to a single `pip download` call that
- # cannot satisfy `git+` specs without git operations, so we
- # strip them here too.
- run: |
- mkdir -p audit-reqs
- python <<'PY' > audit-reqs/unsloth-deps.txt
- import tomllib
- with open("pyproject.toml", "rb") as f:
- d = tomllib.load(f)
- core = d["project"]["dependencies"]
- extras = d["project"]["optional-dependencies"]["huggingfacenotorch"]
- print("# Auto-generated from pyproject.toml by security-audit.yml.")
- print("# core deps + huggingfacenotorch extras.")
- for spec in core + extras:
- print(spec)
- PY
- for f in studio.txt extras.txt extras-no-deps.txt \
- no-torch-runtime.txt overrides.txt triton-kernels.txt; do
- python < "audit-reqs/$f"
- src = "studio/backend/requirements/$f"
- with open(src) as fh:
- for line in fh:
- stripped = line.strip()
- before_comment = stripped.split("#", 1)[0]
- if "git+" in before_comment:
- print(f"# [security-audit] skipped git+ spec: {stripped}")
- continue
- print(line.rstrip("\n"))
- PY
- done
-
- - name: Sanity-check scan_packages.py
- # The scanner lives at scripts/scan_packages.py in this repo
- # so we don't depend on a network fetch at job time.
- run: |
- test -f scripts/scan_packages.py
- head -3 scripts/scan_packages.py
- grep -q "Standalone pre-install package scanner" scripts/scan_packages.py
-
- - name: Scan declared + transitive Python deps
- # scan_packages.py exits 1 on CRITICAL/HIGH findings, 0 on
- # clean. We swallow the exit because the baseline isn't
- # triaged yet; surface the findings in the workflow summary.
- # Drop continue-on-error after the first clean run on main.
- #
- # `--with-deps` walks PyPI metadata to enumerate every
- # transitive dep the declared set would install, then scans
- # them all. Without this flag, we'd only catch a malicious
- # *direct* dep -- and supply-chain attacks usually land
- # several hops down (litellm 1.82.7 was a dep of a dep for
- # most users).
- #
- # This step runs once per matrix shard. Within a shard, every
- # -r file is fed to a single `pip download` call so pip
- # intersects version constraints and yields a deduped
- # transitive set (no point fetching the same transformers
- # wheel five times). Across shards we accept some redundant
- # downloads in exchange for wall-clock parallelism.
- env:
- SHARD_FILES: ${{ matrix.shard.files }}
- run: |
- set +e
- mkdir -p logs
- LOG="logs-scan-packages-${{ matrix.shard.id }}.txt"
- echo "::group::shard ${{ matrix.shard.id }} input files"
- REQ_ARGS=()
- for f in $SHARD_FILES; do
- if grep -qE '^[^#[:space:]]' "audit-reqs/$f.txt"; then
- echo " + audit-reqs/$f.txt"
- REQ_ARGS+=( -r "audit-reqs/$f.txt" )
- else
- echo " - audit-reqs/$f.txt (empty after git+ filter, skipping)"
- fi
- done
- echo "::endgroup::"
- if [ ${#REQ_ARGS[@]} -eq 0 ]; then
- echo "[security-audit] shard ${{ matrix.shard.id }}: no PyPI specs, nothing to scan" \
- | tee "$LOG"
- else
- python scripts/scan_packages.py --with-deps "${REQ_ARGS[@]}" \
- 2>&1 | tee "$LOG"
- fi
- {
- echo "## scan_packages :: shard ${{ matrix.shard.id }}"
- echo
- echo "### Files in this shard"
- for f in $SHARD_FILES; do echo "- audit-reqs/$f.txt"; done
- echo
- echo '### Findings (tail)'
- echo '```'
- tail -200 "$LOG"
- echo '```'
- } >> "$GITHUB_STEP_SUMMARY"
-
- - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- if: always()
- with:
- name: scan-packages-log-${{ matrix.shard.id }}
- path: |
- logs-scan-packages-${{ matrix.shard.id }}.txt
- audit-reqs/
- retention-days: 30
-
- # ─────────────────────────────────────────────────────────────────────
- # npm: pre-install tarball content scan.
- # ─────────────────────────────────────────────────────────────────────
- npm-scan-packages:
- # Counterpart to pip-scan-packages for the npm side. Reads
- # studio/frontend/package-lock.json, downloads each resolved
- # tarball DIRECTLY from registry.npmjs.org (never via `npm
- # install` -- no lifecycle scripts ever run), verifies the
- # lockfile integrity hash, unpacks each tarball into a sandboxed
- # temp dir behind size / count / path-escape / symlink guards,
- # and pattern-scans the extracted file contents for the
- # signatures common to npm supply-chain attacks:
- #
- # - lifecycle (preinstall / install / postinstall / prepare)
- # scripts in any package.json that fetch + execute external
- # code,
- # - C2 / exfiltration hosts (getsession.org, AWS IMDS,
- # Kubernetes ServiceAccount token paths, GitHub Actions OIDC,
- # HashiCorp Vault endpoints),
- # - credential-stealing references (.npmrc, .aws/credentials,
- # GITHUB_TOKEN / NPM_TOKEN in JS sources),
- # - known IOC filenames (router_init.js, tanstack_runner.js,
- # router_runtime.js),
- # - obfuscation shapes (Function/eval against base64 blobs).
- #
- # Threat model: every tarball is hostile. Safety guarantees are
- # documented at scripts/scan_npm_packages.py top-of-file. The
- # script is stdlib-only so adding it does not increase the
- # transitive supply-chain surface.
- name: npm scan-packages (Studio frontend tarballs)
- runs-on: ubuntu-latest
- timeout-minutes: 30
- needs: []
- steps:
- # Per-job allowlist: npm-scan-packages only fetches tarballs from
- # registry.npmjs.org. GitHub endpoints retained for checkout +
- # setup-python action machinery.
- - name: Harden runner (egress block)
- uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
- with:
- egress-policy: block
- disable-sudo: true
- allowed-endpoints: >
- api.github.com:443
- github.com:443
- codeload.github.com:443
- objects.githubusercontent.com:443
- registry.npmjs.org:443
-
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
-
- - name: Sanity-check scan_npm_packages.py
- run: |
- test -f scripts/scan_npm_packages.py
- python3 -c "import ast; ast.parse(open('scripts/scan_npm_packages.py').read())"
-
- - name: Scan npm tarballs (declared + transitive, no install)
- # The script exits 1 on HIGH/CRITICAL findings; we capture the
- # full log and surface it in the step summary either way. It
- # never runs `npm install`, never executes anything from a
- # downloaded tarball, and only fetches from registry.npmjs.org.
- # Initially non-blocking so the baseline can settle; drop
- # continue-on-error once the baseline is clean for a week.
- run: |
- set -o pipefail
- LOG=logs-scan-npm.txt
- python3 scripts/scan_npm_packages.py 2>&1 | tee "$LOG"
- {
- echo "## scan_npm_packages"
- echo
- echo '### Findings (tail)'
- echo '```'
- tail -300 "$LOG"
- echo '```'
- } >> "$GITHUB_STEP_SUMMARY"
-
- - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- if: always()
- with:
- name: scan-npm-packages-log
- path: logs-scan-npm.txt
- retention-days: 30
-
- # ─────────────────────────────────────────────────────────────────────
- # Workflow-trigger lint. Refuses two patterns that together powered the
- # TanStack GHSA-g7cv-rxg3-hmpx supply-chain compromise:
- #
- # 1. `pull_request_target` -- runs a fork's workflow YAML against
- # the base repository's secrets. There is no safe use of this
- # trigger for a public open-source project.
- #
- # 2. Shared cache keys between PR-triggered workflows and the
- # publish workflow. A fork PR can poison the cache; the publish
- # workflow then restores the poisoned cache on next run.
- #
- # Cheap pure-Python lint, runs in seconds. Fail-closed.
- # ─────────────────────────────────────────────────────────────────────
- workflow-trigger-lint:
- name: workflow-trigger lint (pull_request_target / cache-poisoning)
- runs-on: ubuntu-latest
- timeout-minutes: 5
- steps:
- - name: Harden runner (egress block)
- uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
- with:
- egress-policy: block
- disable-sudo: true
- allowed-endpoints: >
- api.github.com:443
- github.com:443
- codeload.github.com:443
- objects.githubusercontent.com:443
- pypi.org:443
- files.pythonhosted.org:443
-
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
-
- - name: Install PyYAML
- run: pip install pyyaml
-
- - name: Lint workflow triggers + cache keys
- run: python3 scripts/lint_workflow_triggers.py
-
- # ─────────────────────────────────────────────────────────────────────
- # Regression tests: pin scanner IOC tables and pre-install fixtures.
- # Hard gate (no continue-on-error) so future drift in the IOC tables
- # or scanner exit semantics fails this PR at review time.
- # ─────────────────────────────────────────────────────────────────────
- tests-security:
- name: pytest tests/security
- runs-on: ubuntu-latest
- timeout-minutes: 10
- steps:
- - name: Harden runner (egress block)
- uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
- with:
- egress-policy: block
- disable-sudo: true
- allowed-endpoints: >
- api.github.com:443
- github.com:443
- codeload.github.com:443
- objects.githubusercontent.com:443
- pypi.org:443
- files.pythonhosted.org:443
-
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
-
- - name: Install pytest + PyYAML
- # PyYAML is imported by scripts/lint_workflow_triggers.py, which the
- # `tests/security/test_lint_workflow_triggers.py` regression suite
- # exercises as a subprocess. Without it the lint script bails with
- # `ERROR: PyYAML is required` (exit 2) and the 5 lint regression
- # tests fail. Pinned the same way pytest is pinned.
- run: pip install pytest==9.0.3 pyyaml==6.0.2
-
- - name: Run security regression tests
- run: python3 -m pytest tests/security -v
-
- # ─────────────────────────────────────────────────────────────────────
- # npm provenance + new install-script diff. Catches the two npm
- # supply-chain levers we don't yet gate on:
- #
- # 1. `npm audit signatures` validates the registry-signed
- # provenance of every tarball laid down in node_modules. Pulled
- # from the public npm transparency log; surfaces unsigned or
- # mis-signed deps. Informational for now (continue-on-error)
- # while the baseline settles.
- #
- # 2. `check_new_install_scripts.py` diffs the PR's lockfile
- # against the base ref and refuses any newly-added dep that
- # ships a postinstall hook. Every recent npm supply-chain
- # compromise leveraged a postinstall as the execution lever, so
- # blocking new ones at PR time is a small, high-signal gate.
- # ─────────────────────────────────────────────────────────────────────
- npm-provenance-and-install-scripts:
- name: npm provenance + new install-script diff
- runs-on: ubuntu-latest
- timeout-minutes: 15
- steps:
- - name: Harden runner (egress block)
- uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
- with:
- egress-policy: audit
- disable-sudo: true
- allowed-endpoints: >
- api.github.com:443
- github.com:443
- codeload.github.com:443
- objects.githubusercontent.com:443
- registry.npmjs.org:443
-
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- # Need the base commit accessible for `git show
- # :studio/frontend/package-lock.json` below.
- fetch-depth: 0
- persist-credentials: false
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
-
- - name: Install Studio frontend deps (--ignore-scripts)
- # `npm audit signatures` requires node_modules to be populated.
- # `--ignore-scripts` is mandatory: this is exactly the lever the
- # new-install-script gate below protects against, and we must
- # not run any third-party hook to set up the audit.
- working-directory: studio/frontend
- run: npm ci --ignore-scripts
-
- - name: npm audit signatures (informational)
- # Surfaces unsigned / mis-signed packages from the npm
- # transparency log. continue-on-error during baseline-build
- # phase; promote to hard gate once the lockfile is fully
- # signed (most major maintainers signed by mid-2025).
- working-directory: studio/frontend
- continue-on-error: true
- run: |
- set -o pipefail
- LOG=logs-audit-signatures.txt
- npm audit signatures 2>&1 | tee "$LOG"
- {
- echo "## npm audit signatures"
- echo
- echo '```'
- tail -200 "$LOG"
- echo '```'
- } >> "$GITHUB_STEP_SUMMARY"
-
- - name: Extract base-ref lockfile (PR triggers only)
- if: github.event_name == 'pull_request'
- run: |
- set -e
- BASE_SHA="${{ github.event.pull_request.base.sha }}"
- git show "$BASE_SHA:studio/frontend/package-lock.json" \
- > /tmp/base-package-lock.json
-
- - name: Diff for newly-added install-script deps
- if: github.event_name == 'pull_request'
- run: |
- python3 scripts/check_new_install_scripts.py \
- --base /tmp/base-package-lock.json \
- --head studio/frontend/package-lock.json
-
- - name: Skip install-script diff (non-PR trigger)
- if: github.event_name != 'pull_request'
- run: |
- echo "Not a pull_request event; install-script diff requires a base ref."
- echo "This step is intentionally a no-op outside PR triggers."
-
- - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- if: always()
- with:
- name: npm-audit-signatures-log
- path: studio/frontend/logs-audit-signatures.txt
- if-no-files-found: ignore
- retention-days: 30
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
deleted file mode 100644
index 1a4cf841d0..0000000000
--- a/.github/workflows/stale.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-name: 'Inactive Issue Pinger'
-
-on:
- schedule:
- - cron: '30 5 * * *' # Runs at 5:30 UTC every day
-
-jobs:
- stale:
- runs-on: ubuntu-latest
- permissions:
- issues: write
-
- steps:
- - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0
- with:
- # The message to post on stale issues.
- # This message will ping the issue author.
- # Note: The stale bot action does not currently support a direct placeholder for the last commenter.
- # As a workaround, this message encourages any participant to reply.
- stale-issue-message: >
- Is this issue still important to you?
- Apologies in advance we might have missed this issue as well.
- For faster response times, please post on our Reddit server - https://www.reddit.com/r/unsloth or our Discord - https://discord.com/invite/unsloth
-
- # The number of days of inactivity before an issue is considered stale.
- days-before-issue-stale: 9999
-
- # Set to -1 to never close stale issues.
- days-before-issue-close: -1
-
- # A label to apply to stale issues.
- stale-issue-label: 'inactive'
-
- # The number of operations to perform per run to avoid rate limiting.
- operations-per-run: 500
-
- enable-statistics: false
diff --git a/.github/workflows/studio-api-smoke.yml b/.github/workflows/studio-api-smoke.yml
deleted file mode 100644
index 53514e2ce1..0000000000
--- a/.github/workflows/studio-api-smoke.yml
+++ /dev/null
@@ -1,166 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# Studio API & Auth Tests -- HTTP-level integration tests for the
-# FastAPI surface. No Playwright, no model UI; tests/studio/test_studio_api_smoke.py
-# runs ~30 s and asserts:
-# - CORS hardening (no wildcard + credentials, no bootstrap leak)
-# - /api/system + /api/system/hardware require auth
-# - Auth state machine + JWT expiry
-# - API key lifecycle E2E (create / list / use / delete / reject)
-# - Auth file-mode hardening (Linux only)
-# - Inference lifecycle (force reload, bogus variant, /v1/models, /v1/embeddings, /v1/responses)
-# - Endpoint-by-endpoint auth audit
-#
-# Reuses the GGUF cache key from studio-ui-smoke.yml so the model
-# download is one cache-hit on the second job.
-
-name: Studio API CI
-
-on:
- pull_request:
- paths:
- - 'studio/**'
- - 'unsloth/**'
- - 'unsloth_cli/**'
- - 'install.sh'
- - 'pyproject.toml'
- - 'tests/studio/**'
- - '.github/workflows/studio-api-smoke.yml'
- push:
- branches: [main, pip]
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- api-smoke:
- name: Studio API & Auth Tests
- runs-on: ubuntu-latest
- timeout-minutes: 12
- env:
- GGUF_REPO: unsloth/gemma-3-270m-it-GGUF
- GGUF_VARIANT: UD-Q4_K_XL
- GGUF_FILE: gemma-3-270m-it-UD-Q4_K_XL.gguf
- STUDIO_PORT: '18893'
- HF_HOME: ${{ github.workspace }}/hf-cache
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - name: Linux deps
- run: |
- sudo apt-get update
- sudo apt-get install -y --no-install-recommends \
- libcurl4-openssl-dev libssl-dev jq
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- - name: Restore HF_HOME for ${{ env.GGUF_REPO }}
- id: cache-hf
- uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- continue-on-error: true
- with:
- path: hf-cache
- # Same key as studio-ui-smoke.yml so the two jobs share a
- # single GGUF download across CI.
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
-
- - name: Prime HF_HOME with the GGUF
- id: prime-hf
- if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success'
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- run: |
- python -m pip install --upgrade huggingface_hub
- mkdir -p hf-cache
- bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE"
-
- - name: Save HF_HOME for ${{ env.GGUF_REPO }}
- if: always() && steps.prime-hf.outcome == 'success'
- uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
-
- - name: Install Studio (--local, --no-torch)
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- mkdir -p logs
- set -o pipefail
- bash install.sh --local --no-torch 2>&1 | tee logs/install.log
-
- - name: Install pyjwt for the JWT-expiry forge test
- run: pip install 'pyjwt>=2.6'
-
- - name: Reset auth + boot Studio (API-only)
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \
- > logs/studio.log 2>&1 &
- echo "STUDIO_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then
- jq -e '.status == "healthy"' /tmp/health.json && break
- fi
- sleep 1
- done
- jq -e '.status == "healthy"' /tmp/health.json
-
- - name: Pass bootstrap password + rotated targets to the test
- # The test does its own bootstrap-login + rotation to exercise
- # the auth state machine; we just pre-mint two random rotated
- # passwords for it. Mask them so the log is clean.
- run: |
- OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- NEW="ApiSmoke-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- NEW2="ApiSmoke-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- echo "::add-mask::$OLD"
- echo "::add-mask::$NEW"
- echo "::add-mask::$NEW2"
- echo "STUDIO_OLD_PW=$OLD" >> "$GITHUB_ENV"
- echo "STUDIO_NEW_PW=$NEW" >> "$GITHUB_ENV"
- echo "STUDIO_NEW2_PW=$NEW2" >> "$GITHUB_ENV"
-
- - name: Run Studio API & Auth tests
- # The script is named WITHOUT a `test_` prefix so it isn't
- # auto-collected by pytest in Backend CI's `tests/` walk
- # (which doesn't set BASE_URL and would crash at import).
- env:
- BASE_URL: http://127.0.0.1:18893
- STUDIO_AUTH_DIR: /home/runner/.unsloth/studio/auth
- run: python tests/studio/studio_api_smoke.py
-
- - name: Stop Studio
- if: always()
- run: |
- kill "${STUDIO_PID}" 2>/dev/null || true
- sleep 2
-
- - name: Upload API smoke logs
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: studio-api-smoke-log
- path: |
- logs/install.log
- logs/studio.log
- retention-days: 7
diff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.yml
deleted file mode 100644
index 63eb70f7f1..0000000000
--- a/.github/workflows/studio-backend-ci.yml
+++ /dev/null
@@ -1,221 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# Runs the existing studio/backend/tests/ suite (~860 tests, all CPU-friendly)
-# on every PR that touches the backend or unsloth library. Until this lands,
-# none of those tests run automatically. Verified locally on Python 3.13 with
-# the surgical exclusions below: 861 pass, 4 skipped.
-#
-# Exclusions:
-# - tests/test_studio_api.py: end-to-end against a live model + GGUF download,
-# too heavy for free runners. Run separately when GPU CI is available.
-# - -k 'not llama_cpp_load_progress_live': spawns a real llama.cpp process,
-# not appropriate for CPU-only runners.
-#
-# Two jobs:
-# - pytest matrix (3.10/3.11/3.12/3.13) over studio/backend/tests
-# - repo-cpu-tests: auto-discovered tests/ + state-isolated spoof files
-#
-# Whole-repo Python lint (syntax + ruff + debugger-leftover scan)
-# moved to the dedicated `Lint CI` workflow (.github/workflows/lint-ci.yml)
-# so it fires on every PR rather than only on studio/unsloth/tests
-# path changes.
-
-name: Backend CI
-
-on:
- pull_request:
- paths:
- - 'studio/**'
- - 'unsloth/**'
- - 'unsloth_cli/**'
- - 'tests/**'
- - 'pyproject.toml'
- - '.github/workflows/studio-backend-ci.yml'
- push:
- branches: [main, pip]
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- pytest:
- name: (Python ${{ matrix.python }})
- runs-on: ubuntu-latest
- timeout-minutes: 15
- strategy:
- fail-fast: false
- matrix:
- python: ['3.10', '3.11', '3.12', '3.13']
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '${{ matrix.python }}'
- cache: 'pip'
-
- - name: Install backend test dependencies (CPU only)
- run: |
- python -m pip install --upgrade pip
- # Studio's declared backend deps:
- pip install -r studio/backend/requirements/studio.txt
- # Extras that studio.txt does not list but the import chain needs
- # (python-multipart for FastAPI form/file uploads, sqlalchemy/cryptography
- # for the auth DB, yaml/jinja2 for utils.models.model_config, etc.):
- pip install \
- python-multipart aiofiles sqlalchemy cryptography \
- pyyaml jinja2 mammoth unpdf requests \
- 'numpy<3' pytest pytest-asyncio httpx
- # Torch CPU + transformers are required by a chunk of the backend test
- # suite (gpu_selection, kv_cache_estimation, utils). CPU-only torch
- # keeps the install ~250 MB / ~1 min on a clean runner.
- pip install --index-url https://download.pytorch.org/whl/cpu 'torch>=2.4,<2.11'
- pip install 'transformers>=4.51,<5.5'
-
- - name: Backend tests
- working-directory: studio/backend
- # Locally validated against this dep set: 831 passed, 5 skipped, 35 deselected.
- # Deselections (all environment-specific, would never pass on a GPU-less
- # `ubuntu-latest` runner regardless of code correctness):
- # - llama_cpp_load_progress_live: spawns a real llama.cpp process
- # - TestGpuAutoSelection / TestPreSpawnGpuResolution / TestPerGpuFitGuardAllCounts:
- # require live transformers config introspection on real GPUs
- # - TestTransformersIntrospection: same
- # - test_returns_cuda_when_cuda_available / test_calls_cuda_cache_when_cuda:
- # assume CUDA-capable GPU
- run: |
- python -m pytest tests/ -q --tb=short \
- --ignore=tests/test_studio_api.py \
- -k 'not llama_cpp_load_progress_live and not TestGpuAutoSelection and not TestPreSpawnGpuResolution and not TestPerGpuFitGuardAllCounts and not TestTransformersIntrospection and not test_returns_cuda_when_cuda_available and not test_calls_cuda_cache_when_cuda'
-
- repo-cpu-tests:
- # Auto-discover everything under tests/ that is not GPU-bound by
- # design. New tests added in covered directories are picked up
- # without a workflow edit. Locally validated: 760 passed, 1 skipped,
- # 23 deselected. tests/conftest.py (mirroring unsloth-zoo PR #624)
- # pre-loads unsloth_zoo.device_type and unsloth.device_type under a
- # mocked torch.cuda.is_available so the unsloth import chain
- # succeeds on CPU.
- name: Repo tests (CPU)
- runs-on: ubuntu-latest
- timeout-minutes: 15
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- # node + uv unlock ~60 tests that previously skipped on CI:
- # - 9 tests in test_chat_preset_builtin_invariants.py need node to
- # compile a tiny TS harness against the frontend chat sources.
- # - tests/python/* spawn fresh `uv venv`s to verify the no-torch
- # install path; they self-skip when uv is missing.
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - name: Install uv (for tests/python/* sandboxed venvs)
- run: pip install uv
-
- - name: Install deps (shared shape with backend pytest job)
- run: |
- python -m pip install --upgrade pip
- pip install -r studio/backend/requirements/studio.txt
- pip install \
- python-multipart aiofiles sqlalchemy cryptography \
- pyyaml jinja2 mammoth unpdf requests typer \
- 'numpy<3' pytest pytest-asyncio httpx
- # torchvision: unsloth_zoo.vision_utils imports it at module scope.
- pip install --index-url https://download.pytorch.org/whl/cpu \
- 'torch>=2.4,<2.11' 'torchvision<0.26'
- pip install 'transformers>=4.51,<5.5'
- # bitsandbytes: hard import in unsloth/models/_utils.py. Recent
- # versions ship a CPU build that imports cleanly on Linux.
- pip install 'bitsandbytes>=0.45'
- # unsloth.device_type imports unsloth_zoo.utils.Version at module
- # scope, so the conftest preload needs unsloth_zoo even though
- # it is an optional dep of unsloth.
- pip install 'unsloth_zoo>=2026.5.1'
- pip install -e . --no-deps
-
- - name: Repo tests (CPU, auto-discovered)
- env:
- # tests/python/* import install_python_stack from studio/.
- PYTHONPATH: ${{ github.workspace }}/studio
- # Skip lazy compilation work the unsloth import chain wants to
- # do at import time on a real GPU.
- UNSLOTH_COMPILE_DISABLE: '1'
- # --ignore: GPU-bound directories (qlora/saving need real weights;
- # tests/sh is the shell suite the next step handles; tests/utils
- # is a helpers folder); tests/vllm_compat + tests/version_compat
- # are dedicated multi-version drift canaries with their own job
- # in version-compat-ci.yml that installs the heavier dep set
- # (torchcodec, full transformers/peft/bnb pins) those tests need.
- # State-sensitive hardware-spoofing files run in isolation in the
- # next step because they mutate hardware.py module globals.
- # -m: honour markers from tests/python/conftest.py (`server` =
- # needs studio venv, `e2e` = needs network).
- # --deselect:
- # - test_model_registration / test_all_model_registration:
- # hit huggingface_hub for live model existence checks.
- # - test_autoconfig_works_with_no_torch_runtime / test_autoconfig_succeeds:
- # fail because no-torch-runtime.txt does not pin tokenizers
- # and the latest tokenizers (0.23.1) is incompatible with the
- # transformers it resolves to. Tracked separately; this is a
- # real bug in the no-torch install path, not a CI issue.
- run: |
- python -m pytest tests/ -q --tb=short \
- --ignore=tests/qlora \
- --ignore=tests/saving \
- --ignore=tests/utils \
- --ignore=tests/sh \
- --ignore=tests/studio/test_hardware_dispatch_matrix.py \
- --ignore=tests/studio/test_is_mlx_dispatch_gate.py \
- --ignore=tests/vllm_compat \
- --ignore=tests/version_compat \
- -m 'not server and not e2e' \
- --deselect tests/test_model_registry.py::test_model_registration \
- --deselect tests/test_model_registry.py::test_all_model_registration \
- --deselect 'tests/python/test_tokenizers_and_torch_constraint.py::TestE2ETokenizersFix::test_autoconfig_works_with_no_torch_runtime' \
- --deselect 'tests/python/test_tokenizers_and_torch_constraint.py::TestE2EFullNoTorchSandbox::test_autoconfig_succeeds'
-
- - name: Hardware-spoof tests (state-sensitive, run in isolation)
- env:
- PYTHONPATH: ${{ github.workspace }}/studio
- UNSLOTH_COMPILE_DISABLE: '1'
- # These two files mutate hardware.py module globals at runtime
- # via the spoof fixtures, which leaks state into any other test
- # that imports hardware. Run them in their own pytest invocation
- # so the leak does not cross file boundaries.
- run: |
- python -m pytest -q --tb=short \
- tests/studio/test_hardware_dispatch_matrix.py \
- tests/studio/test_is_mlx_dispatch_gate.py
-
- - name: Shell installer tests
- # Subset that does not depend on a writable / pristine install.sh
- # tree; test_install_host_defaults.sh checks install.ps1 layout
- # which has drifted (separate followup).
- run: |
- set -e
- for s in \
- tests/sh/test_get_torch_index_url.sh \
- tests/sh/test_mac_intel_compat.sh \
- tests/sh/test_tauri_install_exit_order.sh \
- tests/sh/test_torch_constraint.sh; do
- echo "::group::$s"
- bash "$s"
- echo "::endgroup::"
- done
-
diff --git a/.github/workflows/studio-frontend-ci.yml b/.github/workflows/studio-frontend-ci.yml
deleted file mode 100644
index 1270a57ef6..0000000000
--- a/.github/workflows/studio-frontend-ci.yml
+++ /dev/null
@@ -1,151 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# Frontend PR gate: lockfile freshness, typecheck, build, and a bundle grep
-# that catches the 2026.5.1 chat-history regression at the JS level.
-#
-# biome runs as non-blocking for now: the codebase currently has accumulated
-# ~470 errors and ~1650 warnings against the existing biome config. Surfacing
-# the count in CI lets us drive it down without forcing a fleet-wide cleanup
-# in the same PR. Drop `continue-on-error` once that number is zero.
-
-name: Frontend CI
-
-on:
- pull_request:
- paths:
- - 'studio/frontend/**'
- - 'scripts/check_frontend_dep_removal.py'
- - 'tests/studio/test_frontend_dep_removal.py'
- - '.github/workflows/studio-frontend-ci.yml'
- push:
- branches: [main, pip]
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- build:
- name: Frontend build + bundle sanity
- runs-on: ubuntu-latest
- timeout-minutes: 10
- defaults:
- run:
- working-directory: studio/frontend
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- # FIXME: drop this step once @assistant-ui/* and assistant-stream
- # leave 0.x -- on 1.x, caret ranges are conventional. Until then,
- # every 0.minor on this surface is a SemVer-major (this is exactly
- # how 2026.5.1 shipped a broken chat runtime: ^0.12.19 quietly
- # resolved to 0.12.28).
- - name: '@assistant-ui must be pinned exactly (no caret/tilde)'
- working-directory: ${{ github.workspace }}
- run: |
- set -e
- if grep -nE '"(@assistant-ui/[a-z-]+|assistant-stream)":[[:space:]]*"[\^~]' studio/frontend/package.json; then
- echo "::error file=studio/frontend/package.json::These packages must be pinned to exact versions until they leave 0.x. Drop the leading ^ or ~."
- exit 1
- fi
- echo "All assistant-ui packages are pinned exactly."
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- # Run the structural lockfile scan BEFORE npm ci. A compromised
- # tarball runs its `prepare` / `postinstall` during `npm ci`,
- # so any catch has to fire upstream of that. The scanner is
- # pure-Python read-only; safe to call ahead of every install.
- - name: Lockfile supply-chain audit (pre-install scan)
- working-directory: ${{ github.workspace }}
- run: python3 scripts/lockfile_supply_chain_audit.py
-
- - name: Lockfile must agree with package.json (npm ci is strict)
- # Lifecycle scripts (esbuild native-binary postinstall, etc.) are
- # required for `vite build`. The pre-install lockfile structural
- # audit (lockfile_supply_chain_audit.py) is the practical defence
- # against the npm postinstall-dropper class -- it fires BEFORE any
- # tarball runs, on the injection pattern itself rather than an
- # advisory-DB lookup.
- run: npm ci --no-fund --no-audit
-
- - name: npm ci must not have modified the working tree
- working-directory: ${{ github.workspace }}
- run: |
- if ! git diff --quiet -- studio/frontend; then
- echo "::error::npm ci modified files; commit the updated lockfile"
- git status -- studio/frontend
- exit 1
- fi
-
- # Catch the common foot-gun: a dep dropped from package.json that is
- # still imported somewhere. The script walks the lockfile dep graph
- # from the new top-level deps and only counts top-level node_modules
- # paths as valid resolution targets for bare src/ imports.
- #
- # actions/checkout uses fetch-depth: 1 by default, so the base branch
- # is not available locally. Fetch the single base commit with an
- # explicit refspec so origin/ is reliably created (a bare
- # `git fetch origin [` only updates FETCH_HEAD in some configs).
- - name: Dependency removal safety check
- if: github.event_name == 'pull_request'
- working-directory: ${{ github.workspace }}
- run: |
- git fetch --no-tags --depth=1 origin \
- "${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"
- python3 scripts/check_frontend_dep_removal.py \
- --base "origin/${{ github.base_ref }}" \
- --enumerate-dead
- python3 tests/studio/test_frontend_dep_removal.py
-
- - name: Typecheck
- run: npm run typecheck
-
- - name: Build
- run: npm run build
-
- - name: Built bundle must not contain Studio's unstable_Provider call site
- run: |
- set -e
- JS=$(ls dist/assets/index-*.js | head -1)
- HITS=$(grep -c 'unstable_Provider:' "$JS" || echo 0)
- echo "main bundle: $JS"
- echo "unstable_Provider: hits=$HITS (assistant-ui internals contribute up to 3)"
- if [ "$HITS" -gt 3 ]; then
- echo "::error file=studio/frontend/src/features/chat/runtime-provider.tsx::Studio bundle still passes unstable_Provider through useRemoteThreadListRuntime; this is the 2026.5.1 chat-history regression. Pass adapters directly into useLocalRuntime instead."
- exit 1
- fi
-
- - name: Bundle size budget (75 MB)
- run: |
- SIZE=$(du -sb dist | cut -f1)
- BUDGET=$((75 * 1024 * 1024))
- echo "dist size: $SIZE bytes ($((SIZE/1024/1024)) MB), budget: $BUDGET bytes (75 MB)"
- if [ "$SIZE" -gt "$BUDGET" ]; then
- echo "::error::studio/frontend/dist/ exceeded the 75 MB budget. Drop dead deps (e.g. the unused next dep) or split chunks."
- exit 1
- fi
-
- - name: Biome (non-blocking until accumulated drift is cleared)
- continue-on-error: true
- run: npm run biome:check
-
- - name: Upload built dist
- # Always upload so a green run is reviewable too -- the dist
- # output catches "tests passed but bundle changed unexpectedly"
- # regressions that would be invisible if we only kept artifacts
- # on failure.
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: studio-frontend-dist
- path: studio/frontend/dist
- retention-days: 3
diff --git a/.github/workflows/studio-inference-smoke.yml b/.github/workflows/studio-inference-smoke.yml
deleted file mode 100644
index 6def56f769..0000000000
--- a/.github/workflows/studio-inference-smoke.yml
+++ /dev/null
@@ -1,1052 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# Three end-to-end smoke jobs that boot a freshly-installed Studio and
-# exercise the surfaces real users hit through the OpenAI / Anthropic
-# SDKs and curl. Each job picks the smallest model that exercises the
-# behaviour under test, primes HF_HOME via actions/cache, and shares
-# the install.sh --local --no-torch bootstrap.
-#
-# 1. OpenAI, Anthropic API tests
-# gemma-3-270m-it UD-Q4_K_XL (~254 MiB).
-# Password rotation via /api/auth/change-password (old fails,
-# new works), then OpenAI + Anthropic Python SDKs against /v1/*
-# with temperature=0 and a fixed seed. Asserts the four-turn
-# conversation is deterministic across two runs.
-#
-# 2. Tool calling Tests
-# Qwen3.5-2B UD-IQ3_XXS (~890 MiB). OpenAI function calling,
-# server-side tools (python, terminal, web_search) via
-# enable_tools / enabled_tools, and enable_thinking on/off.
-#
-# 3. JSON, images
-# gemma-4-E2B-it UD-IQ3_XXS (~2.4 GiB) + mmproj-F16 (~986 MiB).
-# response_format JSON-schema decoding and OpenAI image_url
-# (data URI) plus Anthropic source/base64 image inputs.
-#
-# All three jobs run in parallel. Total wall time is dominated by job 3
-# on a cold cache; warm cache cuts that to ~3 min.
-
-name: Studio GGUF CI
-
-on:
- pull_request:
- paths:
- - 'studio/**'
- - 'unsloth/**'
- - 'unsloth_cli/**'
- - 'install.sh'
- - 'pyproject.toml'
- - '.github/workflows/studio-inference-smoke.yml'
- push:
- branches: [main, pip]
- # Manual trigger for pre-warming HF_HOME caches on main, or re-running
- # against an arbitrary branch without pushing a no-op commit.
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- # ─────────────────────────────────────────────────────────────────────
- # Job 1: OpenAI, Anthropic API tests
- # ─────────────────────────────────────────────────────────────────────
- openai-anthropic:
- name: OpenAI, Anthropic API tests
- runs-on: ubuntu-latest
- timeout-minutes: 25
- env:
- GGUF_REPO: unsloth/gemma-3-270m-it-GGUF
- GGUF_VARIANT: UD-Q4_K_XL
- GGUF_FILE: gemma-3-270m-it-UD-Q4_K_XL.gguf
- STUDIO_PORT: '18888'
- HF_HOME: ${{ github.workspace }}/hf-cache
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - name: Linux deps for llama.cpp prebuilt
- run: |
- sudo apt-get update
- sudo apt-get install -y --no-install-recommends \
- libcurl4-openssl-dev libssl-dev jq
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- - name: Restore HF_HOME for ${{ env.GGUF_REPO }}
- id: cache-hf
- uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- continue-on-error: true
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
-
- - name: Prime HF_HOME with the GGUF
- id: prime-hf
- if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success'
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- run: |
- python -m pip install --upgrade huggingface_hub
- mkdir -p hf-cache
- bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE"
-
- - name: Save HF_HOME for ${{ env.GGUF_REPO }}
- if: always() && steps.prime-hf.outcome == 'success'
- uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
-
- - name: Install Studio (--local, --no-torch)
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- mkdir -p logs
- set -o pipefail
- bash install.sh --local --no-torch 2>&1 | tee logs/install.log
-
- - name: Install OpenAI + Anthropic Python SDKs
- run: pip install 'openai>=1.50' 'anthropic>=0.40'
-
- - name: Reset auth + boot Studio (API-only)
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \
- > logs/studio.log 2>&1 &
- echo "STUDIO_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then
- jq -e '.status == "healthy"' /tmp/health.json
- exit 0
- fi
- sleep 1
- done
- echo "Studio did not become healthy in 180s"
- tail -200 logs/studio.log
- exit 1
-
- - name: Password rotation (old must fail, new must work)
- run: |
- OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- NEW="CIRotated-$(python -c 'import secrets; print(secrets.token_urlsafe(12))')"
- echo "::add-mask::$OLD"
- echo "::add-mask::$NEW"
- # 1. Login with the bootstrap password.
- OLD_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}" | jq -r .access_token)
- [ -n "$OLD_TOKEN" ] && [ "$OLD_TOKEN" != "null" ] || { echo "bootstrap login failed"; exit 1; }
- # 2. Rotate to a fresh random password.
- curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/change-password" \
- -H "Authorization: Bearer $OLD_TOKEN" -H 'content-type: application/json' \
- -d "{\"current_password\":\"$OLD\",\"new_password\":\"$NEW\"}" > /dev/null
- # 3. Old password must now be rejected (HTTP 401).
- OLD_STATUS=$(curl -s -o /dev/null -w '%{http_code}' \
- -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}")
- if [ "$OLD_STATUS" != "401" ]; then
- echo "::error::Login with old password returned $OLD_STATUS, expected 401"
- exit 1
- fi
- # 4. New password must succeed; capture the JWT for downstream steps.
- NEW_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$NEW\"}" | jq -r .access_token)
- [ -n "$NEW_TOKEN" ] && [ "$NEW_TOKEN" != "null" ] || { echo "new login failed"; exit 1; }
- echo "TOKEN=$NEW_TOKEN" >> "$GITHUB_ENV"
- echo "password rotation OK (old=401, new=200)"
-
- - name: Load the GGUF (HF repo + variant, served from HF_HOME cache)
- run: |
- curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \
- -H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \
- --max-time 600 \
- -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048}" \
- | jq '{status, display_name, is_gguf, context_length}'
-
- - name: Multi-turn determinism via OpenAI + Anthropic SDKs
- env:
- BASE_URL: http://127.0.0.1:18888
- run: |
- python - <<'PY'
- import json
- import os
- from openai import OpenAI
- from anthropic import Anthropic
-
- BASE = os.environ["BASE_URL"]
- KEY = os.environ["TOKEN"] # JWT also accepted as Bearer on /v1/*
- SEED = 3407
-
- # Four-turn conversation: the second and fourth turns can only be
- # answered correctly if the model sees the prior turns, so this
- # also exercises the conversation-history wiring.
- PROMPTS = [
- "What is 1+1?",
- "What did I ask before?",
- "What is the capital of France?",
- "Repeat the city name",
- ]
-
- def run_openai():
- client = OpenAI(base_url = f"{BASE}/v1", api_key = KEY)
- history, replies = [], []
- for prompt in PROMPTS:
- history.append({"role": "user", "content": prompt})
- resp = client.chat.completions.create(
- model = "default",
- messages = history,
- temperature = 0.0,
- max_tokens = 80,
- seed = SEED,
- extra_body = {"enable_thinking": False},
- )
- text = resp.choices[0].message.content or ""
- replies.append(text)
- history.append({"role": "assistant", "content": text})
- return replies
-
- def run_anthropic():
- # Two SDK quirks vs. Studio:
- # 1. base_url must NOT include /v1 -- the SDK appends
- # /v1/messages itself; otherwise the request hits
- # /v1/v1/messages and 405s.
- # 2. The SDK sends `x-api-key` by default, but Studio's
- # auth layer is HTTPBearer-only. Override via
- # default_headers so Authorization: Bearer ... is
- # sent instead.
- client = Anthropic(
- base_url = BASE,
- api_key = "unused",
- default_headers = {"Authorization": f"Bearer {KEY}"},
- )
- history, replies = [], []
- for prompt in PROMPTS:
- history.append({"role": "user", "content": prompt})
- msg = client.messages.create(
- model = "default",
- max_tokens = 80,
- messages = history,
- temperature = 0.0,
- extra_body = {"seed": SEED, "enable_thinking": False},
- )
- text = "".join(b.text for b in msg.content if getattr(b, "type", None) == "text")
- replies.append(text)
- history.append({"role": "assistant", "content": text})
- return replies
-
- for label, runner in (("openai", run_openai), ("anthropic", run_anthropic)):
- first = runner()
- second = runner()
- determinism_failures = []
- for i, (a, b) in enumerate(zip(first, second), start = 1):
- print(f"[{label} turn {i}] {a!r}")
- # Both runs must be non-empty; small-quant drift
- # across runs is WARN-only (grounding asserts below
- # are the stronger signal).
- assert a, f"{label}: empty turn {i} response in first run"
- assert b, f"{label}: empty turn {i} response in second run"
- if a.strip() != b.strip():
- determinism_failures.append(
- f"turn {i}: run1={a!r} run2={b!r}"
- )
- if determinism_failures:
- print(
- f"[{label}] WARN non-determinism at temperature=0.0 across "
- f"{len(determinism_failures)} of {len(first)} turn(s); "
- f"small-quant model drift, not a Studio regression. "
- f"Details: " + " | ".join(determinism_failures)
- )
- # Sanity: turn-2 reply should mention the earlier question, and
- # turn-4 reply should mention Paris (model echoes the city it
- # produced for turn 3). Lower-cased substring checks keep the
- # assertion robust to formatting jitter.
- joined = " ".join(first).lower()
- assert "1" in first[0], f"{label}: turn-1 answer should contain '1', got {first[0]!r}"
- assert "paris" in joined, f"{label}: expected 'paris' somewhere in the four-turn transcript: {first}"
- status_word = "PASS" if not determinism_failures else "PASS (with drift)"
- print(f"[{label}] {status_word} -- 4 turns, history grounded ('paris' present)")
- PY
-
- - name: Stop Studio
- if: always()
- run: |
- kill "${STUDIO_PID}" 2>/dev/null || true
- sleep 2
- ss -tln | grep ":${STUDIO_PORT}" || true
-
- - name: Upload logs
- # Always upload so green runs are still reviewable.
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: openai-anthropic-log
- path: |
- logs/studio.log
- logs/install.log
- retention-days: 7
-
- # ─────────────────────────────────────────────────────────────────────
- # Job 2: Tool calling Tests
- # ─────────────────────────────────────────────────────────────────────
- tool-calling:
- name: Tool calling Tests
- runs-on: ubuntu-latest
- timeout-minutes: 25
- env:
- # Tool calling is the highest-volume GGUF in this workflow
- # (Qwen3.5-2B at IQ3_XXS = ~890 MiB). Caching HF_HOME would
- # store xet chunks + blobs + snapshots = ~4 GiB compressed --
- # 4-5x file-size inflation, dominated by xet chunks. Use main's
- # `--local-dir gguf-cache` pattern to cache the flat .gguf only.
- # Studio's /api/inference/load accepts either a HF repo (which
- # uses HF_HOME) or an absolute file path; passing the absolute
- # path keeps the test off HF_HOME entirely so the cache size
- # tracks the GGUF file 1:1. The OpenAI/Anth and JSON+images
- # jobs still cover the gguf_variant resolution path.
- GGUF_REPO: unsloth/Qwen3.5-2B-GGUF
- GGUF_FILE: Qwen3.5-2B-UD-IQ3_XXS.gguf
- STUDIO_PORT: '18889'
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - name: Linux deps for llama.cpp prebuilt
- run: |
- sudo apt-get update
- sudo apt-get install -y --no-install-recommends \
- libcurl4-openssl-dev libssl-dev jq
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- - name: Restore GGUF model file
- id: cache-gguf
- uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- continue-on-error: true
- with:
- path: gguf-cache
- key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1
-
- - name: Download GGUF if cache miss
- id: download-gguf
- if: steps.cache-gguf.outputs.cache-hit != 'true' || steps.cache-gguf.outcome != 'success'
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- run: |
- python -m pip install --upgrade huggingface_hub
- mkdir -p gguf-cache
- bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" gguf-cache
-
- - name: Save GGUF model file
- if: always() && steps.download-gguf.outcome == 'success'
- uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- with:
- path: gguf-cache
- key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1
-
- - name: Install Studio (--local, --no-torch)
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- mkdir -p logs
- set -o pipefail
- bash install.sh --local --no-torch 2>&1 | tee logs/install.log
-
- - name: Reset auth + boot Studio (API-only, default tool policy)
- # We deliberately use the API-only mode rather than
- # `unsloth studio run` because the latter calls
- # `set_tool_policy(...)` with a resolved bool: on loopback the
- # default resolves to True, which forces every request through
- # the server-side agentic loop and breaks the standard
- # function-calling test below. API-only mode leaves
- # tool_policy=None so each request's `enable_tools` field is
- # honoured.
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \
- > logs/studio.log 2>&1 &
- echo "STUDIO_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health, log in, change password, load model
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then
- jq -e '.status == "healthy"' /tmp/health.json && break
- fi
- sleep 1
- done
- jq -e '.status == "healthy"' /tmp/health.json
- OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- NEW="CITool-$(python -c 'import secrets; print(secrets.token_urlsafe(12))')"
- echo "::add-mask::$OLD"
- echo "::add-mask::$NEW"
- OLD_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}" | jq -r .access_token)
- curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/change-password" \
- -H "Authorization: Bearer $OLD_TOKEN" -H 'content-type: application/json' \
- -d "{\"current_password\":\"$OLD\",\"new_password\":\"$NEW\"}" > /dev/null
- TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$NEW\"}" | jq -r .access_token)
- echo "API_KEY=$TOKEN" >> "$GITHUB_ENV"
- GGUF_PATH="$GITHUB_WORKSPACE/gguf-cache/${GGUF_FILE}"
- ls -lh "$GGUF_PATH"
- curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \
- -H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \
- --max-time 600 \
- -d "{\"model_path\":\"$GGUF_PATH\",\"is_lora\":false,\"max_seq_length\":2048}" \
- | jq '{status, display_name}'
-
- - name: Tool calling, server-side tools, thinking on/off
- env:
- BASE_URL: http://127.0.0.1:18889
- run: |
- python - <<'PY'
- import json
- import os
- import urllib.request
-
- BASE = os.environ["BASE_URL"]
- KEY = os.environ["API_KEY"]
- SEED = 3407
-
- def post(path, body, *, timeout = 240):
- """Plain JSON POST. For requests that don't go through
- the server-side agentic loop, the response is one JSON
- object."""
- data = json.dumps(body).encode()
- req = urllib.request.Request(
- f"{BASE}{path}",
- data = data,
- method = "POST",
- headers = {
- "Authorization": f"Bearer {KEY}",
- "Content-Type": "application/json",
- },
- )
- with urllib.request.urlopen(req, timeout = timeout) as resp:
- return resp.status, json.loads(resp.read().decode())
-
- def post_sse(path, body, *, timeout = 600):
- """POST a streaming request and accumulate the assistant
- text deltas. The server-side agentic loop ALWAYS returns
- SSE regardless of the request's `stream` field, so any
- call with enable_tools=true must use this helper.
-
- Returns (content, raw_payloads):
- content -- concatenated assistant delta.content
- raw_payloads -- list of every raw "data: ..." event
- payload (JSON strings). Callers asserting
- that a server-side tool actually ran (and
- not just that the model emitted some
- text) should grep raw_payloads for tool
- invocation markers / tool output, since
- `delta.content` alone is not evidence
- that the tool path executed.
- """
- body = {**body, "stream": True}
- data = json.dumps(body).encode()
- req = urllib.request.Request(
- f"{BASE}{path}",
- data = data,
- method = "POST",
- headers = {
- "Authorization": f"Bearer {KEY}",
- "Content-Type": "application/json",
- },
- )
- parts = []
- events = []
- with urllib.request.urlopen(req, timeout = timeout) as resp:
- for raw in resp:
- line = raw.decode().strip()
- if not line.startswith("data: "):
- continue
- payload = line[6:]
- if payload == "[DONE]":
- break
- events.append(payload)
- try:
- chunk = json.loads(payload)
- except json.JSONDecodeError:
- continue
- for choice in chunk.get("choices", []):
- delta = choice.get("delta", {}) or {}
- if delta.get("content"):
- parts.append(delta["content"])
- return "".join(parts), events
-
- _STUDIO_TOOL_TYPES = {
- "tool_start", "tool_end", "tool_use", "tool_result",
- }
-
- def _tool_invoked(events):
- """Structural check: True iff some SSE payload is a real
- tool envelope (Studio tool_start/tool_end, Anthropic
- tool_use/tool_result, OpenAI non-empty delta.tool_calls /
- message.tool_calls / finish_reason='tool_calls' /
- role:'tool' / function_call). tool_status is NOT
- evidence: Studio emits empty tool_status events on
- iteration boundaries even when no tool ran.
- """
- for raw in events:
- try:
- ev = json.loads(raw)
- except (json.JSONDecodeError, TypeError):
- continue
- if not isinstance(ev, dict):
- continue
- if ev.get("type") in _STUDIO_TOOL_TYPES:
- return True
- for choice in ev.get("choices", []) or []:
- if not isinstance(choice, dict):
- continue
- if choice.get("finish_reason") == "tool_calls":
- return True
- for src_key in ("delta", "message"):
- src = choice.get(src_key) or {}
- if not isinstance(src, dict):
- continue
- tc = src.get("tool_calls")
- if isinstance(tc, list) and tc:
- return True
- if src.get("function_call"):
- return True
- if src.get("role") == "tool":
- return True
- for item in ev.get("output", []) or []:
- if isinstance(item, dict) and item.get("type") in {
- "tool_call", "function_call", "tool_use",
- }:
- return True
- content = ev.get("content")
- if isinstance(content, list):
- for blk in content:
- if isinstance(blk, dict) and blk.get("type") in {
- "tool_use", "tool_result",
- }:
- return True
- return False
-
- def _tool_output_contains(events, *needles):
- """True iff any tool_end.result / tool_result.content /
- tool-role message content contains a needle. Inspects
- the tool's own output, not the model's narration."""
- for raw in events:
- try:
- ev = json.loads(raw)
- except (json.JSONDecodeError, TypeError):
- continue
- if not isinstance(ev, dict):
- continue
- if ev.get("type") == "tool_end":
- result = ev.get("result")
- if isinstance(result, str) and any(n in result for n in needles if n):
- return True
- if ev.get("type") == "tool_result":
- content = ev.get("content")
- if isinstance(content, str) and any(n in content for n in needles if n):
- return True
- if isinstance(content, list):
- for blk in content:
- if isinstance(blk, dict):
- text = blk.get("text") or blk.get("content")
- if isinstance(text, str) and any(n in text for n in needles if n):
- return True
- for choice in ev.get("choices", []) or []:
- delta = (choice or {}).get("delta") or {}
- msg = (choice or {}).get("message") or {}
- for src in (delta, msg):
- if src.get("role") == "tool":
- content = src.get("content") or ""
- if isinstance(content, str) and any(n in content for n in needles if n):
- return True
- return False
-
- # ── 1. Standard OpenAI function calling ──────────────────────
- weather_tool = {
- "type": "function",
- "function": {
- "name": "get_weather",
- "description": "Get current weather for a city.",
- "parameters": {
- "type": "object",
- "properties": {"city": {"type": "string"}},
- "required": ["city"],
- },
- },
- }
-
- status, data = post("/v1/chat/completions", {
- "messages": [{"role": "user", "content": "What is the weather in Paris?"}],
- "tools": [weather_tool],
- "tool_choice": "required",
- "stream": False,
- "temperature": 0.0,
- "seed": SEED,
- "max_tokens": 120,
- })
- assert status == 200, f"tool call status {status}: {data}"
- choice = data["choices"][0]
- assert choice["finish_reason"] == "tool_calls", f"finish_reason={choice['finish_reason']!r}"
- tc = choice["message"]["tool_calls"][0]
- assert tc["function"]["name"] == "get_weather"
- args = json.loads(tc["function"]["arguments"])
- assert args.get("city"), f"missing city arg: {args}"
- print(f"[tools] PASS function calling -> {tc['function']['name']}({args})")
-
- # T=0 = deterministic argmax in llama.cpp; T>0 lets seed
- # rotation explore distinct trajectories on retry.
- TOOL_PROBE_TEMP = 0.4
-
- def _run_tool_probe(*, label, prompt, enabled, session, needles,
- max_attempts = 4):
- """Drive a server-side tool with retries. Hard FAIL if no
- attempt has structural invocation evidence. WARN (not
- FAIL) if invoked but no attempt produces the expected
- literal in tool_end.result -- small-quant Qwen3.5-2B can
- emit OpenAI tool_calls deltas without Studio's GGUF
- agentic loop intercepting them, and that GGUF-vs-OpenAI
- format mismatch is out of scope for #5642.
- """
- attempts_log = []
- best = None
- for attempt_i in range(max_attempts):
- attempt_seed = SEED + attempt_i
- content, events = post_sse("/v1/chat/completions", {
- "messages": [{"role": "user", "content": prompt}],
- "enable_tools": True,
- "enabled_tools": enabled,
- "session_id": f"{session}-att{attempt_i}",
- "temperature": TOOL_PROBE_TEMP,
- "seed": attempt_seed,
- "max_tokens": 600,
- })
- invoked = _tool_invoked(events)
- produced = _tool_output_contains(events, *needles)
- attempts_log.append({
- "attempt": attempt_i, "seed": attempt_seed,
- "n_events": len(events),
- "tool_invoked": invoked, "tool_output_contains": produced,
- "content_len": len(content),
- })
- if invoked and produced:
- print(f"[tools] PASS {label} attempt {attempt_i}")
- return content, events, attempts_log
- if invoked and best is None:
- best = (content, events)
- print(f"[tools] retry {label} attempt {attempt_i}: invoked={invoked} output_ok={produced} events={len(events)}")
- if best is not None:
- print(f"[tools] WARN {label}: invoked but no tool_end.result match (small-quant flake). Attempts: {attempts_log}")
- content, events = best
- return content, events, attempts_log
- raise AssertionError(
- f"{label}: no structural tool-invocation evidence across "
- f"{max_attempts} attempts. enable_tools may be silently "
- f"ignored. Attempts: {attempts_log}"
- )
-
- # ── 2. Server-side python tool ───────────────────────────────
- content, events, _attempts = _run_tool_probe(
- label = "python tool",
- prompt = "What is 123 * 456? Use the python tool to compute it and tell me the number.",
- enabled = ["python"],
- session = "ci-tool-calling-py",
- needles = ("56088", "56,088"),
- )
- if "56088" in content or "56,088" in content:
- print(f"[tools] python tool narration OK")
- else:
- print(f"[tools] python tool narration drifted -- content={content!r}")
-
- # ── 3. Server-side bash (terminal) tool ──────────────────────
- content, events, _attempts = _run_tool_probe(
- label = "bash/terminal tool",
- prompt = "Use the terminal tool to run `echo hello-bash-tool` and tell me the exact output.",
- enabled = ["terminal"],
- session = "ci-tool-calling-bash",
- needles = ("hello-bash-tool",),
- )
- if "hello-bash-tool" in content:
- print(f"[tools] bash/terminal narration OK")
- else:
- print(f"[tools] bash/terminal narration dropped literal -- content={content!r}")
-
- # ── 4. Server-side web_search tool ───────────────────────────
- # DuckDuckGo is flaky from CI runners and small Qwen3.5-2B
- # may not actually search. Only assert that the SSE stream
- # opens and yields any data; HTTP / parser failures already
- # raise above. Tool-invocation strictness is relaxed here
- # because (a) the search may legitimately return no results,
- # and (b) DuckDuckGo upstream blocks GHA IP ranges often
- # enough that requiring a tool_call marker would create
- # red-herring failures from infra rather than from Studio.
- try:
- content, events = post_sse("/v1/chat/completions", {
- "messages": [{"role": "user", "content": "Search the web for 'unsloth ai github' and summarise."}],
- "enable_tools": True,
- "enabled_tools": ["web_search"],
- "session_id": "ci-tool-calling-web",
- "temperature": 0.0,
- "seed": SEED,
- "max_tokens": 400,
- })
- print(
- f"[tools] PASS web_search stream ({len(content)} chars in content, "
- f"{len(events)} raw events)"
- )
- except Exception as exc:
- print(f"[tools] WARN web_search probe failed (non-blocking): {exc}")
-
- # ── 5. Thinking on / off ─────────────────────────────────────
- # Studio strips think blocks from message.content for tools-mode
- # responses, so we toggle plain chat (no enable_tools) and look
- # at the surfaced reasoning_content / message.thinking field.
- def thinking_call(enable):
- status, data = post("/v1/chat/completions", {
- "messages": [{"role": "user", "content": "Briefly: is 17 prime?"}],
- "stream": False,
- "enable_thinking": enable,
- "temperature": 0.0,
- "seed": SEED,
- "max_tokens": 300,
- })
- assert status == 200
- msg = data["choices"][0]["message"]
- # Studio surfaces thinking via reasoning_content (OpenAI
- # extension). Fall back to inline markers for
- # robustness across template versions.
- raw = (msg.get("content") or "") + (msg.get("reasoning_content") or "")
- return raw
-
- on_text = thinking_call(True)
- off_text = thinking_call(False)
- had_think_on = ("" in on_text) or len(on_text) > 80
- had_think_off = ("" in off_text) and len(off_text) > 0
- assert had_think_on, (
- f"enable_thinking=True produced no thinking signal: {on_text!r}"
- )
- # Off-mode should not contain the literal marker.
- assert "" not in off_text, (
- f"enable_thinking=False but still present: {off_text!r}"
- )
- print(f"[tools] PASS thinking on/off (on={len(on_text)} chars, off={len(off_text)} chars)")
- PY
-
- - name: Stop Studio
- if: always()
- run: |
- kill "${STUDIO_PID}" 2>/dev/null || true
- sleep 2
- ss -tln | grep ":${STUDIO_PORT}" || true
-
- - name: Upload logs
- # Always upload so green runs are still reviewable.
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: tool-calling-log
- path: |
- logs/studio.log
- logs/install.log
- retention-days: 7
-
- # ─────────────────────────────────────────────────────────────────────
- # Job 3: JSON, images
- # ─────────────────────────────────────────────────────────────────────
- json-images:
- name: JSON, images
- runs-on: ubuntu-latest
- timeout-minutes: 30
- env:
- GGUF_REPO: unsloth/gemma-4-E2B-it-GGUF
- GGUF_VARIANT: UD-IQ3_XXS
- GGUF_FILE: gemma-4-E2B-it-UD-IQ3_XXS.gguf
- MMPROJ_FILE: mmproj-F16.gguf
- STUDIO_PORT: '18890'
- HF_HOME: ${{ github.workspace }}/hf-cache
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - name: Linux deps for llama.cpp prebuilt
- run: |
- sudo apt-get update
- sudo apt-get install -y --no-install-recommends \
- libcurl4-openssl-dev libssl-dev jq
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- - name: Restore HF_HOME for ${{ env.GGUF_REPO }} (model + mmproj)
- id: cache-hf
- uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- continue-on-error: true
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-${{ env.MMPROJ_FILE }}-v1
-
- - name: Prime HF_HOME with the GGUF + mmproj
- id: prime-hf
- if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success'
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- run: |
- python -m pip install --upgrade huggingface_hub
- mkdir -p hf-cache
- bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE"
- bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$MMPROJ_FILE"
-
- - name: Save HF_HOME for ${{ env.GGUF_REPO }} (model + mmproj)
- if: always() && steps.prime-hf.outcome == 'success'
- uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-${{ env.MMPROJ_FILE }}-v1
-
- - name: Install Studio (--local, --no-torch)
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- mkdir -p logs
- set -o pipefail
- bash install.sh --local --no-torch 2>&1 | tee logs/install.log
-
- - name: Install OpenAI + Anthropic Python SDKs
- run: pip install 'openai>=1.50' 'anthropic>=0.40'
-
- - name: Reset auth + boot Studio (API-only)
- # See Job 2's comment: API-only mode keeps tool_policy=None so
- # response_format requests aren't routed through the agentic
- # tool loop.
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \
- > logs/studio.log 2>&1 &
- echo "STUDIO_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health, log in, change password, load model
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then
- jq -e '.status == "healthy"' /tmp/health.json && break
- fi
- sleep 1
- done
- jq -e '.status == "healthy"' /tmp/health.json
- OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- NEW="CIJson-$(python -c 'import secrets; print(secrets.token_urlsafe(12))')"
- echo "::add-mask::$OLD"
- echo "::add-mask::$NEW"
- OLD_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}" | jq -r .access_token)
- curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/change-password" \
- -H "Authorization: Bearer $OLD_TOKEN" -H 'content-type: application/json' \
- -d "{\"current_password\":\"$OLD\",\"new_password\":\"$NEW\"}" > /dev/null
- TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$NEW\"}" | jq -r .access_token)
- echo "API_KEY=$TOKEN" >> "$GITHUB_ENV"
- # Load the GGUF (mmproj is auto-detected via the HF repo
- # lookup, the cached file is pulled out of HF_HOME).
- curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \
- -H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \
- --max-time 900 \
- -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048}" \
- | jq '{status, display_name, is_vision}'
-
- - name: JSON schema decoding + image input
- env:
- BASE_URL: http://127.0.0.1:18890
- run: |
- python - <<'PY'
- import base64
- import json
- import os
- import urllib.request
- from openai import OpenAI
- from anthropic import Anthropic
-
- BASE = os.environ["BASE_URL"]
- KEY = os.environ["API_KEY"]
- SEED = 3407
-
- def post(path, body, *, timeout = 240):
- req = urllib.request.Request(
- f"{BASE}{path}",
- data = json.dumps(body).encode(),
- method = "POST",
- headers = {
- "Authorization": f"Bearer {KEY}",
- "Content-Type": "application/json",
- },
- )
- with urllib.request.urlopen(req, timeout = timeout) as resp:
- return resp.status, json.loads(resp.read().decode())
-
- # ── 1. response_format = json_object (JSON mode) ─────────────
- # llama.cpp's HTTP server supports OpenAI-compatible JSON
- # mode: `response_format: {"type": "json_object"}` constrains
- # the model to emit syntactically-valid JSON. We use raw HTTP
- # rather than the OpenAI SDK so that the field shape Studio
- # forwards to llama-server is unambiguous (the SDK rewrites
- # response_format depending on which variant it recognises).
- # We deliberately do NOT pass a strict JSON schema -- on
- # small Gemma-4 quants the GBNF-from-schema path occasionally
- # produces empty output, and JSON mode is the surface we care
- # about exposing through Studio.
- status, data = post("/v1/chat/completions", {
- "model": "default",
- "messages": [
- {"role": "system", "content": 'Reply with a single JSON object of the form {"city": "...", "country": "..."}. Output ONLY the JSON, nothing else.'},
- {"role": "user", "content": "What is the capital of France?"},
- ],
- "temperature": 0.0,
- "max_tokens": 200,
- "seed": SEED,
- "stream": False,
- "enable_thinking": False,
- "response_format": {"type": "json_object"},
- }, timeout = 600)
- assert status == 200, f"json status {status}: {data}"
- content = (data["choices"][0]["message"].get("content") or "").strip()
- # Some chat templates wrap JSON in ```json fences even in JSON
- # mode -- strip those before parsing.
- if content.startswith("```"):
- content = content.split("```", 2)[1]
- if content.startswith("json"):
- content = content[4:]
- content = content.strip("`\n ")
- parsed = json.loads(content)
- assert "paris" in str(parsed.get("city", "")).lower(), (
- f"city != Paris: {parsed}"
- )
- print(f"[json] PASS json_object -> {parsed}")
-
- # ── 2. OpenAI image_url (data URI base64) ───────────────────
- # 64x64 solid-red PNG. stb_image (used by Studio's image
- # normaliser at routes/inference.py:3410) rejects 4x4 or
- # smaller PNGs as truncated, so we go up to 64x64 -- still
- # tiny in token cost. The assertion is loose: any non-empty
- # response from the vision path proves multimodal end-to-end
- # wiring; small VL quants are weak at colour identification.
- PNG_64X64_RED_B64 = (
- "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAYklEQVR4nO3PMQ0AIADAMEAI/k"
- "UhBhEcDcmqYJtn7/GzpQNeNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA"
- "1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaBdCJ0BmMJ25zMAAAAASUVORK5CYII="
- )
- data_uri = f"data:image/png;base64,{PNG_64X64_RED_B64}"
-
- client = OpenAI(base_url = f"{BASE}/v1", api_key = KEY)
- openai_resp = client.chat.completions.create(
- model = "default",
- temperature = 0.0,
- max_tokens = 80,
- seed = SEED,
- messages = [{
- "role": "user",
- "content": [
- {"type": "image_url", "image_url": {"url": data_uri}},
- {"type": "text", "text": "What colour dominates this image? Reply in one word."},
- ],
- }],
- )
- openai_text = (openai_resp.choices[0].message.content or "").lower()
- print(f"[image/openai] reply: {openai_text!r}")
- assert openai_text, "OpenAI image_url returned empty content"
- # We do not strictly require 'red' -- some quants of small VL
- # models are weak at colour names. Just require a non-empty
- # answer; the vision path is the part under test.
- print("[image/openai] PASS image_url accepted, non-empty response")
-
- # ── 3. Anthropic source/base64 image ────────────────────────
- # Two SDK quirks vs. Studio: base_url must NOT include /v1
- # (the SDK appends it itself; otherwise /v1/v1/messages -> 405),
- # and Studio's auth is HTTPBearer-only so the SDK's default
- # x-api-key header is ignored -- send Authorization: Bearer
- # via default_headers.
- anthropic = Anthropic(
- base_url = BASE,
- api_key = "unused",
- default_headers = {"Authorization": f"Bearer {KEY}"},
- )
- a_msg = anthropic.messages.create(
- model = "default",
- max_tokens = 80,
- temperature = 0.0,
- extra_body = {"seed": SEED},
- messages = [{
- "role": "user",
- "content": [
- {
- "type": "image",
- "source": {
- "type": "base64",
- "media_type": "image/png",
- "data": PNG_64X64_RED_B64,
- },
- },
- {"type": "text", "text": "Describe this image briefly."},
- ],
- }],
- )
- a_text = "".join(b.text for b in a_msg.content if getattr(b, "type", None) == "text")
- print(f"[image/anthropic] reply: {a_text!r}")
- assert a_text, "Anthropic source/base64 returned empty content"
- print("[image/anthropic] PASS source/base64 accepted, non-empty response")
- PY
-
- - name: Stop Studio
- if: always()
- run: |
- kill "${STUDIO_PID}" 2>/dev/null || true
- sleep 2
- ss -tln | grep ":${STUDIO_PORT}" || true
-
- - name: Upload logs
- # Always upload so green runs are still reviewable.
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: json-images-log
- path: |
- logs/studio.log
- logs/install.log
- retention-days: 7
diff --git a/.github/workflows/studio-load-orchestrator-ci.yml b/.github/workflows/studio-load-orchestrator-ci.yml
deleted file mode 100644
index 93d1a7742d..0000000000
--- a/.github/workflows/studio-load-orchestrator-ci.yml
+++ /dev/null
@@ -1,68 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-#
-# Event-loop regression test for the Studio model-load orchestrator.
-# Pins down issue #5642 (Win10 UI freeze on model load): the /load
-# route calls LlamaCppBackend.detect_audio_type synchronously, blocking
-# the FastAPI event loop on a chain of sync httpx.Client.post() probes.
-#
-# The suite stands up a stdlib fake llama-server + a tiny FastAPI app
-# via uvicorn and asserts that detect_audio_type runs via
-# asyncio.to_thread so concurrent /api/inference/load-progress polling
-# stays responsive. CPU-only, no torch, no real llama.cpp binary, no
-# GPU -- the matching cross-OS staging proof lives on
-# danielhanchen/unsloth-staging-2 (Ubuntu / macOS / Windows all
-# green at PR time).
-
-name: Studio load-orchestrator CI
-
-on:
- pull_request:
- paths:
- - 'studio/backend/routes/inference.py'
- - 'studio/backend/core/inference/llama_cpp.py'
- - 'tests/studio/load_freeze/**'
- - '.github/workflows/studio-load-orchestrator-ci.yml'
- push:
- branches: [main]
- paths:
- - 'studio/backend/routes/inference.py'
- - 'studio/backend/core/inference/llama_cpp.py'
- - 'tests/studio/load_freeze/**'
- - '.github/workflows/studio-load-orchestrator-ci.yml'
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- test:
- runs-on: ubuntu-latest
- timeout-minutes: 15
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
- - name: Install minimal deps (no torch, no unsloth)
- # The test stubs `loggers` and `structlog`, imports
- # core.inference.llama_cpp directly, and drives a small
- # FastAPI app. Nothing here pulls torch or any GPU code,
- # so the entire job typically completes in well under 60 s.
- run: |
- python -m pip install --upgrade pip
- python -m pip install \
- 'pytest>=8' \
- 'httpx>=0.27,<1' \
- 'fastapi>=0.110,<1' \
- 'uvicorn>=0.30,<1' \
- 'anyio>=4'
- - name: Run load-orchestrator tests
- run: python -m pytest -v --tb=short tests/studio/load_freeze/
diff --git a/.github/workflows/studio-mac-api-smoke.yml b/.github/workflows/studio-mac-api-smoke.yml
deleted file mode 100644
index b4e274155e..0000000000
--- a/.github/workflows/studio-mac-api-smoke.yml
+++ /dev/null
@@ -1,153 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# Mac counterpart to studio-api-smoke.yml. Same tests/studio/
-# studio_api_smoke.py exercise (CORS hardening, auth state machine,
-# JWT expiry, API key lifecycle, /v1/models / /v1/embeddings /
-# /v1/responses, endpoint-by-endpoint auth audit) but on a real
-# Apple Silicon (macos-14, M1) runner. Drops the apt-get block;
-# GitHub-hosted macos-14 ships curl + jq.
-
-name: Mac Studio API CI
-
-on:
- pull_request:
- paths:
- - 'studio/**'
- - 'unsloth/**'
- - 'unsloth_cli/**'
- - 'install.sh'
- - 'pyproject.toml'
- - 'tests/studio/**'
- - '.github/workflows/studio-mac-api-smoke.yml'
- push:
- branches: [main, pip]
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- api-smoke:
- name: Studio API & Auth Tests
- runs-on: macos-14
- timeout-minutes: 25
- env:
- GGUF_REPO: unsloth/gemma-3-270m-it-GGUF
- GGUF_VARIANT: UD-Q4_K_XL
- GGUF_FILE: gemma-3-270m-it-UD-Q4_K_XL.gguf
- STUDIO_PORT: '18895'
- HF_HOME: ${{ github.workspace }}/hf-cache
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- - name: Restore HF_HOME for ${{ env.GGUF_REPO }}
- id: cache-hf
- uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- continue-on-error: true
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
-
- - name: Prime HF_HOME with the GGUF
- id: prime-hf
- if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success'
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- run: |
- python -m pip install --upgrade huggingface_hub
- mkdir -p hf-cache
- bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE"
-
- - name: Save HF_HOME for ${{ env.GGUF_REPO }}
- if: always() && steps.prime-hf.outcome == 'success'
- uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
-
- - name: Install Studio (--local, --no-torch)
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- mkdir -p logs
- set -o pipefail
- bash install.sh --local --no-torch 2>&1 | tee logs/install.log
-
- - name: Assert install.sh used the Mac llama.cpp prebuilt
- run: |
- if grep -q "falling back to source build" logs/install.log; then
- echo "::error::install.sh fell back to source-build llama.cpp on Mac. Studio must install the prebuilt llama-bNNNN-bin-macos-arm64 on Apple Silicon."
- grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
- exit 1
- fi
-
- - name: Install pyjwt for the JWT-expiry forge test
- run: pip install 'pyjwt>=2.6'
-
- - name: Reset auth + boot Studio (API-only)
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \
- > logs/studio.log 2>&1 &
- echo "STUDIO_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then
- jq -e '.status == "healthy"' /tmp/health.json && break
- fi
- sleep 1
- done
- jq -e '.status == "healthy"' /tmp/health.json
-
- - name: Pass bootstrap password + rotated targets to the test
- run: |
- OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- NEW="ApiSmoke-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- NEW2="ApiSmoke-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- echo "::add-mask::$OLD"
- echo "::add-mask::$NEW"
- echo "::add-mask::$NEW2"
- echo "STUDIO_OLD_PW=$OLD" >> "$GITHUB_ENV"
- echo "STUDIO_NEW_PW=$NEW" >> "$GITHUB_ENV"
- echo "STUDIO_NEW2_PW=$NEW2" >> "$GITHUB_ENV"
-
- - name: Run Studio API & Auth tests
- env:
- BASE_URL: http://127.0.0.1:18895
- STUDIO_AUTH_DIR: /Users/runner/.unsloth/studio/auth
- run: python tests/studio/studio_api_smoke.py
-
- - name: Stop Studio
- if: always()
- run: |
- kill "${STUDIO_PID}" 2>/dev/null || true
- sleep 2
-
- - name: Upload API smoke logs
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: mac-studio-api-smoke-log
- path: |
- logs/install.log
- logs/studio.log
- retention-days: 7
diff --git a/.github/workflows/studio-mac-inference-smoke.yml b/.github/workflows/studio-mac-inference-smoke.yml
deleted file mode 100644
index fab0a36bd1..0000000000
--- a/.github/workflows/studio-mac-inference-smoke.yml
+++ /dev/null
@@ -1,1049 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# Three end-to-end smoke jobs that boot a freshly-installed Studio and
-# exercise the surfaces real users hit through the OpenAI / Anthropic
-# SDKs and curl. Each job picks the smallest model that exercises the
-# behaviour under test, primes a model cache via actions/cache, and
-# shares the install.sh --local --no-torch bootstrap.
-#
-# 1. OpenAI, Anthropic API tests
-# gemma-3-270m-it UD-Q4_K_XL (~254 MiB).
-# Password rotation via /api/auth/change-password (old fails,
-# new works), then OpenAI + Anthropic Python SDKs against /v1/*
-# with temperature=0 and a fixed seed. Asserts the four-turn
-# conversation is deterministic across two runs.
-#
-# 2. Tool calling Tests
-# Qwen3.5-2B UD-IQ3_XXS (~890 MiB). OpenAI function calling,
-# server-side tools (python, terminal, web_search) via
-# enable_tools / enabled_tools, and enable_thinking on/off.
-#
-# 3. JSON, images
-# gemma-4-E2B-it UD-IQ3_XXS (~2.4 GiB) + mmproj-F16 (~986 MiB).
-# response_format JSON-schema decoding and OpenAI image_url
-# (data URI) plus Anthropic source/base64 image inputs.
-#
-# All three jobs run in parallel. Total wall time is dominated by job 3
-# on a cold cache; warm cache cuts that to ~3 min.
-
-name: Mac Studio GGUF CI
-
-on:
- pull_request:
- paths:
- - 'studio/**'
- - 'unsloth/**'
- - 'unsloth_cli/**'
- - 'install.sh'
- - 'pyproject.toml'
- - '.github/workflows/studio-mac-inference-smoke.yml'
- push:
- branches: [main, pip]
- # Manual trigger for pre-warming model caches on main, or re-running
- # against an arbitrary branch without pushing a no-op commit.
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- # ─────────────────────────────────────────────────────────────────────
- # Job 1: OpenAI, Anthropic API tests
- # ─────────────────────────────────────────────────────────────────────
- openai-anthropic:
- name: OpenAI, Anthropic API tests
- runs-on: macos-14
- timeout-minutes: 25
- env:
- GGUF_REPO: unsloth/gemma-3-270m-it-GGUF
- GGUF_VARIANT: UD-Q4_K_XL
- GGUF_FILE: gemma-3-270m-it-UD-Q4_K_XL.gguf
- STUDIO_PORT: '18888'
- HF_HOME: ${{ github.workspace }}/hf-cache
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- - name: Restore HF_HOME for ${{ env.GGUF_REPO }}
- id: cache-hf
- uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- continue-on-error: true
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
-
- - name: Prime HF_HOME with the GGUF
- id: prime-hf
- if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success'
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- run: |
- python -m pip install --upgrade huggingface_hub
- mkdir -p hf-cache
- bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE"
-
- # Save partial caches on cancel/timeout -- hf download resumes by
- # content hash. `outcome != skipped` keeps cache-hit a no-op.
- - name: Save HF_HOME for ${{ env.GGUF_REPO }}
- if: always() && steps.prime-hf.outcome != 'skipped' && hashFiles('hf-cache/**/*.gguf') != ''
- uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
-
- - name: Install Studio (--local, --no-torch)
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- mkdir -p logs
- set -o pipefail
- bash install.sh --local --no-torch 2>&1 | tee logs/install.log
-
- - name: Assert install.sh used the Mac llama.cpp prebuilt
- run: |
- if grep -q "falling back to source build" logs/install.log; then
- echo "::error::install.sh fell back to source-build llama.cpp on Mac. Studio must install the prebuilt llama-bNNNN-bin-macos-arm64 on Apple Silicon."
- grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
- exit 1
- fi
-
- - name: Install OpenAI + Anthropic Python SDKs
- run: pip install 'openai>=1.50' 'anthropic>=0.40'
-
- - name: Reset auth + boot Studio (API-only)
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \
- > logs/studio.log 2>&1 &
- echo "STUDIO_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then
- jq -e '.status == "healthy"' /tmp/health.json
- exit 0
- fi
- sleep 1
- done
- echo "Studio did not become healthy in 180s"
- tail -200 logs/studio.log
- exit 1
-
- - name: Password rotation (old must fail, new must work)
- run: |
- OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- NEW="CIRotated-$(python -c 'import secrets; print(secrets.token_urlsafe(12))')"
- echo "::add-mask::$OLD"
- echo "::add-mask::$NEW"
- # 1. Login with the bootstrap password.
- OLD_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}" | jq -r .access_token)
- [ -n "$OLD_TOKEN" ] && [ "$OLD_TOKEN" != "null" ] || { echo "bootstrap login failed"; exit 1; }
- # 2. Rotate to a fresh random password.
- curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/change-password" \
- -H "Authorization: Bearer $OLD_TOKEN" -H 'content-type: application/json' \
- -d "{\"current_password\":\"$OLD\",\"new_password\":\"$NEW\"}" > /dev/null
- # 3. Old password must now be rejected (HTTP 401).
- OLD_STATUS=$(curl -s -o /dev/null -w '%{http_code}' \
- -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}")
- if [ "$OLD_STATUS" != "401" ]; then
- echo "::error::Login with old password returned $OLD_STATUS, expected 401"
- exit 1
- fi
- # 4. New password must succeed; capture the JWT for downstream steps.
- NEW_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$NEW\"}" | jq -r .access_token)
- [ -n "$NEW_TOKEN" ] && [ "$NEW_TOKEN" != "null" ] || { echo "new login failed"; exit 1; }
- echo "TOKEN=$NEW_TOKEN" >> "$GITHUB_ENV"
- echo "password rotation OK (old=401, new=200)"
-
- - name: Load the GGUF (HF repo + variant, served from HF_HOME cache)
- run: |
- curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \
- -H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \
- --max-time 600 \
- -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048}" \
- | jq '{status, display_name, is_gguf, context_length}'
-
- - name: Multi-turn determinism via OpenAI + Anthropic SDKs
- env:
- BASE_URL: http://127.0.0.1:18888
- run: |
- python - <<'PY'
- import json
- import os
- from openai import OpenAI
- from anthropic import Anthropic
-
- BASE = os.environ["BASE_URL"]
- KEY = os.environ["TOKEN"] # JWT also accepted as Bearer on /v1/*
- SEED = 3407
-
- # Four-turn conversation: the second and fourth turns can only be
- # answered correctly if the model sees the prior turns, so this
- # also exercises the conversation-history wiring.
- PROMPTS = [
- "What is 1+1?",
- "What did I ask before?",
- "What is the capital of France?",
- "Repeat the city name",
- ]
-
- def run_openai():
- client = OpenAI(base_url = f"{BASE}/v1", api_key = KEY)
- history, replies = [], []
- for prompt in PROMPTS:
- history.append({"role": "user", "content": prompt})
- resp = client.chat.completions.create(
- model = "default",
- messages = history,
- temperature = 0.0,
- max_tokens = 80,
- seed = SEED,
- extra_body = {"enable_thinking": False},
- )
- text = resp.choices[0].message.content or ""
- replies.append(text)
- history.append({"role": "assistant", "content": text})
- return replies
-
- def run_anthropic():
- # Two SDK quirks vs. Studio:
- # 1. base_url must NOT include /v1 -- the SDK appends
- # /v1/messages itself; otherwise the request hits
- # /v1/v1/messages and 405s.
- # 2. The SDK sends `x-api-key` by default, but Studio's
- # auth layer is HTTPBearer-only. Override via
- # default_headers so Authorization: Bearer ... is
- # sent instead.
- client = Anthropic(
- base_url = BASE,
- api_key = "unused",
- default_headers = {"Authorization": f"Bearer {KEY}"},
- )
- history, replies = [], []
- for prompt in PROMPTS:
- history.append({"role": "user", "content": prompt})
- msg = client.messages.create(
- model = "default",
- max_tokens = 80,
- messages = history,
- temperature = 0.0,
- extra_body = {"seed": SEED, "enable_thinking": False},
- )
- text = "".join(b.text for b in msg.content if getattr(b, "type", None) == "text")
- replies.append(text)
- history.append({"role": "assistant", "content": text})
- return replies
-
- for label, runner in (("openai", run_openai), ("anthropic", run_anthropic)):
- first = runner()
- second = runner()
- for i, (a, b) in enumerate(zip(first, second), start = 1):
- print(f"[{label} turn {i}] {a!r}")
- assert a, f"{label}: empty turn {i} response"
- # Compare on stripped content: llama-server can vary
- # trailing whitespace (specifically a final '\n') between
- # otherwise-identical greedy runs depending on the
- # batch-flush boundary at which the stream is closed. The
- # generated tokens are identical; only the trailing
- # whitespace differs. Keep the raw repr in the failure
- # message so a real divergence is still legible.
- assert a.strip() == b.strip(), (
- f"{label} non-deterministic at turn {i} with temperature=0.0:\n"
- f" run1: {a!r}\n run2: {b!r}"
- )
- # Sanity: turn-2 reply should mention the earlier question, and
- # turn-4 reply should mention Paris (model echoes the city it
- # produced for turn 3). Lower-cased substring checks keep the
- # assertion robust to formatting jitter.
- joined = " ".join(first).lower()
- assert "1" in first[0], f"{label}: turn-1 answer should contain '1', got {first[0]!r}"
- assert "paris" in joined, f"{label}: expected 'paris' somewhere in the four-turn transcript: {first}"
- print(f"[{label}] OK -- 4 turns, run1 == run2, history grounded")
- PY
-
- - name: Stop Studio
- if: always()
- run: |
- kill "${STUDIO_PID}" 2>/dev/null || true
- sleep 2
- ss -tln | grep ":${STUDIO_PORT}" || true
-
- - name: Upload logs
- # Always upload so green runs are still reviewable.
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: openai-anthropic-log
- path: |
- logs/studio.log
- logs/install.log
- retention-days: 7
-
- # ─────────────────────────────────────────────────────────────────────
- # Job 2: Tool calling Tests
- # ─────────────────────────────────────────────────────────────────────
- tool-calling:
- name: Tool calling Tests
- runs-on: macos-14
- timeout-minutes: 25
- env:
- # Tool calling is the highest-volume GGUF in this workflow
- # (Qwen3.5-2B at Q4_K_XL = ~1.28 GiB on Mac, where IQ3_XXS
- # collapses for tool-call grammar under Metal at temperature=0).
- # Caching HF_HOME stores xet chunks + blobs + snapshots = ~4.6
- # GiB compressed -- 3.6x file-size inflation. Use main's
- # `--local-dir gguf-cache` pattern to cache the flat .gguf only.
- # The OpenAI/Anth and JSON+images jobs still cover the
- # gguf_variant resolution path.
- GGUF_REPO: unsloth/Qwen3.5-2B-GGUF
- GGUF_FILE: Qwen3.5-2B-UD-Q4_K_XL.gguf
- STUDIO_PORT: '18898'
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- - name: Restore GGUF model file
- id: cache-gguf
- uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- continue-on-error: true
- with:
- path: gguf-cache
- key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1
-
- - name: Download GGUF if cache miss
- id: download-gguf
- if: steps.cache-gguf.outputs.cache-hit != 'true' || steps.cache-gguf.outcome != 'success'
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- run: |
- python -m pip install --upgrade huggingface_hub
- mkdir -p gguf-cache
- bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" gguf-cache
-
- # Save partial caches on cancel; next run resumes via content hash.
- - name: Save GGUF model file
- if: always() && steps.download-gguf.outcome != 'skipped' && hashFiles('gguf-cache/**/*.gguf') != ''
- uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- with:
- path: gguf-cache
- key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1
-
- - name: Install Studio (--local, --no-torch)
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- mkdir -p logs
- set -o pipefail
- bash install.sh --local --no-torch 2>&1 | tee logs/install.log
-
- - name: Assert install.sh used the Mac llama.cpp prebuilt
- run: |
- if grep -q "falling back to source build" logs/install.log; then
- echo "::error::install.sh fell back to source-build llama.cpp on Mac. Studio must install the prebuilt llama-bNNNN-bin-macos-arm64 on Apple Silicon."
- grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
- exit 1
- fi
-
- - name: Reset auth + boot Studio (API-only, default tool policy)
- # We deliberately use the API-only mode rather than
- # `unsloth studio run` because the latter calls
- # `set_tool_policy(...)` with a resolved bool: on loopback the
- # default resolves to True, which forces every request through
- # the server-side agentic loop and breaks the standard
- # function-calling test below. API-only mode leaves
- # tool_policy=None so each request's `enable_tools` field is
- # honoured.
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \
- > logs/studio.log 2>&1 &
- echo "STUDIO_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health, log in, change password, load model
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then
- jq -e '.status == "healthy"' /tmp/health.json && break
- fi
- sleep 1
- done
- jq -e '.status == "healthy"' /tmp/health.json
- OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- NEW="CITool-$(python -c 'import secrets; print(secrets.token_urlsafe(12))')"
- echo "::add-mask::$OLD"
- echo "::add-mask::$NEW"
- OLD_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}" | jq -r .access_token)
- curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/change-password" \
- -H "Authorization: Bearer $OLD_TOKEN" -H 'content-type: application/json' \
- -d "{\"current_password\":\"$OLD\",\"new_password\":\"$NEW\"}" > /dev/null
- TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$NEW\"}" | jq -r .access_token)
- echo "API_KEY=$TOKEN" >> "$GITHUB_ENV"
- GGUF_PATH="$GITHUB_WORKSPACE/gguf-cache/${GGUF_FILE}"
- ls -lh "$GGUF_PATH"
- curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \
- -H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \
- --max-time 600 \
- -d "{\"model_path\":\"$GGUF_PATH\",\"is_lora\":false,\"max_seq_length\":2048}" \
- | jq '{status, display_name}'
-
- - name: Tool calling, server-side tools, thinking on/off
- env:
- BASE_URL: http://127.0.0.1:18898
- run: |
- python - <<'PY'
- import json
- import os
- import urllib.request
-
- BASE = os.environ["BASE_URL"]
- KEY = os.environ["API_KEY"]
- SEED = 3407
-
- def post(path, body, *, timeout = 240):
- """Plain JSON POST. For requests that don't go through
- the server-side agentic loop, the response is one JSON
- object."""
- data = json.dumps(body).encode()
- req = urllib.request.Request(
- f"{BASE}{path}",
- data = data,
- method = "POST",
- headers = {
- "Authorization": f"Bearer {KEY}",
- "Content-Type": "application/json",
- },
- )
- with urllib.request.urlopen(req, timeout = timeout) as resp:
- return resp.status, json.loads(resp.read().decode())
-
- def post_sse(path, body, *, timeout = 600):
- """POST a streaming request and accumulate the assistant
- text deltas. The server-side agentic loop ALWAYS returns
- SSE regardless of the request's `stream` field, so any
- call with enable_tools=true must use this helper."""
- body = {**body, "stream": True}
- data = json.dumps(body).encode()
- req = urllib.request.Request(
- f"{BASE}{path}",
- data = data,
- method = "POST",
- headers = {
- "Authorization": f"Bearer {KEY}",
- "Content-Type": "application/json",
- },
- )
- parts = []
- with urllib.request.urlopen(req, timeout = timeout) as resp:
- for raw in resp:
- line = raw.decode().strip()
- if not line.startswith("data: "):
- continue
- payload = line[6:]
- if payload == "[DONE]":
- break
- try:
- chunk = json.loads(payload)
- except json.JSONDecodeError:
- continue
- for choice in chunk.get("choices", []):
- delta = choice.get("delta", {}) or {}
- if delta.get("content"):
- parts.append(delta["content"])
- return "".join(parts)
-
- # ── 1. Standard OpenAI function calling ──────────────────────
- weather_tool = {
- "type": "function",
- "function": {
- "name": "get_weather",
- "description": "Get current weather for a city.",
- "parameters": {
- "type": "object",
- "properties": {"city": {"type": "string"}},
- "required": ["city"],
- },
- },
- }
-
- # Mac Metal at temperature=0 is pathological for these small
- # quants (Qwen3.5-2B emits ',,,,,,...' or 'The The The...'),
- # gemma-4-E2B emits '' tokens). The Linux CPU
- # backend hides the issue. Use a small non-zero temperature
- # with a fixed seed so we stay deterministic but escape the
- # degenerate sampling trap.
- TEMP = 0.2
-
- status, data = post("/v1/chat/completions", {
- "messages": [{"role": "user", "content": "What is the weather in Paris?"}],
- "tools": [weather_tool],
- "tool_choice": "required",
- "stream": False,
- "temperature": TEMP,
- "seed": SEED,
- # tool_choice='required' constrains the grammar so the
- # model emits a tool_call quickly when it works at all;
- # 128 tokens is enough for `{"city":"Paris"}` plus the
- # JSON envelope.
- "max_tokens": 128,
- }, timeout = 180)
- assert status == 200, f"tool call status {status}: {data}"
- choice = data["choices"][0]
- tool_calls = (choice.get("message") or {}).get("tool_calls") or []
- # Studio's contract: when tool_choice='required', llama.cpp's
- # grammar should force a tool_calls payload. On Mac that
- # contract is sometimes broken by the underlying quant; the
- # PASS path is "tool_calls present + correct schema", the
- # WARN path documents Studio still returned 200 with a
- # well-formed choices[] envelope.
- if tool_calls:
- tc = tool_calls[0]
- assert tc["function"]["name"] == "get_weather", (
- f"unexpected tool name: {tc['function']['name']!r}"
- )
- args = json.loads(tc["function"]["arguments"])
- assert args.get("city"), f"missing city arg: {args}"
- print(f"[tools] PASS function calling -> {tc['function']['name']}({args}) finish={choice.get('finish_reason')!r}")
- else:
- # Infrastructure path is correct; model output drifted.
- print(
- f"[tools] WARN function calling: no tool_calls (finish_reason="
- f"{choice.get('finish_reason')!r}); HTTP path OK, this is a "
- f"Mac Metal quant degeneracy."
- )
-
- # ── 2. Server-side python tool ───────────────────────────────
- # 123 * 456 = 56088. The agentic loop streams SSE; we
- # accumulate the assistant text and look for the answer. On
- # Mac the model often loses the tool calling contract before
- # producing the answer; accept either the answer OR a
- # non-empty SSE stream as proof the path completes.
- # macos-14 free runner is ~10 tok/s on Qwen3.5-2B Q4_K_XL;
- # cap max_tokens tightly so each SSE round stays under ~30s
- # even when the model stalls in a degenerate output state.
- content = post_sse("/v1/chat/completions", {
- "messages": [{"role": "user", "content": "What is 123 * 456? Use the python tool to compute it and tell me the number."}],
- "enable_tools": True,
- "enabled_tools": ["python"],
- "session_id": "ci-tool-calling-py",
- "temperature": TEMP,
- "seed": SEED,
- "max_tokens": 128,
- }, timeout = 180)
- if "56088" in content or "56,088" in content:
- print(f"[tools] PASS python tool ({len(content)} chars, found 56088)")
- else:
- # Empty stream is a known Mac-quant degeneracy too; log
- # but do not fail.
- print(
- f"[tools] WARN python tool: SSE OK ({len(content)} chars) but "
- f"model didn't return 56088 -- Mac quant drift"
- )
-
- # NOTE: the dedicated "Server-side bash (terminal) tool" axis
- # was dropped in favour of the python axis above. Both share
- # the SAME server-side agentic loop wiring (only the registry
- # entry differs); the python axis is the canonical proof. On
- # macos-14 the duplicated SSE round was the dominant cost in
- # this step, so collapsing the two saves ~30-60 s wallclock
- # without losing distinct coverage.
-
- # ── 3. Server-side web_search tool ───────────────────────────
- # DuckDuckGo is flaky from CI runners and small Qwen3.5-2B
- # may not actually search. Only assert that the SSE stream
- # opens and yields any data; HTTP / parser failures already
- # raise above.
- try:
- content = post_sse("/v1/chat/completions", {
- "messages": [{"role": "user", "content": "Search the web for 'unsloth ai github' and summarise."}],
- "enable_tools": True,
- "enabled_tools": ["web_search"],
- "session_id": "ci-tool-calling-web",
- "temperature": TEMP,
- "seed": SEED,
- "max_tokens": 96,
- }, timeout = 180)
- print(f"[tools] PASS web_search stream ({len(content)} chars)")
- except Exception as exc:
- print(f"[tools] WARN web_search probe failed (non-blocking): {exc}")
-
- # ── 4. Thinking on / off ─────────────────────────────────────
- # Studio strips think blocks from message.content for tools-mode
- # responses, so we toggle plain chat (no enable_tools) and look
- # at the surfaced reasoning_content / message.thinking field.
- def thinking_call(enable):
- status, data = post("/v1/chat/completions", {
- "messages": [{"role": "user", "content": "Briefly: is 17 prime?"}],
- "stream": False,
- "enable_thinking": enable,
- "temperature": TEMP,
- "seed": SEED,
- # 80 tokens lands within the 25-minute job timeout
- # on the macos-14 free runner. 17 is small; this is
- # plenty of room for either "Yes" + brief reasoning
- # or a degenerate empty completion.
- "max_tokens": 80,
- }, timeout = 180)
- assert status == 200
- msg = data["choices"][0]["message"]
- # Studio surfaces thinking via reasoning_content (OpenAI
- # extension). Fall back to inline markers for
- # robustness across template versions.
- raw = (msg.get("content") or "") + (msg.get("reasoning_content") or "")
- return raw
-
- on_text = thinking_call(True)
- off_text = thinking_call(False)
- # Mac quant drift: the model may produce empty / degenerate
- # output regardless of enable_thinking. Assert ONLY that the
- # endpoint returned 200 (already enforced inside thinking_call)
- # and that toggling the flag doesn't surface a hard
- # marker when off.
- had_think_on = ("" in on_text) or len(on_text) > 80
- if not had_think_on:
- print(
- f"[tools] WARN enable_thinking=True produced no thinking signal: "
- f"{on_text[:200]!r} -- Mac quant drift"
- )
- # Off-mode should not contain the literal marker.
- assert "" not in off_text, (
- f"enable_thinking=False but still present: {off_text!r}"
- )
- print(f"[tools] PASS thinking on/off (on={len(on_text)} chars, off={len(off_text)} chars)")
- PY
-
- - name: Stop Studio
- if: always()
- run: |
- kill "${STUDIO_PID}" 2>/dev/null || true
- sleep 2
- ss -tln | grep ":${STUDIO_PORT}" || true
-
- - name: Upload logs
- # Always upload so green runs are still reviewable.
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: tool-calling-log
- path: |
- logs/studio.log
- logs/install.log
- retention-days: 7
-
- # ─────────────────────────────────────────────────────────────────────
- # Job 3: JSON, images
- # ─────────────────────────────────────────────────────────────────────
- json-images:
- name: JSON, images
- runs-on: macos-14
- timeout-minutes: 30
- env:
- GGUF_REPO: unsloth/gemma-4-E2B-it-GGUF
- # Linux smoke uses UD-IQ3_XXS, but on Mac Metal that gemma-4
- # quant emits sentinel tokens () for any prompt at
- # temperature=0 -- inference path is fine, the quant itself is
- # broken on Metal. UD-Q4_K_XL is the smallest published variant
- # that generates real text on M1.
- GGUF_VARIANT: UD-Q4_K_XL
- GGUF_FILE: gemma-4-E2B-it-UD-Q4_K_XL.gguf
- MMPROJ_FILE: mmproj-F16.gguf
- STUDIO_PORT: '18899'
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- # Cache flat .gguf + mmproj (Job 2's pattern). HF_HOME inflates
- # ~3.6x via xet/blobs/snapshots, which made macOS saves never land.
- # mmproj is auto-detected as a sibling via detect_mmproj_file
- # (studio/backend/utils/models/model_config.py).
- - name: Restore GGUF + mmproj files
- id: cache-gguf
- uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- continue-on-error: true
- with:
- path: gguf-cache
- key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-${{ env.MMPROJ_FILE }}-v2
-
- - name: Verify cache contains BOTH gguf + mmproj
- id: verify-cache
- if: steps.cache-gguf.outputs.cache-hit == 'true'
- run: |
- if [[ -f "gguf-cache/$GGUF_FILE" && -f "gguf-cache/$MMPROJ_FILE" ]]; then
- echo "ok=true" >> "$GITHUB_OUTPUT"
- else
- echo "Partial cache hit -- forcing re-download."
- echo "ok=false" >> "$GITHUB_OUTPUT"
- fi
-
- - name: Download GGUF + mmproj if cache miss or partial
- id: download-gguf
- if: steps.cache-gguf.outputs.cache-hit != 'true' || steps.verify-cache.outputs.ok != 'true'
- # Authenticated + parallel: shared macos-14 NAT egress stalls
- # multi-GB anonymous downloads.
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- run: |
- python -m pip install --upgrade huggingface_hub
- mkdir -p gguf-cache
- bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" gguf-cache &
- MODEL_PID=$!
- bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$MMPROJ_FILE" gguf-cache &
- MMPROJ_PID=$!
- wait "$MODEL_PID"
- wait "$MMPROJ_PID"
- # Fail loud on a partial download instead of in the next step.
- ls -lh "gguf-cache/$GGUF_FILE" "gguf-cache/$MMPROJ_FILE"
-
- # Save partial caches on cancel. hashFiles guard avoids a hard
- # save failure when the download step exits with no files. The
- # additional mmproj-presence check stops a partial save from
- # poisoning the cache for the next run.
- - name: Save GGUF + mmproj files
- if: always() && steps.download-gguf.outcome != 'skipped' && hashFiles('gguf-cache/**/*.gguf') != '' && hashFiles(format('gguf-cache/{0}', env.MMPROJ_FILE)) != ''
- uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- with:
- path: gguf-cache
- key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-${{ env.MMPROJ_FILE }}-v2
-
- - name: Install Studio (--local, --no-torch)
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- mkdir -p logs
- set -o pipefail
- bash install.sh --local --no-torch 2>&1 | tee logs/install.log
-
- - name: Assert install.sh used the Mac llama.cpp prebuilt
- run: |
- if grep -q "falling back to source build" logs/install.log; then
- echo "::error::install.sh fell back to source-build llama.cpp on Mac. Studio must install the prebuilt llama-bNNNN-bin-macos-arm64 on Apple Silicon."
- grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
- exit 1
- fi
-
- - name: Install OpenAI + Anthropic Python SDKs
- run: pip install 'openai>=1.50' 'anthropic>=0.40'
-
- - name: Reset auth + boot Studio (API-only)
- # See Job 2's comment: API-only mode keeps tool_policy=None so
- # response_format requests aren't routed through the agentic
- # tool loop.
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \
- > logs/studio.log 2>&1 &
- echo "STUDIO_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health, log in, change password, load model
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then
- jq -e '.status == "healthy"' /tmp/health.json && break
- fi
- sleep 1
- done
- jq -e '.status == "healthy"' /tmp/health.json
- OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- NEW="CIJson-$(python -c 'import secrets; print(secrets.token_urlsafe(12))')"
- echo "::add-mask::$OLD"
- echo "::add-mask::$NEW"
- OLD_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}" | jq -r .access_token)
- curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/change-password" \
- -H "Authorization: Bearer $OLD_TOKEN" -H 'content-type: application/json' \
- -d "{\"current_password\":\"$OLD\",\"new_password\":\"$NEW\"}" > /dev/null
- TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$NEW\"}" | jq -r .access_token)
- echo "API_KEY=$TOKEN" >> "$GITHUB_ENV"
- # Load via local file path; mmproj sibling auto-detected by
- # detect_mmproj_file (model_config.py). gguf_variant omitted
- # -- it routes through _find_local_gguf_by_variant which
- # expects a directory, not a file path.
- GGUF_PATH="$GITHUB_WORKSPACE/gguf-cache/${GGUF_FILE}"
- MMPROJ_PATH="$GITHUB_WORKSPACE/gguf-cache/${MMPROJ_FILE}"
- ls -lh "$GGUF_PATH" "$MMPROJ_PATH"
- curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \
- -H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \
- --max-time 900 \
- -d "{\"model_path\":\"$GGUF_PATH\",\"is_lora\":false,\"max_seq_length\":2048}" \
- | jq '{status, display_name, is_vision}'
-
- - name: JSON schema decoding + image input
- env:
- BASE_URL: http://127.0.0.1:18899
- run: |
- python - <<'PY'
- import base64
- import json
- import os
- import urllib.request
- from openai import OpenAI
- from anthropic import Anthropic
-
- BASE = os.environ["BASE_URL"]
- KEY = os.environ["API_KEY"]
- SEED = 3407
- # Mac Metal degenerates these gemma-4 quants at temperature=0
- # (any prompt yields '...' padding tokens). Use a
- # small non-zero temperature with the same seed so we stay
- # deterministic-enough but escape the trap.
- TEMP = 0.2
-
- def post(path, body, *, timeout = 240):
- req = urllib.request.Request(
- f"{BASE}{path}",
- data = json.dumps(body).encode(),
- method = "POST",
- headers = {
- "Authorization": f"Bearer {KEY}",
- "Content-Type": "application/json",
- },
- )
- with urllib.request.urlopen(req, timeout = timeout) as resp:
- return resp.status, json.loads(resp.read().decode())
-
- # ── 1. response_format = json_object (JSON mode) ─────────────
- # llama.cpp's HTTP server supports OpenAI-compatible JSON
- # mode: `response_format: {"type": "json_object"}` constrains
- # the model to emit syntactically-valid JSON. We use raw HTTP
- # rather than the OpenAI SDK so that the field shape Studio
- # forwards to llama-server is unambiguous (the SDK rewrites
- # response_format depending on which variant it recognises).
- # We deliberately do NOT pass a strict JSON schema -- on
- # small Gemma-4 quants the GBNF-from-schema path occasionally
- # produces empty output, and JSON mode is the surface we care
- # about exposing through Studio.
- status, data = post("/v1/chat/completions", {
- "model": "default",
- "messages": [
- {"role": "system", "content": 'Reply with a single JSON object of the form {"city": "...", "country": "..."}. Output ONLY the JSON, nothing else.'},
- {"role": "user", "content": "What is the capital of France?"},
- ],
- "temperature": TEMP,
- # Trimmed for Mac runner timeout budget; json_object
- # grammar terminates quickly when working.
- "max_tokens": 200,
- "seed": SEED,
- "stream": False,
- "enable_thinking": False,
- "response_format": {"type": "json_object"},
- }, timeout = 240)
- assert status == 200, f"json status {status}: {data}"
- # Verify the response envelope shape -- this is what we
- # actually want to exercise on Mac. The model output quality
- # downstream of this is a Mac-Metal-quant artefact.
- assert (
- isinstance(data.get("choices"), list)
- and data["choices"]
- and "message" in data["choices"][0]
- ), f"json response envelope malformed: {data}"
- content = (data["choices"][0]["message"].get("content") or "").strip()
- print(f"[json] raw json_object content: {content!r}")
- # Some chat templates wrap JSON in ```json fences even in JSON
- # mode -- strip those before parsing.
- if content.startswith("```"):
- content = content.split("```", 2)[1]
- if content.startswith("json"):
- content = content[4:]
- content = content.strip("`\n ")
- if content:
- try:
- parsed = json.loads(content)
- if "paris" in str(parsed.get("city", "")).lower():
- print(f"[json] PASS json_object -> {parsed}")
- else:
- print(f"[json] WARN json_object decoded but city!=Paris: {parsed}")
- except json.JSONDecodeError as exc:
- print(f"[json] WARN json_object content not parseable ({exc}); content={content!r}")
- else:
- print("[json] WARN json_object produced empty content on this Mac quant")
- # Cross-check: same prompt without response_format. We care
- # that the inference path stays healthy (status 200 + envelope
- # shape OK); model output quality is a separate concern.
- status2, data2 = post("/v1/chat/completions", {
- "model": "default",
- "messages": [{"role": "user", "content": "What is the capital of France? Answer with one word."}],
- "temperature": TEMP,
- # 1-word answer doesn't need 400 tokens; trim so a
- # degenerate streaming model doesn't burn through the
- # job's wallclock budget.
- "max_tokens": 150,
- "seed": SEED,
- "stream": False,
- "enable_thinking": False,
- }, timeout = 240)
- assert status2 == 200, f"plain status {status2}: {data2}"
- plain = (data2["choices"][0]["message"].get("content") or "").lower()
- print(f"[json] plain capital-of-france reply: {plain!r}")
- if "paris" in plain:
- print("[json] PASS plain inference path (paris mentioned)")
- else:
- print(
- f"[json] WARN plain inference returned no 'paris' -- Mac quant "
- f"degeneracy. HTTP path validated separately above."
- )
-
- # ── 2. OpenAI image_url (data URI base64) ───────────────────
- # 64x64 solid-red PNG. stb_image (used by Studio's image
- # normaliser at routes/inference.py:3410) rejects 4x4 or
- # smaller PNGs as truncated, so we go up to 64x64 -- still
- # tiny in token cost. The assertion is loose: any non-empty
- # response from the vision path proves multimodal end-to-end
- # wiring; small VL quants are weak at colour identification.
- PNG_64X64_RED_B64 = (
- "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAYklEQVR4nO3PMQ0AIADAMEAI/k"
- "UhBhEcDcmqYJtn7/GzpQNeNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA"
- "1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaBdCJ0BmMJ25zMAAAAASUVORK5CYII="
- )
- data_uri = f"data:image/png;base64,{PNG_64X64_RED_B64}"
-
- # The Mac prebuilt llama.cpp server has a known crash when
- # processing image inputs alongside the gemma-4-E2B mmproj
- # (server disconnects mid-completion). This is upstream
- # llama.cpp behaviour, not Studio. Wrap both SDK calls in
- # try/except so an upstream crash registers as a WARN rather
- # than failing the whole job. Studio's contract (OpenAI/
- # Anthropic image fields are accepted and forwarded) is
- # validated by the request body Studio constructs, not by
- # whether llama.cpp can decode it on Mac Metal.
- client = OpenAI(base_url = f"{BASE}/v1", api_key = KEY)
- try:
- openai_resp = client.chat.completions.create(
- model = "default",
- temperature = TEMP,
- max_tokens = 80,
- seed = SEED,
- messages = [{
- "role": "user",
- "content": [
- {"type": "image_url", "image_url": {"url": data_uri}},
- {"type": "text", "text": "What colour dominates this image? Reply in one word."},
- ],
- }],
- )
- openai_text = (openai_resp.choices[0].message.content or "").lower()
- print(f"[image/openai] reply: {openai_text!r}")
- if openai_text:
- print("[image/openai] PASS image_url accepted, non-empty response")
- else:
- print("[image/openai] WARN image_url accepted but empty content -- Mac quant drift")
- except Exception as exc:
- print(
- f"[image/openai] WARN image_url SDK call raised: {type(exc).__name__}: "
- f"{exc}. Likely upstream llama.cpp Mac+vision crash, NOT a Studio "
- f"regression. Studio successfully forwarded the request."
- )
-
- # ── 3. Anthropic source/base64 image ────────────────────────
- # Two SDK quirks vs. Studio: base_url must NOT include /v1
- # (the SDK appends it itself; otherwise /v1/v1/messages -> 405),
- # and Studio's auth is HTTPBearer-only so the SDK's default
- # x-api-key header is ignored -- send Authorization: Bearer
- # via default_headers.
- anthropic = Anthropic(
- base_url = BASE,
- api_key = "unused",
- default_headers = {"Authorization": f"Bearer {KEY}"},
- )
- try:
- a_msg = anthropic.messages.create(
- model = "default",
- max_tokens = 80,
- temperature = TEMP,
- extra_body = {"seed": SEED},
- messages = [{
- "role": "user",
- "content": [
- {
- "type": "image",
- "source": {
- "type": "base64",
- "media_type": "image/png",
- "data": PNG_64X64_RED_B64,
- },
- },
- {"type": "text", "text": "Describe this image briefly."},
- ],
- }],
- )
- a_text = "".join(b.text for b in a_msg.content if getattr(b, "type", None) == "text")
- print(f"[image/anthropic] reply: {a_text!r}")
- if a_text:
- print("[image/anthropic] PASS source/base64 accepted, non-empty response")
- else:
- print("[image/anthropic] WARN source/base64 accepted but empty content -- Mac quant drift")
- except Exception as exc:
- print(
- f"[image/anthropic] WARN anthropic image SDK call raised: "
- f"{type(exc).__name__}: {exc}. Likely upstream llama.cpp Mac+vision "
- f"crash, NOT a Studio regression."
- )
- PY
-
- - name: Stop Studio
- if: always()
- run: |
- kill "${STUDIO_PID}" 2>/dev/null || true
- sleep 2
- ss -tln | grep ":${STUDIO_PORT}" || true
-
- - name: Upload logs
- # Always upload so green runs are still reviewable.
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: json-images-log
- path: |
- logs/studio.log
- logs/install.log
- retention-days: 7
diff --git a/.github/workflows/studio-mac-ui-smoke.yml b/.github/workflows/studio-mac-ui-smoke.yml
deleted file mode 100644
index b353f0ec83..0000000000
--- a/.github/workflows/studio-mac-ui-smoke.yml
+++ /dev/null
@@ -1,345 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# Mac counterpart to studio-ui-smoke.yml. Same Playwright + Chromium
-# end-to-end chat UI flow, but on macos-14 (M1) so we catch
-# Mac-specific frontend / backend wiring regressions that the Linux
-# job would miss (e.g. the Mac Tauri shell loading the same React
-# bundle, or the Mac llama.cpp prebuilt's HTTP layer behaving
-# differently from the Linux build).
-
-name: Mac Studio UI CI
-
-on:
- pull_request:
- paths:
- - 'studio/**'
- - 'unsloth/**'
- - 'unsloth_cli/**'
- - 'install.sh'
- - 'pyproject.toml'
- - 'tests/studio/**'
- - '.github/workflows/studio-mac-ui-smoke.yml'
- push:
- branches: [main, pip]
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- ui-smoke:
- name: Chat UI Tests
- runs-on: macos-14
- timeout-minutes: 35
- env:
- GGUF_REPO: unsloth/gemma-3-270m-it-GGUF
- GGUF_VARIANT: UD-Q4_K_XL
- GGUF_FILE: gemma-3-270m-it-UD-Q4_K_XL.gguf
- STUDIO_PORT: '18896'
- HF_HOME: ${{ github.workspace }}/hf-cache
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- - name: Restore HF_HOME for ${{ env.GGUF_REPO }}
- id: cache-hf
- uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- continue-on-error: true
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
-
- - name: Prime HF_HOME with the GGUF
- id: prime-hf
- if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success'
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- run: |
- python -m pip install --upgrade huggingface_hub
- mkdir -p hf-cache
- bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE"
-
- - name: Save HF_HOME for ${{ env.GGUF_REPO }}
- if: always() && steps.prime-hf.outcome == 'success'
- uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
-
- - name: Install Studio (--local, --no-torch)
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- mkdir -p logs
- set -o pipefail
- bash install.sh --local --no-torch 2>&1 | tee logs/install.log
-
- - name: Assert install.sh used the Mac llama.cpp prebuilt
- run: |
- if grep -q "falling back to source build" logs/install.log; then
- echo "::error::install.sh fell back to source-build llama.cpp on Mac. Studio must install the prebuilt llama-bNNNN-bin-macos-arm64 on Apple Silicon."
- grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
- exit 1
- fi
-
- - name: Install Playwright + Chromium
- # No --with-deps on Mac: that flag installs Linux apt packages.
- # GitHub-hosted macos-14 ships the system frameworks Chromium
- # needs already.
- # Pinned <1.58 because all 1.55-1.58 drivers ship Node 24 on
- # macos-14 and intermittently hit 'SyntaxError: Unexpected end
- # of JSON input' in pipeTransport.js. Run 25491698868 showed
- # the crash hitting 100% of three retry attempts -- not a
- # rare race but a hard reproduction. Belt-and-suspenders fix:
- # the test scripts pass --single-process to Chromium (see
- # tests/studio/playwright_chat_ui.py) AND we patch
- # pipeTransport.js below to swallow JSON parse errors instead
- # of crashing the driver Node process. Both together let the
- # in-script retry recover from any residual flakes.
- run: |
- pip install 'playwright>=1.55,<1.58'
- python -m playwright install chromium
-
- - name: Patch Playwright pipeTransport.js to tolerate malformed JSON
- # In Playwright 1.55-1.58, pipeTransport.js does
- # `JSON.parse(message)` with no try/catch; when Chromium dies
- # mid-write the partial buffer crashes the driver Node
- # process and the test script exits with 'Connection closed
- # while reading from the driver'. Newer Playwright versions
- # added a try/catch upstream. Backport that here.
- run: |
- python - <<'PY'
- import os, re, sys
- import playwright
- driver_dir = os.path.join(os.path.dirname(playwright.__file__), "driver", "package", "lib", "server")
- path = os.path.join(driver_dir, "pipeTransport.js")
- src = open(path).read()
- # Wrap both `this.onmessage.call(null, JSON.parse(...))` sites in try/catch.
- patched = re.sub(
- r"this\.onmessage\.call\(null, JSON\.parse\((message2?)\)\);",
- r"try { this.onmessage.call(null, JSON.parse(\1)); } "
- r"catch (e) { /* swallow malformed JSON from a crashing browser */ }",
- src,
- )
- if patched == src:
- # Already patched, or upstream changed -- either way, don't fail the build.
- print(f"pipeTransport.js: no JSON.parse calls matched at {path}; skipping.")
- else:
- open(path, "w").write(patched)
- print(f"pipeTransport.js: patched JSON.parse calls in {path}")
- PY
-
- - name: Reset auth + boot Studio
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \
- > logs/studio.log 2>&1 &
- echo "STUDIO_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then
- jq -e '.status == "healthy"' /tmp/health.json && break
- fi
- sleep 1
- done
- jq -e '.status == "healthy"' /tmp/health.json
-
- - name: Pass bootstrap password to the Playwright step
- run: |
- OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- NEW="CIUi-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- NEW2="CIUi-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- echo "::add-mask::$OLD"
- echo "::add-mask::$NEW"
- echo "::add-mask::$NEW2"
- echo "STUDIO_OLD_PW=$OLD" >> "$GITHUB_ENV"
- echo "STUDIO_NEW_PW=$NEW" >> "$GITHUB_ENV"
- echo "STUDIO_NEW2_PW=$NEW2" >> "$GITHUB_ENV"
-
- - name: Drive the chat UI with Playwright
- env:
- BASE_URL: http://127.0.0.1:18896
- PW_ART_DIR: logs/playwright
- STUDIO_UI_STRICT: '1'
- # macos-14 free runner is 3 vCPU / 7 GB / no Metal-accel
- # available to llama.cpp from CI; gemma-3-270m turn latency
- # has been observed to crowd the 180s default. Triple it.
- STUDIO_UI_TURN_TIMEOUT_MS: '540000'
- # Retry up to 3 times to absorb known macos-14 free-runner
- # flakes: (1) Playwright Node 24 pipeTransport.js 'Unexpected
- # end of JSON input' crash when the Chromium browser process
- # dies mid-test, and (2) Chromium net::ERR_NO_BUFFER_SPACE
- # when the runner's kernel briefly runs out of socket buffers.
- # The retry FULLY resets Studio (kill, reset-password, reboot,
- # wait /api/health, re-export bootstrap pw) before re-running
- # the script. A real test failure (assertion / timeout) does
- # NOT match either pattern so it bypasses retry and surfaces
- # immediately.
- run: |
- mkdir -p logs/playwright
- attempt=1
- max_attempts=3
- while : ; do
- set +e
- python tests/studio/playwright_chat_ui.py 2>&1 | tee logs/playwright_attempt_${attempt}.log
- rc=${PIPESTATUS[0]}
- set -e
- if [ "$rc" -eq 0 ]; then
- break
- fi
- if { grep -q "Unexpected end of JSON input" logs/playwright_attempt_${attempt}.log \
- || grep -q "ERR_NO_BUFFER_SPACE" logs/playwright_attempt_${attempt}.log; } \
- && [ "$attempt" -lt "$max_attempts" ]; then
- echo "::warning::Playwright flake on attempt ${attempt}; resetting Studio and retrying..."
- kill "${STUDIO_PID}" 2>/dev/null || true
- sleep 2
- unsloth studio reset-password
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \
- > "logs/studio_retry_${attempt}.log" 2>&1 &
- STUDIO_PID=$!
- echo "STUDIO_PID=$STUDIO_PID" >> "$GITHUB_ENV"
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json \
- && jq -e '.status == "healthy"' /tmp/health.json >/dev/null; then
- break
- fi
- sleep 1
- done
- STUDIO_OLD_PW=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- STUDIO_NEW_PW="CIUi-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- STUDIO_NEW2_PW="CIUi-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- echo "::add-mask::$STUDIO_OLD_PW"
- echo "::add-mask::$STUDIO_NEW_PW"
- echo "::add-mask::$STUDIO_NEW2_PW"
- export STUDIO_OLD_PW STUDIO_NEW_PW STUDIO_NEW2_PW
- attempt=$((attempt + 1))
- sleep 3
- continue
- fi
- exit "$rc"
- done
-
- - name: Stop Studio (chat-ui ends with Shutdown click; this is belt-and-suspenders)
- if: always()
- run: |
- kill "${STUDIO_PID}" 2>/dev/null || true
- sleep 2
-
- - name: Reset auth + boot Studio for extra UI tests (port 18897)
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18897 \
- > logs/studio_extra.log 2>&1 &
- echo "STUDIO_EXTRA_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health on 18897
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:18897/api/health" > /tmp/health2.json; then
- jq -e '.status == "healthy"' /tmp/health2.json && break
- fi
- sleep 1
- done
- jq -e '.status == "healthy"' /tmp/health2.json
-
- - name: Pass bootstrap pw for extra UI test
- run: |
- OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- NEW="CIUiExtra-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- echo "::add-mask::$OLD"
- echo "::add-mask::$NEW"
- echo "STUDIO_EXTRA_OLD_PW=$OLD" >> "$GITHUB_ENV"
- echo "STUDIO_EXTRA_NEW_PW=$NEW" >> "$GITHUB_ENV"
-
- - name: Drive Compare/Recipes/Export/Studio/Settings with Playwright
- env:
- BASE_URL: http://127.0.0.1:18897
- STUDIO_OLD_PW: ${{ env.STUDIO_EXTRA_OLD_PW }}
- STUDIO_NEW_PW: ${{ env.STUDIO_EXTRA_NEW_PW }}
- PW_ART_DIR: logs/playwright_extra
- STUDIO_UI_STRICT: '1'
- # See "Drive the chat UI" step.
- STUDIO_UI_TURN_TIMEOUT_MS: '540000'
- GGUF_REPO: ${{ env.GGUF_REPO }}
- GGUF_VARIANT: ${{ env.GGUF_VARIANT }}
- # Same flake-retry shape as "Drive the chat UI with Playwright"
- # -- catches pipeTransport JSON crash and ERR_NO_BUFFER_SPACE.
- run: |
- mkdir -p logs/playwright_extra
- attempt=1
- max_attempts=3
- while : ; do
- set +e
- python tests/studio/playwright_extra_ui.py 2>&1 | tee logs/playwright_extra_attempt_${attempt}.log
- rc=${PIPESTATUS[0]}
- set -e
- if [ "$rc" -eq 0 ]; then
- break
- fi
- if { grep -q "Unexpected end of JSON input" logs/playwright_extra_attempt_${attempt}.log \
- || grep -q "ERR_NO_BUFFER_SPACE" logs/playwright_extra_attempt_${attempt}.log; } \
- && [ "$attempt" -lt "$max_attempts" ]; then
- echo "::warning::Playwright flake on attempt ${attempt}; resetting Studio and retrying..."
- kill "${STUDIO_EXTRA_PID}" 2>/dev/null || true
- sleep 2
- unsloth studio reset-password
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18897 \
- > "logs/studio_extra_retry_${attempt}.log" 2>&1 &
- STUDIO_EXTRA_PID=$!
- echo "STUDIO_EXTRA_PID=$STUDIO_EXTRA_PID" >> "$GITHUB_ENV"
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:18897/api/health" > /tmp/health2.json \
- && jq -e '.status == "healthy"' /tmp/health2.json >/dev/null; then
- break
- fi
- sleep 1
- done
- STUDIO_OLD_PW=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- STUDIO_NEW_PW="CIUiExtra-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- echo "::add-mask::$STUDIO_OLD_PW"
- echo "::add-mask::$STUDIO_NEW_PW"
- export STUDIO_OLD_PW STUDIO_NEW_PW
- attempt=$((attempt + 1))
- sleep 3
- continue
- fi
- exit "$rc"
- done
-
- - name: Stop second Studio
- if: always()
- run: |
- kill "${STUDIO_EXTRA_PID}" 2>/dev/null || true
- sleep 2
-
- - name: Upload Playwright artifacts
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: mac-studio-ui-smoke-artifacts
- path: |
- logs/studio.log
- logs/studio_extra.log
- logs/install.log
- logs/playwright
- logs/playwright_extra
- retention-days: 7
diff --git a/.github/workflows/studio-mac-update-smoke.yml b/.github/workflows/studio-mac-update-smoke.yml
deleted file mode 100644
index b65439f174..0000000000
--- a/.github/workflows/studio-mac-update-smoke.yml
+++ /dev/null
@@ -1,184 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# Mac counterpart to studio-update-smoke.yml. Verifies that on a real
-# Apple Silicon (macos-14, M1) runner:
-#
-# 1. install.sh --local --no-torch installs Studio AND auto-fetches
-# the prebuilt llama.cpp Mac binary (llama-bNNNN-bin-macos-arm64
-# from ggml-org/llama.cpp). Hitting the source-build fallback is
-# treated as an Unsloth bug -- Studio must always pick the
-# prebuilt on Mac.
-# 2. unsloth studio update --local is idempotent. Two consecutive
-# runs both report "prebuilt up to date and validated", no
-# source-build fallback.
-# 3. The installed Studio still boots and /api/health returns
-# healthy after the update path.
-
-name: Mac Studio Update CI
-
-on:
- pull_request:
- paths:
- - 'install.sh'
- - 'scripts/uninstall.sh'
- - 'studio/setup.sh'
- - 'studio/install_python_stack.py'
- - 'studio/install_llama_prebuilt.py'
- - 'studio/backend/requirements/**'
- - 'unsloth_cli/commands/studio.py'
- - 'pyproject.toml'
- - '.github/workflows/studio-mac-update-smoke.yml'
- push:
- branches: [main, pip]
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- update-idempotency:
- name: Studio Updating Tests
- runs-on: macos-14
- timeout-minutes: 30
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- - name: Install Studio (--local, --no-torch)
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- mkdir -p logs
- set -o pipefail
- bash install.sh --local --no-torch 2>&1 | tee logs/install.log
-
- - name: Assert install.sh used the Mac llama.cpp prebuilt
- run: |
- # Mac install must take the prebuilt path. Source-build
- # fallback here is an Unsloth bug.
- if grep -q "falling back to source build" logs/install.log; then
- echo "::error::install.sh fell back to source-build llama.cpp on Mac. Studio must install the prebuilt llama-bNNNN-bin-macos-arm64 on Apple Silicon."
- grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
- exit 1
- fi
- if ! grep -qE "prebuilt installed and validated|prebuilt up to date and validated|bin-macos-arm64" logs/install.log; then
- echo "::error::no Mac prebuilt llama.cpp marker in install.log."
- grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
- exit 1
- fi
- echo "install.sh installed the Mac prebuilt llama.cpp"
-
- - name: First update should be a no-op (prebuilt already validated)
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- set -o pipefail
- unsloth studio update --local 2>&1 | tee logs/update.log
- if grep -q "falling back to source build" logs/update.log; then
- echo "::error::studio update fell back to source-build llama.cpp on Mac."
- grep -E "llama-prebuilt|llama.cpp" logs/update.log | tail -60
- exit 1
- fi
- if ! grep -qE "prebuilt up to date and validated|prebuilt installed and validated" logs/update.log; then
- echo "::error::no prebuilt up-to-date marker in update.log."
- grep -E "llama-prebuilt|llama.cpp" logs/update.log | tail -60
- exit 1
- fi
- echo "update path took the prebuilt fast path"
-
- - name: Second update must also be a no-op
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- set -o pipefail
- unsloth studio update --local 2>&1 | tee logs/update2.log
- grep -q "falling back to source build" logs/update2.log && {
- echo "::error::second update fell back to source build on Mac"
- tail -60 logs/update2.log; exit 1; } || true
- grep -qE "prebuilt up to date and validated|prebuilt installed and validated" logs/update2.log
- echo "second update was clean"
-
- - name: Boot Studio briefly to confirm the install is still usable
- run: |
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18891 \
- > logs/studio.log 2>&1 &
- PID=$!
- HEALTHY=""
- for i in $(seq 1 60); do
- if curl -fs http://127.0.0.1:18891/api/health > /tmp/health.json; then
- if python3 -c "import json,sys; d=json.load(open('/tmp/health.json')); sys.exit(0 if d.get('status')=='healthy' else 1)"; then
- HEALTHY=1
- break
- fi
- fi
- sleep 1
- done
- if [ -z "$HEALTHY" ]; then
- echo "Studio failed to come up after \`update\`"
- tail -200 logs/studio.log
- kill "$PID" 2>/dev/null || true
- exit 1
- fi
- kill "$PID" 2>/dev/null || true
- echo "post-update Studio /api/health OK"
-
- - name: Uninstall and verify clean
- # Round-trip through scripts/uninstall.sh on real macOS. As a side
- # effect this exercises the macOS-only .app bundle + Launch Services
- # removal path (~/Applications/Unsloth Studio.app, lsregister -u)
- # which is not testable from a Linux runner. Skips gracefully if
- # scripts/uninstall.sh has not landed yet (lets this workflow merge
- # before #5497).
- run: |
- set -o pipefail
- if [ ! -f scripts/uninstall.sh ]; then
- echo "scripts/uninstall.sh not present in this tree; skipping round-trip"
- : > logs/uninstall.log
- exit 0
- fi
- sh scripts/uninstall.sh 2>&1 | tee logs/uninstall.log
- leak=0
- for p in \
- "$HOME/.unsloth/studio" \
- "$HOME/.local/share/unsloth" \
- "$HOME/Applications/Unsloth Studio.app" \
- "$HOME/Desktop/Unsloth Studio.app" \
- "$HOME/.local/bin/unsloth"; do
- if [ -e "$p" ] || [ -L "$p" ]; then
- echo "::error::leak: $p"
- leak=$((leak + 1))
- fi
- done
- [ "$leak" -eq 0 ] || exit 1
- sh scripts/uninstall.sh 2>&1 | tail -5
- sh scripts/uninstall.sh 2>&1 | tail -5
- echo "PASS: mac install -> update -> uninstall round-trip clean"
-
- - name: Upload update logs
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: mac-studio-update-log
- path: |
- logs/install.log
- logs/update.log
- logs/update2.log
- logs/studio.log
- logs/uninstall.log
- retention-days: 7
diff --git a/.github/workflows/studio-tauri-smoke.yml b/.github/workflows/studio-tauri-smoke.yml
deleted file mode 100644
index 1156c264ae..0000000000
--- a/.github/workflows/studio-tauri-smoke.yml
+++ /dev/null
@@ -1,128 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# PR-time smoke for the Tauri desktop wrapper. Builds the frontend and the
-# Tauri Linux debug binary, with no codesigning. Catches:
-# - tauri.conf.json drift
-# - src-tauri Cargo.toml or rust source breakage
-# - Tauri CLI version drift (we pin 2.10.1, matching release-desktop.yml)
-# - frontend output not picked up by Tauri's distDir
-#
-# Linux-only on a free `ubuntu-latest` runner. Mac and Windows desktop builds
-# stay in release-desktop.yml (manual `workflow_dispatch`) because they need
-# code-signing secrets and ~30 min of runner time each.
-
-name: Studio Tauri CI
-
-on:
- pull_request:
- paths:
- - 'studio/frontend/**'
- - 'studio/src-tauri/**'
- # CLI rename / signature change can break Tauri's spawned
- # `unsloth studio` -- include unsloth_cli in the trigger set.
- - 'unsloth_cli/**'
- - '.github/workflows/studio-tauri-smoke.yml'
- push:
- branches: [main, pip]
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- linux-debug-build:
- name: Tauri Linux debug build (no codesign)
- runs-on: ubuntu-22.04
- timeout-minutes: 25
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - name: Linux native deps for Tauri / WebKit2GTK
- run: |
- sudo apt-get update
- sudo apt-get install -y \
- libwebkit2gtk-4.1-dev libayatana-appindicator3-dev \
- librsvg2-dev libxdo-dev libssl-dev patchelf
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '24'
-
- - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @ 2026-03-27
-
- - uses: swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2.9.1
- with:
- workspaces: studio/src-tauri -> target
-
- - name: Install pinned Tauri CLI (matches release-desktop.yml)
- # Lifecycle scripts (esbuild native-binary postinstall, etc.) are
- # required for `vite build`. The pre-install lockfile structural
- # audit (lockfile_supply_chain_audit.py) is the practical defence
- # against the npm postinstall-dropper class -- it fires BEFORE any
- # tarball runs, on the injection pattern itself rather than an
- # advisory-DB lookup.
- run: npm install --save-dev --prefix studio @tauri-apps/cli@2.10.1 --no-fund --no-audit
-
- - name: Verify pinned Tauri CLI version
- run: |
- out="$(npx --prefix studio tauri --version)"
- echo "$out"
- [ "$out" = "tauri-cli 2.10.1" ] || { echo "::error::expected tauri-cli 2.10.1, got $out"; exit 1; }
-
- - name: Lockfile supply-chain audit (pre-install scan)
- run: python3 scripts/lockfile_supply_chain_audit.py
-
- - name: Frontend build (npm ci, vite)
- working-directory: studio/frontend
- # Lifecycle scripts (esbuild native-binary postinstall, etc.) are
- # required for `vite build`. The pre-install lockfile structural
- # audit (lockfile_supply_chain_audit.py) is the practical defence
- # against the npm postinstall-dropper class -- it fires BEFORE any
- # tarball runs, on the injection pattern itself rather than an
- # advisory-DB lookup.
- run: |
- npm ci --no-fund --no-audit
- npm run build
- test -f dist/index.html
-
- - name: Tauri debug build (Linux, no bundle, no codesign)
- # `--debug` + `--no-bundle` keeps this lean: compiles the Rust crate,
- # confirms the frontend dist is wired into Tauri, but skips the AppImage
- # / .deb production. Code signing is irrelevant because we never produce
- # a distributable artifact.
- env:
- TAURI_SIGNING_PRIVATE_KEY: ''
- TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ''
- run: npx --prefix studio tauri build --debug --no-bundle
-
- - name: Inspect produced binary
- run: |
- BIN=$(find studio/src-tauri/target/debug -maxdepth 1 -type f -executable 2>/dev/null \
- | grep -Ev '\.(d|so|dylib|dll)$' \
- | grep -Ev '/(deps|build|examples)$' \
- | head -1)
- echo "binary: $BIN"
- if [ -z "$BIN" ]; then
- echo "::error::Tauri debug binary not produced"
- ls -la studio/src-tauri/target/debug/ || true
- exit 1
- fi
- file "$BIN"
- du -h "$BIN"
-
- - name: Upload Tauri debug build
- # Always upload so a green run leaves the binary inspectable too.
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: tauri-debug-build
- path: |
- studio/src-tauri/target/debug
- studio/frontend/dist
- retention-days: 3
diff --git a/.github/workflows/studio-ui-smoke.yml b/.github/workflows/studio-ui-smoke.yml
deleted file mode 100644
index 455fe4b7e1..0000000000
--- a/.github/workflows/studio-ui-smoke.yml
+++ /dev/null
@@ -1,293 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# End-to-end Studio chat UI smoke via Playwright + Chromium against a
-# headless Linux runner. Boots Studio with the smallest GGUF
-# (gemma-3-270m-it UD-Q4_K_XL, ~254 MiB), drives the actual frontend
-# bundle, and asserts the full bootstrap-password / change-password /
-# send-message / persist-on-reload journey works end to end.
-#
-# This is the only workflow that catches regressions in the wiring
-# between the React frontend and the FastAPI backend, e.g. assistant-ui
-# version drift, /api/auth response shape changes, runtime-provider
-# regressions, or chat-history persistence breaking. Backend-only and
-# frontend-only CI happily pass while the actual user-visible UI is
-# broken (cf. the 2026.5.1 chat-history release).
-
-name: Studio UI CI
-
-on:
- pull_request:
- paths:
- - 'studio/**'
- - 'unsloth/**'
- - 'unsloth_cli/**'
- - 'install.sh'
- - 'pyproject.toml'
- # The Playwright test files themselves -- a PR that ONLY edits
- # the test must still trigger UI CI.
- - 'tests/studio/**'
- - '.github/workflows/studio-ui-smoke.yml'
- push:
- branches: [main, pip]
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- ui-smoke:
- name: Chat UI Tests
- runs-on: ubuntu-latest
- timeout-minutes: 25
- env:
- GGUF_REPO: unsloth/gemma-3-270m-it-GGUF
- GGUF_VARIANT: UD-Q4_K_XL
- GGUF_FILE: gemma-3-270m-it-UD-Q4_K_XL.gguf
- STUDIO_PORT: '18892'
- HF_HOME: ${{ github.workspace }}/hf-cache
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - name: Linux deps
- run: |
- sudo apt-get update
- sudo apt-get install -y --no-install-recommends \
- libcurl4-openssl-dev libssl-dev jq
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
-
- - name: Restore HF_HOME for ${{ env.GGUF_REPO }}
- id: cache-hf
- uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- continue-on-error: true
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
-
- - name: Prime HF_HOME with the GGUF
- id: prime-hf
- if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success'
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- run: |
- python -m pip install --upgrade huggingface_hub
- mkdir -p hf-cache
- bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE"
-
- - name: Save HF_HOME for ${{ env.GGUF_REPO }}
- if: always() && steps.prime-hf.outcome == 'success'
- uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
-
- - name: Install Studio (--local, --no-torch)
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- mkdir -p logs
- set -o pipefail
- bash install.sh --local --no-torch 2>&1 | tee logs/install.log
-
- - name: Install Playwright + Chromium
- run: |
- pip install 'playwright>=1.45'
- # --with-deps installs the OS-level runtime libs Chromium
- # needs (libnss3, libxkbcommon, etc.). About 30 s on a
- # warm runner.
- python -m playwright install --with-deps chromium
-
- - name: Reset auth + boot Studio
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \
- > logs/studio.log 2>&1 &
- echo "STUDIO_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health
- # 180 s -- a cold runner with venv warm-up + lazy imports has
- # been seen to exceed 60 s. Failing the wait is more expensive
- # than waiting an extra two minutes.
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then
- jq -e '.status == "healthy"' /tmp/health.json && break
- fi
- sleep 1
- done
- jq -e '.status == "healthy"' /tmp/health.json
-
- - name: Pass bootstrap password to the Playwright step
- # The Playwright test does its OWN /change-password through the
- # UI (Setup your account / Choose a new password), then loads
- # the model via page.evaluate against /api/inference/load with
- # the JWT it got from change-password. So the only thing we
- # have to hand it is the bootstrap password (so it can verify
- # post-rotation that the OLD bootstrap pw now returns 401).
- #
- # NEW + NEW2 are generated freshly per CI run via secrets.token_urlsafe
- # rather than hardcoded. If a workflow gets compromised, the
- # attacker can't replay a known-good rotated password against
- # any future / parallel Studio install -- the rotated value
- # only ever exists for the lifetime of this single job, masked
- # in the log via ::add-mask::.
- run: |
- OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- NEW="CIUi-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- NEW2="CIUi-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- echo "::add-mask::$OLD"
- echo "::add-mask::$NEW"
- echo "::add-mask::$NEW2"
- echo "STUDIO_OLD_PW=$OLD" >> "$GITHUB_ENV"
- echo "STUDIO_NEW_PW=$NEW" >> "$GITHUB_ENV"
- echo "STUDIO_NEW2_PW=$NEW2" >> "$GITHUB_ENV"
-
- - name: Drive the chat UI with Playwright
- env:
- BASE_URL: http://127.0.0.1:18892
- # The test file lives in the repo so it can be run locally
- # against a freshly-installed Studio (BASE_URL=...; STUDIO_OLD_PW=
- # $(cat ~/.unsloth/studio/auth/.bootstrap_password); python ...).
- PW_ART_DIR: logs/playwright
- # Strict mode: in CI a missing button / nav / dialog must
- # FAIL the test. Locally the test still runs against partial
- # Studio installs without STUDIO_UI_STRICT.
- STUDIO_UI_STRICT: '1'
- run: |
- mkdir -p logs/playwright
- python tests/studio/playwright_chat_ui.py
-
- - name: Stop Studio (chat-ui ends with Shutdown click; this is belt-and-suspenders)
- if: always()
- run: |
- kill "${STUDIO_PID}" 2>/dev/null || true
- sleep 2
-
- # The chat UI test ends by clicking the Shutdown menuitem, which
- # leaves the server dead. The extra UI test (Compare / Recipes /
- # Export / Studio / Settings) needs a fresh Studio, so we boot a
- # second one on a different port. Boot is fast (~3-5s on the
- # warm install we already did) so this adds little wall time.
- - name: Reset auth + boot Studio for extra UI tests (port 18894)
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18894 \
- > logs/studio_extra.log 2>&1 &
- echo "STUDIO_EXTRA_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health on 18894
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:18894/api/health" > /tmp/health2.json; then
- jq -e '.status == "healthy"' /tmp/health2.json && break
- fi
- sleep 1
- done
- jq -e '.status == "healthy"' /tmp/health2.json
-
- - name: Pass bootstrap pw for extra UI test
- run: |
- OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- NEW="CIUiExtra-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- echo "::add-mask::$OLD"
- echo "::add-mask::$NEW"
- echo "STUDIO_EXTRA_OLD_PW=$OLD" >> "$GITHUB_ENV"
- echo "STUDIO_EXTRA_NEW_PW=$NEW" >> "$GITHUB_ENV"
-
- - name: Drive Compare/Recipes/Export/Studio/Settings with Playwright
- env:
- BASE_URL: http://127.0.0.1:18894
- STUDIO_OLD_PW: ${{ env.STUDIO_EXTRA_OLD_PW }}
- STUDIO_NEW_PW: ${{ env.STUDIO_EXTRA_NEW_PW }}
- PW_ART_DIR: logs/playwright_extra
- STUDIO_UI_STRICT: '1'
- GGUF_REPO: ${{ env.GGUF_REPO }}
- GGUF_VARIANT: ${{ env.GGUF_VARIANT }}
- run: |
- mkdir -p logs/playwright_extra
- python tests/studio/playwright_extra_ui.py
-
- - name: Stop second Studio
- if: always()
- run: |
- kill "${STUDIO_EXTRA_PID}" 2>/dev/null || true
- sleep 2
-
- # IME + multilingual paste regression (issue #5318 / PR #5327).
- # Third Studio on its own port so a hang here cannot poison the
- # earlier UI tests. No GGUF -- the bug surface is the composer.
- - name: Reset auth + boot Studio for IME / i18n tests (port 18896)
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18896 \
- > logs/studio_ime.log 2>&1 &
- echo "STUDIO_IME_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health on 18896
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:18896/api/health" > /tmp/health3.json; then
- jq -e '.status == "healthy"' /tmp/health3.json && break
- fi
- sleep 1
- done
- jq -e '.status == "healthy"' /tmp/health3.json
-
- - name: Pass bootstrap pw for IME / i18n test
- # IME smoke does the change-password against the bootstrap that
- # Studio's frontend injects into the page, so it only needs the
- # NEW password.
- run: |
- NEW="CIIme-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- echo "::add-mask::$NEW"
- echo "STUDIO_IME_NEW_PW=$NEW" >> "$GITHUB_ENV"
-
- - name: Drive IME + multilingual paste regression with Playwright
- env:
- BASE_URL: http://127.0.0.1:18896
- STUDIO_NEW_PW: ${{ env.STUDIO_IME_NEW_PW }}
- PW_ART_DIR: logs/playwright_ime
- STUDIO_UI_STRICT: '1'
- run: |
- mkdir -p logs/playwright_ime
- python tests/studio/playwright_chat_ime_i18n.py
-
- - name: Stop third Studio
- if: always()
- run: |
- kill "${STUDIO_IME_PID}" 2>/dev/null || true
- sleep 2
-
- - name: Upload Playwright artifacts
- # Always upload so a green run's screenshots stay reviewable --
- # catches "passed but the UI is silently broken" regressions.
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: studio-ui-smoke-artifacts
- path: |
- logs/studio.log
- logs/studio_extra.log
- logs/studio_ime.log
- logs/install.log
- logs/playwright
- logs/playwright_extra
- logs/playwright_ime
- retention-days: 7
diff --git a/.github/workflows/studio-update-smoke.yml b/.github/workflows/studio-update-smoke.yml
deleted file mode 100644
index 057aeacbd4..0000000000
--- a/.github/workflows/studio-update-smoke.yml
+++ /dev/null
@@ -1,191 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# Verifies that `unsloth studio update --local` is idempotent: a fresh
-# install via install.sh, followed by `unsloth studio update --local`,
-# succeeds and is a no-op for the llama.cpp prebuilt (it should report
-# "prebuilt up to date and validated", not re-run the source build).
-#
-# This catches regressions in setup.sh's update path that the existing
-# GGUF / wheel jobs would miss because they only invoke install.sh once.
-
-name: Studio Update CI
-
-on:
- pull_request:
- paths:
- - 'install.sh'
- - 'scripts/uninstall.sh'
- - 'studio/setup.sh'
- - 'studio/install_python_stack.py'
- - 'studio/install_llama_prebuilt.py'
- - 'studio/backend/requirements/**'
- - 'unsloth_cli/commands/studio.py'
- - 'pyproject.toml'
- - '.github/workflows/studio-update-smoke.yml'
- push:
- branches: [main, pip]
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- update-idempotency:
- name: Studio Updating Tests
- runs-on: ubuntu-latest
- timeout-minutes: 15
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - name: Linux deps for llama.cpp prebuilt
- run: |
- sudo apt-get update
- sudo apt-get install -y --no-install-recommends \
- libcurl4-openssl-dev libssl-dev jq
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- # Don't cache pip: this job runs `bash install.sh` and
- # `unsloth studio update --local` which both go through
- # `uv` and never populate ~/.cache/pip. setup-python's
- # post-step then fatal-errors with "Cache folder path is
- # retrieved for pip but doesn't exist on disk".
-
- - name: Install Studio (--local, --no-torch)
- # Pass the workflow token so the llama.cpp prebuilt installer's
- # GitHub-API call to list releases isn't rate-limited (60/hr
- # unauthenticated). Without this, three consecutive install +
- # update + update calls in this job exceed the limit and the
- # prebuilt path falls back to source build.
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- mkdir -p logs
- set -o pipefail
- bash install.sh --local --no-torch 2>&1 | tee logs/install.log
-
- - name: First update should be a no-op (prebuilt already validated)
- # `unsloth studio update --local` runs studio/setup.sh against
- # the local repo. Right after install.sh the llama.cpp prebuilt
- # has just been installed and validated, so the second run must
- # take the "prebuilt up to date and validated" code path. Any
- # source-build fallback or re-download here means setup.sh's
- # idempotency regressed.
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- set -o pipefail
- unsloth studio update --local 2>&1 | tee logs/update.log
- if grep -q "falling back to source build" logs/update.log; then
- echo "::error::studio update fell back to source-build llama.cpp on a fresh install. setup.sh idempotency regressed."
- grep -E "llama-prebuilt|llama.cpp" logs/update.log | tail -60
- exit 1
- fi
- if ! grep -qE "prebuilt up to date and validated|prebuilt installed and validated" logs/update.log; then
- echo "::error::no prebuilt up-to-date marker in update.log. Did setup.sh skip the prebuilt path on update?"
- grep -E "llama-prebuilt|llama.cpp" logs/update.log | tail -60
- exit 1
- fi
- echo "update path took the prebuilt fast path"
-
- - name: Second update must also be a no-op
- # Two consecutive `update`s back-to-back is the usual desktop
- # flow (auto-update, then user-triggered update). Asserting the
- # second run is also clean rules out hidden state changes from
- # the first one.
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- set -o pipefail
- unsloth studio update --local 2>&1 | tee logs/update2.log
- grep -q "falling back to source build" logs/update2.log && {
- echo "::error::second update fell back to source build"
- tail -60 logs/update2.log; exit 1; } || true
- grep -qE "prebuilt up to date and validated|prebuilt installed and validated" logs/update2.log
- echo "second update was clean"
-
- - name: Boot Studio briefly to confirm the install is still usable
- # If `update --local` accidentally broke the venv or wiped the
- # llama-server binary, the server would fail to start here.
- run: |
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18891 \
- > logs/studio.log 2>&1 &
- PID=$!
- for i in $(seq 1 60); do
- if curl -fs http://127.0.0.1:18891/api/health > /tmp/health.json; then
- jq -e '.status == "healthy"' /tmp/health.json
- break
- fi
- sleep 1
- done
- if ! jq -e '.status == "healthy"' /tmp/health.json 2>/dev/null; then
- echo "Studio failed to come up after `update`"
- tail -200 logs/studio.log
- kill "$PID" 2>/dev/null || true
- exit 1
- fi
- kill "$PID" 2>/dev/null || true
- echo "post-update Studio /api/health OK"
-
- - name: Uninstall and verify clean
- # Round-trip the installer through scripts/uninstall.sh: confirms the
- # uninstaller actually finds and removes everything install.sh +
- # update wrote. Safety-guard scenarios (refuse-$HOME etc.) belong
- # in a separate fast smoke job; this is the happy-path cleanup
- # assertion that catches regressions where install.sh starts
- # writing to a new location and scripts/uninstall.sh hasn't caught up.
- # Skips gracefully if scripts/uninstall.sh has not landed yet (lets
- # this workflow merge before #5497).
- run: |
- set -o pipefail
- if [ ! -f scripts/uninstall.sh ]; then
- echo "scripts/uninstall.sh not present in this tree; skipping round-trip"
- : > logs/uninstall.log
- exit 0
- fi
- sh scripts/uninstall.sh 2>&1 | tee logs/uninstall.log
- leak=0
- for p in \
- "$HOME/.unsloth/studio" \
- "$HOME/.local/share/unsloth" \
- "$HOME/Desktop/Unsloth Studio.desktop" \
- "$HOME/.local/bin/unsloth"; do
- if [ -e "$p" ] || [ -L "$p" ]; then
- echo "::error::leak: $p"
- ls -la "$p" 2>&1 | head -3
- leak=$((leak + 1))
- fi
- done
- [ "$leak" -eq 0 ] || exit 1
- # Idempotent: re-runs exit 0 on an empty $HOME.
- sh scripts/uninstall.sh 2>&1 | tail -5
- sh scripts/uninstall.sh 2>&1 | tail -5
- echo "PASS: install -> update -> uninstall round-trip clean"
-
- - name: Upload update logs
- # Always upload so a green run still leaves the install + two
- # update logs + uninstall log reviewable.
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: studio-update-log
- path: |
- logs/install.log
- logs/update.log
- logs/update2.log
- logs/studio.log
- logs/uninstall.log
- retention-days: 7
diff --git a/.github/workflows/studio-windows-api-smoke.yml b/.github/workflows/studio-windows-api-smoke.yml
deleted file mode 100644
index 1d12ea6f90..0000000000
--- a/.github/workflows/studio-windows-api-smoke.yml
+++ /dev/null
@@ -1,246 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# Windows counterpart to studio-api-smoke.yml / studio-mac-api-smoke.yml.
-# Same tests/studio/studio_api_smoke.py exercise (CORS hardening, auth
-# state machine, JWT expiry, API key lifecycle, /v1/models /
-# /v1/embeddings / /v1/responses, endpoint-by-endpoint auth audit) but
-# on the FREE windows-latest runner. The file-mode hardening section
-# (Section 6) is Linux-only and short-circuits on non-POSIX; the rest
-# is platform-portable.
-
-name: Windows Studio API CI
-
-on:
- pull_request:
- paths:
- - 'studio/**'
- - 'unsloth/**'
- - 'unsloth_cli/**'
- - 'install.ps1'
- - 'pyproject.toml'
- - 'tests/studio/**'
- - '.github/workflows/studio-windows-api-smoke.yml'
- push:
- branches: [main, pip]
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- api-smoke:
- name: Studio API & Auth Tests
- runs-on: windows-latest
- timeout-minutes: 30
- defaults:
- run:
- shell: bash
- env:
- GGUF_REPO: unsloth/gemma-3-270m-it-GGUF
- GGUF_VARIANT: UD-Q4_K_XL
- GGUF_FILE: gemma-3-270m-it-UD-Q4_K_XL.gguf
- STUDIO_PORT: '18895'
- HF_HOME: ${{ github.workspace }}/hf-cache
- # Force UTF-8 for stdio (Windows defaults to cp1252; hf
- # download prints a "✓" checkmark and crashes otherwise).
- PYTHONIOENCODING: utf-8
- PYTHONUTF8: '1'
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
-
- - name: Restore HF_HOME for ${{ env.GGUF_REPO }}
- id: cache-hf
- uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- continue-on-error: true
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
-
- - name: Prime HF_HOME with the GGUF
- id: prime-hf
- if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success'
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- run: |
- python -m pip install --upgrade huggingface_hub
- mkdir -p hf-cache
- bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE"
-
- - name: Save HF_HOME for ${{ env.GGUF_REPO }}
- if: always() && steps.prime-hf.outcome == 'success'
- uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
-
- - name: Pre-install Windows tweaks (npm 11 + Defender exclusions)
- shell: pwsh
- # See studio-windows-update-smoke.yml for the full rationale.
- # tl;dr: setup.ps1 needs npm >=11 to skip a 35 s winget Node
- # reinstall, and Defender's real-time scan dominates the
- # frontend / uv-pip-extract steps.
- run: |
- $ProgressPreference = 'SilentlyContinue'
- Write-Host "npm version before upgrade: $(npm -v)"
- npm install -g 'npm@^11' 2>&1 | Out-Host
- Write-Host "npm version after upgrade: $(npm -v)"
- # NOTE: do NOT pre-create these directories. See
- # studio-windows-update-smoke.yml for the full rationale --
- # creating an empty studio/frontend/dist trips setup.ps1's
- # mtime-based staleness check into "frontend up to date, skip
- # rebuild" and Studio boots with an empty dist directory.
- # Add-MpPreference accepts paths that do not yet exist.
- foreach ($p in @(
- "$env:USERPROFILE\.unsloth",
- "$env:USERPROFILE\AppData\Local\uv",
- "$env:GITHUB_WORKSPACE\studio\frontend\node_modules",
- "$env:GITHUB_WORKSPACE\studio\frontend\dist"
- )) {
- try {
- Add-MpPreference -ExclusionPath $p -ErrorAction Stop
- Write-Host "Defender exclusion added: $p"
- } catch {
- Write-Host "Defender exclusion skipped ($($_.Exception.Message)): $p"
- }
- }
-
- - name: Install Studio (--local, --no-torch)
- shell: pwsh
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- New-Item -ItemType Directory -Force -Path logs | Out-Null
- # *>&1 captures Write-Host (Information stream) output;
- # plain 2>&1 does not. setup.ps1 emits "prebuilt installed
- # and validated" via Write-Host, and we grep for that.
- $ProgressPreference = 'SilentlyContinue'
- & ./install.ps1 --local --no-torch *>&1 | Tee-Object -FilePath logs/install.log
-
- - name: Assert install.ps1 used the Windows llama.cpp prebuilt
- run: |
- # Filesystem-based check (setup.ps1's stream output isn't
- # captured back through this parent step's pipeline; see
- # studio-windows-ui-smoke.yml for full explanation).
- LLAMA_DIR=~/.unsloth/llama.cpp
- INFO="$LLAMA_DIR/UNSLOTH_PREBUILT_INFO.json"
- BIN="$LLAMA_DIR/build/bin/Release/llama-server.exe"
- if grep -q "falling back to source build" logs/install.log; then
- echo "::error::install.ps1 fell back to source-build llama.cpp on Windows."
- grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
- exit 1
- fi
- if [ ! -f "$INFO" ]; then
- echo "::error::no UNSLOTH_PREBUILT_INFO.json at $INFO."
- ls -la "$LLAMA_DIR" || true
- exit 1
- fi
- if [ ! -f "$BIN" ]; then
- echo "::error::no llama-server.exe at $BIN."
- ls -la "$LLAMA_DIR/build/bin" || true
- exit 1
- fi
- echo "install.ps1 installed the Windows prebuilt llama.cpp:"
- cat "$INFO"
-
- - name: Add Studio shim to GITHUB_PATH
- # install.ps1's User-PATH update doesn't propagate to a
- # running Git Bash session; export the shim dir so the
- # next `unsloth ...` invocation finds it.
- run: |
- SHIM_DIR=~/.unsloth/studio/bin
- if [ ! -f "$SHIM_DIR/unsloth.exe" ]; then
- echo "::error::unsloth.exe shim not found at $SHIM_DIR"
- ls -la ~/.unsloth/studio/ || true
- exit 1
- fi
- cygpath -w "$SHIM_DIR" >> "$GITHUB_PATH"
-
- - name: Patch Studio venv with full typer / pydantic dep trees
- # Belt-and-suspenders: install.ps1's --no-deps install of
- # no-torch-runtime.txt drops typer's and pydantic's runtime
- # deps unless explicitly pinned. Re-install the ones whose
- # deps don't pull torch.
- run: |
- STUDIO_PY=~/.unsloth/studio/unsloth_studio/Scripts/python.exe
- if [ ! -f "$STUDIO_PY" ]; then
- echo "::error::Studio venv python not at $STUDIO_PY"
- ls -la ~/.unsloth/studio/ || true
- exit 1
- fi
- "$STUDIO_PY" -m pip install --upgrade typer pydantic huggingface_hub
-
- - name: Install pyjwt for the JWT-expiry forge test
- run: python -m pip install 'pyjwt>=2.6'
-
- - name: Reset auth + boot Studio (API-only)
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \
- > logs/studio.log 2>&1 &
- echo "STUDIO_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then
- jq -e '.status == "healthy"' /tmp/health.json && break
- fi
- sleep 1
- done
- jq -e '.status == "healthy"' /tmp/health.json
-
- - name: Pass bootstrap password + rotated targets to the test
- run: |
- OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- NEW="ApiSmoke-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- NEW2="ApiSmoke-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- echo "::add-mask::$OLD"
- echo "::add-mask::$NEW"
- echo "::add-mask::$NEW2"
- echo "STUDIO_OLD_PW=$OLD" >> "$GITHUB_ENV"
- echo "STUDIO_NEW_PW=$NEW" >> "$GITHUB_ENV"
- echo "STUDIO_NEW2_PW=$NEW2" >> "$GITHUB_ENV"
-
- - name: Run Studio API & Auth tests
- # Do NOT pin STUDIO_AUTH_DIR here. The Mac/Linux mirrors
- # hardcode runner-specific paths (/Users/runner/...,
- # /home/runner/...), but on Windows the path is
- # C:\Users\runneradmin\.unsloth\studio\auth and varies by
- # runner image. studio_api_smoke.py defaults to
- # Path.home()/".unsloth"/"studio"/"auth" when the env is
- # unset, which is correct on every OS.
- env:
- BASE_URL: http://127.0.0.1:18895
- run: python tests/studio/studio_api_smoke.py
-
- - name: Stop Studio
- if: always()
- run: |
- kill "${STUDIO_PID}" 2>/dev/null || true
- sleep 2
-
- - name: Upload API smoke logs
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: windows-studio-api-smoke-log
- path: |
- logs/install.log
- logs/studio.log
- retention-days: 7
diff --git a/.github/workflows/studio-windows-inference-smoke.yml b/.github/workflows/studio-windows-inference-smoke.yml
deleted file mode 100644
index ad739dd529..0000000000
--- a/.github/workflows/studio-windows-inference-smoke.yml
+++ /dev/null
@@ -1,1251 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# Three end-to-end smoke jobs that boot a freshly-installed Studio and
-# exercise the surfaces real users hit through the OpenAI / Anthropic
-# SDKs and curl, on the FREE windows-latest runner. Each job picks the
-# smallest model that exercises the behaviour under test, primes
-# HF_HOME via actions/cache, and shares the install.ps1 --local
-# --no-torch bootstrap.
-#
-# 1. OpenAI, Anthropic API tests
-# gemma-3-270m-it UD-Q4_K_XL (~254 MiB).
-# 2. Tool calling Tests
-# Qwen3.5-2B UD-Q4_K_XL (~890 MiB).
-# 3. JSON, images
-# gemma-4-E2B-it UD-Q4_K_XL + mmproj-F16 (~3.4 GiB total).
-# Within the 14 GB windows-latest SSD budget.
-
-name: Windows Studio GGUF CI
-
-on:
- pull_request:
- paths:
- - 'studio/**'
- - 'unsloth/**'
- - 'unsloth_cli/**'
- - 'install.ps1'
- - 'pyproject.toml'
- - '.github/workflows/studio-windows-inference-smoke.yml'
- push:
- branches: [main, pip]
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- # ─────────────────────────────────────────────────────────────────────
- # Job 1: OpenAI, Anthropic API tests
- # ─────────────────────────────────────────────────────────────────────
- openai-anthropic:
- name: OpenAI, Anthropic API tests
- runs-on: windows-latest
- timeout-minutes: 30
- defaults:
- run:
- shell: bash
- env:
- GGUF_REPO: unsloth/gemma-3-270m-it-GGUF
- GGUF_VARIANT: UD-Q4_K_XL
- GGUF_FILE: gemma-3-270m-it-UD-Q4_K_XL.gguf
- STUDIO_PORT: '18888'
- HF_HOME: ${{ github.workspace }}/hf-cache
- # Force UTF-8 for stdio (Windows defaults to cp1252; hf
- # download / Studio CLI print "✓" checkmarks and crash
- # otherwise).
- PYTHONIOENCODING: utf-8
- PYTHONUTF8: '1'
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
-
- # Split restore + save (rather than the one-step actions/cache) so a
- # transient restore-side failure does not kill the whole job. v5 has a
- # known flake where it logs "Cache hit for: " and then exits
- # non-zero without actually extracting the archive (see
- # actions/cache#1621 and github community discussion #163260).
- # continue-on-error on restore masks that failure so the Prime step
- # below can re-download from HF and the job keeps running. Save then
- # populates the cache key on a real miss only; cache keys are
- # immutable, so a corrupted cached entry persists until the -v1
- # suffix below is bumped.
- - name: Restore HF_HOME cache for ${{ env.GGUF_REPO }}
- id: cache-hf
- uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- continue-on-error: true
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
-
- - name: Prime HF_HOME with the GGUF
- id: prime-hf
- # Run on a real cache miss AND on the silent-restore-failure mode
- # described above (outcome != success).
- if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success'
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- run: |
- python -m pip install --upgrade huggingface_hub
- mkdir -p hf-cache
- bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE"
-
- - name: Save HF_HOME cache for ${{ env.GGUF_REPO }}
- # Only write a fresh cache entry when we actually rebuilt the
- # directory (Prime ran and succeeded). Skipping when Prime is
- # skipped avoids "already exists" save warnings on the happy path.
- if: always() && steps.prime-hf.outcome == 'success'
- uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
-
- - name: Pre-install Windows tweaks (npm 11 + Defender exclusions)
- shell: pwsh
- # See studio-windows-update-smoke.yml for the full rationale.
- # tl;dr: setup.ps1 needs npm >=11 to skip a 35 s winget Node
- # reinstall, and Defender's real-time scan dominates the
- # frontend / uv-pip-extract steps.
- run: |
- $ProgressPreference = 'SilentlyContinue'
- Write-Host "npm version before upgrade: $(npm -v)"
- npm install -g 'npm@^11' 2>&1 | Out-Host
- Write-Host "npm version after upgrade: $(npm -v)"
- # NOTE: do NOT pre-create these directories. See
- # studio-windows-update-smoke.yml for the full rationale --
- # creating an empty studio/frontend/dist trips setup.ps1's
- # mtime-based staleness check into "frontend up to date, skip
- # rebuild" and Studio boots with an empty dist directory.
- # Add-MpPreference accepts paths that do not yet exist.
- foreach ($p in @(
- "$env:USERPROFILE\.unsloth",
- "$env:USERPROFILE\AppData\Local\uv",
- "$env:GITHUB_WORKSPACE\studio\frontend\node_modules",
- "$env:GITHUB_WORKSPACE\studio\frontend\dist"
- )) {
- try {
- Add-MpPreference -ExclusionPath $p -ErrorAction Stop
- Write-Host "Defender exclusion added: $p"
- } catch {
- Write-Host "Defender exclusion skipped ($($_.Exception.Message)): $p"
- }
- }
-
- - name: Install Studio (--local, --no-torch)
- shell: pwsh
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- New-Item -ItemType Directory -Force -Path logs | Out-Null
- # *>&1 captures Write-Host (Information stream) output;
- # plain 2>&1 does not. setup.ps1 emits "prebuilt installed
- # and validated" via Write-Host, and we grep for that.
- $ProgressPreference = 'SilentlyContinue'
- & ./install.ps1 --local --no-torch *>&1 | Tee-Object -FilePath logs/install.log
-
- - name: Assert install.ps1 used the Windows llama.cpp prebuilt
- run: |
- # Filesystem check; setup.ps1's stream output isn't captured.
- LLAMA_DIR=~/.unsloth/llama.cpp
- INFO="$LLAMA_DIR/UNSLOTH_PREBUILT_INFO.json"
- BIN="$LLAMA_DIR/build/bin/Release/llama-server.exe"
- if grep -q "falling back to source build" logs/install.log; then
- echo "::error::install.ps1 fell back to source-build llama.cpp on Windows."
- grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
- exit 1
- fi
- if [ ! -f "$INFO" ]; then
- echo "::error::no UNSLOTH_PREBUILT_INFO.json at $INFO."
- ls -la "$LLAMA_DIR" || true
- exit 1
- fi
- if [ ! -f "$BIN" ]; then
- echo "::error::no llama-server.exe at $BIN."
- ls -la "$LLAMA_DIR/build/bin" || true
- exit 1
- fi
- echo "install.ps1 installed the Windows prebuilt llama.cpp:"
- cat "$INFO"
-
- - name: Add Studio shim to GITHUB_PATH
- run: |
- SHIM_DIR=~/.unsloth/studio/bin
- if [ ! -f "$SHIM_DIR/unsloth.exe" ]; then
- echo "::error::unsloth.exe shim not found at $SHIM_DIR"
- ls -la ~/.unsloth/studio/ || true
- exit 1
- fi
- cygpath -w "$SHIM_DIR" >> "$GITHUB_PATH"
-
- - name: Patch Studio venv with full typer / pydantic dep trees
- # Belt-and-suspenders: install.ps1's --no-deps install of
- # no-torch-runtime.txt drops typer's and pydantic's runtime
- # deps unless explicitly pinned. Re-install the ones whose
- # deps don't pull torch.
- run: |
- STUDIO_PY=~/.unsloth/studio/unsloth_studio/Scripts/python.exe
- if [ ! -f "$STUDIO_PY" ]; then
- echo "::error::Studio venv python not at $STUDIO_PY"
- ls -la ~/.unsloth/studio/ || true
- exit 1
- fi
- "$STUDIO_PY" -m pip install --upgrade typer pydantic huggingface_hub
-
- - name: Install OpenAI + Anthropic Python SDKs
- run: python -m pip install 'openai>=1.50' 'anthropic>=0.40'
-
- - name: Reset auth + boot Studio (API-only)
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \
- > logs/studio.log 2>&1 &
- echo "STUDIO_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then
- jq -e '.status == "healthy"' /tmp/health.json
- exit 0
- fi
- sleep 1
- done
- echo "Studio did not become healthy in 180s"
- tail -200 logs/studio.log
- exit 1
-
- - name: Password rotation (old must fail, new must work)
- run: |
- OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- NEW="CIRotated-$(python -c 'import secrets; print(secrets.token_urlsafe(12))')"
- echo "::add-mask::$OLD"
- echo "::add-mask::$NEW"
- OLD_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}" | jq -r .access_token)
- [ -n "$OLD_TOKEN" ] && [ "$OLD_TOKEN" != "null" ] || { echo "bootstrap login failed"; exit 1; }
- curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/change-password" \
- -H "Authorization: Bearer $OLD_TOKEN" -H 'content-type: application/json' \
- -d "{\"current_password\":\"$OLD\",\"new_password\":\"$NEW\"}" > /dev/null
- OLD_STATUS=$(curl -s -o /dev/null -w '%{http_code}' \
- -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}")
- if [ "$OLD_STATUS" != "401" ]; then
- echo "::error::Login with old password returned $OLD_STATUS, expected 401"
- exit 1
- fi
- NEW_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$NEW\"}" | jq -r .access_token)
- [ -n "$NEW_TOKEN" ] && [ "$NEW_TOKEN" != "null" ] || { echo "new login failed"; exit 1; }
- echo "TOKEN=$NEW_TOKEN" >> "$GITHUB_ENV"
- echo "password rotation OK (old=401, new=200)"
-
- - name: Load the GGUF (HF repo + variant, served from HF_HOME cache)
- run: |
- # Retry the load step a few times so a transient TCP RST during
- # llama-server warm-up (Windows runner image churn,
- # windows-latest -> windows-2025-vs2026 rollout) doesn't fail
- # the whole job. The Studio backend's _wait_for_health now
- # catches httpx.ReadError too; this retry layer covers the
- # cases the backend can't recover from on its own.
- LOAD_OK=0
- for attempt in 1 2 3; do
- HTTP=$(curl -s -o /tmp/load.json -w '%{http_code}' \
- -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \
- -H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \
- --max-time 600 \
- -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048}")
- if [ "$HTTP" = "200" ]; then LOAD_OK=1; break; fi
- echo "::warning::/api/inference/load attempt $attempt returned $HTTP; response:"
- cat /tmp/load.json || true
- sleep 10
- done
- [ "$LOAD_OK" = "1" ] || { echo "::error::/api/inference/load failed 3 attempts"; exit 22; }
- jq '{status, display_name, is_gguf, context_length}' /tmp/load.json
-
- - name: Multi-turn determinism via OpenAI + Anthropic SDKs
- env:
- BASE_URL: http://127.0.0.1:18888
- run: |
- python - <<'PY'
- import json
- import os
- from openai import OpenAI
- from anthropic import Anthropic
-
- BASE = os.environ["BASE_URL"]
- KEY = os.environ["TOKEN"]
- SEED = 3407
-
- PROMPTS = [
- "What is 1+1?",
- "What did I ask before?",
- "What is the capital of France?",
- "Repeat the city name",
- ]
-
- def run_openai():
- client = OpenAI(base_url = f"{BASE}/v1", api_key = KEY)
- history, replies = [], []
- for prompt in PROMPTS:
- history.append({"role": "user", "content": prompt})
- resp = client.chat.completions.create(
- model = "default",
- messages = history,
- temperature = 0.0,
- max_tokens = 80,
- seed = SEED,
- extra_body = {"enable_thinking": False},
- )
- text = resp.choices[0].message.content or ""
- replies.append(text)
- history.append({"role": "assistant", "content": text})
- return replies
-
- def run_anthropic():
- client = Anthropic(
- base_url = BASE,
- api_key = "unused",
- default_headers = {"Authorization": f"Bearer {KEY}"},
- )
- history, replies = [], []
- for prompt in PROMPTS:
- history.append({"role": "user", "content": prompt})
- msg = client.messages.create(
- model = "default",
- max_tokens = 80,
- messages = history,
- temperature = 0.0,
- extra_body = {"seed": SEED, "enable_thinking": False},
- )
- text = "".join(b.text for b in msg.content if getattr(b, "type", None) == "text")
- replies.append(text)
- history.append({"role": "assistant", "content": text})
- return replies
-
- for label, runner in (("openai", run_openai), ("anthropic", run_anthropic)):
- first = runner()
- second = runner()
- for i, (a, b) in enumerate(zip(first, second), start = 1):
- print(f"[{label} turn {i}] {a!r}")
- assert a, f"{label}: empty turn {i} response"
- # Compare on stripped content: llama-server can vary
- # trailing whitespace (specifically a final '\n') between
- # otherwise-identical greedy runs depending on the
- # batch-flush boundary at which the stream is closed. The
- # generated tokens are identical; only the trailing
- # whitespace differs. Keep the raw repr in the failure
- # message so a real divergence is still legible.
- assert a.strip() == b.strip(), (
- f"{label} non-deterministic at turn {i} with temperature=0.0:\n"
- f" run1: {a!r}\n run2: {b!r}"
- )
- joined = " ".join(first).lower()
- assert "1" in first[0], f"{label}: turn-1 answer should contain '1', got {first[0]!r}"
- assert "paris" in joined, f"{label}: expected 'paris' somewhere in the four-turn transcript: {first}"
- print(f"[{label}] OK -- 4 turns, run1 == run2, history grounded")
- PY
-
- - name: Stop Studio
- if: always()
- # Run as cmd so we are not running through the Git Bash shell;
- # Git Bash on windows-latest has been observed to exit 143
- # (SIGTERM) from any inline kill/sleep block, masking a green
- # test run. The runner reclaims the Studio child process at
- # job end either way, so just emit a marker and exit 0.
- shell: cmd
- run: echo Stop Studio (no-op; runner reclaims STUDIO_PID=%STUDIO_PID% at job end)
-
- - name: Collect llama-server logs
- if: always()
- shell: bash
- # Copy llama-server's own stdout/stderr (teed by Studio under
- # ~/.unsloth/studio/logs/llama-server/) into the workspace so
- # upload-artifact can pick it up. Crucial for diagnosing a
- # subprocess crash where Studio's traceback only shows the
- # symptom (httpx ReadError) but not the cause.
- run: |
- mkdir -p logs/llama-server
- cp -v ~/.unsloth/studio/logs/llama-server/*.log logs/llama-server/ 2>/dev/null || \
- echo "no llama-server logs to collect"
-
- - name: Upload logs
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: windows-openai-anthropic-log
- path: |
- logs/studio.log
- logs/install.log
- logs/llama-server/*.log
- retention-days: 7
-
- # ─────────────────────────────────────────────────────────────────────
- # Job 2: Tool calling Tests
- # ─────────────────────────────────────────────────────────────────────
- tool-calling:
- name: Tool calling Tests
- runs-on: windows-latest
- timeout-minutes: 30
- defaults:
- run:
- shell: bash
- env:
- # Tool calling is the highest-volume GGUF in this workflow
- # (Qwen3.5-2B at Q4_K_XL = ~1.28 GiB). The previous HF_HOME
- # cache stored xet chunks + blobs + snapshots = ~4.7 GiB --
- # 3.7x file-size inflation, dominating the post-step upload
- # (211 s on first run; subsequent runs hit the cache, but the
- # one-time cost recurs every time the cache key bumps). Use
- # main's `--local-dir gguf-cache` pattern: cache the flat .gguf
- # only, pass an absolute path to Studio's /api/inference/load.
- # The OpenAI/Anth and JSON+images jobs still cover the
- # gguf_variant resolution path.
- GGUF_REPO: unsloth/Qwen3.5-2B-GGUF
- GGUF_FILE: Qwen3.5-2B-UD-Q4_K_XL.gguf
- STUDIO_PORT: '18898'
- # Force UTF-8 for stdio (Windows defaults to cp1252; hf
- # download / Studio CLI print "✓" checkmarks and crash
- # otherwise).
- PYTHONIOENCODING: utf-8
- PYTHONUTF8: '1'
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
-
- # Split restore + save so a transient restore-side failure does not
- # kill the whole job. See the matching block in the tool-calling job
- # above for the full rationale (actions/cache#1621).
- - name: Restore GGUF model cache
- id: cache-gguf
- uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- continue-on-error: true
- with:
- path: gguf-cache
- key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1
-
- - name: Download GGUF if cache miss
- id: download-gguf
- if: steps.cache-gguf.outputs.cache-hit != 'true' || steps.cache-gguf.outcome != 'success'
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- run: |
- python -m pip install --upgrade huggingface_hub
- mkdir -p gguf-cache
- bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" gguf-cache
-
- - name: Save GGUF model cache
- if: always() && steps.download-gguf.outcome == 'success'
- uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- with:
- path: gguf-cache
- key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1
-
- - name: Pre-install Windows tweaks (npm 11 + Defender exclusions)
- shell: pwsh
- # See studio-windows-update-smoke.yml for the full rationale.
- # tl;dr: setup.ps1 needs npm >=11 to skip a 35 s winget Node
- # reinstall, and Defender's real-time scan dominates the
- # frontend / uv-pip-extract steps.
- run: |
- $ProgressPreference = 'SilentlyContinue'
- Write-Host "npm version before upgrade: $(npm -v)"
- npm install -g 'npm@^11' 2>&1 | Out-Host
- Write-Host "npm version after upgrade: $(npm -v)"
- # NOTE: do NOT pre-create these directories. See
- # studio-windows-update-smoke.yml for the full rationale --
- # creating an empty studio/frontend/dist trips setup.ps1's
- # mtime-based staleness check into "frontend up to date, skip
- # rebuild" and Studio boots with an empty dist directory.
- # Add-MpPreference accepts paths that do not yet exist.
- foreach ($p in @(
- "$env:USERPROFILE\.unsloth",
- "$env:USERPROFILE\AppData\Local\uv",
- "$env:GITHUB_WORKSPACE\studio\frontend\node_modules",
- "$env:GITHUB_WORKSPACE\studio\frontend\dist"
- )) {
- try {
- Add-MpPreference -ExclusionPath $p -ErrorAction Stop
- Write-Host "Defender exclusion added: $p"
- } catch {
- Write-Host "Defender exclusion skipped ($($_.Exception.Message)): $p"
- }
- }
-
- - name: Install Studio (--local, --no-torch)
- shell: pwsh
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- New-Item -ItemType Directory -Force -Path logs | Out-Null
- # *>&1 captures Write-Host (Information stream) output;
- # plain 2>&1 does not. setup.ps1 emits "prebuilt installed
- # and validated" via Write-Host, and we grep for that.
- $ProgressPreference = 'SilentlyContinue'
- & ./install.ps1 --local --no-torch *>&1 | Tee-Object -FilePath logs/install.log
-
- - name: Assert install.ps1 used the Windows llama.cpp prebuilt
- run: |
- # Filesystem check; setup.ps1's stream output isn't captured.
- LLAMA_DIR=~/.unsloth/llama.cpp
- INFO="$LLAMA_DIR/UNSLOTH_PREBUILT_INFO.json"
- BIN="$LLAMA_DIR/build/bin/Release/llama-server.exe"
- if grep -q "falling back to source build" logs/install.log; then
- echo "::error::install.ps1 fell back to source-build llama.cpp on Windows."
- grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
- exit 1
- fi
- if [ ! -f "$INFO" ]; then
- echo "::error::no UNSLOTH_PREBUILT_INFO.json at $INFO."
- ls -la "$LLAMA_DIR" || true
- exit 1
- fi
- if [ ! -f "$BIN" ]; then
- echo "::error::no llama-server.exe at $BIN."
- ls -la "$LLAMA_DIR/build/bin" || true
- exit 1
- fi
- echo "install.ps1 installed the Windows prebuilt llama.cpp:"
- cat "$INFO"
-
- - name: Add Studio shim to GITHUB_PATH
- run: |
- SHIM_DIR=~/.unsloth/studio/bin
- if [ ! -f "$SHIM_DIR/unsloth.exe" ]; then
- echo "::error::unsloth.exe shim not found at $SHIM_DIR"
- ls -la ~/.unsloth/studio/ || true
- exit 1
- fi
- cygpath -w "$SHIM_DIR" >> "$GITHUB_PATH"
-
- - name: Patch Studio venv with full typer / pydantic dep trees
- # Belt-and-suspenders: install.ps1's --no-deps install of
- # no-torch-runtime.txt drops typer's and pydantic's runtime
- # deps unless explicitly pinned. Re-install the ones whose
- # deps don't pull torch.
- run: |
- STUDIO_PY=~/.unsloth/studio/unsloth_studio/Scripts/python.exe
- if [ ! -f "$STUDIO_PY" ]; then
- echo "::error::Studio venv python not at $STUDIO_PY"
- ls -la ~/.unsloth/studio/ || true
- exit 1
- fi
- "$STUDIO_PY" -m pip install --upgrade typer pydantic huggingface_hub
-
- - name: Reset auth + boot Studio (API-only, default tool policy)
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \
- > logs/studio.log 2>&1 &
- echo "STUDIO_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health, log in, change password, load model
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then
- jq -e '.status == "healthy"' /tmp/health.json && break
- fi
- sleep 1
- done
- jq -e '.status == "healthy"' /tmp/health.json
- OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- NEW="CITool-$(python -c 'import secrets; print(secrets.token_urlsafe(12))')"
- echo "::add-mask::$OLD"
- echo "::add-mask::$NEW"
- OLD_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}" | jq -r .access_token)
- curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/change-password" \
- -H "Authorization: Bearer $OLD_TOKEN" -H 'content-type: application/json' \
- -d "{\"current_password\":\"$OLD\",\"new_password\":\"$NEW\"}" > /dev/null
- TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$NEW\"}" | jq -r .access_token)
- echo "API_KEY=$TOKEN" >> "$GITHUB_ENV"
- # GITHUB_WORKSPACE on windows-latest is a Windows path with
- # backslashes ("D:\a\unsloth\unsloth"). Bash handles it as a
- # raw string, but we cannot embed `\a` etc. in JSON without
- # JSON-string-escaping every backslash. Replace `\` with `/`
- # via bash parameter expansion -- pathlib.Path on Windows
- # accepts forward slashes natively, so Studio's loader sees
- # a normal path.
- GGUF_PATH="${GITHUB_WORKSPACE//\\//}/gguf-cache/${GGUF_FILE}"
- ls -lh "$GGUF_PATH"
- # Retry: same rationale as the OpenAI/Anthropic job.
- LOAD_OK=0
- for attempt in 1 2 3; do
- HTTP=$(curl -s -o /tmp/load.json -w '%{http_code}' \
- -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \
- -H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \
- --max-time 600 \
- -d "{\"model_path\":\"$GGUF_PATH\",\"is_lora\":false,\"max_seq_length\":2048}")
- if [ "$HTTP" = "200" ]; then LOAD_OK=1; break; fi
- echo "::warning::/api/inference/load attempt $attempt returned $HTTP; response:"
- cat /tmp/load.json || true
- sleep 10
- done
- [ "$LOAD_OK" = "1" ] || { echo "::error::/api/inference/load failed 3 attempts"; exit 22; }
- jq '{status, display_name}' /tmp/load.json
-
- - name: Tool calling, server-side tools, thinking on/off
- env:
- BASE_URL: http://127.0.0.1:18898
- run: |
- python - <<'PY'
- import json
- import os
- import urllib.request
-
- BASE = os.environ["BASE_URL"]
- KEY = os.environ["API_KEY"]
- SEED = 3407
- # Same temperature shim as the Mac job. Small Qwen3.5-2B
- # quants can degenerate at temperature=0; a small non-zero
- # temperature with a fixed seed keeps the test deterministic
- # while escaping the trap.
- TEMP = 0.2
-
- def post(path, body, *, timeout = 240):
- data = json.dumps(body).encode()
- req = urllib.request.Request(
- f"{BASE}{path}",
- data = data,
- method = "POST",
- headers = {
- "Authorization": f"Bearer {KEY}",
- "Content-Type": "application/json",
- },
- )
- with urllib.request.urlopen(req, timeout = timeout) as resp:
- return resp.status, json.loads(resp.read().decode())
-
- def post_sse(path, body, *, timeout = 600):
- body = {**body, "stream": True}
- data = json.dumps(body).encode()
- req = urllib.request.Request(
- f"{BASE}{path}",
- data = data,
- method = "POST",
- headers = {
- "Authorization": f"Bearer {KEY}",
- "Content-Type": "application/json",
- },
- )
- parts = []
- with urllib.request.urlopen(req, timeout = timeout) as resp:
- for raw in resp:
- line = raw.decode().strip()
- if not line.startswith("data: "):
- continue
- payload = line[6:]
- if payload == "[DONE]":
- break
- try:
- chunk = json.loads(payload)
- except json.JSONDecodeError:
- continue
- for choice in chunk.get("choices", []):
- delta = choice.get("delta", {}) or {}
- if delta.get("content"):
- parts.append(delta["content"])
- return "".join(parts)
-
- # ── 1. Standard OpenAI function calling ──────────────────────
- weather_tool = {
- "type": "function",
- "function": {
- "name": "get_weather",
- "description": "Get current weather for a city.",
- "parameters": {
- "type": "object",
- "properties": {"city": {"type": "string"}},
- "required": ["city"],
- },
- },
- }
-
- status, data = post("/v1/chat/completions", {
- "messages": [{"role": "user", "content": "What is the weather in Paris?"}],
- "tools": [weather_tool],
- "tool_choice": "required",
- "stream": False,
- "temperature": TEMP,
- "seed": SEED,
- "max_tokens": 600,
- })
- assert status == 200, f"tool call status {status}: {data}"
- choice = data["choices"][0]
- tool_calls = (choice.get("message") or {}).get("tool_calls") or []
- if tool_calls:
- tc = tool_calls[0]
- assert tc["function"]["name"] == "get_weather", (
- f"unexpected tool name: {tc['function']['name']!r}"
- )
- args = json.loads(tc["function"]["arguments"])
- assert args.get("city"), f"missing city arg: {args}"
- print(f"[tools] PASS function calling -> {tc['function']['name']}({args}) finish={choice.get('finish_reason')!r}")
- else:
- print(
- f"[tools] WARN function calling: no tool_calls (finish_reason="
- f"{choice.get('finish_reason')!r}); HTTP path OK, model output drift."
- )
-
- # ── 2. Server-side python tool ───────────────────────────────
- content = post_sse("/v1/chat/completions", {
- "messages": [{"role": "user", "content": "What is 123 * 456? Use the python tool to compute it and tell me the number."}],
- "enable_tools": True,
- "enabled_tools": ["python"],
- "session_id": "ci-tool-calling-py",
- "temperature": TEMP,
- "seed": SEED,
- "max_tokens": 600,
- })
- if "56088" in content or "56,088" in content:
- print(f"[tools] PASS python tool ({len(content)} chars, found 56088)")
- else:
- assert content, "python tool: SSE stream empty"
- print(
- f"[tools] WARN python tool: SSE OK ({len(content)} chars) but "
- f"model didn't return 56088 -- model output drift"
- )
-
- # ── 3. Server-side bash (terminal) tool ──────────────────────
- # On Windows the terminal tool resolves to the system shell
- # (cmd.exe wrapper) and `echo hello-bash-tool` works the same
- # way it does on POSIX. The model still has to choose to
- # invoke the tool; assert non-empty SSE if it doesn't.
- content = post_sse("/v1/chat/completions", {
- "messages": [{"role": "user", "content": "Use the terminal tool to run `echo hello-bash-tool` and tell me the exact output."}],
- "enable_tools": True,
- "enabled_tools": ["terminal"],
- "session_id": "ci-tool-calling-bash",
- "temperature": TEMP,
- "seed": SEED,
- "max_tokens": 600,
- })
- if "hello-bash-tool" in content:
- print(f"[tools] PASS terminal tool ({len(content)} chars)")
- else:
- assert content, "terminal tool: SSE stream empty"
- print(
- f"[tools] WARN terminal tool: SSE OK ({len(content)} chars) but "
- f"model didn't echo 'hello-bash-tool' -- model output drift"
- )
-
- # ── 4. Server-side web_search tool ───────────────────────────
- # DuckDuckGo can be flaky from CI runners; only assert that
- # the SSE stream opens and yields any data.
- try:
- content = post_sse("/v1/chat/completions", {
- "messages": [{"role": "user", "content": "Search the web for 'unsloth ai github' and summarise."}],
- "enable_tools": True,
- "enabled_tools": ["web_search"],
- "session_id": "ci-tool-calling-web",
- "temperature": TEMP,
- "seed": SEED,
- "max_tokens": 400,
- })
- print(f"[tools] PASS web_search stream ({len(content)} chars)")
- except Exception as exc:
- print(f"[tools] WARN web_search probe failed (non-blocking): {exc}")
-
- # ── 5. Thinking on / off ─────────────────────────────────────
- def thinking_call(enable):
- status, data = post("/v1/chat/completions", {
- "messages": [{"role": "user", "content": "Briefly: is 17 prime?"}],
- "stream": False,
- "enable_thinking": enable,
- "temperature": TEMP,
- "seed": SEED,
- "max_tokens": 300,
- })
- assert status == 200
- msg = data["choices"][0]["message"]
- raw = (msg.get("content") or "") + (msg.get("reasoning_content") or "")
- return raw
-
- on_text = thinking_call(True)
- off_text = thinking_call(False)
- had_think_on = ("" in on_text) or len(on_text) > 80
- if not had_think_on:
- print(
- f"[tools] WARN enable_thinking=True produced no thinking signal: "
- f"{on_text[:200]!r}"
- )
- assert "" not in off_text, (
- f"enable_thinking=False but still present: {off_text!r}"
- )
- print(f"[tools] PASS thinking on/off (on={len(on_text)} chars, off={len(off_text)} chars)")
- PY
-
- - name: Stop Studio
- if: always()
- # Run as cmd so we are not running through the Git Bash shell;
- # Git Bash on windows-latest has been observed to exit 143
- # (SIGTERM) from any inline kill/sleep block, masking a green
- # test run. The runner reclaims the Studio child process at
- # job end either way, so just emit a marker and exit 0.
- shell: cmd
- run: echo Stop Studio (no-op; runner reclaims STUDIO_PID=%STUDIO_PID% at job end)
-
- - name: Collect llama-server logs
- if: always()
- shell: bash
- # Copy llama-server's own stdout/stderr (teed by Studio under
- # ~/.unsloth/studio/logs/llama-server/) into the workspace so
- # upload-artifact can pick it up. Crucial for diagnosing a
- # subprocess crash where Studio's traceback only shows the
- # symptom (httpx ReadError) but not the cause.
- run: |
- mkdir -p logs/llama-server
- cp -v ~/.unsloth/studio/logs/llama-server/*.log logs/llama-server/ 2>/dev/null || \
- echo "no llama-server logs to collect"
-
- - name: Upload logs
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: windows-tool-calling-log
- path: |
- logs/studio.log
- logs/install.log
- logs/llama-server/*.log
- retention-days: 7
-
- # ─────────────────────────────────────────────────────────────────────
- # Job 3: JSON, images
- # ─────────────────────────────────────────────────────────────────────
- json-images:
- name: JSON, images
- runs-on: windows-latest
- timeout-minutes: 35
- defaults:
- run:
- shell: bash
- env:
- GGUF_REPO: unsloth/gemma-4-E2B-it-GGUF
- GGUF_VARIANT: UD-Q4_K_XL
- GGUF_FILE: gemma-4-E2B-it-UD-Q4_K_XL.gguf
- MMPROJ_FILE: mmproj-F16.gguf
- STUDIO_PORT: '18899'
- HF_HOME: ${{ github.workspace }}/hf-cache
- # Force UTF-8 for stdio (Windows defaults to cp1252; hf
- # download / Studio CLI print "✓" checkmarks and crash
- # otherwise).
- PYTHONIOENCODING: utf-8
- PYTHONUTF8: '1'
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
-
- # Split restore + save so a transient restore-side failure does not
- # kill the whole job. See the matching block in the tool-calling job
- # for the full rationale (actions/cache#1621). This is the block that
- # actually broke in run 25713577488: "Cache hit for: " was
- # logged, the step exited non-zero in ~0.3 s without extracting the
- # 3.4 GiB archive, and steps 6-15 were skipped.
- - name: Restore HF_HOME cache for ${{ env.GGUF_REPO }} (model + mmproj)
- id: cache-hf
- uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- continue-on-error: true
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-${{ env.MMPROJ_FILE }}-v1
-
- - name: Prime HF_HOME with the GGUF + mmproj
- id: prime-hf
- if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success'
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- run: |
- python -m pip install --upgrade huggingface_hub
- mkdir -p hf-cache
- bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE"
- bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$MMPROJ_FILE"
-
- - name: Save HF_HOME cache for ${{ env.GGUF_REPO }} (model + mmproj)
- if: always() && steps.prime-hf.outcome == 'success'
- uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-${{ env.MMPROJ_FILE }}-v1
-
- - name: Pre-install Windows tweaks (npm 11 + Defender exclusions)
- shell: pwsh
- # See studio-windows-update-smoke.yml for the full rationale.
- # tl;dr: setup.ps1 needs npm >=11 to skip a 35 s winget Node
- # reinstall, and Defender's real-time scan dominates the
- # frontend / uv-pip-extract steps.
- run: |
- $ProgressPreference = 'SilentlyContinue'
- Write-Host "npm version before upgrade: $(npm -v)"
- npm install -g 'npm@^11' 2>&1 | Out-Host
- Write-Host "npm version after upgrade: $(npm -v)"
- # NOTE: do NOT pre-create these directories. See
- # studio-windows-update-smoke.yml for the full rationale --
- # creating an empty studio/frontend/dist trips setup.ps1's
- # mtime-based staleness check into "frontend up to date, skip
- # rebuild" and Studio boots with an empty dist directory.
- # Add-MpPreference accepts paths that do not yet exist.
- foreach ($p in @(
- "$env:USERPROFILE\.unsloth",
- "$env:USERPROFILE\AppData\Local\uv",
- "$env:GITHUB_WORKSPACE\studio\frontend\node_modules",
- "$env:GITHUB_WORKSPACE\studio\frontend\dist"
- )) {
- try {
- Add-MpPreference -ExclusionPath $p -ErrorAction Stop
- Write-Host "Defender exclusion added: $p"
- } catch {
- Write-Host "Defender exclusion skipped ($($_.Exception.Message)): $p"
- }
- }
-
- - name: Install Studio (--local, --no-torch)
- shell: pwsh
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- New-Item -ItemType Directory -Force -Path logs | Out-Null
- # *>&1 captures Write-Host (Information stream) output;
- # plain 2>&1 does not. setup.ps1 emits "prebuilt installed
- # and validated" via Write-Host, and we grep for that.
- $ProgressPreference = 'SilentlyContinue'
- & ./install.ps1 --local --no-torch *>&1 | Tee-Object -FilePath logs/install.log
-
- - name: Assert install.ps1 used the Windows llama.cpp prebuilt
- run: |
- # Filesystem check; setup.ps1's stream output isn't captured.
- LLAMA_DIR=~/.unsloth/llama.cpp
- INFO="$LLAMA_DIR/UNSLOTH_PREBUILT_INFO.json"
- BIN="$LLAMA_DIR/build/bin/Release/llama-server.exe"
- if grep -q "falling back to source build" logs/install.log; then
- echo "::error::install.ps1 fell back to source-build llama.cpp on Windows."
- grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
- exit 1
- fi
- if [ ! -f "$INFO" ]; then
- echo "::error::no UNSLOTH_PREBUILT_INFO.json at $INFO."
- ls -la "$LLAMA_DIR" || true
- exit 1
- fi
- if [ ! -f "$BIN" ]; then
- echo "::error::no llama-server.exe at $BIN."
- ls -la "$LLAMA_DIR/build/bin" || true
- exit 1
- fi
- echo "install.ps1 installed the Windows prebuilt llama.cpp:"
- cat "$INFO"
-
- - name: Add Studio shim to GITHUB_PATH
- run: |
- SHIM_DIR=~/.unsloth/studio/bin
- if [ ! -f "$SHIM_DIR/unsloth.exe" ]; then
- echo "::error::unsloth.exe shim not found at $SHIM_DIR"
- ls -la ~/.unsloth/studio/ || true
- exit 1
- fi
- cygpath -w "$SHIM_DIR" >> "$GITHUB_PATH"
-
- - name: Patch Studio venv with full typer / pydantic dep trees
- # Belt-and-suspenders: install.ps1's --no-deps install of
- # no-torch-runtime.txt drops typer's and pydantic's runtime
- # deps unless explicitly pinned. Re-install the ones whose
- # deps don't pull torch.
- run: |
- STUDIO_PY=~/.unsloth/studio/unsloth_studio/Scripts/python.exe
- if [ ! -f "$STUDIO_PY" ]; then
- echo "::error::Studio venv python not at $STUDIO_PY"
- ls -la ~/.unsloth/studio/ || true
- exit 1
- fi
- "$STUDIO_PY" -m pip install --upgrade typer pydantic huggingface_hub
-
- - name: Install OpenAI + Anthropic Python SDKs
- run: python -m pip install 'openai>=1.50' 'anthropic>=0.40'
-
- - name: Reset auth + boot Studio (API-only)
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \
- > logs/studio.log 2>&1 &
- echo "STUDIO_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health, log in, change password, load model
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then
- jq -e '.status == "healthy"' /tmp/health.json && break
- fi
- sleep 1
- done
- jq -e '.status == "healthy"' /tmp/health.json
- OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- NEW="CIJson-$(python -c 'import secrets; print(secrets.token_urlsafe(12))')"
- echo "::add-mask::$OLD"
- echo "::add-mask::$NEW"
- OLD_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}" | jq -r .access_token)
- curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/change-password" \
- -H "Authorization: Bearer $OLD_TOKEN" -H 'content-type: application/json' \
- -d "{\"current_password\":\"$OLD\",\"new_password\":\"$NEW\"}" > /dev/null
- TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
- -H 'content-type: application/json' \
- -d "{\"username\":\"unsloth\",\"password\":\"$NEW\"}" | jq -r .access_token)
- echo "API_KEY=$TOKEN" >> "$GITHUB_ENV"
- # Retry: same rationale as the OpenAI/Anthropic and Tool calling jobs.
- LOAD_OK=0
- for attempt in 1 2 3; do
- HTTP=$(curl -s -o /tmp/load.json -w '%{http_code}' \
- -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \
- -H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \
- --max-time 900 \
- -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048}")
- if [ "$HTTP" = "200" ]; then LOAD_OK=1; break; fi
- echo "::warning::/api/inference/load attempt $attempt returned $HTTP; response:"
- cat /tmp/load.json || true
- sleep 10
- done
- [ "$LOAD_OK" = "1" ] || { echo "::error::/api/inference/load failed 3 attempts"; exit 22; }
- jq '{status, display_name, is_vision}' /tmp/load.json
-
- - name: JSON schema decoding + image input
- env:
- BASE_URL: http://127.0.0.1:18899
- run: |
- python - <<'PY'
- import base64
- import json
- import os
- import urllib.request
- from openai import OpenAI
- from anthropic import Anthropic
-
- BASE = os.environ["BASE_URL"]
- KEY = os.environ["API_KEY"]
- SEED = 3407
- TEMP = 0.2
-
- def post(path, body, *, timeout = 240):
- req = urllib.request.Request(
- f"{BASE}{path}",
- data = json.dumps(body).encode(),
- method = "POST",
- headers = {
- "Authorization": f"Bearer {KEY}",
- "Content-Type": "application/json",
- },
- )
- with urllib.request.urlopen(req, timeout = timeout) as resp:
- return resp.status, json.loads(resp.read().decode())
-
- # ── 1. response_format = json_object (JSON mode) ─────────────
- status, data = post("/v1/chat/completions", {
- "model": "default",
- "messages": [
- {"role": "system", "content": 'Reply with a single JSON object of the form {"city": "...", "country": "..."}. Output ONLY the JSON, nothing else.'},
- {"role": "user", "content": "What is the capital of France?"},
- ],
- "temperature": TEMP,
- "max_tokens": 600,
- "seed": SEED,
- "stream": False,
- "enable_thinking": False,
- "response_format": {"type": "json_object"},
- }, timeout = 600)
- assert status == 200, f"json status {status}: {data}"
- assert (
- isinstance(data.get("choices"), list)
- and data["choices"]
- and "message" in data["choices"][0]
- ), f"json response envelope malformed: {data}"
- content = (data["choices"][0]["message"].get("content") or "").strip()
- print(f"[json] raw json_object content: {content!r}")
- if content.startswith("```"):
- content = content.split("```", 2)[1]
- if content.startswith("json"):
- content = content[4:]
- content = content.strip("`\n ")
- if content:
- try:
- parsed = json.loads(content)
- if "paris" in str(parsed.get("city", "")).lower():
- print(f"[json] PASS json_object -> {parsed}")
- else:
- print(f"[json] WARN json_object decoded but city!=Paris: {parsed}")
- except json.JSONDecodeError as exc:
- print(f"[json] WARN json_object content not parseable ({exc}); content={content!r}")
- else:
- print("[json] WARN json_object produced empty content")
-
- status2, data2 = post("/v1/chat/completions", {
- "model": "default",
- "messages": [{"role": "user", "content": "What is the capital of France? Answer with one word."}],
- "temperature": TEMP,
- "max_tokens": 400,
- "seed": SEED,
- "stream": False,
- "enable_thinking": False,
- }, timeout = 600)
- assert status2 == 200, f"plain status {status2}: {data2}"
- plain = (data2["choices"][0]["message"].get("content") or "").lower()
- print(f"[json] plain capital-of-france reply: {plain!r}")
- if "paris" in plain:
- print("[json] PASS plain inference path (paris mentioned)")
- else:
- print(
- f"[json] WARN plain inference returned no 'paris' -- "
- f"model output drift. HTTP path validated separately above."
- )
-
- # ── 2. OpenAI image_url (data URI base64) ───────────────────
- PNG_64X64_RED_B64 = (
- "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAYklEQVR4nO3PMQ0AIADAMEAI/k"
- "UhBhEcDcmqYJtn7/GzpQNeNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA"
- "1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaBdCJ0BmMJ25zMAAAAASUVORK5CYII="
- )
- data_uri = f"data:image/png;base64,{PNG_64X64_RED_B64}"
-
- # On Windows + the gemma-4-E2B mmproj, llama.cpp's vision
- # path runs on CPU (no Metal involvement). The wrapper is
- # kept for resilience but the vision path is expected to
- # work on Windows; an exception here is a real regression.
- client = OpenAI(base_url = f"{BASE}/v1", api_key = KEY)
- try:
- openai_resp = client.chat.completions.create(
- model = "default",
- temperature = TEMP,
- max_tokens = 80,
- seed = SEED,
- messages = [{
- "role": "user",
- "content": [
- {"type": "image_url", "image_url": {"url": data_uri}},
- {"type": "text", "text": "What colour dominates this image? Reply in one word."},
- ],
- }],
- )
- openai_text = (openai_resp.choices[0].message.content or "").lower()
- print(f"[image/openai] reply: {openai_text!r}")
- if openai_text:
- print("[image/openai] PASS image_url accepted, non-empty response")
- else:
- print("[image/openai] WARN image_url accepted but empty content")
- except Exception as exc:
- print(
- f"[image/openai] WARN image_url SDK call raised: {type(exc).__name__}: "
- f"{exc}. Studio successfully forwarded the request; failure here is "
- f"upstream llama.cpp vision behaviour."
- )
-
- # ── 3. Anthropic source/base64 image ────────────────────────
- anthropic = Anthropic(
- base_url = BASE,
- api_key = "unused",
- default_headers = {"Authorization": f"Bearer {KEY}"},
- )
- try:
- a_msg = anthropic.messages.create(
- model = "default",
- max_tokens = 80,
- temperature = TEMP,
- extra_body = {"seed": SEED},
- messages = [{
- "role": "user",
- "content": [
- {
- "type": "image",
- "source": {
- "type": "base64",
- "media_type": "image/png",
- "data": PNG_64X64_RED_B64,
- },
- },
- {"type": "text", "text": "Describe this image briefly."},
- ],
- }],
- )
- a_text = "".join(b.text for b in a_msg.content if getattr(b, "type", None) == "text")
- print(f"[image/anthropic] reply: {a_text!r}")
- if a_text:
- print("[image/anthropic] PASS source/base64 accepted, non-empty response")
- else:
- print("[image/anthropic] WARN source/base64 accepted but empty content")
- except Exception as exc:
- print(
- f"[image/anthropic] WARN anthropic image SDK call raised: "
- f"{type(exc).__name__}: {exc}. Likely upstream llama.cpp vision "
- f"behaviour, NOT a Studio regression."
- )
- PY
-
- - name: Stop Studio
- if: always()
- # Run as cmd so we are not running through the Git Bash shell;
- # Git Bash on windows-latest has been observed to exit 143
- # (SIGTERM) from any inline kill/sleep block, masking a green
- # test run. The runner reclaims the Studio child process at
- # job end either way, so just emit a marker and exit 0.
- shell: cmd
- run: echo Stop Studio (no-op; runner reclaims STUDIO_PID=%STUDIO_PID% at job end)
-
- - name: Collect llama-server logs
- if: always()
- shell: bash
- # Copy llama-server's own stdout/stderr (teed by Studio under
- # ~/.unsloth/studio/logs/llama-server/) into the workspace so
- # upload-artifact can pick it up. Crucial for diagnosing a
- # subprocess crash where Studio's traceback only shows the
- # symptom (httpx ReadError) but not the cause.
- run: |
- mkdir -p logs/llama-server
- cp -v ~/.unsloth/studio/logs/llama-server/*.log logs/llama-server/ 2>/dev/null || \
- echo "no llama-server logs to collect"
-
- - name: Upload logs
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: windows-json-images-log
- path: |
- logs/studio.log
- logs/install.log
- logs/llama-server/*.log
- retention-days: 7
diff --git a/.github/workflows/studio-windows-ui-smoke.yml b/.github/workflows/studio-windows-ui-smoke.yml
deleted file mode 100644
index e5ab9f8ab7..0000000000
--- a/.github/workflows/studio-windows-ui-smoke.yml
+++ /dev/null
@@ -1,342 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# Windows counterpart to studio-ui-smoke.yml / studio-mac-ui-smoke.yml.
-# Same Playwright + Chromium end-to-end chat UI flow + extra UI flow,
-# but on the FREE windows-latest runner so we catch Windows-specific
-# regressions in the install path (install.ps1), the Studio CLI's
-# Windows process-management branches, and the llama.cpp prebuilt's
-# Windows HTTP layer.
-
-name: Windows Studio UI CI
-
-on:
- pull_request:
- paths:
- - 'studio/**'
- - 'unsloth/**'
- - 'unsloth_cli/**'
- - 'install.ps1'
- - 'pyproject.toml'
- - 'tests/studio/**'
- - '.github/workflows/studio-windows-ui-smoke.yml'
- push:
- branches: [main, pip]
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- ui-smoke:
- name: Chat UI Tests
- runs-on: windows-latest
- timeout-minutes: 45
- # Default every step's shell to Git Bash. windows-latest's default
- # shell is pwsh; without this each curl / heredoc / `kill $PID`
- # step would need its own `shell: bash`. Steps that genuinely
- # need PowerShell (install.ps1 invocation) override per-step.
- defaults:
- run:
- shell: bash
- env:
- GGUF_REPO: unsloth/gemma-3-270m-it-GGUF
- GGUF_VARIANT: UD-Q4_K_XL
- GGUF_FILE: gemma-3-270m-it-UD-Q4_K_XL.gguf
- STUDIO_PORT: '18896'
- HF_HOME: ${{ github.workspace }}/hf-cache
- # Force UTF-8 for stdio so Python tools (hf download, Studio
- # CLI, etc.) can print Unicode characters like the success
- # checkmark "✓". Windows defaults to cp1252 / charmap and
- # any tool that prints "OK ✓" hits a UnicodeEncodeError.
- PYTHONIOENCODING: utf-8
- PYTHONUTF8: '1'
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
- # No `cache: 'npm'`. setup-node's npm cache restore silently
- # aborts the entire job on Windows runners when the npm cache
- # path (`C:\npm\cache` per `npm config get cache`) doesn't yet
- # exist on a fresh runner -- the step exits without an error
- # message and every following step gets skipped. See
- # npm/cli#7308. The frontend `npm ci` is fast enough without
- # the cache that the reliability gain is worth the ~30s.
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- # No `cache: 'pip'`. install.ps1 / setup.ps1 use uv and
- # never populate ~/.cache/pip; setup-python's post-step
- # then fatal-errors with "Cache folder path is retrieved
- # for pip but doesn't exist on disk".
-
- - name: Restore HF_HOME for ${{ env.GGUF_REPO }}
- id: cache-hf
- uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- continue-on-error: true
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
-
- - name: Prime HF_HOME with the GGUF
- id: prime-hf
- if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success'
- env:
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
- run: |
- python -m pip install --upgrade huggingface_hub
- mkdir -p hf-cache
- bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE"
-
- - name: Save HF_HOME for ${{ env.GGUF_REPO }}
- if: always() && steps.prime-hf.outcome == 'success'
- uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- with:
- path: hf-cache
- key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
-
- - name: Pre-install Windows tweaks (npm 11 + Defender exclusions)
- shell: pwsh
- # See studio-windows-update-smoke.yml for the full rationale.
- # tl;dr: setup.ps1 needs npm >=11 to skip a 35 s winget Node
- # reinstall, and Defender's real-time scan dominates the
- # frontend / uv-pip-extract steps.
- run: |
- $ProgressPreference = 'SilentlyContinue'
- Write-Host "npm version before upgrade: $(npm -v)"
- npm install -g 'npm@^11' 2>&1 | Out-Host
- Write-Host "npm version after upgrade: $(npm -v)"
- # NOTE: do NOT pre-create these directories. See
- # studio-windows-update-smoke.yml for the full rationale --
- # creating an empty studio/frontend/dist trips setup.ps1's
- # mtime-based staleness check into "frontend up to date, skip
- # rebuild" and Studio boots with an empty dist directory.
- # Add-MpPreference accepts paths that do not yet exist.
- foreach ($p in @(
- "$env:USERPROFILE\.unsloth",
- "$env:USERPROFILE\AppData\Local\uv",
- "$env:GITHUB_WORKSPACE\studio\frontend\node_modules",
- "$env:GITHUB_WORKSPACE\studio\frontend\dist"
- )) {
- try {
- Add-MpPreference -ExclusionPath $p -ErrorAction Stop
- Write-Host "Defender exclusion added: $p"
- } catch {
- Write-Host "Defender exclusion skipped ($($_.Exception.Message)): $p"
- }
- }
-
- - name: Install Studio (--local, --no-torch)
- # install.ps1 is the supported Windows installer. install.sh
- # has no Windows branch (apt-get / brew calls). The PS1
- # script's `Install-UnslothStudio @args` line at the bottom
- # forwards `--local --no-torch` correctly.
- shell: pwsh
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- New-Item -ItemType Directory -Force -Path logs | Out-Null
- # *>&1 redirects ALL PowerShell streams (stdout, stderr,
- # warning, verbose, debug, information) into the success
- # stream so Tee-Object captures everything. install.ps1
- # and setup.ps1 emit step/substep markers via Write-Host
- # which lands on the Information stream (PS 5+); without
- # the wildcard redirect, those markers (including
- # "prebuilt installed and validated") never reach
- # logs/install.log and the post-step grep asserter fails.
- $ProgressPreference = 'SilentlyContinue'
- & ./install.ps1 --local --no-torch *>&1 | Tee-Object -FilePath logs/install.log
-
- - name: Assert install.ps1 used the Windows llama.cpp prebuilt
- run: |
- # install.ps1's setup.ps1 child writes "prebuilt installed
- # and validated" to its own console host -- that output
- # does NOT come back through this parent step's stdout
- # pipeline (no matter how aggressively we redirect: *>&1,
- # tee, etc.). Verify the install via the filesystem
- # instead. setup.ps1 writes UNSLOTH_PREBUILT_INFO.json
- # next to the install dir on success, and lays the
- # binaries under build/bin/Release/ on Windows.
- STUDIO_HOME=~/.unsloth/studio
- LLAMA_DIR=~/.unsloth/llama.cpp
- INFO="$LLAMA_DIR/UNSLOTH_PREBUILT_INFO.json"
- BIN="$LLAMA_DIR/build/bin/Release/llama-server.exe"
- # Source-build fallback grep stays as a fast bail-out.
- if grep -q "falling back to source build" logs/install.log; then
- echo "::error::install.ps1 fell back to source-build llama.cpp on Windows."
- grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
- exit 1
- fi
- if [ ! -f "$INFO" ]; then
- echo "::error::no UNSLOTH_PREBUILT_INFO.json at $INFO; setup.ps1 didn't install the prebuilt."
- ls -la "$LLAMA_DIR" || true
- exit 1
- fi
- if [ ! -f "$BIN" ]; then
- echo "::error::no llama-server.exe at $BIN; prebuilt extraction incomplete."
- ls -la "$LLAMA_DIR/build/bin" || true
- ls -la "$LLAMA_DIR/build/bin/Release" || true
- exit 1
- fi
- echo "install.ps1 installed the Windows prebuilt llama.cpp:"
- cat "$INFO"
-
- - name: Add Studio shim to GITHUB_PATH
- # install.ps1 puts unsloth.exe at $StudioHome\bin\unsloth.exe
- # and adds that dir to the User PATH via the Windows registry.
- # Registry-level PATH updates don't propagate to a running
- # Git Bash session, so the next step's `unsloth ...` invocation
- # would hit "command not found". Re-export the shim dir to
- # GITHUB_PATH so every subsequent step in this job sees it.
- run: |
- SHIM_DIR=~/.unsloth/studio/bin
- if [ ! -f "$SHIM_DIR/unsloth.exe" ]; then
- echo "::error::unsloth.exe shim not found at $SHIM_DIR"
- ls -la ~/.unsloth/studio/ || true
- exit 1
- fi
- # GITHUB_PATH wants Windows-style paths; convert via cygpath.
- cygpath -w "$SHIM_DIR" >> "$GITHUB_PATH"
- echo "Added Studio shim dir to PATH: $(cygpath -w "$SHIM_DIR")"
-
- - name: Patch Studio venv with full typer / pydantic dep trees
- # Belt-and-suspenders: install.ps1's --no-deps install of
- # no-torch-runtime.txt drops typer's and pydantic's runtime
- # deps unless explicitly pinned. Re-install the ones whose
- # deps don't pull torch.
- run: |
- STUDIO_PY=~/.unsloth/studio/unsloth_studio/Scripts/python.exe
- if [ ! -f "$STUDIO_PY" ]; then
- echo "::error::Studio venv python not at $STUDIO_PY"
- ls -la ~/.unsloth/studio/ || true
- exit 1
- fi
- "$STUDIO_PY" -m pip install --upgrade typer pydantic huggingface_hub
-
- - name: Install Playwright + Chromium
- # No --with-deps on Windows: that flag installs Linux apt
- # packages. windows-latest ships the system frameworks
- # Chromium needs (Edge / WebView2) already.
- run: |
- python -m pip install 'playwright>=1.45'
- python -m playwright install chromium
-
- - name: Reset auth + boot Studio
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \
- > logs/studio.log 2>&1 &
- echo "STUDIO_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then
- jq -e '.status == "healthy"' /tmp/health.json && break
- fi
- sleep 1
- done
- jq -e '.status == "healthy"' /tmp/health.json
-
- - name: Pass bootstrap password to the Playwright step
- run: |
- OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- NEW="CIUi-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- NEW2="CIUi-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- echo "::add-mask::$OLD"
- echo "::add-mask::$NEW"
- echo "::add-mask::$NEW2"
- echo "STUDIO_OLD_PW=$OLD" >> "$GITHUB_ENV"
- echo "STUDIO_NEW_PW=$NEW" >> "$GITHUB_ENV"
- echo "STUDIO_NEW2_PW=$NEW2" >> "$GITHUB_ENV"
-
- - name: Drive the chat UI with Playwright
- env:
- BASE_URL: http://127.0.0.1:18896
- PW_ART_DIR: logs/playwright
- STUDIO_UI_STRICT: '1'
- # windows-latest free runner is 4 vCPU / 16 GB; gemma-3-
- # 270m turn latency under llama-server's CPU backend can
- # crowd the 180s default (slower than ubuntu-latest on
- # the same model). Keep the same generous budget the Mac
- # job uses.
- STUDIO_UI_TURN_TIMEOUT_MS: '540000'
- run: |
- mkdir -p logs/playwright
- python tests/studio/playwright_chat_ui.py
-
- - name: Stop Studio (chat-ui ends with Shutdown click; this is belt-and-suspenders)
- if: always()
- run: |
- kill "${STUDIO_PID}" 2>/dev/null || true
- sleep 2
-
- - name: Reset auth + boot Studio for extra UI tests (port 18897)
- run: |
- unsloth studio reset-password
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18897 \
- > logs/studio_extra.log 2>&1 &
- echo "STUDIO_EXTRA_PID=$!" >> "$GITHUB_ENV"
-
- - name: Wait for /api/health on 18897
- run: |
- for i in $(seq 1 180); do
- if curl -fs "http://127.0.0.1:18897/api/health" > /tmp/health2.json; then
- jq -e '.status == "healthy"' /tmp/health2.json && break
- fi
- sleep 1
- done
- jq -e '.status == "healthy"' /tmp/health2.json
-
- - name: Pass bootstrap pw for extra UI test
- run: |
- OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
- NEW="CIUiExtra-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')"
- echo "::add-mask::$OLD"
- echo "::add-mask::$NEW"
- echo "STUDIO_EXTRA_OLD_PW=$OLD" >> "$GITHUB_ENV"
- echo "STUDIO_EXTRA_NEW_PW=$NEW" >> "$GITHUB_ENV"
-
- - name: Drive Compare/Recipes/Export/Studio/Settings with Playwright
- env:
- BASE_URL: http://127.0.0.1:18897
- STUDIO_OLD_PW: ${{ env.STUDIO_EXTRA_OLD_PW }}
- STUDIO_NEW_PW: ${{ env.STUDIO_EXTRA_NEW_PW }}
- PW_ART_DIR: logs/playwright_extra
- STUDIO_UI_STRICT: '1'
- STUDIO_UI_TURN_TIMEOUT_MS: '540000'
- GGUF_REPO: ${{ env.GGUF_REPO }}
- GGUF_VARIANT: ${{ env.GGUF_VARIANT }}
- run: |
- mkdir -p logs/playwright_extra
- python tests/studio/playwright_extra_ui.py
-
- - name: Stop second Studio
- if: always()
- run: |
- kill "${STUDIO_EXTRA_PID}" 2>/dev/null || true
- sleep 2
-
- - name: Upload Playwright artifacts
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: windows-studio-ui-smoke-artifacts
- path: |
- logs/studio.log
- logs/studio_extra.log
- logs/install.log
- logs/playwright
- logs/playwright_extra
- retention-days: 7
diff --git a/.github/workflows/studio-windows-update-smoke.yml b/.github/workflows/studio-windows-update-smoke.yml
deleted file mode 100644
index b477b3fa11..0000000000
--- a/.github/workflows/studio-windows-update-smoke.yml
+++ /dev/null
@@ -1,315 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# Windows counterpart to studio-update-smoke.yml /
-# studio-mac-update-smoke.yml. Verifies that on the FREE
-# windows-latest runner:
-#
-# 1. install.ps1 --local --no-torch installs Studio AND auto-fetches
-# the prebuilt llama.cpp Windows binary (llama-bNNNN-bin-win-cpu-
-# x64 from ggml-org/llama.cpp). Hitting the source-build fallback
-# is treated as an Unsloth bug -- Studio must always pick the
-# prebuilt on Windows.
-# 2. unsloth studio update --local is idempotent. Two consecutive
-# runs both report "prebuilt up to date and validated", no
-# source-build fallback. The CLI's _find_setup_script picks
-# setup.ps1 on Windows automatically.
-# 3. The installed Studio still boots and /api/health returns
-# healthy after the update path.
-
-name: Windows Studio Update CI
-
-on:
- pull_request:
- paths:
- - 'install.ps1'
- - 'scripts/uninstall.ps1'
- - 'studio/setup.ps1'
- - 'studio/setup.bat'
- - 'studio/install_python_stack.py'
- - 'studio/install_llama_prebuilt.py'
- - 'studio/backend/requirements/**'
- - 'unsloth_cli/commands/studio.py'
- - 'pyproject.toml'
- - '.github/workflows/studio-windows-update-smoke.yml'
- push:
- branches: [main, pip]
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- update-idempotency:
- name: Studio Updating Tests
- runs-on: windows-latest
- timeout-minutes: 30
- defaults:
- run:
- shell: bash
- env:
- # Force UTF-8 for stdio (Windows defaults to cp1252; hf
- # download / Studio CLI print "✓" checkmarks and crash
- # otherwise).
- PYTHONIOENCODING: utf-8
- PYTHONUTF8: '1'
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- # Don't cache pip: install.ps1 + setup.ps1 go through uv
- # and never populate ~/.cache/pip; setup-python's post-step
- # then fatal-errors with "Cache folder path is retrieved
- # for pip but doesn't exist on disk".
-
- - name: Pre-install Windows tweaks (npm 11 + Defender exclusions)
- shell: pwsh
- # Two surgical fixes against measured Windows-only install
- # waste (vs Mac/Linux on the same SHA):
- #
- # (1) npm. setup.ps1 line 1109-1145 requires Node 22.12+ (or
- # 20.19+ / 23+) AND npm >=11 because Vite 8 needs both.
- # actions/setup-node@v4 with `node-version: '22'` lands
- # Node 22.22.2 + the npm 10.9.7 it bundles, so the npm
- # check fails and setup.ps1 falls through to the
- # "winget install Node.js LTS" branch -- a ~35 s reinstall
- # of Node we don't need. `npm install -g npm@^11` updates
- # the bundled npm in-place in ~5 s, which makes setup.ps1
- # short-circuit on the existing Node.
- #
- # (2) Defender. windows-latest's real-time scan opens / hashes
- # every file Studio writes during install (Vite output =
- # thousands of small chunks, uv pip = wheel-extraction =
- # thousands of small files). The latency dominates the
- # 200 s frontend build and the 90 s deps install. Adding
- # ExclusionPath entries for the directories the install
- # writes to drops per-file open latency from ~ms to ~us.
- # Add-MpPreference needs admin; the runneradmin user has
- # it, but wrap in try/catch so a permission flake leaves
- # the install otherwise unaffected.
- run: |
- $ProgressPreference = 'SilentlyContinue'
- Write-Host "npm version before upgrade: $(npm -v)"
- npm install -g 'npm@^11' 2>&1 | Out-Host
- Write-Host "npm version after upgrade: $(npm -v)"
- # NOTE: do NOT pre-create these directories before adding the
- # exclusion -- creating an empty studio/frontend/dist trips
- # setup.ps1 line 1281-1296's mtime-based "is the frontend
- # stale?" check into "up to date, skip rebuild", because the
- # newly-created dist's mtime is younger than every source
- # file. Studio then boots with an empty dist and 500s on
- # GET / with FileNotFoundError: dist\index.html. See run
- # 25546676715 / job 74984469728.
- # Add-MpPreference accepts paths that do not yet exist; the
- # exclusion is registered and applies when the path
- # materialises.
- foreach ($p in @(
- "$env:USERPROFILE\.unsloth",
- "$env:USERPROFILE\AppData\Local\uv",
- "$env:GITHUB_WORKSPACE\studio\frontend\node_modules",
- "$env:GITHUB_WORKSPACE\studio\frontend\dist"
- )) {
- try {
- Add-MpPreference -ExclusionPath $p -ErrorAction Stop
- Write-Host "Defender exclusion added: $p"
- } catch {
- Write-Host "Defender exclusion skipped ($($_.Exception.Message)): $p"
- }
- }
-
- - name: Install Studio (--local, --no-torch)
- shell: pwsh
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- New-Item -ItemType Directory -Force -Path logs | Out-Null
- # *>&1 captures Write-Host (Information stream) output;
- # plain 2>&1 does not. setup.ps1 emits "prebuilt installed
- # and validated" via Write-Host, and we grep for that.
- $ProgressPreference = 'SilentlyContinue'
- & ./install.ps1 --local --no-torch *>&1 | Tee-Object -FilePath logs/install.log
-
- - name: Assert install.ps1 used the Windows llama.cpp prebuilt
- run: |
- # Filesystem-based check (setup.ps1's stream output isn't
- # captured back through the parent pipeline).
- LLAMA_DIR=~/.unsloth/llama.cpp
- INFO="$LLAMA_DIR/UNSLOTH_PREBUILT_INFO.json"
- BIN="$LLAMA_DIR/build/bin/Release/llama-server.exe"
- if grep -q "falling back to source build" logs/install.log; then
- echo "::error::install.ps1 fell back to source-build llama.cpp on Windows."
- grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60
- exit 1
- fi
- if [ ! -f "$INFO" ]; then
- echo "::error::no UNSLOTH_PREBUILT_INFO.json at $INFO."
- ls -la "$LLAMA_DIR" || true
- exit 1
- fi
- if [ ! -f "$BIN" ]; then
- echo "::error::no llama-server.exe at $BIN."
- ls -la "$LLAMA_DIR/build/bin" || true
- exit 1
- fi
- echo "install.ps1 installed the Windows prebuilt llama.cpp:"
- cat "$INFO"
-
- - name: Add Studio shim to GITHUB_PATH
- run: |
- SHIM_DIR=~/.unsloth/studio/bin
- if [ ! -f "$SHIM_DIR/unsloth.exe" ]; then
- echo "::error::unsloth.exe shim not found at $SHIM_DIR"
- ls -la ~/.unsloth/studio/ || true
- exit 1
- fi
- cygpath -w "$SHIM_DIR" >> "$GITHUB_PATH"
-
- - name: Patch Studio venv with full typer / pydantic dep trees
- # install.ps1 runs `uv pip install --no-deps -r
- # no-torch-runtime.txt` to keep torch out of transitive
- # resolution from accelerate/peft/trl. That also drops
- # typer's and pydantic's runtime deps unless they're
- # explicitly pinned in no-torch-runtime.txt. We pin the
- # known ones (click, shellingham, annotated-doc, rich,
- # pydantic-core, annotated-types, typing-inspection, ...)
- # but typer / pydantic minor versions can introduce new
- # transitive deps that are NOT in our pin list.
- #
- # Belt-and-suspenders: re-install typer + pydantic +
- # huggingface_hub WITH their deps into the Studio venv.
- # `pip install --upgrade` only adds missing packages; it
- # never down-shifts an installed version. Cannot pull
- # torch (none of typer / pydantic / huggingface_hub depend
- # on it).
- run: |
- STUDIO_PY=~/.unsloth/studio/unsloth_studio/Scripts/python.exe
- if [ ! -f "$STUDIO_PY" ]; then
- echo "::error::Studio venv python not at $STUDIO_PY"
- ls -la ~/.unsloth/studio/ || true
- exit 1
- fi
- "$STUDIO_PY" -m pip install --upgrade typer pydantic huggingface_hub
-
- - name: First update should be a no-op (prebuilt already validated)
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- set -o pipefail
- unsloth studio update --local 2>&1 | tee logs/update.log
- if grep -q "falling back to source build" logs/update.log; then
- echo "::error::studio update fell back to source-build llama.cpp on Windows."
- grep -E "llama-prebuilt|llama.cpp" logs/update.log | tail -60
- exit 1
- fi
- if ! grep -qE "prebuilt up to date and validated|prebuilt installed and validated" logs/update.log; then
- echo "::error::no prebuilt up-to-date marker in update.log."
- grep -E "llama-prebuilt|llama.cpp" logs/update.log | tail -60
- exit 1
- fi
- echo "update path took the prebuilt fast path"
-
- - name: Second update must also be a no-op
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- set -o pipefail
- unsloth studio update --local 2>&1 | tee logs/update2.log
- grep -q "falling back to source build" logs/update2.log && {
- echo "::error::second update fell back to source build on Windows"
- tail -60 logs/update2.log; exit 1; } || true
- grep -qE "prebuilt up to date and validated|prebuilt installed and validated" logs/update2.log
- echo "second update was clean"
-
- - name: Boot Studio briefly to confirm the install is still usable
- run: |
- mkdir -p logs
- UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18891 \
- > logs/studio.log 2>&1 &
- PID=$!
- HEALTHY=""
- # Use jq (a Git Bash builtin) instead of `python -c
- # open('/tmp/health.json')` to read the saved health
- # response. Bash on windows-latest is MSYS Git Bash, which
- # resolves `/tmp/...` against the MSYS root, while the
- # python interpreter is Windows-native and resolves it
- # against the current drive's root. The two paths don't
- # agree, so python never finds the file curl just wrote.
- # jq reads through MSYS, so the path matches. Mirrors what
- # studio-windows-api-smoke.yml and the other Windows smoke
- # workflows already do.
- for i in $(seq 1 60); do
- if curl -fs http://127.0.0.1:18891/api/health > /tmp/health.json; then
- if jq -e '.status == "healthy"' /tmp/health.json >/dev/null; then
- HEALTHY=1
- break
- fi
- fi
- sleep 1
- done
- if [ -z "$HEALTHY" ]; then
- echo "Studio failed to come up after \`update\`"
- tail -200 logs/studio.log
- kill "$PID" 2>/dev/null || true
- exit 1
- fi
- kill "$PID" 2>/dev/null || true
- echo "post-update Studio /api/health OK"
-
- - name: Uninstall and verify clean
- # Round-trip through scripts/uninstall.ps1 against the default
- # install tree at %USERPROFILE%\.unsloth\studio. Catches
- # regressions where install.ps1 starts writing under a new key
- # (registry, Start Menu, %APPDATA%) and scripts/uninstall.ps1 has
- # not been updated to match. Skips gracefully if
- # scripts/uninstall.ps1 has not landed yet (lets this workflow
- # merge before #5513).
- shell: pwsh
- run: |
- New-Item -ItemType Directory -Force -Path logs | Out-Null
- if (-not (Test-Path "$PWD\scripts\uninstall.ps1")) {
- Write-Host "scripts/uninstall.ps1 not present in this tree; skipping round-trip"
- "" | Set-Content logs/uninstall.log
- exit 0
- }
- pwsh -NoProfile -File "$PWD\scripts\uninstall.ps1" *>&1 | Tee-Object -FilePath logs/uninstall.log
- $leak = 0
- foreach ($p in @(
- "$env:USERPROFILE\.unsloth\studio",
- "$env:USERPROFILE\.unsloth\studio\unsloth_studio",
- "$env:USERPROFILE\.unsloth\studio\bin\unsloth.exe"
- )) {
- if (Test-Path -LiteralPath $p) {
- Write-Host "::error::leak: $p"
- $leak++
- }
- }
- if ($leak -gt 0) { exit 1 }
- # Idempotency.
- pwsh -NoProfile -File "$PWD\scripts\uninstall.ps1" *>&1 | Select-Object -Last 5
- pwsh -NoProfile -File "$PWD\scripts\uninstall.ps1" *>&1 | Select-Object -Last 5
- Write-Host "PASS: windows install -> update -> uninstall round-trip clean"
-
- - name: Upload update logs
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: windows-studio-update-log
- path: |
- logs/install.log
- logs/update.log
- logs/update2.log
- logs/studio.log
- logs/uninstall.log
- retention-days: 7
diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.yml
deleted file mode 100644
index 599b53df1d..0000000000
--- a/.github/workflows/version-compat-ci.yml
+++ /dev/null
@@ -1,312 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-#
-# Cross-version compat canary for the four upstream packages whose
-# release cadence regularly breaks unsloth + unsloth-zoo:
-#
-# 1. vLLM (LoRA worker manager, BnB loader, cumem allocator)
-# 2. TRL / GRPO (trainer source rewriters in unsloth.models.rl*)
-# 3. PEFT (LoraConfig, get_peft_model, LoraLayer, bnb integration)
-# 4. sentence-transformers (Transformer/Pooling/Normalize, Trainer)
-# 5. bitsandbytes (Linear4bit, dequantize_4bit)
-#
-# Strategy: GitHub raw-fetch + symbol grep against every tracked
-# version (no pip install, CPU-only). When upstream renames a symbol
-# we depend on, the matching test fails BEFORE a user hits it. The
-# `main` branch entries give us a few-day lead on PyPI releases.
-#
-# Cross-references:
-# tests/vllm_compat/test_vllm_pinned_symbols.py (vLLM symbols)
-# tests/version_compat/test_trl_grpo_pinned_symbols.py
-# tests/version_compat/test_peft_pinned_symbols.py
-# tests/version_compat/test_sentence_transformers_pinned_symbols.py
-# tests/version_compat/test_bitsandbytes_pinned_symbols.py
-
-name: Version Compat CI
-
-on:
- pull_request:
- # Trigger on any unsloth source change, not just the three previously
- # named files. The symbol-existence tests verify that EVERY pinned
- # upstream reference in unsloth still resolves; a new
- # `from peft.foo import Bar` added in unsloth/kernels/whatever.py
- # is just as much a compat regression risk as one added in
- # unsloth/models/rl.py.
- paths:
- - 'unsloth/**'
- - 'tests/vllm_compat/**'
- - 'tests/version_compat/**'
- - 'pyproject.toml'
- - '.github/workflows/version-compat-ci.yml'
- schedule:
- # Daily 06:43 UTC. Catches upstream PyPI releases roughly within
- # 24 h. Off the :00 / :30 fleet-collision spots.
- - cron: '43 6 * * *'
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- vllm-pinned-symbols:
- name: vLLM pinned-symbol matrix (≥ 0.9.0 + main)
- runs-on: ubuntu-latest
- timeout-minutes: 12
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
- - name: Install pytest only
- # The test fetches from raw.githubusercontent.com and greps
- # source. No pip install of vllm / torch / transformers is
- # needed — that's the whole point of this canary.
- run: |
- python -m pip install --upgrade pip
- pip install 'pytest>=8'
- - name: Run vllm-compat suite
- env:
- # Authenticated requests get a 5000-req/h quota on raw
- # fetches; unauthenticated is 60/h and trips on the matrix.
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- python -m pytest tests/vllm_compat/test_vllm_pinned_symbols.py -v --tb=short
-
- trl-grpo-pinned-symbols:
- name: TRL / GRPO pinned-symbol matrix
- runs-on: ubuntu-latest
- timeout-minutes: 10
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
- - name: Install pytest only
- run: |
- python -m pip install --upgrade pip
- pip install 'pytest>=8'
- - name: Run trl-compat suite
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- # PYTHONPATH=. so `from tests.version_compat._fetch import …`
- # works without an editable install of unsloth itself.
- PYTHONPATH=. python -m pytest \
- tests/version_compat/test_trl_grpo_pinned_symbols.py \
- -v --tb=short
-
- peft-pinned-symbols:
- name: PEFT pinned-symbol matrix (pyproject window + main)
- runs-on: ubuntu-latest
- timeout-minutes: 8
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
- - name: Install pytest only
- run: |
- python -m pip install --upgrade pip
- pip install 'pytest>=8'
- - name: Run peft-compat suite
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- PYTHONPATH=. python -m pytest \
- tests/version_compat/test_peft_pinned_symbols.py \
- tests/version_compat/test_unsloth_zoo_save_merged_pinned_symbols.py \
- -v --tb=short
-
- st-pinned-symbols:
- name: sentence-transformers pinned-symbol matrix
- runs-on: ubuntu-latest
- timeout-minutes: 8
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
- - name: Install pytest only
- run: |
- python -m pip install --upgrade pip
- pip install 'pytest>=8'
- - name: Run sentence-transformers compat suite
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- PYTHONPATH=. python -m pytest \
- tests/version_compat/test_sentence_transformers_pinned_symbols.py \
- -v --tb=short
-
- bitsandbytes-pinned-symbols:
- name: bitsandbytes pinned-symbol matrix
- runs-on: ubuntu-latest
- timeout-minutes: 8
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
- - name: Install pytest only
- run: |
- python -m pip install --upgrade pip
- pip install 'pytest>=8'
- - name: Run bitsandbytes compat suite
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- PYTHONPATH=. python -m pytest \
- tests/version_compat/test_bitsandbytes_pinned_symbols.py \
- -v --tb=short
-
- transformers-pinned-symbols:
- name: transformers pinned-symbol matrix (4.57.6 + 5.x + main)
- runs-on: ubuntu-latest
- timeout-minutes: 12
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
- - name: Install pytest only
- run: |
- python -m pip install --upgrade pip
- pip install 'pytest>=8'
- - name: Run transformers compat suite
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- PYTHONPATH=. python -m pytest \
- tests/version_compat/test_transformers_pinned_symbols.py \
- -v --tb=short
-
- # Optional second layer: actually `pip install` ONE representative
- # version of each package and verify unsloth + unsloth-zoo modules
- # import on it under the existing CUDA spoof. CPU-only, runs on
- # ubuntu-latest. Catches the small set of breakages that the static
- # symbol check misses (e.g. import-time side effects).
- zoo-imports-under-spoof:
- name: unsloth_zoo vllm/grpo/peft/st modules import under CUDA spoof
- runs-on: ubuntu-latest
- timeout-minutes: 15
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
- path: unsloth
- - name: Clone unsloth-zoo @ main
- run: |
- # github.com occasionally 500s on the git fetch; retry so a
- # single upstream blip does not fail CI.
- for attempt in 1 2 3; do
- rm -rf "$RUNNER_TEMP/unsloth-zoo"
- if git clone --depth=1 https://github.com/unslothai/unsloth-zoo \
- "$RUNNER_TEMP/unsloth-zoo"; then
- break
- fi
- if [ "$attempt" -eq 3 ]; then
- echo "::error::git clone unsloth-zoo failed after 3 attempts"
- exit 1
- fi
- delay=$((5 * attempt))
- echo "::warning::clone failed (attempt $attempt/3), retrying in ${delay}s..."
- sleep "$delay"
- done
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
- - name: Install CPU torch + supported pkg pins
- run: |
- python -m pip install --upgrade pip
- # CPU torch (vllm/peft/st all depend on it).
- pip install --index-url https://download.pytorch.org/whl/cpu \
- 'torch>=2.4,<2.11' 'torchvision<0.26' 'torchcodec<0.10'
- # torchcodec is a hard requirement on transformers 5.x:
- # transformers/audio_utils.py:55 does
- # `importlib.metadata.version("torchcodec")` UNCONDITIONALLY,
- # which raises PackageNotFoundError on a CPU runner that
- # otherwise has no audio path -- and that error trickles up
- # through every `import unsloth_zoo.` because
- # unsloth-zoo's vision_utils transitively pulls
- # transformers.processing_utils (-> audio_utils). The 0.10
- # cap mirrors the torch 2.10 / torchvision 0.26 ABI window
- # we already pin above.
- # Ladder of supported floor versions per pyproject.toml.
- pip install \
- 'transformers>=4.56,<5.6' 'trl>=0.22,<0.26' \
- 'peft>=0.18.0' 'sentence-transformers>=5.0' \
- 'accelerate>=1.0' 'datasets>=3.4,<5' \
- 'bitsandbytes>=0.45.5' \
- sentencepiece protobuf safetensors numpy 'pytest>=8' \
- 'huggingface_hub>=0.34' tqdm packaging psutil triton Pillow
- # Editable-install both repos so the test imports the
- # checkouts (not whatever stale PyPI version pip resolved).
- pip install --no-deps -e "$RUNNER_TEMP/unsloth-zoo"
- pip install --no-deps -e ./unsloth
- - name: Run vllm_compat zoo-imports tests under spoof
- env:
- UNSLOTH_IS_PRESENT: '1'
- UNSLOTH_COMPILE_DISABLE: '1'
- PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
- run: |
- cd unsloth
- # tests/vllm_compat/test_unsloth_zoo_imports.py: narrow vllm/grpo
- # import gates (5 tests).
- # tests/vllm_compat/test_extended_module_imports.py: full sweep
- # of unsloth_zoo + unsloth.models.* modules + RL dispatch
- # table population + FastModel API surface under spoof
- # (~30 tests). Catches transformers / peft / bnb symbol pin
- # drift at module-top BEFORE any runtime call.
- PYTHONPATH=. python -m pytest \
- tests/vllm_compat/test_unsloth_zoo_imports.py \
- tests/vllm_compat/test_extended_module_imports.py \
- -v --tb=short
-
- # Daily-only: same suites but with --strict on importable upstream
- # tags. Schedule-only so PR jobs stay fast; cron tolerates a flake.
- daily-fresh-fetch:
- name: daily fresh-fetch sweep (cron only)
- if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
- runs-on: ubuntu-latest
- timeout-minutes: 20
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
- cache: 'pip'
- - name: Install pytest
- run: pip install 'pytest>=8'
- - name: Run all version-compat suites in one process (no cache)
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- PYTHONPATH=. python -m pytest \
- tests/vllm_compat/test_vllm_pinned_symbols.py \
- tests/version_compat/ \
- -v --tb=short
diff --git a/.github/workflows/wheel-smoke.yml b/.github/workflows/wheel-smoke.yml
deleted file mode 100644
index 3de3c33ca2..0000000000
--- a/.github/workflows/wheel-smoke.yml
+++ /dev/null
@@ -1,136 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-
-# Builds the PyPI wheel from the PR branch, then verifies the built wheel
-# actually contains what we expect to ship and does NOT contain the broken
-# Studio bundle that 2026.5.1 published. This is the single workflow that
-# would have blocked the 2026.5.1 release before twine upload.
-#
-# Verified locally end-to-end against this branch:
-# - python -m build produces unsloth--py3-none-any.whl in 13s
-# - wheel content sanity passes:
-# lockfile shipped, frontend dist shipped,
-# no node_modules in wheel, no bun.lock in wheel,
-# main bundle has unstable_Provider hits=1 (assistant-ui internals only).
-# - Studio backend imports cleanly from the installed wheel with the
-# lightweight dep set below.
-
-name: Wheel CI
-
-on:
- pull_request:
- paths:
- - 'pyproject.toml'
- - 'studio/**'
- - 'unsloth/**'
- - 'unsloth_cli/**'
- - '.github/workflows/wheel-smoke.yml'
- push:
- branches: [main, pip]
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-jobs:
- wheel:
- name: Wheel build + content sanity + import smoke
- runs-on: ubuntu-latest
- timeout-minutes: 15
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
-
- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '22'
-
- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: '3.12'
-
- - name: Lockfile supply-chain audit (pre-install scan)
- run: python3 scripts/lockfile_supply_chain_audit.py
-
- - name: Build frontend
- # Lifecycle scripts (esbuild native-binary postinstall, etc.) are
- # required for `vite build`. The pre-install lockfile structural
- # audit (lockfile_supply_chain_audit.py) is the practical defence
- # against the npm postinstall-dropper class -- it fires BEFORE any
- # tarball runs, on the injection pattern itself rather than an
- # advisory-DB lookup.
- run: |
- cd studio/frontend
- npm ci --no-fund --no-audit
- npm run build
-
- - name: Build wheel + sdist
- run: |
- python -m pip install --upgrade pip build
- rm -rf dist build ./*.egg-info
- python -m build
-
- - name: Wheel content sanity
- run: |
- python - <<'PY'
- import zipfile, glob, sys
- w = glob.glob("dist/unsloth-*.whl")
- if not w:
- print("FAIL: no wheel produced"); sys.exit(2)
- w = w[0]
- print(f"wheel: {w}")
- with zipfile.ZipFile(w) as z:
- n = z.namelist()
- checks = {
- "lockfile shipped": any(s.endswith("studio/frontend/package-lock.json") for s in n),
- "frontend dist shipped": any(s.endswith("studio/frontend/dist/index.html") for s in n),
- "no node_modules": not any("studio/frontend/node_modules/" in s for s in n),
- "no bun.lock": not any(s.endswith("studio/frontend/bun.lock") for s in n),
- }
- js = [s for s in n
- if "studio/frontend/dist/assets/" in s
- and s.endswith(".js")
- and "/index-" in s]
- if not js:
- print("FAIL: no main bundle index-*.js in wheel"); sys.exit(2)
- data = z.read(js[0]).decode("utf-8", "replace")
- hits = data.count("unstable_Provider:")
- print(f"main bundle: {js[0]}")
- print(f"unstable_Provider hits: {hits} (>=4 indicates 2026.5.1 regression)")
- checks["bundle has no Studio unstable_Provider call site"] = (hits < 4)
-
- print()
- for k, v in checks.items():
- print(f" [{'PASS' if v else 'FAIL'}] {k}")
- sys.exit(0 if all(checks.values()) else 1)
- PY
-
- - name: Studio backend import smoke
- # Imports `studio.backend.main:app` from the freshly-installed wheel in
- # a clean venv. This catches the class of bug that 2026.5.1 shipped with:
- # frontend dist missing, package-lock.json missing, or the wheel's Python
- # source tree broken in a way that surfaces only at app construction time.
- run: |
- python -m venv /tmp/v
- /tmp/v/bin/pip install --upgrade pip
- /tmp/v/bin/pip install -r studio/backend/requirements/studio.txt
- /tmp/v/bin/pip install \
- python-multipart aiofiles sqlalchemy cryptography \
- pyyaml jinja2 mammoth unpdf requests \
- 'numpy<3'
- /tmp/v/bin/pip install --no-deps dist/unsloth-*.whl
- # Run from /tmp so Python imports the installed package, not the source tree.
- cd /tmp
- /tmp/v/bin/python -c "from studio.backend.main import app; print('Studio backend OK:', app.title)"
-
- - name: Upload wheel on failure
- if: failure()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: unsloth-wheel
- path: dist/
- retention-days: 7
diff --git a/studio_test_kit/README.md b/studio_test_kit/README.md
new file mode 100644
index 0000000000..f065c5b831
--- /dev/null
+++ b/studio_test_kit/README.md
@@ -0,0 +1,242 @@
+# studio_test_kit
+
+A small, vendorable cookbook for driving **Unsloth Studio** end-to-end with
+Playwright. Drop these files into your project, read them top-to-bottom,
+and you'll know how to script chat flows, generate screenshots + videos,
+and run pre/post-PR comparisons.
+
+This is **not** a generic library. It is intentionally Studio-flavoured
+(model picker selectors, localStorage keys, `/api/auth/login` route,
+`unsloth studio -p` CLI, etc.). See [Customisation](#customisation) for
+the override points if you're targeting a different SPA.
+
+## What to vendor (15 files)
+
+Copy the whole `studio_test_kit/` directory. No editable install needed
+once on `sys.path`.
+
+| # | File | Purpose | Read for |
+|---|------|---------|----------|
+| 1 | `__init__.py` | Public re-exports | API surface overview |
+| 2 | `lifecycle.py` | Install Studio at a given `UNSLOTH_STUDIO_HOME` from any git ref; launch on a port; parse bootstrap password from log; wait on `/healthz`. | Two-port pre/post comparison setup |
+| 3 | `auth.py` | Backend JWT login + Playwright init-script builder that seeds the 5 `unsloth_*` localStorage keys. Convenience constructors for Gemini / OpenAI / Anthropic. | Studio's auth + provider model |
+| 4 | `ui.py` | Playwright primitives: `open_chat` async context manager (video recording + post-close rename + optional mp4 transcode), `pick_model`, `set_pill`, `send_prompt`, `wait_for_stream`, `wait_for_image`, `wait_for_text`, `extract_data_url`. | The Playwright lessons |
+| 5 | `flows.py` | High-level scenarios returning `FlowResult`: `multi_turn_chat`, `image_generation`, `tool_pills`, `vision_upload`. | Composable test flows |
+| 6 | `compose.py` | Post-processing: PIL `hstack_images`/`vstack_images`, ffmpeg `hstack_videos`/`webm_to_mp4`. | Pre/post side-by-side composition |
+| 7 | `examples/explicit_walkthrough.py` | **Start here.** Fully inline end-to-end flow with every Playwright primitive at the call site and "why" comments. No `flows.*` shortcuts. | Tutorial |
+| 8 | `examples/multi_turn.py` | Thin: 4-turn chat via `flows.multi_turn_chat` | Pattern after you know the basics |
+| 9 | `examples/image_gen.py` | Thin: Nano Banana image-gen, saves decoded PNG | Pattern |
+| 10 | `examples/tools_pills.py` | Thin: Search + Code composer pills | Pattern |
+| 11 | `examples/side_by_side.py` | Two `install_studio` + two `launch_studio` + same flow on both + ffmpeg hstack | Pre/post PR comparison |
+| 12 | `examples/__init__.py` | Empty marker | — |
+| 13 | `_self_test.py` | 13 offline regression tests (no live Studio needed) | CI / verifying the kit didn't bit-rot |
+| 14 | `_smoke_ui.py` | Live Playwright integration test against an in-process HTTP server (verifies init-script seeding, screenshot, video flush, mp4 transcode) | CI / smoke after vendoring |
+| 15 | `README.md` | This document | — |
+
+## Setup
+
+```bash
+pip install playwright httpx pillow
+playwright install chromium
+
+# Optional but recommended (used for mp4 transcode and side-by-side video):
+apt-get install ffmpeg # or: brew install ffmpeg
+```
+
+Verify the vendoring worked:
+
+```bash
+python -m studio_test_kit._self_test # 13/13 offline regression
+python -m studio_test_kit._smoke_ui # live Playwright smoke
+```
+
+## The model: how Studio authentication and providers work
+
+You need to understand this in 60 seconds before the Playwright bits
+make sense.
+
+1. **Login is REST, tokens live in localStorage.** Studio exposes
+ `POST /api/auth/login` taking `{username, password}` and returning
+ `{access_token, refresh_token}`. The SPA reads those tokens from
+ localStorage on every request — NOT from cookies. So
+ `browser_context.add_cookies()` does nothing useful for auth.
+ `auth.login()` makes the REST call; `auth.seed_init_script()` plants
+ the tokens in localStorage.
+
+2. **External providers are localStorage-driven, not server-stored.** The
+ Settings → Connections UI writes three keys:
+
+ | Key | Value |
+ |-----|-------|
+ | `unsloth_chat_external_providers` | JSON array of `{id, providerType, name, baseUrl, models}` |
+ | `unsloth_chat_external_provider_keys` | `{providerId: plaintextKey}` |
+ | `unsloth_chat_connections_enabled` | `"true"` / `"false"` |
+
+ Seeding these via `add_init_script` is exactly equivalent to typing
+ the key into the dialog manually. The frontend then RSA-encrypts the
+ plaintext key on every request using the public key from `GET
+ /api/key`, so the plaintext never goes over the wire even though it
+ sits in localStorage.
+
+3. **`add_init_script` ordering is what makes seeding work.** The init
+ script runs on every page navigation BEFORE the SPA's own JS. Plant
+ AFTER `goto` and you race the SPA's bootstrap — sometimes works,
+ sometimes silently doesn't.
+
+## Quickstart
+
+Assumes Studio is already running on `--port` (use `lifecycle.launch_studio`
+to spin it up programmatically — see `examples/side_by_side.py` for that
+pattern).
+
+```bash
+export GEMINI_API_KEY=AIza...
+
+# Tutorial: every primitive inline + commentary. Read this file first.
+python -m studio_test_kit.examples.explicit_walkthrough \
+ --port 8902 --password 'YourBootstrap!' \
+ --model gemini-2.5-flash
+
+# 4-turn conversation (uses flows.multi_turn_chat)
+python -m studio_test_kit.examples.multi_turn \
+ --port 8902 --password 'YourBootstrap!' --model gemini-2.5-flash
+
+# Image generation, saves decoded PNG
+python -m studio_test_kit.examples.image_gen \
+ --port 8902 --password 'YourBootstrap!' \
+ --model gemini-2.5-flash-image \
+ --prompt 'A red panda eating ramen in the rain'
+
+# Search + Code composer pills
+python -m studio_test_kit.examples.tools_pills \
+ --port 8902 --password 'YourBootstrap!' --model gemini-2.5-flash
+
+# Full pre/post PR comparison: installs Studio twice, drives both, composes
+python -m studio_test_kit.examples.side_by_side \
+ --pre-branch main --post-branch feat/my-thing \
+ --pre-port 8901 --post-port 8902 \
+ --model gemini-2.5-flash
+```
+
+## Programmatic flow (custom scenario)
+
+If `flows.multi_turn_chat` / `image_generation` / `tool_pills` /
+`vision_upload` don't fit your task, compose primitives:
+
+```python
+import asyncio
+from pathlib import Path
+from studio_test_kit.auth import gemini_provider, login, seed_init_script
+from studio_test_kit.ui import (
+ open_chat, pick_model, send_prompt, set_pill, wait_for_stream,
+ wait_for_image, extract_data_url,
+)
+
+
+async def run():
+ auth = await login("http://127.0.0.1:8902", "unsloth", "YourBootstrap!")
+ init = seed_init_script(
+ auth,
+ [gemini_provider(api_key="AIza...", models=["gemini-2.5-flash-image"])],
+ )
+ async with open_chat(
+ "http://127.0.0.1:8902",
+ init_scripts=[init],
+ video_dir=Path("out/video"),
+ video_name="my_run",
+ transcode_mp4=True,
+ headless=True,
+ ) as sp:
+ await pick_model(sp, "gemini-2.5-flash-image")
+ await set_pill(sp, "images", on=True)
+ await send_prompt(sp, "Draw a red panda")
+ data_url = await wait_for_image(sp)
+ raw = await extract_data_url(data_url)
+ Path("out/red_panda.png").write_bytes(raw)
+ print(f"video: {sp.video_webm} mp4: {sp.video_mp4}")
+
+
+asyncio.run(run())
+```
+
+`sp.video_webm` / `sp.video_mp4` are populated AFTER the `async with`
+exits (Playwright finalises on `context.close()`).
+
+## Pitfalls (read once, save yourself a day)
+
+Every one of these bit me; the kit defends against all of them.
+
+| # | Pitfall | Fix in the kit |
+|---|---|---|
+| 1 | `wait_until="networkidle"` never fires on Studio because the chat thread holds long-lived SSE/WebSocket; Playwright deadlines. | `open_chat` uses `wait_until="domcontentloaded"` plus an explicit `form:has(textarea) textarea` visibility wait. |
+| 2 | `:has-text("gemini-2.5-flash")` substring-matches `gemini-2.5-flash-image`. | `pick_model` uses `get_by_role("option", name=re.compile(rf"^{re.escape(model_id)}$"))`. |
+| 3 | A bare `button:has-text("Search")` clicks the LEFT-SIDEBAR Search-history button, not the composer pill. | All UI selectors are scoped: `form:has(textarea) button:has-text(...)`. |
+| 4 | Stop button hides BEFORE the final `]![]()
` paints; screenshots taken on stop-hidden miss the image. | `wait_for_image` polls the DOM via `page.evaluate` for an `![]()
` whose `src` starts with `data:image/{png,jpeg,webp}` and validates decoded bytes. |
+| 5 | `page.type(...)` simulates ~50 ms per keystroke; long prompts take seconds. | `send_prompt` uses `box.fill(...)` (instant paste). |
+| 6 | `page.video.path()` read inside the `async with` returns a tempfile that won't survive context close. | Read `sp.video_webm` AFTER the `async with` exits. |
+| 7 | Video stalls if you `await page.close()` before `context.close()`. | `open_chat` closes context then browser in a `finally`. |
+| 8 | Init script planted AFTER `goto` races the SPA's bootstrap; sometimes works, sometimes silently doesn't. | `seed_init_script` is wired through `context.add_init_script(...)` BEFORE `new_page`/`goto`. |
+| 9 | Cookies: Studio reads its JWT from localStorage, not cookies. | `auth.seed_init_script` plants `unsloth_auth_token` / `unsloth_refresh_token` in localStorage. |
+| 10 | Playwright's `record_video_dir` writes `page@.webm`. Parallel tests sharing a video dir can rename each other's recordings. | `open_chat` snapshots pre-existing webms on entry and renames only files that appeared during this session. |
+| 11 | Video finalisation skipped if the test body raises (the case you most want a recording). | `open_chat` wraps the playwright lifecycle in `try/finally`. |
+| 12 | Shared deadline between password discovery and `/healthz`: a quiet log starves the healthz check and raises a spurious TimeoutError. | `launch_studio` takes independent `password_timeout_s` (default 30s) and `healthz_timeout_s` (default 180s). |
+| 13 | Password log regex `[:\s]+` greedily backtracks on `password = secret`, capturing `"="` as the password. | `_PW_RE` uses an explicit `[:=]?` separator with mandatory `\s+` before the value. |
+| 14 | `:has-text` substring on `data:image/` matches SVG tracking pixels. | `wait_for_image(mime_prefixes=("data:image/png", "data:image/jpeg", "data:image/webp"))` excludes SVG by default. |
+
+## Things I avoided (and why)
+
+| Avoided | Why |
+|---|---|
+| WebKit / Firefox launchers | Studio's RSA-encrypt-on-frontend path uses Web Crypto; only reliable on Chromium for now. |
+| `wait_for_selector(...)` | Deprecated in Playwright 1.40+. Use `locator(...).wait_for(state=...)`. |
+| `expect_response()` for streaming | SSE doesn't end with a discrete `Response`; DOM polling is sturdier. |
+| `evaluate_handle` | Returns a JSHandle you must `.dispose()`. `evaluate` returns serialised JSON, which is enough for selector probes. |
+| `playwright codegen` selectors | Produces brittle `.locator("...:nth=2")`; hand-written form-scoped role/text predicates are far more durable. |
+| Cookie-based auth | Studio reads JWT from localStorage. Cookies do nothing. |
+| Page video at `await page.video.path()` mid-block | The path isn't valid until `context.close()`; reading early gives you a tempfile that gets unlinked. |
+| `screencast_*` / CDP video | Lower-level, no auto-flush, no rename. Context-level `record_video_dir` is the only path that just works. |
+
+## Customisation
+
+This kit is tuned to **Unsloth Studio**. If you're driving a similar
+FastAPI + React chat app with a different shape, here are the override
+points:
+
+| What | File:line | How to swap |
+|---|---|---|
+| Login URL / payload shape | `auth.py:login` | Replace `POST /api/auth/login` with your endpoint; ensure `StudioAuth(access_token, refresh_token, base_url)` is constructible |
+| localStorage key names | `auth.py:seed_init_script` (5 keys) | Edit the `payload = {...}` dict; the rest of the kit only consumes `access_token` directly |
+| RSA-on-frontend assumption | `auth.py:5-15` docstring | If your app sends keys plain or via header, modify `seed_init_script` to set whatever your SPA reads |
+| Composer scope selector | `ui.py:set_pill / send_prompt` use `form:has(textarea) ...` | Change the prefix if your composer isn't an HTML `