[CI] Add E2E CI for Tinker API with SkyRLTrainBackend

.github/workflows/gpu_e2e_ci_tinker.yaml

@@ -0,0 +1,47 @@
+name: SkyRL-GPU-E2E-CI-Tinker
+
+on:
+  schedule:
+    - cron: '17 8 * * *'   # Every day at 08:17 UTC, offset from the other nightly E2Es
+  workflow_dispatch:
+
+permissions:
+  checks: write   # for status checks to appear
+  contents: read
+
+jobs:
+
+  skyrl_gpu_e2e_test_tinker:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: .
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        # This is the version of the action for setting up Python, not the Python version.
+        uses: actions/setup-python@v5
+        with:
+          # Semantic version range syntax or exact version of a Python version
+          python-version: '3.12'
+          cache: 'pip'
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          activate-environment: true
+      - name: Install basic dependencies
+        run: uv pip install anyscale==0.24.79 typer==0.9.0
+      - name: Install envsubst
+        run: sudo apt-get update && sudo apt-get install -y gettext-base
+      - name: Tinker rl_loop convergence test
+        env:
+          ANYSCALE_CLI_TOKEN: ${{ secrets.ANYSCALE_CLI_TOKEN }}
+          ANYSCALE_HOST: https://console.anyscale.com
+          WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
+        run: |
+          envsubst < ci/anyscale_gpu_e2e_test_tinker.yaml > ci/anyscale_gpu_e2e_test_tinker_envsubst.yaml
+          anyscale job submit -f ci/anyscale_gpu_e2e_test_tinker_envsubst.yaml --timeout 12000
+          anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name skyrl-train-gpu-e2e-test-tinker --timeout 12000
+          rm -f ci/anyscale_gpu_e2e_test_tinker_envsubst.yaml

.github/workflows/gpu_e2e_ci_tinker_fully_async.yaml

@@ -0,0 +1,47 @@
+name: SkyRL-GPU-E2E-CI-Tinker-Fully-Async
+
+on:
+  schedule:
+    - cron: '29 8 * * *'   # Every day at 08:29 UTC, offset from the other nightly E2Es
+  workflow_dispatch:
+
+permissions:
+  checks: write   # for status checks to appear
+  contents: read
+
+jobs:
+
+  skyrl_gpu_e2e_test_tinker_fully_async:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: .
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        # This is the version of the action for setting up Python, not the Python version.
+        uses: actions/setup-python@v5
+        with:
+          # Semantic version range syntax or exact version of a Python version
+          python-version: '3.12'
+          cache: 'pip'
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          activate-environment: true
+      - name: Install basic dependencies
+        run: uv pip install anyscale==0.24.79 typer==0.9.0
+      - name: Install envsubst
+        run: sudo apt-get update && sudo apt-get install -y gettext-base
+      - name: Tinker fully-async rl_loop convergence test
+        env:
+          ANYSCALE_CLI_TOKEN: ${{ secrets.ANYSCALE_CLI_TOKEN }}
+          ANYSCALE_HOST: https://console.anyscale.com
+          WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
+        run: |
+          envsubst < ci/anyscale_gpu_e2e_test_tinker_fully_async.yaml > ci/anyscale_gpu_e2e_test_tinker_fully_async_envsubst.yaml
+          anyscale job submit -f ci/anyscale_gpu_e2e_test_tinker_fully_async_envsubst.yaml --timeout 12000
+          anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name skyrl-train-gpu-e2e-test-tinker-fully-async --timeout 12000
+          rm -f ci/anyscale_gpu_e2e_test_tinker_fully_async_envsubst.yaml

ci/anyscale_gpu_e2e_test_tinker.yaml

@@ -0,0 +1,10 @@
+name: skyrl-train-gpu-e2e-test-tinker
+entrypoint: bash ci/gpu_e2e_test_run_tinker.sh
+image_uri: novaskyai/skyrl-train-ray-2.51.1-py3.12-cu12.8 # (Optional) Exclusive with `containerfile`.
+cloud: sky-anyscale-aws-us-east-1
+ray_version: "2.51.1"
+compute_config: l4_ci
+working_dir: . # (Optional) Use current working directory "." as the working_dir. Can be any local path or remote .zip file in cloud storage.
+env_vars:
+  WANDB_API_KEY: $WANDB_API_KEY
+max_retries: 1 # (Optional) Maximum number of times the job will be retried before being marked failed. Defaults to `1`.

ci/anyscale_gpu_e2e_test_tinker_fully_async.yaml

@@ -0,0 +1,10 @@
+name: skyrl-train-gpu-e2e-test-tinker-fully-async
+entrypoint: bash ci/gpu_e2e_test_run_tinker_fully_async.sh
+image_uri: novaskyai/skyrl-train-ray-2.51.1-py3.12-cu12.8 # (Optional) Exclusive with `containerfile`.
+cloud: sky-anyscale-aws-us-east-1
+ray_version: "2.51.1"
+compute_config: l4_ci
+working_dir: . # (Optional) Use current working directory "." as the working_dir. Can be any local path or remote .zip file in cloud storage.
+env_vars:
+  WANDB_API_KEY: $WANDB_API_KEY
+max_retries: 1 # (Optional) Maximum number of times the job will be retried before being marked failed. Defaults to `1`.

ci/gpu_e2e_test_run_tinker.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+bash tests/train/gpu_e2e_test/gsm8k_tinker.sh

ci/gpu_e2e_test_run_tinker_fully_async.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+bash tests/train/gpu_e2e_test/gsm8k_tinker_fully_async.sh

tests/train/gpu_e2e_test/gsm8k_tinker.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+RUN_NAME="run_$(date +%Y%m%d%H)"
+PROJECT_NAME="gsm8k_tinker_ci"
+SCRIPT_DIR=$(dirname $(realpath $0))
+SKYRL_REPO_ROOT=$(realpath "$SCRIPT_DIR/../../..")
+LOG_DIR="$HOME/tinker_logs/$RUN_NAME"
+mkdir -p "$LOG_DIR"
+
+# TODO: tighten thresholds after 3-5 nightly runs (5% allowance from min observed),
+# matching the convention in gsm8k_colocate.sh.
+REWARD_MIN_VALUE=0.0
+
+BACKEND_CONFIG='{"trainer.placement.colocate_all": true, "trainer.placement.policy_num_gpus_per_node": 4, "trainer.micro_forward_batch_size_per_gpu": 8, "trainer.micro_train_batch_size_per_gpu": 8, "generator.inference_engine.num_engines": 4, "generator.inference_engine.tensor_parallel_size": 1, "generator.inference_engine.backend": "vllm", "generator.inference_engine.run_engines_locally": true, "generator.inference_engine.weight_sync_backend": "nccl", "generator.inference_engine.async_engine": true, "generator.inference_engine.gpu_memory_utilization": 0.8, "generator.batched": true}'
+
+# Start tinker server in its own process group so we can clean up the engine subprocess too.
+setsid uv run --extra tinker --extra fsdp -m skyrl.tinker.api \
+  --base-model "Qwen/Qwen3-0.6B" --backend fsdp --port 8000 \
+  --backend-config "$BACKEND_CONFIG" >"$LOG_DIR/server.log" 2>&1 &
+SERVER_PID=$!
+trap 'kill -TERM -- -$SERVER_PID 2>/dev/null || true; sleep 5; kill -KILL -- -$SERVER_PID 2>/dev/null || true' EXIT
+
+deadline=$(( $(date +%s) + 1800 ))
+until curl -sSf http://localhost:8000/docs >/dev/null 2>&1; do
+  if (( $(date +%s) > deadline )); then
+    echo "Tinker server did not become ready within 30 minutes" >&2
+    tail -n 200 "$LOG_DIR/server.log" >&2 || true
+    exit 1
+  fi
+  if ! kill -0 $SERVER_PID 2>/dev/null; then
+    echo "Tinker server exited early" >&2
+    tail -n 200 "$LOG_DIR/server.log" >&2 || true
+    exit 1
+  fi
+  sleep 5
+done
+
+COOKBOOK_DIR="$HOME/tinker-cookbook"
+[ -d "$COOKBOOK_DIR" ] || git clone --depth 1 https://github.com/thinking-machines-lab/tinker-cookbook.git "$COOKBOOK_DIR"
+
+# math_rl.train builds on tinker_cookbook/rl/train.py and exposes wandb_project /
+# wandb_name natively, so we get the same wandb-driven flow as the other E2E
+# nightlies (no client-side metrics publisher needed).
+cd "$COOKBOOK_DIR"
+TINKER_API_KEY=tml-dummy uv run --extra math-rl --extra wandb --with tinker --with datasets --with torch \
+  python -m tinker_cookbook.recipes.math_rl.train \
+  base_url=http://localhost:8000 \
+  model_name="Qwen/Qwen3-0.6B" \
+  env=gsm8k \
+  log_path="$LOG_DIR" \
+  groups_per_batch=512 \
+  group_size=4 \
+  max_tokens=512 \
+  max_steps=14 \
+  eval_every=10000 \
+  save_every=10000 \
+  wandb_project="$PROJECT_NAME" \
+  wandb_name="$RUN_NAME" \
+  behavior_if_log_dir_exists=delete
+
+cd "$SKYRL_REPO_ROOT"
+uv run --isolated --extra fsdp "$SCRIPT_DIR/get_summary.py" \
+  --run_name "$RUN_NAME" --project_name "$PROJECT_NAME" \
+  --asserts "env/all/reward/total >= $REWARD_MIN_VALUE"

tests/train/gpu_e2e_test/gsm8k_tinker_fully_async.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+RUN_NAME="run_$(date +%Y%m%d%H)"
+PROJECT_NAME="gsm8k_tinker_fully_async_ci"
+SCRIPT_DIR=$(dirname $(realpath $0))
+SKYRL_REPO_ROOT=$(realpath "$SCRIPT_DIR/../../..")
+LOG_DIR="$HOME/tinker_fully_async_logs/$RUN_NAME"
+mkdir -p "$LOG_DIR"
+
+# TODO: tighten thresholds after 3-5 nightly runs (5% allowance from min observed),
+# matching the convention in gsm8k_colocate.sh.
+REWARD_MIN_VALUE=0.0
+
+# Non-colocated layout: 2 GPUs for the trainer (FSDP policy) and 2 GPUs for the
+# inference engines (vLLM). colocate_all=false keeps training and inference on
+# disjoint GPUs so they can run concurrently with the cookbook's async loop.
+BACKEND_CONFIG='{"trainer.placement.colocate_all": false, "trainer.placement.policy_num_gpus_per_node": 2, "trainer.micro_forward_batch_size_per_gpu": 8, "trainer.micro_train_batch_size_per_gpu": 8, "generator.inference_engine.num_engines": 2, "generator.inference_engine.tensor_parallel_size": 1, "generator.inference_engine.backend": "vllm", "generator.inference_engine.run_engines_locally": true, "generator.inference_engine.weight_sync_backend": "nccl", "generator.inference_engine.async_engine": true, "generator.inference_engine.gpu_memory_utilization": 0.8, "generator.batched": true}'
+
+# Start tinker server in its own process group so we can clean up the engine subprocess too.
+setsid uv run --extra tinker --extra fsdp -m skyrl.tinker.api \
+  --base-model "Qwen/Qwen3-0.6B" --backend fsdp --port 8000 \
+  --backend-config "$BACKEND_CONFIG" >"$LOG_DIR/server.log" 2>&1 &
+SERVER_PID=$!
+trap 'kill -TERM -- -$SERVER_PID 2>/dev/null || true; sleep 5; kill -KILL -- -$SERVER_PID 2>/dev/null || true' EXIT
+
+deadline=$(( $(date +%s) + 1800 ))
+until curl -sSf http://localhost:8000/docs >/dev/null 2>&1; do
+  if (( $(date +%s) > deadline )); then
+    echo "Tinker server did not become ready within 30 minutes" >&2
+    tail -n 200 "$LOG_DIR/server.log" >&2 || true
+    exit 1
+  fi
+  if ! kill -0 $SERVER_PID 2>/dev/null; then
+    echo "Tinker server exited early" >&2
+    tail -n 200 "$LOG_DIR/server.log" >&2 || true
+    exit 1
+  fi
+  sleep 5
+done
+
+COOKBOOK_DIR="$HOME/tinker-cookbook"
+[ -d "$COOKBOOK_DIR" ] || git clone --depth 1 https://github.com/thinking-machines-lab/tinker-cookbook.git "$COOKBOOK_DIR"
+
+# math_rl.train builds on tinker_cookbook/rl/train.py and exposes wandb_project /
+# wandb_name natively, so we get the same wandb-driven flow as the other E2E
+# nightlies (no client-side metrics publisher needed).
+# max_steps_off_policy enables the cookbook's async mode: sampling and training
+# run concurrently and samples up to N steps stale are accepted into the batch.
+cd "$COOKBOOK_DIR"
+TINKER_API_KEY=tml-dummy uv run --extra math-rl --extra wandb --with tinker --with datasets --with torch \
+  python -m tinker_cookbook.recipes.math_rl.train \
+  base_url=http://localhost:8000 \
+  model_name="Qwen/Qwen3-0.6B" \
+  env=gsm8k \
+  log_path="$LOG_DIR" \
+  groups_per_batch=512 \
+  group_size=4 \
+  max_tokens=512 \
+  max_steps=14 \
+  max_steps_off_policy=4 \
+  eval_every=10000 \
+  save_every=10000 \
+  wandb_project="$PROJECT_NAME" \
+  wandb_name="$RUN_NAME" \
+  behavior_if_log_dir_exists=delete
+
+cd "$SKYRL_REPO_ROOT"
+uv run --isolated --extra fsdp "$SCRIPT_DIR/get_summary.py" \
+  --run_name "$RUN_NAME" --project_name "$PROJECT_NAME" \
+  --asserts "env/all/reward/total >= $REWARD_MIN_VALUE"

tests/train/gpu_e2e_test/official_tinker_vs_skyrl/gsm8k_official_tinker_qwen3-8b_baseline.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+# -----------------------------------------------------------------------------
+# NOT PART OF CI.
+#
+# One-off comparison script: SkyRL's local tinker server vs. the hosted Tinker
+# API, both training Qwen3-8B on GSM8K under the colocated layout. Used to
+# generate side-by-side baselines in wandb; not referenced from any workflow
+# or ci/*.yaml. Hyperparameters follow the tinker-cookbook math_rl README
+# recommendation for Qwen3-8B (group_size=16, groups_per_batch=64,
+# learning_rate=2e-5). max_steps stays low because this is a smoke-style
+# convergence check, not a full training run.
+# -----------------------------------------------------------------------------
+set -euo pipefail
+
+RUN_NAME="${RUN_NAME:-skyrl_baseline_Qwen3-8B}"
+PROJECT_NAME="${PROJECT_NAME:-gsm8k_tinker_ci}"
+MODEL_NAME="${MODEL_NAME:-Qwen/Qwen3-8B}"
+SCRIPT_DIR=$(dirname $(realpath $0))
+SKYRL_REPO_ROOT=$(realpath "$SCRIPT_DIR/../../..")
+LOG_DIR="$HOME/tinker_logs/$RUN_NAME"
+mkdir -p "$LOG_DIR"
+
+REWARD_MIN_VALUE=0.0
+
+BACKEND_CONFIG='{"trainer.placement.colocate_all": true, "trainer.placement.policy_num_gpus_per_node": 4, "trainer.micro_forward_batch_size_per_gpu": 4, "trainer.micro_train_batch_size_per_gpu": 4, "generator.inference_engine.num_engines": 4, "generator.inference_engine.tensor_parallel_size": 1, "generator.inference_engine.backend": "vllm", "generator.inference_engine.run_engines_locally": true, "generator.inference_engine.weight_sync_backend": "nccl", "generator.inference_engine.async_engine": true, "generator.inference_engine.gpu_memory_utilization": 0.8, "generator.batched": true}'
+
+# Start tinker server in its own process group so we can clean up the engine subprocess too.
+setsid uv run --extra tinker --extra fsdp -m skyrl.tinker.api \
+  --base-model "$MODEL_NAME" --backend fsdp --port 8000 \
+  --backend-config "$BACKEND_CONFIG" >"$LOG_DIR/server.log" 2>&1 &
+SERVER_PID=$!
+trap 'kill -TERM -- -$SERVER_PID 2>/dev/null || true; sleep 5; kill -KILL -- -$SERVER_PID 2>/dev/null || true' EXIT
+
+deadline=$(( $(date +%s) + 1800 ))
+until curl -sSf http://localhost:8000/docs >/dev/null 2>&1; do
+  if (( $(date +%s) > deadline )); then
+    echo "Tinker server did not become ready within 30 minutes" >&2
+    tail -n 200 "$LOG_DIR/server.log" >&2 || true
+    exit 1
+  fi
+  if ! kill -0 $SERVER_PID 2>/dev/null; then
+    echo "Tinker server exited early" >&2
+    tail -n 200 "$LOG_DIR/server.log" >&2 || true
+    exit 1
+  fi
+  sleep 5
+done
+
+COOKBOOK_DIR="$HOME/tinker-cookbook"
+[ -d "$COOKBOOK_DIR" ] || git clone --depth 1 https://github.com/thinking-machines-lab/tinker-cookbook.git "$COOKBOOK_DIR"
+
+cd "$COOKBOOK_DIR"
+TINKER_API_KEY=tml-dummy uv run --extra math-rl --extra wandb --with tinker --with datasets --with torch \
+  python -m tinker_cookbook.recipes.math_rl.train \
+  base_url=http://localhost:8000 \
+  model_name="$MODEL_NAME" \
+  env=gsm8k \
+  log_path="$LOG_DIR" \
+  groups_per_batch=64 \
+  group_size=16 \
+  learning_rate=2e-5 \
+  max_tokens=512 \
+  max_steps=14 \
+  eval_every=10000 \
+  save_every=10000 \
+  wandb_project="$PROJECT_NAME" \
+  wandb_name="$RUN_NAME" \
+  behavior_if_log_dir_exists=delete
+
+cd "$SKYRL_REPO_ROOT"
+uv run --isolated --extra fsdp "$SCRIPT_DIR/get_summary.py" \
+  --run_name "$RUN_NAME" --project_name "$PROJECT_NAME" \
+  --asserts "env/all/reward/total >= $REWARD_MIN_VALUE"