feat(flydsl): sglang-kimi-k26-flydsl-mi355x eval instance

instances/primus-qwen3-30b-mfu/Dockerfile

@@ -0,0 +1,65 @@
+# Primus Qwen3-30B-A3B MoE pretraining MFU eval (Issue #1).
+#
+# Base image is the locally-built primus-mi355x-flat:v1, which already
+# has /workspace/primus_train/Primus + Primus-Turbo + the triton 3.4.0
+# patches applied. We layer two things on top:
+#
+#  1. The canonical bench wrapper / test harness / task description.
+#  2. uv (the Python package manager) at /root/.local/bin so the
+#     kimi-cli runtime can `source $HOME/.local/bin/env` on container
+#     start — without this the executor hits "No such file or
+#     directory" and trials exit 137 immediately.
+FROM primus-mi355x-flat:v1
+
+WORKDIR /workspace
+
+# Install uv with its activation script so kimi-cli wheel install works.
+RUN mkdir -p /tmp && \
+    curl -LsSf https://astral.sh/uv/install.sh | TMPDIR=/tmp sh && \
+    test -x /root/.local/bin/uv && \
+    test -f /root/.local/bin/env
+
+# Overlay the canonical bench wrapper, test harness, and task description.
+COPY bench_mfu.sh /workspace/bench_mfu.sh
+RUN chmod +x /workspace/bench_mfu.sh
+
+COPY test_harness.py /workspace/test_harness.py
+COPY task_description.md /workspace/task_description.md
+
+# Replace the original detect_interface.sh (which depends on `ip`,
+# unavailable in this slim base) with a /proc-based detector. Works
+# in any container without iproute2.
+RUN cat > /workspace/detect_interface.sh <<'EOF' && chmod +x /workspace/detect_interface.sh
+#!/bin/bash
+# Find the interface that handles the default route (no `ip` dependency).
+IFACE=$(awk '$2 == "00000000" { print $1; exit }' /proc/net/route 2>/dev/null)
+if [ -z "$IFACE" ]; then
+    # Fallback: first non-loopback / non-docker interface from /sys.
+    for cand in $(ls /sys/class/net/); do
+        case "$cand" in
+            lo|docker*|veth*|br-*|tailscale*|usb*) continue ;;
+        esac
+        if [ -d "/sys/class/net/$cand" ]; then IFACE="$cand"; break; fi
+    done
+fi
+if [ -n "$IFACE" ]; then
+    echo "export GLOO_SOCKET_IFNAME=$IFACE"
+    echo "export NCCL_SOCKET_IFNAME=$IFACE"
+fi
+EOF
+
+# gfx950 (MI355X) torch HIP runtime needs these set for kernel
+# dispatch. Setting them here too (not just task.yaml env) so bare
+# `docker run` works for ad-hoc smoke tests.
+ENV PYTORCH_ROCM_ARCH=gfx950 \
+    AITER_ROCM_ARCH="gfx942;gfx950" \
+    HSA_NO_SCRATCH_RECLAIM=1 \
+    HIP_FORCE_DEV_KERNARG=1 \
+    LD_LIBRARY_PATH=/opt/rocm/lib:/usr/local/lib: \
+    PATH=/opt/rocm/llvm/bin:/opt/venv/bin:/opt/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+# Seed bench_config.env with PATH so verifications source it cleanly.
+# bench_mfu.sh auto-detects GLOO/NCCL socket interfaces at runtime.
+RUN echo 'export PATH="/opt/venv/bin:$PATH"' > /workspace/bench_config.env
+
+CMD ["sleep", "infinity"]

instances/primus-qwen3-30b-mfu/bench_mfu.sh

@@ -0,0 +1,125 @@
+#!/bin/bash
+set -euo pipefail
+
+# Auto-detect the host network interface for Gloo/NCCL sockets if the
+# bench_config.env doesn't already pin one. Without GLOO_SOCKET_IFNAME
+# the Megatron distributed init fails fast (3-5s) before training
+# starts and Phase 1 / executor trials can't get a real metric.
+if [ -f /workspace/bench_config.env ]; then
+    echo '>>> Loading bench_config.env overrides'
+    source /workspace/bench_config.env
+fi
+if [ -z "${GLOO_SOCKET_IFNAME:-}" ] && [ -x /workspace/detect_interface.sh ]; then
+    echo '>>> Auto-detecting GLOO_SOCKET_IFNAME via /workspace/detect_interface.sh'
+    eval "$(/workspace/detect_interface.sh)"
+fi
+echo ">>> GLOO_SOCKET_IFNAME=${GLOO_SOCKET_IFNAME:-<unset>} NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-<unset>}"
+
+cd /workspace/primus_train/Primus
+
+echo '=== Starting Qwen3-30B-A3B MFU Benchmark ==='
+echo "Timestamp: $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+
+TRAIN_ITERS="${TRAIN_ITERS:-10}"
+MICRO_BATCH_SIZE="${MICRO_BATCH_SIZE:-1}"
+GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-8}"
+SEQ_LENGTH="${SEQ_LENGTH:-8192}"
+MAX_POSITION_EMBEDDINGS="${MAX_POSITION_EMBEDDINGS:-8192}"
+EP_SIZE="${EP_SIZE:-8}"
+RECOMPUTE_NUM_LAYERS="${RECOMPUTE_NUM_LAYERS:-5}"
+RECOMPUTE_GRANULARITY="${RECOMPUTE_GRANULARITY:-full}"
+RECOMPUTE_METHOD="${RECOMPUTE_METHOD:-block}"
+TURBO_DEEPEP_NUM_CU="${TURBO_DEEPEP_NUM_CU:-80}"
+TURBO_SYNC_FREE_MOE_STAGE="${TURBO_SYNC_FREE_MOE_STAGE:-1}"
+EXTRA_FLAGS="${EXTRA_FLAGS:-}"
+
+echo ">>> Config: mbs=${MICRO_BATCH_SIZE}, gbs=${GLOBAL_BATCH_SIZE}, seq=${SEQ_LENGTH}, EP=${EP_SIZE}"
+echo ">>> Recompute: ${RECOMPUTE_NUM_LAYERS} layers, ${RECOMPUTE_GRANULARITY}/${RECOMPUTE_METHOD}"
+echo ">>> Train iters: ${TRAIN_ITERS}"
+
+OUTPUT=$(./primus-cli direct \
+  -- train pretrain --config examples/megatron/configs/MI355X/qwen3_30B_A3B-BF16-pretrain.yaml \
+  --train_iters ${TRAIN_ITERS} \
+  --micro_batch_size ${MICRO_BATCH_SIZE} \
+  --global_batch_size ${GLOBAL_BATCH_SIZE} \
+  --seq_length ${SEQ_LENGTH} \
+  --max_position_embeddings ${MAX_POSITION_EMBEDDINGS} \
+  --expert_model_parallel_size ${EP_SIZE} \
+  --mock_data True \
+  --disable_last_saving True \
+  --moe_use_legacy_grouped_gemm True \
+  --use_turbo_grouped_mlp True \
+  --use_turbo_attention True \
+  --enable_primus_turbo True \
+  --use_turbo_deepep True \
+  --turbo_deepep_num_cu ${TURBO_DEEPEP_NUM_CU} \
+  --turbo_sync_free_moe_stage ${TURBO_SYNC_FREE_MOE_STAGE} \
+  --enable_experimental True \
+  --apply_rope_fusion True \
+  --cross_entropy_fusion_impl te \
+  --cross_entropy_loss_fusion True \
+  --use_precision_aware_optimizer True \
+  --main_grads_dtype bf16 \
+  --exp_avg_dtype bf16 \
+  --exp_avg_sq_dtype bf16 \
+  --recompute_num_layers ${RECOMPUTE_NUM_LAYERS} \
+  --recompute_granularity ${RECOMPUTE_GRANULARITY} \
+  --recompute_method ${RECOMPUTE_METHOD} \
+  --disable_wandb True \
+  --disable_tensorboard True \
+  ${EXTRA_FLAGS} 2>&1) || true
+
+echo "$OUTPUT"
+echo "$OUTPUT" > /workspace/bench_output.log
+
+# Extract per-iteration instant TFLOP/s/GPU from the Megatron log.
+# Format: "throughput per GPU (TFLOP/s/GPU): <instant>/<running_avg>"
+# We extract <instant> (the number before the slash).
+TFLOPS_VALUES=$(echo "$OUTPUT" | grep -oP 'throughput per GPU \(TFLOP/s/GPU\):\s*\K[\d.]+' || true)
+
+if [ -z "$TFLOPS_VALUES" ]; then
+    # Fallback: try "tflops/gpu:" pattern
+    TFLOPS_VALUES=$(echo "$OUTPUT" | grep -oP 'tflops/gpu:\s*\K[\d.]+' || true)
+fi
+
+if [ -z "$TFLOPS_VALUES" ]; then
+    TFLOPS_VALUES=$(echo "$OUTPUT" | grep -oP 'TFLOP/s/GPU[):\s]*\K[\d.]+' || true)
+fi
+
+if [ -z "$TFLOPS_VALUES" ]; then
+    echo 'ERROR: Could not extract TFLOP/s/GPU from output'
+    echo 'TFLOPS_PER_GPU: 0'
+    echo 'METRIC: 0'
+    exit 1
+fi
+
+readarray -t VALUES <<< "$TFLOPS_VALUES"
+TOTAL_ITERS=${#VALUES[@]}
+echo "=== Found $TOTAL_ITERS iteration TFLOP/s values ==="
+
+SKIP=2
+SUM=0
+COUNT=0
+for i in "${!VALUES[@]}"; do
+    val="${VALUES[$i]}"
+    echo "  Iter $((i+1)): ${val} TFLOP/s/GPU"
+    if [ "$i" -ge "$SKIP" ]; then
+        SUM=$(python3 -c "print($SUM + $val)")
+        COUNT=$((COUNT + 1))
+    fi
+done
+
+if [ "$COUNT" -eq 0 ]; then
+    echo 'ERROR: Not enough iterations after skipping warmup'
+    echo 'TFLOPS_PER_GPU: 0'
+    echo 'METRIC: 0'
+    exit 1
+fi
+
+AVG=$(python3 -c "print(round($SUM / $COUNT, 2))")
+echo ''
+echo "=== Steady-state average (iters $((SKIP+1))-${TOTAL_ITERS}): ${AVG} TFLOP/s/GPU ==="
+# Emit both the domain-specific label and the generic METRIC label so
+# existing task.yaml regexes and newer canonical wrappers both work.
+echo "TFLOPS_PER_GPU: ${AVG}"
+echo "METRIC: ${AVG}"

instances/primus-qwen3-30b-mfu/metadata.json

@@ -0,0 +1,19 @@
+{
+  "name": "primus-qwen3-30b-mfu",
+  "category": "optimize",
+  "difficulty": "hard",
+  "issue_url": "https://github.com/amdpilot-org/Primus/issues/1",
+  "repo": "amdpilot-org/Primus",
+  "branch": "mfu-optimization",
+  "primus_commit": "e50a78b",
+  "primus_turbo_commit": "3cd482d",
+  "metric_name": "TFLOPS_PER_GPU",
+  "metric_direction": "higher",
+  "expected_baseline": 300.0,
+  "tolerance": 0.07,
+  "gpu_shape": "8xMI355X",
+  "requires_gpu": true,
+  "gpu_count": 8,
+  "gpu_arch": "mi355x",
+  "purpose": "A/B target for the Phase 1 baseline reproduction agent (feat/phase1-baseline-opus)."
+}

instances/primus-qwen3-30b-mfu/task.yaml

@@ -0,0 +1,109 @@
+name: primus-qwen3-30b-mfu
+type: optimize
+repo: https://github.com/amdpilot-org/Primus.git
+repo_branch: mfu-optimization
+base_image: primus-qwen3-30b-mfu-base:v1
+
+# Phase 1 + supervisor + nudge run on Opus via the AMD Gateway. The
+# executor (kimi-cli ralph mode) runs on the K2.6 sglang endpoint.
+frontier_model: true
+phase1_baseline: true
+phase1_max_turns: 45
+phase1_budget_minutes: 90
+
+# Tag + push the Phase 1 baseline image to ghcr.io/amdpilot-org so
+# every other node can docker pull it and skip re-running Phase 1.
+# Requires the docker daemon to be `docker login ghcr.io` with a
+# classic PAT scoped to `write:packages` (and the user must be an
+# amdpilot-org member). The first push under amdpilot-org has to
+# upload the 42 GB base layer; subsequent pushes only upload the
+# Phase-1-commit delta (~500 MB - 1 GB) via cross-repo mount.
+phase1_publish:
+  enabled: true
+  repository: "ghcr.io/amdpilot-org/primus-qwen3-30b-mfu-phase1"
+  tag_template: "{date}-{metric}"
+  also_tag_latest: true
+  timeout_s: 9000  # 2.5h — covers the one-time 42 GB seed upload
+
+model_endpoint:
+  model: "Kimi-K2.6"
+  base_url: "http://10.235.24.154:30000/v1"
+  api_key: "EMPTY"
+  # Lock to K2.6 even if .env exports AMDPILOT_MODEL_URL/AMDPILOT_MODEL.
+  allow_env_base_url_override: false
+  allow_env_model_override: false
+
+kimi_cli:
+  thinking: true
+  max_steps_per_turn: 200
+  max_retries_per_step: 12
+  ralph_iterations: -1
+
+container:
+  name: amdpilot_primus_qwen3_mfu
+  gpu: "0,1,2,3,4,5,6,7"
+  shm_size: 128g
+  devices: [/dev/kfd, /dev/dri]
+  env:
+    # Same env that r54 + r36 used to reach 300+ TFLOP/s/GPU on this
+    # node. The gfx950 knobs (PYTORCH_HIP_ALLOC_CONF, NVTE_ROCM_ARCH,
+    # HIP_FORCE_DEV_KERNARG, etc.) are baked into the base image and
+    # MUST NOT be overridden.
+    GLOO_SOCKET_IFNAME: "enp81s0f1"
+    NCCL_SOCKET_IFNAME: "enp81s0f1"
+    IP_INTERFACE: "enp81s0f1"
+    TOKENIZERS_PARALLELISM: "false"
+    GPU_COREDUMP_ENABLE: "0"
+    NVTE_CK_USES_BWD_V3: "1"
+    # Override amdpilot's default TMPDIR=/scratch/tmp which triggers
+    # torchrun / primus-cli HIP kernel-loading failures on this box
+    # (docker_manager.py:220). /tmp is safe and works.
+    TMPDIR: "/tmp"
+
+workload:
+  description: "Maximize Qwen3-30B-A3B MoE pretraining MFU on 8x MI355X (Megatron + Primus + Primus-Turbo)"
+  function: "primus.train.pretrain"
+  framework: PyTorch
+
+benchmark:
+  command: "bash /workspace/bench_mfu.sh"
+  script: "evals/instances/primus-qwen3-30b-mfu/bench_mfu.sh"
+  metric_name: TFLOPS_PER_GPU
+  metric_pattern: 'TFLOPS_PER_GPU:\s+([\d.]+)'
+  metric_direction: higher
+  preflight_timeout: 1500
+  fixed_params:
+    model: Qwen3-30B-A3B
+    micro_batch_size: 1
+    global_batch_size: 8
+    seq_length: 8192
+    expert_model_parallel_size: 8
+    train_iters: 10
+  output_format: "TFLOPS_PER_GPU: <value> | mbs=1 gbs=8 seq=8192 ep=8"
+
+# Minimum contract — phase1 fills in expected_metric / required_flags /
+# active_backends. immutable_artifacts protect the bench wrapper and the
+# Primus baseline script from in-trial mutation.
+baseline_contract:
+  metric_name: TFLOPS_PER_GPU
+  metric_direction: higher
+  tolerance: 0.07
+  reproduce_command: "bash /workspace/bench_mfu.sh"
+  gpu_shape: "8xMI355X"
+  base_image: "primus-qwen3-30b-mfu-base:v1"
+  immutable_artifacts:
+    - /workspace/bench_mfu.sh
+    - /workspace/test_harness.py
+    - /workspace/primus_train/Primus/scripts/run_qwen3_30b_mfu_baseline.sh
+
+task:
+  description_file: evals/instances/primus-qwen3-30b-mfu/task_description.md
+
+stages: auto
+
+max_retries_per_stage: 5
+max_total_hours: 4
+gpu_required: 8
+
+slack:
+  enabled: true