From e7ec73f2973d48d035fc392b0e7860efc0f408ff Mon Sep 17 00:00:00 2001 From: adithya-s-k Date: Wed, 30 Sep 2026 07:29:02 +0000 Subject: [PATCH 01/48] Add reproducible three-mode multi-harness RL recipe --- .github/workflows/multi-harness-rl.yml | 21 + 05-multi-harness-rl/.gitignore | 8 + 05-multi-harness-rl/README.md | 75 ++ 05-multi-harness-rl/REPRODUCE.md | 89 ++ 05-multi-harness-rl/VALIDATION.md | 60 + 05-multi-harness-rl/configs/default.json | 53 + 05-multi-harness-rl/configs/runtime-lock.json | 61 + 05-multi-harness-rl/data/test.json | 252 +++++ 05-multi-harness-rl/data/train.json | 1002 +++++++++++++++++ 05-multi-harness-rl/eval/evaluate.py | 165 +++ 05-multi-harness-rl/eval/watch.py | 99 ++ 05-multi-harness-rl/prepare.py | 80 ++ 05-multi-harness-rl/project.yaml | 32 + 05-multi-harness-rl/recipe.py | 109 ++ 05-multi-harness-rl/requirements.lock | 312 +++++ 05-multi-harness-rl/requirements.txt | 40 + 05-multi-harness-rl/run.py | 182 +++ 05-multi-harness-rl/runtime/bootstrap.py | 88 ++ 05-multi-harness-rl/runtime/checkpoints.py | 57 + 05-multi-harness-rl/runtime/environment.py | 46 + 05-multi-harness-rl/runtime/job.sh | 16 + 05-multi-harness-rl/runtime/launch.py | 122 ++ 05-multi-harness-rl/runtime/models.py | 73 ++ 05-multi-harness-rl/runtime/patches.py | 55 + .../runtime/whitebox_reward.py | 18 + 05-multi-harness-rl/tests/test_contracts.py | 160 +++ 05-multi-harness-rl/tests/test_runtime.py | 78 ++ 05-multi-harness-rl/train/adapters.py | 129 +++ 05-multi-harness-rl/train/blackbox.py | 82 ++ 05-multi-harness-rl/train/generation.py | 34 + 05-multi-harness-rl/train/logging_utils.py | 38 + 05-multi-harness-rl/train/whitebox.py | 84 ++ README.md | 1 + 33 files changed, 3721 insertions(+) create mode 100644 .github/workflows/multi-harness-rl.yml create mode 100644 05-multi-harness-rl/.gitignore create mode 100644 05-multi-harness-rl/README.md create mode 100644 05-multi-harness-rl/REPRODUCE.md create mode 100644 05-multi-harness-rl/VALIDATION.md create mode 100644 05-multi-harness-rl/configs/default.json create mode 100644 05-multi-harness-rl/configs/runtime-lock.json create mode 100644 05-multi-harness-rl/data/test.json create mode 100644 05-multi-harness-rl/data/train.json create mode 100644 05-multi-harness-rl/eval/evaluate.py create mode 100644 05-multi-harness-rl/eval/watch.py create mode 100644 05-multi-harness-rl/prepare.py create mode 100644 05-multi-harness-rl/project.yaml create mode 100644 05-multi-harness-rl/recipe.py create mode 100644 05-multi-harness-rl/requirements.lock create mode 100644 05-multi-harness-rl/requirements.txt create mode 100644 05-multi-harness-rl/run.py create mode 100644 05-multi-harness-rl/runtime/bootstrap.py create mode 100644 05-multi-harness-rl/runtime/checkpoints.py create mode 100644 05-multi-harness-rl/runtime/environment.py create mode 100644 05-multi-harness-rl/runtime/job.sh create mode 100644 05-multi-harness-rl/runtime/launch.py create mode 100644 05-multi-harness-rl/runtime/models.py create mode 100644 05-multi-harness-rl/runtime/patches.py create mode 100644 05-multi-harness-rl/runtime/whitebox_reward.py create mode 100644 05-multi-harness-rl/tests/test_contracts.py create mode 100644 05-multi-harness-rl/tests/test_runtime.py create mode 100644 05-multi-harness-rl/train/adapters.py create mode 100644 05-multi-harness-rl/train/blackbox.py create mode 100644 05-multi-harness-rl/train/generation.py create mode 100644 05-multi-harness-rl/train/logging_utils.py create mode 100644 05-multi-harness-rl/train/whitebox.py diff --git a/.github/workflows/multi-harness-rl.yml b/.github/workflows/multi-harness-rl.yml new file mode 100644 index 00000000..2f7ef525 --- /dev/null +++ b/.github/workflows/multi-harness-rl.yml @@ -0,0 +1,21 @@ +name: SmolDataEnv RL recipe +on: + pull_request: + paths: ['05-multi-harness-rl/**', '.github/workflows/multi-harness-rl.yml'] + push: + branches: [main] + paths: ['05-multi-harness-rl/**', '.github/workflows/multi-harness-rl.yml'] +permissions: + contents: read +jobs: + contracts: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install pytest==9.1.1 requests + - name: Check recipe contracts without GPUs or sandbox credentials + run: python -m pytest 05-multi-harness-rl/tests -q diff --git a/05-multi-harness-rl/.gitignore b/05-multi-harness-rl/.gitignore new file mode 100644 index 00000000..c1f1293c --- /dev/null +++ b/05-multi-harness-rl/.gitignore @@ -0,0 +1,8 @@ +.runtime/ +.venv/ +__pycache__/ +.pytest_cache/ +runs/ +prepared/ +.env +.gradio/ diff --git a/05-multi-harness-rl/README.md b/05-multi-harness-rl/README.md new file mode 100644 index 00000000..474ee4eb --- /dev/null +++ b/05-multi-harness-rl/README.md @@ -0,0 +1,75 @@ +# SmolDataEnv: whitebox and multi-harness RL + +Train the same data-analysis policy through three agent interfaces. Use **LFM2.5-2.6B** or **Qwen3.5-2B**, with thinking disabled, the same task split, and correctness plus a small tool-efficiency bonus. + +This is a focused successor to the archived `04-data-agent` experiment. Start with [REPRODUCE.md](REPRODUCE.md) for HF Jobs and Slurm commands. The new recipe is under qualification; historical runs do not establish that this new non-thinking configuration is trained or validated. + +| Mode | Who drives the agent? | Training | Evaluation | +|---|---|---|---| +| `whitebox` | TRL calls native bash/SETA methods | Synchronous GRPO | 250 tasks with the same bash/SETA tools | +| `opencode` | Native OpenCode, without Harbor execution | AsyncGRPO | 250 tasks through each of four Harbor harnesses | +| `multi-harness` | Harbor runs OpenCode, Claude Code, Codex or Mini-SWE-Agent | AsyncGRPO | The same 250 tasks through all four harnesses | + +The OpenEnv server runs **inside each training or evaluation job**. Daytona runs the isolated task containers. No environment Space is needed. Training and evaluation use separate GPU allocations, inference engines and environment processes; they still share your sandbox quota and storage bandwidth. + +## Shared experiment + +| Setting | Value | +|---|---| +| Training data | Fixed 1,000 tasks: 400 medium, 600 hard; no test task or notebook overlap | +| Schedule | Two passes; one harness per task group; rotate the harness on the second pass | +| Rollouts per group | 8, all using the same task and harness | +| Learning rate / sampling | `3e-6`; temperature `0.8`, top-p `1`, top-k disabled | +| Async limits | 32 inflight; 16 outstanding rollouts including queued work; staleness 4 | +| Optimizer | Full fine-tuning, paged AdamW 8-bit, BF16, gradient checkpointing | +| Async batching | Batch 4 × accumulation 4; whole rollouts admitted together; 40,960-token packing target | +| Synchronous batching | Microbatch 1 × accumulation 16 to limit logits memory | +| Episode limits | 17 model/tool iterations, 600 seconds; 4,096 output tokens per model call | +| Trainer limits | 16,384 completion tokens; 131,072 context; 1,000 optimizer-step ceiling | +| Save / evaluate | Save every 50 updates; evaluate every 100 and the final checkpoint | +| Evaluation | Pass@1; TP1/DP2; default concurrency 35, configurable | + +Task groups and optimizer updates are different units. Async packing, discarded stale rollouts, and groups without reward contrast affect how many groups produce an update. Training stops at schedule exhaustion or the step ceiling, whichever comes first. Two scheduled passes do not guarantee two completed passes; checkpoints record the groups actually used. + +The reward is `correctness × (1 + 0.1 × 15 / (15 + tool_calls))`. Incorrect answers earn zero. A zero or unverified tool count receives no bonus. Infrastructure failures remain ungraded. Native action records provide the counts; the sandbox grader returns correctness only. Pass@1 reports binary correctness separately from shaping and always includes coverage. + +Qwen uses `enable_thinking=False`. The pinned LFM template ignores that flag, so this recipe explicitly closes its generation-time thinking block. Serving omits reasoning parsing: a streaming reasoning parser can otherwise hide tool calls from the harness. Startup checks that both streaming and ordinary replies are visible. Earlier LFM RL results used thinking and are not a baseline for this setting. + +## Code comparison + +The learning boundary is small: + +```python +# Whitebox: the trainer generates, calls tools and masks tool results. +GRPOTrainer(environment_factory=white_box_bash_env(...), ...) + +# Native OpenCode: the native environment runs OpenCode and captures its model calls. +DataAgentSession(...) + +# Multi-harness: Harbor owns the agent loop and returns captured model calls. +HarborSession(harness=group["harness"], ...) +``` + +Both blackbox modes use the same finite AsyncGRPO worker and whole-rollout admission. Engine token IDs, sampled logprobs and loss masks are preserved, including when a harness rewrites its history. Those rewrites can create several training rows from one rollout, so row-averaged training reward is not the same as rollout-level correctness. LFM's packed convolution states are isolated between sequences. + +| File | Purpose | +|---|---| +| [configs/default.json](configs/default.json) | Shared hyperparameters and model/harness pins | +| [prepare.py](prepare.py) | Fixed task selection and corrected Harbor grader checks | +| [train/whitebox.py](train/whitebox.py) | Synchronous tool-based training | +| [train/blackbox.py](train/blackbox.py), [train/adapters.py](train/adapters.py) | Async training and native/Harbor session boundary | +| [eval/evaluate.py](eval/evaluate.py) | Resumable pass@1 evaluation and usage metrics | +| [eval/watch.py](eval/watch.py) | Checkpoint verification and separate evaluation jobs | +| [runtime/launch.py](runtime/launch.py) | HF Jobs and Slurm submission; dry-run by default | + +`runtime/bootstrap.py` fetches 45 existing environment/admission files and pinned TRL/OpenEnv checkouts into ignored `.runtime/`. Their source hashes are in [runtime-lock.json](configs/runtime-lock.json). Small native-environment adaptations are explicit in [runtime/patches.py](runtime/patches.py). The recipe does not require an old local experiment checkout. The pinned TRL integration is experimental; dependency upgrades require another smoke. + +## Logs and previous results + +Each run keeps its resolved configuration, task schedule, scalar `metrics.jsonl`, local Trackio data, rollout reward/count evidence, service logs and full optimizer checkpoints. Each eval keeps a record per task/harness, coverage, missing pairs, pass@1, tool calls and generated/prompt tokens. Set `trackio_space_id` in the config to mirror to an online dashboard. + +[SmolDataEnv RL dashboard](https://huggingface.co/spaces/FineEnvs/data-agent-training-comparison-trackio) contains the historical experiments. Their full record remains on [the archived branch](https://github.com/adithya-s-k/FineEnvs/tree/archive/data-agent-experiments-20260930/04-data-agent). This recipe has no new published scores yet. + +The September 30 Harbor grader update is pinned here: null tool-efficiency fields must not discard a valid correctness reward. [SmolDataEnvs prompt updates](https://huggingface.co/datasets/FineEnvs/SmolDataEnvs/discussions/2) also change some single-program baselines. Do not combine those scores with an older protocol without identifying the revisions. + +Historical impact depends on each run's frozen files. The corrected Hub tasks match this recipe's instruction hashes; 302 selected training tasks use a JSON reward wrapper where the old local copy used `reward.txt`. The recipe checks the corrected grader before serving either split. See [validation notes](VALIDATION.md) for smoke coverage and remaining checks. diff --git a/05-multi-harness-rl/REPRODUCE.md b/05-multi-harness-rl/REPRODUCE.md new file mode 100644 index 00000000..d5184c7c --- /dev/null +++ b/05-multi-harness-rl/REPRODUCE.md @@ -0,0 +1,89 @@ +# Run the recipe + +All commands below start in `05-multi-harness-rl/`. Python 3.12 is required. Choose `--mode whitebox`, `--mode opencode` or `--mode multi-harness`, and `--model lfm` or `--model qwen`. + +## 1. Prepare and inspect + +```bash +uv venv --python 3.12 +source .venv/bin/activate +uv pip install -r requirements.lock +python runtime/bootstrap.py +python prepare.py +python -m pytest -q tests +python run.py plan --model lfm --mode multi-harness +``` + +`prepare.py` downloads immutable revisions of the corrected Harbor train/test datasets. It selects the exact IDs in `data/`, checks instruction and grader hashes, sets each task to one CPU/4 GB, and records file hashes. It never selects a fresh random test set. Data and downloaded runtime code are ignored by Git. + +Log in with `hf auth login` and set `DAYTONA_API_KEY` in your shell. Do not put credentials into a config or commit an `.env` file. The job-local service stages private task data; model-serving access is scoped to capture sessions. + +## 2. HF Jobs + +Create a durable bucket you own. Supply its `owner/name` below. Launch commands print a plan unless you add `--submit`; neither Git push nor an environment Space is required. The launcher uploads only the allowlisted recipe source. + +```bash +# Disposable two-update training smoke. +python runtime/launch.py hf smoke --model lfm --mode opencode \ + --run-name smoke-lfm-native --bucket FineEnvs/YOUR_BUCKET --timeout 2h + +# Inspect, then add --submit to launch. +python runtime/launch.py hf train --model lfm --mode multi-harness \ + --run-name lfm-multi-nonthinking-v1 --bucket FineEnvs/YOUR_BUCKET + +# Separate CPU watcher submits evaluations on separate GPU Jobs. +python runtime/launch.py hf watch --run-name eval-watcher \ + --bucket FineEnvs/YOUR_BUCKET --concurrency 35 --max-active-evals 1 +``` + +Repeat the smoke for each model/mode before launching that combination. The default GPU flavor is `h200x2`: one trainer GPU and one serving GPU; evaluation uses both GPUs as DP2 replicas. HF Jobs currently has no two-A100 flavor. Override `--flavor` only with hardware that has enough GPUs and memory. The base image is pinned by digest; Python packages and source revisions are pinned separately. + +The bucket is mounted at `/outputs`. A run writes to `/outputs/RUN_NAME`; the watcher hashes completed checkpoints before launching evals. Keep the watcher alive until all final evaluations finish. Failures remain in `eval-watcher.json` for investigation rather than resubmitting indefinitely. Relaunch a failed evaluation explicitly with the same output name to retain already graded pairs. + +```bash +python runtime/launch.py hf eval --model lfm --mode multi-harness \ + --run-name lfm-multi-nonthinking-v1-eval-checkpoint-100 \ + --checkpoint /outputs/lfm-multi-nonthinking-v1/checkpoint-100 \ + --bucket FineEnvs/YOUR_BUCKET --concurrency 35 +``` + +For online Trackio, set `trackio_space_id` in `configs/default.json` before submission. Logs and offline Trackio data remain in the bucket regardless. Use `hf jobs ps` and `hf jobs logs JOB_ID` to monitor. The new HF container path still needs an actual GPU Job smoke; a successful local smoke alone does not qualify it. + +## 3. Slurm or a local two-GPU machine + +The same Python environment and prepared files must be visible on compute nodes. Choose a free GPU partition; no cluster name is hardcoded into the recipe. + +```bash +python runtime/launch.py slurm smoke --partition YOUR_GPU_PARTITION \ + --model lfm --mode opencode --run-name smoke-lfm-native + +python runtime/launch.py slurm train --partition YOUR_GPU_PARTITION \ + --model qwen --mode multi-harness --run-name qwen-multi-v1 + +# On a persistent CPU/login process, submit separate GPU eval allocations. +python eval/watch.py --root runs --backend slurm --partition YOUR_GPU_PARTITION \ + --concurrency 50 --max-active 1 --submit +``` + +Add `--submit` to the launch commands after inspecting them. They request one node, two GPUs, 16 CPUs and 192 GB RAM. Local execution without Slurm is `CUDA_VISIBLE_DEVICES=0,1 python run.py smoke --model lfm --mode opencode`. + +Each job starts its own vLLM and OpenEnv services on unused local ports, and stops its own process groups on exit. Their URLs are recorded in `services.json`. The capture proxy uses OpenEnv's Gradio tunnel so remote Daytona containers can reach it. Sandbox egress and tunnel reliability must be checked in the rollout smoke. + +## 4. Checkpoint and evaluation checks + +The smoke saves at updates 1 and 2. Run the watcher with `--once` without `--submit` to verify checkpoint files and inspect eval commands. Then reload a smoke checkpoint for a two-task evaluation: + +```bash +python eval/watch.py --root runs --backend slurm --partition YOUR_GPU_PARTITION --once +python run.py eval --model lfm --mode opencode \ + --checkpoint runs/smoke-lfm-native/checkpoint-2 \ + --output runs/smoke-lfm-native-eval --limit 2 --concurrency 2 +``` + +That is eight pass@1 pairs for a blackbox policy, or two SETA episodes for whitebox. Require a complete summary, valid captured tokens/logprobs/masks, verified tool counts, finite training metrics and a loadable checkpoint. Inspect `train.log`, `environment.log` and `vllm.log` when a check fails. + +Production saves every 50 updates; the watcher evaluates every 100 and the final checkpoint. Full blackbox eval is 1,000 pairs. An incorrect answer is retained as zero; only ungraded failures are retried. Incomplete evaluations are explicitly provisional. Do not retry completed incorrect answers and call the result pass@1. + +Resume with `--resume /path/to/checkpoint-N`. Blackbox checkpoints preserve committed group IDs, so groups already used by the optimizer are not replayed. Unconsumed tails of partially committed groups are recorded and abandoned; groups with no committed work can be regenerated. Keep the task schedule, model and reward configuration fixed across a resume. + +References: [HF Jobs](https://huggingface.co/docs/huggingface_hub/guides/jobs), [volume configuration](https://huggingface.co/docs/hub/jobs-configuration), [OpenCode CLI](https://dev.opencode.ai/docs/cli/). diff --git a/05-multi-harness-rl/VALIDATION.md b/05-multi-harness-rl/VALIDATION.md new file mode 100644 index 00000000..2ca0e354 --- /dev/null +++ b/05-multi-harness-rl/VALIDATION.md @@ -0,0 +1,60 @@ +# Qualification notes + +This is a new non-thinking recipe. Historical training results are not its baseline. + +## Data and runtime + +- Public Hub preparation passed for all 1,000 training tasks and 250 test tasks. Instruction hashes match the fixed selection; the corrected numeric-only grader is checked for every task. +- Train revision: `4719635555de1666f374d847baebb500d368e493`. Test revision: `b130595b579fc6026ad5762925a9a3b40e72f5bb`. +- Compared with the old staged files, 302 training tasks now use the JSON reward wrapper. Task metadata also changed. The test instructions and grader did not change. +- OpenEnv is pinned to public commit `4f4c85fb9038f43efc2f51858a27638277f16355`, including partial-mask validation. No unpublished OpenEnv checkout is required. +- TRL, native environment sources and file hashes are in `configs/runtime-lock.json`. The Python dependency set resolves successfully; `requirements.lock` records the resolved versions. + +## Checks on 30 September 2026 + +| Check | Result | +|---|---| +| CPU contracts and fetched-runtime integration | 21 passed | +| Clean, minimal CI environment | 17 passed; four runtime checks skipped without fetched dependencies | +| Native OpenEnv to TRL partial mask | `[1, 0]` preserved | +| Native LFM checkpoint handoff | Both initial smoke checkpoints saved; file hashes verified | +| Corrected four-harness reload eval | 8/8 pairs graded; all token checks passed; four correct answers | +| Qwen whitebox training | Two updates and both checkpoints saved; second update had reward contrast and nonzero gradient | +| Qwen whitebox token audit | Both batches passed; 68 captured calls in total | +| Corrected non-thinking HTTP and streaming replies | Passed for LFM and Qwen | +| Both models, all three training modes | All six completed two updates and saved both checkpoints | +| Whitebox checkpoint reload eval | 2/2 tasks graded; one correct answer | +| Async checkpoint resume | Resumed step 1 to 2; committed group 0 was not replayed | +| HF Jobs setup | Fresh Python 3.12 environment, locked installation and all 1,250 tasks prepared successfully | +| HF Jobs GPU smoke | Corrected submission accepted; allocation pending | + +| Model | Mode | Local smoke wall time | Nonzero gradient observed? | +|---|---|---|---| +| Qwen3.5-2B | Whitebox | 10m 26s | Yes | +| Qwen3.5-2B | Native OpenCode | 8m 35s | No reward contrast in this sample | +| Qwen3.5-2B | Harbor multi-harness | 9m 07s | Yes | +| LFM2.5-2.6B | Whitebox | 6m 37s | No reward contrast in this sample | +| LFM2.5-2.6B | Native OpenCode | 9m 30s | No reward contrast in this sample | +| LFM2.5-2.6B | Harbor multi-harness | 13m 01s | Yes | + +Wall time includes model loading, sandbox setup, two updates, saving and cleanup. Smoke batches use two rollouts and low concurrency, not the full training batch or eval concurrency 35. Local smokes use the existing cluster runtime; the separate HF check qualifies a fresh installation from the lockfile. + +The initial LFM smoke exposed a serving error: the reasoning parser classified tool-call text as reasoning, leaving Claude Code with empty assistant replies. Non-thinking serving now omits that parser. Claude Code subsequently emitted native tool calls, and a Harbor OpenCode rollout produced correctness `1` with nine calls and shaped reward `1.0625`. Initial LFM smoke scores must not be reused as a baseline. + +Other fixes found by the smokes: job-specific ports on shared Slurm nodes, explicit HF login forwarding to Harbor task staging, the SDK's typed bucket mount, explicit tool-response templates, the whitebox cache-reset response, and the text context limit on Qwen's outer model config. The pinned base image has Python 3.11; the HF entrypoint now creates its own Python 3.12 environment before installing the lockfile. + +## Evidence + +Local smoke artifacts are ignored under `runs/qualification/`: configuration, service logs, rollout reward records, `token-audit.jsonl`, local Trackio data and checkpoints. CPU logs are `cpu-tests-final.log` and `ci-tests.log`; public data preparation is `prepare-hub.log` and `prepared-hub/`. + +- Qwen whitebox: `qwen-whitebox-v5`, Slurm job `92750`. +- Corrected LFM Harbor: `lfm-multi-harness-v4`, job `92751`. +- Corrected native OpenCode: `lfm-opencode-v4` / `qwen-opencode-v1`, jobs `92752` / `92753`. +- Corrected LFM reload evaluation: `lfm-nonthinking-eval-v2`, job `92757`. +- LFM whitebox / Qwen Harbor: jobs `92759` / `92760`. +- Whitebox checkpoint eval: `qwen-whitebox-v5-eval`, job `92821`. +- Async resume: `lfm-native-resume-v1`, job `92822`. Only optimizer step 2 was executed; the new admission record contains group 2, while the restored state retains group 0. +- [HF setup check](https://huggingface.co/jobs/FineEnvs/6abcb82f031314b696343d7d): completed with the corrected Python 3.12 bootstrap. +- [HF GPU smoke](https://huggingface.co/jobs/FineEnvs/6abcb8c8031314b696343daf): current submission, bounded to 40 minutes of runtime. Superseded queued submissions were canceled before allocation. + +Before a long HF run, finish the GPU smoke and verify bucket checkpoint handoff. Then collect a fresh non-thinking baseline on the fixed 250-task test set. A tiny smoke establishes execution, not benchmark performance or throughput at concurrency 35. diff --git a/05-multi-harness-rl/configs/default.json b/05-multi-harness-rl/configs/default.json new file mode 100644 index 00000000..a6cf2185 --- /dev/null +++ b/05-multi-harness-rl/configs/default.json @@ -0,0 +1,53 @@ +{ + "model": "lfm", + "mode": "multi-harness", + "seed": 0, + "passes": 2, + "max_steps": 1000, + "learning_rate": 0.000003, + "num_generations": 8, + "max_inflight": 32, + "max_outstanding_rollouts": 16, + "max_staleness": 4, + "batch_size": 4, + "gradient_accumulation_steps": 4, + "token_budget": 40960, + "max_model_len": 131072, + "max_completion_length": 16384, + "max_output_tokens": 4096, + "temperature": 0.8, + "top_p": 1.0, + "top_k": 0, + "agent_step_limit": 17, + "agent_timeout_sec": 600, + "save_steps": 50, + "eval_steps": 100, + "eval_concurrency": 35, + "eval_dp": 2, + "efficiency_weight": 0.1, + "tool_budget": 15, + "sandbox": "daytona", + "project": "smoldataenv-rl", + "trackio_space_id": null, + "models": { + "lfm": { + "id": "LiquidAI/LFM2.5-2.6B", + "revision": "654f9463ce32b05d0429d76fe1f580b27d4c1ac0", + "tool_parser": "lfm2", + "template": "lfm-closed-thinking" + }, + "qwen": { + "id": "Qwen/Qwen3.5-2B", + "revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", + "tool_parser": "qwen3_xml", + "template": "enable_thinking_false" + } + }, + "harnesses": ["opencode", "claude-code", "codex", "mini-swe-agent"], + "harness_versions": { + "opencode": "1.18.31", + "claude-code": "2.1.270", + "codex": "0.154.0", + "mini-swe-agent": "2.4.6" + } +} diff --git a/05-multi-harness-rl/configs/runtime-lock.json b/05-multi-harness-rl/configs/runtime-lock.json new file mode 100644 index 00000000..0d8f3db6 --- /dev/null +++ b/05-multi-harness-rl/configs/runtime-lock.json @@ -0,0 +1,61 @@ +{ + "archive": { + "repo": "adithya-s-k/FineEnvs", + "revision": "bff1b69310db76bf048ae26f7ba1d70134d5f2e3", + "files": { + "envs/blackbox-opencode/tasks.py": "1a99bfb65008366993ce6794fe70a762985aadec01320cb9de9039bd762af802", + "envs/blackbox-opencode/config.py": "19914b42ea832893e52b1c8671e69aefd3908df76dc8bd97498507ecb64385ad", + "envs/blackbox-opencode/curriculum.py": "d3c2d9b47a4a517d8a671ca2a46c1e0682f3c2786b79ba2d6731022f279ec2c2", + "envs/blackbox-opencode/verifier.py": "02e2c9ab9d7340fa450f75e36f1418a8acfb6d6dc7593a77fb3d1cc3dbc797c3", + "envs/blackbox-opencode/__init__.py": "0eabe01dc2ba02aed86167f3c746d9a096379dfb26b0a917b59502c8affe8ebc", + "envs/blackbox-opencode/task.py": "8e492535a8bf6fd7b13e9e00a19a4ec6c03a801246143db842b185b9cc7d8d97", + "envs/blackbox-opencode/client.py": "18cee684f7f3ae377c565bdeeabcaa6e790b366fd859efce93938ed6ea7a52d7", + "envs/blackbox-opencode/reward.py": "fed11e459a0795f624ee9a18121890f0a6aa460b9e1b75763e9d66279fde5129", + "envs/blackbox-opencode/harness.py": "2037caa67cf1a8100f1acff3bbd6d510471261c402738c0634b97beac4ed5fbe", + "envs/blackbox-opencode/grader.py": "58220f93a7d78767f9152c7db6f33a67800d694b4f15eca8e53dbca02625f457", + "envs/blackbox-opencode/rollout.py": "be6b7c787ee187a74b5dfa9af7eef1e56efedaa14aa99dc3b72132dd15e6e6bc", + "envs/blackbox-opencode/models.py": "5fcc4a4b1e7d35b359d87463b83e4f35263733384768b7cc07c924227ae8d0fd", + "envs/blackbox-opencode/server/__init__.py": "0b63c98ec0d6e82d66fa0b13523d417d87dbba0e7aeea1d786aa13a8388e7184", + "envs/blackbox-opencode/server/app.py": "0e318a153dbb420e7a95e05a815466da8bdaf49f45bdeb1b9656ad2296f20a58", + "envs/blackbox-opencode/server/rollout.py": "eb961258c432798403b4bc41709b3ac8fa01c087041911022bac8adba283447b", + "envs/blackbox-opencode/server/capture.py": "97108b4a961e1c0217e04fd0f94b51de6f1a4356d232e49f1511a45bc911da49", + "envs/blackbox-opencode/server/environment.py": "95dfae62baa042742840efc67dad816e1deabcb62b568ad69204bf005ae01229", + "envs/blackbox-opencode/sandbox/hf.py": "8d2d1b3e0eba9819833f52ac714ff693bd1ae2cf467d12313a531837b5f2d795", + "envs/blackbox-opencode/sandbox/__init__.py": "399b02c8050f151072c32b59760ccc4c4c397cb129f406186cd153d974741556", + "envs/blackbox-opencode/sandbox/base.py": "2385954579f8061d34bcbc3e1a20b13658557d1dc8f872a8058aaf4670899d93", + "envs/blackbox-opencode/sandbox/e2b.py": "b30faa2628bc23a2ba3ebaa1b21ef467f3b5d8fac93a317d23bd5f4ce6c8bffa", + "envs/blackbox-opencode/sandbox/daytona.py": "9407386abc7ace49f6b6615ceb20ed8ab8e922840614d2cc863e5e211e079707", + "envs/whitebox-bash/grader.py": "4333d1526a3307a0b9b72a37199009831f7162f2d30cf3d8c7bbd656fdd7b859", + "envs/whitebox-bash/dataagent.py": "7d3b9d91e8f2f8c3679c86e10bba47c4ffcc234a9a065095fe5ee0a327d11943", + "envs/whitebox-bash/suite.py": "a0e162f042d6f6bf5efc4165e3e117d474df2723417e42abd4b3a1875502dc80", + "envs/whitebox-bash/client.py": "b3888637c79ae079c66e1a434090da89cfa5e2a90ec60b64a6c42ecf22782e7f", + "envs/whitebox-bash/__init__.py": "8f34adaf64792f79887b9a1cf72d9767db3a8425028e3566ee377050e1de6b08", + "envs/whitebox-bash/tasks.py": "465c5c721150a4f46929b8e4750c99565c165dbe25930b2ada226551b71c49ba", + "envs/whitebox-bash/models.py": "ea2a29f0c9dd0db0fc201069ed7183089b277cfd3b94bd562a7c19b6929291f8", + "envs/whitebox-bash/tools.py": "02eee5f7ce958ae48702f219859c59432929f7f7f4c99aeee8051472a2c6b35a", + "envs/whitebox-bash/server/__init__.py": "515cd1e723b755ac5281840119a4dd30fe6c63eb98b1b3a6d9b6b9b6c8978cd3", + "envs/whitebox-bash/server/environment.py": "f372126214413d7cddbc631ce6a388c23412effb5713931ee38c703a8c787836", + "envs/whitebox-bash/server/sandbox.py": "238d142a1f2ad114ddea12fb975670e5c1d39d361d80abd8983a326a58643b21", + "envs/whitebox-bash/server/app.py": "9d3cfae74f414a236a95a38844b8b2a78f28df6de2f872b06d1b6fd839d4cdd7", + "train/atomic_rollouts.py": "2c3a1918caff0d5c580f7dacaa914e6707ea7ec78365beb0411f92fb30c63616", + "train/training_audit.py": "2345477c719f15c3c8dd1b98bed0ca7d7c09c6eb3f7b00630dd8fb9de34bcb9b", + "train/checkpoint_artifacts.py": "9bc1055b9bdc1ec389fb73320c2519ab14dad6ace02e6e617b3c1d00365f06db", + "train/harness_schedule.py": "acca0746f9168c50d5e1ef1197cedba8c09126b28dc88fa34da01248cc36a0ba", + "train/multi_harness.py": "6609a24ee58ea1dfbf5ce10a7fea997e4028f82cf65cb4699d5a92a125673259", + "train/standalone_comparison.py": "bd0768a108b96129a2a5c8d80029b63a78fb8b578b96044eca19c90a50031d09", + "hf/runtime/hard_curriculum_train.py": "52c71aa1749536683901832817cd280a107ef7d66d9e7fa85efc2d0ff3dc8777", + "hf/runtime/daytona_whitebox_backend.py": "26677b370d16d8d9f8c4634d169d7ad9aff371d59c3b66583e028f85d2e8549c", + "hf/runtime/service_policy.py": "00c161987b6118c77dcde301fdd0eaeeba2708a4f386934bee1844af38c0701f", + "hf/runtime/whitebox_tito.py": "aee498fdb47f1b66e67118372e8641de79d8f01a8a6e63f9ca90cc6261f88be5", + "model_runs/lfm25/packing.py": "411c8be4ad0966f36ca5260b96d6c04fc20f4c8d7454b89cb64eda762a6df820" + } + }, + "trl": { + "repo": "adithya-s-k/trl", + "revision": "3cb8efdd8586f1eb1b4d4f9b346f9bdad7e86bb4" + }, + "openenv": { + "repo": "huggingface/OpenEnv", + "revision": "4f4c85fb9038f43efc2f51858a27638277f16355" + } +} diff --git a/05-multi-harness-rl/data/test.json b/05-multi-harness-rl/data/test.json new file mode 100644 index 00000000..25cd82ed --- /dev/null +++ b/05-multi-harness-rl/data/test.json @@ -0,0 +1,252 @@ +[ + {"name": "0000_324_324293_qa_2", "source": "0000/324/324293.ipynb_qa_2", "notebook": "0000/324/324293.ipynb", "difficulty": "medium", "instruction_sha256": "03bd60bb664c4cc96b3ab85db3e3a18039dcfbfee06d3935696f002a2bb26bc5"}, + {"name": "0000_340_340599_qa_1", "source": "0000/340/340599.ipynb_qa_1", "notebook": "0000/340/340599.ipynb", "difficulty": "medium", "instruction_sha256": "dd7911b3032aa469ef4f7cd9c71748c62cb560dcfc2bcc897e6111f4197f0383"}, + {"name": "0000_340_340599_qa_3", "source": "0000/340/340599.ipynb_qa_3", "notebook": "0000/340/340599.ipynb", "difficulty": "easy", "instruction_sha256": "dac579f315382a7411af51143da96cfddafa1a68a0271d8e80e7da09a2e49d8c"}, + {"name": "0000_347_347102_qa_1", "source": "0000/347/347102.ipynb_qa_1", "notebook": "0000/347/347102.ipynb", "difficulty": "medium", "instruction_sha256": "75ecfd8d28ed2da9ab44452168e9d8a84edcf7f24ca532ae0bc5f1c122cd5868"}, + {"name": "0000_376_376301_qa_1", "source": "0000/376/376301.ipynb_qa_1", "notebook": "0000/376/376301.ipynb", "difficulty": "hard", "instruction_sha256": "9fbf3484b5169e2dcdcdc52b8022796ae1186c5a6f82d0116726a1311a86c556"}, + {"name": "0000_407_407872_qa_4", "source": "0000/407/407872.ipynb_qa_4", "notebook": "0000/407/407872.ipynb", "difficulty": "medium", "instruction_sha256": "5e9308a8d05b4c1ee0e80a6d875574f2a7a63f18d1e299920bbc111aa78ad9a5"}, + {"name": "0000_414_414083_qa_5", "source": "0000/414/414083.ipynb_qa_5", "notebook": "0000/414/414083.ipynb", "difficulty": "medium", "instruction_sha256": "db2224a25820236d26c5e3dbe269570ff6151c687bc6417a6cdc627c9bc2651c"}, + {"name": "0000_416_416942_qa_3", "source": "0000/416/416942.ipynb_qa_3", "notebook": "0000/416/416942.ipynb", "difficulty": "medium", "instruction_sha256": "1f2d4a1984d64cfea338e269fb3cb6b4c46d0f4ae9dcaefaedec564fe62b42b9"}, + {"name": "0000_419_419818_qa_1", "source": "0000/419/419818.ipynb_qa_1", "notebook": "0000/419/419818.ipynb", "difficulty": "hard", "instruction_sha256": "7021d93a934f65a61cd06d818b5effa346a9191cfb30235cb6553f26dea7666c"}, + {"name": "0000_419_419825_qa_1", "source": "0000/419/419825.ipynb_qa_1", "notebook": "0000/419/419825.ipynb", "difficulty": "medium", "instruction_sha256": "f93eb525f14c051111544a4806e681302ca4f7311c50b8d061d2ebda76edeea2"}, + {"name": "0000_419_419825_qa_2", "source": "0000/419/419825.ipynb_qa_2", "notebook": "0000/419/419825.ipynb", "difficulty": "medium", "instruction_sha256": "d5e8f840b0e2de1deb83b89f234cad36f3dd94945240609f8d43cdf1fb7e6918"}, + {"name": "0000_421_421481_qa_2", "source": "0000/421/421481.ipynb_qa_2", "notebook": "0000/421/421481.ipynb", "difficulty": "medium", "instruction_sha256": "a1bb39b52801ea85dd8d1456d8b6fe30c3525f09f6b8c329cb6a25daccef7a21"}, + {"name": "0000_421_421838_qa_1", "source": "0000/421/421838.ipynb_qa_1", "notebook": "0000/421/421838.ipynb", "difficulty": "medium", "instruction_sha256": "b289ee5a9751a43d8b63f7a5bdf733e056fda721f3df5ed7c7a0019045ef5b26"}, + {"name": "0000_426_426505_qa_3", "source": "0000/426/426505.ipynb_qa_3", "notebook": "0000/426/426505.ipynb", "difficulty": "medium", "instruction_sha256": "709cd8d12afdb1145198a6deb65e6fd958390d8727a92835fc78f68a88de6fff"}, + {"name": "0000_428_428622_qa_2", "source": "0000/428/428622.ipynb_qa_2", "notebook": "0000/428/428622.ipynb", "difficulty": "medium", "instruction_sha256": "d45a384ac73300ab8528dec90095dac3ee472f6ddf472050bd9b28f35dce957b"}, + {"name": "0000_429_429367_qa_3", "source": "0000/429/429367.ipynb_qa_3", "notebook": "0000/429/429367.ipynb", "difficulty": "medium", "instruction_sha256": "9128a3704e211c2069495847044fed9e3c56c114190a6e595c1b385103f7710f"}, + {"name": "0000_430_430116_qa_1", "source": "0000/430/430116.ipynb_qa_1", "notebook": "0000/430/430116.ipynb", "difficulty": "medium", "instruction_sha256": "f432746e61ae31e982fcbafc25fe0bb0491251447e2d259f3d63ed1d63abf494"}, + {"name": "0000_430_430116_qa_3", "source": "0000/430/430116.ipynb_qa_3", "notebook": "0000/430/430116.ipynb", "difficulty": "medium", "instruction_sha256": "d1a13cd9d37fef259d1a19a6d7b3dace1eb194eb15250b8f9d133475695ba916"}, + {"name": "0000_430_430387_qa_5", "source": "0000/430/430387.ipynb_qa_5", "notebook": "0000/430/430387.ipynb", "difficulty": "medium", "instruction_sha256": "fd3b7794d150b61e106e5bae16a4ee8491a45685aedd28de59752bd2523ca5f9"}, + {"name": "0000_430_430797_qa_3", "source": "0000/430/430797.ipynb_qa_3", "notebook": "0000/430/430797.ipynb", "difficulty": "easy", "instruction_sha256": "70807740f335d615d2015b4e3fc9385a97dabd1f76a71bd60704a59372d705f4"}, + {"name": "0000_430_430797_qa_4", "source": "0000/430/430797.ipynb_qa_4", "notebook": "0000/430/430797.ipynb", "difficulty": "easy", "instruction_sha256": "afa7f31692c87906f8546e92b9b38d1f687d3b21f5f735636d2f73e18ffd2596"}, + {"name": "0000_431_431678_qa_5", "source": "0000/431/431678.ipynb_qa_5", "notebook": "0000/431/431678.ipynb", "difficulty": "easy", "instruction_sha256": "f70373d7853741d97ee7a0c080c86a14090d2fff6a1dfb5cbe1824649311e71b"}, + {"name": "0000_432_432250_qa_4", "source": "0000/432/432250.ipynb_qa_4", "notebook": "0000/432/432250.ipynb", "difficulty": "medium", "instruction_sha256": "ac96191139e8122668d2112fa375a1ac91bca61263440d9a15200ed9c1238602"}, + {"name": "0000_435_435800_qa_4", "source": "0000/435/435800.ipynb_qa_4", "notebook": "0000/435/435800.ipynb", "difficulty": "medium", "instruction_sha256": "7d094b3cad8c48e1b22fc98b6fc49d8e5e73c476b04ab1a811d00105126446e2"}, + {"name": "0000_440_440038_qa_4", "source": "0000/440/440038.ipynb_qa_4", "notebook": "0000/440/440038.ipynb", "difficulty": "medium", "instruction_sha256": "f49c9f362f95d66700eb9e036fe413f525985abbe46df8032de743265b61f9f7"}, + {"name": "0000_440_440038_qa_5", "source": "0000/440/440038.ipynb_qa_5", "notebook": "0000/440/440038.ipynb", "difficulty": "easy", "instruction_sha256": "a500e2ff0801452ecf7230def6e5ab9db96faa80a6b7775a49c3cab694c53600"}, + {"name": "0000_441_441126_qa_2", "source": "0000/441/441126.ipynb_qa_2", "notebook": "0000/441/441126.ipynb", "difficulty": "easy", "instruction_sha256": "b83ab32da1d187f5fc4b1f3a3de5828e5d014ae675904397b6945733a3a5c600"}, + {"name": "0000_441_441126_qa_5", "source": "0000/441/441126.ipynb_qa_5", "notebook": "0000/441/441126.ipynb", "difficulty": "medium", "instruction_sha256": "d83c61fcd757aa118b085c267ba3dcd45f23784fa9e997a161eda6ffe62d03d6"}, + {"name": "0000_445_445307_qa_2", "source": "0000/445/445307.ipynb_qa_2", "notebook": "0000/445/445307.ipynb", "difficulty": "medium", "instruction_sha256": "37b5b8bcc12079ead32027f392959b7fa43739f6b545b21608f3ec99182d751d"}, + {"name": "0000_445_445912_qa_3", "source": "0000/445/445912.ipynb_qa_3", "notebook": "0000/445/445912.ipynb", "difficulty": "medium", "instruction_sha256": "65e658c39d354dc010c1d0a177edefb1f964acbea18791adaeb69e0c6d3273f9"}, + {"name": "0000_455_455459_qa_5", "source": "0000/455/455459.ipynb_qa_5", "notebook": "0000/455/455459.ipynb", "difficulty": "hard", "instruction_sha256": "3fabd97dc24653b0d9261de8ceb05829a5c593c9183d82422b44de9398c7b77d"}, + {"name": "0000_455_455476_qa_5", "source": "0000/455/455476.ipynb_qa_5", "notebook": "0000/455/455476.ipynb", "difficulty": "hard", "instruction_sha256": "1351770e709a28a0b76d417d0e407adda497e4957268f11ca9a361346c2756f1"}, + {"name": "0000_457_457079_qa_2", "source": "0000/457/457079.ipynb_qa_2", "notebook": "0000/457/457079.ipynb", "difficulty": "easy", "instruction_sha256": "249caa6706e1e9e7e879b48478b7ddb274fae57fe12b90e4d433560bf4316588"}, + {"name": "0000_458_458419_qa_1", "source": "0000/458/458419.ipynb_qa_1", "notebook": "0000/458/458419.ipynb", "difficulty": "easy", "instruction_sha256": "e4e99f0db8569d98903fc4c0740e1d8df000bc3c8fa0f923107a653c3821187e"}, + {"name": "0000_458_458419_qa_3", "source": "0000/458/458419.ipynb_qa_3", "notebook": "0000/458/458419.ipynb", "difficulty": "easy", "instruction_sha256": "4aab5d85a1a669b3f3c0bfd4ca2cc097938cdc73bd6ac787b9c12869343656a5"}, + {"name": "0000_458_458607_qa_2", "source": "0000/458/458607.ipynb_qa_2", "notebook": "0000/458/458607.ipynb", "difficulty": "medium", "instruction_sha256": "0146592334708eeacc34f66ea61b139029bab3dc6ee394fd949a84e0aa18edbd"}, + {"name": "0000_458_458607_qa_3", "source": "0000/458/458607.ipynb_qa_3", "notebook": "0000/458/458607.ipynb", "difficulty": "medium", "instruction_sha256": "37b2b7c62f22411583d0557e362733d2d79c1d3f1ed241099638569055cd478b"}, + {"name": "0000_458_458607_qa_4", "source": "0000/458/458607.ipynb_qa_4", "notebook": "0000/458/458607.ipynb", "difficulty": "medium", "instruction_sha256": "3d4d72429aeb58095357510d9c20586237c89f4a5abe870662db4bce5d2a8755"}, + {"name": "0000_458_458607_qa_5", "source": "0000/458/458607.ipynb_qa_5", "notebook": "0000/458/458607.ipynb", "difficulty": "medium", "instruction_sha256": "ce591a3f66473fb68068d47d7ac36c3e1796812d4ce2593fe47dabc5e9882c3f"}, + {"name": "0000_461_461615_qa_4", "source": "0000/461/461615.ipynb_qa_4", "notebook": "0000/461/461615.ipynb", "difficulty": "medium", "instruction_sha256": "5e5f5c76ef29edb47b170a647a97a635d86c45d5d5a172f963c21824b9a9d6b8"}, + {"name": "0000_462_462872_qa_2", "source": "0000/462/462872.ipynb_qa_2", "notebook": "0000/462/462872.ipynb", "difficulty": "medium", "instruction_sha256": "d7928f6dc559541b8e3967f8c8032f732bc40138f28780504b3d72ed6f9888a8"}, + {"name": "0000_470_470407_qa_4", "source": "0000/470/470407.ipynb_qa_4", "notebook": "0000/470/470407.ipynb", "difficulty": "easy", "instruction_sha256": "0f558d39988dbe2904f8d05a927501290d59a139e9d22ecdae93c13171e322d7"}, + {"name": "0000_471_471618_qa_2", "source": "0000/471/471618.ipynb_qa_2", "notebook": "0000/471/471618.ipynb", "difficulty": "easy", "instruction_sha256": "efdfd2d18da4a302166f9c6ee54670705993111db3e916565023e259ac41cc3f"}, + {"name": "0000_471_471618_qa_5", "source": "0000/471/471618.ipynb_qa_5", "notebook": "0000/471/471618.ipynb", "difficulty": "medium", "instruction_sha256": "cae783d2b4f4ee815ab56726b3b88e55db0601c21d7dab76a0f7cc603db950a4"}, + {"name": "0000_473_473331_qa_1", "source": "0000/473/473331.ipynb_qa_1", "notebook": "0000/473/473331.ipynb", "difficulty": "easy", "instruction_sha256": "4bf9b29b157bdc702d190ab0f83793afbb19c18e1021ba836a7400d6981bcab9"}, + {"name": "0000_500_500452_qa_3", "source": "0000/500/500452.ipynb_qa_3", "notebook": "0000/500/500452.ipynb", "difficulty": "medium", "instruction_sha256": "b9bf81fea1fc5dbc6c98a7d000f7f5fe5c9b5d0b41fcdf4776eb60ed1d4df753"}, + {"name": "0000_513_513191_qa_3", "source": "0000/513/513191.ipynb_qa_3", "notebook": "0000/513/513191.ipynb", "difficulty": "medium", "instruction_sha256": "d830524a3aa9c50209cb92c37b21fde973cfb9357817a7dd157a91e34925d675"}, + {"name": "0000_526_526379_qa_1", "source": "0000/526/526379.ipynb_qa_1", "notebook": "0000/526/526379.ipynb", "difficulty": "medium", "instruction_sha256": "58e54442a78cf2c4600e35cad0298abec06d359ae11c5417ea02a4db1f4b7204"}, + {"name": "0000_528_528067_qa_5", "source": "0000/528/528067.ipynb_qa_5", "notebook": "0000/528/528067.ipynb", "difficulty": "medium", "instruction_sha256": "dfde5bf6704b342502587bda9fbe65d5f20b3665d4359a12c6947ac62991cde3"}, + {"name": "0000_531_531577_qa_1", "source": "0000/531/531577.ipynb_qa_1", "notebook": "0000/531/531577.ipynb", "difficulty": "easy", "instruction_sha256": "6459848e9a6bce5061a333107e5ba97c8b954567e1305237b7f744737f4f4e17"}, + {"name": "0000_531_531577_qa_2", "source": "0000/531/531577.ipynb_qa_2", "notebook": "0000/531/531577.ipynb", "difficulty": "medium", "instruction_sha256": "ace88bde6c834d6a92948204e7d6ca9e9573210369174d1a251f4d1aded9ba95"}, + {"name": "0000_537_537525_qa_4", "source": "0000/537/537525.ipynb_qa_4", "notebook": "0000/537/537525.ipynb", "difficulty": "medium", "instruction_sha256": "742b20af32c570a0c8f84c73652d1939377d88f50de27dd867fed275a6589d95"}, + {"name": "0000_541_541073_qa_1", "source": "0000/541/541073.ipynb_qa_1", "notebook": "0000/541/541073.ipynb", "difficulty": "easy", "instruction_sha256": "71d60f5f4ef8349c3d0acca7cb98ac08f9b3451e4238accacc98e778c39ccd31"}, + {"name": "0000_545_545324_qa_1", "source": "0000/545/545324.ipynb_qa_1", "notebook": "0000/545/545324.ipynb", "difficulty": "medium", "instruction_sha256": "abe5e6253bcc116547dcfd59412e78fa2611943790485e0b5e4d5305c2cae360"}, + {"name": "0000_549_549519_qa_1", "source": "0000/549/549519.ipynb_qa_1", "notebook": "0000/549/549519.ipynb", "difficulty": "hard", "instruction_sha256": "89a11870b7bba0b86d0d813c99147889da41c05b61bf081165cdc45744175460"}, + {"name": "0000_551_551429_qa_1", "source": "0000/551/551429.ipynb_qa_1", "notebook": "0000/551/551429.ipynb", "difficulty": "easy", "instruction_sha256": "a2efaaa0bcaff4125bf9b836449d5fe6f4332e52843048675e7f19db9174f404"}, + {"name": "0000_561_561293_qa_4", "source": "0000/561/561293.ipynb_qa_4", "notebook": "0000/561/561293.ipynb", "difficulty": "medium", "instruction_sha256": "efebaef984a86ae3ae705ff7757e26a5b78626d06f3ffbdde0fde0c1d7394eb8"}, + {"name": "0000_566_566544_qa_3", "source": "0000/566/566544.ipynb_qa_3", "notebook": "0000/566/566544.ipynb", "difficulty": "hard", "instruction_sha256": "6bd12a2b5c344e8fb22f8034a803acc0da8921a94e02b317db9651d7377fd33a"}, + {"name": "0000_577_577064_qa_3", "source": "0000/577/577064.ipynb_qa_3", "notebook": "0000/577/577064.ipynb", "difficulty": "hard", "instruction_sha256": "b06ff2b2b5a1df6280812c080f9327b720c8b2c46dcc3392d917257dd7e23659"}, + {"name": "0000_577_577064_qa_5", "source": "0000/577/577064.ipynb_qa_5", "notebook": "0000/577/577064.ipynb", "difficulty": "hard", "instruction_sha256": "24e9b8a97daa06efb4cd4e94d8a55b0682ed80eef42bfb28502922d6e83f5102"}, + {"name": "0000_582_582934_qa_3", "source": "0000/582/582934.ipynb_qa_3", "notebook": "0000/582/582934.ipynb", "difficulty": "medium", "instruction_sha256": "ab8fb0eafd3beff9817b968a2f570fb623e3447713df3cd93122e93de7acb9ff"}, + {"name": "0000_593_593356_qa_1", "source": "0000/593/593356.ipynb_qa_1", "notebook": "0000/593/593356.ipynb", "difficulty": "easy", "instruction_sha256": "837b40e81db2c27e18b939bfbac4c56f4b26a637fca4cd4f98bbd27a514b756c"}, + {"name": "0000_593_593356_qa_4", "source": "0000/593/593356.ipynb_qa_4", "notebook": "0000/593/593356.ipynb", "difficulty": "easy", "instruction_sha256": "522dec56afb0fb74dfcee671aaa1ac5cee0bccc4fc90ec67ac5b9c909185c6f5"}, + {"name": "0000_603_603758_qa_1", "source": "0000/603/603758.ipynb_qa_1", "notebook": "0000/603/603758.ipynb", "difficulty": "medium", "instruction_sha256": "319f092ed597b1432c409918b65b6b8dce99b93f885d0c63b9f70ec67ef8e009"}, + {"name": "0000_621_621324_qa_4", "source": "0000/621/621324.ipynb_qa_4", "notebook": "0000/621/621324.ipynb", "difficulty": "medium", "instruction_sha256": "e06b0002cb35bb3215fb2d7e322186654c8a54b6fbeba726d1496089b5bbf259"}, + {"name": "0000_641_641256_qa_3", "source": "0000/641/641256.ipynb_qa_3", "notebook": "0000/641/641256.ipynb", "difficulty": "medium", "instruction_sha256": "df1c0b9d077fa19786f972b08fc92c0a3aae908ee10f38f2990e9fbaed7d1e78"}, + {"name": "0000_647_647453_qa_3", "source": "0000/647/647453.ipynb_qa_3", "notebook": "0000/647/647453.ipynb", "difficulty": "medium", "instruction_sha256": "65050bea6f1b595a92d5c0d1a9523e9d3803f5fe3756384aa38b0bd8da998402"}, + {"name": "0000_649_649690_qa_4", "source": "0000/649/649690.ipynb_qa_4", "notebook": "0000/649/649690.ipynb", "difficulty": "easy", "instruction_sha256": "e1b1f2fe41bdabcf329b20e7a394a945b995360f9082bf7d0e19d01314457e0d"}, + {"name": "0000_651_651325_qa_3", "source": "0000/651/651325.ipynb_qa_3", "notebook": "0000/651/651325.ipynb", "difficulty": "medium", "instruction_sha256": "4680edb05fa3f91dadfe50522c56fa1a29b4c5b9f92907b480ee909acf1d93b4"}, + {"name": "0000_651_651325_qa_4", "source": "0000/651/651325.ipynb_qa_4", "notebook": "0000/651/651325.ipynb", "difficulty": "easy", "instruction_sha256": "9e7db4e2f4660256317f9aa610c417a21b6aa90b19f92501cd0dbc750da599c5"}, + {"name": "0000_656_656399_qa_4", "source": "0000/656/656399.ipynb_qa_4", "notebook": "0000/656/656399.ipynb", "difficulty": "hard", "instruction_sha256": "7c45aced8fe16adba54b514a5f3d25001ff53d5c2123d0015f67a73981e67853"}, + {"name": "0000_658_658395_qa_1", "source": "0000/658/658395.ipynb_qa_1", "notebook": "0000/658/658395.ipynb", "difficulty": "medium", "instruction_sha256": "ae339d42c3b652cf4bef4138a0bd1de3bfd7838b087d879ebea99ac32048331d"}, + {"name": "0000_658_658395_qa_3", "source": "0000/658/658395.ipynb_qa_3", "notebook": "0000/658/658395.ipynb", "difficulty": "medium", "instruction_sha256": "5cbc681e1ba1c21a15d66d5b67781a5e7825a106008973fe4cf326804338aaff"}, + {"name": "0000_660_660223_qa_1", "source": "0000/660/660223.ipynb_qa_1", "notebook": "0000/660/660223.ipynb", "difficulty": "easy", "instruction_sha256": "daf19149ff52aacdcb8166eaae1c15d26db61c28a6a8f534c86d32d352abc3db"}, + {"name": "0000_663_663164_qa_1", "source": "0000/663/663164.ipynb_qa_1", "notebook": "0000/663/663164.ipynb", "difficulty": "medium", "instruction_sha256": "67be63bb477517ca71cda922ecbd717565ee3073f9159d44107f4b20b113d245"}, + {"name": "0000_667_667350_qa_3", "source": "0000/667/667350.ipynb_qa_3", "notebook": "0000/667/667350.ipynb", "difficulty": "medium", "instruction_sha256": "64a938c15df7869a21ab28844f489498f87684ce8409fd9c139dc355a24e77c6"}, + {"name": "0000_667_667579_qa_1", "source": "0000/667/667579.ipynb_qa_1", "notebook": "0000/667/667579.ipynb", "difficulty": "medium", "instruction_sha256": "59d7cd5d15a780f8cf9f55d2841c8e42204abb6c4602c52be642e3fd7706b521"}, + {"name": "0000_667_667579_qa_3", "source": "0000/667/667579.ipynb_qa_3", "notebook": "0000/667/667579.ipynb", "difficulty": "easy", "instruction_sha256": "999914485081a8c8742d4709dd2f1c8becbcb2ca0e15d1c50d652a239da8e2c3"}, + {"name": "0000_672_672627_qa_1", "source": "0000/672/672627.ipynb_qa_1", "notebook": "0000/672/672627.ipynb", "difficulty": "medium", "instruction_sha256": "a118ca39a13553379f59c2f52ac8e5f57451188d78fcc4ff3eb7c7d7100325d7"}, + {"name": "0000_672_672627_qa_2", "source": "0000/672/672627.ipynb_qa_2", "notebook": "0000/672/672627.ipynb", "difficulty": "medium", "instruction_sha256": "1aa6da56fd3c1df99c271c1589949b2b5bbf9e4225c22319d06428d449b301e7"}, + {"name": "0000_672_672627_qa_4", "source": "0000/672/672627.ipynb_qa_4", "notebook": "0000/672/672627.ipynb", "difficulty": "hard", "instruction_sha256": "c208af7dc59b41a590023a8f434f2c9cd5abadf4e4ca838f1dfcb430d06299b1"}, + {"name": "0000_680_680306_qa_1", "source": "0000/680/680306.ipynb_qa_1", "notebook": "0000/680/680306.ipynb", "difficulty": "easy", "instruction_sha256": "992b85fe340bbb85c44200a28b66a4d34d6ce73b018e2ca1f7bbad911a779690"}, + {"name": "0000_682_682562_qa_1", "source": "0000/682/682562.ipynb_qa_1", "notebook": "0000/682/682562.ipynb", "difficulty": "medium", "instruction_sha256": "8ca32e6cb9dafec55faefe0ef9b94f6c24b038bfeb6bf8a05fcaf4161125c17f"}, + {"name": "0000_682_682562_qa_2", "source": "0000/682/682562.ipynb_qa_2", "notebook": "0000/682/682562.ipynb", "difficulty": "medium", "instruction_sha256": "9117cd8c4654316aa03ddc453247a3a43d8878cabf21999961a169c72e77ed06"}, + {"name": "0000_684_684849_qa_1", "source": "0000/684/684849.ipynb_qa_1", "notebook": "0000/684/684849.ipynb", "difficulty": "easy", "instruction_sha256": "a6568d2d48d6c8613d846247b43b12302053db7312d67496cbb6414a17c95769"}, + {"name": "0000_684_684849_qa_2", "source": "0000/684/684849.ipynb_qa_2", "notebook": "0000/684/684849.ipynb", "difficulty": "easy", "instruction_sha256": "4d2cd7576e5941a4129630685267c7aa0c410d55a0b6c23f07816fe1554ac0df"}, + {"name": "0000_684_684849_qa_3", "source": "0000/684/684849.ipynb_qa_3", "notebook": "0000/684/684849.ipynb", "difficulty": "medium", "instruction_sha256": "232a31c7db1871385e04f4a26243a6effd9c47305ba4ce0a15993bff939ccfda"}, + {"name": "0000_684_684849_qa_4", "source": "0000/684/684849.ipynb_qa_4", "notebook": "0000/684/684849.ipynb", "difficulty": "easy", "instruction_sha256": "aa9b52a1677e5b04e548f5d90c5a01193f11af078bc3d13a9db793471771522f"}, + {"name": "0000_689_689967_qa_5", "source": "0000/689/689967.ipynb_qa_5", "notebook": "0000/689/689967.ipynb", "difficulty": "medium", "instruction_sha256": "6f36e1970f9283442ccd37f89764501c40860ea4efe04e16348008fc1fe8cf9e"}, + {"name": "0000_696_696467_qa_5", "source": "0000/696/696467.ipynb_qa_5", "notebook": "0000/696/696467.ipynb", "difficulty": "hard", "instruction_sha256": "81599d9fc2ed82843e1a0d746d2a784f5681ee6029076d322e3c833b0fa4e398"}, + {"name": "0000_710_710837_qa_4", "source": "0000/710/710837.ipynb_qa_4", "notebook": "0000/710/710837.ipynb", "difficulty": "easy", "instruction_sha256": "9429c146379c49f0c2371404dd1d41f4150004d9652a73f3d632b9f2d4a4bda1"}, + {"name": "0000_717_717375_qa_1", "source": "0000/717/717375.ipynb_qa_1", "notebook": "0000/717/717375.ipynb", "difficulty": "medium", "instruction_sha256": "06506d29207e5c4312258735c95818212b40b4e5928cb96596e9488fd3b465ca"}, + {"name": "0000_730_730302_qa_3", "source": "0000/730/730302.ipynb_qa_3", "notebook": "0000/730/730302.ipynb", "difficulty": "medium", "instruction_sha256": "503979c88723ef843f326f0c3e365496c620b3472ea2efdf222a28f19855ac76"}, + {"name": "0000_731_731312_qa_3", "source": "0000/731/731312.ipynb_qa_3", "notebook": "0000/731/731312.ipynb", "difficulty": "medium", "instruction_sha256": "c76ec2b00cc490c7530815c1f06f4a5d44ddd6b619fd62a6e170af54c3da36a0"}, + {"name": "0000_731_731973_qa_4", "source": "0000/731/731973.ipynb_qa_4", "notebook": "0000/731/731973.ipynb", "difficulty": "medium", "instruction_sha256": "7416dc6ec3b5dd4d20f77f85c10de337f835f444368a7936e6dbb1dd53d0c3e4"}, + {"name": "0000_731_731973_qa_5", "source": "0000/731/731973.ipynb_qa_5", "notebook": "0000/731/731973.ipynb", "difficulty": "medium", "instruction_sha256": "3372b3a8e455a924afc9d1f3fce0c4cbde7993455ee1abbd8946be8c3393f83e"}, + {"name": "0000_740_740974_qa_1", "source": "0000/740/740974.ipynb_qa_1", "notebook": "0000/740/740974.ipynb", "difficulty": "medium", "instruction_sha256": "51076b6f80b88235589e60d2e5689f40fc56d53214a692e13c16623d06595411"}, + {"name": "0000_746_746224_qa_4", "source": "0000/746/746224.ipynb_qa_4", "notebook": "0000/746/746224.ipynb", "difficulty": "easy", "instruction_sha256": "bc0ce441ef8c2529a24d09ef9e2ead9d520fac6241c2e9660ff0b050d85cfaf4"}, + {"name": "0000_748_748225_qa_1", "source": "0000/748/748225.ipynb_qa_1", "notebook": "0000/748/748225.ipynb", "difficulty": "hard", "instruction_sha256": "d5d9ddd66f2d34b2d792f811f0eb090e3103c13649d9caa557bd0c6b08b7a6f9"}, + {"name": "0000_748_748225_qa_2", "source": "0000/748/748225.ipynb_qa_2", "notebook": "0000/748/748225.ipynb", "difficulty": "easy", "instruction_sha256": "058e577f4f88ddb5a012f1cf70356c3e5b35028de6565df29921fdbebce1c2e6"}, + {"name": "0000_757_757598_qa_2", "source": "0000/757/757598.ipynb_qa_2", "notebook": "0000/757/757598.ipynb", "difficulty": "hard", "instruction_sha256": "7fc59ba6822b520cf5f4548b7f7a3cbfc1c920f2db7021afdac7ae24c4b8f802"}, + {"name": "0000_761_761305_qa_2", "source": "0000/761/761305.ipynb_qa_2", "notebook": "0000/761/761305.ipynb", "difficulty": "hard", "instruction_sha256": "9bbfa570560a5df0ddce5aadbed32bdbd2eb495ae23a99fd86b4b479b0dfc329"}, + {"name": "0000_764_764519_qa_3", "source": "0000/764/764519.ipynb_qa_3", "notebook": "0000/764/764519.ipynb", "difficulty": "medium", "instruction_sha256": "f78e0e7e974d8b537cf39fe956ea9d4a4e7a9f61a8b6b17b4aad7fb1183a3eb1"}, + {"name": "0000_774_774985_qa_2", "source": "0000/774/774985.ipynb_qa_2", "notebook": "0000/774/774985.ipynb", "difficulty": "medium", "instruction_sha256": "383b5f3438bda3280a234ba459d6bc2f4ca31f2ee130b06e98581490407dd718"}, + {"name": "0000_774_774985_qa_3", "source": "0000/774/774985.ipynb_qa_3", "notebook": "0000/774/774985.ipynb", "difficulty": "easy", "instruction_sha256": "949b9d763834af414d2fd8f764de90bf7f05546e3529ed692cb1a42380ffcd2a"}, + {"name": "0000_780_780974_qa_2", "source": "0000/780/780974.ipynb_qa_2", "notebook": "0000/780/780974.ipynb", "difficulty": "medium", "instruction_sha256": "9b0097f35a24e14c9b234459c810528139656fc50db9693ebf242e5ea8873ee7"}, + {"name": "0000_785_785253_qa_1", "source": "0000/785/785253.ipynb_qa_1", "notebook": "0000/785/785253.ipynb", "difficulty": "hard", "instruction_sha256": "a0cdabdf41fdd56f6f32c91271be895b6823e91e9559358222b1e0fae445b474"}, + {"name": "0000_787_787044_qa_1", "source": "0000/787/787044.ipynb_qa_1", "notebook": "0000/787/787044.ipynb", "difficulty": "medium", "instruction_sha256": "56e9269d71068da68325d160c21c1db23946c0ad91d095d4fd88f1c8e3a2dfd3"}, + {"name": "0000_791_791422_qa_3", "source": "0000/791/791422.ipynb_qa_3", "notebook": "0000/791/791422.ipynb", "difficulty": "medium", "instruction_sha256": "01cc9e42d9ac2304377fac1f60892168e6f97e47e7f527d4b684de505d74f3b1"}, + {"name": "0000_806_806826_qa_5", "source": "0000/806/806826.ipynb_qa_5", "notebook": "0000/806/806826.ipynb", "difficulty": "medium", "instruction_sha256": "3de49764252480764f3f0d90b927bdf77bc213b14ab2515cf7c7ca86e166db97"}, + {"name": "0000_807_807075_qa_1", "source": "0000/807/807075.ipynb_qa_1", "notebook": "0000/807/807075.ipynb", "difficulty": "medium", "instruction_sha256": "64e9944e641909a06e53be02255df6a121f6367346f91b386d2e49042c842e07"}, + {"name": "0000_812_812048_qa_1", "source": "0000/812/812048.ipynb_qa_1", "notebook": "0000/812/812048.ipynb", "difficulty": "medium", "instruction_sha256": "bd581e171c6c52642986e824fa2ea148fab451b5b70d11b4ab0ea41ccba03e9f"}, + {"name": "0000_812_812048_qa_2", "source": "0000/812/812048.ipynb_qa_2", "notebook": "0000/812/812048.ipynb", "difficulty": "medium", "instruction_sha256": "4118a257dae241287e88c2519550cc91e5e5cc9ea8d04dd9a50f26d4d3760d1b"}, + {"name": "0000_813_813578_qa_5", "source": "0000/813/813578.ipynb_qa_5", "notebook": "0000/813/813578.ipynb", "difficulty": "medium", "instruction_sha256": "5e5b029471fa5d3382713fce73975e53419e6b516db800d56c495c6a5336dfa1"}, + {"name": "0000_816_816225_qa_3", "source": "0000/816/816225.ipynb_qa_3", "notebook": "0000/816/816225.ipynb", "difficulty": "medium", "instruction_sha256": "5238d69faf6d78129e9c754683ea8f5be3a95d0dca5d3092d773b27e0f3facb2"}, + {"name": "0000_822_822279_qa_1", "source": "0000/822/822279.ipynb_qa_1", "notebook": "0000/822/822279.ipynb", "difficulty": "medium", "instruction_sha256": "c641d9bcbd08566c8f3e76bd51be904d268790ec0f612c6ed766bd7e84dd3761"}, + {"name": "0000_822_822279_qa_2", "source": "0000/822/822279.ipynb_qa_2", "notebook": "0000/822/822279.ipynb", "difficulty": "medium", "instruction_sha256": "dff13714238cf63f28ba853c76e0d9973309a5e35755651eb57e38353978f67d"}, + {"name": "0000_822_822279_qa_4", "source": "0000/822/822279.ipynb_qa_4", "notebook": "0000/822/822279.ipynb", "difficulty": "easy", "instruction_sha256": "1d6077fabd21bf78ec5d947d119b3b4a84080bb3e448651ff66158fa886f7d5c"}, + {"name": "0000_822_822279_qa_5", "source": "0000/822/822279.ipynb_qa_5", "notebook": "0000/822/822279.ipynb", "difficulty": "easy", "instruction_sha256": "5fe0b11c6fcafdc22863c2d49c6b2dc76416863fa44f23648802c566a92e5bf8"}, + {"name": "0000_837_837727_qa_4", "source": "0000/837/837727.ipynb_qa_4", "notebook": "0000/837/837727.ipynb", "difficulty": "medium", "instruction_sha256": "19511e8fed5300ae1dd69e7c905666f59aa2e549c2221e911a1489f599de01c7"}, + {"name": "0000_838_838183_qa_2", "source": "0000/838/838183.ipynb_qa_2", "notebook": "0000/838/838183.ipynb", "difficulty": "easy", "instruction_sha256": "881b0c1d40d29e5c09520eebe281d092a4a11d8f8caecbbeb2a00582267f0b68"}, + {"name": "0000_840_840617_qa_3", "source": "0000/840/840617.ipynb_qa_3", "notebook": "0000/840/840617.ipynb", "difficulty": "easy", "instruction_sha256": "b15b0bfe48ac8dd2e6b1c579273bbf19ea6c44036ea9ba74fbbf71397194d454"}, + {"name": "0000_847_847719_qa_3", "source": "0000/847/847719.ipynb_qa_3", "notebook": "0000/847/847719.ipynb", "difficulty": "medium", "instruction_sha256": "991fb62da199af9a7f02d375ae55293441d0e2fe046cb8300913e083ec663563"}, + {"name": "0000_847_847719_qa_5", "source": "0000/847/847719.ipynb_qa_5", "notebook": "0000/847/847719.ipynb", "difficulty": "medium", "instruction_sha256": "4a32e4206835aada938093a99b2d2bd1498cfe797a6498cb932d5ad37059464c"}, + {"name": "0000_856_856349_qa_4", "source": "0000/856/856349.ipynb_qa_4", "notebook": "0000/856/856349.ipynb", "difficulty": "medium", "instruction_sha256": "1e09d9a7fe835c9e29384332d173389ca4f7c94253771311c305bb1928635f29"}, + {"name": "0000_862_862257_qa_5", "source": "0000/862/862257.ipynb_qa_5", "notebook": "0000/862/862257.ipynb", "difficulty": "hard", "instruction_sha256": "4a0a4312b4d4317884201814e1d2de398d6d1ec8cc5b76e3f7a79b4b9a2f7ee1"}, + {"name": "0000_868_868084_qa_4", "source": "0000/868/868084.ipynb_qa_4", "notebook": "0000/868/868084.ipynb", "difficulty": "medium", "instruction_sha256": "ea8e3c0d1dc5da296a20c87caec90a0d70b8f5471f1f5fd8b506899f223138e4"}, + {"name": "0000_903_903504_qa_3", "source": "0000/903/903504.ipynb_qa_3", "notebook": "0000/903/903504.ipynb", "difficulty": "medium", "instruction_sha256": "29e60737681a3783b9bd23ae666d48b8201246456f52d30132ee0050b60f8ac8"}, + {"name": "0000_965_965367_qa_1", "source": "0000/965/965367.ipynb_qa_1", "notebook": "0000/965/965367.ipynb", "difficulty": "medium", "instruction_sha256": "6816747df4881b4bb6ce32cde598fe72bbaadd092cb5119ae339134102c4f348"}, + {"name": "0000_965_965367_qa_2", "source": "0000/965/965367.ipynb_qa_2", "notebook": "0000/965/965367.ipynb", "difficulty": "medium", "instruction_sha256": "e49395a1842c7b131e2d145ca6fb0933a0c15052e15ac7311bf80bc1fba0711e"}, + {"name": "0000_965_965367_qa_3", "source": "0000/965/965367.ipynb_qa_3", "notebook": "0000/965/965367.ipynb", "difficulty": "medium", "instruction_sha256": "474b8a360beb674a0de536b8df301383ffb909a721b39b5920b0412a3ada5d65"}, + {"name": "0000_982_982214_qa_4", "source": "0000/982/982214.ipynb_qa_4", "notebook": "0000/982/982214.ipynb", "difficulty": "hard", "instruction_sha256": "8ce6f8f48e37de38aafe61a2d581bbf94e743d880dcdead194c1fa23c25aa89d"}, + {"name": "0000_982_982214_qa_5", "source": "0000/982/982214.ipynb_qa_5", "notebook": "0000/982/982214.ipynb", "difficulty": "hard", "instruction_sha256": "6b0a9cc988d8a058809b310e74b2b5701eef666abc3b262715a36edb703dac7a"}, + {"name": "0001_004_1004716_qa_5", "source": "0001/004/1004716.ipynb_qa_5", "notebook": "0001/004/1004716.ipynb", "difficulty": "medium", "instruction_sha256": "0fafb8b18a1c295c88082e0239ca2aea3830cabe109dfbec80f6673b136d2c4a"}, + {"name": "0001_028_1028511_qa_3", "source": "0001/028/1028511.ipynb_qa_3", "notebook": "0001/028/1028511.ipynb", "difficulty": "hard", "instruction_sha256": "b5dac5cd464bc78ebb9592f22edfcd3f7499bad2425144ce4f84055043f6c6ab"}, + {"name": "0001_043_1043420_qa_4", "source": "0001/043/1043420.ipynb_qa_4", "notebook": "0001/043/1043420.ipynb", "difficulty": "hard", "instruction_sha256": "c1bed5635cc7ad8c51427cee3b0f2418f370111be204e64258c9a19ef76c61f6"}, + {"name": "0001_043_1043981_qa_3", "source": "0001/043/1043981.ipynb_qa_3", "notebook": "0001/043/1043981.ipynb", "difficulty": "hard", "instruction_sha256": "997c4417ac706b868b96fde55bc51f6cfb41a5b7a128904e8306e7f3e4da6c54"}, + {"name": "0001_046_1046894_qa_2", "source": "0001/046/1046894.ipynb_qa_2", "notebook": "0001/046/1046894.ipynb", "difficulty": "hard", "instruction_sha256": "a18650c85945e4948b5440fe752d2f124a8ef75c8bac7741d774d08995f45719"}, + {"name": "0001_046_1046894_qa_3", "source": "0001/046/1046894.ipynb_qa_3", "notebook": "0001/046/1046894.ipynb", "difficulty": "hard", "instruction_sha256": "bf8d039ac3da6a9c3e5b0fdd60f6cf5e91f3e686412f50a77efa95e5feeaa623"}, + {"name": "0001_069_1069844_qa_2", "source": "0001/069/1069844.ipynb_qa_2", "notebook": "0001/069/1069844.ipynb", "difficulty": "medium", "instruction_sha256": "838761d0699ccb38146436237103695e44d753e9ba3783df958d815d82e1a025"}, + {"name": "0001_090_1090831_qa_3", "source": "0001/090/1090831.ipynb_qa_3", "notebook": "0001/090/1090831.ipynb", "difficulty": "hard", "instruction_sha256": "39a053a0020b43cac6b6657d156bbe5cb5b32701d9419bd6dcea9ea8fd7b32fe"}, + {"name": "0001_095_1095590_qa_3", "source": "0001/095/1095590.ipynb_qa_3", "notebook": "0001/095/1095590.ipynb", "difficulty": "medium", "instruction_sha256": "e157c394f248df37704749b03c4980cf63c6416eb4878619b80a9321874f7d0a"}, + {"name": "0001_102_1102505_qa_1", "source": "0001/102/1102505.ipynb_qa_1", "notebook": "0001/102/1102505.ipynb", "difficulty": "medium", "instruction_sha256": "4abe08f4466db57272556c0b4632ef34b86161f9496b655b78a4f1e8aaef8dca"}, + {"name": "0001_127_1127835_qa_2", "source": "0001/127/1127835.ipynb_qa_2", "notebook": "0001/127/1127835.ipynb", "difficulty": "hard", "instruction_sha256": "82f99890ae4960ddb82cf263da81b9f5e08759cb6d332c0ab148aaec67afb731"}, + {"name": "0001_137_1137537_qa_3", "source": "0001/137/1137537.ipynb_qa_3", "notebook": "0001/137/1137537.ipynb", "difficulty": "hard", "instruction_sha256": "42eaa0e8a3c15efdfe5c04b022dcd43cbc55627286921224764626edb105ea6d"}, + {"name": "0001_155_1155013_qa_2", "source": "0001/155/1155013.ipynb_qa_2", "notebook": "0001/155/1155013.ipynb", "difficulty": "medium", "instruction_sha256": "2fe4eb92819f25c1d37a2bf09052f9dda8794d836acbba8f12812b449b907911"}, + {"name": "0001_155_1155013_qa_3", "source": "0001/155/1155013.ipynb_qa_3", "notebook": "0001/155/1155013.ipynb", "difficulty": "hard", "instruction_sha256": "91e9722d858487535f93aa0b0160858aa5916676cd675d4ed5377c10265cf26b"}, + {"name": "0001_163_1163742_qa_1", "source": "0001/163/1163742.ipynb_qa_1", "notebook": "0001/163/1163742.ipynb", "difficulty": "hard", "instruction_sha256": "722485a71d9ac79b21771c1ecc8d49720c2d3b869a4d85fab1007f5ccb811744"}, + {"name": "0001_163_1163742_qa_5", "source": "0001/163/1163742.ipynb_qa_5", "notebook": "0001/163/1163742.ipynb", "difficulty": "hard", "instruction_sha256": "7dc62cc314b1514cbe35b802e95a315f5f062ed4a18e5d1422357aaa9815c0f7"}, + {"name": "0001_176_1176622_qa_1", "source": "0001/176/1176622.ipynb_qa_1", "notebook": "0001/176/1176622.ipynb", "difficulty": "medium", "instruction_sha256": "dbfc495af0c45110e712aba34f1a5398b5641f236a541566364c4f1a8735c0fc"}, + {"name": "0001_181_1181828_qa_3", "source": "0001/181/1181828.ipynb_qa_3", "notebook": "0001/181/1181828.ipynb", "difficulty": "medium", "instruction_sha256": "9523d75f20c6562e8e0734fdc5855b11eb87c8c8ee54841309ebdd6b1fcdd048"}, + {"name": "0001_188_1188888_qa_3", "source": "0001/188/1188888.ipynb_qa_3", "notebook": "0001/188/1188888.ipynb", "difficulty": "hard", "instruction_sha256": "ca96fcd878ce0fd8ab286683ba0a4f028a3842c97fe2418b943d903b9fd5b5f8"}, + {"name": "0001_214_1214633_qa_4", "source": "0001/214/1214633.ipynb_qa_4", "notebook": "0001/214/1214633.ipynb", "difficulty": "medium", "instruction_sha256": "0f4ed63efd45b428f01828b6958f26f735078fa3de70558874354b69ddff154e"}, + {"name": "0001_232_1232019_qa_4", "source": "0001/232/1232019.ipynb_qa_4", "notebook": "0001/232/1232019.ipynb", "difficulty": "hard", "instruction_sha256": "ba0a76a802c4dd2a0cb2e3392124bd574e22d932369dfc8e49a704eaff292160"}, + {"name": "0001_233_1233959_qa_3", "source": "0001/233/1233959.ipynb_qa_3", "notebook": "0001/233/1233959.ipynb", "difficulty": "hard", "instruction_sha256": "330f0027e19f1ed16b4646923ff5b5d1e369cff4f77ff89ed44753d4fba36031"}, + {"name": "0001_237_1237016_qa_1", "source": "0001/237/1237016.ipynb_qa_1", "notebook": "0001/237/1237016.ipynb", "difficulty": "medium", "instruction_sha256": "cba63fbc25ee575acc2838bd59f5eb8c1d3d2be8bd27505516705083d3c378aa"}, + {"name": "0001_238_1238983_qa_2", "source": "0001/238/1238983.ipynb_qa_2", "notebook": "0001/238/1238983.ipynb", "difficulty": "medium", "instruction_sha256": "edfc76425a5d3ebdc2578557cfaca1f974b32f043b14601690744654557ca94a"}, + {"name": "0001_242_1242706_qa_1", "source": "0001/242/1242706.ipynb_qa_1", "notebook": "0001/242/1242706.ipynb", "difficulty": "medium", "instruction_sha256": "c36dc0c58b996988c105ddbfb3a68333d9b33490f8fd3ee3f3d15bc58a563add"}, + {"name": "0001_250_1250662_qa_1", "source": "0001/250/1250662.ipynb_qa_1", "notebook": "0001/250/1250662.ipynb", "difficulty": "medium", "instruction_sha256": "856b8a0a6f19dbae4ec1c3cb8924af0e76b6d8bb1bce27d53e5272cf6e3b53ac"}, + {"name": "0001_250_1250662_qa_3", "source": "0001/250/1250662.ipynb_qa_3", "notebook": "0001/250/1250662.ipynb", "difficulty": "medium", "instruction_sha256": "0dd23bb948b52aaab3e85f9c48e2066f8e7e09efde78de864cfa9c7a5469e364"}, + {"name": "0001_264_1264151_qa_2", "source": "0001/264/1264151.ipynb_qa_2", "notebook": "0001/264/1264151.ipynb", "difficulty": "medium", "instruction_sha256": "51d44403f57225e8d51cb0f467a5d99ae52c8105cdbbd4ee034885acf297428a"}, + {"name": "0001_264_1264151_qa_3", "source": "0001/264/1264151.ipynb_qa_3", "notebook": "0001/264/1264151.ipynb", "difficulty": "medium", "instruction_sha256": "710ff6f875e40a87d338f54b5f15fea535f897ff9e8b501ad1d31e440f948e6a"}, + {"name": "0001_269_1269679_qa_5", "source": "0001/269/1269679.ipynb_qa_5", "notebook": "0001/269/1269679.ipynb", "difficulty": "hard", "instruction_sha256": "144a03daaf7b90d481b732f6d078422ce9aa018642c8bf4abe36fbf3155b6b8c"}, + {"name": "0001_273_1273208_qa_3", "source": "0001/273/1273208.ipynb_qa_3", "notebook": "0001/273/1273208.ipynb", "difficulty": "medium", "instruction_sha256": "6fb83f2bdbd7976be9e0c19aaedd3b65c92dbbb19ebf15e118b12fba0ad87b7c"}, + {"name": "0001_296_1296784_qa_5", "source": "0001/296/1296784.ipynb_qa_5", "notebook": "0001/296/1296784.ipynb", "difficulty": "hard", "instruction_sha256": "fb1d4b9d73ec1f73a3833335f03e23e67ab2d14f68ae4eb568c83ecb0d66cf41"}, + {"name": "0001_312_1312239_qa_5", "source": "0001/312/1312239.ipynb_qa_5", "notebook": "0001/312/1312239.ipynb", "difficulty": "medium", "instruction_sha256": "87b33ab8856dad1de9ff210b679d980fb0902614432db1289f493eaad3cd6ec1"}, + {"name": "0001_325_1325110_qa_4", "source": "0001/325/1325110.ipynb_qa_4", "notebook": "0001/325/1325110.ipynb", "difficulty": "medium", "instruction_sha256": "cbe2f999b974327095f42c5917614145ec964724fd2810c100a26158be8ee843"}, + {"name": "0001_325_1325260_qa_1", "source": "0001/325/1325260.ipynb_qa_1", "notebook": "0001/325/1325260.ipynb", "difficulty": "medium", "instruction_sha256": "06635327abeea0bd5897b11420ad334cefe2d4b2a6a4195f0df39388c7898ce9"}, + {"name": "0001_333_1333685_qa_2", "source": "0001/333/1333685.ipynb_qa_2", "notebook": "0001/333/1333685.ipynb", "difficulty": "medium", "instruction_sha256": "af4d45f475874e49779a0e422cf2a44ca7aa898eb5dba75acfcb3be74ca16f5c"}, + {"name": "0001_341_1341821_qa_4", "source": "0001/341/1341821.ipynb_qa_4", "notebook": "0001/341/1341821.ipynb", "difficulty": "medium", "instruction_sha256": "441857e97034ee06adfd44de3c8ba153571c86ab8e371ab669eec14c2a4ec4b6"}, + {"name": "0001_347_1347384_qa_1", "source": "0001/347/1347384.ipynb_qa_1", "notebook": "0001/347/1347384.ipynb", "difficulty": "medium", "instruction_sha256": "f15076959c13118fb4b4f7b8e1c1e3cce5bf2119e8bdad4639618f659dd6f326"}, + {"name": "0001_364_1364973_qa_2", "source": "0001/364/1364973.ipynb_qa_2", "notebook": "0001/364/1364973.ipynb", "difficulty": "medium", "instruction_sha256": "e7a54d2480a8261e20ae54c08c0826e171df5b36d2e9413462fc58333cbca24b"}, + {"name": "0001_365_1365152_qa_1", "source": "0001/365/1365152.ipynb_qa_1", "notebook": "0001/365/1365152.ipynb", "difficulty": "hard", "instruction_sha256": "4c949bc61c05ead7427f41b6432a89c2bcc1cbd52371812f0755097d41be3faf"}, + {"name": "0001_381_1381379_qa_2", "source": "0001/381/1381379.ipynb_qa_2", "notebook": "0001/381/1381379.ipynb", "difficulty": "medium", "instruction_sha256": "7c21434aa1d22af52db0d88bcb93863fda79bddc7f62dfa8546541581bb08e89"}, + {"name": "0001_404_1404790_qa_5", "source": "0001/404/1404790.ipynb_qa_5", "notebook": "0001/404/1404790.ipynb", "difficulty": "hard", "instruction_sha256": "3116b3f4848a11b7d80d0ccdf720ff1a1595da30df17e4419b5a9aa5bf8a8554"}, + {"name": "0001_436_1436410_qa_4", "source": "0001/436/1436410.ipynb_qa_4", "notebook": "0001/436/1436410.ipynb", "difficulty": "medium", "instruction_sha256": "a71d099d9eb856c7dee8fe1cecb8b54dddf83a185e93d3e0f76aa69af1edaeb0"}, + {"name": "0001_439_1439907_qa_3", "source": "0001/439/1439907.ipynb_qa_3", "notebook": "0001/439/1439907.ipynb", "difficulty": "medium", "instruction_sha256": "bce2b562750901aca9a0f390bd443ed412c77f3e80fabe691e2e360c79a47431"}, + {"name": "0001_441_1441202_qa_4", "source": "0001/441/1441202.ipynb_qa_4", "notebook": "0001/441/1441202.ipynb", "difficulty": "hard", "instruction_sha256": "b9817f1c748eaa954e2dcbc97d768dac43ba56fbe93654a89acff0720c7a2bc5"}, + {"name": "0001_443_1443137_qa_3", "source": "0001/443/1443137.ipynb_qa_3", "notebook": "0001/443/1443137.ipynb", "difficulty": "medium", "instruction_sha256": "ddf913a66e2981749b73131c7d243e94022b176be2bec0e78c6941c741428220"}, + {"name": "0001_445_1445988_qa_2", "source": "0001/445/1445988.ipynb_qa_2", "notebook": "0001/445/1445988.ipynb", "difficulty": "hard", "instruction_sha256": "a0f71504c474fab7356714ba844f7a4f3577ac51d322229f5332d92149656b6c"}, + {"name": "0001_446_1446067_qa_1", "source": "0001/446/1446067.ipynb_qa_1", "notebook": "0001/446/1446067.ipynb", "difficulty": "hard", "instruction_sha256": "ed4d597ef6df8f6aaf0aa76369919569c49af37dc20c8687b75c798e49b185e5"}, + {"name": "0001_446_1446695_qa_3", "source": "0001/446/1446695.ipynb_qa_3", "notebook": "0001/446/1446695.ipynb", "difficulty": "medium", "instruction_sha256": "19b3ad1506cf88e5e99d8ee8864a5b7c5838b5a70019369c445aaeaa2acd2e32"}, + {"name": "0001_466_1466571_qa_1", "source": "0001/466/1466571.ipynb_qa_1", "notebook": "0001/466/1466571.ipynb", "difficulty": "medium", "instruction_sha256": "a2b3eef5a1d70646e315f84abfd84b7284169dc5d59349f82312d99514aeca16"}, + {"name": "0001_481_1481274_qa_3", "source": "0001/481/1481274.ipynb_qa_3", "notebook": "0001/481/1481274.ipynb", "difficulty": "hard", "instruction_sha256": "c22a2070c360c06e7ace1973566b3c512ebb4f98ca1eefb289e224d3f065bb0d"}, + {"name": "0001_487_1487950_qa_5", "source": "0001/487/1487950.ipynb_qa_5", "notebook": "0001/487/1487950.ipynb", "difficulty": "medium", "instruction_sha256": "545722def39cdec59a4f25010308c6935a5779437c2819760a81522100c3ea53"}, + {"name": "0001_490_1490464_qa_3", "source": "0001/490/1490464.ipynb_qa_3", "notebook": "0001/490/1490464.ipynb", "difficulty": "medium", "instruction_sha256": "292718ad58c6169403cd70a769f6d633b8f638a848f04a6fcf3866c4f8347f1d"}, + {"name": "0001_497_1497755_qa_4", "source": "0001/497/1497755.ipynb_qa_4", "notebook": "0001/497/1497755.ipynb", "difficulty": "hard", "instruction_sha256": "3496acd25633477adbe148a2463880843546cf23bdb22cdaef0653965024f0f9"}, + {"name": "0001_505_1505707_qa_1", "source": "0001/505/1505707.ipynb_qa_1", "notebook": "0001/505/1505707.ipynb", "difficulty": "hard", "instruction_sha256": "9c026b86ed13305dc74184292a99b8a2b308725e4a0e35d312e24d4b8c90804c"}, + {"name": "0001_526_1526465_qa_2", "source": "0001/526/1526465.ipynb_qa_2", "notebook": "0001/526/1526465.ipynb", "difficulty": "hard", "instruction_sha256": "3a8d7f0f42717081e88d3fdef068a9320732dc354d1f96277636898aa12b4615"}, + {"name": "0001_527_1527107_qa_4", "source": "0001/527/1527107.ipynb_qa_4", "notebook": "0001/527/1527107.ipynb", "difficulty": "medium", "instruction_sha256": "6f7f953d21fcdbebb6b16d65edea41203cb94d04d2a9731cb264229f1dcb9e68"}, + {"name": "0001_530_1530114_qa_1", "source": "0001/530/1530114.ipynb_qa_1", "notebook": "0001/530/1530114.ipynb", "difficulty": "medium", "instruction_sha256": "78ff74f9f28f71888fa33257bc25a2fbf6cfda045f17e965049da913e60262b2"}, + {"name": "0001_532_1532354_qa_3", "source": "0001/532/1532354.ipynb_qa_3", "notebook": "0001/532/1532354.ipynb", "difficulty": "medium", "instruction_sha256": "88dea3f2d0be4d0e062c67aafaa84001376bd3fdf96afff063e28426bed20845"}, + {"name": "0001_533_1533644_qa_3", "source": "0001/533/1533644.ipynb_qa_3", "notebook": "0001/533/1533644.ipynb", "difficulty": "hard", "instruction_sha256": "522650762f2063ed143b8782977475d5313d0efc15f281abb1f7dec69adf7ee1"}, + {"name": "0001_541_1541084_qa_3", "source": "0001/541/1541084.ipynb_qa_3", "notebook": "0001/541/1541084.ipynb", "difficulty": "medium", "instruction_sha256": "6b227926e2cd6a3892297c0e3fab6f0947990c650bfe80d637ae9c0076c3c05d"}, + {"name": "0001_545_1545936_qa_2", "source": "0001/545/1545936.ipynb_qa_2", "notebook": "0001/545/1545936.ipynb", "difficulty": "medium", "instruction_sha256": "fe631c21c45a6b452f5964a6fbf9ecc01f8cd24bfc9151abc8865d38a694e561"}, + {"name": "0001_570_1570721_qa_4", "source": "0001/570/1570721.ipynb_qa_4", "notebook": "0001/570/1570721.ipynb", "difficulty": "hard", "instruction_sha256": "eb32513b83e397a1a88d9ff073ecb1a31c66056dca2b1ba7c4f658983419e8fc"}, + {"name": "0001_570_1570721_qa_5", "source": "0001/570/1570721.ipynb_qa_5", "notebook": "0001/570/1570721.ipynb", "difficulty": "hard", "instruction_sha256": "a736fcdcf7407ac5367d7a7df9114290a4289da852f35c39e1ecd61f525dacf2"}, + {"name": "0001_602_1602735_qa_3", "source": "0001/602/1602735.ipynb_qa_3", "notebook": "0001/602/1602735.ipynb", "difficulty": "hard", "instruction_sha256": "8005b14ddb96e5d55dcccf05d3eb3c8450b0bd0a727f72fead9d9e0bcf363d94"}, + {"name": "0001_632_1632381_qa_1", "source": "0001/632/1632381.ipynb_qa_1", "notebook": "0001/632/1632381.ipynb", "difficulty": "hard", "instruction_sha256": "311fb38f89fde5e7c47c2997ea1247cfe4c1417b3247088837f54804edd8200f"}, + {"name": "0001_638_1638152_qa_5", "source": "0001/638/1638152.ipynb_qa_5", "notebook": "0001/638/1638152.ipynb", "difficulty": "hard", "instruction_sha256": "518e685a36ef006fb519af29e1941ee36aae3510ca522689041c2a73a5ca9cf1"}, + {"name": "0001_640_1640467_qa_2", "source": "0001/640/1640467.ipynb_qa_2", "notebook": "0001/640/1640467.ipynb", "difficulty": "hard", "instruction_sha256": "43fed3cf658f70d92ee5c253dd8eb8cc9f10405412fbef0a33c3b2863663c684"}, + {"name": "0001_665_1665514_qa_2", "source": "0001/665/1665514.ipynb_qa_2", "notebook": "0001/665/1665514.ipynb", "difficulty": "hard", "instruction_sha256": "24393c1db5b9440fb89b3ceda99646d5c2f5fcff5269d7532737b29a77a44dc3"}, + {"name": "0001_668_1668567_qa_4", "source": "0001/668/1668567.ipynb_qa_4", "notebook": "0001/668/1668567.ipynb", "difficulty": "hard", "instruction_sha256": "2f44a2179a399d502dd54b67e280d48e231f8503c9f3df2ce9c724394ce3e1c9"}, + {"name": "0001_669_1669063_qa_3", "source": "0001/669/1669063.ipynb_qa_3", "notebook": "0001/669/1669063.ipynb", "difficulty": "hard", "instruction_sha256": "03244124453c72910872ad5f0867981b1a5acfee8091014eca187870380b5cd0"}, + {"name": "0001_672_1672543_qa_5", "source": "0001/672/1672543.ipynb_qa_5", "notebook": "0001/672/1672543.ipynb", "difficulty": "hard", "instruction_sha256": "1ba665756c4f3fcb0d07f2abdc7a9853170e4c81642b9fff058c5bc10a127c24"}, + {"name": "0001_696_1696691_qa_1", "source": "0001/696/1696691.ipynb_qa_1", "notebook": "0001/696/1696691.ipynb", "difficulty": "hard", "instruction_sha256": "6c14738467827b31deda06561554f1c3521fd11647a4ab5a848d26927d847787"}, + {"name": "0001_696_1696691_qa_3", "source": "0001/696/1696691.ipynb_qa_3", "notebook": "0001/696/1696691.ipynb", "difficulty": "hard", "instruction_sha256": "3cb33e4394b6837aa9942b597b5e88f2fdb1f65ff80a1e98cfd15653ec76c8b7"}, + {"name": "0001_727_1727043_qa_4", "source": "0001/727/1727043.ipynb_qa_4", "notebook": "0001/727/1727043.ipynb", "difficulty": "hard", "instruction_sha256": "429ffd1a334140546f178c4f4018e23e43532f13175dc7ea15684c0bd0cd3bbc"}, + {"name": "0001_757_1757917_qa_3", "source": "0001/757/1757917.ipynb_qa_3", "notebook": "0001/757/1757917.ipynb", "difficulty": "hard", "instruction_sha256": "a27ba8c5f95a5f3526c800182637473d31ce521f96ccc7a0e7bf754ade0b8a7c"}, + {"name": "0001_770_1770623_qa_1", "source": "0001/770/1770623.ipynb_qa_1", "notebook": "0001/770/1770623.ipynb", "difficulty": "hard", "instruction_sha256": "1f234bae8511b49bedfe4bd30176e7d1a4c31bedf2ea5cdf85f559f75559b942"}, + {"name": "0001_875_1875067_qa_3", "source": "0001/875/1875067.ipynb_qa_3", "notebook": "0001/875/1875067.ipynb", "difficulty": "hard", "instruction_sha256": "0a7f4c30e394b577f8b0222c643047494a68d82bd0b17d859f9924a4139539c5"}, + {"name": "0001_892_1892658_qa_3", "source": "0001/892/1892658.ipynb_qa_3", "notebook": "0001/892/1892658.ipynb", "difficulty": "hard", "instruction_sha256": "fc99f18b9f27e2c5aa143bed56acffe6e6897b68f5435c78659e456c24e0640e"}, + {"name": "0001_893_1893978_qa_5", "source": "0001/893/1893978.ipynb_qa_5", "notebook": "0001/893/1893978.ipynb", "difficulty": "hard", "instruction_sha256": "73b861c118c1bdf4a9bf5cac14f52422f69f4412de02e3c9338552542689e31a"}, + {"name": "0001_895_1895893_qa_2", "source": "0001/895/1895893.ipynb_qa_2", "notebook": "0001/895/1895893.ipynb", "difficulty": "hard", "instruction_sha256": "6132d4a5dcb443f53cb6d53348a1a76d56d95b9af841f88f42f9f847f934f92b"}, + {"name": "0001_903_1903634_qa_1", "source": "0001/903/1903634.ipynb_qa_1", "notebook": "0001/903/1903634.ipynb", "difficulty": "hard", "instruction_sha256": "9943c21454817f59a1b0a64a64802d7d7d3d876a63e3bb9a8ef7a17fdcedfc80"}, + {"name": "0001_903_1903634_qa_4", "source": "0001/903/1903634.ipynb_qa_4", "notebook": "0001/903/1903634.ipynb", "difficulty": "hard", "instruction_sha256": "bc69bbc99fe61207b27eda417f0304091e95c82f18bd8c3d69505ce0e6203456"}, + {"name": "0001_903_1903634_qa_5", "source": "0001/903/1903634.ipynb_qa_5", "notebook": "0001/903/1903634.ipynb", "difficulty": "hard", "instruction_sha256": "dfd05c52775022a3a62f8506c5124e61d2264900a31bc742d6969f5c8da7d1a7"}, + {"name": "0001_904_1904638_qa_1", "source": "0001/904/1904638.ipynb_qa_1", "notebook": "0001/904/1904638.ipynb", "difficulty": "hard", "instruction_sha256": "47a1a9cfe9326e2f6bfc7592637a473eb2c195fc31f0e722f3a176d60be32baf"}, + {"name": "0001_904_1904638_qa_2", "source": "0001/904/1904638.ipynb_qa_2", "notebook": "0001/904/1904638.ipynb", "difficulty": "hard", "instruction_sha256": "69c8eedddd4580a3cb1ac73d6b93911b7e3be20aea7d8d7079fac7dae148785d"}, + {"name": "0001_912_1912867_qa_2", "source": "0001/912/1912867.ipynb_qa_2", "notebook": "0001/912/1912867.ipynb", "difficulty": "hard", "instruction_sha256": "7d7d5fe2651b4ab7cb0f28d7f5b07fd249e4d64c136e68a4f91bd30923042925"}, + {"name": "0001_912_1912867_qa_5", "source": "0001/912/1912867.ipynb_qa_5", "notebook": "0001/912/1912867.ipynb", "difficulty": "hard", "instruction_sha256": "03e34fcaeab8293895ab6769b24729fceae98e25c019963214c9cd56b1514de8"}, + {"name": "0001_918_1918012_qa_1", "source": "0001/918/1918012.ipynb_qa_1", "notebook": "0001/918/1918012.ipynb", "difficulty": "hard", "instruction_sha256": "413ec54174c4fee088e32ea8fa2304eab6383ead00a0ad8b649d02c8ff19d060"}, + {"name": "0001_922_1922748_qa_4", "source": "0001/922/1922748.ipynb_qa_4", "notebook": "0001/922/1922748.ipynb", "difficulty": "hard", "instruction_sha256": "1d0234376e455d67f95f2e112ab1e9be75814ecf85182614d60292529cca4cec"}, + {"name": "0001_991_1991000_qa_3", "source": "0001/991/1991000.ipynb_qa_3", "notebook": "0001/991/1991000.ipynb", "difficulty": "hard", "instruction_sha256": "1cb0f2552b0eff9384273ded32b00f1a609f5840dd1ff516ee6b2820f434b55f"}, + {"name": "0001_991_1991376_qa_1", "source": "0001/991/1991376.ipynb_qa_1", "notebook": "0001/991/1991376.ipynb", "difficulty": "hard", "instruction_sha256": "3e7e7c3a681602573090f8446187bbe6615ee1c5a043733c59b7231b075468ad"}, + {"name": "0001_991_1991376_qa_5", "source": "0001/991/1991376.ipynb_qa_5", "notebook": "0001/991/1991376.ipynb", "difficulty": "hard", "instruction_sha256": "a0aba7aa773ec3c3352f677b70131c33083b135e3d0bd776d525a3f81ecfb435"}, + {"name": "0002_134_2134700_qa_5", "source": "0002/134/2134700.ipynb_qa_5", "notebook": "0002/134/2134700.ipynb", "difficulty": "hard", "instruction_sha256": "d2bd38963709deb5780643df0b05fd1bb8491da390a3c0b400ae35fc8d16db54"}, + {"name": "0002_169_2169008_qa_1", "source": "0002/169/2169008.ipynb_qa_1", "notebook": "0002/169/2169008.ipynb", "difficulty": "hard", "instruction_sha256": "1ec8adf67ef5d0bbf5bc364bbc7c51a06815f721736560d9a664997f756bf100"}, + {"name": "0002_187_2187459_qa_3", "source": "0002/187/2187459.ipynb_qa_3", "notebook": "0002/187/2187459.ipynb", "difficulty": "hard", "instruction_sha256": "70b17eebda3078b92ef0b110633ced726155d4722af943a520dcdf021b09e645"}, + {"name": "0002_189_2189266_qa_2", "source": "0002/189/2189266.ipynb_qa_2", "notebook": "0002/189/2189266.ipynb", "difficulty": "hard", "instruction_sha256": "57caf0bf7a5e06ccbf95d0749c7456e092cb472e32ea7d54e3ddc29795b22548"}, + {"name": "0014_516_14516985_qa_4", "source": "0014/516/14516985.ipynb_qa_4", "notebook": "0014/516/14516985.ipynb", "difficulty": "hard", "instruction_sha256": "6097e844a6f616ec28fc8c8ddcd5defd6d794cd01c729d66ada1c2aa4a30812e"}, + {"name": "0032_521_32521554_qa_3", "source": "0032/521/32521554.ipynb_qa_3", "notebook": "0032/521/32521554.ipynb", "difficulty": "hard", "instruction_sha256": "67a31e3f106b773fd6c1225f700f30be451668cac4a0aebebc57e2f261bb9571"}, + {"name": "0036_284_36284344_qa_5", "source": "0036/284/36284344.ipynb_qa_5", "notebook": "0036/284/36284344.ipynb", "difficulty": "hard", "instruction_sha256": "dc64eec5ba515a04b10f692ec9a102389309c2d2015995e5de59f61645fcb49b"}, + {"name": "0041_214_41214590_qa_4", "source": "0041/214/41214590.ipynb_qa_4", "notebook": "0041/214/41214590.ipynb", "difficulty": "hard", "instruction_sha256": "1b60982b46ee4257d89781b0d8411fd5e9de960223680028457bce8f002f2316"}, + {"name": "0048_996_48996091_qa_5", "source": "0048/996/48996091.ipynb_qa_5", "notebook": "0048/996/48996091.ipynb", "difficulty": "hard", "instruction_sha256": "adf2ebed947e03a15c108ed522b720b068371f65629c7c9f719e2a6c877ba745"}, + {"name": "0049_302_49302290_qa_3", "source": "0049/302/49302290.ipynb_qa_3", "notebook": "0049/302/49302290.ipynb", "difficulty": "hard", "instruction_sha256": "ba6cf04cb5182218a7bdffb662e3384bfc9cb9c38b367c37849a9eb2b42fa3b6"}, + {"name": "0051_018_51018932_qa_4", "source": "0051/018/51018932.ipynb_qa_4", "notebook": "0051/018/51018932.ipynb", "difficulty": "hard", "instruction_sha256": "299852a89a08120ce05b91762ce1e13467107d00c3eb23ffd76c24fc5a774612"}, + {"name": "0060_085_60085489_qa_4", "source": "0060/085/60085489.ipynb_qa_4", "notebook": "0060/085/60085489.ipynb", "difficulty": "hard", "instruction_sha256": "db2a6a0d4db73ddfce498cb4bae9542c73849ccb39fa6d43862bbee0457858c9"}, + {"name": "0064_089_64089732_qa_5", "source": "0064/089/64089732.ipynb_qa_5", "notebook": "0064/089/64089732.ipynb", "difficulty": "hard", "instruction_sha256": "01459031a14193b4fbe987f4e4c1466488f33b4283451476e04dda4d5906aab1"}, + {"name": "0065_262_65262139_qa_5", "source": "0065/262/65262139.ipynb_qa_5", "notebook": "0065/262/65262139.ipynb", "difficulty": "hard", "instruction_sha256": "0ab09f6bd0d928239f60bcde97931436721925aa77348c2d91ca342518cbb68e"}, + {"name": "0075_374_75374659_qa_2", "source": "0075/374/75374659.ipynb_qa_2", "notebook": "0075/374/75374659.ipynb", "difficulty": "hard", "instruction_sha256": "3323a5edef46c2ffe235d7a7a215cfecda1601808e48ae91eabecffa13f65ab0"}, + {"name": "0092_637_92637915_qa_2", "source": "0092/637/92637915.ipynb_qa_2", "notebook": "0092/637/92637915.ipynb", "difficulty": "hard", "instruction_sha256": "902721593434f0b3ed7435c7ac9c0c2d09799697e1aa03e4f33f3b7ab141ba53"}, + {"name": "0096_776_96776427_qa_5", "source": "0096/776/96776427.ipynb_qa_5", "notebook": "0096/776/96776427.ipynb", "difficulty": "hard", "instruction_sha256": "cb45bd02629e73b9fc7ea3629d1300a521ec74b9bcb89392ae59e8792cc1356d"}, + {"name": "0099_089_99089980_qa_5", "source": "0099/089/99089980.ipynb_qa_5", "notebook": "0099/089/99089980.ipynb", "difficulty": "hard", "instruction_sha256": "e934a92dd25e11ac99781af75ab5433c4c8804cfb8a4fd4ead47b9b24b954227"}, + {"name": "0104_079_104079493_qa_4", "source": "0104/079/104079493.ipynb_qa_4", "notebook": "0104/079/104079493.ipynb", "difficulty": "hard", "instruction_sha256": "906e1a55aec2a70fa6c5310a87f36d0d6121fbce5c3ac32a4641f1c765d0c06d"}, + {"name": "0115_735_115735943_qa_1", "source": "0115/735/115735943.ipynb_qa_1", "notebook": "0115/735/115735943.ipynb", "difficulty": "hard", "instruction_sha256": "dbfafc2ca0f02cbc4b1e07e68b62efde9450653099780475121f1304c465b269"}, + {"name": "0117_122_117122383_qa_3", "source": "0117/122/117122383.ipynb_qa_3", "notebook": "0117/122/117122383.ipynb", "difficulty": "hard", "instruction_sha256": "bd17eea23c6cb3ae28e35ede16102611d27634015e351807154ff37fe43cb4b1"}, + {"name": "0122_938_122938717_qa_4", "source": "0122/938/122938717.ipynb_qa_4", "notebook": "0122/938/122938717.ipynb", "difficulty": "hard", "instruction_sha256": "ba8b1221265a98d6ced0f68c49dd8fb8ea04e2afc03733b77475c3ee98ec353c"}, + {"name": "0126_223_126223416_qa_5", "source": "0126/223/126223416.ipynb_qa_5", "notebook": "0126/223/126223416.ipynb", "difficulty": "hard", "instruction_sha256": "bf9fdef8013dfd12241731291f01a41c77d85f4f0f423f4612147d71949a74f2"}, + {"name": "0138_500_138500445_qa_1", "source": "0138/500/138500445.ipynb_qa_1", "notebook": "0138/500/138500445.ipynb", "difficulty": "hard", "instruction_sha256": "f1ecbcaab9a48a1ab5481192443d0749968e40988ca0f7df06ffc13230ac5277"} +] diff --git a/05-multi-harness-rl/data/train.json b/05-multi-harness-rl/data/train.json new file mode 100644 index 00000000..6352014b --- /dev/null +++ b/05-multi-harness-rl/data/train.json @@ -0,0 +1,1002 @@ +[ + {"name": "0119_907_119907848_qa_1", "source": "0119/907/119907848.ipynb_qa_1", "notebook": "0119/907/119907848.ipynb", "difficulty": "hard", "instruction_sha256": "e7d59f1d15e5fc4ec7de2284b95cb51d41ee70cd4c4561d6f2aaaaea3dc98a40"}, + {"name": "0107_327_107327801_qa_2", "source": "0107/327/107327801.ipynb_qa_2", "notebook": "0107/327/107327801.ipynb", "difficulty": "medium", "instruction_sha256": "9a6ebd2c66c59bf55a52c198511ebc052c5e2be49c7030ef39d96918f833cd62"}, + {"name": "0087_372_87372002_qa_1", "source": "0087/372/87372002.ipynb_qa_1", "notebook": "0087/372/87372002.ipynb", "difficulty": "hard", "instruction_sha256": "d687b1a69cc28509d558114112a7f612f24c84494e4a55db98af738c5ceb1d33"}, + {"name": "0010_624_10624322_qa_4", "source": "0010/624/10624322.ipynb_qa_4", "notebook": "0010/624/10624322.ipynb", "difficulty": "medium", "instruction_sha256": "ef47b6f4382341777c279c25785990f10c89bb648d6849caebfc928fb0036b77"}, + {"name": "0062_391_62391096_qa_3", "source": "0062/391/62391096.ipynb_qa_3", "notebook": "0062/391/62391096.ipynb", "difficulty": "hard", "instruction_sha256": "ff3e791d65beca2aab26ef3e571dcba087db9201c1b1c12806a91da5eeca3fcd"}, + {"name": "0126_339_126339086_qa_5", "source": "0126/339/126339086.ipynb_qa_5", "notebook": "0126/339/126339086.ipynb", "difficulty": "hard", "instruction_sha256": "317f9a8301952a7d678c437e2e75a739d343c0eebdf20fc07f95a9b3e3005c30"}, + {"name": "0037_895_37895109_qa_2", "source": "0037/895/37895109.ipynb_qa_2", "notebook": "0037/895/37895109.ipynb", "difficulty": "hard", "instruction_sha256": "f0891895b4342de3be4ac490e90db39983b0958b72cf222ceb72ff695638028d"}, + {"name": "0019_978_19978878_qa_2", "source": "0019/978/19978878.ipynb_qa_2", "notebook": "0019/978/19978878.ipynb", "difficulty": "medium", "instruction_sha256": "33cd1ec7e01f4367cbc659f019c5a34e90238ece745895a3a6ef3f18ba257c67"}, + {"name": "0119_590_119590030_qa_2", "source": "0119/590/119590030.ipynb_qa_2", "notebook": "0119/590/119590030.ipynb", "difficulty": "hard", "instruction_sha256": "33cda10eca59f883f27ac0003d9c24740a14c43d509c00db2ea627a100d81240"}, + {"name": "0100_986_100986965_qa_1", "source": "0100/986/100986965.ipynb_qa_1", "notebook": "0100/986/100986965.ipynb", "difficulty": "medium", "instruction_sha256": "afa11b2120f8a5c1a7d8b3fadfcc3b23a7496b676414c5a0aa1e5f7c1e20bb46"}, + {"name": "0065_789_65789409_qa_2", "source": "0065/789/65789409.ipynb_qa_2", "notebook": "0065/789/65789409.ipynb", "difficulty": "hard", "instruction_sha256": "747a8a085267c7377d33ffad8d334cd6d24365d053e4f5eea64cd38e222f288b"}, + {"name": "0027_179_27179583_qa_2", "source": "0027/179/27179583.ipynb_qa_2", "notebook": "0027/179/27179583.ipynb", "difficulty": "medium", "instruction_sha256": "ce0905eb8fb4c9a38826938e130fba5d793e0a52382c166fd5b7f84803112880"}, + {"name": "0047_266_47266614_qa_3", "source": "0047/266/47266614.ipynb_qa_3", "notebook": "0047/266/47266614.ipynb", "difficulty": "hard", "instruction_sha256": "8756fec9edb4cc354a989ce201694483223b3aedc3bf1b020f8a1224b42fd6aa"}, + {"name": "0039_008_39008775_qa_5", "source": "0039/008/39008775.ipynb_qa_5", "notebook": "0039/008/39008775.ipynb", "difficulty": "hard", "instruction_sha256": "e672a103c229ce2fdeaf53e97e4a7b1eef22951f88665e997d30cf59c22388d5"}, + {"name": "0045_913_45913671_qa_3", "source": "0045/913/45913671.ipynb_qa_3", "notebook": "0045/913/45913671.ipynb", "difficulty": "medium", "instruction_sha256": "802aab3af1e7082a10b85ec877c017f80cafed9bbc0ac9d165189bc6f6effff5"}, + {"name": "0038_790_38790514_qa_3", "source": "0038/790/38790514.ipynb_qa_3", "notebook": "0038/790/38790514.ipynb", "difficulty": "hard", "instruction_sha256": "e5a087a1e4efcd094252a9afdcb08d3dc4a318a63a83ff5e59cfbfea962cbab9"}, + {"name": "0087_222_87222139_qa_2", "source": "0087/222/87222139.ipynb_qa_2", "notebook": "0087/222/87222139.ipynb", "difficulty": "hard", "instruction_sha256": "971c737213fe1f140d3012dc265fff2851a1228a9d4ab4b15db293a2d3f2d0fe"}, + {"name": "0043_338_43338909_qa_3", "source": "0043/338/43338909.ipynb_qa_3", "notebook": "0043/338/43338909.ipynb", "difficulty": "hard", "instruction_sha256": "4247c6cc10c35dddc2a93bb1a247b914b030fcb318ed11f68748aef1c94e5a3b"}, + {"name": "0087_720_87720557_qa_2", "source": "0087/720/87720557.ipynb_qa_2", "notebook": "0087/720/87720557.ipynb", "difficulty": "medium", "instruction_sha256": "56b518e0b18d665f27fa18a386cdca081e4700c2aef54b39216c54063b786121"}, + {"name": "0131_072_131072256_qa_3", "source": "0131/072/131072256.ipynb_qa_3", "notebook": "0131/072/131072256.ipynb", "difficulty": "medium", "instruction_sha256": "a765c3a814287e1455add8a538799d49eba9319e2423880c6e3a1eac59a84ed2"}, + {"name": "0042_738_42738840_qa_2", "source": "0042/738/42738840.ipynb_qa_2", "notebook": "0042/738/42738840.ipynb", "difficulty": "medium", "instruction_sha256": "b285fb0f0f6904dd9061d502ea49ab52d15c9abb8c0f0f7665c4c10f96469001"}, + {"name": "0120_979_120979302_qa_5", "source": "0120/979/120979302.ipynb_qa_5", "notebook": "0120/979/120979302.ipynb", "difficulty": "hard", "instruction_sha256": "99eb9be647b5c673e916ea891a7edc00e79d695ca44d87c99739e0ef4f8836e8"}, + {"name": "0064_330_64330026_qa_1", "source": "0064/330/64330026.ipynb_qa_1", "notebook": "0064/330/64330026.ipynb", "difficulty": "hard", "instruction_sha256": "907a1d62feb3e609f3a35082b1699e21164b0962396901df84792fff169fa018"}, + {"name": "0025_160_25160247_qa_2", "source": "0025/160/25160247.ipynb_qa_2", "notebook": "0025/160/25160247.ipynb", "difficulty": "medium", "instruction_sha256": "c7dea306695c187ad14bc1fa1890ed613037174066a4a3c9f53e68a9685c377f"}, + {"name": "0130_713_130713633_qa_5", "source": "0130/713/130713633.ipynb_qa_5", "notebook": "0130/713/130713633.ipynb", "difficulty": "hard", "instruction_sha256": "d6897f9826782f37642f32d8833696f3ec206cc74bb03587f1b120dcf225cfee"}, + {"name": "0001_379_1379568_qa_1", "source": "0001/379/1379568.ipynb_qa_1", "notebook": "0001/379/1379568.ipynb", "difficulty": "medium", "instruction_sha256": "e0ea9649d12a0cc76b1f2131f084b25d16a0d7b47179f54169525de311128f47"}, + {"name": "0020_397_20397857_qa_2", "source": "0020/397/20397857.ipynb_qa_2", "notebook": "0020/397/20397857.ipynb", "difficulty": "medium", "instruction_sha256": "632d3fc5a7a602573eda7bf447418e6dcde70409ec651ffaf5898fc45b162a42"}, + {"name": "0030_890_30890976_qa_5", "source": "0030/890/30890976.ipynb_qa_5", "notebook": "0030/890/30890976.ipynb", "difficulty": "hard", "instruction_sha256": "d68694d422c18ba6b35f9e896dcf10d72b0170966cbd23eeaf20ebabb3f17102"}, + {"name": "0072_216_72216676_qa_1", "source": "0072/216/72216676.ipynb_qa_1", "notebook": "0072/216/72216676.ipynb", "difficulty": "medium", "instruction_sha256": "af608195fe5d91802f489d2df6323bf28ebbaa2bf95d287f11cec39cc12f67da"}, + {"name": "0079_852_79852470_qa_3", "source": "0079/852/79852470.ipynb_qa_3", "notebook": "0079/852/79852470.ipynb", "difficulty": "medium", "instruction_sha256": "2d3431c6ea6f2e2f4a86a02e101ce42070632c0bf59076b987edde783cf64241"}, + {"name": "0001_261_1261978_qa_5", "source": "0001/261/1261978.ipynb_qa_5", "notebook": "0001/261/1261978.ipynb", "difficulty": "medium", "instruction_sha256": "c8b03bd151066ed4103bbc08f5983b6c6c7441e7eb62218887a46d3d29164ebd"}, + {"name": "0136_396_136396252_qa_2", "source": "0136/396/136396252.ipynb_qa_2", "notebook": "0136/396/136396252.ipynb", "difficulty": "medium", "instruction_sha256": "7fb038cef6f4d63109417bfb79fd471261f3ff791e10dbf6dd12ba538bda2bae"}, + {"name": "0045_452_45452619_qa_3", "source": "0045/452/45452619.ipynb_qa_3", "notebook": "0045/452/45452619.ipynb", "difficulty": "hard", "instruction_sha256": "cc35f9f2c726f91378079f6e482d87b0fd4e4cb2b09bd90748bcbe866143bf7f"}, + {"name": "0039_964_39964259_qa_2", "source": "0039/964/39964259.ipynb_qa_2", "notebook": "0039/964/39964259.ipynb", "difficulty": "hard", "instruction_sha256": "8be6384d917f9880ac9c6f665c84f6a66414649ee261cdb3f3bc13da061cf42b"}, + {"name": "0072_767_72767167_qa_1", "source": "0072/767/72767167.ipynb_qa_1", "notebook": "0072/767/72767167.ipynb", "difficulty": "medium", "instruction_sha256": "5175cab0d13c09856db170389ff685f3a263cf62419e0b828fab8a473142b151"}, + {"name": "0131_609_131609730_qa_1", "source": "0131/609/131609730.ipynb_qa_1", "notebook": "0131/609/131609730.ipynb", "difficulty": "medium", "instruction_sha256": "8fac43defd31a2df6067c0d33e899fb3aba30ba2984a71eec4d326ff0cc1490c"}, + {"name": "0070_945_70945386_qa_3", "source": "0070/945/70945386.ipynb_qa_3", "notebook": "0070/945/70945386.ipynb", "difficulty": "hard", "instruction_sha256": "318195db0ef587c84c33156177317fd5ade938e1e80c1a7e00626c9fd14bdd3f"}, + {"name": "0104_194_104194599_qa_4", "source": "0104/194/104194599.ipynb_qa_4", "notebook": "0104/194/104194599.ipynb", "difficulty": "medium", "instruction_sha256": "474735f6477943566a092e5ba787479074035a8a5383839f3f6039cfba2f8aee"}, + {"name": "0050_257_50257627_qa_3", "source": "0050/257/50257627.ipynb_qa_3", "notebook": "0050/257/50257627.ipynb", "difficulty": "hard", "instruction_sha256": "af42bbc55613a2ec7566c4a1b955437489a0d666bf65e7ce94f2e8224175904f"}, + {"name": "0101_507_101507712_qa_1", "source": "0101/507/101507712.ipynb_qa_1", "notebook": "0101/507/101507712.ipynb", "difficulty": "medium", "instruction_sha256": "5a57e1c96dad24baa0b7f781f4eab1dd919a830dd860effa38fce3821f6c6226"}, + {"name": "0101_507_101507409_qa_3", "source": "0101/507/101507409.ipynb_qa_3", "notebook": "0101/507/101507409.ipynb", "difficulty": "medium", "instruction_sha256": "9aa74e72ef4665d6cfaaf1c7d7a8b30ccf37c995e639f4a40b1b7716b9a7e09f"}, + {"name": "0113_929_113929776_qa_3", "source": "0113/929/113929776.ipynb_qa_3", "notebook": "0113/929/113929776.ipynb", "difficulty": "hard", "instruction_sha256": "150569f3fa0bccf29a4cb608f053fe4d2d582a3c342da042ee22eda5dcb87eed"}, + {"name": "0037_513_37513711_qa_2", "source": "0037/513/37513711.ipynb_qa_2", "notebook": "0037/513/37513711.ipynb", "difficulty": "hard", "instruction_sha256": "0f936841e1c5a249e7c9f5e7636363d8938446df13c45b660201dde52d227b94"}, + {"name": "0019_469_19469309_qa_5", "source": "0019/469/19469309.ipynb_qa_5", "notebook": "0019/469/19469309.ipynb", "difficulty": "hard", "instruction_sha256": "c8ba24107e8f6968ad0aa9dc97e88bbd8e6b105a53803fdb2d7467f1aa31f306"}, + {"name": "0102_190_102190555_qa_2", "source": "0102/190/102190555.ipynb_qa_2", "notebook": "0102/190/102190555.ipynb", "difficulty": "hard", "instruction_sha256": "c33181ae95222b839c79c93affb4a7cddeb38a2fb0f6da25f891a19a75f99379"}, + {"name": "0019_416_19416448_qa_4", "source": "0019/416/19416448.ipynb_qa_4", "notebook": "0019/416/19416448.ipynb", "difficulty": "hard", "instruction_sha256": "7b2a3098520263df167dab7a8cf4c0bd5a309231e07467093807c43de8894780"}, + {"name": "0018_278_18278432_qa_1", "source": "0018/278/18278432.ipynb_qa_1", "notebook": "0018/278/18278432.ipynb", "difficulty": "medium", "instruction_sha256": "23a361f7d9c7383401297bfcd447bf9930e31a4fcbef48b9e22682eeaf6ce942"}, + {"name": "0058_529_58529660_qa_3", "source": "0058/529/58529660.ipynb_qa_3", "notebook": "0058/529/58529660.ipynb", "difficulty": "hard", "instruction_sha256": "b66b4ef27f26f155e4e91f7e79bcc63b558e2886dd94ceaef3245523d78d6638"}, + {"name": "0113_907_113907466_qa_1", "source": "0113/907/113907466.ipynb_qa_1", "notebook": "0113/907/113907466.ipynb", "difficulty": "hard", "instruction_sha256": "17be490a1095f5e40cd293c8180e4c4cfe2379fc7a88300a7dc0d2c9d652cd48"}, + {"name": "0105_060_105060710_qa_5", "source": "0105/060/105060710.ipynb_qa_5", "notebook": "0105/060/105060710.ipynb", "difficulty": "medium", "instruction_sha256": "2a5e848c63264921e1e05cecff9895101acc9988d27c0585f7ad1ff4f5d8f9e3"}, + {"name": "0021_643_21643452_qa_5", "source": "0021/643/21643452.ipynb_qa_5", "notebook": "0021/643/21643452.ipynb", "difficulty": "medium", "instruction_sha256": "9bc59c7a28e0db5177f7d05ce52229d39ec046a4e2674de8cc7f6ed80265489e"}, + {"name": "0043_313_43313326_qa_5", "source": "0043/313/43313326.ipynb_qa_5", "notebook": "0043/313/43313326.ipynb", "difficulty": "hard", "instruction_sha256": "44fc1c8d2d1928aa507b03ca0051b72479d1ca966e47ee1ccc891b8f3dc059d0"}, + {"name": "0040_808_40808132_qa_5", "source": "0040/808/40808132.ipynb_qa_5", "notebook": "0040/808/40808132.ipynb", "difficulty": "hard", "instruction_sha256": "353ceee79ef3f0fccf3a6898f373261134308aba82c9af1ac250ce1d9a4dc2bd"}, + {"name": "0101_507_101507409_qa_4", "source": "0101/507/101507409.ipynb_qa_4", "notebook": "0101/507/101507409.ipynb", "difficulty": "hard", "instruction_sha256": "b21a2d3a81170eb62330a738312e9b65270eec3004da5d45ac8a3f3475ce4dc0"}, + {"name": "0067_290_67290658_qa_1", "source": "0067/290/67290658.ipynb_qa_1", "notebook": "0067/290/67290658.ipynb", "difficulty": "hard", "instruction_sha256": "04a99f1d7cab9816988b860426dfdfd53cdc9613c4d5a689ac50c768330623b4"}, + {"name": "0065_074_65074869_qa_2", "source": "0065/074/65074869.ipynb_qa_2", "notebook": "0065/074/65074869.ipynb", "difficulty": "hard", "instruction_sha256": "d97ffaadaa8b563b3da4a300fb53dbc1b7ab8ae4ff8d40c12331a749556235df"}, + {"name": "0094_883_94883618_qa_1", "source": "0094/883/94883618.ipynb_qa_1", "notebook": "0094/883/94883618.ipynb", "difficulty": "hard", "instruction_sha256": "71e0db78e7ac243f9bf4258057dce3dab7c8f230572250c794b07a69fccc15a2"}, + {"name": "0044_954_44954372_qa_2", "source": "0044/954/44954372.ipynb_qa_2", "notebook": "0044/954/44954372.ipynb", "difficulty": "hard", "instruction_sha256": "01e69f497be869f87f9c2efc6329f230cf7df6fc16d7f6eb742f8d1ad79e2582"}, + {"name": "0043_124_43124508_qa_2", "source": "0043/124/43124508.ipynb_qa_2", "notebook": "0043/124/43124508.ipynb", "difficulty": "hard", "instruction_sha256": "46446fa4a0360093174dd2037e0d3231c8c0f538c823573333e709070c037ceb"}, + {"name": "0012_748_12748314_qa_4", "source": "0012/748/12748314.ipynb_qa_4", "notebook": "0012/748/12748314.ipynb", "difficulty": "medium", "instruction_sha256": "1d75dd3fd05f9a1023f6f93ee2a54e544d752233aabc56fd2a16b8130c534562"}, + {"name": "0039_850_39850014_qa_3", "source": "0039/850/39850014.ipynb_qa_3", "notebook": "0039/850/39850014.ipynb", "difficulty": "medium", "instruction_sha256": "878f30b7ec81b271a82221584c4b27fb508aa85ab6b9c57484017e3619b88d74"}, + {"name": "0055_050_55050231_qa_4", "source": "0055/050/55050231.ipynb_qa_4", "notebook": "0055/050/55050231.ipynb", "difficulty": "hard", "instruction_sha256": "7b96f2d6da82d7514169a4954b285f04257eff20e4e24a6a36b40ee125edefcd"}, + {"name": "0061_879_61879031_qa_3", "source": "0061/879/61879031.ipynb_qa_3", "notebook": "0061/879/61879031.ipynb", "difficulty": "hard", "instruction_sha256": "7d2a04f670987b5cef0a8af8080cc4af211e905e2ca48a244506feafd966b285"}, + {"name": "0112_804_112804674_qa_2", "source": "0112/804/112804674.ipynb_qa_2", "notebook": "0112/804/112804674.ipynb", "difficulty": "medium", "instruction_sha256": "d76072bf5e07c087b856ac945239ca93cd5b92b6cc69561ca84a4709a93aeb4a"}, + {"name": "0025_024_25024460_qa_1", "source": "0025/024/25024460.ipynb_qa_1", "notebook": "0025/024/25024460.ipynb", "difficulty": "hard", "instruction_sha256": "ade914b962e0bbf5effa5980a1c08331101fdca9a7d3afe622ead966a08d154e"}, + {"name": "0131_609_131609730_qa_5", "source": "0131/609/131609730.ipynb_qa_5", "notebook": "0131/609/131609730.ipynb", "difficulty": "hard", "instruction_sha256": "ffd4985690cbc68aa32e5bd529c1178ac673d24125e3f4dbf8f2002a095d53d3"}, + {"name": "0125_976_125976958_qa_4", "source": "0125/976/125976958.ipynb_qa_4", "notebook": "0125/976/125976958.ipynb", "difficulty": "medium", "instruction_sha256": "189398f6f5744d7de4de58142c0f4295c96923e21de871b50606f3f629f1ca0a"}, + {"name": "0039_154_39154395_qa_2", "source": "0039/154/39154395.ipynb_qa_2", "notebook": "0039/154/39154395.ipynb", "difficulty": "medium", "instruction_sha256": "1f3cfc597dc75180f8479e8690ab165f28834501dfe60d1bc346c5d94bc9ea3a"}, + {"name": "0078_661_78661279_qa_5", "source": "0078/661/78661279.ipynb_qa_5", "notebook": "0078/661/78661279.ipynb", "difficulty": "medium", "instruction_sha256": "9d083b9b5916ddfff8d1575b5ab3a5033f20f3ea2c848fbc0b22c81e0de909ee"}, + {"name": "0046_487_46487066_qa_4", "source": "0046/487/46487066.ipynb_qa_4", "notebook": "0046/487/46487066.ipynb", "difficulty": "hard", "instruction_sha256": "a81b44b810a62ceed9d0129c6178269ab7a0b5f2234451744bc758a9a4b9a7ba"}, + {"name": "0001_452_1452536_qa_1", "source": "0001/452/1452536.ipynb_qa_1", "notebook": "0001/452/1452536.ipynb", "difficulty": "hard", "instruction_sha256": "81b6b416725ca1bb71121d1f13c4631b96082b4f03e5845e7151204c993737c8"}, + {"name": "0057_443_57443371_qa_1", "source": "0057/443/57443371.ipynb_qa_1", "notebook": "0057/443/57443371.ipynb", "difficulty": "hard", "instruction_sha256": "df60a08c0300eb0de61816f75e4ac12eb713ecb4c991b1f3c1569972e980dcd7"}, + {"name": "0072_990_72990356_qa_1", "source": "0072/990/72990356.ipynb_qa_1", "notebook": "0072/990/72990356.ipynb", "difficulty": "hard", "instruction_sha256": "b70d74147061e1a66ca93ac529b5cd6a96bfe2d461732d71490a1f5f0422f2ba"}, + {"name": "0040_288_40288007_qa_3", "source": "0040/288/40288007.ipynb_qa_3", "notebook": "0040/288/40288007.ipynb", "difficulty": "hard", "instruction_sha256": "8c4e0e033e3063e6287dd3fa90aa9ab2d2cc04a67d2fd757129d9c24fa1370ae"}, + {"name": "0030_588_30588724_qa_4", "source": "0030/588/30588724.ipynb_qa_4", "notebook": "0030/588/30588724.ipynb", "difficulty": "hard", "instruction_sha256": "a77d75d2088b79882f5b1cabc7eb220a50f23d9a62918f0faccb495dacbdb1c9"}, + {"name": "0049_585_49585863_qa_3", "source": "0049/585/49585863.ipynb_qa_3", "notebook": "0049/585/49585863.ipynb", "difficulty": "medium", "instruction_sha256": "18a35d3bb47d1a7c60e9dda30b8af2f136236770fc80e8e4d77ca4296ff8e4e5"}, + {"name": "0034_588_34588161_qa_5", "source": "0034/588/34588161.ipynb_qa_5", "notebook": "0034/588/34588161.ipynb", "difficulty": "hard", "instruction_sha256": "c37f4fb16214ea3f1a3b11e87b06ccf8da5fff06a77e706e536fb952e7180fe2"}, + {"name": "0040_288_40288007_qa_1", "source": "0040/288/40288007.ipynb_qa_1", "notebook": "0040/288/40288007.ipynb", "difficulty": "hard", "instruction_sha256": "fa740f01b8a80da80a24fbd60845d8cf11f1890cbe698b45e8f55eb83aa9d8fc"}, + {"name": "0126_954_126954425_qa_3", "source": "0126/954/126954425.ipynb_qa_3", "notebook": "0126/954/126954425.ipynb", "difficulty": "medium", "instruction_sha256": "62d65ceefd4b81ba37c4918e24424801e6e5df21b90832d3307abd48c0075592"}, + {"name": "0034_886_34886359_qa_5", "source": "0034/886/34886359.ipynb_qa_5", "notebook": "0034/886/34886359.ipynb", "difficulty": "hard", "instruction_sha256": "1ecbeeab961db4a242aec45be1a66453d93ee0f3c5f36397ce5fb6abfd36f0df"}, + {"name": "0107_521_107521812_qa_2", "source": "0107/521/107521812.ipynb_qa_2", "notebook": "0107/521/107521812.ipynb", "difficulty": "hard", "instruction_sha256": "ecfa131bf90cf4db054d763c3a83b1d9ba866628328f22c31959e1b6abaa6cba"}, + {"name": "0021_656_21656365_qa_5", "source": "0021/656/21656365.ipynb_qa_5", "notebook": "0021/656/21656365.ipynb", "difficulty": "hard", "instruction_sha256": "ddf779c156debeb6062f2a7eef1ca5e7c2cf6f136237f08cb252a870f5b7dad0"}, + {"name": "0121_209_121209389_qa_1", "source": "0121/209/121209389.ipynb_qa_1", "notebook": "0121/209/121209389.ipynb", "difficulty": "hard", "instruction_sha256": "410563a2411410d7265739393bf2e25c91b341af8e813d3c4ae72264df764bc4"}, + {"name": "0134_941_134941943_qa_1", "source": "0134/941/134941943.ipynb_qa_1", "notebook": "0134/941/134941943.ipynb", "difficulty": "hard", "instruction_sha256": "f826e2825e7f5c72bd91443f800346d8f847993ec832ff3c184ebde0c8c0e764"}, + {"name": "0026_895_26895005_qa_1", "source": "0026/895/26895005.ipynb_qa_1", "notebook": "0026/895/26895005.ipynb", "difficulty": "medium", "instruction_sha256": "ba7563fc8a8baae554b812e8d650d3ae2ca4a7ad9a08b7050f5225fcff8f2514"}, + {"name": "0001_483_1483764_qa_5", "source": "0001/483/1483764.ipynb_qa_5", "notebook": "0001/483/1483764.ipynb", "difficulty": "medium", "instruction_sha256": "a06dea3397cd2b6b9f53b1e070802d7b0d5b8c43969c80a285e30fb97a5b3ed5"}, + {"name": "0000_804_804467_qa_1", "source": "0000/804/804467.ipynb_qa_1", "notebook": "0000/804/804467.ipynb", "difficulty": "hard", "instruction_sha256": "df61bcfd3ec120b7bc876d6c0b2211fc4b88768189597122a3bd30feaaccf970"}, + {"name": "0098_812_98812520_qa_1", "source": "0098/812/98812520.ipynb_qa_1", "notebook": "0098/812/98812520.ipynb", "difficulty": "medium", "instruction_sha256": "a05a08c52e59814c13a1ea9e90db429c426386108a5495bbc54df98c37b3d8e3"}, + {"name": "0108_199_108199780_qa_2", "source": "0108/199/108199780.ipynb_qa_2", "notebook": "0108/199/108199780.ipynb", "difficulty": "hard", "instruction_sha256": "0057c018d6b7ff2a749fcb1b5ba59919674f6e776e900ecac42c43a4604403c9"}, + {"name": "0067_729_67729269_qa_4", "source": "0067/729/67729269.ipynb_qa_4", "notebook": "0067/729/67729269.ipynb", "difficulty": "medium", "instruction_sha256": "2355c93e5b05bf9f53ca99f01f30e82a591d28fc87ecbb3713379c3b27659ace"}, + {"name": "0064_908_64908177_qa_2", "source": "0064/908/64908177.ipynb_qa_2", "notebook": "0064/908/64908177.ipynb", "difficulty": "hard", "instruction_sha256": "fa46d30922591120c328dec77c2fe8bbbcb8d9977e1235e5510efae07e514b2c"}, + {"name": "0073_798_73798390_qa_4", "source": "0073/798/73798390.ipynb_qa_4", "notebook": "0073/798/73798390.ipynb", "difficulty": "hard", "instruction_sha256": "58756c1cc2bc611d570f49980f779f755964ab10054afc553f223e2a68f7efd5"}, + {"name": "0082_498_82498397_qa_3", "source": "0082/498/82498397.ipynb_qa_3", "notebook": "0082/498/82498397.ipynb", "difficulty": "medium", "instruction_sha256": "df256f443450fa7ed25e25e59abdde77b7ed49679cd71dac2b356e0032d3f361"}, + {"name": "0137_970_137970410_qa_4", "source": "0137/970/137970410.ipynb_qa_4", "notebook": "0137/970/137970410.ipynb", "difficulty": "hard", "instruction_sha256": "b2173297bfe3424e8470b78624d32cec9ea47c83128afe0a3f85e7c7def65b1d"}, + {"name": "0018_456_18456045_qa_5", "source": "0018/456/18456045.ipynb_qa_5", "notebook": "0018/456/18456045.ipynb", "difficulty": "hard", "instruction_sha256": "b7383a67bed445a6b6e69a670e68646e6cdc4bce37ae89dcd3523e8ab9ef8669"}, + {"name": "0036_579_36579520_qa_4", "source": "0036/579/36579520.ipynb_qa_4", "notebook": "0036/579/36579520.ipynb", "difficulty": "hard", "instruction_sha256": "e210421800ccce3f096590159c6b696dcab0f2ad7e6e290ea148555bb1d01637"}, + {"name": "0127_374_127374855_qa_4", "source": "0127/374/127374855.ipynb_qa_4", "notebook": "0127/374/127374855.ipynb", "difficulty": "medium", "instruction_sha256": "350f98d0ec919322514af87a907db3d13a3d29b254b6e5bc9998b8475aa22031"}, + {"name": "0048_917_48917297_qa_4", "source": "0048/917/48917297.ipynb_qa_4", "notebook": "0048/917/48917297.ipynb", "difficulty": "hard", "instruction_sha256": "e6d2904900a73e30e6f44cc28e929a064afc2f6baa5334617d7b26f148e4464a"}, + {"name": "0066_447_66447966_qa_4", "source": "0066/447/66447966.ipynb_qa_4", "notebook": "0066/447/66447966.ipynb", "difficulty": "medium", "instruction_sha256": "c41d409e90a85fffaf9e978d8b469343b6aba037dab4144eb19c32195808c4f9"}, + {"name": "0024_602_24602868_qa_2", "source": "0024/602/24602868.ipynb_qa_2", "notebook": "0024/602/24602868.ipynb", "difficulty": "medium", "instruction_sha256": "aa6cddad802f7b53bb3849faf84784025e778b8aecb37df6a6dedd020a1dc3bf"}, + {"name": "0001_866_1866353_qa_1", "source": "0001/866/1866353.ipynb_qa_1", "notebook": "0001/866/1866353.ipynb", "difficulty": "medium", "instruction_sha256": "8e83ccfbdd19ea3e9a8d9aba28413b6a1f034ac65e9a2e9f14475be0e7325740"}, + {"name": "0117_594_117594280_qa_2", "source": "0117/594/117594280.ipynb_qa_2", "notebook": "0117/594/117594280.ipynb", "difficulty": "hard", "instruction_sha256": "d1e2c767b83dacc9180545bd4301a19867009971d9f91611594d5d5fae55c246"}, + {"name": "0002_094_2094844_qa_1", "source": "0002/094/2094844.ipynb_qa_1", "notebook": "0002/094/2094844.ipynb", "difficulty": "medium", "instruction_sha256": "8a167134732d638368bd8f4efa12eec2b0c6d81e10d82749a011f9cb4c5ba612"}, + {"name": "0094_620_94620248_qa_2", "source": "0094/620/94620248.ipynb_qa_2", "notebook": "0094/620/94620248.ipynb", "difficulty": "medium", "instruction_sha256": "94b8a892b459f89e7dba4222b7bcc037cd7c1c0f935da9283f638255e4914919"}, + {"name": "0034_890_34890721_qa_3", "source": "0034/890/34890721.ipynb_qa_3", "notebook": "0034/890/34890721.ipynb", "difficulty": "hard", "instruction_sha256": "3779789f2ffacff223691d5386f54e5234045a2668469fab95542329821f2ed1"}, + {"name": "0112_574_112574806_qa_5", "source": "0112/574/112574806.ipynb_qa_5", "notebook": "0112/574/112574806.ipynb", "difficulty": "medium", "instruction_sha256": "18864213a0d414db02d6b8b8db1de1ae8de68baed88408bed24c5e97fb0907a5"}, + {"name": "0074_813_74813463_qa_5", "source": "0074/813/74813463.ipynb_qa_5", "notebook": "0074/813/74813463.ipynb", "difficulty": "hard", "instruction_sha256": "86b625077d4355a3471fcf1af4541e7ff58a0e8a7631a2e0b8f58be6183da355"}, + {"name": "0041_035_41035825_qa_1", "source": "0041/035/41035825.ipynb_qa_1", "notebook": "0041/035/41035825.ipynb", "difficulty": "medium", "instruction_sha256": "dddc58fec071fd8aa8953df6555b95fb53a76b1cba28882e9db89ce65d916a0b"}, + {"name": "0095_395_95395894_qa_2", "source": "0095/395/95395894.ipynb_qa_2", "notebook": "0095/395/95395894.ipynb", "difficulty": "medium", "instruction_sha256": "d46129d1a92e9673c20f26eeaf3b98ca530bcd4f9c7878439becbe48a93c0eb5"}, + {"name": "0099_829_99829565_qa_3", "source": "0099/829/99829565.ipynb_qa_3", "notebook": "0099/829/99829565.ipynb", "difficulty": "medium", "instruction_sha256": "7129a18b4d8e6d46fc3f6f7c73fc5ecb41f1a523af32876b19832b561fb5e99c"}, + {"name": "0021_960_21960570_qa_4", "source": "0021/960/21960570.ipynb_qa_4", "notebook": "0021/960/21960570.ipynb", "difficulty": "medium", "instruction_sha256": "6c43f1570d416fc5b21df30d85c97e04324b8d97bab63425298df2c6cda778d1"}, + {"name": "0042_372_42372757_qa_5", "source": "0042/372/42372757.ipynb_qa_5", "notebook": "0042/372/42372757.ipynb", "difficulty": "medium", "instruction_sha256": "e02364d423b10015226fead3507d2ca91623dff303d170a77a747742e325a2b3"}, + {"name": "0056_266_56266405_qa_3", "source": "0056/266/56266405.ipynb_qa_3", "notebook": "0056/266/56266405.ipynb", "difficulty": "medium", "instruction_sha256": "50b09aaa0504afc655b382cab7850af26a9b48da26ebcea55c4570a73a841a37"}, + {"name": "0035_921_35921172_qa_1", "source": "0035/921/35921172.ipynb_qa_1", "notebook": "0035/921/35921172.ipynb", "difficulty": "hard", "instruction_sha256": "59f8f184dde34cabc9fbb0c12099adf8f059728c66932bae3cf1e6cc6425aee9"}, + {"name": "0075_391_75391350_qa_3", "source": "0075/391/75391350.ipynb_qa_3", "notebook": "0075/391/75391350.ipynb", "difficulty": "medium", "instruction_sha256": "b92fceaa0ad37b5aad84aada4d9472d7b744de5136ffb960f28e0506add2fc10"}, + {"name": "0122_097_122097207_qa_3", "source": "0122/097/122097207.ipynb_qa_3", "notebook": "0122/097/122097207.ipynb", "difficulty": "medium", "instruction_sha256": "11edfa2a8994c96702b42836f7813affb47e0b214e9c8f24e7843108f1566543"}, + {"name": "0114_582_114582781_qa_5", "source": "0114/582/114582781.ipynb_qa_5", "notebook": "0114/582/114582781.ipynb", "difficulty": "hard", "instruction_sha256": "2b96f512706a0c2d3b4ee8479b560cf56b8a2360e86e36618cde63bd7eaf51e8"}, + {"name": "0022_668_22668393_qa_3", "source": "0022/668/22668393.ipynb_qa_3", "notebook": "0022/668/22668393.ipynb", "difficulty": "hard", "instruction_sha256": "e6d170a1754a8c30334e4100da3bf83c8f1f48d99b528ed4796ebe127dfcac0e"}, + {"name": "0096_765_96765273_qa_4", "source": "0096/765/96765273.ipynb_qa_4", "notebook": "0096/765/96765273.ipynb", "difficulty": "medium", "instruction_sha256": "67ac21d464e4a98782d544cdc7792991c5215a63fc1fde4ce17bdc4b5618b06f"}, + {"name": "0051_768_51768802_qa_2", "source": "0051/768/51768802.ipynb_qa_2", "notebook": "0051/768/51768802.ipynb", "difficulty": "medium", "instruction_sha256": "e37f311f5e8b4f3727b1a1da331c7b0412634997ae57ac86614fd2066f9b7d96"}, + {"name": "0060_573_60573328_qa_3", "source": "0060/573/60573328.ipynb_qa_3", "notebook": "0060/573/60573328.ipynb", "difficulty": "medium", "instruction_sha256": "4f3dd8ffbd80a8bb13cd87f59537885de1f220054b4852f0cafdc2c3e85041d1"}, + {"name": "0001_234_1234901_qa_3", "source": "0001/234/1234901.ipynb_qa_3", "notebook": "0001/234/1234901.ipynb", "difficulty": "medium", "instruction_sha256": "f5bd6aa688047809358eb17fc637223175b4f0376432528c86326cbcad7f47b7"}, + {"name": "0033_326_33326125_qa_3", "source": "0033/326/33326125.ipynb_qa_3", "notebook": "0033/326/33326125.ipynb", "difficulty": "hard", "instruction_sha256": "8d1df6387d3b1b19414616a9a8c1f9bf2e1c6ac7fc7249dbd73fb18c688883d5"}, + {"name": "0027_992_27992214_qa_3", "source": "0027/992/27992214.ipynb_qa_3", "notebook": "0027/992/27992214.ipynb", "difficulty": "hard", "instruction_sha256": "6366fef4ea8cb3b9d9c24edf50e077b6083795f6a2bfd0791baec69e7bb92c8e"}, + {"name": "0045_454_45454053_qa_3", "source": "0045/454/45454053.ipynb_qa_3", "notebook": "0045/454/45454053.ipynb", "difficulty": "hard", "instruction_sha256": "dfee2c2d580d22662a5746bcff49912055435a9a2a1d0cef4546e554526e2cb7"}, + {"name": "0031_128_31128554_qa_3", "source": "0031/128/31128554.ipynb_qa_3", "notebook": "0031/128/31128554.ipynb", "difficulty": "medium", "instruction_sha256": "7ab51677f2bcb3d8abb4422eec7d9a3877859d094e9e31b69d91958852406be0"}, + {"name": "0131_667_131667789_qa_4", "source": "0131/667/131667789.ipynb_qa_4", "notebook": "0131/667/131667789.ipynb", "difficulty": "hard", "instruction_sha256": "1437114b67d4477f3afccaea8226579a26677764ee206ccfcdce3653aa9cdc09"}, + {"name": "0027_004_27004450_qa_2", "source": "0027/004/27004450.ipynb_qa_2", "notebook": "0027/004/27004450.ipynb", "difficulty": "medium", "instruction_sha256": "097583b3106ed9b854267b0d029e2d3bf445b5d4334aa8f7310b124e005c2843"}, + {"name": "0105_054_105054048_qa_4", "source": "0105/054/105054048.ipynb_qa_4", "notebook": "0105/054/105054048.ipynb", "difficulty": "hard", "instruction_sha256": "f28cd6c4f1dc313e092a1297be1e53431f59b9882404f106465704f4dd2cdacf"}, + {"name": "0090_257_90257990_qa_1", "source": "0090/257/90257990.ipynb_qa_1", "notebook": "0090/257/90257990.ipynb", "difficulty": "medium", "instruction_sha256": "d3bc3913126cb35138c75e6c4b880c7518f41023629a9bb0e3b02c9fdc66108b"}, + {"name": "0039_785_39785108_qa_1", "source": "0039/785/39785108.ipynb_qa_1", "notebook": "0039/785/39785108.ipynb", "difficulty": "hard", "instruction_sha256": "4dbd50281474449f2d4867a2a9eb39fd4cde0f4b1b2325ca5c9889bcd0080455"}, + {"name": "0040_843_40843458_qa_4", "source": "0040/843/40843458.ipynb_qa_4", "notebook": "0040/843/40843458.ipynb", "difficulty": "medium", "instruction_sha256": "502bf0572b3a75b514e7d74f51ab18965527a84c0b4ecd5156078a5ce4c2a7a7"}, + {"name": "0047_564_47564024_qa_3", "source": "0047/564/47564024.ipynb_qa_3", "notebook": "0047/564/47564024.ipynb", "difficulty": "hard", "instruction_sha256": "a085c253dc9f982a09f4652e26be4ea66b970cd951f07673e4002e45d0b0e23d"}, + {"name": "0055_521_55521574_qa_5", "source": "0055/521/55521574.ipynb_qa_5", "notebook": "0055/521/55521574.ipynb", "difficulty": "hard", "instruction_sha256": "be0e2783fa6f3c206fc8ce51ffed4598050bcd171371a86e399dfc15e262cb49"}, + {"name": "0054_670_54670437_qa_5", "source": "0054/670/54670437.ipynb_qa_5", "notebook": "0054/670/54670437.ipynb", "difficulty": "medium", "instruction_sha256": "3152a1e2d1b99f797d63de62e2a68ed24cd47b4bf8dcb737141b32b9e213fc49"}, + {"name": "0084_747_84747621_qa_5", "source": "0084/747/84747621.ipynb_qa_5", "notebook": "0084/747/84747621.ipynb", "difficulty": "hard", "instruction_sha256": "00cdec951b383597c0653c2cec0467d4dd211cfc3abade86eb68435c21e8e36f"}, + {"name": "0083_238_83238182_qa_2", "source": "0083/238/83238182.ipynb_qa_2", "notebook": "0083/238/83238182.ipynb", "difficulty": "medium", "instruction_sha256": "1bce11a46232708444d5e1f3cf0d187c258306d95b15bd8c33ae09e19815d4f2"}, + {"name": "0083_620_83620010_qa_4", "source": "0083/620/83620010.ipynb_qa_4", "notebook": "0083/620/83620010.ipynb", "difficulty": "medium", "instruction_sha256": "34ed65be29726fb5036ef1cc7e3c23e0a42634d8225743af7d59108aba8bdf14"}, + {"name": "0096_057_96057481_qa_1", "source": "0096/057/96057481.ipynb_qa_1", "notebook": "0096/057/96057481.ipynb", "difficulty": "medium", "instruction_sha256": "325ec57bca26b2332d8bb4d7a2999e72c7975b43b730198fe1b240c50be76f8d"}, + {"name": "0063_931_63931182_qa_4", "source": "0063/931/63931182.ipynb_qa_4", "notebook": "0063/931/63931182.ipynb", "difficulty": "hard", "instruction_sha256": "ff0cc8673c8f3ab8d29f00533aa11817326597acdf324da926c47bd9b34852c4"}, + {"name": "0032_222_32222371_qa_1", "source": "0032/222/32222371.ipynb_qa_1", "notebook": "0032/222/32222371.ipynb", "difficulty": "medium", "instruction_sha256": "a2a866ced02e751b07ce4d79c6e1e574e41a4dce7e9f16eb9f1973e3f4fe7a45"}, + {"name": "0011_544_11544512_qa_4", "source": "0011/544/11544512.ipynb_qa_4", "notebook": "0011/544/11544512.ipynb", "difficulty": "medium", "instruction_sha256": "3eabe5d9cc5e36c09f0f8b02679882582bb85a1680a332adcf3e744b89226b66"}, + {"name": "0042_124_42124908_qa_3", "source": "0042/124/42124908.ipynb_qa_3", "notebook": "0042/124/42124908.ipynb", "difficulty": "hard", "instruction_sha256": "c16c93dbc78b674f85821bc677b28e3030b059b9711ecc319638f3dd82565e8b"}, + {"name": "0021_868_21868084_qa_1", "source": "0021/868/21868084.ipynb_qa_1", "notebook": "0021/868/21868084.ipynb", "difficulty": "medium", "instruction_sha256": "cb6bb0cad868dd2b387f19133cfa5ef5e2ca5a48e9dd4a6ba3a36ee8a696c9a8"}, + {"name": "0025_241_25241426_qa_2", "source": "0025/241/25241426.ipynb_qa_2", "notebook": "0025/241/25241426.ipynb", "difficulty": "hard", "instruction_sha256": "e60be3a26f6004c61c1a50ee78d57764420d3604b3b09ccf582617d4cc79fdd1"}, + {"name": "0092_008_92008043_qa_2", "source": "0092/008/92008043.ipynb_qa_2", "notebook": "0092/008/92008043.ipynb", "difficulty": "medium", "instruction_sha256": "25b1e6ff5b0851d9a6963c2fb03f853178b88eeb107d76345794cac78329c58c"}, + {"name": "0061_664_61664085_qa_2", "source": "0061/664/61664085.ipynb_qa_2", "notebook": "0061/664/61664085.ipynb", "difficulty": "hard", "instruction_sha256": "cc449af8013609dc6c9e0d723300e9b1b610a74175edd74ce64ae89dc94a10b0"}, + {"name": "0089_696_89696158_qa_1", "source": "0089/696/89696158.ipynb_qa_1", "notebook": "0089/696/89696158.ipynb", "difficulty": "hard", "instruction_sha256": "558cfa5812240704e021fcf30157530568d9cbc24d970cf92cb2d7a99479e731"}, + {"name": "0021_803_21803688_qa_1", "source": "0021/803/21803688.ipynb_qa_1", "notebook": "0021/803/21803688.ipynb", "difficulty": "hard", "instruction_sha256": "10a2461abcdfa3388b2c8be1e89ba5c9113d616bfaf1e12a17ef94d9f86451ed"}, + {"name": "0133_615_133615287_qa_4", "source": "0133/615/133615287.ipynb_qa_4", "notebook": "0133/615/133615287.ipynb", "difficulty": "hard", "instruction_sha256": "30f60e6d7e977c3c17c0517c05e7bbab525cc0f766b1775ce46a284a0e684ee8"}, + {"name": "0125_028_125028860_qa_5", "source": "0125/028/125028860.ipynb_qa_5", "notebook": "0125/028/125028860.ipynb", "difficulty": "medium", "instruction_sha256": "c6ba8317c523f9034bd7fecddc95a0a3e7e8f327f438e54488811bdd68e24723"}, + {"name": "0084_834_84834261_qa_4", "source": "0084/834/84834261.ipynb_qa_4", "notebook": "0084/834/84834261.ipynb", "difficulty": "medium", "instruction_sha256": "e5f2cc7e67be040df52840282b4e15a2d0d4d45cbcc47005f8cdffa5b5aa42ec"}, + {"name": "0063_942_63942638_qa_2", "source": "0063/942/63942638.ipynb_qa_2", "notebook": "0063/942/63942638.ipynb", "difficulty": "hard", "instruction_sha256": "7f97d8bf09791202102cdc461b5fb3e9f384a029d09216acc26c3c0bddb7c792"}, + {"name": "0084_135_84135613_qa_4", "source": "0084/135/84135613.ipynb_qa_4", "notebook": "0084/135/84135613.ipynb", "difficulty": "hard", "instruction_sha256": "f8c9a3f9fe3cd74032552afd63ecb79bdcce09c04b4d90161a266db5b42b3c98"}, + {"name": "0039_392_39392401_qa_3", "source": "0039/392/39392401.ipynb_qa_3", "notebook": "0039/392/39392401.ipynb", "difficulty": "hard", "instruction_sha256": "c5ebaff0a26c2253f59dfac8e1df666a268c80c66a5d18cedb0fab7d2e94af2b"}, + {"name": "0033_684_33684962_qa_5", "source": "0033/684/33684962.ipynb_qa_5", "notebook": "0033/684/33684962.ipynb", "difficulty": "hard", "instruction_sha256": "b33afc6ee5ea36ee1bb4836cf33191e946d97a0927a4b5f0d2a3bf06b522de32"}, + {"name": "0035_336_35336797_qa_5", "source": "0035/336/35336797.ipynb_qa_5", "notebook": "0035/336/35336797.ipynb", "difficulty": "hard", "instruction_sha256": "f6725a254b180350f2d21ff88eddf4362d687cf04ab11daafd1abf2099dbde87"}, + {"name": "0124_195_124195561_qa_1", "source": "0124/195/124195561.ipynb_qa_1", "notebook": "0124/195/124195561.ipynb", "difficulty": "hard", "instruction_sha256": "2fc959d0f490b8653abe56bbc89ef645244c412c2557b6f76ace8ee5473f5c82"}, + {"name": "0010_624_10624322_qa_3", "source": "0010/624/10624322.ipynb_qa_3", "notebook": "0010/624/10624322.ipynb", "difficulty": "medium", "instruction_sha256": "f88aed650292f79c6a7dabb30c9d55dac50211b30c2ffef72cf43b6133c89de0"}, + {"name": "0001_690_1690621_qa_2", "source": "0001/690/1690621.ipynb_qa_2", "notebook": "0001/690/1690621.ipynb", "difficulty": "medium", "instruction_sha256": "267b81b7b96fda04b08816c8e838ce6f28313d868d5abed4d863294c53b7cc3e"}, + {"name": "0053_977_53977236_qa_3", "source": "0053/977/53977236.ipynb_qa_3", "notebook": "0053/977/53977236.ipynb", "difficulty": "medium", "instruction_sha256": "ac5f351a6e53748fb9d801666a03dc139c868e4a56049cd6391674701f04fa0a"}, + {"name": "0054_296_54296946_qa_1", "source": "0054/296/54296946.ipynb_qa_1", "notebook": "0054/296/54296946.ipynb", "difficulty": "medium", "instruction_sha256": "2ea2629f6d7f8ab9a2fadb96638bf8c6c9078572eb4544b774d8bcc270d1fb73"}, + {"name": "0074_849_74849685_qa_5", "source": "0074/849/74849685.ipynb_qa_5", "notebook": "0074/849/74849685.ipynb", "difficulty": "hard", "instruction_sha256": "78ba883181653e249f90536ad98c75b7a7b5e6861c15671044531be1f49ea804"}, + {"name": "0049_820_49820417_qa_1", "source": "0049/820/49820417.ipynb_qa_1", "notebook": "0049/820/49820417.ipynb", "difficulty": "hard", "instruction_sha256": "c309c1ce9efb92249d09d8b002119accb4d52c76620004ab19dfacdc297c7fbc"}, + {"name": "0070_945_70945386_qa_4", "source": "0070/945/70945386.ipynb_qa_4", "notebook": "0070/945/70945386.ipynb", "difficulty": "hard", "instruction_sha256": "067f75a1619acf962e36b2f131ca277ca3939c66eeb231964b4ba4c502032bcd"}, + {"name": "0012_761_12761066_qa_2", "source": "0012/761/12761066.ipynb_qa_2", "notebook": "0012/761/12761066.ipynb", "difficulty": "medium", "instruction_sha256": "fd70b4e59f197fccad6113e8b3bef9b338a7f75d2eb60c84b0080e5b2667ca5b"}, + {"name": "0060_533_60533129_qa_5", "source": "0060/533/60533129.ipynb_qa_5", "notebook": "0060/533/60533129.ipynb", "difficulty": "hard", "instruction_sha256": "39945287f2518e1d4bc8aecc989d262b4e26cf2920296e59117f269a2c664308"}, + {"name": "0035_513_35513650_qa_5", "source": "0035/513/35513650.ipynb_qa_5", "notebook": "0035/513/35513650.ipynb", "difficulty": "medium", "instruction_sha256": "dbda6946172257b90cb7f3fa9a6e1c5ebf2a1ba00a57441550fa898369c9f84c"}, + {"name": "0074_813_74813463_qa_2", "source": "0074/813/74813463.ipynb_qa_2", "notebook": "0074/813/74813463.ipynb", "difficulty": "hard", "instruction_sha256": "d8b13832659f18ae0e9c2ce42237e818df0fdd90d165508dfd77fef72d147ee5"}, + {"name": "0042_684_42684548_qa_4", "source": "0042/684/42684548.ipynb_qa_4", "notebook": "0042/684/42684548.ipynb", "difficulty": "hard", "instruction_sha256": "f5cf4922850b55dbfda9a4e2dbd6bcc90d1f4de083ae80cf98033fc993f1f2a5"}, + {"name": "0001_955_1955907_qa_3", "source": "0001/955/1955907.ipynb_qa_3", "notebook": "0001/955/1955907.ipynb", "difficulty": "medium", "instruction_sha256": "616dccadc00d8db2d906dd74491e60ba27b6cf10a0031275986b732151c36b21"}, + {"name": "0087_136_87136363_qa_5", "source": "0087/136/87136363.ipynb_qa_5", "notebook": "0087/136/87136363.ipynb", "difficulty": "hard", "instruction_sha256": "f62b885df654ac73767b9afb5f52774a5ef6d2fbe7de316b68dc04f63af67ef1"}, + {"name": "0070_420_70420149_qa_1", "source": "0070/420/70420149.ipynb_qa_1", "notebook": "0070/420/70420149.ipynb", "difficulty": "medium", "instruction_sha256": "4b0d18e4745ee0c10a0976e586760f7210b346b27d8516fa5b96ecd655f632cb"}, + {"name": "0115_281_115281076_qa_2", "source": "0115/281/115281076.ipynb_qa_2", "notebook": "0115/281/115281076.ipynb", "difficulty": "medium", "instruction_sha256": "9c03be417149f5156caf55e9f86ef96f232f061ec2568fc9b47c9108df7e4cb8"}, + {"name": "0040_340_40340401_qa_3", "source": "0040/340/40340401.ipynb_qa_3", "notebook": "0040/340/40340401.ipynb", "difficulty": "hard", "instruction_sha256": "4a5f19322640d8d6ee25aa96a23a65ed6eb49cb095b8d4ec149ede7d0b7ecdd8"}, + {"name": "0064_173_64173346_qa_2", "source": "0064/173/64173346.ipynb_qa_2", "notebook": "0064/173/64173346.ipynb", "difficulty": "medium", "instruction_sha256": "9be0ca321876168ba4288ffd37cc1e9c2263e5da9c74b5ffb00635cd7a116e55"}, + {"name": "0041_243_41243008_qa_3", "source": "0041/243/41243008.ipynb_qa_3", "notebook": "0041/243/41243008.ipynb", "difficulty": "hard", "instruction_sha256": "8a7f06f1084d538b3aeea07087cde9396d0ba37e9e413a860a23f2c407fab8b3"}, + {"name": "0010_515_10515511_qa_1", "source": "0010/515/10515511.ipynb_qa_1", "notebook": "0010/515/10515511.ipynb", "difficulty": "medium", "instruction_sha256": "2a86307cd3cb4c4dfab855c1056f9e8c2d11f3d9434548f3d771ff1362f18b90"}, + {"name": "0062_048_62048358_qa_3", "source": "0062/048/62048358.ipynb_qa_3", "notebook": "0062/048/62048358.ipynb", "difficulty": "hard", "instruction_sha256": "ea2b2c566b9e9bd23664774c27cfffb7e33323b2b69130ec9aa30e2a172ef9d8"}, + {"name": "0085_687_85687102_qa_1", "source": "0085/687/85687102.ipynb_qa_1", "notebook": "0085/687/85687102.ipynb", "difficulty": "hard", "instruction_sha256": "141061354cb6c5061478b7e42ed8748d8aee68489a7eb47203a319b56aa4c34f"}, + {"name": "0042_902_42902013_qa_3", "source": "0042/902/42902013.ipynb_qa_3", "notebook": "0042/902/42902013.ipynb", "difficulty": "hard", "instruction_sha256": "90aefa4f273c0eb5887a49341ea9b42bd56de115853bbc3b7cc2077272fdc3b2"}, + {"name": "0069_427_69427691_qa_5", "source": "0069/427/69427691.ipynb_qa_5", "notebook": "0069/427/69427691.ipynb", "difficulty": "medium", "instruction_sha256": "4199a5832d637f9228a5d474564b68081bdf68d5623ddcddaba76d8c833cc6ba"}, + {"name": "0126_180_126180239_qa_2", "source": "0126/180/126180239.ipynb_qa_2", "notebook": "0126/180/126180239.ipynb", "difficulty": "medium", "instruction_sha256": "fdeaf21c036db1c81f3ffd2de473f20e3ab4d655145846354b8a2486bf447926"}, + {"name": "0023_598_23598239_qa_4", "source": "0023/598/23598239.ipynb_qa_4", "notebook": "0023/598/23598239.ipynb", "difficulty": "medium", "instruction_sha256": "233eadf89319bb795adb948db8a5be2b7275cef8519872f1d2b900610a27b69d"}, + {"name": "0002_094_2094789_qa_1", "source": "0002/094/2094789.ipynb_qa_1", "notebook": "0002/094/2094789.ipynb", "difficulty": "medium", "instruction_sha256": "fc2c6a2a51dd1e222c605738f4bc5fbb22ffc946ad6d70b657d914338defefc5"}, + {"name": "0059_141_59141107_qa_4", "source": "0059/141/59141107.ipynb_qa_4", "notebook": "0059/141/59141107.ipynb", "difficulty": "hard", "instruction_sha256": "4e843f9b2694ebe7381302d11242101bea0c19858a8a1da230fbc63d0fff4c2a"}, + {"name": "0054_266_54266894_qa_3", "source": "0054/266/54266894.ipynb_qa_3", "notebook": "0054/266/54266894.ipynb", "difficulty": "hard", "instruction_sha256": "fdf20747f279f120c53716f203181eaa21524f4cd1982d69d82fc14456b787d1"}, + {"name": "0027_503_27503967_qa_5", "source": "0027/503/27503967.ipynb_qa_5", "notebook": "0027/503/27503967.ipynb", "difficulty": "hard", "instruction_sha256": "780aab30c95dcc0974d27ce1ec1b432039273e1073f438850c22a94404109288"}, + {"name": "0012_949_12949007_qa_4", "source": "0012/949/12949007.ipynb_qa_4", "notebook": "0012/949/12949007.ipynb", "difficulty": "medium", "instruction_sha256": "eff44b73e4a749f76f8f40839541b60bae3ab306a8aa56a23aa0bbb59a1ea5a7"}, + {"name": "0119_037_119037763_qa_4", "source": "0119/037/119037763.ipynb_qa_4", "notebook": "0119/037/119037763.ipynb", "difficulty": "hard", "instruction_sha256": "78cb1f4d2ef7b5b8e194037e5b82e6e0e35a6b5fbd8c9a516a813874e107f8eb"}, + {"name": "0077_572_77572617_qa_3", "source": "0077/572/77572617.ipynb_qa_3", "notebook": "0077/572/77572617.ipynb", "difficulty": "hard", "instruction_sha256": "4f915cdd4e7a0d7057efbc7dd61a25e335c46bde81739c4f36d49281935935ff"}, + {"name": "0030_991_30991339_qa_4", "source": "0030/991/30991339.ipynb_qa_4", "notebook": "0030/991/30991339.ipynb", "difficulty": "hard", "instruction_sha256": "c47d536852210ad104df0b6b13a2e0819883f0543990359970a2d182af2bde5d"}, + {"name": "0052_814_52814699_qa_5", "source": "0052/814/52814699.ipynb_qa_5", "notebook": "0052/814/52814699.ipynb", "difficulty": "hard", "instruction_sha256": "a7d7aa6aaa9409135fccea60d0eb5f16aefeb3bffc74d6bb8b347e64856056dd"}, + {"name": "0045_118_45118642_qa_2", "source": "0045/118/45118642.ipynb_qa_2", "notebook": "0045/118/45118642.ipynb", "difficulty": "hard", "instruction_sha256": "8a929d71348ec044937a9a1d2ef8911a95c8f0bdb295c161ef2e6a674cb7efcb"}, + {"name": "0026_528_26528881_qa_3", "source": "0026/528/26528881.ipynb_qa_3", "notebook": "0026/528/26528881.ipynb", "difficulty": "hard", "instruction_sha256": "634342d29cf5084a00d81a959d4e00e42e170cd1531d97690b7ede9262b5f7ab"}, + {"name": "0028_601_28601781_qa_4", "source": "0028/601/28601781.ipynb_qa_4", "notebook": "0028/601/28601781.ipynb", "difficulty": "hard", "instruction_sha256": "76258fbe4ed18f3647db577fba89d161438ab7b6ba08e93358097273e7e50a35"}, + {"name": "0103_514_103514732_qa_1", "source": "0103/514/103514732.ipynb_qa_1", "notebook": "0103/514/103514732.ipynb", "difficulty": "hard", "instruction_sha256": "d072befe0b98d9bb2e09e67a9305bc717f3c23dfcf7628f7bfea07c010ceaf40"}, + {"name": "0024_081_24081260_qa_3", "source": "0024/081/24081260.ipynb_qa_3", "notebook": "0024/081/24081260.ipynb", "difficulty": "medium", "instruction_sha256": "3a54192daef7f30eb77029350ff18f975e9647df807a7a70de2292bdae63ba20"}, + {"name": "0058_494_58494583_qa_2", "source": "0058/494/58494583.ipynb_qa_2", "notebook": "0058/494/58494583.ipynb", "difficulty": "hard", "instruction_sha256": "957005b070a94baaefafc5c80ff87332479160b147d24d6361d46146fb7e2a6f"}, + {"name": "0071_437_71437256_qa_3", "source": "0071/437/71437256.ipynb_qa_3", "notebook": "0071/437/71437256.ipynb", "difficulty": "hard", "instruction_sha256": "3c161cbb60d8bd1104f83d1252a7d5b3252bd2f94d1f4bcd47ad144ab57e4044"}, + {"name": "0061_837_61837525_qa_5", "source": "0061/837/61837525.ipynb_qa_5", "notebook": "0061/837/61837525.ipynb", "difficulty": "hard", "instruction_sha256": "bbc873eb4a12f1552af5a90ced5c12174e64b9bc7f117cf087c3265183ca66c5"}, + {"name": "0048_488_48488979_qa_4", "source": "0048/488/48488979.ipynb_qa_4", "notebook": "0048/488/48488979.ipynb", "difficulty": "medium", "instruction_sha256": "8b1291e194b2448df5f78d6f5b876a5baed67b7f4a29b025ee4b2f3b9e98e15f"}, + {"name": "0030_991_30991339_qa_2", "source": "0030/991/30991339.ipynb_qa_2", "notebook": "0030/991/30991339.ipynb", "difficulty": "hard", "instruction_sha256": "4f5bf3a4c19a9a00abde7c328c55d515680f271d7befc0ef4399222962a256e8"}, + {"name": "0012_689_12689524_qa_4", "source": "0012/689/12689524.ipynb_qa_4", "notebook": "0012/689/12689524.ipynb", "difficulty": "medium", "instruction_sha256": "72c1d988b33bc7e423905a1d321473f020af480052c57b0fb1ae0d62a1cd99b3"}, + {"name": "0011_587_11587305_qa_5", "source": "0011/587/11587305.ipynb_qa_5", "notebook": "0011/587/11587305.ipynb", "difficulty": "medium", "instruction_sha256": "cf896007870b0b19274ce6cb35f29277d171ddd87e42eee58ddfe2b8c01d7749"}, + {"name": "0042_383_42383310_qa_2", "source": "0042/383/42383310.ipynb_qa_2", "notebook": "0042/383/42383310.ipynb", "difficulty": "medium", "instruction_sha256": "5ef5cef46204edf0ae71a1661104a7678b6b7073569ffc4ed8718e9c299bdfbe"}, + {"name": "0028_492_28492741_qa_1", "source": "0028/492/28492741.ipynb_qa_1", "notebook": "0028/492/28492741.ipynb", "difficulty": "hard", "instruction_sha256": "bd05773c52206f4eeddbc0329b227f3056a428832df7a0a77c4c7c1b77a0b4e7"}, + {"name": "0085_903_85903531_qa_1", "source": "0085/903/85903531.ipynb_qa_1", "notebook": "0085/903/85903531.ipynb", "difficulty": "hard", "instruction_sha256": "70e7fa805675ee92767470dc4804de9226a022903ac97d2bcbac0fc6d110382d"}, + {"name": "0121_603_121603419_qa_3", "source": "0121/603/121603419.ipynb_qa_3", "notebook": "0121/603/121603419.ipynb", "difficulty": "hard", "instruction_sha256": "9a5b28c0317b6485c3bcdda1e493da8895debf08e47656fe1d3ae59fbe11864f"}, + {"name": "0039_968_39968264_qa_3", "source": "0039/968/39968264.ipynb_qa_3", "notebook": "0039/968/39968264.ipynb", "difficulty": "hard", "instruction_sha256": "52a1faaec48b3e79232dd6dc09a257a6fd1959d30d91e4daa2b2507cfbaf370c"}, + {"name": "0043_124_43124508_qa_1", "source": "0043/124/43124508.ipynb_qa_1", "notebook": "0043/124/43124508.ipynb", "difficulty": "hard", "instruction_sha256": "f2e721bcac87e14862d53cef4c63be67ef220824060cd247ec9e989b07713d1a"}, + {"name": "0023_980_23980311_qa_3", "source": "0023/980/23980311.ipynb_qa_3", "notebook": "0023/980/23980311.ipynb", "difficulty": "hard", "instruction_sha256": "c895c10d0a472a0511d67e83ce3388bed44625397028f83358018a9d4b93ea1a"}, + {"name": "0056_538_56538672_qa_5", "source": "0056/538/56538672.ipynb_qa_5", "notebook": "0056/538/56538672.ipynb", "difficulty": "hard", "instruction_sha256": "7d28eb22e98c9cebda3f7586074ce1d692e30023f4e1cc8d8912835c2e78c723"}, + {"name": "0039_006_39006547_qa_4", "source": "0039/006/39006547.ipynb_qa_4", "notebook": "0039/006/39006547.ipynb", "difficulty": "medium", "instruction_sha256": "1892500d1b16d2e5a5c8ad4079e3765386666fbef94bf671be6875e7cbbabde7"}, + {"name": "0076_072_76072290_qa_5", "source": "0076/072/76072290.ipynb_qa_5", "notebook": "0076/072/76072290.ipynb", "difficulty": "hard", "instruction_sha256": "c7a8ad6c3e537f0fa3b957836c515b08327cc62e53a4016a1d26a791645010ab"}, + {"name": "0074_094_74094547_qa_2", "source": "0074/094/74094547.ipynb_qa_2", "notebook": "0074/094/74094547.ipynb", "difficulty": "hard", "instruction_sha256": "dffafa179055767f33ebba21ac25a92a96f4b34c4b74e26b2705a0c6e48fc3eb"}, + {"name": "0044_721_44721454_qa_1", "source": "0044/721/44721454.ipynb_qa_1", "notebook": "0044/721/44721454.ipynb", "difficulty": "medium", "instruction_sha256": "8419ba26968d493a31ff63390afbae04cd81262429e8e0436b59caa00968c797"}, + {"name": "0108_363_108363990_qa_2", "source": "0108/363/108363990.ipynb_qa_2", "notebook": "0108/363/108363990.ipynb", "difficulty": "medium", "instruction_sha256": "4e68782388d8d699d8b1957dbfbe5402c8d253022e6c18afb139fd1e6f917e6e"}, + {"name": "0075_579_75579600_qa_1", "source": "0075/579/75579600.ipynb_qa_1", "notebook": "0075/579/75579600.ipynb", "difficulty": "medium", "instruction_sha256": "d2e81921ae60926887e85bdd3cac8c47e9a5387f64641f2d5f0461019758c285"}, + {"name": "0072_403_72403187_qa_1", "source": "0072/403/72403187.ipynb_qa_1", "notebook": "0072/403/72403187.ipynb", "difficulty": "hard", "instruction_sha256": "b405dc570a296356f5cfc0ff001f3c3fcdbbcb2f55918a1c8490308105101106"}, + {"name": "0032_205_32205594_qa_1", "source": "0032/205/32205594.ipynb_qa_1", "notebook": "0032/205/32205594.ipynb", "difficulty": "medium", "instruction_sha256": "d43f91de355a155fe6d0d92fbdc700d5ca844b4ba25688884a118149386123e4"}, + {"name": "0088_996_88996860_qa_1", "source": "0088/996/88996860.ipynb_qa_1", "notebook": "0088/996/88996860.ipynb", "difficulty": "hard", "instruction_sha256": "7b417a716f29649e2616ce8dd41fdd42a2f64ca977ac8ab6b42e511fa6b9304c"}, + {"name": "0088_409_88409114_qa_4", "source": "0088/409/88409114.ipynb_qa_4", "notebook": "0088/409/88409114.ipynb", "difficulty": "hard", "instruction_sha256": "626a3535ddc7a1ca748a895388219e3cbaf38a9ce1f7fafb9a111a3284f49d54"}, + {"name": "0139_186_139186690_qa_5", "source": "0139/186/139186690.ipynb_qa_5", "notebook": "0139/186/139186690.ipynb", "difficulty": "hard", "instruction_sha256": "36027fcd15b1e04452eddf8b7bf985a8dab64a65c732c489d0a9fb7a47edceb2"}, + {"name": "0030_991_30991339_qa_3", "source": "0030/991/30991339.ipynb_qa_3", "notebook": "0030/991/30991339.ipynb", "difficulty": "hard", "instruction_sha256": "e64adc72be3271d1a62b60b2149f6dc9795687b6286089cfe8caa9af4df68a19"}, + {"name": "0001_930_1930122_qa_2", "source": "0001/930/1930122.ipynb_qa_2", "notebook": "0001/930/1930122.ipynb", "difficulty": "medium", "instruction_sha256": "b9f20709b47b9f5258316892192fff32c10934211009292459069d656bcd6f68"}, + {"name": "0070_379_70379910_qa_5", "source": "0070/379/70379910.ipynb_qa_5", "notebook": "0070/379/70379910.ipynb", "difficulty": "hard", "instruction_sha256": "f8f1fc9fd6629ac0dced46f9a537578d9dfbf31bcbbce543fa2f1786ac689d1a"}, + {"name": "0039_354_39354291_qa_3", "source": "0039/354/39354291.ipynb_qa_3", "notebook": "0039/354/39354291.ipynb", "difficulty": "hard", "instruction_sha256": "20d23fca67e6895361d63326b374833b650d9089fdd56a87710a40d00439b53c"}, + {"name": "0114_503_114503160_qa_2", "source": "0114/503/114503160.ipynb_qa_2", "notebook": "0114/503/114503160.ipynb", "difficulty": "hard", "instruction_sha256": "4e4c30147de8286eaa996bbe75bfcb66cc21e199245c4018b93d53dce5ff4e29"}, + {"name": "0034_773_34773515_qa_5", "source": "0034/773/34773515.ipynb_qa_5", "notebook": "0034/773/34773515.ipynb", "difficulty": "medium", "instruction_sha256": "baa2d060c61719650eca6a42aeb77ab30c49e7d7f7b080b9870037945159513c"}, + {"name": "0035_586_35586526_qa_2", "source": "0035/586/35586526.ipynb_qa_2", "notebook": "0035/586/35586526.ipynb", "difficulty": "hard", "instruction_sha256": "dc82a5d468aee141c04a8a8d16d998edc44eb36d6d25d68e91c8dcd9caadaff5"}, + {"name": "0061_879_61879031_qa_2", "source": "0061/879/61879031.ipynb_qa_2", "notebook": "0061/879/61879031.ipynb", "difficulty": "hard", "instruction_sha256": "c58ca0f676fedf7112333caa2be58aa5b50702b02e8a4820863750cf26333ead"}, + {"name": "0069_294_69294198_qa_5", "source": "0069/294/69294198.ipynb_qa_5", "notebook": "0069/294/69294198.ipynb", "difficulty": "hard", "instruction_sha256": "2f42507e590c70bc5447500e30625603df902afccc461a3c5fd99671fd2e3aa8"}, + {"name": "0065_745_65745045_qa_3", "source": "0065/745/65745045.ipynb_qa_3", "notebook": "0065/745/65745045.ipynb", "difficulty": "medium", "instruction_sha256": "7fa2757ef182ea57c02e85bf9a77c6f326d8f0d6eec6b89874a5563ebeb6c36e"}, + {"name": "0053_345_53345266_qa_3", "source": "0053/345/53345266.ipynb_qa_3", "notebook": "0053/345/53345266.ipynb", "difficulty": "hard", "instruction_sha256": "119f29f65fdd37804699db3c4e5ef5684312a86c1654088a5fce40f464f60f4f"}, + {"name": "0106_674_106674377_qa_1", "source": "0106/674/106674377.ipynb_qa_1", "notebook": "0106/674/106674377.ipynb", "difficulty": "medium", "instruction_sha256": "7802ef20ea613af8d637548079a4a8977a6536cde1419d1547d91f95b777b1e9"}, + {"name": "0087_388_87388798_qa_1", "source": "0087/388/87388798.ipynb_qa_1", "notebook": "0087/388/87388798.ipynb", "difficulty": "medium", "instruction_sha256": "9e49ca2d4c2c8e8e78ceabd70b2e5e85ac3858564e8490290666f2e4f50d8327"}, + {"name": "0016_952_16952441_qa_4", "source": "0016/952/16952441.ipynb_qa_4", "notebook": "0016/952/16952441.ipynb", "difficulty": "medium", "instruction_sha256": "07555513d2eeceb9a55abbb96680bf5bd5d3f3c378e9dc922ecc79714635d718"}, + {"name": "0116_436_116436448_qa_2", "source": "0116/436/116436448.ipynb_qa_2", "notebook": "0116/436/116436448.ipynb", "difficulty": "medium", "instruction_sha256": "98bb797a25d77a12e9bbcf6ddf53e7720bb5e8c3cc5969482e14f4aada672378"}, + {"name": "0029_204_29204719_qa_2", "source": "0029/204/29204719.ipynb_qa_2", "notebook": "0029/204/29204719.ipynb", "difficulty": "hard", "instruction_sha256": "b2c7a410ee56dd2d4199610f9e6bf603457c0da7c79fff0bd0c33b45e5d1e0aa"}, + {"name": "0098_463_98463251_qa_3", "source": "0098/463/98463251.ipynb_qa_3", "notebook": "0098/463/98463251.ipynb", "difficulty": "hard", "instruction_sha256": "35ec89b142db9376af7fddc863d9b42a0488e161e6e68e25e2cdd778b124ee58"}, + {"name": "0085_451_85451661_qa_2", "source": "0085/451/85451661.ipynb_qa_2", "notebook": "0085/451/85451661.ipynb", "difficulty": "medium", "instruction_sha256": "bbcb5940a16ed9a96121af2a381e53396fc586c47e2ac7045b2905a34a9be5e9"}, + {"name": "0057_686_57686190_qa_1", "source": "0057/686/57686190.ipynb_qa_1", "notebook": "0057/686/57686190.ipynb", "difficulty": "hard", "instruction_sha256": "6529e986f8b50866774e6fae93a0ce8d5b47cff1fff69169a5e617941c695c3e"}, + {"name": "0064_808_64808166_qa_5", "source": "0064/808/64808166.ipynb_qa_5", "notebook": "0064/808/64808166.ipynb", "difficulty": "hard", "instruction_sha256": "d0d22d43f157af5dc5ddff453c4e1e9543789ec63fc4829a877ea7bc17e1ad82"}, + {"name": "0076_746_76746181_qa_2", "source": "0076/746/76746181.ipynb_qa_2", "notebook": "0076/746/76746181.ipynb", "difficulty": "medium", "instruction_sha256": "5f7d6fddb878fc79cb3bdbf905614f24bdece5917c5b6c867b43671235e5c5b6"}, + {"name": "0089_051_89051135_qa_2", "source": "0089/051/89051135.ipynb_qa_2", "notebook": "0089/051/89051135.ipynb", "difficulty": "hard", "instruction_sha256": "ab5720940e12ad017dde0afe3e7efd8f69a24c237f48aae60cd869ba30250130"}, + {"name": "0077_266_77266642_qa_3", "source": "0077/266/77266642.ipynb_qa_3", "notebook": "0077/266/77266642.ipynb", "difficulty": "hard", "instruction_sha256": "29e1b23f5e0b9ce116389632cb197f0b890c6b805e25383f46b69765e6759211"}, + {"name": "0021_828_21828791_qa_2", "source": "0021/828/21828791.ipynb_qa_2", "notebook": "0021/828/21828791.ipynb", "difficulty": "hard", "instruction_sha256": "52dcaf9745cc1dd65e6dc2d085e41bc30c67dac85c473834dda6c9910123a9af"}, + {"name": "0040_785_40785152_qa_2", "source": "0040/785/40785152.ipynb_qa_2", "notebook": "0040/785/40785152.ipynb", "difficulty": "medium", "instruction_sha256": "3b0f0a10445e536500533f4e9a436c90591f402ab160f0e9a1fad61796924df9"}, + {"name": "0104_947_104947882_qa_3", "source": "0104/947/104947882.ipynb_qa_3", "notebook": "0104/947/104947882.ipynb", "difficulty": "medium", "instruction_sha256": "c94f588ce965ae32bc514e0cfcb51f5dc80a61c9b611eb5e61a0f6e903c8e408"}, + {"name": "0061_528_61528663_qa_5", "source": "0061/528/61528663.ipynb_qa_5", "notebook": "0061/528/61528663.ipynb", "difficulty": "hard", "instruction_sha256": "272216d79c508eb5923230541d395fb9e896ed01ec46c6bb95d1968158bcce37"}, + {"name": "0132_903_132903502_qa_3", "source": "0132/903/132903502.ipynb_qa_3", "notebook": "0132/903/132903502.ipynb", "difficulty": "hard", "instruction_sha256": "fff14c17142d808a7b60528ce9790de77dee75dc09092509cf6383a1a56ffc9b"}, + {"name": "0039_392_39392401_qa_1", "source": "0039/392/39392401.ipynb_qa_1", "notebook": "0039/392/39392401.ipynb", "difficulty": "hard", "instruction_sha256": "b361f9f5d32f0e2f98dc7c0f48828104fd7458f44cdc1b75e56793050859cd85"}, + {"name": "0046_487_46487066_qa_1", "source": "0046/487/46487066.ipynb_qa_1", "notebook": "0046/487/46487066.ipynb", "difficulty": "hard", "instruction_sha256": "72230aa9e9a44a9b4ed97e0d192ccf2471abf90f074748f35912f2a2826b03be"}, + {"name": "0098_681_98681805_qa_3", "source": "0098/681/98681805.ipynb_qa_3", "notebook": "0098/681/98681805.ipynb", "difficulty": "hard", "instruction_sha256": "601d501ae6ef83f28952de6af8b18cb56f3f00a356b7443e2cf182470153fa4b"}, + {"name": "0103_452_103452394_qa_2", "source": "0103/452/103452394.ipynb_qa_2", "notebook": "0103/452/103452394.ipynb", "difficulty": "medium", "instruction_sha256": "5779157472bb11efe645be236d06d98cc8201438dd1a4fe422367f092f4ff111"}, + {"name": "0103_923_103923129_qa_1", "source": "0103/923/103923129.ipynb_qa_1", "notebook": "0103/923/103923129.ipynb", "difficulty": "hard", "instruction_sha256": "7e6a3e0a5d2da58dfb7ddd047dfd39145cc5c30f4672daed00576cb412fede0c"}, + {"name": "0071_997_71997138_qa_3", "source": "0071/997/71997138.ipynb_qa_3", "notebook": "0071/997/71997138.ipynb", "difficulty": "hard", "instruction_sha256": "36fe43313d2ae4bff3854099b0d60c9974bb5bfb5ce4d3b973650c83660f1b31"}, + {"name": "0090_648_90648013_qa_4", "source": "0090/648/90648013.ipynb_qa_4", "notebook": "0090/648/90648013.ipynb", "difficulty": "medium", "instruction_sha256": "7d838c79752f840ccbd4722169e5129223ba7a67ad5fb4479a1e8aed32012497"}, + {"name": "0126_746_126746817_qa_1", "source": "0126/746/126746817.ipynb_qa_1", "notebook": "0126/746/126746817.ipynb", "difficulty": "hard", "instruction_sha256": "d4a682e5dbd29035da9f26bb9d434b1cde51812af398fc81b19d1a01678adc9b"}, + {"name": "0035_835_35835284_qa_3", "source": "0035/835/35835284.ipynb_qa_3", "notebook": "0035/835/35835284.ipynb", "difficulty": "hard", "instruction_sha256": "f72ef11cb1fadf972cc2f055d79e347e84747c27a7146fa85a07af18e9dca74e"}, + {"name": "0114_673_114673575_qa_5", "source": "0114/673/114673575.ipynb_qa_5", "notebook": "0114/673/114673575.ipynb", "difficulty": "medium", "instruction_sha256": "e01af876d6698d2c52e07964610d8c865c3b68fe09517734d941f5535a13e8a6"}, + {"name": "0071_835_71835908_qa_1", "source": "0071/835/71835908.ipynb_qa_1", "notebook": "0071/835/71835908.ipynb", "difficulty": "medium", "instruction_sha256": "26fa80d52a37c2e46f122819646c9a5ec2e9cf3d07ea955cd9d04de752c911f3"}, + {"name": "0077_529_77529364_qa_3", "source": "0077/529/77529364.ipynb_qa_3", "notebook": "0077/529/77529364.ipynb", "difficulty": "hard", "instruction_sha256": "3968614f10e02c5a5fad629b1a8b1d0e4279246a7529ab3855d7e4960f39d127"}, + {"name": "0044_609_44609132_qa_3", "source": "0044/609/44609132.ipynb_qa_3", "notebook": "0044/609/44609132.ipynb", "difficulty": "medium", "instruction_sha256": "ce99be0e7028d5ffd55026ebea5efa174fd8e0193e42d75f82c675686379ae12"}, + {"name": "0024_650_24650942_qa_2", "source": "0024/650/24650942.ipynb_qa_2", "notebook": "0024/650/24650942.ipynb", "difficulty": "medium", "instruction_sha256": "bb7aa06b21681e0fbf3a56091a617cb81f40d35a7028be283a96cef962260d6a"}, + {"name": "0032_734_32734784_qa_3", "source": "0032/734/32734784.ipynb_qa_3", "notebook": "0032/734/32734784.ipynb", "difficulty": "medium", "instruction_sha256": "e659102c615e05922862e94a9d39307a7890f1140c852458b5afb3911e9a33b1"}, + {"name": "0025_241_25241426_qa_3", "source": "0025/241/25241426.ipynb_qa_3", "notebook": "0025/241/25241426.ipynb", "difficulty": "medium", "instruction_sha256": "be813de48c8f5c286ed8af3578f401cd65d82066fc237daeef96de8c4310f373"}, + {"name": "0125_379_125379106_qa_4", "source": "0125/379/125379106.ipynb_qa_4", "notebook": "0125/379/125379106.ipynb", "difficulty": "hard", "instruction_sha256": "dd8d80883e605f04752c9bc6d19477ef34ff46b28c25a3399642690cf0f5cbe2"}, + {"name": "0064_469_64469829_qa_4", "source": "0064/469/64469829.ipynb_qa_4", "notebook": "0064/469/64469829.ipynb", "difficulty": "hard", "instruction_sha256": "6b5c1333941b17a863aa13ebb4353186856a7df9cb15aea9191f0d9d4d4db88d"}, + {"name": "0001_599_1599781_qa_5", "source": "0001/599/1599781.ipynb_qa_5", "notebook": "0001/599/1599781.ipynb", "difficulty": "medium", "instruction_sha256": "e3b9ae2d15fd254f885a9350db6d513420750ffa9a560242364fb2f3858ff148"}, + {"name": "0109_525_109525152_qa_1", "source": "0109/525/109525152.ipynb_qa_1", "notebook": "0109/525/109525152.ipynb", "difficulty": "medium", "instruction_sha256": "fc05a287816056cdc22535714f6125390112a77554e17999f9857bfb5dc0d855"}, + {"name": "0124_966_124966489_qa_5", "source": "0124/966/124966489.ipynb_qa_5", "notebook": "0124/966/124966489.ipynb", "difficulty": "medium", "instruction_sha256": "5b6e97d69731207296c4e1bbb8e21a727a6c7b6c8b043009bd2e925850640f9c"}, + {"name": "0020_610_20610563_qa_5", "source": "0020/610/20610563.ipynb_qa_5", "notebook": "0020/610/20610563.ipynb", "difficulty": "hard", "instruction_sha256": "1887008fb11615ea64d5fbb5df844d3bc3ddca535a8f268e65e41ed425805026"}, + {"name": "0053_295_53295093_qa_1", "source": "0053/295/53295093.ipynb_qa_1", "notebook": "0053/295/53295093.ipynb", "difficulty": "medium", "instruction_sha256": "8004ef6032efdd6b6110bcee146b0ca5c046715958d73a653b3e6bf85adecd54"}, + {"name": "0029_467_29467495_qa_2", "source": "0029/467/29467495.ipynb_qa_2", "notebook": "0029/467/29467495.ipynb", "difficulty": "hard", "instruction_sha256": "996febbafa05617d0b90d34e12e8063e3265a7cbcd7aa0a5d52ae1f993d91f7d"}, + {"name": "0127_275_127275270_qa_1", "source": "0127/275/127275270.ipynb_qa_1", "notebook": "0127/275/127275270.ipynb", "difficulty": "hard", "instruction_sha256": "e5da9f36f35cee0d058083fc9d520d2661d4309a13617fdbe117c76b8d64a98f"}, + {"name": "0055_388_55388977_qa_5", "source": "0055/388/55388977.ipynb_qa_5", "notebook": "0055/388/55388977.ipynb", "difficulty": "hard", "instruction_sha256": "df2376db7cd9efc79dff767045872b16030821585c368ecb731db7eecb413012"}, + {"name": "0089_775_89775918_qa_4", "source": "0089/775/89775918.ipynb_qa_4", "notebook": "0089/775/89775918.ipynb", "difficulty": "hard", "instruction_sha256": "c5a21a39cfc4665a189ec1a41f3f95ad5ddaa73ee125d7dc96f8541b0b7e18d4"}, + {"name": "0030_284_30284375_qa_1", "source": "0030/284/30284375.ipynb_qa_1", "notebook": "0030/284/30284375.ipynb", "difficulty": "hard", "instruction_sha256": "cbe623ff6d37b13c7a26d12e32f0501594aa08138dad770db2c2d5a556f87dc4"}, + {"name": "0080_564_80564946_qa_5", "source": "0080/564/80564946.ipynb_qa_5", "notebook": "0080/564/80564946.ipynb", "difficulty": "hard", "instruction_sha256": "3aefcfbdb24e654cc32e830751703a01d9939f12c576a4b5e728e0f35e21cd0f"}, + {"name": "0066_515_66515711_qa_4", "source": "0066/515/66515711.ipynb_qa_4", "notebook": "0066/515/66515711.ipynb", "difficulty": "medium", "instruction_sha256": "6d0f2a56f0eb184a346d54d9df73402875ce314720d2eb9321097bd7e7b735ed"}, + {"name": "0072_216_72216676_qa_4", "source": "0072/216/72216676.ipynb_qa_4", "notebook": "0072/216/72216676.ipynb", "difficulty": "medium", "instruction_sha256": "0e8e7619fd768519ca616a80c2e83f66e485692c842f7971e4c07a5c4353a381"}, + {"name": "0047_266_47266614_qa_1", "source": "0047/266/47266614.ipynb_qa_1", "notebook": "0047/266/47266614.ipynb", "difficulty": "hard", "instruction_sha256": "c5e24ee9a66ffbc372815fb2130f09aab22258e2e59906efd985d0c0af036e3a"}, + {"name": "0114_928_114928761_qa_5", "source": "0114/928/114928761.ipynb_qa_5", "notebook": "0114/928/114928761.ipynb", "difficulty": "hard", "instruction_sha256": "58174854ce4c5b7d576e22c0f38a3bbb9972f38669757b9906af4f368a127023"}, + {"name": "0085_383_85383242_qa_5", "source": "0085/383/85383242.ipynb_qa_5", "notebook": "0085/383/85383242.ipynb", "difficulty": "hard", "instruction_sha256": "0592f82f0154680f676f4b42038dbc9d800f03e4df1045d2491c6e41706895de"}, + {"name": "0110_622_110622713_qa_2", "source": "0110/622/110622713.ipynb_qa_2", "notebook": "0110/622/110622713.ipynb", "difficulty": "medium", "instruction_sha256": "1b210e77fe0f7ba22dc91b63fc8c9237ab1485f6e969da4438f879bc78cde661"}, + {"name": "0037_236_37236169_qa_4", "source": "0037/236/37236169.ipynb_qa_4", "notebook": "0037/236/37236169.ipynb", "difficulty": "medium", "instruction_sha256": "2621e151c407548cb07adbe1eafb94c3f645a8652f3f99cfff5e1d75324b2472"}, + {"name": "0087_941_87941186_qa_3", "source": "0087/941/87941186.ipynb_qa_3", "notebook": "0087/941/87941186.ipynb", "difficulty": "medium", "instruction_sha256": "32a6f01eb5b85a1690d1c665ab79e0ae9a23799ffda110f24c3698b5d2dbf9cb"}, + {"name": "0096_673_96673455_qa_1", "source": "0096/673/96673455.ipynb_qa_1", "notebook": "0096/673/96673455.ipynb", "difficulty": "hard", "instruction_sha256": "99fe287aa036e306e2720e539cecf9493cb915e9e5c939bd3e4d96da7d2baa11"}, + {"name": "0077_724_77724747_qa_3", "source": "0077/724/77724747.ipynb_qa_3", "notebook": "0077/724/77724747.ipynb", "difficulty": "hard", "instruction_sha256": "e2f61646423e33448f46ab207ba322af85570e742dd6c0f230c88dc1b8c2031e"}, + {"name": "0022_383_22383259_qa_5", "source": "0022/383/22383259.ipynb_qa_5", "notebook": "0022/383/22383259.ipynb", "difficulty": "hard", "instruction_sha256": "883856e470d44bf7790d5fa71e7a6d69037929b6c27b34b8ea138d660bc9cf23"}, + {"name": "0056_907_56907084_qa_3", "source": "0056/907/56907084.ipynb_qa_3", "notebook": "0056/907/56907084.ipynb", "difficulty": "hard", "instruction_sha256": "0dcc811708d4d7cfc4f2844decbe13b5d8df88eef6e646cbe7744ab6071cf76f"}, + {"name": "0056_414_56414713_qa_4", "source": "0056/414/56414713.ipynb_qa_4", "notebook": "0056/414/56414713.ipynb", "difficulty": "hard", "instruction_sha256": "24e1d6feaa8520bae7310b38e4c9d2070316f5587380b8b752e5db1d8ec8d5e4"}, + {"name": "0117_642_117642587_qa_5", "source": "0117/642/117642587.ipynb_qa_5", "notebook": "0117/642/117642587.ipynb", "difficulty": "medium", "instruction_sha256": "6ebdbf165971bd2ea0df5f5d78da9a0302797a69d01b67bda234a7b15893238a"}, + {"name": "0055_474_55474406_qa_3", "source": "0055/474/55474406.ipynb_qa_3", "notebook": "0055/474/55474406.ipynb", "difficulty": "hard", "instruction_sha256": "3853b0efacf11b2d0d24cd1d7699b1f0b5557cc2b087d8bd0b3ad92c12cc8697"}, + {"name": "0035_362_35362667_qa_4", "source": "0035/362/35362667.ipynb_qa_4", "notebook": "0035/362/35362667.ipynb", "difficulty": "medium", "instruction_sha256": "7f269f1c3d658fc83a3914f3b30dd199271dae13ffce2023e6dcc9eec581dfe4"}, + {"name": "0113_924_113924850_qa_4", "source": "0113/924/113924850.ipynb_qa_4", "notebook": "0113/924/113924850.ipynb", "difficulty": "hard", "instruction_sha256": "36d8d82066783f9f08cd20ac257e7fd7263c9f7cc0e7e641935e333ca2a9772f"}, + {"name": "0078_483_78483410_qa_3", "source": "0078/483/78483410.ipynb_qa_3", "notebook": "0078/483/78483410.ipynb", "difficulty": "hard", "instruction_sha256": "1d2c0e115eebdb3c1a045609d7633630f17e9b1ad5b113655f5b11c129395403"}, + {"name": "0092_990_92990642_qa_2", "source": "0092/990/92990642.ipynb_qa_2", "notebook": "0092/990/92990642.ipynb", "difficulty": "hard", "instruction_sha256": "e99d5e8ebe32238a3297d3e69c8c7e6bf457335b7600698716d08ce1faca6621"}, + {"name": "0136_198_136198797_qa_1", "source": "0136/198/136198797.ipynb_qa_1", "notebook": "0136/198/136198797.ipynb", "difficulty": "medium", "instruction_sha256": "5ad4ace2fdf22b636df6eb89d5fe93c680bf97da2f327111b2961a031c2ffe10"}, + {"name": "0057_639_57639553_qa_5", "source": "0057/639/57639553.ipynb_qa_5", "notebook": "0057/639/57639553.ipynb", "difficulty": "hard", "instruction_sha256": "a811da17130b2afe0a006ab5a9907e719c7455c438aaf9019365fc252ab75baf"}, + {"name": "0041_261_41261699_qa_2", "source": "0041/261/41261699.ipynb_qa_2", "notebook": "0041/261/41261699.ipynb", "difficulty": "hard", "instruction_sha256": "5b9e7c0dbd3859e1c02f8d7f5129401b4f873851bb7d6ded0562e874d41e21cb"}, + {"name": "0133_040_133040358_qa_3", "source": "0133/040/133040358.ipynb_qa_3", "notebook": "0133/040/133040358.ipynb", "difficulty": "hard", "instruction_sha256": "e47861958eef842d133abe6cdf324d080c931e4fbcc93d1b1650de7f0bfb93c6"}, + {"name": "0041_015_41015677_qa_4", "source": "0041/015/41015677.ipynb_qa_4", "notebook": "0041/015/41015677.ipynb", "difficulty": "hard", "instruction_sha256": "7a85bc4be80317b64f68758e7742270a3ce413f338bfd88ebc236861dad95c43"}, + {"name": "0020_991_20991108_qa_5", "source": "0020/991/20991108.ipynb_qa_5", "notebook": "0020/991/20991108.ipynb", "difficulty": "hard", "instruction_sha256": "583eea808eedfb77fc7491e5359c723829309fa42508e497d952f0fb67d29c68"}, + {"name": "0050_885_50885329_qa_3", "source": "0050/885/50885329.ipynb_qa_3", "notebook": "0050/885/50885329.ipynb", "difficulty": "hard", "instruction_sha256": "f89b00bcaad3dca7e62e892f09f2f5fdca9c8309d821dc3cf1791d9840541835"}, + {"name": "0046_933_46933248_qa_5", "source": "0046/933/46933248.ipynb_qa_5", "notebook": "0046/933/46933248.ipynb", "difficulty": "medium", "instruction_sha256": "84cda5302531d6806c3bd7d0a494dfe4f55fa2cb943bd85adbb38e8badfc2055"}, + {"name": "0109_914_109914712_qa_2", "source": "0109/914/109914712.ipynb_qa_2", "notebook": "0109/914/109914712.ipynb", "difficulty": "hard", "instruction_sha256": "25c9c92bf0ceb782cf2653ca11ea4aab272500a174ff9c7a9d4011bec87f8c9c"}, + {"name": "0105_059_105059592_qa_5", "source": "0105/059/105059592.ipynb_qa_5", "notebook": "0105/059/105059592.ipynb", "difficulty": "hard", "instruction_sha256": "cd443f2230705ca314c73561183512c66cfb6f8bb8e4be6471d8530b2a0e2c8d"}, + {"name": "0032_057_32057827_qa_4", "source": "0032/057/32057827.ipynb_qa_4", "notebook": "0032/057/32057827.ipynb", "difficulty": "medium", "instruction_sha256": "a7d503cf19511f2beb4094f1f9a787d4ad579ec52c18b208e73cf2c5f39cac44"}, + {"name": "0085_154_85154775_qa_2", "source": "0085/154/85154775.ipynb_qa_2", "notebook": "0085/154/85154775.ipynb", "difficulty": "hard", "instruction_sha256": "2a2a8e44527c3c553ea459c90e0a3f1c8020ef32b8b718aebdd7dedb421167f5"}, + {"name": "0025_211_25211872_qa_4", "source": "0025/211/25211872.ipynb_qa_4", "notebook": "0025/211/25211872.ipynb", "difficulty": "medium", "instruction_sha256": "b7a462e9ee3d0a99f458c91b43ac459f850d009fa7fe181399485423fd7830a0"}, + {"name": "0010_489_10489077_qa_1", "source": "0010/489/10489077.ipynb_qa_1", "notebook": "0010/489/10489077.ipynb", "difficulty": "medium", "instruction_sha256": "61fa226e78c20770a86d6b1353e5cb7578d2ed71108d956eec396397753ca1e6"}, + {"name": "0035_332_35332810_qa_3", "source": "0035/332/35332810.ipynb_qa_3", "notebook": "0035/332/35332810.ipynb", "difficulty": "medium", "instruction_sha256": "43defef59efbb85fbb6b22bbc884817eb459de4854831cf8d14be1b36739f6a4"}, + {"name": "0109_469_109469247_qa_3", "source": "0109/469/109469247.ipynb_qa_3", "notebook": "0109/469/109469247.ipynb", "difficulty": "hard", "instruction_sha256": "a1810cf2a4f902b4c57723c51a1c7ec764a3f43ff391e250b54fbc9034d3ee3b"}, + {"name": "0124_384_124384181_qa_1", "source": "0124/384/124384181.ipynb_qa_1", "notebook": "0124/384/124384181.ipynb", "difficulty": "medium", "instruction_sha256": "1ac9c34b78f0a8ef3cd3e4ec4ca234584ec5af0c917e913ecdcc98ec8bcf5dad"}, + {"name": "0039_714_39714962_qa_1", "source": "0039/714/39714962.ipynb_qa_1", "notebook": "0039/714/39714962.ipynb", "difficulty": "medium", "instruction_sha256": "9c589092ef6a03e177997c2863a87fb5cfc709c54819afa10c960d526ade049a"}, + {"name": "0117_445_117445192_qa_4", "source": "0117/445/117445192.ipynb_qa_4", "notebook": "0117/445/117445192.ipynb", "difficulty": "hard", "instruction_sha256": "d83dd0d5d8318164524b06521eb7e29f55550093e6d93e5eb8e715a6f1879016"}, + {"name": "0012_992_12992267_qa_5", "source": "0012/992/12992267.ipynb_qa_5", "notebook": "0012/992/12992267.ipynb", "difficulty": "medium", "instruction_sha256": "115d588161185e7b295c3077ffb00673381a885f500f6a71caf07059272dd531"}, + {"name": "0111_571_111571326_qa_2", "source": "0111/571/111571326.ipynb_qa_2", "notebook": "0111/571/111571326.ipynb", "difficulty": "hard", "instruction_sha256": "ef67827d018b75c2ddfd8d1ddaef8a6121e2e505c9a22dd44d11abf0a142e53f"}, + {"name": "0127_467_127467979_qa_4", "source": "0127/467/127467979.ipynb_qa_4", "notebook": "0127/467/127467979.ipynb", "difficulty": "medium", "instruction_sha256": "f59ea809db6c992f8f36724d38894cfa67ca300b617ea48bd18f0c1710d4ac1e"}, + {"name": "0083_317_83317900_qa_1", "source": "0083/317/83317900.ipynb_qa_1", "notebook": "0083/317/83317900.ipynb", "difficulty": "hard", "instruction_sha256": "a0548466a86d13cf62db6be589312e994fdc3169385ed1f480279fb832fa345c"}, + {"name": "0115_972_115972863_qa_3", "source": "0115/972/115972863.ipynb_qa_3", "notebook": "0115/972/115972863.ipynb", "difficulty": "medium", "instruction_sha256": "0c337e56281619108792d4b270079927469d97197b94214032acca83c1e849c2"}, + {"name": "0063_482_63482132_qa_1", "source": "0063/482/63482132.ipynb_qa_1", "notebook": "0063/482/63482132.ipynb", "difficulty": "medium", "instruction_sha256": "a16dbacec8b0a9afa15939182adcf313a5c907c4e85954d990de28afb979278a"}, + {"name": "0039_613_39613543_qa_3", "source": "0039/613/39613543.ipynb_qa_3", "notebook": "0039/613/39613543.ipynb", "difficulty": "hard", "instruction_sha256": "5c157406dbdba2c8e48728130c571807ee4d197ad0983df985b7c3948c8231a8"}, + {"name": "0051_554_51554569_qa_5", "source": "0051/554/51554569.ipynb_qa_5", "notebook": "0051/554/51554569.ipynb", "difficulty": "hard", "instruction_sha256": "0d70ce52e83275a4cac8968b912c5cf3dacbf47a741a5f32e30dfbd36a8b678b"}, + {"name": "0078_655_78655226_qa_3", "source": "0078/655/78655226.ipynb_qa_3", "notebook": "0078/655/78655226.ipynb", "difficulty": "hard", "instruction_sha256": "d739288482951f442f2b4c4745afe6da48ffa6f54d9c146453215e74dea99e7e"}, + {"name": "0138_805_138805260_qa_1", "source": "0138/805/138805260.ipynb_qa_1", "notebook": "0138/805/138805260.ipynb", "difficulty": "hard", "instruction_sha256": "78b24e3a1dc81797095b14cb9c45fa7a5598bc3c6ca37fed8c1083e2517d4783"}, + {"name": "0069_075_69075979_qa_5", "source": "0069/075/69075979.ipynb_qa_5", "notebook": "0069/075/69075979.ipynb", "difficulty": "hard", "instruction_sha256": "82a3ad43348e9f9475e29f37ce83fc200981f217fb667f7e21ce5f456aed2d1d"}, + {"name": "0040_981_40981860_qa_1", "source": "0040/981/40981860.ipynb_qa_1", "notebook": "0040/981/40981860.ipynb", "difficulty": "hard", "instruction_sha256": "9328a727a7d1047cb9252ba82e5010c652011df28da987e395e9a2bf610abe8b"}, + {"name": "0091_774_91774768_qa_4", "source": "0091/774/91774768.ipynb_qa_4", "notebook": "0091/774/91774768.ipynb", "difficulty": "hard", "instruction_sha256": "9338c935e25c14cd89547c35c97c9aea7673d5b6b981e1222879c57e023011ec"}, + {"name": "0051_530_51530318_qa_3", "source": "0051/530/51530318.ipynb_qa_3", "notebook": "0051/530/51530318.ipynb", "difficulty": "hard", "instruction_sha256": "704c061b0856d5a8aa12358312c4f8e40487f813b0b9c427797b59ada2f5eedf"}, + {"name": "0051_054_51054366_qa_5", "source": "0051/054/51054366.ipynb_qa_5", "notebook": "0051/054/51054366.ipynb", "difficulty": "hard", "instruction_sha256": "85ac166f13d78054f7db8d873b2006b451d94988a34bc849235d186ec9d0b564"}, + {"name": "0110_889_110889035_qa_1", "source": "0110/889/110889035.ipynb_qa_1", "notebook": "0110/889/110889035.ipynb", "difficulty": "hard", "instruction_sha256": "bc4ada0d2ef1b44dd04d6b104abbbfc758790a5f90e11e9b717a2a2163410119"}, + {"name": "0060_106_60106319_qa_2", "source": "0060/106/60106319.ipynb_qa_2", "notebook": "0060/106/60106319.ipynb", "difficulty": "hard", "instruction_sha256": "7c50fb463ff613ffd6326d20de24887a6c20d5af6d95fa9cefd0683c4bbc9338"}, + {"name": "0054_170_54170809_qa_3", "source": "0054/170/54170809.ipynb_qa_3", "notebook": "0054/170/54170809.ipynb", "difficulty": "hard", "instruction_sha256": "e19f1513a3f58a1faf643860ced7b984fc94f9ee8433e6b3d722816e8e5fc465"}, + {"name": "0067_744_67744711_qa_4", "source": "0067/744/67744711.ipynb_qa_4", "notebook": "0067/744/67744711.ipynb", "difficulty": "medium", "instruction_sha256": "c20b91b666cdd4549f518f4618830bf5e0a4cbf1007853419862ad93ee405fb0"}, + {"name": "0070_379_70379910_qa_2", "source": "0070/379/70379910.ipynb_qa_2", "notebook": "0070/379/70379910.ipynb", "difficulty": "hard", "instruction_sha256": "a602e62bfbe666fdbccb85ed68d2f7d639ace311cbd45338f8b981f2ec21bba0"}, + {"name": "0104_115_104115755_qa_1", "source": "0104/115/104115755.ipynb_qa_1", "notebook": "0104/115/104115755.ipynb", "difficulty": "medium", "instruction_sha256": "fb57148284b1a794197e334091561a11eff14d86c432d4a2251dda2c91d57a9b"}, + {"name": "0068_060_68060356_qa_4", "source": "0068/060/68060356.ipynb_qa_4", "notebook": "0068/060/68060356.ipynb", "difficulty": "hard", "instruction_sha256": "3332dcc2cc57328c75512a7b1bac675afd5ba9d274d63a85a1958940a9e18cd9"}, + {"name": "0053_008_53008449_qa_4", "source": "0053/008/53008449.ipynb_qa_4", "notebook": "0053/008/53008449.ipynb", "difficulty": "hard", "instruction_sha256": "7d514b8e01a94157a91e9f9c7d9cf83b381acd40065450a7d3f5de19c1f08a07"}, + {"name": "0022_886_22886038_qa_4", "source": "0022/886/22886038.ipynb_qa_4", "notebook": "0022/886/22886038.ipynb", "difficulty": "hard", "instruction_sha256": "c1846cd220f8f76b384dc2e49cd9f9fbd25fa512b1d782887150d14daad62ef3"}, + {"name": "0127_099_127099750_qa_5", "source": "0127/099/127099750.ipynb_qa_5", "notebook": "0127/099/127099750.ipynb", "difficulty": "hard", "instruction_sha256": "3c85f86e0cb5e7c9bc50be505bc9514cccf12871a00a6886cf74f0a12e37c420"}, + {"name": "0057_378_57378613_qa_3", "source": "0057/378/57378613.ipynb_qa_3", "notebook": "0057/378/57378613.ipynb", "difficulty": "hard", "instruction_sha256": "13cdced0f433eeb06f836ceb7914c09167842a7e8d245cc1c1a065d1e1f27daf"}, + {"name": "0113_941_113941848_qa_4", "source": "0113/941/113941848.ipynb_qa_4", "notebook": "0113/941/113941848.ipynb", "difficulty": "hard", "instruction_sha256": "4ac807730bf423ab211516ec899776c53cf1e39f28377c6114135a5e723e726c"}, + {"name": "0062_653_62653158_qa_5", "source": "0062/653/62653158.ipynb_qa_5", "notebook": "0062/653/62653158.ipynb", "difficulty": "hard", "instruction_sha256": "01a819a4660f4663dd8c7bad441769f269e2c636a146464e6f96b257328a20b6"}, + {"name": "0031_829_31829521_qa_5", "source": "0031/829/31829521.ipynb_qa_5", "notebook": "0031/829/31829521.ipynb", "difficulty": "medium", "instruction_sha256": "b74cefa8a46f639b75d2cebf2088bd65b67adf5a8479b01f280b5211477cb25f"}, + {"name": "0075_890_75890673_qa_1", "source": "0075/890/75890673.ipynb_qa_1", "notebook": "0075/890/75890673.ipynb", "difficulty": "hard", "instruction_sha256": "daf1563ff5e668ca331b04aeb7e62e78db38ac9da765bdbe5b07f92dec1cbb37"}, + {"name": "0124_488_124488738_qa_1", "source": "0124/488/124488738.ipynb_qa_1", "notebook": "0124/488/124488738.ipynb", "difficulty": "medium", "instruction_sha256": "1023a1153f9c98788db0b6a1a3ea8b87f823ad8fef23144ee4a3bd7a15ab5b07"}, + {"name": "0128_435_128435180_qa_1", "source": "0128/435/128435180.ipynb_qa_1", "notebook": "0128/435/128435180.ipynb", "difficulty": "medium", "instruction_sha256": "f50a79db778b53ca0b4262f173cc6c629214c527ead743bb3f1b6e4521708cd1"}, + {"name": "0044_572_44572407_qa_4", "source": "0044/572/44572407.ipynb_qa_4", "notebook": "0044/572/44572407.ipynb", "difficulty": "medium", "instruction_sha256": "3cb16706b1688c8d03ad0358705ae3494f53d0ee78aa9e6955510911b25506a6"}, + {"name": "0024_285_24285301_qa_2", "source": "0024/285/24285301.ipynb_qa_2", "notebook": "0024/285/24285301.ipynb", "difficulty": "medium", "instruction_sha256": "983b615d6287bb2f6a03af428367ce87e8e55a752e95687bacd5b2951aecdceb"}, + {"name": "0044_977_44977269_qa_5", "source": "0044/977/44977269.ipynb_qa_5", "notebook": "0044/977/44977269.ipynb", "difficulty": "hard", "instruction_sha256": "7338e7a9a282a9afaf8ba21e668d34325d2915470c61be4b14f9d9ad61c80bf2"}, + {"name": "0055_127_55127360_qa_4", "source": "0055/127/55127360.ipynb_qa_4", "notebook": "0055/127/55127360.ipynb", "difficulty": "hard", "instruction_sha256": "620b1b2f58fed99a2ecd47f26a5352e89f1af8344ce87b4aa190839aad75c3b1"}, + {"name": "0049_677_49677120_qa_1", "source": "0049/677/49677120.ipynb_qa_1", "notebook": "0049/677/49677120.ipynb", "difficulty": "hard", "instruction_sha256": "7ce88c80aa3fe019dfd8d4566cbd65f9356070ff7a0474d55708c8d77ccbdaf2"}, + {"name": "0038_280_38280511_qa_5", "source": "0038/280/38280511.ipynb_qa_5", "notebook": "0038/280/38280511.ipynb", "difficulty": "hard", "instruction_sha256": "3578c7183c08d6acf666d02afa5ea52a256711549cff2f534b486b31681c6df5"}, + {"name": "0015_817_15817041_qa_2", "source": "0015/817/15817041.ipynb_qa_2", "notebook": "0015/817/15817041.ipynb", "difficulty": "medium", "instruction_sha256": "57b6e52fa4e41bc5b4f585191069f93410aa65cbfbe42b7fcb60361cddbf3790"}, + {"name": "0137_213_137213710_qa_1", "source": "0137/213/137213710.ipynb_qa_1", "notebook": "0137/213/137213710.ipynb", "difficulty": "medium", "instruction_sha256": "89e7f3f567f52de155a1ff4e3e05e63c89d562a5d20bf8a58e3f0c82ac0eb4bd"}, + {"name": "0108_778_108778094_qa_5", "source": "0108/778/108778094.ipynb_qa_5", "notebook": "0108/778/108778094.ipynb", "difficulty": "hard", "instruction_sha256": "4eabacd0e1b8202f47a13056abbf0453668fd43b71c721175461071fc484e73e"}, + {"name": "0106_933_106933480_qa_2", "source": "0106/933/106933480.ipynb_qa_2", "notebook": "0106/933/106933480.ipynb", "difficulty": "medium", "instruction_sha256": "0c8355c1e1661efcf1b799b3c5bf35762041e05cab11913115b85239c62b3f74"}, + {"name": "0138_107_138107886_qa_1", "source": "0138/107/138107886.ipynb_qa_1", "notebook": "0138/107/138107886.ipynb", "difficulty": "medium", "instruction_sha256": "bc7465306aeeef74c417729ad6ca289002589acc5008182864b67e9f2434f9e8"}, + {"name": "0041_501_41501839_qa_5", "source": "0041/501/41501839.ipynb_qa_5", "notebook": "0041/501/41501839.ipynb", "difficulty": "medium", "instruction_sha256": "d06f8db807dad25e0e07fa08f11b8e89fe3f58d264efbdd32e91ec10cb73e984"}, + {"name": "0130_119_130119508_qa_4", "source": "0130/119/130119508.ipynb_qa_4", "notebook": "0130/119/130119508.ipynb", "difficulty": "medium", "instruction_sha256": "0536cf52891a86b9e1a6397f7e535a63c946e59c0f4e97db1831b6165a582816"}, + {"name": "0059_844_59844912_qa_5", "source": "0059/844/59844912.ipynb_qa_5", "notebook": "0059/844/59844912.ipynb", "difficulty": "medium", "instruction_sha256": "9d0ce145531dd1d2252396471ed7b85f5afb5b4218f57590703eccc7c8e623bc"}, + {"name": "0069_548_69548302_qa_1", "source": "0069/548/69548302.ipynb_qa_1", "notebook": "0069/548/69548302.ipynb", "difficulty": "hard", "instruction_sha256": "79ced4bd31e48f365eb8dbe9ba259744e6b41dedaea44d48a5d98c6d9edca93d"}, + {"name": "0103_797_103797974_qa_1", "source": "0103/797/103797974.ipynb_qa_1", "notebook": "0103/797/103797974.ipynb", "difficulty": "hard", "instruction_sha256": "f5f3c915f380c8dfdb4ee9b41f3cc3d77eff70c91e17b8e6f8867789f04b8862"}, + {"name": "0121_603_121603419_qa_1", "source": "0121/603/121603419.ipynb_qa_1", "notebook": "0121/603/121603419.ipynb", "difficulty": "hard", "instruction_sha256": "3113a8907218aead1457463f9d8e1636f1d474634cf1949e08abb2644ecaed4e"}, + {"name": "0128_954_128954233_qa_2", "source": "0128/954/128954233.ipynb_qa_2", "notebook": "0128/954/128954233.ipynb", "difficulty": "medium", "instruction_sha256": "9f7208773e81a0a3b91a970afe7c94af36444b3d68ffeba5b1a020cafd728060"}, + {"name": "0016_971_16971389_qa_1", "source": "0016/971/16971389.ipynb_qa_1", "notebook": "0016/971/16971389.ipynb", "difficulty": "medium", "instruction_sha256": "0778ccacc5bdffbbeeebfdfbf8506436a2ef3d360479c53fee02b41a2dc1b34b"}, + {"name": "0132_672_132672509_qa_3", "source": "0132/672/132672509.ipynb_qa_3", "notebook": "0132/672/132672509.ipynb", "difficulty": "hard", "instruction_sha256": "b3ca7c05422b35e1c06dc12bed4a88af5e2961e6c0ac1776f430f54f46457c9a"}, + {"name": "0129_784_129784913_qa_2", "source": "0129/784/129784913.ipynb_qa_2", "notebook": "0129/784/129784913.ipynb", "difficulty": "hard", "instruction_sha256": "b057db45643258f5e4ce8ebda25596525ce8566a5f595d1b4a0a253c9bae1c9c"}, + {"name": "0097_585_97585155_qa_2", "source": "0097/585/97585155.ipynb_qa_2", "notebook": "0097/585/97585155.ipynb", "difficulty": "medium", "instruction_sha256": "054360ae56914d72383f81fbae91c66ecca060478e8bac5b6c2933668d6b5b94"}, + {"name": "0103_040_103040019_qa_4", "source": "0103/040/103040019.ipynb_qa_4", "notebook": "0103/040/103040019.ipynb", "difficulty": "hard", "instruction_sha256": "c3fb354265a62ac94b959f752546b75f831b3cd58dfe52d31225d84ba2fcaee3"}, + {"name": "0106_208_106208028_qa_1", "source": "0106/208/106208028.ipynb_qa_1", "notebook": "0106/208/106208028.ipynb", "difficulty": "hard", "instruction_sha256": "d0633ff92f1a6cae13cdd5478d73d775be46bfcda9f3444d433d9f91f5cf56ba"}, + {"name": "0065_282_65282332_qa_2", "source": "0065/282/65282332.ipynb_qa_2", "notebook": "0065/282/65282332.ipynb", "difficulty": "hard", "instruction_sha256": "9669b5c32838a07e7219f69f1d042e549360087abbac6f6f98808acc57b5b2d7"}, + {"name": "0015_404_15404226_qa_2", "source": "0015/404/15404226.ipynb_qa_2", "notebook": "0015/404/15404226.ipynb", "difficulty": "medium", "instruction_sha256": "2ac2936fb0a0790c7fa5dc89b222b301e616c73bc1a581800e65a125af60c2d0"}, + {"name": "0095_690_95690898_qa_3", "source": "0095/690/95690898.ipynb_qa_3", "notebook": "0095/690/95690898.ipynb", "difficulty": "medium", "instruction_sha256": "920f208961dd77822d8ce13b536e4e283f7912cb1fd3a521dc78963e547b1fd9"}, + {"name": "0082_960_82960295_qa_4", "source": "0082/960/82960295.ipynb_qa_4", "notebook": "0082/960/82960295.ipynb", "difficulty": "hard", "instruction_sha256": "8988e697771c0e3ec294638cd416d810ded37b01755660d56b709e8edd04ce98"}, + {"name": "0129_784_129784913_qa_3", "source": "0129/784/129784913.ipynb_qa_3", "notebook": "0129/784/129784913.ipynb", "difficulty": "hard", "instruction_sha256": "e2f289a88fad26520e3b2d6d1210acfd81a590f00ff53d0fb5cadd560fbe16c5"}, + {"name": "0036_305_36305200_qa_3", "source": "0036/305/36305200.ipynb_qa_3", "notebook": "0036/305/36305200.ipynb", "difficulty": "hard", "instruction_sha256": "18e8b99f258ae0cdab1e946fed220a5cf713088356653614bc12051e5832a851"}, + {"name": "0121_896_121896638_qa_2", "source": "0121/896/121896638.ipynb_qa_2", "notebook": "0121/896/121896638.ipynb", "difficulty": "hard", "instruction_sha256": "a6e943c606f8cfa1ea49d08a10ce0d691b86f7df164964039ce10ebe14d7381d"}, + {"name": "0090_233_90233145_qa_2", "source": "0090/233/90233145.ipynb_qa_2", "notebook": "0090/233/90233145.ipynb", "difficulty": "medium", "instruction_sha256": "60d185b0851dfa004bd67e6810d6fd42247c59b8dd7b22903f8455c23fa3f0ae"}, + {"name": "0088_472_88472757_qa_2", "source": "0088/472/88472757.ipynb_qa_2", "notebook": "0088/472/88472757.ipynb", "difficulty": "hard", "instruction_sha256": "642e1eedb73efacf4222705d6c40d54b8689ffc5dfb7b10338c10944b0cae5a2"}, + {"name": "0019_309_19309067_qa_5", "source": "0019/309/19309067.ipynb_qa_5", "notebook": "0019/309/19309067.ipynb", "difficulty": "medium", "instruction_sha256": "bd3fe4b51c58214960b73298da62d3314cbeab8c3b0c56e7d3f4ebbe0c2945c2"}, + {"name": "0107_679_107679548_qa_5", "source": "0107/679/107679548.ipynb_qa_5", "notebook": "0107/679/107679548.ipynb", "difficulty": "medium", "instruction_sha256": "7f1ce94950bcc8d1e34d4480a6a482a4e3d4bdc4ab63f41cee78fd037ea31141"}, + {"name": "0047_667_47667613_qa_5", "source": "0047/667/47667613.ipynb_qa_5", "notebook": "0047/667/47667613.ipynb", "difficulty": "hard", "instruction_sha256": "fd088b7f74ad43b1dbe68bc28ced3b6a1c4061d9babbf3dfe637d516bd809e58"}, + {"name": "0035_712_35712790_qa_3", "source": "0035/712/35712790.ipynb_qa_3", "notebook": "0035/712/35712790.ipynb", "difficulty": "hard", "instruction_sha256": "bcd033a3b0540dd1f648f36f02be9887c428e97055187594dcff45c652baaf7b"}, + {"name": "0082_385_82385327_qa_5", "source": "0082/385/82385327.ipynb_qa_5", "notebook": "0082/385/82385327.ipynb", "difficulty": "medium", "instruction_sha256": "b996e9731903787ad9a1ce3e05c24a4a5c6a2021cce2cef2f7859dbf16015c6e"}, + {"name": "0023_580_23580177_qa_2", "source": "0023/580/23580177.ipynb_qa_2", "notebook": "0023/580/23580177.ipynb", "difficulty": "hard", "instruction_sha256": "32b3c69a0718ed7b4d2f5990855a67125e3f7ecad4110f354bf41f8157d4062d"}, + {"name": "0040_528_40528853_qa_4", "source": "0040/528/40528853.ipynb_qa_4", "notebook": "0040/528/40528853.ipynb", "difficulty": "hard", "instruction_sha256": "c7a3e5afa537f7ee20e83cc388d548fc9cc5ad8242050a28d96c836fb67540e8"}, + {"name": "0102_158_102158700_qa_5", "source": "0102/158/102158700.ipynb_qa_5", "notebook": "0102/158/102158700.ipynb", "difficulty": "medium", "instruction_sha256": "d763ac69c9e2afcf88fba9e99d398f14c471f4721f97650e9673b9b454654e01"}, + {"name": "0120_574_120574465_qa_1", "source": "0120/574/120574465.ipynb_qa_1", "notebook": "0120/574/120574465.ipynb", "difficulty": "hard", "instruction_sha256": "4dfee78ee9a85f6d5da99cd7f88613a6949bb7702bdef40f5f21b2ed9b6a932d"}, + {"name": "0075_402_75402843_qa_3", "source": "0075/402/75402843.ipynb_qa_3", "notebook": "0075/402/75402843.ipynb", "difficulty": "medium", "instruction_sha256": "9528bc44fef9d97b241a8b3d6117f918e242350a0ec61f34557d90b173754597"}, + {"name": "0120_102_120102768_qa_1", "source": "0120/102/120102768.ipynb_qa_1", "notebook": "0120/102/120102768.ipynb", "difficulty": "hard", "instruction_sha256": "f108e7a21625fe27b48e1e80c7586eb0e34738eb481348f2f0819673cf11a1d5"}, + {"name": "0035_228_35228737_qa_5", "source": "0035/228/35228737.ipynb_qa_5", "notebook": "0035/228/35228737.ipynb", "difficulty": "hard", "instruction_sha256": "5712569a483b7a1d3b811ee15807e7db00d2c88f613a63c2155ce719128962d3"}, + {"name": "0061_330_61330810_qa_1", "source": "0061/330/61330810.ipynb_qa_1", "notebook": "0061/330/61330810.ipynb", "difficulty": "hard", "instruction_sha256": "e2c4aa616a41427e65c8ed453996021e2b1cd929f9f77a1ade974a88604fe873"}, + {"name": "0048_966_48966571_qa_3", "source": "0048/966/48966571.ipynb_qa_3", "notebook": "0048/966/48966571.ipynb", "difficulty": "hard", "instruction_sha256": "0400e1d20d92fe1888f28de8b5bc1f6c8cf0494bc6a90241ebd2220adb48337c"}, + {"name": "0033_760_33760253_qa_4", "source": "0033/760/33760253.ipynb_qa_4", "notebook": "0033/760/33760253.ipynb", "difficulty": "medium", "instruction_sha256": "1abad5bd40394fdcfa597dd64a62ff3bf095d7f36c5b0f40393ee0db86f0f574"}, + {"name": "0094_146_94146987_qa_5", "source": "0094/146/94146987.ipynb_qa_5", "notebook": "0094/146/94146987.ipynb", "difficulty": "medium", "instruction_sha256": "9369d1253a47f82d1cf789835b8a386be812d5252f0e1dd73f119c0e3f90cc55"}, + {"name": "0114_582_114582781_qa_1", "source": "0114/582/114582781.ipynb_qa_1", "notebook": "0114/582/114582781.ipynb", "difficulty": "hard", "instruction_sha256": "6c2c6d7ff73652cbce6b0608bd932e2497b4c081dca272b48b34409fe6f78c95"}, + {"name": "0114_928_114928761_qa_2", "source": "0114/928/114928761.ipynb_qa_2", "notebook": "0114/928/114928761.ipynb", "difficulty": "medium", "instruction_sha256": "0d926b5c76855155c36c37b58def2bc85d0c9179dd23a31ffea345abd74c24f2"}, + {"name": "0098_375_98375855_qa_2", "source": "0098/375/98375855.ipynb_qa_2", "notebook": "0098/375/98375855.ipynb", "difficulty": "hard", "instruction_sha256": "e92d1ef685d481772545ee9ceb8400871fa39aa250565b2b6bc29ae80d73671f"}, + {"name": "0082_994_82994330_qa_5", "source": "0082/994/82994330.ipynb_qa_5", "notebook": "0082/994/82994330.ipynb", "difficulty": "medium", "instruction_sha256": "9c8589a342ea4d08f7bb544ae947ca39964e0b38a37d7d094b778596981e188c"}, + {"name": "0031_906_31906980_qa_1", "source": "0031/906/31906980.ipynb_qa_1", "notebook": "0031/906/31906980.ipynb", "difficulty": "hard", "instruction_sha256": "1325b9d9d24ce22ab69bfb0328ce0f7b10a4b95b5c4ee4c06a1ca1ec406bb7d1"}, + {"name": "0129_208_129208819_qa_2", "source": "0129/208/129208819.ipynb_qa_2", "notebook": "0129/208/129208819.ipynb", "difficulty": "medium", "instruction_sha256": "c8676de995efa2133fb28c64db40ccde0b2f2a109d5aa66ad69068ec71d62f9a"}, + {"name": "0091_442_91442113_qa_2", "source": "0091/442/91442113.ipynb_qa_2", "notebook": "0091/442/91442113.ipynb", "difficulty": "hard", "instruction_sha256": "8c1b5fa72206c0ef1a97ff760542b8a54a53e2761b15ee0d3edd7ba0ab74974f"}, + {"name": "0057_925_57925686_qa_2", "source": "0057/925/57925686.ipynb_qa_2", "notebook": "0057/925/57925686.ipynb", "difficulty": "hard", "instruction_sha256": "40040024155d0e7c8dac562101e9fc717bc3ccd59e38435ee62c079551526f92"}, + {"name": "0041_258_41258552_qa_1", "source": "0041/258/41258552.ipynb_qa_1", "notebook": "0041/258/41258552.ipynb", "difficulty": "medium", "instruction_sha256": "f622db55ce57a8c5fece39f865b8d84150179bb8eed0241562ab966ed20fdd09"}, + {"name": "0096_904_96904544_qa_4", "source": "0096/904/96904544.ipynb_qa_4", "notebook": "0096/904/96904544.ipynb", "difficulty": "hard", "instruction_sha256": "545700c023704e47135e2b0ec212517bf52e02d5766aaaad20d2895e4f662eb1"}, + {"name": "0010_376_10376491_qa_3", "source": "0010/376/10376491.ipynb_qa_3", "notebook": "0010/376/10376491.ipynb", "difficulty": "medium", "instruction_sha256": "9e9a9daf37a41ac463acb321de44df84997f99bf2bceb37d53cec5ad3f80ee7d"}, + {"name": "0072_866_72866029_qa_5", "source": "0072/866/72866029.ipynb_qa_5", "notebook": "0072/866/72866029.ipynb", "difficulty": "hard", "instruction_sha256": "fce94ca2b74ac4b511a69eeb8fcac7e59a2c832ef1c73de7b2e3d9cfe7b51a10"}, + {"name": "0045_605_45605559_qa_4", "source": "0045/605/45605559.ipynb_qa_4", "notebook": "0045/605/45605559.ipynb", "difficulty": "hard", "instruction_sha256": "46df82fd93c952f327970c5919d51df9d2c330f3605a26c85dfe84bf1ec554e5"}, + {"name": "0137_338_137338171_qa_1", "source": "0137/338/137338171.ipynb_qa_1", "notebook": "0137/338/137338171.ipynb", "difficulty": "medium", "instruction_sha256": "cec2dfcea3f6d5c5ae1e449dd33fcef5271674c7b69a1ead5e838231911a4f36"}, + {"name": "0064_766_64766227_qa_2", "source": "0064/766/64766227.ipynb_qa_2", "notebook": "0064/766/64766227.ipynb", "difficulty": "medium", "instruction_sha256": "badfb0450f71032edc20079ea3834d1dd9738685d5bce9780edc09124cb7943d"}, + {"name": "0062_015_62015834_qa_1", "source": "0062/015/62015834.ipynb_qa_1", "notebook": "0062/015/62015834.ipynb", "difficulty": "hard", "instruction_sha256": "e1b479b0828486357b3f449a8ff7333a34e3ed3a11d0476628b056b95e31c97e"}, + {"name": "0041_560_41560306_qa_4", "source": "0041/560/41560306.ipynb_qa_4", "notebook": "0041/560/41560306.ipynb", "difficulty": "medium", "instruction_sha256": "2848801d0cc9f9c0b48ce3903dc869d249043eb723fc1ad64da1caf70f7bc782"}, + {"name": "0107_391_107391967_qa_2", "source": "0107/391/107391967.ipynb_qa_2", "notebook": "0107/391/107391967.ipynb", "difficulty": "hard", "instruction_sha256": "8d44dbc08177f4d8a1bfc63bb8526dc57b733a62699b927460e53f6bb5d51c4d"}, + {"name": "0080_304_80304821_qa_1", "source": "0080/304/80304821.ipynb_qa_1", "notebook": "0080/304/80304821.ipynb", "difficulty": "medium", "instruction_sha256": "3d76e496c9b19fdc00e5c7bc946eb34e7ba56f7b463bd6f7e552393f91bc4f1a"}, + {"name": "0038_705_38705868_qa_4", "source": "0038/705/38705868.ipynb_qa_4", "notebook": "0038/705/38705868.ipynb", "difficulty": "hard", "instruction_sha256": "f0b10366bb1942582d3dc929566c48bef2242035873b415ad1e68700d5d4d532"}, + {"name": "0060_511_60511698_qa_4", "source": "0060/511/60511698.ipynb_qa_4", "notebook": "0060/511/60511698.ipynb", "difficulty": "hard", "instruction_sha256": "95f32917b81eea5485b10c0e4fdfb0dd9264c28ed22cdaee91b04938d9903a89"}, + {"name": "0037_168_37168956_qa_5", "source": "0037/168/37168956.ipynb_qa_5", "notebook": "0037/168/37168956.ipynb", "difficulty": "hard", "instruction_sha256": "e7e86442541ea3c57c087616407aefe7a26d3e7ab08ef4339ef73743965e0f11"}, + {"name": "0038_439_38439618_qa_2", "source": "0038/439/38439618.ipynb_qa_2", "notebook": "0038/439/38439618.ipynb", "difficulty": "hard", "instruction_sha256": "73accc109d8bdc6a259b3e010c22d0fba79de5f89cd9fa815276f9bb6a1db927"}, + {"name": "0067_633_67633156_qa_3", "source": "0067/633/67633156.ipynb_qa_3", "notebook": "0067/633/67633156.ipynb", "difficulty": "hard", "instruction_sha256": "776508d94ac20831b48875e4423eab658cfd91eae93b0145f00ff87b47b6d720"}, + {"name": "0041_216_41216352_qa_1", "source": "0041/216/41216352.ipynb_qa_1", "notebook": "0041/216/41216352.ipynb", "difficulty": "medium", "instruction_sha256": "0969df368f483a2488a87e043f7f8f37e860cc4fa2c1e5d83a3fa144eb897dab"}, + {"name": "0060_106_60106319_qa_1", "source": "0060/106/60106319.ipynb_qa_1", "notebook": "0060/106/60106319.ipynb", "difficulty": "medium", "instruction_sha256": "6cbc3ea05e0c3012a09bef6cc9f6fce9b2f2807a1388bae5503a99eaca4416d1"}, + {"name": "0120_295_120295119_qa_2", "source": "0120/295/120295119.ipynb_qa_2", "notebook": "0120/295/120295119.ipynb", "difficulty": "hard", "instruction_sha256": "90a861910efd599f12d065c270fdf748d9e104923ce964b928b4867f30ec610c"}, + {"name": "0018_550_18550354_qa_4", "source": "0018/550/18550354.ipynb_qa_4", "notebook": "0018/550/18550354.ipynb", "difficulty": "medium", "instruction_sha256": "c898b8ca0737c009a5ca54c302e421944be0b2cdf62b84f8f33a18ffe9c5016b"}, + {"name": "0038_681_38681339_qa_3", "source": "0038/681/38681339.ipynb_qa_3", "notebook": "0038/681/38681339.ipynb", "difficulty": "hard", "instruction_sha256": "8c4d26c0fdc060f53cee1e40381705da9426a117d62bafe169687c89694612ef"}, + {"name": "0108_808_108808607_qa_2", "source": "0108/808/108808607.ipynb_qa_2", "notebook": "0108/808/108808607.ipynb", "difficulty": "hard", "instruction_sha256": "7ffd8f3b3d3c5eab93265505539110b13a51b7cf2313cae15ae7354eafaf9597"}, + {"name": "0017_914_17914820_qa_1", "source": "0017/914/17914820.ipynb_qa_1", "notebook": "0017/914/17914820.ipynb", "difficulty": "medium", "instruction_sha256": "ddd2c9a1bc8893d4dc06249c4fea6b52f20fd070fd41c31345f5cc7b24509fee"}, + {"name": "0093_755_93755212_qa_1", "source": "0093/755/93755212.ipynb_qa_1", "notebook": "0093/755/93755212.ipynb", "difficulty": "hard", "instruction_sha256": "995ff4a034de0c0a59359f2ed24831925f14e8f32e93a0bc934d2ed6f084056c"}, + {"name": "0087_320_87320991_qa_1", "source": "0087/320/87320991.ipynb_qa_1", "notebook": "0087/320/87320991.ipynb", "difficulty": "hard", "instruction_sha256": "8ebca6caf2803ab33a15fa8a8799cd8fdd2d00e6009caf34f56247b6d14f902d"}, + {"name": "0076_410_76410509_qa_2", "source": "0076/410/76410509.ipynb_qa_2", "notebook": "0076/410/76410509.ipynb", "difficulty": "hard", "instruction_sha256": "f4f63b4e3f2b70d232b166942cc3dc2d0f6ae34283677f06c3584c8399703670"}, + {"name": "0043_694_43694920_qa_5", "source": "0043/694/43694920.ipynb_qa_5", "notebook": "0043/694/43694920.ipynb", "difficulty": "hard", "instruction_sha256": "6f04a613a7aaf140f98e0363614f49662434e474f5c028bdde0fa891b48aa578"}, + {"name": "0063_631_63631791_qa_5", "source": "0063/631/63631791.ipynb_qa_5", "notebook": "0063/631/63631791.ipynb", "difficulty": "hard", "instruction_sha256": "64a0c5f1ad801740f6c9d161fd257882eedeeb80a6a68ac6aa5ae9cc02a77911"}, + {"name": "0070_990_70990012_qa_1", "source": "0070/990/70990012.ipynb_qa_1", "notebook": "0070/990/70990012.ipynb", "difficulty": "hard", "instruction_sha256": "f6306679c1311195f9f99ce97cc86bb44f3e3a5e3a3d838a32a40f663e008a32"}, + {"name": "0089_589_89589668_qa_1", "source": "0089/589/89589668.ipynb_qa_1", "notebook": "0089/589/89589668.ipynb", "difficulty": "hard", "instruction_sha256": "e6f4a234d13e12edea9ac3328552e22b443d465da084c11ec3ab6081273cb562"}, + {"name": "0061_109_61109875_qa_1", "source": "0061/109/61109875.ipynb_qa_1", "notebook": "0061/109/61109875.ipynb", "difficulty": "medium", "instruction_sha256": "97be8a46323c71cf6b46d4e634d4d5fd10feecb1f32acdc142c4c152f6fd0b51"}, + {"name": "0068_964_68964852_qa_2", "source": "0068/964/68964852.ipynb_qa_2", "notebook": "0068/964/68964852.ipynb", "difficulty": "hard", "instruction_sha256": "9712c1bd968ccdc29244529be5e48fc074ce6ee8cfe93e71c82101565382d020"}, + {"name": "0039_489_39489585_qa_2", "source": "0039/489/39489585.ipynb_qa_2", "notebook": "0039/489/39489585.ipynb", "difficulty": "medium", "instruction_sha256": "44f4003408017cf85fc19a23c97d1b25862770e14025a32f24a7614495edff2b"}, + {"name": "0048_093_48093392_qa_5", "source": "0048/093/48093392.ipynb_qa_5", "notebook": "0048/093/48093392.ipynb", "difficulty": "hard", "instruction_sha256": "fe754ac2aa5cacd36338cac61fab0b16898118a981c4890820b8f77625b7d4be"}, + {"name": "0104_138_104138961_qa_2", "source": "0104/138/104138961.ipynb_qa_2", "notebook": "0104/138/104138961.ipynb", "difficulty": "medium", "instruction_sha256": "415f733fae9756c817526c0040d00203024fad8277122033a98f0154d716e8b2"}, + {"name": "0037_718_37718370_qa_2", "source": "0037/718/37718370.ipynb_qa_2", "notebook": "0037/718/37718370.ipynb", "difficulty": "medium", "instruction_sha256": "406af30b3d9e60197cd94b44869ce586636b360a699b379d47724009b4d03b52"}, + {"name": "0127_323_127323479_qa_1", "source": "0127/323/127323479.ipynb_qa_1", "notebook": "0127/323/127323479.ipynb", "difficulty": "medium", "instruction_sha256": "745781f7b04e0cbef3173972197a4f5eaf81aad5eeb90d948452de51f43e86e6"}, + {"name": "0032_314_32314190_qa_4", "source": "0032/314/32314190.ipynb_qa_4", "notebook": "0032/314/32314190.ipynb", "difficulty": "medium", "instruction_sha256": "de4fdedb736ca6b8f8d0d770241df6408046a6d965d664d34c7136a2d3518607"}, + {"name": "0125_045_125045351_qa_1", "source": "0125/045/125045351.ipynb_qa_1", "notebook": "0125/045/125045351.ipynb", "difficulty": "hard", "instruction_sha256": "cd3fb40ef15f5caa78eba15c0b5696748fe714c02acc1ca6a409348e17e48362"}, + {"name": "0044_873_44873237_qa_3", "source": "0044/873/44873237.ipynb_qa_3", "notebook": "0044/873/44873237.ipynb", "difficulty": "hard", "instruction_sha256": "e551387894a9ec034bc4e3244dfe1eaf4f2b9fa44cc6beeeb914097d126d3637"}, + {"name": "0002_170_2170781_qa_1", "source": "0002/170/2170781.ipynb_qa_1", "notebook": "0002/170/2170781.ipynb", "difficulty": "hard", "instruction_sha256": "0c34d4bf217dc5a8fe58197dc34f9fb1a4a1e0a48534851dd50d43240f87c289"}, + {"name": "0001_239_1239559_qa_4", "source": "0001/239/1239559.ipynb_qa_4", "notebook": "0001/239/1239559.ipynb", "difficulty": "medium", "instruction_sha256": "0ef68c3c324b5d549727cd16aa969d1e2e0ffafe856257451952521d465adc89"}, + {"name": "0088_614_88614765_qa_5", "source": "0088/614/88614765.ipynb_qa_5", "notebook": "0088/614/88614765.ipynb", "difficulty": "hard", "instruction_sha256": "79b7b366245667067b875698529483ee8b0e5020e425e255bdbc1a3ec97eeeb0"}, + {"name": "0118_722_118722407_qa_2", "source": "0118/722/118722407.ipynb_qa_2", "notebook": "0118/722/118722407.ipynb", "difficulty": "hard", "instruction_sha256": "eb7283c51897aca75067fa84bb366e103392ce5caababe3c5ae36835fefb28d5"}, + {"name": "0116_195_116195791_qa_2", "source": "0116/195/116195791.ipynb_qa_2", "notebook": "0116/195/116195791.ipynb", "difficulty": "hard", "instruction_sha256": "7a67b218183f1329230701e4f31f57a3977dfee4969a5b96b31a75cb5f66afd7"}, + {"name": "0044_155_44155408_qa_2", "source": "0044/155/44155408.ipynb_qa_2", "notebook": "0044/155/44155408.ipynb", "difficulty": "medium", "instruction_sha256": "2b0214854f159dffae40376a4799a7caef96910d3a876b16007ac42c9ada0fe7"}, + {"name": "0061_283_61283870_qa_3", "source": "0061/283/61283870.ipynb_qa_3", "notebook": "0061/283/61283870.ipynb", "difficulty": "hard", "instruction_sha256": "e4b960e67767ce1afa46de79efd95f27a7906fa439d8c97503ac77bb6b63b66e"}, + {"name": "0063_879_63879344_qa_3", "source": "0063/879/63879344.ipynb_qa_3", "notebook": "0063/879/63879344.ipynb", "difficulty": "hard", "instruction_sha256": "1f646416db4573587a0c2efde2738a0a6e4c1a1f8fdcc3d8f39202397241b244"}, + {"name": "0048_371_48371332_qa_5", "source": "0048/371/48371332.ipynb_qa_5", "notebook": "0048/371/48371332.ipynb", "difficulty": "hard", "instruction_sha256": "5b284ca008e5d87a3002de1485883e16cb1665e25fd43de1ed516ecef1317db2"}, + {"name": "0046_660_46660345_qa_2", "source": "0046/660/46660345.ipynb_qa_2", "notebook": "0046/660/46660345.ipynb", "difficulty": "hard", "instruction_sha256": "4ecf8911f4c49bb5da430236ebdb2cf270fb2a7092ceb4a9462989c0c4327b83"}, + {"name": "0032_101_32101676_qa_1", "source": "0032/101/32101676.ipynb_qa_1", "notebook": "0032/101/32101676.ipynb", "difficulty": "medium", "instruction_sha256": "82b0cb489ae103d107941d96d3962e06f71793b261b0d23263ad711f60e0a9d9"}, + {"name": "0031_566_31566788_qa_1", "source": "0031/566/31566788.ipynb_qa_1", "notebook": "0031/566/31566788.ipynb", "difficulty": "medium", "instruction_sha256": "0bfcbfc74758079eaa3fb078517ecf8c1e2ab11838dc0180d61b3703bf76572f"}, + {"name": "0051_975_51975834_qa_4", "source": "0051/975/51975834.ipynb_qa_4", "notebook": "0051/975/51975834.ipynb", "difficulty": "hard", "instruction_sha256": "ad80a3282937603b681ff73cc69780d2accd920460310388cf5158cd96edea98"}, + {"name": "0065_069_65069276_qa_4", "source": "0065/069/65069276.ipynb_qa_4", "notebook": "0065/069/65069276.ipynb", "difficulty": "hard", "instruction_sha256": "6c8f5f2772601c7de5604ae145c5aea9c47731fecfc32f5b0d224bde626fce1b"}, + {"name": "0064_762_64762975_qa_2", "source": "0064/762/64762975.ipynb_qa_2", "notebook": "0064/762/64762975.ipynb", "difficulty": "hard", "instruction_sha256": "2b97d5c2c8d800fbdc82726695e4de994ad1ec1325e5d543b81d548c1a825c4b"}, + {"name": "0065_850_65850570_qa_4", "source": "0065/850/65850570.ipynb_qa_4", "notebook": "0065/850/65850570.ipynb", "difficulty": "medium", "instruction_sha256": "6f567e2c9d4df0e65870b9dc8e5ffb087378f2013d1ef16bdb4cade53d882387"}, + {"name": "0128_750_128750159_qa_3", "source": "0128/750/128750159.ipynb_qa_3", "notebook": "0128/750/128750159.ipynb", "difficulty": "hard", "instruction_sha256": "6a3bb38df8f5ff4666139975455511cbe11db1f5bec40cf0a50ebd763b811220"}, + {"name": "0103_567_103567040_qa_1", "source": "0103/567/103567040.ipynb_qa_1", "notebook": "0103/567/103567040.ipynb", "difficulty": "hard", "instruction_sha256": "a56558819634ef2956d5651766f4800d1d97529349b38b8d4c834d9887367cad"}, + {"name": "0046_267_46267192_qa_3", "source": "0046/267/46267192.ipynb_qa_3", "notebook": "0046/267/46267192.ipynb", "difficulty": "medium", "instruction_sha256": "c90064063c56a529450e466838821fff5dd940bdde9d4e52094b0ac8415894c7"}, + {"name": "0085_343_85343577_qa_5", "source": "0085/343/85343577.ipynb_qa_5", "notebook": "0085/343/85343577.ipynb", "difficulty": "hard", "instruction_sha256": "b80cf4974da951808ea93c889c8b936340b1207c7837ab929c93eff75fa04713"}, + {"name": "0021_687_21687757_qa_1", "source": "0021/687/21687757.ipynb_qa_1", "notebook": "0021/687/21687757.ipynb", "difficulty": "medium", "instruction_sha256": "c8bcac1da1ea9b02ff711fc1018a4ebd10f496e6dd562803d033679ca4ad5f03"}, + {"name": "0096_064_96064872_qa_1", "source": "0096/064/96064872.ipynb_qa_1", "notebook": "0096/064/96064872.ipynb", "difficulty": "hard", "instruction_sha256": "191511606752d30507e7f6534ef7276ea5d0886bd5d15acba21cf483f15ac5e9"}, + {"name": "0040_008_40008810_qa_1", "source": "0040/008/40008810.ipynb_qa_1", "notebook": "0040/008/40008810.ipynb", "difficulty": "hard", "instruction_sha256": "92bbd9bc642aba780fad5269c47303eb1d5d8d7ba799268052ec97f632f2e1d3"}, + {"name": "0026_293_26293551_qa_3", "source": "0026/293/26293551.ipynb_qa_3", "notebook": "0026/293/26293551.ipynb", "difficulty": "medium", "instruction_sha256": "62c16b0e03f4236e9380888d3960ce1060bd17ccec8b1d8469b1e2ff46add2fd"}, + {"name": "0086_170_86170046_qa_3", "source": "0086/170/86170046.ipynb_qa_3", "notebook": "0086/170/86170046.ipynb", "difficulty": "hard", "instruction_sha256": "351c6189af309135611211332620070ac759ec0a416388fb276206a84fcbc1b5"}, + {"name": "0043_528_43528813_qa_2", "source": "0043/528/43528813.ipynb_qa_2", "notebook": "0043/528/43528813.ipynb", "difficulty": "hard", "instruction_sha256": "24bd8bfe2c1d2ee755decce2cffc48082b1f83939647da0900bc60e0598c62b6"}, + {"name": "0073_796_73796901_qa_2", "source": "0073/796/73796901.ipynb_qa_2", "notebook": "0073/796/73796901.ipynb", "difficulty": "hard", "instruction_sha256": "f0a8473418927e479d0b4ed80a0ea7bfb2a97ae0075eb1d3dabbd200babc9e87"}, + {"name": "0086_358_86358959_qa_2", "source": "0086/358/86358959.ipynb_qa_2", "notebook": "0086/358/86358959.ipynb", "difficulty": "medium", "instruction_sha256": "f716d46b1e8c7b886416d5ab5eae05e123dcf4960100964ee82781e0d63e8eff"}, + {"name": "0001_247_1247152_qa_1", "source": "0001/247/1247152.ipynb_qa_1", "notebook": "0001/247/1247152.ipynb", "difficulty": "medium", "instruction_sha256": "70d74a945db2b98c269d3469562d6c74b9c2488684c4be50ef9b78bcccfd16bd"}, + {"name": "0038_509_38509914_qa_4", "source": "0038/509/38509914.ipynb_qa_4", "notebook": "0038/509/38509914.ipynb", "difficulty": "hard", "instruction_sha256": "7b8882193c75be190ee7e97a9cc71eba8191fd18cac6064825754ba7f2cdaadc"}, + {"name": "0110_717_110717969_qa_1", "source": "0110/717/110717969.ipynb_qa_1", "notebook": "0110/717/110717969.ipynb", "difficulty": "hard", "instruction_sha256": "7fa64e36f67be33102e6fbd763ebc8943f60ad6712f817deade152d63e3c3793"}, + {"name": "0092_004_92004246_qa_1", "source": "0092/004/92004246.ipynb_qa_1", "notebook": "0092/004/92004246.ipynb", "difficulty": "hard", "instruction_sha256": "228a6faaa46d80183c24ef4d804017d898b720edd5afb2892a32d83bd535ceb0"}, + {"name": "0022_169_22169428_qa_5", "source": "0022/169/22169428.ipynb_qa_5", "notebook": "0022/169/22169428.ipynb", "difficulty": "hard", "instruction_sha256": "f078b702c817accbad0f92e43d52b31559cef0349c4f6a729a4618cbe850b308"}, + {"name": "0001_683_1683832_qa_2", "source": "0001/683/1683832.ipynb_qa_2", "notebook": "0001/683/1683832.ipynb", "difficulty": "medium", "instruction_sha256": "d824f4a2b3f32897d1ff2d8c5d0cff3bef3f8ee1d64f304618c1045af8ad9d4d"}, + {"name": "0107_427_107427822_qa_5", "source": "0107/427/107427822.ipynb_qa_5", "notebook": "0107/427/107427822.ipynb", "difficulty": "medium", "instruction_sha256": "1e893b079852cb57810c7f3007852d23182a4f318bdc6b7688ffa6a407d0495f"}, + {"name": "0051_543_51543222_qa_5", "source": "0051/543/51543222.ipynb_qa_5", "notebook": "0051/543/51543222.ipynb", "difficulty": "hard", "instruction_sha256": "35c60efaa449d81a4bd13b8604e44ace5540cc50b1494363108687a8f934a438"}, + {"name": "0062_214_62214042_qa_5", "source": "0062/214/62214042.ipynb_qa_5", "notebook": "0062/214/62214042.ipynb", "difficulty": "medium", "instruction_sha256": "538716968fdf493dde10946f07112d0bb856761ae9e1b7828a666567fbaed13e"}, + {"name": "0042_488_42488426_qa_1", "source": "0042/488/42488426.ipynb_qa_1", "notebook": "0042/488/42488426.ipynb", "difficulty": "medium", "instruction_sha256": "c946ce0814edb7bbe6c12abb43edd03c4da3d3ee82c347af2a3a0a2f8372b05c"}, + {"name": "0135_735_135735954_qa_4", "source": "0135/735/135735954.ipynb_qa_4", "notebook": "0135/735/135735954.ipynb", "difficulty": "hard", "instruction_sha256": "170af06528baf8afd9e1b38c8101d6206bc3de038505681e28dd356a84ce3f54"}, + {"name": "0119_557_119557726_qa_5", "source": "0119/557/119557726.ipynb_qa_5", "notebook": "0119/557/119557726.ipynb", "difficulty": "hard", "instruction_sha256": "eb524c75702fd37c811b4c2881f5d0beb4a41650e9b1ed706b32f2ab34727e36"}, + {"name": "0113_768_113768809_qa_2", "source": "0113/768/113768809.ipynb_qa_2", "notebook": "0113/768/113768809.ipynb", "difficulty": "hard", "instruction_sha256": "d352f131b9090dcacf611bc68072c5828d48aa5e57247c910c60df814de3917f"}, + {"name": "0031_523_31523942_qa_3", "source": "0031/523/31523942.ipynb_qa_3", "notebook": "0031/523/31523942.ipynb", "difficulty": "medium", "instruction_sha256": "25508a0417cd0847c9bc71a97680f659c95ba0965b75b231f852ca7db9f326cf"}, + {"name": "0103_336_103336687_qa_4", "source": "0103/336/103336687.ipynb_qa_4", "notebook": "0103/336/103336687.ipynb", "difficulty": "hard", "instruction_sha256": "9ce0edc512fd09ecba321ceb0e3811ab75cee5ce48105070758079b0a05104bc"}, + {"name": "0012_556_12556896_qa_1", "source": "0012/556/12556896.ipynb_qa_1", "notebook": "0012/556/12556896.ipynb", "difficulty": "medium", "instruction_sha256": "9f8e99fbbfc98940b61a08c96489b031eb9cfa24ecc871ea262b3a3e4bffe3b6"}, + {"name": "0119_926_119926274_qa_1", "source": "0119/926/119926274.ipynb_qa_1", "notebook": "0119/926/119926274.ipynb", "difficulty": "hard", "instruction_sha256": "346ae26e82d53849f5e45bdc4979e0a4e8a749e9e06b16f9880e7fd1ddcbef0e"}, + {"name": "0113_504_113504478_qa_3", "source": "0113/504/113504478.ipynb_qa_3", "notebook": "0113/504/113504478.ipynb", "difficulty": "medium", "instruction_sha256": "676e18d8c140d83dd925e955ce38837ddfb31e6d1a2f8e93dc4d21ee2f5341ab"}, + {"name": "0052_654_52654015_qa_1", "source": "0052/654/52654015.ipynb_qa_1", "notebook": "0052/654/52654015.ipynb", "difficulty": "hard", "instruction_sha256": "f4061814e5b1f9e30e2e63f783ee86539286297af23d6359e4e022c328d60557"}, + {"name": "0103_567_103567040_qa_3", "source": "0103/567/103567040.ipynb_qa_3", "notebook": "0103/567/103567040.ipynb", "difficulty": "medium", "instruction_sha256": "7b75f59cabb86f7a21730b17c652b556ac89a34d60dd86441b587708241ad516"}, + {"name": "0087_180_87180327_qa_4", "source": "0087/180/87180327.ipynb_qa_4", "notebook": "0087/180/87180327.ipynb", "difficulty": "hard", "instruction_sha256": "b796312ee306fa0668f4e1330caabc5ec174ae3f30bbaf1c3a9542c0f720580e"}, + {"name": "0035_558_35558843_qa_1", "source": "0035/558/35558843.ipynb_qa_1", "notebook": "0035/558/35558843.ipynb", "difficulty": "medium", "instruction_sha256": "ba37eabc4a6ed1e5897e63d67ca27e19bc0f79a93587beaed0f67e0d90147a87"}, + {"name": "0106_400_106400337_qa_4", "source": "0106/400/106400337.ipynb_qa_4", "notebook": "0106/400/106400337.ipynb", "difficulty": "medium", "instruction_sha256": "13f9802b34ab5601770753191532c3fba5851252b753e6f2af37150b0f4b2245"}, + {"name": "0075_579_75579600_qa_3", "source": "0075/579/75579600.ipynb_qa_3", "notebook": "0075/579/75579600.ipynb", "difficulty": "medium", "instruction_sha256": "0488b591d4a6b63cea73b8acb57e901bfea9fc8296ec2cddbaef8ac6003f96a4"}, + {"name": "0043_338_43338909_qa_1", "source": "0043/338/43338909.ipynb_qa_1", "notebook": "0043/338/43338909.ipynb", "difficulty": "hard", "instruction_sha256": "4dc33b9e9c7e7b188afc7c9336d7dcf71e50376f138dddf59cb1a8f0fded5103"}, + {"name": "0047_417_47417997_qa_1", "source": "0047/417/47417997.ipynb_qa_1", "notebook": "0047/417/47417997.ipynb", "difficulty": "hard", "instruction_sha256": "3bcda977beb1c7a536d8eae2d12ec1e011f437c0b4c1da3b1ecb328703b50531"}, + {"name": "0038_189_38189340_qa_1", "source": "0038/189/38189340.ipynb_qa_1", "notebook": "0038/189/38189340.ipynb", "difficulty": "hard", "instruction_sha256": "bb9a142c5a112f82458972309bdfeaaf7bbbda0679b9e49e1f240ac645b6f248"}, + {"name": "0054_928_54928271_qa_2", "source": "0054/928/54928271.ipynb_qa_2", "notebook": "0054/928/54928271.ipynb", "difficulty": "hard", "instruction_sha256": "9dbf82b57623031ab46aeec0c905c00058f897bfbf87bccc102719be11045fcd"}, + {"name": "0044_352_44352035_qa_5", "source": "0044/352/44352035.ipynb_qa_5", "notebook": "0044/352/44352035.ipynb", "difficulty": "medium", "instruction_sha256": "38d6f804e1eae16097e4adcd9850182adc26717d326e52a00a85b6e2e867d0d8"}, + {"name": "0083_648_83648661_qa_3", "source": "0083/648/83648661.ipynb_qa_3", "notebook": "0083/648/83648661.ipynb", "difficulty": "hard", "instruction_sha256": "3bf539b7251b70a6e41cce9780d877047223094a77eebb11e1a149b6becc3383"}, + {"name": "0040_627_40627093_qa_4", "source": "0040/627/40627093.ipynb_qa_4", "notebook": "0040/627/40627093.ipynb", "difficulty": "hard", "instruction_sha256": "79d74681e735cfc057e00a647b587d318d2cba8911891a85a6400c767e5bc1e9"}, + {"name": "0120_304_120304069_qa_1", "source": "0120/304/120304069.ipynb_qa_1", "notebook": "0120/304/120304069.ipynb", "difficulty": "medium", "instruction_sha256": "5cbf41862a5b65fc50fb498c7c12bf24a34e4d5550f4d1b8cbe1be603aaa9fe6"}, + {"name": "0028_183_28183894_qa_1", "source": "0028/183/28183894.ipynb_qa_1", "notebook": "0028/183/28183894.ipynb", "difficulty": "medium", "instruction_sha256": "fbfb6391f8791093d4735985ca0670cfdabe27ace5733e9d2a0257cb815a3a2c"}, + {"name": "0085_618_85618243_qa_2", "source": "0085/618/85618243.ipynb_qa_2", "notebook": "0085/618/85618243.ipynb", "difficulty": "medium", "instruction_sha256": "b8992e1be8ef4728fe2e4ff48b2b74ed1e8435d1d597ca4470377dd1f21fb5fb"}, + {"name": "0054_137_54137484_qa_1", "source": "0054/137/54137484.ipynb_qa_1", "notebook": "0054/137/54137484.ipynb", "difficulty": "medium", "instruction_sha256": "31bce982813ee87270b106da02940bf6b93fe52a4784a72f939e91d7cbaecb40"}, + {"name": "0128_083_128083387_qa_3", "source": "0128/083/128083387.ipynb_qa_3", "notebook": "0128/083/128083387.ipynb", "difficulty": "medium", "instruction_sha256": "7452905bbed4131155e0c99a7ba19116cc8b5ca63b3a70f40e20e46c34dae277"}, + {"name": "0103_361_103361984_qa_4", "source": "0103/361/103361984.ipynb_qa_4", "notebook": "0103/361/103361984.ipynb", "difficulty": "hard", "instruction_sha256": "00dc8599adbe69edcee06d3726b3284de10217bf7a318730bbba60286e42f689"}, + {"name": "0075_094_75094397_qa_3", "source": "0075/094/75094397.ipynb_qa_3", "notebook": "0075/094/75094397.ipynb", "difficulty": "medium", "instruction_sha256": "ca5e81f119165a67c5088ce14298d54c0c90fc7f2bd5ae9c4f1a36b3541793dd"}, + {"name": "0120_102_120102768_qa_2", "source": "0120/102/120102768.ipynb_qa_2", "notebook": "0120/102/120102768.ipynb", "difficulty": "hard", "instruction_sha256": "1d899d170f2c2459e232da89b26e605d35edee0c4f7c753e8a81805402722bed"}, + {"name": "0020_098_20098950_qa_1", "source": "0020/098/20098950.ipynb_qa_1", "notebook": "0020/098/20098950.ipynb", "difficulty": "hard", "instruction_sha256": "cbd29e1a23e9a1465a357ec24256d852671229823e07c831eaaecd4c4da7265a"}, + {"name": "0084_320_84320235_qa_1", "source": "0084/320/84320235.ipynb_qa_1", "notebook": "0084/320/84320235.ipynb", "difficulty": "hard", "instruction_sha256": "f30bacb26dd8c7248ea9ed98939915b31dfa0ea78e580de8f10014f3cd4e2a6f"}, + {"name": "0021_093_21093883_qa_3", "source": "0021/093/21093883.ipynb_qa_3", "notebook": "0021/093/21093883.ipynb", "difficulty": "medium", "instruction_sha256": "5fe276b29beca36ff7b68c4df2e8d4ffe14093f64d04367809657ee7a448016e"}, + {"name": "0039_873_39873778_qa_5", "source": "0039/873/39873778.ipynb_qa_5", "notebook": "0039/873/39873778.ipynb", "difficulty": "hard", "instruction_sha256": "1fbb7d4273d89f2a9ed9b09f61b9b31fdb7c2e7b35eeedc6046f0cac883cf9b9"}, + {"name": "0126_259_126259861_qa_2", "source": "0126/259/126259861.ipynb_qa_2", "notebook": "0126/259/126259861.ipynb", "difficulty": "hard", "instruction_sha256": "98f1be1d841d799e4df5c65ae65fccae4a0a5e42b3e64d692764aca66b43d79c"}, + {"name": "0108_883_108883493_qa_1", "source": "0108/883/108883493.ipynb_qa_1", "notebook": "0108/883/108883493.ipynb", "difficulty": "medium", "instruction_sha256": "d7ab6282424956e83a8813449b546d5b03d2ea4ff09b40f81c02819870f64de6"}, + {"name": "0132_967_132967098_qa_4", "source": "0132/967/132967098.ipynb_qa_4", "notebook": "0132/967/132967098.ipynb", "difficulty": "medium", "instruction_sha256": "dca443a5198798d4d1d26f85abbbb559d0e112bb2fc954b753aadf1034cf2002"}, + {"name": "0053_105_53105677_qa_2", "source": "0053/105/53105677.ipynb_qa_2", "notebook": "0053/105/53105677.ipynb", "difficulty": "hard", "instruction_sha256": "25d4d939f4383a01dae2e8faae023a09dc01df9853ecb98c394b49e872e51f9c"}, + {"name": "0024_991_24991983_qa_2", "source": "0024/991/24991983.ipynb_qa_2", "notebook": "0024/991/24991983.ipynb", "difficulty": "medium", "instruction_sha256": "809aa5a472851c3bbe2288962958e1767d7ab9802ac77fbdb83eb3a3ffff3b29"}, + {"name": "0012_112_12112066_qa_4", "source": "0012/112/12112066.ipynb_qa_4", "notebook": "0012/112/12112066.ipynb", "difficulty": "medium", "instruction_sha256": "bd8f3332f2f2762ae2dc1aa6c0a980f3fd8d1fc913c7aa34857d11377bad5364"}, + {"name": "0061_519_61519361_qa_1", "source": "0061/519/61519361.ipynb_qa_1", "notebook": "0061/519/61519361.ipynb", "difficulty": "hard", "instruction_sha256": "5d88433c58e2f8177372aafb51f18d6fea35a3409fdc87f441c38202a95cdd7b"}, + {"name": "0062_663_62663778_qa_2", "source": "0062/663/62663778.ipynb_qa_2", "notebook": "0062/663/62663778.ipynb", "difficulty": "hard", "instruction_sha256": "8b5aba04f443705a905798e5247fa686f047c5ce38e49e30ddd0b5c061a02b65"}, + {"name": "0045_433_45433575_qa_3", "source": "0045/433/45433575.ipynb_qa_3", "notebook": "0045/433/45433575.ipynb", "difficulty": "medium", "instruction_sha256": "035983a6f7040073db935cc5f5a15c5b3b464df7fc4aa01b52d4cf4be8c49375"}, + {"name": "0104_905_104905979_qa_3", "source": "0104/905/104905979.ipynb_qa_3", "notebook": "0104/905/104905979.ipynb", "difficulty": "hard", "instruction_sha256": "2f4e7c1a266cdf6a3680814b0db6256cc98373a45bf065f34cf0f9ea9a2f9bb1"}, + {"name": "0137_314_137314470_qa_3", "source": "0137/314/137314470.ipynb_qa_3", "notebook": "0137/314/137314470.ipynb", "difficulty": "medium", "instruction_sha256": "01a94ac7b52feb53e081f5c0c7efbe7721ac72bd3c99694b54bde8474357be93"}, + {"name": "0092_606_92606348_qa_3", "source": "0092/606/92606348.ipynb_qa_3", "notebook": "0092/606/92606348.ipynb", "difficulty": "hard", "instruction_sha256": "27e0c1daf02628790cee29fe3fdb8d0aae854b852e3e2d7ebadc6bcc46c21b78"}, + {"name": "0021_030_21030697_qa_5", "source": "0021/030/21030697.ipynb_qa_5", "notebook": "0021/030/21030697.ipynb", "difficulty": "medium", "instruction_sha256": "12490769065374b8bd27a64604f4e83fff42c00a56bdfd34f97e253b6e067ad0"}, + {"name": "0069_205_69205815_qa_5", "source": "0069/205/69205815.ipynb_qa_5", "notebook": "0069/205/69205815.ipynb", "difficulty": "hard", "instruction_sha256": "bcd22519515297ec0974a1b173ee392f970b624d1a23949a60e629d2e9c70a00"}, + {"name": "0060_427_60427905_qa_2", "source": "0060/427/60427905.ipynb_qa_2", "notebook": "0060/427/60427905.ipynb", "difficulty": "hard", "instruction_sha256": "ab2cb531e89bc30e49e125124eb3d8445935ee71b2d3adf0e58ccf96576e0138"}, + {"name": "0049_548_49548242_qa_2", "source": "0049/548/49548242.ipynb_qa_2", "notebook": "0049/548/49548242.ipynb", "difficulty": "hard", "instruction_sha256": "ea60797d72d422c954877798917d568dc0abe7f7be02d5a9333c191135af8af1"}, + {"name": "0046_305_46305988_qa_2", "source": "0046/305/46305988.ipynb_qa_2", "notebook": "0046/305/46305988.ipynb", "difficulty": "hard", "instruction_sha256": "6ca7629e510f1c9407f0eabed99725d14b83ebe22753115dfded6a3bd70d6f48"}, + {"name": "0131_881_131881806_qa_2", "source": "0131/881/131881806.ipynb_qa_2", "notebook": "0131/881/131881806.ipynb", "difficulty": "medium", "instruction_sha256": "7008835fd6e067970f26f7739342c0ce7a80442497b09c0f1515033a391d1a04"}, + {"name": "0045_071_45071899_qa_5", "source": "0045/071/45071899.ipynb_qa_5", "notebook": "0045/071/45071899.ipynb", "difficulty": "medium", "instruction_sha256": "595df467b3cd744033a0f43fcaa31d28510923c361ec66a295e42637fed6d931"}, + {"name": "0019_517_19517366_qa_3", "source": "0019/517/19517366.ipynb_qa_3", "notebook": "0019/517/19517366.ipynb", "difficulty": "hard", "instruction_sha256": "07d2b6689dd0cb0d248324b572d9bfb1909bfd92352ca70ba5ae6bc660c57539"}, + {"name": "0132_566_132566728_qa_4", "source": "0132/566/132566728.ipynb_qa_4", "notebook": "0132/566/132566728.ipynb", "difficulty": "hard", "instruction_sha256": "0ff4c8c57d45e1c887e249c4b3b96b06c0bf4bd4512f08e0333c5fee90c1ac09"}, + {"name": "0027_804_27804225_qa_2", "source": "0027/804/27804225.ipynb_qa_2", "notebook": "0027/804/27804225.ipynb", "difficulty": "hard", "instruction_sha256": "5e608114a50b66b416bd365f9fe2ebb9585ae5abf09b802fc061e235858f0c52"}, + {"name": "0042_170_42170738_qa_4", "source": "0042/170/42170738.ipynb_qa_4", "notebook": "0042/170/42170738.ipynb", "difficulty": "medium", "instruction_sha256": "1299dad1a6ba38cbdf38ca73b3d13e1c5308e11c44a5c8204661f0f673ade48a"}, + {"name": "0044_575_44575651_qa_3", "source": "0044/575/44575651.ipynb_qa_3", "notebook": "0044/575/44575651.ipynb", "difficulty": "hard", "instruction_sha256": "5979b27a9ea420c04dda0915eb0c48af4535bc7ad702bec1c36107260a1ad4d6"}, + {"name": "0041_888_41888820_qa_1", "source": "0041/888/41888820.ipynb_qa_1", "notebook": "0041/888/41888820.ipynb", "difficulty": "medium", "instruction_sha256": "4cb69b2892609da20954c05bb99efc114d10579cf1b8e91e76e972aae6998b1c"}, + {"name": "0047_271_47271508_qa_4", "source": "0047/271/47271508.ipynb_qa_4", "notebook": "0047/271/47271508.ipynb", "difficulty": "hard", "instruction_sha256": "08886be08ad4cb55183ac498c3e01691e6e6641aabb086c59096cbbcf3017210"}, + {"name": "0077_004_77004274_qa_5", "source": "0077/004/77004274.ipynb_qa_5", "notebook": "0077/004/77004274.ipynb", "difficulty": "hard", "instruction_sha256": "eac6ca9fccacb05e4f1718e6dba5e99e914c3ac56752ed69167d1bd04a3319a5"}, + {"name": "0001_667_1667317_qa_1", "source": "0001/667/1667317.ipynb_qa_1", "notebook": "0001/667/1667317.ipynb", "difficulty": "medium", "instruction_sha256": "e2514c6f2c15ddaea7229d417626805908a1839e72ebf29e73f7522c83079962"}, + {"name": "0034_043_34043351_qa_3", "source": "0034/043/34043351.ipynb_qa_3", "notebook": "0034/043/34043351.ipynb", "difficulty": "hard", "instruction_sha256": "aea5c1e11aa21332f003991b347e163d86d6c2cf147a55e5f92e44bbc00779a3"}, + {"name": "0033_061_33061669_qa_1", "source": "0033/061/33061669.ipynb_qa_1", "notebook": "0033/061/33061669.ipynb", "difficulty": "hard", "instruction_sha256": "f763dd64e809019fccb4561a5c68bd7024e48e23ea308fe091c052789f058fe4"}, + {"name": "0042_644_42644185_qa_2", "source": "0042/644/42644185.ipynb_qa_2", "notebook": "0042/644/42644185.ipynb", "difficulty": "hard", "instruction_sha256": "b1a89483b8c741bb38adbcc218520afc1dda89ad4c036df0f517cafabdcde274"}, + {"name": "0095_202_95202118_qa_5", "source": "0095/202/95202118.ipynb_qa_5", "notebook": "0095/202/95202118.ipynb", "difficulty": "hard", "instruction_sha256": "3001605e5da3c18a31976dcffb9f09375d98c4ca6d99f39b4b97477867f09231"}, + {"name": "0099_308_99308206_qa_3", "source": "0099/308/99308206.ipynb_qa_3", "notebook": "0099/308/99308206.ipynb", "difficulty": "hard", "instruction_sha256": "6b70a4a72eb2170beb292b548617c905eddc53b65ebe5fae36394346bf5763c5"}, + {"name": "0102_213_102213799_qa_1", "source": "0102/213/102213799.ipynb_qa_1", "notebook": "0102/213/102213799.ipynb", "difficulty": "medium", "instruction_sha256": "d2eb195e061e4256ae74c4047d852762de2646408b558227dc7775a3033fc0d9"}, + {"name": "0068_754_68754620_qa_1", "source": "0068/754/68754620.ipynb_qa_1", "notebook": "0068/754/68754620.ipynb", "difficulty": "hard", "instruction_sha256": "a40743e0b3a89b6da96511d3824294fa3d5753fcbe11bccff1b8060f3929a89d"}, + {"name": "0041_703_41703265_qa_5", "source": "0041/703/41703265.ipynb_qa_5", "notebook": "0041/703/41703265.ipynb", "difficulty": "hard", "instruction_sha256": "fdf5b80b663297a651a29fa05f43e4ebf056aa01ed3c59d27aaabbf9efb4c816"}, + {"name": "0052_615_52615149_qa_2", "source": "0052/615/52615149.ipynb_qa_2", "notebook": "0052/615/52615149.ipynb", "difficulty": "hard", "instruction_sha256": "09e8b64ba2cec338c06f84d16b5fe4cc6c6ae4b65be644af7d590007fdb98707"}, + {"name": "0036_430_36430763_qa_1", "source": "0036/430/36430763.ipynb_qa_1", "notebook": "0036/430/36430763.ipynb", "difficulty": "hard", "instruction_sha256": "55e95bf3247a4d1753bd1b03af495895ccdf1dc064c3c33be1376d836c1782f6"}, + {"name": "0028_488_28488702_qa_1", "source": "0028/488/28488702.ipynb_qa_1", "notebook": "0028/488/28488702.ipynb", "difficulty": "hard", "instruction_sha256": "632c091027e11ee4369fa080682d3c7bd89266e9068f80c2b156db553ad7d1c1"}, + {"name": "0136_825_136825945_qa_1", "source": "0136/825/136825945.ipynb_qa_1", "notebook": "0136/825/136825945.ipynb", "difficulty": "medium", "instruction_sha256": "9b11924a9ca7040737459baba7094f11f30578290fbfe2408a849accdc21f383"}, + {"name": "0107_477_107477079_qa_5", "source": "0107/477/107477079.ipynb_qa_5", "notebook": "0107/477/107477079.ipynb", "difficulty": "hard", "instruction_sha256": "4f508e45fe3cc996cdbeb5e706453fde33122c6bf29fa28d42b7bb6d0cfa85ff"}, + {"name": "0065_735_65735171_qa_4", "source": "0065/735/65735171.ipynb_qa_4", "notebook": "0065/735/65735171.ipynb", "difficulty": "hard", "instruction_sha256": "f073edb87fc4d1412a6967477f35c6f6b3efa65b2220c262c4d231166ef4402b"}, + {"name": "0107_704_107704788_qa_3", "source": "0107/704/107704788.ipynb_qa_3", "notebook": "0107/704/107704788.ipynb", "difficulty": "hard", "instruction_sha256": "eb8428d43c91f4cb5ce8998b153d66cd30e89a7d7c772ae6004166ecc0ba503f"}, + {"name": "0121_988_121988886_qa_5", "source": "0121/988/121988886.ipynb_qa_5", "notebook": "0121/988/121988886.ipynb", "difficulty": "hard", "instruction_sha256": "89444f5b54730b0db3ec55f77f847cee00c3091dbca88c8e91c6f097793bb42a"}, + {"name": "0055_625_55625808_qa_5", "source": "0055/625/55625808.ipynb_qa_5", "notebook": "0055/625/55625808.ipynb", "difficulty": "hard", "instruction_sha256": "bbd44c0b248888fe3e2617d564c1138cddfa83acb4eb0ea34afa1b510ab0c4e1"}, + {"name": "0057_406_57406107_qa_4", "source": "0057/406/57406107.ipynb_qa_4", "notebook": "0057/406/57406107.ipynb", "difficulty": "hard", "instruction_sha256": "1e151e9ea22f2ccd4858e748223a72b01ab7444eec79d80118d5e0b1317569c3"}, + {"name": "0132_151_132151183_qa_2", "source": "0132/151/132151183.ipynb_qa_2", "notebook": "0132/151/132151183.ipynb", "difficulty": "hard", "instruction_sha256": "2fe32eb36acc6003ba314653661adf962566e6a8eee00ddd82b1fc465df2b63e"}, + {"name": "0046_660_46660345_qa_5", "source": "0046/660/46660345.ipynb_qa_5", "notebook": "0046/660/46660345.ipynb", "difficulty": "hard", "instruction_sha256": "a5dace8de7a90703e2bc964f74b71d4abe49d625951d61aaa9787c2441bfc1df"}, + {"name": "0104_503_104503087_qa_3", "source": "0104/503/104503087.ipynb_qa_3", "notebook": "0104/503/104503087.ipynb", "difficulty": "hard", "instruction_sha256": "2d64a1e4a84587061410928dcf08613f10d3d32e38f9dbb15159e887beeed8df"}, + {"name": "0056_349_56349410_qa_4", "source": "0056/349/56349410.ipynb_qa_4", "notebook": "0056/349/56349410.ipynb", "difficulty": "hard", "instruction_sha256": "3f0fa676e9c7baef189df32958680763f841c0462badc80f6836375af2d9d3ae"}, + {"name": "0096_973_96973759_qa_3", "source": "0096/973/96973759.ipynb_qa_3", "notebook": "0096/973/96973759.ipynb", "difficulty": "hard", "instruction_sha256": "78ef0548dbb724d668d89bf7e4034f3edf428e5a4514e905f5459d3833b4981f"}, + {"name": "0031_954_31954480_qa_3", "source": "0031/954/31954480.ipynb_qa_3", "notebook": "0031/954/31954480.ipynb", "difficulty": "medium", "instruction_sha256": "395eb68fe8123da9ef2880430119d58be648c8bd1bacd4551d6365082556a908"}, + {"name": "0109_469_109469247_qa_1", "source": "0109/469/109469247.ipynb_qa_1", "notebook": "0109/469/109469247.ipynb", "difficulty": "hard", "instruction_sha256": "0767eaa2f93e3b2b773fa00e249efdad58755ab69e051d9865d24d21ce8a510d"}, + {"name": "0115_767_115767675_qa_2", "source": "0115/767/115767675.ipynb_qa_2", "notebook": "0115/767/115767675.ipynb", "difficulty": "medium", "instruction_sha256": "d8fdb33692f08a7a77235d9aa068719b25bec187b6801dc6d49e3b6ffc512856"}, + {"name": "0068_041_68041367_qa_5", "source": "0068/041/68041367.ipynb_qa_5", "notebook": "0068/041/68041367.ipynb", "difficulty": "hard", "instruction_sha256": "948e258d28ab82a4b91305f1f0865d9a965a4bac841c69d8dfc86fff286ec4db"}, + {"name": "0026_818_26818249_qa_5", "source": "0026/818/26818249.ipynb_qa_5", "notebook": "0026/818/26818249.ipynb", "difficulty": "hard", "instruction_sha256": "4a86f4596c7f67cec0d1f22365efb876aeadb3108e999798cb362fb96fb856f3"}, + {"name": "0035_911_35911748_qa_2", "source": "0035/911/35911748.ipynb_qa_2", "notebook": "0035/911/35911748.ipynb", "difficulty": "hard", "instruction_sha256": "736e077a870a6b5ab4c82586367f38547fceb7777047cb735951b3cb0d67442c"}, + {"name": "0032_092_32092871_qa_2", "source": "0032/092/32092871.ipynb_qa_2", "notebook": "0032/092/32092871.ipynb", "difficulty": "hard", "instruction_sha256": "719e950a6db39083186b3e992423544dd98ae1dba15b6b854b901ba5f28c2bc9"}, + {"name": "0062_945_62945279_qa_3", "source": "0062/945/62945279.ipynb_qa_3", "notebook": "0062/945/62945279.ipynb", "difficulty": "hard", "instruction_sha256": "63f09994d1a7864714a15abdc022536ad67152254296baf10d3451af92e3c5b5"}, + {"name": "0042_503_42503346_qa_4", "source": "0042/503/42503346.ipynb_qa_4", "notebook": "0042/503/42503346.ipynb", "difficulty": "medium", "instruction_sha256": "658787677aa0b9e876781bcad6094d7cf67dce1c679c059010a8ecadd188a0c2"}, + {"name": "0135_324_135324126_qa_5", "source": "0135/324/135324126.ipynb_qa_5", "notebook": "0135/324/135324126.ipynb", "difficulty": "hard", "instruction_sha256": "7869fbb8d25301ac1bca0d5e5545db81a543cf238c570521c9c8cad794895e50"}, + {"name": "0114_484_114484617_qa_4", "source": "0114/484/114484617.ipynb_qa_4", "notebook": "0114/484/114484617.ipynb", "difficulty": "hard", "instruction_sha256": "e01294715c920050dbf7fe810d698fab28af34b4c9134582c2f19b528d98c324"}, + {"name": "0034_890_34890721_qa_4", "source": "0034/890/34890721.ipynb_qa_4", "notebook": "0034/890/34890721.ipynb", "difficulty": "hard", "instruction_sha256": "2b89dfe10973bcda072d9892c82b71d8b910523aff35864a2e30bd1a138854bb"}, + {"name": "0073_719_73719424_qa_2", "source": "0073/719/73719424.ipynb_qa_2", "notebook": "0073/719/73719424.ipynb", "difficulty": "hard", "instruction_sha256": "3b29d67ed5450cc889679e93524a6438cb1c15c2e22366a77ff7e1e00845ec80"}, + {"name": "0041_074_41074160_qa_1", "source": "0041/074/41074160.ipynb_qa_1", "notebook": "0041/074/41074160.ipynb", "difficulty": "medium", "instruction_sha256": "5977a46ef79c2f2ccae88236f18967039b185474c1cf593a231c2163ef973d22"}, + {"name": "0130_998_130998781_qa_1", "source": "0130/998/130998781.ipynb_qa_1", "notebook": "0130/998/130998781.ipynb", "difficulty": "hard", "instruction_sha256": "1b21ccdb67dea3270cbc20f3dad3f82e410471d62d14395fbeeb309fce89f517"}, + {"name": "0050_885_50885329_qa_2", "source": "0050/885/50885329.ipynb_qa_2", "notebook": "0050/885/50885329.ipynb", "difficulty": "hard", "instruction_sha256": "135db4c791fefc00d86f58d2847f5fd71f6dc9dc1ff898bc01ac32ab3938cd2b"}, + {"name": "0046_115_46115058_qa_5", "source": "0046/115/46115058.ipynb_qa_5", "notebook": "0046/115/46115058.ipynb", "difficulty": "hard", "instruction_sha256": "1cb83d2dd12dad40f30442b91078c67d0ab0cbb15a7f0e295504975e6a101bcc"}, + {"name": "0035_350_35350224_qa_5", "source": "0035/350/35350224.ipynb_qa_5", "notebook": "0035/350/35350224.ipynb", "difficulty": "hard", "instruction_sha256": "ac311080ca0fcc186ceef1a16f029f5aefb1533c53351fc1d8c64e2a424fe7ce"}, + {"name": "0050_513_50513011_qa_5", "source": "0050/513/50513011.ipynb_qa_5", "notebook": "0050/513/50513011.ipynb", "difficulty": "medium", "instruction_sha256": "13b7a1ba52160ee38305f93669d2fb3ac407c5824daf4a8d6ee17f07818ffa57"}, + {"name": "0022_609_22609620_qa_3", "source": "0022/609/22609620.ipynb_qa_3", "notebook": "0022/609/22609620.ipynb", "difficulty": "hard", "instruction_sha256": "01e70cda1b78c0b186bcbddbece36c44d7ab141dddd8d1ded8d0e71a411e9560"}, + {"name": "0045_892_45892166_qa_5", "source": "0045/892/45892166.ipynb_qa_5", "notebook": "0045/892/45892166.ipynb", "difficulty": "hard", "instruction_sha256": "fe66f713e379ccf7e116dbe7c11c12d12642334abb416b576318dc7e628d25b6"}, + {"name": "0039_513_39513695_qa_4", "source": "0039/513/39513695.ipynb_qa_4", "notebook": "0039/513/39513695.ipynb", "difficulty": "medium", "instruction_sha256": "ca3efa9b20bfc9fd2707559c6be88a983d965bdc5d0a0354b93540958739a090"}, + {"name": "0051_802_51802195_qa_4", "source": "0051/802/51802195.ipynb_qa_4", "notebook": "0051/802/51802195.ipynb", "difficulty": "medium", "instruction_sha256": "60efdec5883de81b7c9852162c554e27cf209aee0922abc82e3552ea522f5d33"}, + {"name": "0086_158_86158878_qa_4", "source": "0086/158/86158878.ipynb_qa_4", "notebook": "0086/158/86158878.ipynb", "difficulty": "medium", "instruction_sha256": "0b5ed2f2c51bdec1d82fd512b9062411d70be5f85dd14c34795dde545991ca56"}, + {"name": "0047_766_47766276_qa_2", "source": "0047/766/47766276.ipynb_qa_2", "notebook": "0047/766/47766276.ipynb", "difficulty": "medium", "instruction_sha256": "bb0accf322b7ea46cf8ba8ed1b225bb375d659e8f83d11452e23466ac23fe7df"}, + {"name": "0042_642_42642764_qa_2", "source": "0042/642/42642764.ipynb_qa_2", "notebook": "0042/642/42642764.ipynb", "difficulty": "hard", "instruction_sha256": "046fac1d8d344136bc1b5bb6bd64d90134c7b8d8472312ace86f2cc7d2803265"}, + {"name": "0096_460_96460762_qa_2", "source": "0096/460/96460762.ipynb_qa_2", "notebook": "0096/460/96460762.ipynb", "difficulty": "hard", "instruction_sha256": "76c77d7ed7e82f43fd977d87142865ed21c06bab07175d043c17f7df8096e34a"}, + {"name": "0039_729_39729863_qa_2", "source": "0039/729/39729863.ipynb_qa_2", "notebook": "0039/729/39729863.ipynb", "difficulty": "medium", "instruction_sha256": "1546adcc72d5d5a58d97780c30d3feb04f1ca8edfccc4fa4f2370b73fe308516"}, + {"name": "0108_379_108379534_qa_2", "source": "0108/379/108379534.ipynb_qa_2", "notebook": "0108/379/108379534.ipynb", "difficulty": "medium", "instruction_sha256": "2fffab1b8e086849e984fe0a04cdee14031730eb6e1e6e186edef85848cb2949"}, + {"name": "0041_652_41652253_qa_1", "source": "0041/652/41652253.ipynb_qa_1", "notebook": "0041/652/41652253.ipynb", "difficulty": "medium", "instruction_sha256": "58cc0a273f93ed4f000bfbce1130ce606410005c6218ee58e62d27bb83bd9182"}, + {"name": "0122_965_122965866_qa_5", "source": "0122/965/122965866.ipynb_qa_5", "notebook": "0122/965/122965866.ipynb", "difficulty": "hard", "instruction_sha256": "7f333ce37f75cf38717f554516db0f07a5d80479e74cf3225bc19cb2666751a5"}, + {"name": "0012_834_12834655_qa_4", "source": "0012/834/12834655.ipynb_qa_4", "notebook": "0012/834/12834655.ipynb", "difficulty": "hard", "instruction_sha256": "e668e303e0656fc1466cc7df83e0be99ac66cacd1cee6bfb2c7b6f1b38303f7c"}, + {"name": "0068_164_68164064_qa_1", "source": "0068/164/68164064.ipynb_qa_1", "notebook": "0068/164/68164064.ipynb", "difficulty": "medium", "instruction_sha256": "f4ace518480e8da18c2c3d537f5f272544c9e209315a71f55167c0b46bf1b1c0"}, + {"name": "0043_869_43869885_qa_5", "source": "0043/869/43869885.ipynb_qa_5", "notebook": "0043/869/43869885.ipynb", "difficulty": "hard", "instruction_sha256": "6d998a89afa5fa2ed09df3b670c3dae555bf0f1faefa4910fa56a3612b5f9f38"}, + {"name": "0048_518_48518667_qa_4", "source": "0048/518/48518667.ipynb_qa_4", "notebook": "0048/518/48518667.ipynb", "difficulty": "medium", "instruction_sha256": "6a274c3ba858b3cd31e7f9164f55ba2c8511605721bbc69f8de6d00d99929879"}, + {"name": "0091_713_91713569_qa_1", "source": "0091/713/91713569.ipynb_qa_1", "notebook": "0091/713/91713569.ipynb", "difficulty": "hard", "instruction_sha256": "435a263496dc794e6b1d5de436416c889f3859da0916d86e13b2a8e9f7fdb082"}, + {"name": "0001_170_1170198_qa_3", "source": "0001/170/1170198.ipynb_qa_3", "notebook": "0001/170/1170198.ipynb", "difficulty": "medium", "instruction_sha256": "98e7505d3e2d5028b54277a080a6dad1c9b9ef74f3d1990b79f39caa4110764f"}, + {"name": "0081_472_81472707_qa_3", "source": "0081/472/81472707.ipynb_qa_3", "notebook": "0081/472/81472707.ipynb", "difficulty": "hard", "instruction_sha256": "662df6a371daaf1d8c96b5aa252eccab06a69777c6aea4d6f52321ef19f31fa9"}, + {"name": "0017_914_17914820_qa_5", "source": "0017/914/17914820.ipynb_qa_5", "notebook": "0017/914/17914820.ipynb", "difficulty": "medium", "instruction_sha256": "44a8d4dec79605c389b22458aefd7faed8685b6b20170dfa067eaa59a564091a"}, + {"name": "0057_778_57778333_qa_1", "source": "0057/778/57778333.ipynb_qa_1", "notebook": "0057/778/57778333.ipynb", "difficulty": "medium", "instruction_sha256": "cd34a7be4343a859b0afffc30d5ea39c11c073919a9dee639fb2c9a3e8e7b55f"}, + {"name": "0088_860_88860848_qa_2", "source": "0088/860/88860848.ipynb_qa_2", "notebook": "0088/860/88860848.ipynb", "difficulty": "hard", "instruction_sha256": "1d8390ed98ad903d62d41502f8f9203b170bbe9a7914eb272b7fbf8fc4a1907a"}, + {"name": "0114_713_114713370_qa_2", "source": "0114/713/114713370.ipynb_qa_2", "notebook": "0114/713/114713370.ipynb", "difficulty": "medium", "instruction_sha256": "78d76cc017f1ebc485c99c5b0d45b30f0e05967b2c7a9a23f273b9dba0507efc"}, + {"name": "0079_597_79597687_qa_2", "source": "0079/597/79597687.ipynb_qa_2", "notebook": "0079/597/79597687.ipynb", "difficulty": "hard", "instruction_sha256": "06fb81994c346198180977259e68f193e6060febe38a234a86f3d5d29d3dfca0"}, + {"name": "0020_991_20991289_qa_3", "source": "0020/991/20991289.ipynb_qa_3", "notebook": "0020/991/20991289.ipynb", "difficulty": "hard", "instruction_sha256": "f18efe2764c9bcef57c9b3f01f395a7a5436d45832ae56b7e1eddaeae04d8206"}, + {"name": "0107_177_107177097_qa_4", "source": "0107/177/107177097.ipynb_qa_4", "notebook": "0107/177/107177097.ipynb", "difficulty": "medium", "instruction_sha256": "95d240e7d9039ec429913c185a6f7c653ccab89534425dbb698494ae20294650"}, + {"name": "0067_856_67856541_qa_2", "source": "0067/856/67856541.ipynb_qa_2", "notebook": "0067/856/67856541.ipynb", "difficulty": "hard", "instruction_sha256": "3b9c1f91f344f84d273cf44f594be057b71c8c82e7229a07d2c4138af36a7ed0"}, + {"name": "0124_358_124358327_qa_3", "source": "0124/358/124358327.ipynb_qa_3", "notebook": "0124/358/124358327.ipynb", "difficulty": "medium", "instruction_sha256": "a08852333ea230475e82c4f0974c64a7b56c4ef96d809b3fb1246b6f73d1db5d"}, + {"name": "0086_824_86824902_qa_2", "source": "0086/824/86824902.ipynb_qa_2", "notebook": "0086/824/86824902.ipynb", "difficulty": "hard", "instruction_sha256": "cc8ccd7eec2a9215c0f6b76c12e7ea2a1f85f8491858791a6cdf963c4db1d0b7"}, + {"name": "0038_970_38970193_qa_5", "source": "0038/970/38970193.ipynb_qa_5", "notebook": "0038/970/38970193.ipynb", "difficulty": "medium", "instruction_sha256": "e99f5da55c7fdc54829fbc90f27dabcb6973357ec6d10834ac5075fec6802f29"}, + {"name": "0014_312_14312432_qa_4", "source": "0014/312/14312432.ipynb_qa_4", "notebook": "0014/312/14312432.ipynb", "difficulty": "medium", "instruction_sha256": "b096485d08ecba418863386c3ccc6f74427617dc84cd4ad3ac39c4190e77250e"}, + {"name": "0108_363_108363990_qa_3", "source": "0108/363/108363990.ipynb_qa_3", "notebook": "0108/363/108363990.ipynb", "difficulty": "medium", "instruction_sha256": "3382c7fd72622e38c481b8b8653729034f4ef400ea11d2604389385c2f3a5f9a"}, + {"name": "0043_419_43419580_qa_1", "source": "0043/419/43419580.ipynb_qa_1", "notebook": "0043/419/43419580.ipynb", "difficulty": "hard", "instruction_sha256": "d4ba6c427effbe5e2da9df20e385c860439cf245f08eac4241a1dac9c9389b38"}, + {"name": "0010_637_10637554_qa_1", "source": "0010/637/10637554.ipynb_qa_1", "notebook": "0010/637/10637554.ipynb", "difficulty": "medium", "instruction_sha256": "5f4eeca58ccdfac2747ef8fabf321cd7d93d42382ac2a7ebc6e6f0930db30fd3"}, + {"name": "0040_788_40788908_qa_4", "source": "0040/788/40788908.ipynb_qa_4", "notebook": "0040/788/40788908.ipynb", "difficulty": "medium", "instruction_sha256": "3cd70f93b2d17ecae079140d08d56bcef33cfe8d85919a8fd8b77e3f3c8016de"}, + {"name": "0066_392_66392992_qa_3", "source": "0066/392/66392992.ipynb_qa_3", "notebook": "0066/392/66392992.ipynb", "difficulty": "hard", "instruction_sha256": "357cbc9c23751dc3a67b8075af66d0dc371b3fca0b4906e424181d7e5283466b"}, + {"name": "0001_969_1969476_qa_3", "source": "0001/969/1969476.ipynb_qa_3", "notebook": "0001/969/1969476.ipynb", "difficulty": "medium", "instruction_sha256": "c5e654248ed707ee898d1062015afe4fa137ac83b871ec0b85746c39baa4f9f0"}, + {"name": "0032_769_32769596_qa_1", "source": "0032/769/32769596.ipynb_qa_1", "notebook": "0032/769/32769596.ipynb", "difficulty": "hard", "instruction_sha256": "c1f6ca57895446f3320fb54572825e5bc26cbf2075e082148f48d771f9205d53"}, + {"name": "0062_401_62401866_qa_3", "source": "0062/401/62401866.ipynb_qa_3", "notebook": "0062/401/62401866.ipynb", "difficulty": "hard", "instruction_sha256": "72a1bd3cf9bb623924741c903ed4211748bcd118070eb741d2fdaa1c8287313b"}, + {"name": "0074_849_74849685_qa_3", "source": "0074/849/74849685.ipynb_qa_3", "notebook": "0074/849/74849685.ipynb", "difficulty": "hard", "instruction_sha256": "9151bea0a5fc95de67c870d898527c75ac5e216eb8a5ad4c11bc4f6ca4d5a4e8"}, + {"name": "0040_738_40738491_qa_4", "source": "0040/738/40738491.ipynb_qa_4", "notebook": "0040/738/40738491.ipynb", "difficulty": "medium", "instruction_sha256": "53783b41f7faa8d82961ec4eca6a3a30edf42447c9eae67374c50f5060284a8a"}, + {"name": "0124_096_124096448_qa_1", "source": "0124/096/124096448.ipynb_qa_1", "notebook": "0124/096/124096448.ipynb", "difficulty": "hard", "instruction_sha256": "2a4b31099bee1b7f9d30f5e0b18d544a523395674d3fa43d4dbb8fb666ddea35"}, + {"name": "0039_968_39968264_qa_1", "source": "0039/968/39968264.ipynb_qa_1", "notebook": "0039/968/39968264.ipynb", "difficulty": "hard", "instruction_sha256": "f36f1731f4c289e3c062fed785372d58eb0efa49e36cd1b09e2ba89a547c601a"}, + {"name": "0124_259_124259674_qa_1", "source": "0124/259/124259674.ipynb_qa_1", "notebook": "0124/259/124259674.ipynb", "difficulty": "medium", "instruction_sha256": "17f69790b818b9920dbfef1385a072fbd67bee4bd4f6877960402d0e1b061f4f"}, + {"name": "0032_337_32337163_qa_5", "source": "0032/337/32337163.ipynb_qa_5", "notebook": "0032/337/32337163.ipynb", "difficulty": "hard", "instruction_sha256": "4bb9f089ef0eeb8e7cf292ff36d6099019224b5b72c65d18b43091e39ed6b988"}, + {"name": "0095_624_95624401_qa_2", "source": "0095/624/95624401.ipynb_qa_2", "notebook": "0095/624/95624401.ipynb", "difficulty": "medium", "instruction_sha256": "98d9d4af08f9d9cd05dcb7901205204137627236fb91bc1cb79e3fc6dce4f807"}, + {"name": "0075_648_75648287_qa_1", "source": "0075/648/75648287.ipynb_qa_1", "notebook": "0075/648/75648287.ipynb", "difficulty": "medium", "instruction_sha256": "476fb703f70ec5a6cb50058dbae064caaed9649e33bcfb514940e0464cd6c59f"}, + {"name": "0041_018_41018707_qa_4", "source": "0041/018/41018707.ipynb_qa_4", "notebook": "0041/018/41018707.ipynb", "difficulty": "hard", "instruction_sha256": "50b793a56601dea816e472b3af68166141c13bf40a5323b0f8f5123d4b2fbfca"}, + {"name": "0051_855_51855195_qa_2", "source": "0051/855/51855195.ipynb_qa_2", "notebook": "0051/855/51855195.ipynb", "difficulty": "hard", "instruction_sha256": "194d55717dc491f871ca57e553f227c59ddd050f8037cf49fa2822f00dc9275f"}, + {"name": "0125_252_125252436_qa_3", "source": "0125/252/125252436.ipynb_qa_3", "notebook": "0125/252/125252436.ipynb", "difficulty": "medium", "instruction_sha256": "41a6290ab049817f6f3ced44b5799077785f6790a6fefc887e5869913618c386"}, + {"name": "0058_240_58240435_qa_1", "source": "0058/240/58240435.ipynb_qa_1", "notebook": "0058/240/58240435.ipynb", "difficulty": "hard", "instruction_sha256": "71a1ba373fb138a679e8de03cd513eee84ce12a3c30d3aaeeea87f03339a2b89"}, + {"name": "0060_240_60240221_qa_2", "source": "0060/240/60240221.ipynb_qa_2", "notebook": "0060/240/60240221.ipynb", "difficulty": "medium", "instruction_sha256": "4bb158dee8d7f512785ef85d66c7ac6e0a397f1bb194662cf2c3a6bd25c35201"}, + {"name": "0090_301_90301818_qa_4", "source": "0090/301/90301818.ipynb_qa_4", "notebook": "0090/301/90301818.ipynb", "difficulty": "medium", "instruction_sha256": "4e8a93b5a849aa291261c2bc38d659a955172953d090fd700c926d9b0b8a1b83"}, + {"name": "0088_860_88860848_qa_4", "source": "0088/860/88860848.ipynb_qa_4", "notebook": "0088/860/88860848.ipynb", "difficulty": "hard", "instruction_sha256": "06fd7cb9c6d74269158db2f1aaa5aadcc3ce67da1c805cc375030640f29bd4ac"}, + {"name": "0029_212_29212385_qa_2", "source": "0029/212/29212385.ipynb_qa_2", "notebook": "0029/212/29212385.ipynb", "difficulty": "hard", "instruction_sha256": "21a8e3614c484d109cf3fe0a9c878e4d0b9d681d1621cddb5651035112bec19d"}, + {"name": "0081_459_81459159_qa_3", "source": "0081/459/81459159.ipynb_qa_3", "notebook": "0081/459/81459159.ipynb", "difficulty": "hard", "instruction_sha256": "17b7e5e001839b310f26f484c0f7d9d36512581f25b5f9415a571178a06d1a04"}, + {"name": "0054_315_54315694_qa_2", "source": "0054/315/54315694.ipynb_qa_2", "notebook": "0054/315/54315694.ipynb", "difficulty": "medium", "instruction_sha256": "5a4555abf97f0e57c7fbaf5acfbb69df402d95a196c6432a4b19e287374f57f3"}, + {"name": "0064_330_64330026_qa_5", "source": "0064/330/64330026.ipynb_qa_5", "notebook": "0064/330/64330026.ipynb", "difficulty": "hard", "instruction_sha256": "ab33e15220294dbd50925d3a8a6f8f5f717aa5e45fc89401e7507b656080fa7b"}, + {"name": "0061_353_61353471_qa_4", "source": "0061/353/61353471.ipynb_qa_4", "notebook": "0061/353/61353471.ipynb", "difficulty": "hard", "instruction_sha256": "5aeede7af5bd0e0c449d3f4a3004461621cb03bd34763262ae0080c1fca6baa5"}, + {"name": "0058_565_58565902_qa_3", "source": "0058/565/58565902.ipynb_qa_3", "notebook": "0058/565/58565902.ipynb", "difficulty": "hard", "instruction_sha256": "d9e740cffe35448695d25cad56645b3fe9ff803117363152c1c727041a62765f"}, + {"name": "0022_712_22712745_qa_5", "source": "0022/712/22712745.ipynb_qa_5", "notebook": "0022/712/22712745.ipynb", "difficulty": "hard", "instruction_sha256": "2181deb8326cea452782b5aa3906102715f0a3689e812b2c2a2287746bcd4235"}, + {"name": "0079_677_79677746_qa_5", "source": "0079/677/79677746.ipynb_qa_5", "notebook": "0079/677/79677746.ipynb", "difficulty": "hard", "instruction_sha256": "662fd16d4a9c6c59c9265831ab65cd0b0d95644a9254d51bdc4c83e18ac5676e"}, + {"name": "0001_170_1170198_qa_4", "source": "0001/170/1170198.ipynb_qa_4", "notebook": "0001/170/1170198.ipynb", "difficulty": "medium", "instruction_sha256": "091fb77b2dc53da06a3bb0d859ec20ca998c081012bf528ad9e4ebab6fffe87e"}, + {"name": "0078_329_78329159_qa_3", "source": "0078/329/78329159.ipynb_qa_3", "notebook": "0078/329/78329159.ipynb", "difficulty": "hard", "instruction_sha256": "2ff694593e0fd5ef914d142ef50dfe380dab1c3c1c6bb7b65d2dc2339174b40b"}, + {"name": "0029_784_29784165_qa_1", "source": "0029/784/29784165.ipynb_qa_1", "notebook": "0029/784/29784165.ipynb", "difficulty": "medium", "instruction_sha256": "c998f3340d6e61f6eb92ab7d9d04f85fc83473fa418ded45872bfbb04114c8ed"}, + {"name": "0043_043_43043497_qa_2", "source": "0043/043/43043497.ipynb_qa_2", "notebook": "0043/043/43043497.ipynb", "difficulty": "medium", "instruction_sha256": "42323876a8734075efd5a68d45745b8d2c3fb394aa780e29df48652f6bc9666e"}, + {"name": "0104_261_104261137_qa_1", "source": "0104/261/104261137.ipynb_qa_1", "notebook": "0104/261/104261137.ipynb", "difficulty": "hard", "instruction_sha256": "5d582ba6ddcc3e8ecb99a54fba62229f0bfa503e490d67d6f8745095447e4d4c"}, + {"name": "0020_162_20162109_qa_3", "source": "0020/162/20162109.ipynb_qa_3", "notebook": "0020/162/20162109.ipynb", "difficulty": "hard", "instruction_sha256": "48dcfb13afb7c7c785d594f5b682d3c69e72169282495c023890ccabe2a6cc84"}, + {"name": "0026_826_26826730_qa_1", "source": "0026/826/26826730.ipynb_qa_1", "notebook": "0026/826/26826730.ipynb", "difficulty": "hard", "instruction_sha256": "41ab7724881e9f743c9a239359e4a75f69347221c241b06c7f6f1b10d1db49e9"}, + {"name": "0071_835_71835908_qa_3", "source": "0071/835/71835908.ipynb_qa_3", "notebook": "0071/835/71835908.ipynb", "difficulty": "medium", "instruction_sha256": "fe6acc42de6f3e7c7a0154f825a6d28634a435eaf11ef5f31bccdf15d1b231d2"}, + {"name": "0138_805_138805260_qa_2", "source": "0138/805/138805260.ipynb_qa_2", "notebook": "0138/805/138805260.ipynb", "difficulty": "hard", "instruction_sha256": "cdefd4adf8e8310893c6fbd089c7fae54c416b8b7b87f4f2039a6223a4db46cd"}, + {"name": "0073_138_73138449_qa_1", "source": "0073/138/73138449.ipynb_qa_1", "notebook": "0073/138/73138449.ipynb", "difficulty": "medium", "instruction_sha256": "bfd9891e1f75ad20fa258bc06c845de47481c502bd0d3d4dc291942be2d77627"}, + {"name": "0019_787_19787626_qa_3", "source": "0019/787/19787626.ipynb_qa_3", "notebook": "0019/787/19787626.ipynb", "difficulty": "medium", "instruction_sha256": "8e74a46320b0548ae7f5cb7643e9daadd9efbb0f65b5b28e60cadc0ecdaf4f7f"}, + {"name": "0001_916_1916491_qa_4", "source": "0001/916/1916491.ipynb_qa_4", "notebook": "0001/916/1916491.ipynb", "difficulty": "medium", "instruction_sha256": "151d10e108103bdb9edef05a8402010ff0fc4ed0ae3dea1107105de0b1c32e08"}, + {"name": "0045_006_45006527_qa_2", "source": "0045/006/45006527.ipynb_qa_2", "notebook": "0045/006/45006527.ipynb", "difficulty": "medium", "instruction_sha256": "169dd09d138e8a9f4abd43a249b13e02e1a37a084eda5caa3a5d1c1dd33e4c35"}, + {"name": "0064_766_64766227_qa_3", "source": "0064/766/64766227.ipynb_qa_3", "notebook": "0064/766/64766227.ipynb", "difficulty": "medium", "instruction_sha256": "146aca791858f79ec9aa8b46bcd660432372833f993f9eaf0c95f06b4a1215e1"}, + {"name": "0074_450_74450579_qa_5", "source": "0074/450/74450579.ipynb_qa_5", "notebook": "0074/450/74450579.ipynb", "difficulty": "hard", "instruction_sha256": "d837127086c675b74a388509fb78508891767b72fa03e85f1e315f460ea70fc6"}, + {"name": "0066_730_66730048_qa_2", "source": "0066/730/66730048.ipynb_qa_2", "notebook": "0066/730/66730048.ipynb", "difficulty": "hard", "instruction_sha256": "d74229ac04a413c89610d6f7b62cf66cf26c68e8ff7865e64821393f9344cf50"}, + {"name": "0104_503_104503087_qa_2", "source": "0104/503/104503087.ipynb_qa_2", "notebook": "0104/503/104503087.ipynb", "difficulty": "hard", "instruction_sha256": "ab1e011eba407d8433454fb95edbcbf445f567db102577f9eccafaccef7fca4a"}, + {"name": "0096_460_96460762_qa_5", "source": "0096/460/96460762.ipynb_qa_5", "notebook": "0096/460/96460762.ipynb", "difficulty": "hard", "instruction_sha256": "ec68d46d7cbef4608e95ab86aa993ac1dd326731dee495738692b9b7a53d90bd"}, + {"name": "0074_849_74849685_qa_4", "source": "0074/849/74849685.ipynb_qa_4", "notebook": "0074/849/74849685.ipynb", "difficulty": "hard", "instruction_sha256": "c299799ac1223275b9510712d809f4f6c29634d0b87390cb416a3a5c13260bee"}, + {"name": "0014_358_14358635_qa_1", "source": "0014/358/14358635.ipynb_qa_1", "notebook": "0014/358/14358635.ipynb", "difficulty": "medium", "instruction_sha256": "70248c4ee1498e88d1f38d7a7848a15d3e19767bb4970d376da6c8a4732be95e"}, + {"name": "0053_842_53842474_qa_2", "source": "0053/842/53842474.ipynb_qa_2", "notebook": "0053/842/53842474.ipynb", "difficulty": "hard", "instruction_sha256": "36ebaec31dd10e91fc784d0a6eb18718e18e3a05a08f219d0f34f0027666918c"}, + {"name": "0089_811_89811907_qa_3", "source": "0089/811/89811907.ipynb_qa_3", "notebook": "0089/811/89811907.ipynb", "difficulty": "medium", "instruction_sha256": "ebb733003cba03b2138160b2387551a6cad03bd745dc371cd6a0aa1ead3581a4"}, + {"name": "0101_773_101773827_qa_4", "source": "0101/773/101773827.ipynb_qa_4", "notebook": "0101/773/101773827.ipynb", "difficulty": "hard", "instruction_sha256": "adc70956273813b880ffb9ca02a74fe104905e6eb5575b977235ba424996c06a"}, + {"name": "0001_257_1257756_qa_3", "source": "0001/257/1257756.ipynb_qa_3", "notebook": "0001/257/1257756.ipynb", "difficulty": "medium", "instruction_sha256": "ee99af100b1e2570b039d860dbb4ad035858ded35ca7c4100165c2aa088f303b"}, + {"name": "0031_326_31326436_qa_3", "source": "0031/326/31326436.ipynb_qa_3", "notebook": "0031/326/31326436.ipynb", "difficulty": "hard", "instruction_sha256": "477602640c33cfb238181983e74062c51b2e5db2379093dd75a64e4bf69c6496"}, + {"name": "0075_890_75890673_qa_3", "source": "0075/890/75890673.ipynb_qa_3", "notebook": "0075/890/75890673.ipynb", "difficulty": "hard", "instruction_sha256": "0b73e825a91784190f6f2888f6b0da1ea023d4dad27e2d37ced2e5519a521b38"}, + {"name": "0075_131_75131662_qa_1", "source": "0075/131/75131662.ipynb_qa_1", "notebook": "0075/131/75131662.ipynb", "difficulty": "medium", "instruction_sha256": "fef6843ac6c728f45833123acccd637fe54855764848b6d4e0f72d85d503a27a"}, + {"name": "0122_669_122669772_qa_3", "source": "0122/669/122669772.ipynb_qa_3", "notebook": "0122/669/122669772.ipynb", "difficulty": "hard", "instruction_sha256": "6abff784f72c2978231b30446a6f60e23a5320f7238d8c41a116d886cf8747de"}, + {"name": "0063_552_63552439_qa_4", "source": "0063/552/63552439.ipynb_qa_4", "notebook": "0063/552/63552439.ipynb", "difficulty": "hard", "instruction_sha256": "a594852899507ae63c0a6adc16efcf5886346ec8e7aaccf20b3fe1012bdc8d74"}, + {"name": "0139_066_139066922_qa_3", "source": "0139/066/139066922.ipynb_qa_3", "notebook": "0139/066/139066922.ipynb", "difficulty": "medium", "instruction_sha256": "f11478ea401b382188d3bca6eae2acdecb3cf4f589dbb689239e62ec3e272071"}, + {"name": "0058_565_58565902_qa_1", "source": "0058/565/58565902.ipynb_qa_1", "notebook": "0058/565/58565902.ipynb", "difficulty": "medium", "instruction_sha256": "6fb4796061b00147c8c3779ec6ba7500e81b9b7387081d0a048723925e98b277"}, + {"name": "0010_482_10482813_qa_1", "source": "0010/482/10482813.ipynb_qa_1", "notebook": "0010/482/10482813.ipynb", "difficulty": "hard", "instruction_sha256": "2856d624dad59bf1c4ce7318d76f50643133411b931e0116c3cf47f93e5a9b98"}, + {"name": "0084_977_84977670_qa_3", "source": "0084/977/84977670.ipynb_qa_3", "notebook": "0084/977/84977670.ipynb", "difficulty": "hard", "instruction_sha256": "db2f3e4308423217c4cc9a8418958e98379bfffcc5f8f6401c5b23d4a0221e8f"}, + {"name": "0075_108_75108038_qa_2", "source": "0075/108/75108038.ipynb_qa_2", "notebook": "0075/108/75108038.ipynb", "difficulty": "hard", "instruction_sha256": "109a030528b3a555073bb0fab0cd6e623533ea69314fe7e4f4e9244722d4a298"}, + {"name": "0017_257_17257933_qa_3", "source": "0017/257/17257933.ipynb_qa_3", "notebook": "0017/257/17257933.ipynb", "difficulty": "medium", "instruction_sha256": "a0ad4d6f99d03b5a87b91670c439fc4c900b3e3d10e622d5b015bda6e04df094"}, + {"name": "0046_641_46641346_qa_1", "source": "0046/641/46641346.ipynb_qa_1", "notebook": "0046/641/46641346.ipynb", "difficulty": "medium", "instruction_sha256": "f300c65b54fdf30056b797ac25a56937bef9ca0709100c2427f7cc9ff062e96a"}, + {"name": "0115_621_115621359_qa_4", "source": "0115/621/115621359.ipynb_qa_4", "notebook": "0115/621/115621359.ipynb", "difficulty": "hard", "instruction_sha256": "6241db6e0d7eda68bac7320dc3364f967b974740aa469d1f15f07871864f918d"}, + {"name": "0021_384_21384330_qa_2", "source": "0021/384/21384330.ipynb_qa_2", "notebook": "0021/384/21384330.ipynb", "difficulty": "medium", "instruction_sha256": "d2e3efee0aefe82cd4737a0c5929749edb43f5f9ab98d74cef0886c44d55a215"}, + {"name": "0038_189_38189340_qa_2", "source": "0038/189/38189340.ipynb_qa_2", "notebook": "0038/189/38189340.ipynb", "difficulty": "hard", "instruction_sha256": "7eb19d24ccfd5e253b03cddc858d63d98593190d674d08d7ab031aa4fdba71d7"}, + {"name": "0032_386_32386426_qa_2", "source": "0032/386/32386426.ipynb_qa_2", "notebook": "0032/386/32386426.ipynb", "difficulty": "medium", "instruction_sha256": "e120d22a6877ec48a82ac9a11321e9ebcee6788f6f1d92d57c4886f95d1180a8"}, + {"name": "0135_920_135920645_qa_4", "source": "0135/920/135920645.ipynb_qa_4", "notebook": "0135/920/135920645.ipynb", "difficulty": "hard", "instruction_sha256": "100d3e9a2ab5b1453c4abe9036cbd2c0a64738ff4998bfbf1772e0290afe63e5"}, + {"name": "0064_383_64383055_qa_5", "source": "0064/383/64383055.ipynb_qa_5", "notebook": "0064/383/64383055.ipynb", "difficulty": "hard", "instruction_sha256": "e9f2bc03c0a91f203c2336ef7ac50e2f6f2126fdf41ab1f8793a03fc1d13484a"}, + {"name": "0104_143_104143863_qa_3", "source": "0104/143/104143863.ipynb_qa_3", "notebook": "0104/143/104143863.ipynb", "difficulty": "hard", "instruction_sha256": "190553f98fb93682b3da187fdc01cd217fc3c4903a889a90697ecc9c596e665d"}, + {"name": "0098_341_98341485_qa_2", "source": "0098/341/98341485.ipynb_qa_2", "notebook": "0098/341/98341485.ipynb", "difficulty": "hard", "instruction_sha256": "5bd3b84741fab9814c74d3bc8b0b06143e239509e78724a123789aa8c437138a"}, + {"name": "0110_937_110937668_qa_2", "source": "0110/937/110937668.ipynb_qa_2", "notebook": "0110/937/110937668.ipynb", "difficulty": "hard", "instruction_sha256": "5bd387e41a53aa281d4c34bf2e5832f0ba47548227fb354b1bb0cb985c63c1ea"}, + {"name": "0135_380_135380627_qa_1", "source": "0135/380/135380627.ipynb_qa_1", "notebook": "0135/380/135380627.ipynb", "difficulty": "hard", "instruction_sha256": "4bcff09ed1cfc705b2e5454d962d39ba53f702fd7ce7f975e58e156fa3ece44e"}, + {"name": "0055_859_55859289_qa_2", "source": "0055/859/55859289.ipynb_qa_2", "notebook": "0055/859/55859289.ipynb", "difficulty": "hard", "instruction_sha256": "306ddee8529019b82b97117d1ceea053b097b6d49b3d457e6ae30b2424cd7766"}, + {"name": "0031_576_31576713_qa_1", "source": "0031/576/31576713.ipynb_qa_1", "notebook": "0031/576/31576713.ipynb", "difficulty": "hard", "instruction_sha256": "09cbc0f5b02217d7dc2c9cdaf0232e9152dc900df1477c5b67760e60bf360e4a"}, + {"name": "0112_393_112393758_qa_1", "source": "0112/393/112393758.ipynb_qa_1", "notebook": "0112/393/112393758.ipynb", "difficulty": "medium", "instruction_sha256": "a490eca516d19ae58e04633ffbf85e628f08d0a8b51b6629d5788a9fee2e0aad"}, + {"name": "0127_699_127699013_qa_1", "source": "0127/699/127699013.ipynb_qa_1", "notebook": "0127/699/127699013.ipynb", "difficulty": "medium", "instruction_sha256": "286168efd5eeb01b94cbb04d3faabbbbbfdeaca3edbd59f17fb80e02ff77df62"}, + {"name": "0128_998_128998647_qa_1", "source": "0128/998/128998647.ipynb_qa_1", "notebook": "0128/998/128998647.ipynb", "difficulty": "medium", "instruction_sha256": "1c121048ec38fa4150ec0b548e5b00c04e5c820a8e2e44e9e1e1ca580ee68331"}, + {"name": "0077_322_77322684_qa_3", "source": "0077/322/77322684.ipynb_qa_3", "notebook": "0077/322/77322684.ipynb", "difficulty": "hard", "instruction_sha256": "3a5c0de45956c7e17fb3c950c0579312c44d13a9f07a2fa8cef28bd12d839b2f"}, + {"name": "0113_222_113222481_qa_3", "source": "0113/222/113222481.ipynb_qa_3", "notebook": "0113/222/113222481.ipynb", "difficulty": "medium", "instruction_sha256": "52da7eb9a279a6b801041c5b9a3e2b48697c22a67b0fa9c17dbf82b4d5b7ed8a"}, + {"name": "0102_436_102436012_qa_1", "source": "0102/436/102436012.ipynb_qa_1", "notebook": "0102/436/102436012.ipynb", "difficulty": "hard", "instruction_sha256": "9f83d085bf1bdbf70b5781661911a12b87a44a81157fe711ddeaf5c3c2b09ead"}, + {"name": "0052_041_52041171_qa_2", "source": "0052/041/52041171.ipynb_qa_2", "notebook": "0052/041/52041171.ipynb", "difficulty": "hard", "instruction_sha256": "a222cb753e363dea77d1befbfb40e01d090c43fd22b8eff34fa8aa4abe819776"}, + {"name": "0042_642_42642764_qa_3", "source": "0042/642/42642764.ipynb_qa_3", "notebook": "0042/642/42642764.ipynb", "difficulty": "hard", "instruction_sha256": "1496bc749015c188c67b88d4535d0b94d8d2b25096f6faed914f0148be8d140e"}, + {"name": "0106_481_106481442_qa_3", "source": "0106/481/106481442.ipynb_qa_3", "notebook": "0106/481/106481442.ipynb", "difficulty": "medium", "instruction_sha256": "b800789714b563e927225549b1256e3a6301a57c3313de8e209bc6f92fd8ea71"}, + {"name": "0000_539_539873_qa_3", "source": "0000/539/539873.ipynb_qa_3", "notebook": "0000/539/539873.ipynb", "difficulty": "medium", "instruction_sha256": "573c833f716b94e3c5a1cfd729a30fd625a1b798ce81b060c426d3dfa443f870"}, + {"name": "0064_691_64691450_qa_2", "source": "0064/691/64691450.ipynb_qa_2", "notebook": "0064/691/64691450.ipynb", "difficulty": "hard", "instruction_sha256": "aebf5c32304afb347630033c1a8099d2054f699713b096f84e0496f66a84753a"}, + {"name": "0086_400_86400737_qa_1", "source": "0086/400/86400737.ipynb_qa_1", "notebook": "0086/400/86400737.ipynb", "difficulty": "hard", "instruction_sha256": "59aed094d3bdcae49e199bfad0303f13dc308bc1edc1e0ed86b3837ab034013c"}, + {"name": "0038_466_38466860_qa_2", "source": "0038/466/38466860.ipynb_qa_2", "notebook": "0038/466/38466860.ipynb", "difficulty": "hard", "instruction_sha256": "01fb858103841b1904c16501ed2e8c0dda60c54dd2c3e3ff8d0628175e9bb4bf"}, + {"name": "0002_105_2105769_qa_3", "source": "0002/105/2105769.ipynb_qa_3", "notebook": "0002/105/2105769.ipynb", "difficulty": "medium", "instruction_sha256": "f4ba087b33e587e769bbc99094b75c54aa698c3e73eecec2e08192e48bc0f0da"}, + {"name": "0059_341_59341953_qa_1", "source": "0059/341/59341953.ipynb_qa_1", "notebook": "0059/341/59341953.ipynb", "difficulty": "hard", "instruction_sha256": "7ca74d574f0cf01fc4d022b2a9c630d3e29c240d7bdb4b01fa5a41a5e36f74bd"}, + {"name": "0115_943_115943948_qa_3", "source": "0115/943/115943948.ipynb_qa_3", "notebook": "0115/943/115943948.ipynb", "difficulty": "medium", "instruction_sha256": "dcd012242d85e8991e19293a39bdec2f3320b5efe9509e35e6d864e29b4cfafc"}, + {"name": "0040_298_40298967_qa_4", "source": "0040/298/40298967.ipynb_qa_4", "notebook": "0040/298/40298967.ipynb", "difficulty": "medium", "instruction_sha256": "410a60be178f34c7ffa92a11d29612420814fa7ee76212482124d03b79240aa9"}, + {"name": "0078_661_78661279_qa_3", "source": "0078/661/78661279.ipynb_qa_3", "notebook": "0078/661/78661279.ipynb", "difficulty": "hard", "instruction_sha256": "34440d0136dcc660829fe13a6c7bf823a77d427f50a4b4e6cb5a76a1c82c039e"}, + {"name": "0132_672_132672509_qa_1", "source": "0132/672/132672509.ipynb_qa_1", "notebook": "0132/672/132672509.ipynb", "difficulty": "hard", "instruction_sha256": "85dbae438a4f404fae0b5ca06b26fc280d69e838edfd428e83d8a3b80f397e82"}, + {"name": "0013_126_13126909_qa_3", "source": "0013/126/13126909.ipynb_qa_3", "notebook": "0013/126/13126909.ipynb", "difficulty": "medium", "instruction_sha256": "479b24c3e6452b7cad7fd69c2ed0e35906ca14fdfaf1bed78ba1e70e1f0fe9b4"}, + {"name": "0070_620_70620378_qa_5", "source": "0070/620/70620378.ipynb_qa_5", "notebook": "0070/620/70620378.ipynb", "difficulty": "hard", "instruction_sha256": "34b174465ba266bb643c9b64d4513b391819a5fecfabf20858aa5451886e1c5e"}, + {"name": "0032_769_32769596_qa_3", "source": "0032/769/32769596.ipynb_qa_3", "notebook": "0032/769/32769596.ipynb", "difficulty": "hard", "instruction_sha256": "bdc8f20d13eebf2dcfff40077db1d105f5d0c7398c31b666e027a3e0faa848bd"}, + {"name": "0043_374_43374447_qa_2", "source": "0043/374/43374447.ipynb_qa_2", "notebook": "0043/374/43374447.ipynb", "difficulty": "hard", "instruction_sha256": "ec7b75447fc4c5ad4e3552187996d95d57a114c9baf777f0b90407c5696776e3"}, + {"name": "0043_032_43032812_qa_4", "source": "0043/032/43032812.ipynb_qa_4", "notebook": "0043/032/43032812.ipynb", "difficulty": "medium", "instruction_sha256": "c33c8bacd08561e50ec5c57319e676a2ed73ca96a5b0c387324b4328906f680b"}, + {"name": "0019_267_19267356_qa_5", "source": "0019/267/19267356.ipynb_qa_5", "notebook": "0019/267/19267356.ipynb", "difficulty": "hard", "instruction_sha256": "21d40df2b2df7bcb48d4dbb6f31ef86de5d01b622e3372843ffcc6b9c4de777c"}, + {"name": "0046_875_46875284_qa_1", "source": "0046/875/46875284.ipynb_qa_1", "notebook": "0046/875/46875284.ipynb", "difficulty": "medium", "instruction_sha256": "72c9e8a48dfaf42b2d8168a395a37af73e4dde43596671b7e5bb0c97b1265f0b"}, + {"name": "0061_736_61736140_qa_1", "source": "0061/736/61736140.ipynb_qa_1", "notebook": "0061/736/61736140.ipynb", "difficulty": "hard", "instruction_sha256": "b3da80d9e190ec0f7bfca0aea877e0fe30dcd0607cb43421b71d504b4ba64696"}, + {"name": "0115_722_115722970_qa_3", "source": "0115/722/115722970.ipynb_qa_3", "notebook": "0115/722/115722970.ipynb", "difficulty": "medium", "instruction_sha256": "a89ca83955e1dc8f720dbcf0150424fe31ff3f1215602455c85fc5d74b9263f5"}, + {"name": "0067_857_67857075_qa_4", "source": "0067/857/67857075.ipynb_qa_4", "notebook": "0067/857/67857075.ipynb", "difficulty": "medium", "instruction_sha256": "d9e0299ef4351ddcfd255c99ccfda66c1fae3e65c111b2da938ef4e4e220e80b"}, + {"name": "0092_190_92190264_qa_4", "source": "0092/190/92190264.ipynb_qa_4", "notebook": "0092/190/92190264.ipynb", "difficulty": "hard", "instruction_sha256": "a2d9f5f43910ae250c5c1519bc8e784e41a7423937b988f01a37e0062fd3bf7d"}, + {"name": "0113_428_113428593_qa_1", "source": "0113/428/113428593.ipynb_qa_1", "notebook": "0113/428/113428593.ipynb", "difficulty": "hard", "instruction_sha256": "80959c7b9520d6d8e05b2e94f4606ee6db5f815cab45254eaf62d0a4c132ee81"}, + {"name": "0038_821_38821925_qa_1", "source": "0038/821/38821925.ipynb_qa_1", "notebook": "0038/821/38821925.ipynb", "difficulty": "medium", "instruction_sha256": "39b62feb9a27b4809166e552312da2eba83b0780bf82c91681e6d954b84e9e5e"}, + {"name": "0100_729_100729925_qa_4", "source": "0100/729/100729925.ipynb_qa_4", "notebook": "0100/729/100729925.ipynb", "difficulty": "hard", "instruction_sha256": "b546d8aa594ff64619dc1e38df2bb5e7355994e8515e5f6aaaaddb0f6aec779e"}, + {"name": "0001_293_1293142_qa_5", "source": "0001/293/1293142.ipynb_qa_5", "notebook": "0001/293/1293142.ipynb", "difficulty": "medium", "instruction_sha256": "2b4050786cf7487fe28859ac72ccc2127e66c06eb46381853422e5d831750256"}, + {"name": "0135_355_135355749_qa_3", "source": "0135/355/135355749.ipynb_qa_3", "notebook": "0135/355/135355749.ipynb", "difficulty": "hard", "instruction_sha256": "8a93e67143fc14340d529e568bbd5f9ae043d1a4069626675b8979e9d3d6749f"}, + {"name": "0113_007_113007141_qa_2", "source": "0113/007/113007141.ipynb_qa_2", "notebook": "0113/007/113007141.ipynb", "difficulty": "hard", "instruction_sha256": "0c190531184178f56bfaef7665341df99e29f0e67e7dc1dc3233bb13aa4f7ee2"}, + {"name": "0026_920_26920486_qa_1", "source": "0026/920/26920486.ipynb_qa_1", "notebook": "0026/920/26920486.ipynb", "difficulty": "hard", "instruction_sha256": "d7c919bcb90eec57f1aac95335025359aa9df5f8544eef8f168df6089988631d"}, + {"name": "0037_918_37918087_qa_1", "source": "0037/918/37918087.ipynb_qa_1", "notebook": "0037/918/37918087.ipynb", "difficulty": "hard", "instruction_sha256": "2e206df6b98e52b8c408b6c5e371ae08eb84608722bc9b6b031443bf6f393da3"}, + {"name": "0037_918_37918087_qa_4", "source": "0037/918/37918087.ipynb_qa_4", "notebook": "0037/918/37918087.ipynb", "difficulty": "hard", "instruction_sha256": "7d788b84e72a921fa2a305c80255edc7a3b69292fbe07a27d9b38b67bb74883c"}, + {"name": "0038_900_38900780_qa_4", "source": "0038/900/38900780.ipynb_qa_4", "notebook": "0038/900/38900780.ipynb", "difficulty": "medium", "instruction_sha256": "38a71051a733a237a9323bcf4413bcce355d188f38c36ca2abdeeb8f3a6e1443"}, + {"name": "0022_916_22916338_qa_1", "source": "0022/916/22916338.ipynb_qa_1", "notebook": "0022/916/22916338.ipynb", "difficulty": "hard", "instruction_sha256": "a5313196fe34688ff74571365481876afe4da65d3092db941c8d98061b66543f"}, + {"name": "0041_451_41451584_qa_1", "source": "0041/451/41451584.ipynb_qa_1", "notebook": "0041/451/41451584.ipynb", "difficulty": "medium", "instruction_sha256": "a07dfac002529d8242ae5254f4ca370adeffbce33c401d0930453b587acf7b72"}, + {"name": "0136_646_136646417_qa_4", "source": "0136/646/136646417.ipynb_qa_4", "notebook": "0136/646/136646417.ipynb", "difficulty": "medium", "instruction_sha256": "9feeaa6b519dadb60c3b09119db251875c5e9469fb959ad7bad926ee2d1c358c"}, + {"name": "0083_022_83022280_qa_4", "source": "0083/022/83022280.ipynb_qa_4", "notebook": "0083/022/83022280.ipynb", "difficulty": "hard", "instruction_sha256": "93fe0f764454342aa904d4bb8a0b7d4edd34f9c64e9c4c05aef1ebc3c32085aa"}, + {"name": "0073_798_73798390_qa_5", "source": "0073/798/73798390.ipynb_qa_5", "notebook": "0073/798/73798390.ipynb", "difficulty": "medium", "instruction_sha256": "b85b99126a74576c23a2723d0e5a45a755337fbac26fa3d43111f7bd9dc5a373"}, + {"name": "0104_591_104591892_qa_4", "source": "0104/591/104591892.ipynb_qa_4", "notebook": "0104/591/104591892.ipynb", "difficulty": "medium", "instruction_sha256": "b662b11f8000472a89898bc4600ec5266134231b326d56e785100e69d973dc91"}, + {"name": "0085_154_85154775_qa_3", "source": "0085/154/85154775.ipynb_qa_3", "notebook": "0085/154/85154775.ipynb", "difficulty": "hard", "instruction_sha256": "3e22ce973d343f8adc80c611fbd9c1d626d92a102d961ea80a0523887615bdc6"}, + {"name": "0017_810_17810500_qa_2", "source": "0017/810/17810500.ipynb_qa_2", "notebook": "0017/810/17810500.ipynb", "difficulty": "hard", "instruction_sha256": "47cd0919e43a94f6702841d7a193f596d58ece1f86d52af631714efd22c95685"}, + {"name": "0067_657_67657879_qa_5", "source": "0067/657/67657879.ipynb_qa_5", "notebook": "0067/657/67657879.ipynb", "difficulty": "medium", "instruction_sha256": "9bb616e79698db66977e8d3fef8216b32b86c1c09ba0694d6d67f64692917b31"}, + {"name": "0139_071_139071391_qa_4", "source": "0139/071/139071391.ipynb_qa_4", "notebook": "0139/071/139071391.ipynb", "difficulty": "hard", "instruction_sha256": "7fe21b6c2fc30a58ce709165eafb4ffe2790c3224615eb31f28e9019b76a1626"}, + {"name": "0135_920_135920645_qa_5", "source": "0135/920/135920645.ipynb_qa_5", "notebook": "0135/920/135920645.ipynb", "difficulty": "hard", "instruction_sha256": "12946a850b39de8bd1ce3a8d2c07b0a2e37c010accda31bbb79896e713a73454"}, + {"name": "0137_061_137061428_qa_2", "source": "0137/061/137061428.ipynb_qa_2", "notebook": "0137/061/137061428.ipynb", "difficulty": "medium", "instruction_sha256": "464999068e7577a9b0f8b519bbb0353875f00c8b2562d75d79cb04c65dc0e040"}, + {"name": "0044_708_44708511_qa_4", "source": "0044/708/44708511.ipynb_qa_4", "notebook": "0044/708/44708511.ipynb", "difficulty": "medium", "instruction_sha256": "92137a61929d47918ea9bccae7a1e9ab8f6f48f4834d247bf3ded299f575b8dc"}, + {"name": "0044_175_44175301_qa_3", "source": "0044/175/44175301.ipynb_qa_3", "notebook": "0044/175/44175301.ipynb", "difficulty": "hard", "instruction_sha256": "49b5c1c9d891b7c7bf01977096534d9763c53927554827a801d56ff945af87c3"}, + {"name": "0135_485_135485471_qa_4", "source": "0135/485/135485471.ipynb_qa_4", "notebook": "0135/485/135485471.ipynb", "difficulty": "hard", "instruction_sha256": "9080b6cb0d28a394af2d004eaf74a23e2bcbb2a87da6bceec6d469bf37deb406"}, + {"name": "0036_916_36916974_qa_5", "source": "0036/916/36916974.ipynb_qa_5", "notebook": "0036/916/36916974.ipynb", "difficulty": "medium", "instruction_sha256": "a5cea3559a2aaa8e2eadbd3458a4875dcc819d1101e62a4f29b03b7f5bf0fbdb"}, + {"name": "0055_050_55050231_qa_3", "source": "0055/050/55050231.ipynb_qa_3", "notebook": "0055/050/55050231.ipynb", "difficulty": "hard", "instruction_sha256": "8cf7dd41f7c8a613f582f9abed472797919a08c22673fe76a04c48a156aa0a18"}, + {"name": "0054_291_54291213_qa_5", "source": "0054/291/54291213.ipynb_qa_5", "notebook": "0054/291/54291213.ipynb", "difficulty": "hard", "instruction_sha256": "d34ef22514f1edb0dc3ca6ab69334b9ccb24e1e4aae90f54ad6d8ca1a49a0f44"}, + {"name": "0031_829_31829521_qa_1", "source": "0031/829/31829521.ipynb_qa_1", "notebook": "0031/829/31829521.ipynb", "difficulty": "medium", "instruction_sha256": "5426be35a04ed0436f182372bb447d6a734faf2c5ac7fa49026bb2f76705ce92"}, + {"name": "0043_694_43694920_qa_1", "source": "0043/694/43694920.ipynb_qa_1", "notebook": "0043/694/43694920.ipynb", "difficulty": "hard", "instruction_sha256": "8ca149f612bf1c0a7cd50788d443cc59e3d6bf616349bb8f929dbad0f63a26d8"}, + {"name": "0104_985_104985612_qa_4", "source": "0104/985/104985612.ipynb_qa_4", "notebook": "0104/985/104985612.ipynb", "difficulty": "hard", "instruction_sha256": "f7aebb23ab30ee9b731ab035093527aff8566314cdbc7822113a18aca2da2e2a"}, + {"name": "0139_112_139112787_qa_2", "source": "0139/112/139112787.ipynb_qa_2", "notebook": "0139/112/139112787.ipynb", "difficulty": "hard", "instruction_sha256": "fbb1573509c68a1ecb918bef8775bdb8c19f558d58fbcc54bc0f87835267ee8e"}, + {"name": "0049_034_49034706_qa_4", "source": "0049/034/49034706.ipynb_qa_4", "notebook": "0049/034/49034706.ipynb", "difficulty": "hard", "instruction_sha256": "0ae27bcc40254394059d7da47f3865aa0d02149b8e23375c815714ed33712d8e"}, + {"name": "0074_281_74281388_qa_1", "source": "0074/281/74281388.ipynb_qa_1", "notebook": "0074/281/74281388.ipynb", "difficulty": "medium", "instruction_sha256": "ac9c32c5e00500c7cd78ce0ed5d6c52e251d4ad0af666897a61567736c306b59"}, + {"name": "0044_575_44575268_qa_1", "source": "0044/575/44575268.ipynb_qa_1", "notebook": "0044/575/44575268.ipynb", "difficulty": "medium", "instruction_sha256": "56d52076506baaa0396e33c1c267743f08f2465341629f3b0c549c756dea290c"}, + {"name": "0057_142_57142499_qa_5", "source": "0057/142/57142499.ipynb_qa_5", "notebook": "0057/142/57142499.ipynb", "difficulty": "hard", "instruction_sha256": "9992236d803bc3086246d100ed54ec8d5889fb55cda3de9e15b56cb24acfde9c"}, + {"name": "0001_741_1741873_qa_4", "source": "0001/741/1741873.ipynb_qa_4", "notebook": "0001/741/1741873.ipynb", "difficulty": "medium", "instruction_sha256": "fafb462b16ef0a64d82d607b2c733c5fe23a37b3867eb6ceb08ea85abf9564b4"}, + {"name": "0023_715_23715177_qa_1", "source": "0023/715/23715177.ipynb_qa_1", "notebook": "0023/715/23715177.ipynb", "difficulty": "medium", "instruction_sha256": "8e33485f7ceda61cd22c46c57c6dea2320bb896090c84ce570877c605c5ea169"}, + {"name": "0012_593_12593899_qa_2", "source": "0012/593/12593899.ipynb_qa_2", "notebook": "0012/593/12593899.ipynb", "difficulty": "medium", "instruction_sha256": "f989296330617dc973784285231481f151e5f029ef2e66223555a8e1e3ee14ff"}, + {"name": "0034_651_34651965_qa_2", "source": "0034/651/34651965.ipynb_qa_2", "notebook": "0034/651/34651965.ipynb", "difficulty": "hard", "instruction_sha256": "92dec9d71628122e7c2a7e5e3484353e65ba41837b5ac96dea7decd8741bfd3c"}, + {"name": "0138_777_138777709_qa_1", "source": "0138/777/138777709.ipynb_qa_1", "notebook": "0138/777/138777709.ipynb", "difficulty": "medium", "instruction_sha256": "b5cfbbe172fe9337998747aa3c26a61a89d76a7d02655b8d0a0f46dccdc22892"}, + {"name": "0122_938_122938999_qa_1", "source": "0122/938/122938999.ipynb_qa_1", "notebook": "0122/938/122938999.ipynb", "difficulty": "hard", "instruction_sha256": "2817d7c641a2c642762f8f21d119602ea7d79a6da821b21ba86370f2f1b2060e"}, + {"name": "0063_575_63575701_qa_2", "source": "0063/575/63575701.ipynb_qa_2", "notebook": "0063/575/63575701.ipynb", "difficulty": "hard", "instruction_sha256": "d920d1ff375d45a2c63d59f391c537f64b2c668e363d5bd8a37b2e5bc07e29a6"}, + {"name": "0066_435_66435240_qa_4", "source": "0066/435/66435240.ipynb_qa_4", "notebook": "0066/435/66435240.ipynb", "difficulty": "hard", "instruction_sha256": "d4d81e1dbc114922f98d7324cdab1ed9d626ca14dc37504b1ce2d41aa351f0ea"}, + {"name": "0133_196_133196028_qa_4", "source": "0133/196/133196028.ipynb_qa_4", "notebook": "0133/196/133196028.ipynb", "difficulty": "hard", "instruction_sha256": "8e73c47ae243733f76832e79f9285fb2b33153cc4fe395a66b8ae6996d60e35e"}, + {"name": "0040_783_40783338_qa_3", "source": "0040/783/40783338.ipynb_qa_3", "notebook": "0040/783/40783338.ipynb", "difficulty": "hard", "instruction_sha256": "0a390998be523f55ca18c649bcc13347bde52b8d95da4808b823f45327ee52c8"}, + {"name": "0107_830_107830483_qa_1", "source": "0107/830/107830483.ipynb_qa_1", "notebook": "0107/830/107830483.ipynb", "difficulty": "hard", "instruction_sha256": "6f61a1eabf357cd96802e14b87766af2c57442714d703eb896391a6f5487c6d5"}, + {"name": "0102_982_102982156_qa_3", "source": "0102/982/102982156.ipynb_qa_3", "notebook": "0102/982/102982156.ipynb", "difficulty": "hard", "instruction_sha256": "9b19c23edb3ef8a34c7e0b01d6ee1c5da40b77acce616eef0b0fb91ffee45e19"}, + {"name": "0038_570_38570610_qa_3", "source": "0038/570/38570610.ipynb_qa_3", "notebook": "0038/570/38570610.ipynb", "difficulty": "medium", "instruction_sha256": "6d61783935340c79f08260043a12de16ac98f50b4adbf1379f8ccf04978a17a1"}, + {"name": "0120_295_120295119_qa_5", "source": "0120/295/120295119.ipynb_qa_5", "notebook": "0120/295/120295119.ipynb", "difficulty": "medium", "instruction_sha256": "dd39c5a25a6afb19691005ad6900218fb4716ada277dd9f3f1f2edff475acb57"}, + {"name": "0082_302_82302927_qa_3", "source": "0082/302/82302927.ipynb_qa_3", "notebook": "0082/302/82302927.ipynb", "difficulty": "hard", "instruction_sha256": "16f76842c0b187d16eb4b93df91f27cac7d817b9fb1d413f6dde6e63cb2d4af4"}, + {"name": "0039_815_39815076_qa_3", "source": "0039/815/39815076.ipynb_qa_3", "notebook": "0039/815/39815076.ipynb", "difficulty": "hard", "instruction_sha256": "33b8a375e512e2cb9aee75d69820da7f4380258de6dc696498033589e97e85d4"}, + {"name": "0084_737_84737869_qa_3", "source": "0084/737/84737869.ipynb_qa_3", "notebook": "0084/737/84737869.ipynb", "difficulty": "hard", "instruction_sha256": "7123c5a9e119d2c9433847f99d1ff1a771795bd8fad373e12e12d57922c15d24"}, + {"name": "0101_509_101509299_qa_5", "source": "0101/509/101509299.ipynb_qa_5", "notebook": "0101/509/101509299.ipynb", "difficulty": "hard", "instruction_sha256": "79200fb2e0b48bf60d3bf8341f4a74f86b64c393f42885144af80251df503594"}, + {"name": "0105_115_105115907_qa_1", "source": "0105/115/105115907.ipynb_qa_1", "notebook": "0105/115/105115907.ipynb", "difficulty": "medium", "instruction_sha256": "80290d0a3bd906b9a38a8d4af5f17431bff88a7d0e8a626a97dbcc1ff7a1fc05"}, + {"name": "0097_377_97377124_qa_1", "source": "0097/377/97377124.ipynb_qa_1", "notebook": "0097/377/97377124.ipynb", "difficulty": "hard", "instruction_sha256": "ec4047100e19a8435fb6d6c4c69c7256ef99643f25dfcafa63b63345ec992772"}, + {"name": "0013_708_13708498_qa_4", "source": "0013/708/13708498.ipynb_qa_4", "notebook": "0013/708/13708498.ipynb", "difficulty": "medium", "instruction_sha256": "96576a714a3aa8b33dfeda09aaba768a24e7008d2b66589e668d7d0a29b993ee"}, + {"name": "0042_528_42528802_qa_3", "source": "0042/528/42528802.ipynb_qa_3", "notebook": "0042/528/42528802.ipynb", "difficulty": "hard", "instruction_sha256": "b9edf95eaa2b451ae01cc724f77ba495d3bf268eaaec172fc662783ed8ce7370"}, + {"name": "0042_928_42928888_qa_1", "source": "0042/928/42928888.ipynb_qa_1", "notebook": "0042/928/42928888.ipynb", "difficulty": "medium", "instruction_sha256": "0e4b714406300c0c2ae80aea0f8bd379d89981f7cf9c054fd210ead9664e0d5b"}, + {"name": "0010_860_10860955_qa_1", "source": "0010/860/10860955.ipynb_qa_1", "notebook": "0010/860/10860955.ipynb", "difficulty": "hard", "instruction_sha256": "598282265c9d7db7a822906f4b13e24b5fe0575be0843c60eb5cf290850e41bf"}, + {"name": "0039_156_39156457_qa_3", "source": "0039/156/39156457.ipynb_qa_3", "notebook": "0039/156/39156457.ipynb", "difficulty": "hard", "instruction_sha256": "2fe3c2e54ec0c68d47b7b20bf3b5a613976b34fb95db39bf82c56f814bb439db"}, + {"name": "0017_291_17291057_qa_5", "source": "0017/291/17291057.ipynb_qa_5", "notebook": "0017/291/17291057.ipynb", "difficulty": "medium", "instruction_sha256": "77e39eb82075f4d9b734b7edb9211319de09efc0dceeec1904ee30d91942f11e"}, + {"name": "0139_196_139196764_qa_4", "source": "0139/196/139196764.ipynb_qa_4", "notebook": "0139/196/139196764.ipynb", "difficulty": "medium", "instruction_sha256": "8d097b4709f6b524b688a4f691b53d8138268f107a84f4d56d113b46064fc4fd"}, + {"name": "0069_094_69094334_qa_4", "source": "0069/094/69094334.ipynb_qa_4", "notebook": "0069/094/69094334.ipynb", "difficulty": "hard", "instruction_sha256": "41a8097603ad2380490cc353b100e4e76a0a8b3b5dd51a18d0aa1013e5fa04d5"}, + {"name": "0079_606_79606942_qa_1", "source": "0079/606/79606942.ipynb_qa_1", "notebook": "0079/606/79606942.ipynb", "difficulty": "hard", "instruction_sha256": "24dbccd42613d7a9990fd0f61de0fe2abcde0a45157bd68d135a387dbc60f0ae"}, + {"name": "0010_909_10909980_qa_3", "source": "0010/909/10909980.ipynb_qa_3", "notebook": "0010/909/10909980.ipynb", "difficulty": "medium", "instruction_sha256": "5f00d93da1cda3a267f9a5e4cd86354efe442b64254df4aaa4fe26232ae77f26"}, + {"name": "0062_017_62017707_qa_4", "source": "0062/017/62017707.ipynb_qa_4", "notebook": "0062/017/62017707.ipynb", "difficulty": "hard", "instruction_sha256": "db79a515aa917340d16d6ee2cd8608ee4f0f9a4f80fe88ae10c1ef5a268e94c5"}, + {"name": "0121_319_121319744_qa_1", "source": "0121/319/121319744.ipynb_qa_1", "notebook": "0121/319/121319744.ipynb", "difficulty": "hard", "instruction_sha256": "dc794c23ea4dd635fda1bc96fea8ad9a5249e39d447b96bdd11096df964c81d1"}, + {"name": "0030_152_30152211_qa_2", "source": "0030/152/30152211.ipynb_qa_2", "notebook": "0030/152/30152211.ipynb", "difficulty": "medium", "instruction_sha256": "ef2e218e0024016c475096daa222541ddf2fed4c24abc880d7c408f91e434091"}, + {"name": "0054_291_54291213_qa_2", "source": "0054/291/54291213.ipynb_qa_2", "notebook": "0054/291/54291213.ipynb", "difficulty": "hard", "instruction_sha256": "86122e98199249f4d0c2826b90a47c8e2789835ec277c880cefc393a1c17e48d"}, + {"name": "0125_603_125603788_qa_2", "source": "0125/603/125603788.ipynb_qa_2", "notebook": "0125/603/125603788.ipynb", "difficulty": "medium", "instruction_sha256": "f7b1da3bdbf0c48ee6f291e39ea40188ede97f55db41ee37f326e8b5a17b1493"}, + {"name": "0132_903_132903502_qa_5", "source": "0132/903/132903502.ipynb_qa_5", "notebook": "0132/903/132903502.ipynb", "difficulty": "medium", "instruction_sha256": "bcc5477f1fc78530dc314e2119a9977f73143c45d1ca0a84070b55add37d1a7d"}, + {"name": "0064_907_64907248_qa_1", "source": "0064/907/64907248.ipynb_qa_1", "notebook": "0064/907/64907248.ipynb", "difficulty": "medium", "instruction_sha256": "e1523d54819abc51c60560f1ee12e874de375b38e28bbe2bacaae21b261b4102"}, + {"name": "0102_401_102401343_qa_1", "source": "0102/401/102401343.ipynb_qa_1", "notebook": "0102/401/102401343.ipynb", "difficulty": "medium", "instruction_sha256": "d9f56dca32858e1e9e11e9d936db42b446931de2a9cde07aa45f294b321a97fe"}, + {"name": "0055_335_55335995_qa_1", "source": "0055/335/55335995.ipynb_qa_1", "notebook": "0055/335/55335995.ipynb", "difficulty": "hard", "instruction_sha256": "b2d257d7ccedaaf5cd1aef7525862316dfcdfcda7366bfac297b08fd33f2f060"}, + {"name": "0039_714_39714962_qa_2", "source": "0039/714/39714962.ipynb_qa_2", "notebook": "0039/714/39714962.ipynb", "difficulty": "hard", "instruction_sha256": "a40cbaf828540803e3b06435196223e79f167b7d6fd5ba2b59d493573a873f70"}, + {"name": "0061_736_61736140_qa_4", "source": "0061/736/61736140.ipynb_qa_4", "notebook": "0061/736/61736140.ipynb", "difficulty": "hard", "instruction_sha256": "5ee4016b62386798d6d73f7ce67ea72d846d7957b48e8a1b987677036fe46695"}, + {"name": "0096_064_96064872_qa_2", "source": "0096/064/96064872.ipynb_qa_2", "notebook": "0096/064/96064872.ipynb", "difficulty": "hard", "instruction_sha256": "86087d56aa02c551477369c4788a4a675972802fc3c9007282fbf5472a1021f9"}, + {"name": "0042_806_42806362_qa_1", "source": "0042/806/42806362.ipynb_qa_1", "notebook": "0042/806/42806362.ipynb", "difficulty": "medium", "instruction_sha256": "e8817f0c8cb9c00c89134e38cce939cf63016fa714a98837b9743c321f4475b3"}, + {"name": "0031_872_31872499_qa_4", "source": "0031/872/31872499.ipynb_qa_4", "notebook": "0031/872/31872499.ipynb", "difficulty": "medium", "instruction_sha256": "63d62912914ca21e9ab6ff8437c5debd990c3374f5d8394403702f5253254f36"}, + {"name": "0070_908_70908084_qa_2", "source": "0070/908/70908084.ipynb_qa_2", "notebook": "0070/908/70908084.ipynb", "difficulty": "hard", "instruction_sha256": "487ac97585cacddb75d3a02790e7cb63d907f422205cdebcb0a19acf74e22214"}, + {"name": "0128_624_128624067_qa_1", "source": "0128/624/128624067.ipynb_qa_1", "notebook": "0128/624/128624067.ipynb", "difficulty": "medium", "instruction_sha256": "a10fdc1e5ac246aba50d9ededd8b017c2116930a1a0a7be53674d6be35d3a075"}, + {"name": "0038_708_38708794_qa_3", "source": "0038/708/38708794.ipynb_qa_3", "notebook": "0038/708/38708794.ipynb", "difficulty": "medium", "instruction_sha256": "837bbeb051a204e59939f5a5cdb7179a4dda5d1062cd8bf8ae5290d5750aa505"}, + {"name": "0095_057_95057992_qa_2", "source": "0095/057/95057992.ipynb_qa_2", "notebook": "0095/057/95057992.ipynb", "difficulty": "medium", "instruction_sha256": "27b45113356d734717e8c77d7255b7f71551573380bb63de9bcdfcb769b76784"}, + {"name": "0112_515_112515781_qa_2", "source": "0112/515/112515781.ipynb_qa_2", "notebook": "0112/515/112515781.ipynb", "difficulty": "medium", "instruction_sha256": "72b8314c640bfba93bfe3c15312fe12d8b1b78e2359666439f9fc5ae8d4ac578"}, + {"name": "0084_634_84634252_qa_2", "source": "0084/634/84634252.ipynb_qa_2", "notebook": "0084/634/84634252.ipynb", "difficulty": "medium", "instruction_sha256": "ec727399dff73b53049417b59e12a5d1c0f67341d647e9d22133738ac07a04be"}, + {"name": "0084_932_84932987_qa_5", "source": "0084/932/84932987.ipynb_qa_5", "notebook": "0084/932/84932987.ipynb", "difficulty": "medium", "instruction_sha256": "0eadcf8217cd3b70eb3b60374ab9995a30ccfb4bf3fead6468e57a722d898182"}, + {"name": "0066_819_66819249_qa_2", "source": "0066/819/66819249.ipynb_qa_2", "notebook": "0066/819/66819249.ipynb", "difficulty": "hard", "instruction_sha256": "6b36f038bd4f021671e7d0ed6908033a0e2a7c8a0754c8980460c05348abba17"}, + {"name": "0035_025_35025726_qa_2", "source": "0035/025/35025726.ipynb_qa_2", "notebook": "0035/025/35025726.ipynb", "difficulty": "medium", "instruction_sha256": "e581b205614aac15d8046dc8804720b8ef866d1c82a6b2e52eebfb7e562681fe"}, + {"name": "0063_482_63482132_qa_5", "source": "0063/482/63482132.ipynb_qa_5", "notebook": "0063/482/63482132.ipynb", "difficulty": "hard", "instruction_sha256": "c59f3246e45316c264cd634fb9ec3d16d6dff34e3c12815bc39d133f5ecab745"}, + {"name": "0056_167_56167051_qa_3", "source": "0056/167/56167051.ipynb_qa_3", "notebook": "0056/167/56167051.ipynb", "difficulty": "medium", "instruction_sha256": "d996b1981baa5bd88ed5acda60040b1e61a4a52bae7905c3b0529453395c5967"}, + {"name": "0051_591_51591719_qa_1", "source": "0051/591/51591719.ipynb_qa_1", "notebook": "0051/591/51591719.ipynb", "difficulty": "hard", "instruction_sha256": "6c6895037b473ec1372dcd1893e84ecbeb13f6904b67bee59b4cc2e0a50dd236"}, + {"name": "0135_541_135541718_qa_1", "source": "0135/541/135541718.ipynb_qa_1", "notebook": "0135/541/135541718.ipynb", "difficulty": "hard", "instruction_sha256": "bf5550237925c2b955f6c21fdd51c5f3e936fe92774b1f844fd9ccda8c53e7fd"}, + {"name": "0102_919_102919379_qa_1", "source": "0102/919/102919379.ipynb_qa_1", "notebook": "0102/919/102919379.ipynb", "difficulty": "hard", "instruction_sha256": "57207744dcf1b099b381fe3c0a9d3c05dfd09c696542d773b807cdc62d489c05"}, + {"name": "0114_325_114325622_qa_4", "source": "0114/325/114325622.ipynb_qa_4", "notebook": "0114/325/114325622.ipynb", "difficulty": "hard", "instruction_sha256": "8d9109408fe313a2eea55d19c9f9dcfaaefaab8e7f54d19dce81ab0b3e68b5da"}, + {"name": "0031_106_31106192_qa_2", "source": "0031/106/31106192.ipynb_qa_2", "notebook": "0031/106/31106192.ipynb", "difficulty": "hard", "instruction_sha256": "b9b9cb6906cc09e58fd204b9119743a2164c626847ba77982bc95d388dc32123"}, + {"name": "0110_432_110432344_qa_4", "source": "0110/432/110432344.ipynb_qa_4", "notebook": "0110/432/110432344.ipynb", "difficulty": "medium", "instruction_sha256": "518b92786267735c652929d7fc662115e16547c490486d4535cf28d9e61eecc9"}, + {"name": "0029_340_29340222_qa_5", "source": "0029/340/29340222.ipynb_qa_5", "notebook": "0029/340/29340222.ipynb", "difficulty": "hard", "instruction_sha256": "6d2cc62486de5caeab7e9f6b9f58b22491d1bd59380e7f0f702ba543432db5e4"}, + {"name": "0002_375_2375931_qa_3", "source": "0002/375/2375931.ipynb_qa_3", "notebook": "0002/375/2375931.ipynb", "difficulty": "medium", "instruction_sha256": "1be885a86bb287487066330eed28765190ba31878c4462cc7297205a2b5c69fa"}, + {"name": "0135_324_135324126_qa_2", "source": "0135/324/135324126.ipynb_qa_2", "notebook": "0135/324/135324126.ipynb", "difficulty": "hard", "instruction_sha256": "0babc2a88664ca6d329b5d388d4f6bfd6ceacef0c7d9d2bacf47c82fbdfe1d71"}, + {"name": "0087_320_87320991_qa_2", "source": "0087/320/87320991.ipynb_qa_2", "notebook": "0087/320/87320991.ipynb", "difficulty": "medium", "instruction_sha256": "949785afb3bad617e58d951f037c9aea6ddcb27fcd346d5703fdbee76525b6d9"}, + {"name": "0137_745_137745793_qa_4", "source": "0137/745/137745793.ipynb_qa_4", "notebook": "0137/745/137745793.ipynb", "difficulty": "hard", "instruction_sha256": "f90ae6b3562f929deb7cfca4d9a88561f12d8598a1a56ce21f50f5d649ebee6e"}, + {"name": "0042_455_42455252_qa_2", "source": "0042/455/42455252.ipynb_qa_2", "notebook": "0042/455/42455252.ipynb", "difficulty": "hard", "instruction_sha256": "4897f84f1434e61d1b4f0d76c7e737ff838afcbd23639deba02c2cdce3aad4ba"}, + {"name": "0066_715_66715498_qa_1", "source": "0066/715/66715498.ipynb_qa_1", "notebook": "0066/715/66715498.ipynb", "difficulty": "hard", "instruction_sha256": "14db58282e987c9bee1d36d4eb821d12636ec9e9d92ac1cc9af9d9d92ed1072e"}, + {"name": "0063_386_63386776_qa_1", "source": "0063/386/63386776.ipynb_qa_1", "notebook": "0063/386/63386776.ipynb", "difficulty": "medium", "instruction_sha256": "d6a54fcb46624120c81857893f0e9d307482ed7f6847d95ddae95275e076d6ed"}, + {"name": "0125_624_125624875_qa_2", "source": "0125/624/125624875.ipynb_qa_2", "notebook": "0125/624/125624875.ipynb", "difficulty": "medium", "instruction_sha256": "035279ec02721e78b1a7ecdd1784d216747a3c754dc40271b094ebfe07ca928a"}, + {"name": "0098_340_98340653_qa_5", "source": "0098/340/98340653.ipynb_qa_5", "notebook": "0098/340/98340653.ipynb", "difficulty": "hard", "instruction_sha256": "3df21d9c86e6a7803055219f7e56da3df5638470d16871298523db14a5695aca"}, + {"name": "0039_234_39234687_qa_1", "source": "0039/234/39234687.ipynb_qa_1", "notebook": "0039/234/39234687.ipynb", "difficulty": "medium", "instruction_sha256": "974336441083a8001ac6411c9eb83696ef650092d8642ca7ed25fd9b944a5fc5"}, + {"name": "0091_859_91859529_qa_4", "source": "0091/859/91859529.ipynb_qa_4", "notebook": "0091/859/91859529.ipynb", "difficulty": "hard", "instruction_sha256": "02aef507ef7fe4408090ccf3384f3d261c1faeebf1c7a5439c423686d1711ce9"}, + {"name": "0072_839_72839930_qa_3", "source": "0072/839/72839930.ipynb_qa_3", "notebook": "0072/839/72839930.ipynb", "difficulty": "hard", "instruction_sha256": "9bf9d2c74e546a66595fcacd3ccd6745af445920f8f72b6380c63e1c74ad3df4"}, + {"name": "0033_684_33684962_qa_2", "source": "0033/684/33684962.ipynb_qa_2", "notebook": "0033/684/33684962.ipynb", "difficulty": "hard", "instruction_sha256": "1b4a5cd68ccebaf71e022d42115b68eef840dde87ee5f7f0a9fc7af8b9cfde2b"}, + {"name": "0136_175_136175034_qa_5", "source": "0136/175/136175034.ipynb_qa_5", "notebook": "0136/175/136175034.ipynb", "difficulty": "hard", "instruction_sha256": "ab267510f3d83575ad35a8c1dd1b0a0d1880d6fd260372c4db88eba13c9147f1"}, + {"name": "0090_327_90327261_qa_3", "source": "0090/327/90327261.ipynb_qa_3", "notebook": "0090/327/90327261.ipynb", "difficulty": "hard", "instruction_sha256": "649ebe9ed4cade4d600e965ddce1df75ca759a2462b767ee150c748285f96897"}, + {"name": "0044_107_44107622_qa_1", "source": "0044/107/44107622.ipynb_qa_1", "notebook": "0044/107/44107622.ipynb", "difficulty": "medium", "instruction_sha256": "b30372af43aa6920bab280e93dd9deeec9e37c7406f06ee2bd6a54d4a7d9f5bd"}, + {"name": "0128_173_128173004_qa_2", "source": "0128/173/128173004.ipynb_qa_2", "notebook": "0128/173/128173004.ipynb", "difficulty": "medium", "instruction_sha256": "364add06b3b59f10d7a53f70b1a8df3483bf1f71881ae93422c7f4dc415d6d7c"}, + {"name": "0138_141_138141265_qa_5", "source": "0138/141/138141265.ipynb_qa_5", "notebook": "0138/141/138141265.ipynb", "difficulty": "hard", "instruction_sha256": "bacb08c7c64f291a295600c0c2215f6d3d3f1186b62a1389c4383a811df97d08"}, + {"name": "0094_557_94557527_qa_3", "source": "0094/557/94557527.ipynb_qa_3", "notebook": "0094/557/94557527.ipynb", "difficulty": "hard", "instruction_sha256": "278311c6f91a056938330886aabe7c387aa3aedb7246a38e394b2fbdf61dc1e1"}, + {"name": "0084_821_84821146_qa_1", "source": "0084/821/84821146.ipynb_qa_1", "notebook": "0084/821/84821146.ipynb", "difficulty": "medium", "instruction_sha256": "deaa6cde2c7c7a9c2e398b12a6ce6ce504aa540136d14276e6b762687458224c"}, + {"name": "0102_882_102882563_qa_1", "source": "0102/882/102882563.ipynb_qa_1", "notebook": "0102/882/102882563.ipynb", "difficulty": "medium", "instruction_sha256": "9937ccf3f4241f0ac8c924fd40f829afa4fc80622f8d3abbcf27aa1fe85ab4c7"}, + {"name": "0018_550_18550354_qa_2", "source": "0018/550/18550354.ipynb_qa_2", "notebook": "0018/550/18550354.ipynb", "difficulty": "hard", "instruction_sha256": "0ded4c715819c33d56b68c82185ede954f7ef4397e82a73599bd1c824f3f426d"}, + {"name": "0050_467_50467216_qa_3", "source": "0050/467/50467216.ipynb_qa_3", "notebook": "0050/467/50467216.ipynb", "difficulty": "hard", "instruction_sha256": "8d3534bca14fb074a61779e3074a36c2dfaf8788bea15562c345f3771610c425"}, + {"name": "0026_551_26551206_qa_3", "source": "0026/551/26551206.ipynb_qa_3", "notebook": "0026/551/26551206.ipynb", "difficulty": "hard", "instruction_sha256": "417b1e39262a5ca28303aaa2514186cb1a8f814753c338a3e8d54441ca9cdb69"}, + {"name": "0034_886_34886359_qa_3", "source": "0034/886/34886359.ipynb_qa_3", "notebook": "0034/886/34886359.ipynb", "difficulty": "hard", "instruction_sha256": "a501d5fca3ebc08a79d7b3c905292fba51bd703e6d2050612259de537c742c75"}, + {"name": "0012_748_12748314_qa_1", "source": "0012/748/12748314.ipynb_qa_1", "notebook": "0012/748/12748314.ipynb", "difficulty": "medium", "instruction_sha256": "859057dbfcbf59e19cf07bd0d922e83b57ca5d6b6a7fa233cf5e62d596e4a64b"}, + {"name": "0073_670_73670678_qa_5", "source": "0073/670/73670678.ipynb_qa_5", "notebook": "0073/670/73670678.ipynb", "difficulty": "medium", "instruction_sha256": "6f83226616fdce428b4e5bce9cf1c1e7d7f5cc0c84c4fd83f9d1fbdb17b4fd7d"}, + {"name": "0040_788_40788057_qa_5", "source": "0040/788/40788057.ipynb_qa_5", "notebook": "0040/788/40788057.ipynb", "difficulty": "hard", "instruction_sha256": "076326c18842ab5a8e178941255ef6b42046e91a1db7d4764535ddb6b5d651aa"}, + {"name": "0128_485_128485352_qa_1", "source": "0128/485/128485352.ipynb_qa_1", "notebook": "0128/485/128485352.ipynb", "difficulty": "hard", "instruction_sha256": "16a37ca8ad928b074c2d696d6bd12e74df93cdfc85bbe098ed87a46c9bb62904"}, + {"name": "0045_032_45032719_qa_5", "source": "0045/032/45032719.ipynb_qa_5", "notebook": "0045/032/45032719.ipynb", "difficulty": "medium", "instruction_sha256": "bd77a5cf93f72266137060d7239091f8044dbb0c7128f3f0a8fd9d054ee48091"}, + {"name": "0051_561_51561248_qa_4", "source": "0051/561/51561248.ipynb_qa_4", "notebook": "0051/561/51561248.ipynb", "difficulty": "medium", "instruction_sha256": "ff28ed7824a8aa9e6c22b4cc573c642efb192af2421ebaa859b5b4ceb9158065"}, + {"name": "0091_122_91122496_qa_5", "source": "0091/122/91122496.ipynb_qa_5", "notebook": "0091/122/91122496.ipynb", "difficulty": "hard", "instruction_sha256": "264617e6735d12b239594dd297e453c9fb0cf0ee817fc8f47d53f9ff526e2dd3"}, + {"name": "0020_611_20611134_qa_1", "source": "0020/611/20611134.ipynb_qa_1", "notebook": "0020/611/20611134.ipynb", "difficulty": "hard", "instruction_sha256": "37f046ec90953431d6bfc336e8c4e39cf0e9fd9000d34287fc77c843367b647c"}, + {"name": "0050_587_50587128_qa_4", "source": "0050/587/50587128.ipynb_qa_4", "notebook": "0050/587/50587128.ipynb", "difficulty": "medium", "instruction_sha256": "12ca1f58c6d22f219084f44668c5d0bd9af0c5c67c0cdeef392678fb599da56e"}, + {"name": "0065_360_65360161_qa_1", "source": "0065/360/65360161.ipynb_qa_1", "notebook": "0065/360/65360161.ipynb", "difficulty": "hard", "instruction_sha256": "2f49e2be88ebd2171b938f72e38369a453c65f666327512551e95c26747830b3"}, + {"name": "0017_257_17257933_qa_4", "source": "0017/257/17257933.ipynb_qa_4", "notebook": "0017/257/17257933.ipynb", "difficulty": "medium", "instruction_sha256": "cee7c2e2aebfb0f59ef384ac79d98c9d369fe7aee32b314f06f4f442502a9c57"}, + {"name": "0034_339_34339717_qa_3", "source": "0034/339/34339717.ipynb_qa_3", "notebook": "0034/339/34339717.ipynb", "difficulty": "hard", "instruction_sha256": "ba71ae506e4f5f68beb7b8744fc1003f6603275a4715b6a66502eee1859b1b05"}, + {"name": "0001_085_1085629_qa_4", "source": "0001/085/1085629.ipynb_qa_4", "notebook": "0001/085/1085629.ipynb", "difficulty": "hard", "instruction_sha256": "12e4acdf38208263832bdfb3c13834beae27a06a2cde9b66c107d3a3ba4e9560"}, + {"name": "0027_151_27151900_qa_5", "source": "0027/151/27151900.ipynb_qa_5", "notebook": "0027/151/27151900.ipynb", "difficulty": "hard", "instruction_sha256": "8df46092b816814aa431d9b9430bdb68382ddc25538b44d9ad023d102108e523"}, + {"name": "0101_296_101296258_qa_3", "source": "0101/296/101296258.ipynb_qa_3", "notebook": "0101/296/101296258.ipynb", "difficulty": "hard", "instruction_sha256": "ba77db3e3da4ed644cf79c3377db8534d6c61cfcc2434162561b923dd1b97003"}, + {"name": "0039_458_39458303_qa_5", "source": "0039/458/39458303.ipynb_qa_5", "notebook": "0039/458/39458303.ipynb", "difficulty": "hard", "instruction_sha256": "9dbad988038f95e37f3878cca926316e7399e417f6e286ca0f8cf3e1f7cb9501"}, + {"name": "0038_910_38910811_qa_5", "source": "0038/910/38910811.ipynb_qa_5", "notebook": "0038/910/38910811.ipynb", "difficulty": "medium", "instruction_sha256": "fd30eea9e390b7c1ddda593b875841aa3efcd6eb8c812e17a6b0115ef426eaf4"}, + {"name": "0013_170_13170004_qa_5", "source": "0013/170/13170004.ipynb_qa_5", "notebook": "0013/170/13170004.ipynb", "difficulty": "medium", "instruction_sha256": "380f9a212a63161c33cbb930d3a2fcee2dce3bf156ebb7a9c326218161c4e256"}, + {"name": "0088_548_88548685_qa_4", "source": "0088/548/88548685.ipynb_qa_4", "notebook": "0088/548/88548685.ipynb", "difficulty": "hard", "instruction_sha256": "088d1de768bb29680caff8758cc83ea1aae871d6d41adad30d0a99522238abae"}, + {"name": "0040_717_40717045_qa_2", "source": "0040/717/40717045.ipynb_qa_2", "notebook": "0040/717/40717045.ipynb", "difficulty": "hard", "instruction_sha256": "e5286d7a9b1648abef64157a640ae602a02e38fd54d7cafa8512937841b7faa2"}, + {"name": "0031_954_31954799_qa_2", "source": "0031/954/31954799.ipynb_qa_2", "notebook": "0031/954/31954799.ipynb", "difficulty": "medium", "instruction_sha256": "6162f17c8f6b718d4f9ddccf03034f3ae2bfe218580251f85ea862001955d7cc"}, + {"name": "0059_410_59410836_qa_1", "source": "0059/410/59410836.ipynb_qa_1", "notebook": "0059/410/59410836.ipynb", "difficulty": "medium", "instruction_sha256": "746b8995ad135eaa3a7f5abda022b3de71f7e1f5f9c444d6b10b29a8a31fb009"}, + {"name": "0060_331_60331255_qa_3", "source": "0060/331/60331255.ipynb_qa_3", "notebook": "0060/331/60331255.ipynb", "difficulty": "hard", "instruction_sha256": "0914bdefa6011bbb279a4d64b4a5e501d7882a9fa5348be66ac13cf1d51746aa"}, + {"name": "0095_909_95909431_qa_3", "source": "0095/909/95909431.ipynb_qa_3", "notebook": "0095/909/95909431.ipynb", "difficulty": "medium", "instruction_sha256": "4d89fbf323792e89076cf2be80af5595f2f4ab4765898138185d545a72e5ec98"}, + {"name": "0034_339_34339717_qa_1", "source": "0034/339/34339717.ipynb_qa_1", "notebook": "0034/339/34339717.ipynb", "difficulty": "hard", "instruction_sha256": "d4281908ad7b9d799d2d04f304fc540b8ea307153eb4730979c80c9bd29b8502"}, + {"name": "0026_168_26168363_qa_1", "source": "0026/168/26168363.ipynb_qa_1", "notebook": "0026/168/26168363.ipynb", "difficulty": "hard", "instruction_sha256": "a7be7c9e70ad601222560b3dcb932277e8ddce577b5290bc5e6f74930175e328"}, + {"name": "0033_684_33684962_qa_3", "source": "0033/684/33684962.ipynb_qa_3", "notebook": "0033/684/33684962.ipynb", "difficulty": "hard", "instruction_sha256": "d24eb8ee0e2f12439d71b74730743243e6ab2b439ccef4568eb898148229a548"}, + {"name": "0047_417_47417997_qa_3", "source": "0047/417/47417997.ipynb_qa_3", "notebook": "0047/417/47417997.ipynb", "difficulty": "hard", "instruction_sha256": "aafcd626c2357304e41dff122d2f4a0ba7c9750772a958d1ad49fd66f1d69934"}, + {"name": "0113_130_113130564_qa_2", "source": "0113/130/113130564.ipynb_qa_2", "notebook": "0113/130/113130564.ipynb", "difficulty": "medium", "instruction_sha256": "08571dff1ef050e43f2f54cbe9f8b429642c19ac7741d63841aa5e0a8e8f73e3"}, + {"name": "0123_042_123042543_qa_1", "source": "0123/042/123042543.ipynb_qa_1", "notebook": "0123/042/123042543.ipynb", "difficulty": "medium", "instruction_sha256": "cb7e11923c660ed4b92b42b69037a81e3f34013e9aeec158d6038cdf5e901c10"}, + {"name": "0120_313_120313409_qa_1", "source": "0120/313/120313409.ipynb_qa_1", "notebook": "0120/313/120313409.ipynb", "difficulty": "hard", "instruction_sha256": "8976ec79a401b338bc0cf40411af6a2a93a9541eb952a6fd930043929d969438"}, + {"name": "0039_477_39477592_qa_5", "source": "0039/477/39477592.ipynb_qa_5", "notebook": "0039/477/39477592.ipynb", "difficulty": "hard", "instruction_sha256": "142bdd2cee2ee2a74801ae7f0ab06a9bf59d2c50cc25b3f90d9a3f614f0d2b9b"}, + {"name": "0056_896_56896780_qa_4", "source": "0056/896/56896780.ipynb_qa_4", "notebook": "0056/896/56896780.ipynb", "difficulty": "hard", "instruction_sha256": "cfec4235101707eb10427eceda5fd5628542e2e285de86cf7d7794fd4aa885cd"}, + {"name": "0090_941_90941195_qa_2", "source": "0090/941/90941195.ipynb_qa_2", "notebook": "0090/941/90941195.ipynb", "difficulty": "hard", "instruction_sha256": "463030410f1c0b35e2a3091168b8d08c4a3665664ad03d0503280bf1e4aa3959"}, + {"name": "0037_728_37728227_qa_2", "source": "0037/728/37728227.ipynb_qa_2", "notebook": "0037/728/37728227.ipynb", "difficulty": "medium", "instruction_sha256": "ae02eac3553584b87749cb7a457a15684ebbaf4d932c482a055691983e192c09"}, + {"name": "0118_607_118607538_qa_2", "source": "0118/607/118607538.ipynb_qa_2", "notebook": "0118/607/118607538.ipynb", "difficulty": "medium", "instruction_sha256": "a695a5e0a00068ab2a287658697bf10fa3d9c3dd29cfdfef72879a897e3a46a7"}, + {"name": "0041_143_41143507_qa_1", "source": "0041/143/41143507.ipynb_qa_1", "notebook": "0041/143/41143507.ipynb", "difficulty": "hard", "instruction_sha256": "a3582fe435345eaa65a4899157826fbd6d465185805ac494565e4ff1b9ffb262"}, + {"name": "0096_997_96997158_qa_1", "source": "0096/997/96997158.ipynb_qa_1", "notebook": "0096/997/96997158.ipynb", "difficulty": "hard", "instruction_sha256": "b47618f7c20c9f3d00a5a46d1284e3c193efcab5757a2fdbd099e729101fcb46"}, + {"name": "0044_988_44988061_qa_5", "source": "0044/988/44988061.ipynb_qa_5", "notebook": "0044/988/44988061.ipynb", "difficulty": "hard", "instruction_sha256": "8aef986225794c23af840e3f28146fa30f737d0efcf6de0fc8cbe654ce9701d0"}, + {"name": "0077_006_77006094_qa_1", "source": "0077/006/77006094.ipynb_qa_1", "notebook": "0077/006/77006094.ipynb", "difficulty": "medium", "instruction_sha256": "fc438470bd7aee57f1c6022ae0429b3c7ac4e52c3e30b9e7e9b8dfd6f05650bd"}, + {"name": "0035_755_35755658_qa_3", "source": "0035/755/35755658.ipynb_qa_3", "notebook": "0035/755/35755658.ipynb", "difficulty": "hard", "instruction_sha256": "d23e219a2b092f09f3dd6a6297a265b9bf3ebddc6c9817f5c31cc257c42cbf2a"}, + {"name": "0085_154_85154775_qa_5", "source": "0085/154/85154775.ipynb_qa_5", "notebook": "0085/154/85154775.ipynb", "difficulty": "hard", "instruction_sha256": "2a7ca0492432ca3efd893186a45892abc1b5e41c68268346a7711530a62c7339"}, + {"name": "0002_240_2240465_qa_4", "source": "0002/240/2240465.ipynb_qa_4", "notebook": "0002/240/2240465.ipynb", "difficulty": "medium", "instruction_sha256": "2e3b1883858317d0a95006d8cd0b49627eb1581fd16a56a3d18a3fd8b783bd8e"}, + {"name": "0119_045_119045887_qa_3", "source": "0119/045/119045887.ipynb_qa_3", "notebook": "0119/045/119045887.ipynb", "difficulty": "hard", "instruction_sha256": "9eec201729be0b7af6c485e96f37d61ed838f4e95f95ec8d52489c91537c325d"}, + {"name": "0075_579_75579600_qa_4", "source": "0075/579/75579600.ipynb_qa_4", "notebook": "0075/579/75579600.ipynb", "difficulty": "medium", "instruction_sha256": "8dca9146b4b4a37191e81fdba14507c0af8af4915d2b792a657bd24ce1c18f50"}, + {"name": "0029_204_29204719_qa_1", "source": "0029/204/29204719.ipynb_qa_1", "notebook": "0029/204/29204719.ipynb", "difficulty": "hard", "instruction_sha256": "14c745894cfd3a17ca7dc7e22b86de2f34179f2b884b3cfcc932fb8a5b9d222d"}, + {"name": "0070_655_70655259_qa_5", "source": "0070/655/70655259.ipynb_qa_5", "notebook": "0070/655/70655259.ipynb", "difficulty": "medium", "instruction_sha256": "1df4137d0d6be97ee758aa80f7689eaf786b7c70bceea0942dc1a4005632ed72"}, + {"name": "0093_813_93813737_qa_1", "source": "0093/813/93813737.ipynb_qa_1", "notebook": "0093/813/93813737.ipynb", "difficulty": "hard", "instruction_sha256": "9d6d2537695535758f895bc4ffae7a9965f68bd459effe12fe4760e2346ab1fb"}, + {"name": "0076_310_76310705_qa_3", "source": "0076/310/76310705.ipynb_qa_3", "notebook": "0076/310/76310705.ipynb", "difficulty": "medium", "instruction_sha256": "1f38d09e993145cc0114c909d923e59b1faa7098a5eb00afe088aa0a311d1039"}, + {"name": "0080_653_80653569_qa_2", "source": "0080/653/80653569.ipynb_qa_2", "notebook": "0080/653/80653569.ipynb", "difficulty": "hard", "instruction_sha256": "2d4b45c28dee6874a11378b765577d994452ff012548cd45ed23c5772ec25864"}, + {"name": "0079_597_79597687_qa_5", "source": "0079/597/79597687.ipynb_qa_5", "notebook": "0079/597/79597687.ipynb", "difficulty": "hard", "instruction_sha256": "d6eb7c058787701960d2973183b5a201b33823b7f40e866f8dcccfd533ff5f7f"}, + {"name": "0112_690_112690397_qa_5", "source": "0112/690/112690397.ipynb_qa_5", "notebook": "0112/690/112690397.ipynb", "difficulty": "hard", "instruction_sha256": "615726a5a4df2f4d62b03fd9c482ee884c9db336d25aac8d2bda46888a324623"}, + {"name": "0103_724_103724181_qa_4", "source": "0103/724/103724181.ipynb_qa_4", "notebook": "0103/724/103724181.ipynb", "difficulty": "medium", "instruction_sha256": "8be793b7301f545ea6f53f053a938469e74b7b93d58c12d6dc8e109f427a6c1a"}, + {"name": "0052_465_52465579_qa_3", "source": "0052/465/52465579.ipynb_qa_3", "notebook": "0052/465/52465579.ipynb", "difficulty": "medium", "instruction_sha256": "d60dc917a8113cca89f127eb8bdde08a4a2960c9b9c1bb985207cd41d769b756"}, + {"name": "0066_820_66820736_qa_3", "source": "0066/820/66820736.ipynb_qa_3", "notebook": "0066/820/66820736.ipynb", "difficulty": "hard", "instruction_sha256": "4c2b0a43283dc028dc6f17650145a584b23b01e9e3029ba0ac7f59fb9b5c954f"}, + {"name": "0098_812_98812520_qa_4", "source": "0098/812/98812520.ipynb_qa_4", "notebook": "0098/812/98812520.ipynb", "difficulty": "hard", "instruction_sha256": "999ee4bb510ef092ff0a94a62ba7dff87db8fcc6a705992422090afbff1013b3"}, + {"name": "0052_026_52026780_qa_3", "source": "0052/026/52026780.ipynb_qa_3", "notebook": "0052/026/52026780.ipynb", "difficulty": "hard", "instruction_sha256": "a2e0a5b2dd5295fd07418167041becff9a8534ae01ff05baa0fb2aab11b01732"}, + {"name": "0044_401_44401900_qa_2", "source": "0044/401/44401900.ipynb_qa_2", "notebook": "0044/401/44401900.ipynb", "difficulty": "medium", "instruction_sha256": "74ab8dc579c8e1586cd407dca32917706aa6b2668db4ed0a7f01bdc18e002f1c"}, + {"name": "0062_755_62755543_qa_4", "source": "0062/755/62755543.ipynb_qa_4", "notebook": "0062/755/62755543.ipynb", "difficulty": "hard", "instruction_sha256": "b0abb92ed9801f122c84236a8d1e95a39bc150351411c8d1e22343d3265536b6"}, + {"name": "0000_526_526258_qa_2", "source": "0000/526/526258.ipynb_qa_2", "notebook": "0000/526/526258.ipynb", "difficulty": "hard", "instruction_sha256": "0b06956a35a16588f84ccd46bf66ef1c82a3c2ccbe41a13ebb899624921f096d"}, + {"name": "0090_285_90285188_qa_5", "source": "0090/285/90285188.ipynb_qa_5", "notebook": "0090/285/90285188.ipynb", "difficulty": "hard", "instruction_sha256": "fca23d031360bd4b7487f3cb1c88726484760c565041d197b2985ff8de684ab8"}, + {"name": "0087_222_87222139_qa_1", "source": "0087/222/87222139.ipynb_qa_1", "notebook": "0087/222/87222139.ipynb", "difficulty": "hard", "instruction_sha256": "23407e9f45c78816cb729f48bc37a13a1092e83f8dd54c0d8b808c2193c779a7"}, + {"name": "0104_905_104905979_qa_1", "source": "0104/905/104905979.ipynb_qa_1", "notebook": "0104/905/104905979.ipynb", "difficulty": "hard", "instruction_sha256": "245980b86f1749c90b19a5e07dbc521342f64eb29e69795e91590b9ce3c02093"}, + {"name": "0039_708_39708837_qa_1", "source": "0039/708/39708837.ipynb_qa_1", "notebook": "0039/708/39708837.ipynb", "difficulty": "hard", "instruction_sha256": "bef4d7a2a52ee51ea321b48676c560f8d2f78d98104a216c426a5b9c2ab07b44"}, + {"name": "0014_367_14367970_qa_1", "source": "0014/367/14367970.ipynb_qa_1", "notebook": "0014/367/14367970.ipynb", "difficulty": "medium", "instruction_sha256": "7122a5739345732b2be6816b8306e3812139f4dc779a376b9de01fbca71e19c1"}, + {"name": "0139_186_139186690_qa_1", "source": "0139/186/139186690.ipynb_qa_1", "notebook": "0139/186/139186690.ipynb", "difficulty": "hard", "instruction_sha256": "a6b65a85d68fd501736cef2b9b8ef0e7c67c34291d71b4c7d4cb0cfea03cc351"}, + {"name": "0051_169_51169168_qa_3", "source": "0051/169/51169168.ipynb_qa_3", "notebook": "0051/169/51169168.ipynb", "difficulty": "medium", "instruction_sha256": "2c1fce245888b9be2fa734525204beea587f812dea175a1be8bb10f3048af916"}, + {"name": "0001_579_1579979_qa_2", "source": "0001/579/1579979.ipynb_qa_2", "notebook": "0001/579/1579979.ipynb", "difficulty": "medium", "instruction_sha256": "a67cd24885e36fc86f16f4a1a68b4ae84196aed4c315a98208311eba309bb662"}, + {"name": "0038_899_38899462_qa_2", "source": "0038/899/38899462.ipynb_qa_2", "notebook": "0038/899/38899462.ipynb", "difficulty": "hard", "instruction_sha256": "205b61250629c6618417700889a9ab5bc73d934c6a3458577730824cf66cb637"}, + {"name": "0082_759_82759610_qa_5", "source": "0082/759/82759610.ipynb_qa_5", "notebook": "0082/759/82759610.ipynb", "difficulty": "hard", "instruction_sha256": "680a33700b89a33e166763c460f0d2b562e920a7a4cc877b7ddd25ebb315b0a6"}, + {"name": "0033_558_33558610_qa_3", "source": "0033/558/33558610.ipynb_qa_3", "notebook": "0033/558/33558610.ipynb", "difficulty": "hard", "instruction_sha256": "3c38aeded5f6096b4c89e580b6f695d6ab4aed824741969437dd4a8d4d207dd4"}, + {"name": "0139_181_139181470_qa_3", "source": "0139/181/139181470.ipynb_qa_3", "notebook": "0139/181/139181470.ipynb", "difficulty": "hard", "instruction_sha256": "657458823b2dfb907adffead4ccc4ea6c59799c613472735ff8abe16e195305b"}, + {"name": "0065_756_65756425_qa_1", "source": "0065/756/65756425.ipynb_qa_1", "notebook": "0065/756/65756425.ipynb", "difficulty": "medium", "instruction_sha256": "7e4c6dc2acd775b1f6bc27aeba1ef941958fb47bf08f0191035dfebe4f4f6b2a"}, + {"name": "0050_281_50281280_qa_1", "source": "0050/281/50281280.ipynb_qa_1", "notebook": "0050/281/50281280.ipynb", "difficulty": "hard", "instruction_sha256": "53fbb8ac2998022461b29af2e709e9d13ee9a55a1c79defe1c9a0e2f0c113feb"}, + {"name": "0103_407_103407932_qa_1", "source": "0103/407/103407932.ipynb_qa_1", "notebook": "0103/407/103407932.ipynb", "difficulty": "hard", "instruction_sha256": "7f2f88dd4bf321fe82832d2ba550552ed382f93e4cf765e9c32a0b46dd4a93a8"}, + {"name": "0036_916_36916974_qa_1", "source": "0036/916/36916974.ipynb_qa_1", "notebook": "0036/916/36916974.ipynb", "difficulty": "hard", "instruction_sha256": "6f64c47adbf4b436414e9fc893046d4819e60ad4252123aee0e359f3ff0b18a2"}, + {"name": "0035_236_35236430_qa_3", "source": "0035/236/35236430.ipynb_qa_3", "notebook": "0035/236/35236430.ipynb", "difficulty": "hard", "instruction_sha256": "3c457effc43bd0a8460fac9b94789464426ea8ebf78234f735f21310c2ab5a98"}, + {"name": "0084_135_84135613_qa_1", "source": "0084/135/84135613.ipynb_qa_1", "notebook": "0084/135/84135613.ipynb", "difficulty": "medium", "instruction_sha256": "f57dac35e5d8a6f248f392e4b1e6f51fb2a963593fe8081877256d48164d9365"}, + {"name": "0065_166_65166426_qa_3", "source": "0065/166/65166426.ipynb_qa_3", "notebook": "0065/166/65166426.ipynb", "difficulty": "medium", "instruction_sha256": "202caa77a912c318b098e47b0bf45f24e4370eb367334bb6d00c6f65b71894dd"}, + {"name": "0114_986_114986805_qa_2", "source": "0114/986/114986805.ipynb_qa_2", "notebook": "0114/986/114986805.ipynb", "difficulty": "hard", "instruction_sha256": "7131f0db67b6e9d7179ba9be5618b49722596cb3dc46d9537f1fe6628c649f7d"}, + {"name": "0107_118_107118417_qa_3", "source": "0107/118/107118417.ipynb_qa_3", "notebook": "0107/118/107118417.ipynb", "difficulty": "medium", "instruction_sha256": "412bb5e8465ab7598c0f8a8eb997890aed6ad2aa9c6149056abb800c39b3e007"}, + {"name": "0034_376_34376500_qa_4", "source": "0034/376/34376500.ipynb_qa_4", "notebook": "0034/376/34376500.ipynb", "difficulty": "hard", "instruction_sha256": "fbca36a123a92f343009b60399103a1d098b1ef3ce13a6e15eca7a4f91d76745"}, + {"name": "0135_380_135380654_qa_4", "source": "0135/380/135380654.ipynb_qa_4", "notebook": "0135/380/135380654.ipynb", "difficulty": "hard", "instruction_sha256": "5d027a5724f1a206ab76ce18977dfc0b79b3438dc8f0ce3e6690a8de7c657446"}, + {"name": "0047_591_47591946_qa_1", "source": "0047/591/47591946.ipynb_qa_1", "notebook": "0047/591/47591946.ipynb", "difficulty": "hard", "instruction_sha256": "28138371808bcf4807e2c9c051afe8d948fdbf56270cabe4dfe9ea26d12d90c1"}, + {"name": "0031_852_31852116_qa_2", "source": "0031/852/31852116.ipynb_qa_2", "notebook": "0031/852/31852116.ipynb", "difficulty": "hard", "instruction_sha256": "859e4a61bae4ca99c06a24a6359a57d23056f75267bb9dafdbc4a2435e6e0cc6"}, + {"name": "0104_115_104115755_qa_3", "source": "0104/115/104115755.ipynb_qa_3", "notebook": "0104/115/104115755.ipynb", "difficulty": "medium", "instruction_sha256": "5825b57a6ab282ff8327f4f30b0736a1d09f460510e9619ec89f19ce307483b8"}, + {"name": "0105_131_105131890_qa_1", "source": "0105/131/105131890.ipynb_qa_1", "notebook": "0105/131/105131890.ipynb", "difficulty": "medium", "instruction_sha256": "080b52588c526725a61f01d45705a23e7a1e3bc30df1bad62c74159291d00bcc"}, + {"name": "0077_578_77578267_qa_4", "source": "0077/578/77578267.ipynb_qa_4", "notebook": "0077/578/77578267.ipynb", "difficulty": "medium", "instruction_sha256": "b17e5c98e1cb6cb587f3e800054f85bbe6dc39cbfa079e0a49ff63645cf85fb1"}, + {"name": "0047_591_47591946_qa_3", "source": "0047/591/47591946.ipynb_qa_3", "notebook": "0047/591/47591946.ipynb", "difficulty": "hard", "instruction_sha256": "514ee212ec79d52cd0bc0e4c2387913f38ecd2321bc9de73700863dd0016abed"}, + {"name": "0125_581_125581362_qa_5", "source": "0125/581/125581362.ipynb_qa_5", "notebook": "0125/581/125581362.ipynb", "difficulty": "hard", "instruction_sha256": "e1230f44a72f5109406dfb0b575cb4af9246bfb5311a7c0c7cc50704fbacc0b7"}, + {"name": "0021_336_21336465_qa_2", "source": "0021/336/21336465.ipynb_qa_2", "notebook": "0021/336/21336465.ipynb", "difficulty": "medium", "instruction_sha256": "2a0169dc568ec583a6002da083408b2fe8a09d2f0c05116232cbfe9a5cfdc0b4"}, + {"name": "0036_637_36637598_qa_4", "source": "0036/637/36637598.ipynb_qa_4", "notebook": "0036/637/36637598.ipynb", "difficulty": "hard", "instruction_sha256": "f19672a3e18767aae7374c7bd487fcff95dd0676470f277a25741364e84d343e"}, + {"name": "0075_210_75210599_qa_3", "source": "0075/210/75210599.ipynb_qa_3", "notebook": "0075/210/75210599.ipynb", "difficulty": "hard", "instruction_sha256": "a81bd2b53e57f8c8848b7854d9d192d7bed8f0a11f1ae4755230ee1dc05ee0a5"}, + {"name": "0057_476_57476983_qa_4", "source": "0057/476/57476983.ipynb_qa_4", "notebook": "0057/476/57476983.ipynb", "difficulty": "hard", "instruction_sha256": "1ea3861d89107dd47d5255868260c0643f4d06eda6338eb934c5589e8ee1316d"}, + {"name": "0052_546_52546501_qa_4", "source": "0052/546/52546501.ipynb_qa_4", "notebook": "0052/546/52546501.ipynb", "difficulty": "hard", "instruction_sha256": "2a237719f32aa1b432d8df7101892d23aeaea59ef6fd0ddf38a1eb5de080d7aa"}, + {"name": "0001_074_1074738_qa_1", "source": "0001/074/1074738.ipynb_qa_1", "notebook": "0001/074/1074738.ipynb", "difficulty": "medium", "instruction_sha256": "3a745062096d53056a67c66d281191ff7f15b9370dcac75f997b986620598cd2"}, + {"name": "0042_124_42124908_qa_1", "source": "0042/124/42124908.ipynb_qa_1", "notebook": "0042/124/42124908.ipynb", "difficulty": "hard", "instruction_sha256": "e084112d74e00d89da032ec9112d3d76b9bdcd0dfd4fb4d2d6d7cb92c8261fda"}, + {"name": "0057_771_57771025_qa_3", "source": "0057/771/57771025.ipynb_qa_3", "notebook": "0057/771/57771025.ipynb", "difficulty": "hard", "instruction_sha256": "fc02527a1d0ef1395910ef6b804abfa994ce8a8cf74e7471e356e4082bbb7fcc"}, + {"name": "0113_431_113431641_qa_1", "source": "0113/431/113431641.ipynb_qa_1", "notebook": "0113/431/113431641.ipynb", "difficulty": "medium", "instruction_sha256": "186730e29202845520f4d00fb6ea0a0a3403ac7b36f18bfaae4ea1ce18fef58d"}, + {"name": "0059_864_59864963_qa_3", "source": "0059/864/59864963.ipynb_qa_3", "notebook": "0059/864/59864963.ipynb", "difficulty": "hard", "instruction_sha256": "5cca79e4ea43d04385b8adf3fa8b5e100f6f50262971c3e574775a347d2c4a03"}, + {"name": "0106_409_106409576_qa_3", "source": "0106/409/106409576.ipynb_qa_3", "notebook": "0106/409/106409576.ipynb", "difficulty": "hard", "instruction_sha256": "4d07208d9cd96556640b45f7bcf0817fbfa3790631f58088932dc806243807e0"}, + {"name": "0123_785_123785154_qa_3", "source": "0123/785/123785154.ipynb_qa_3", "notebook": "0123/785/123785154.ipynb", "difficulty": "medium", "instruction_sha256": "5d664c2d1fc82ae9e7535c3896a5794247428a523c106975891e6e0cb5ccd9b7"}, + {"name": "0099_268_99268253_qa_1", "source": "0099/268/99268253.ipynb_qa_1", "notebook": "0099/268/99268253.ipynb", "difficulty": "hard", "instruction_sha256": "bcd80d14b8f44f6353ecc8224ad19fd72e47eebcf65e3063badaa6dc8ec96e9b"}, + {"name": "0094_028_94028594_qa_1", "source": "0094/028/94028594.ipynb_qa_1", "notebook": "0094/028/94028594.ipynb", "difficulty": "hard", "instruction_sha256": "85b241fa27832d786471cc738e73bd17539a709ee096421a32520821014e83a6"}, + {"name": "0039_667_39667959_qa_5", "source": "0039/667/39667959.ipynb_qa_5", "notebook": "0039/667/39667959.ipynb", "difficulty": "medium", "instruction_sha256": "bbe4f894aaa7435d3ef28f672841247a74f46a43c01184f8e2ee99b614914393"}, + {"name": "0024_252_24252070_qa_2", "source": "0024/252/24252070.ipynb_qa_2", "notebook": "0024/252/24252070.ipynb", "difficulty": "hard", "instruction_sha256": "bcf6fc001a3121057b73b9da559e43a65c2bc52e984525eb51c02d601e9537ea"}, + {"name": "0076_340_76340399_qa_5", "source": "0076/340/76340399.ipynb_qa_5", "notebook": "0076/340/76340399.ipynb", "difficulty": "medium", "instruction_sha256": "3fdf32afc2d1375b3c12b0a755aec76ebccf1e6255d8d97a1330915372e96131"}, + {"name": "0065_988_65988202_qa_2", "source": "0065/988/65988202.ipynb_qa_2", "notebook": "0065/988/65988202.ipynb", "difficulty": "hard", "instruction_sha256": "e9f7c57eda61c1fb5b5930666f14738afb70918884a9259b51d9c3177969e5c2"}, + {"name": "0065_929_65929308_qa_5", "source": "0065/929/65929308.ipynb_qa_5", "notebook": "0065/929/65929308.ipynb", "difficulty": "hard", "instruction_sha256": "4ee2585888f301427c080d35e5216eaa9d70c8fea820ec46bcca689e0c139fdf"}, + {"name": "0110_774_110774218_qa_3", "source": "0110/774/110774218.ipynb_qa_3", "notebook": "0110/774/110774218.ipynb", "difficulty": "medium", "instruction_sha256": "71ff0f97c1d9dba09a65e2c30beddae0c1fa144483957d0973e9e5c8a16bdefd"}, + {"name": "0054_541_54541486_qa_1", "source": "0054/541/54541486.ipynb_qa_1", "notebook": "0054/541/54541486.ipynb", "difficulty": "medium", "instruction_sha256": "8e897a158298c88c64f4105d1f25085d2e58184a52574e441cb5cb0d5728cd42"}, + {"name": "0041_018_41018707_qa_1", "source": "0041/018/41018707.ipynb_qa_1", "notebook": "0041/018/41018707.ipynb", "difficulty": "hard", "instruction_sha256": "c3b770bbd3fa245572f3ea8d3c5e494d10c27043fb4e98a0e0fcdd90504dc89e"}, + {"name": "0048_357_48357285_qa_5", "source": "0048/357/48357285.ipynb_qa_5", "notebook": "0048/357/48357285.ipynb", "difficulty": "hard", "instruction_sha256": "bfb787c92f924f61b41fe5d159c0dc4d1a267d9dd0620a306210bef0963ca868"}, + {"name": "0001_660_1660748_qa_2", "source": "0001/660/1660748.ipynb_qa_2", "notebook": "0001/660/1660748.ipynb", "difficulty": "medium", "instruction_sha256": "9f4deaa8223416d843da4ba61d468409eb26a28888d4b31aec290476645ddee9"}, + {"name": "0088_548_88548025_qa_1", "source": "0088/548/88548025.ipynb_qa_1", "notebook": "0088/548/88548025.ipynb", "difficulty": "hard", "instruction_sha256": "69a885906f54a1c522ce1d779274cfe334bd2765f7cc8dad31bc72d11dcbfb22"}, + {"name": "0014_579_14579474_qa_3", "source": "0014/579/14579474.ipynb_qa_3", "notebook": "0014/579/14579474.ipynb", "difficulty": "medium", "instruction_sha256": "2bb8d6eb3fb190649dd866a7844338e8b242953964a2751192630e66e4c407ea"}, + {"name": "0138_302_138302934_qa_3", "source": "0138/302/138302934.ipynb_qa_3", "notebook": "0138/302/138302934.ipynb", "difficulty": "medium", "instruction_sha256": "9de62086cb531840ecf1b7043ab44af8cb1deb0f1d7ca1c27a6a14f421ff784e"}, + {"name": "0040_776_40776095_qa_2", "source": "0040/776/40776095.ipynb_qa_2", "notebook": "0040/776/40776095.ipynb", "difficulty": "hard", "instruction_sha256": "a5932dbb71758ceca9ddb2d0eadd5312180e22bc9d171a61724145b45399c1b6"}, + {"name": "0018_395_18395380_qa_5", "source": "0018/395/18395380.ipynb_qa_5", "notebook": "0018/395/18395380.ipynb", "difficulty": "medium", "instruction_sha256": "d26f04e7f7e9defd6093f9a27c6dd3f140100274cbf0feb21cd2bd4a2d7ba635"}, + {"name": "0089_255_89255842_qa_4", "source": "0089/255/89255842.ipynb_qa_4", "notebook": "0089/255/89255842.ipynb", "difficulty": "hard", "instruction_sha256": "eb643aa40eb949fd06c6954460be09348fc4d7f1e7f727cd75a5d39b47bd73e7"}, + {"name": "0056_410_56410958_qa_1", "source": "0056/410/56410958.ipynb_qa_1", "notebook": "0056/410/56410958.ipynb", "difficulty": "hard", "instruction_sha256": "297ed4204edd95c0bb4256b03b471e863c7a910923de0327f9656cb16f1f2025"}, + {"name": "0038_692_38692630_qa_4", "source": "0038/692/38692630.ipynb_qa_4", "notebook": "0038/692/38692630.ipynb", "difficulty": "hard", "instruction_sha256": "8fa1552fbedb2ef3c72ed94d5e091ca11000b3e2f4fa0891cd0b8400aa319d82"}, + {"name": "0046_905_46905660_qa_1", "source": "0046/905/46905660.ipynb_qa_1", "notebook": "0046/905/46905660.ipynb", "difficulty": "medium", "instruction_sha256": "f76124fa688dc023adb493532405f71205ac3d412ba64a5df7d6d313ef133394"}, + {"name": "0016_714_16714217_qa_4", "source": "0016/714/16714217.ipynb_qa_4", "notebook": "0016/714/16714217.ipynb", "difficulty": "medium", "instruction_sha256": "4e2c527426a98bf19880a5013328690ea02abf8b73b6e76db55582cf895839ad"}, + {"name": "0040_929_40929691_qa_1", "source": "0040/929/40929691.ipynb_qa_1", "notebook": "0040/929/40929691.ipynb", "difficulty": "hard", "instruction_sha256": "ed04d1895994e38114822e6ab837359dbc668fd2007354acf456711b4f6440be"}, + {"name": "0028_007_28007632_qa_1", "source": "0028/007/28007632.ipynb_qa_1", "notebook": "0028/007/28007632.ipynb", "difficulty": "hard", "instruction_sha256": "6ce9e1d51fdc9ac6ffa5628952dccccd39a8f813c925fd52cf6d81ebfd3f501c"}, + {"name": "0067_492_67492049_qa_4", "source": "0067/492/67492049.ipynb_qa_4", "notebook": "0067/492/67492049.ipynb", "difficulty": "hard", "instruction_sha256": "f7bc2233c12ff48343bbb774b55c9cb076afb49d5c65d0df940b354bc7d80192"}, + {"name": "0093_490_93490141_qa_3", "source": "0093/490/93490141.ipynb_qa_3", "notebook": "0093/490/93490141.ipynb", "difficulty": "hard", "instruction_sha256": "ed82cf207e6857658f9874c9fef50f139c0d917f96e4ac0b8d78ff6afe0208f5"}, + {"name": "0011_054_11054618_qa_1", "source": "0011/054/11054618.ipynb_qa_1", "notebook": "0011/054/11054618.ipynb", "difficulty": "medium", "instruction_sha256": "1f0947ffe4e56153452364ad87d9ee1a8fbbb1524131b628b09e94332f0174ac"}, + {"name": "0091_296_91296853_qa_1", "source": "0091/296/91296853.ipynb_qa_1", "notebook": "0091/296/91296853.ipynb", "difficulty": "hard", "instruction_sha256": "57ee3b336365e0be93f5a131e6de4c98c0dcebbd5c9794ecc2c446b7e1b0dad0"}, + {"name": "0067_990_67990197_qa_1", "source": "0067/990/67990197.ipynb_qa_1", "notebook": "0067/990/67990197.ipynb", "difficulty": "medium", "instruction_sha256": "9f7a3b411717713c1bfd4ce2692fae3554586f33e5b5166f8f6b3e5732bb334d"}, + {"name": "0053_910_53910928_qa_5", "source": "0053/910/53910928.ipynb_qa_5", "notebook": "0053/910/53910928.ipynb", "difficulty": "hard", "instruction_sha256": "5200a161448e1f16ac96255315395fecfd949219565217eae6ad6aacb8280850"}, + {"name": "0088_822_88822681_qa_1", "source": "0088/822/88822681.ipynb_qa_1", "notebook": "0088/822/88822681.ipynb", "difficulty": "medium", "instruction_sha256": "2e1f876874f1282defa806195d830b711adb9acab1a4d527f55e724dd16727b6"}, + {"name": "0001_527_1527054_qa_2", "source": "0001/527/1527054.ipynb_qa_2", "notebook": "0001/527/1527054.ipynb", "difficulty": "medium", "instruction_sha256": "3e1ec2b4e0ab278dbb876521fb4b1551ad0b79a63a53d29852ffda60fd78e033"}, + {"name": "0106_834_106834319_qa_1", "source": "0106/834/106834319.ipynb_qa_1", "notebook": "0106/834/106834319.ipynb", "difficulty": "medium", "instruction_sha256": "637da8811aee08d64f5c6e19ecc9e0083464a3c6f2bedcfe9a6b7ad37fa746f2"}, + {"name": "0023_626_23626093_qa_1", "source": "0023/626/23626093.ipynb_qa_1", "notebook": "0023/626/23626093.ipynb", "difficulty": "hard", "instruction_sha256": "3fa61ab7b1454faf531802086d701d246c0b232c1bd724a715714445462bc3ec"}, + {"name": "0021_121_21121885_qa_3", "source": "0021/121/21121885.ipynb_qa_3", "notebook": "0021/121/21121885.ipynb", "difficulty": "hard", "instruction_sha256": "098920eaef9216d6d5d10fa2e607c19126e1336a6509c29b612f596fd46ccdbf"}, + {"name": "0085_309_85309051_qa_1", "source": "0085/309/85309051.ipynb_qa_1", "notebook": "0085/309/85309051.ipynb", "difficulty": "hard", "instruction_sha256": "2a9357a973f34e9c864046771c0c2b3e1b6f85052a24000a14b5310bd012a37a"}, + {"name": "0113_768_113768809_qa_5", "source": "0113/768/113768809.ipynb_qa_5", "notebook": "0113/768/113768809.ipynb", "difficulty": "hard", "instruction_sha256": "edf0e83dfac4ae49f5a612f167576c8b5d8128a94447838fed67c499299607af"}, + {"name": "0054_065_54065855_qa_5", "source": "0054/065/54065855.ipynb_qa_5", "notebook": "0054/065/54065855.ipynb", "difficulty": "hard", "instruction_sha256": "dc30fa15604f467cd7443abc7e601fe4ac6a3d0cca7e4bae4416f45492ed0f40"}, + {"name": "0040_288_40288007_qa_4", "source": "0040/288/40288007.ipynb_qa_4", "notebook": "0040/288/40288007.ipynb", "difficulty": "hard", "instruction_sha256": "c6434e6c5b002a021f382e21ea804e02087cbdf60f669b62e902513bd8b713ef"}, + {"name": "0044_412_44412230_qa_1", "source": "0044/412/44412230.ipynb_qa_1", "notebook": "0044/412/44412230.ipynb", "difficulty": "medium", "instruction_sha256": "fbc3a4777712e9e1de7f8199e0c7bb0a50d80c9579d53e472a6d80d1f1f5805d"}, + {"name": "0083_191_83191425_qa_3", "source": "0083/191/83191425.ipynb_qa_3", "notebook": "0083/191/83191425.ipynb", "difficulty": "hard", "instruction_sha256": "88c6821b7b33ffcdaaddde57e4e701b73291a2cd3888010a2f69b86cc2286f83"}, + {"name": "0119_666_119666168_qa_2", "source": "0119/666/119666168.ipynb_qa_2", "notebook": "0119/666/119666168.ipynb", "difficulty": "medium", "instruction_sha256": "133f73998e7587118d354b8fd0dd9171cb41df81684a052a06a53e0a0cdc2f9e"}, + {"name": "0122_390_122390551_qa_4", "source": "0122/390/122390551.ipynb_qa_4", "notebook": "0122/390/122390551.ipynb", "difficulty": "medium", "instruction_sha256": "65b7c9fd25993e900ac938312bbf40001e94b5a96c2a8dcbc6016ffdccda7687"}, + {"name": "0132_865_132865792_qa_4", "source": "0132/865/132865792.ipynb_qa_4", "notebook": "0132/865/132865792.ipynb", "difficulty": "hard", "instruction_sha256": "f6b16926f5640b7629204c2e484163c4c6e82083f77a75b21730cd9c1e06f04a"}, + {"name": "0122_517_122517438_qa_3", "source": "0122/517/122517438.ipynb_qa_3", "notebook": "0122/517/122517438.ipynb", "difficulty": "hard", "instruction_sha256": "4b3e13801dec3a4d66e9accbb769f2f5efd7f5d70ab682c7e9bb34535e895af6"}, + {"name": "0076_259_76259625_qa_4", "source": "0076/259/76259625.ipynb_qa_4", "notebook": "0076/259/76259625.ipynb", "difficulty": "hard", "instruction_sha256": "1cff4c1b3495bd193958157ea96a1b1354c9ec9ef8380d46d1e2bffc85941534"} +] diff --git a/05-multi-harness-rl/eval/evaluate.py b/05-multi-harness-rl/eval/evaluate.py new file mode 100644 index 00000000..01f59fc4 --- /dev/null +++ b/05-multi-harness-rl/eval/evaluate.py @@ -0,0 +1,165 @@ +"""Pass@1 with durable per-pair results and retries only for ungraded infrastructure failures.""" +from concurrent.futures import ThreadPoolExecutor, as_completed +import json +import math +from pathlib import Path +import time +import uuid + +from recipe import digest, summary, task_rows, write_json +from train.adapters import Factory + + +def capture_metrics(result): + turns = [t for t in result.turns if t.trainable] + if not turns: + raise ValueError("No trainable capture from the token-enabled engine") + for turn in turns: + prompt, tokens, logps = turn.prompt_token_ids, turn.completion_token_ids, turn.per_token_logps + mask = turn.loss_mask + if not prompt or not tokens or len(tokens) != len(logps) or not all(math.isfinite(x) for x in logps): + raise ValueError("Invalid captured token/logprob alignment") + if mask is None or len(mask) != len(prompt) + len(tokens) or any(mask[:len(prompt)]): + raise ValueError("Invalid full-sequence loss mask") + if set(mask) - {0, 1} or not any(mask[len(prompt):]): + raise ValueError("No valid supervised completion span") + return {"tito_pass": True, "model_calls": len(turns), + "generated_tokens": sum(len(t.completion_token_ids) for t in turns), + "prompt_tokens": sum(len(t.prompt_token_ids) for t in turns)} + + +def blackbox_episode(factory, group_id): + group = factory.groups[group_id] + session = factory.create(factory.rows()[group_id]["prompt"], seed=group_id, episode_id=uuid.uuid4().hex) + try: + session.wait_for_completion(timeout_s=factory.cfg["agent_timeout_sec"] + 600) + session.verify([]) + record = json.loads((Path(factory.cfg["output"]) / "rollouts" / f'{session.evidence["episode_id"]}.json').read_text()) + return {**record, **capture_metrics(session.result)} + finally: + session.close() + + +def whitebox_episode(cfg, server, vllm, index): + from openai import OpenAI + from transformers.utils import get_json_schema + from whitebox_bash import white_box_bash_env + import inspect + from train.whitebox import SYSTEM + + env = white_box_bash_env(server, split="test", toolsets="bash,seta", step_limit=cfg["agent_step_limit"])() + client = OpenAI(base_url=vllm.rstrip("/") + "/v1", api_key="unused", timeout=cfg["agent_timeout_sec"]) + calls, generated, inputs = 0, 0, 0 + try: + text = env.reset(split="test", index=index) + tools = [get_json_schema(m) for name, m in inspect.getmembers(env, inspect.ismethod) + if not name.startswith("_") and name not in {"reset", "get_reward"}] + messages = [{"role": "system", "content": SYSTEM}, {"role": "user", "content": text}] + for _ in range(cfg["agent_step_limit"]): + response = client.chat.completions.create(model=cfg["profile"]["id"], messages=messages, tools=tools, + temperature=cfg["temperature"], top_p=1.0, max_tokens=cfg["max_output_tokens"], + extra_body={"chat_template_kwargs": {"enable_thinking": False, "preserve_thinking": True}}) + generated += response.usage.completion_tokens + inputs += response.usage.prompt_tokens + msg = response.choices[0].message + messages.append(msg.model_dump(exclude_none=True)) + if not msg.tool_calls: + break + submitted = False + for call in msg.tool_calls: + allowed = {t["function"]["name"] for t in tools} + if call.function.name not in allowed: + raise ValueError("Model called an unexposed tool") + value = str(getattr(env, call.function.name)(**json.loads(call.function.arguments))) + calls += call.function.name != "submit_solution" + submitted |= call.function.name == "submit_solution" + messages.append({"role": "tool", "tool_call_id": call.id, "content": value}) + if submitted: + break + correctness = env.get_reward() + if not math.isfinite(correctness): + raise ValueError("Ungraded whitebox episode") + return {"correctness": float(correctness >= 1), "tool_calls": calls, + "generated_tokens": generated, "prompt_tokens": inputs} + finally: + try: + if env._session is not None: + env.get_reward() + finally: + env._mcp.close() + client.close() + + +def evaluate(cfg, data, server, vllm, *, limit=None, checkpoint="baseline", max_attempts=3): + out = Path(cfg["output"]) / "eval" + out.mkdir(parents=True, exist_ok=True) + tasks = sorted(task_rows("test"), key=lambda row: row["name"]) + selected = tasks[:limit] if limit else tasks + harnesses = ["whitebox"] if cfg["mode"] == "whitebox" else cfg["harnesses"] + groups = [{"task_name": row["name"], "task_index": i, "harness": h} + for i, row in enumerate(selected) for h in harnesses] + identity = {"model": cfg["profile"], "checkpoint": str(checkpoint), "tasks": selected, + "harnesses": harnesses, "template": cfg["profile"]["template"], "config": cfg} + signature = digest(identity) + identity_path = out / "identity.json" + if identity_path.exists() and json.loads(identity_path.read_text())["sha256"] != signature: + raise ValueError("Evaluation output belongs to a different checkpoint or protocol") + write_json(identity_path, {"sha256": signature, **identity}) + # Both blackbox policies use the same four Harbor evaluation harnesses. + factory = Factory({**cfg, "mode": "multi-harness"}, data, server, vllm, groups, + Path(cfg["output"]) / "trials", split="test") if cfg["mode"] != "whitebox" else None + + def one(item): + group_id, group = item + row = tasks[group["task_index"]] + path = out / "pairs" / f'{row["name"]}--{group["harness"]}.json' + if path.exists(): + prior = json.loads(path.read_text()) + if prior.get("correctness") in (0, 1) and not prior.get("error"): + return prior + for attempt in range(max_attempts): + started = time.time() + try: + result = (blackbox_episode(factory, group_id) if factory else + whitebox_episode(cfg, server, vllm, group["task_index"])) + if result.get("correctness") not in (0, 1): + raise ValueError("Verifier did not produce a binary grade") + record = {**group, "difficulty": row["difficulty"], **result, "attempt": attempt + 1, + "elapsed_sec": time.time() - started, "error": None} + except Exception as exc: + record = {**group, "difficulty": row["difficulty"], "correctness": None, + "error": type(exc).__name__, "attempt": attempt + 1} + write_json(path, record) + if record["correctness"] is not None: + return record + return record + + records = [] + with ThreadPoolExecutor(max_workers=cfg["eval_concurrency"]) as pool: + for future in as_completed([pool.submit(one, pair) for pair in enumerate(groups)]): + records.append(future.result()) + write_json(out / "progress.json", summary(records, len(groups))) + report = summary(records, len(groups)) + report["harnesses"] = {h: summary([r for r in records if r["harness"] == h], len(selected)) for h in harnesses} + report["difficulty"] = {d: summary([r for r in records if r["difficulty"] == d], + sum(t["difficulty"] == d for t in selected) * len(harnesses)) for d in ("easy", "medium", "hard")} + report["missing"] = [{"task_name": r["task_name"], "harness": r["harness"], "error": r["error"]} + for r in records if r["correctness"] is None] + write_json(out / "summary.json", report) + import trackio + run = trackio.init(project=cfg["project"], name=cfg["run_name"] + "-eval", + space_id=cfg["trackio_space_id"], config={"protocol_sha256": signature, "checkpoint": str(checkpoint)}) + step = 0 + if checkpoint != "baseline": + step = json.loads((Path(checkpoint) / "trainer_state.json").read_text())["global_step"] + metrics = {"eval/coverage": report["coverage"], "eval/graded": report["graded"]} + if report["pass_at_1"] is not None: + metrics["eval/pass_at_1_observed"] = report["pass_at_1"] + for harness, values in report["harnesses"].items(): + for key in ("pass_at_1", "coverage", "mean_tool_calls", "mean_generated_tokens"): + if values[key] is not None: + metrics[f"eval/{harness}/{key}"] = values[key] + run.log(metrics, step=step) + run.finish() + print(json.dumps(report, indent=2)) + return report diff --git a/05-multi-harness-rl/eval/watch.py b/05-multi-harness-rl/eval/watch.py new file mode 100644 index 00000000..4f132842 --- /dev/null +++ b/05-multi-harness-rl/eval/watch.py @@ -0,0 +1,99 @@ +"""Hash finished checkpoints and submit evaluations on separate GPU allocations.""" +import argparse +import fcntl +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from recipe import write_json +from runtime.checkpoints import make_ready + + +def scan(root): + for request in sorted(Path(root).glob("*/checkpoint-*/eval.request.json")): + checkpoint = request.parent + if not (checkpoint / "checkpoint.ready.json").exists(): + try: + make_ready(checkpoint) + except (OSError, ValueError) as exc: + print(json.dumps({"checkpoint": str(checkpoint), "status": "waiting_for_complete_files", + "error": type(exc).__name__}), flush=True) + continue + if json.loads(request.read_text())["evaluate"]: + yield checkpoint + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--root", type=Path, required=True) + p.add_argument("--backend", choices=["hf", "slurm"], required=True) + p.add_argument("--bucket") + p.add_argument("--namespace", default="FineEnvs") + p.add_argument("--partition") + p.add_argument("--flavor", default="h200x2") + p.add_argument("--data", type=Path, default=ROOT / "prepared") + p.add_argument("--concurrency", type=int, default=35) + p.add_argument("--max-active", type=int, default=1) + p.add_argument("--once", action="store_true") + p.add_argument("--submit", action="store_true") + args = p.parse_args() + args.root = args.root.resolve() + if args.backend == "hf" and not args.bucket: + p.error("--bucket is required for HF Jobs") + if args.backend == "slurm" and not args.partition: + p.error("--partition is required for Slurm") + if args.max_active < 1 or args.concurrency < 1: + p.error("Concurrency and max-active must be positive") + args.root.mkdir(parents=True, exist_ok=True) + state_path = args.root / "eval-watcher.json" + with (args.root / "eval-watcher.lock").open("w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + while True: + ledger = json.loads(state_path.read_text()) if state_path.exists() else {} + for key, entry in ledger.items(): + if entry["status"] != "active": + continue + if args.backend == "hf": + from huggingface_hub import inspect_job + stage = inspect_job(job_id=entry["id"], namespace=args.namespace).status.stage + if stage in {"COMPLETED", "ERROR", "CANCELED", "DELETED"}: + entry["status"] = "done" if stage == "COMPLETED" else "failed" + else: + status = subprocess.check_output(["sacct", "-j", entry["id"], "--noheader", "--format=State", "--parsable2"], text=True).splitlines() + if status and status[0] not in {"RUNNING", "PENDING", "CONFIGURING", "COMPLETING"}: + entry["status"] = "done" if status[0] == "COMPLETED" else "failed" + active = sum(e["status"] == "active" for e in ledger.values()) + for checkpoint in scan(args.root): + key = str(checkpoint.relative_to(args.root)) + if key in ledger or active >= args.max_active: + continue + cfg = json.loads((checkpoint / "recipe.json").read_text()) + name = f'{checkpoint.parent.name}-eval-{checkpoint.name}' + cmd = [sys.executable, str(ROOT / "runtime/launch.py"), args.backend, "eval", + "--mode", cfg["mode"], "--model", cfg["model"], "--run-name", name, + "--checkpoint", str(checkpoint), "--output-root", str(args.root), + "--config", str(checkpoint / "recipe.json"), + "--data", str(args.data), "--concurrency", str(args.concurrency)] + if args.backend == "hf": + cmd += ["--bucket", args.bucket, "--namespace", args.namespace, "--flavor", args.flavor] + else: + cmd += ["--partition", args.partition] + print(json.dumps({"checkpoint": key, "command": cmd}), flush=True) + if args.submit: + lines = subprocess.check_output(cmd + ["--submit"], text=True).strip().splitlines() + job_id = json.loads(lines[-1])["id"] if args.backend == "hf" else lines[-1].split(";")[0] + ledger[key] = {"id": job_id, "status": "active", "submitted_at": time.time()} + write_json(state_path, ledger) + active += 1 + write_json(state_path, ledger) + if args.once: + break + time.sleep(60) + + +if __name__ == "__main__": + main() diff --git a/05-multi-harness-rl/prepare.py b/05-multi-harness-rl/prepare.py new file mode 100644 index 00000000..dac6a8b4 --- /dev/null +++ b/05-multi-harness-rl/prepare.py @@ -0,0 +1,80 @@ +"""Materialize the fixed train/test tasks from corrected, pinned Harbor datasets.""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import shutil +import tomllib + +from recipe import ROOT, task_rows, write_json + +DATASETS = { + "train": ("FineEnvs/SmolDataEnvs-harbor-train", "4719635555de1666f374d847baebb500d368e493"), + "test": ("FineEnvs/SmolDataEnvs-harbor-test", "b130595b579fc6026ad5762925a9a3b40e72f5bb"), +} +GRADER_SHA256 = "16f906e9248fa8b7e1141939e425e88e59d52d65e98e360bbacef3720a43629f" + + +def prepare(destination, sources=None): + import tomli_w + from huggingface_hub import snapshot_download + + destination = Path(destination).resolve() + manifests = {} + for split in ("train", "test"): + rows = task_rows(split) + repo, revision = DATASETS[split] + source = (sources or {}).get(split) + if source is None: + source = snapshot_download(repo, repo_type="dataset", revision=revision, + allow_patterns=[f'tasks/{r["name"]}/*' for r in rows], max_workers=16) + source = Path(source) + tasks = [] + for row in rows: + folder = source / "tasks" / row["name"] + instruction = (folder / "instruction.md").read_bytes() + if hashlib.sha256(instruction).hexdigest() != row["instruction_sha256"]: + raise ValueError(f"Historical task instruction changed: {split}/{row['name']}") + grader = (folder / "tests/grader.py").read_bytes() + if hashlib.sha256(grader).hexdigest() != GRADER_SHA256: + raise ValueError(f"Expected corrected numeric-only Harbor grader: {folder}") + target = destination / "datasets" / split / "tasks" / row["name"] + target.mkdir(parents=True, exist_ok=True) + for rel in ("instruction.md", "task.toml", "environment/Dockerfile", "environment/pull_bucket.py", + "tests/grader.py", "tests/test.sh"): + dst = target / rel + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(folder / rel, dst) + spec = tomllib.loads((target / "task.toml").read_text()) + spec["environment"]["memory_mb"] = 4096 + spec["environment"]["cpus"] = 1 + (target / "task.toml").write_text(tomli_w.dumps(spec)) + hashes = {str(f.relative_to(destination / "datasets" / split)): hashlib.sha256(f.read_bytes()).hexdigest() + for f in sorted(target.rglob("*")) if f.is_file()} + tasks.append({**row, "file_hashes": hashes}) + local = (sources or {}).get(split) + manifest = {"source_dataset": repo, "source_revision": None if local else revision, + "expected_hub_revision": revision, "local_source": str(local) if local else None, + "task_count": len(tasks), "tasks": tasks} + write_json(destination / f"{split}_manifest.json", manifest) + manifests[split] = manifest + train = {r["source"] for r in manifests["train"]["tasks"]} + test = {r["source"] for r in manifests["test"]["tasks"]} + notebooks = [{r.get("notebook") for r in manifests[s]["tasks"]} - {None} for s in ("train", "test")] + if train & test or notebooks[0] & notebooks[1]: + raise ValueError("Train/test overlap") + write_json(destination / "ready.json", {"train": len(train), "test": len(test), + "grader_sha256": GRADER_SHA256, "train_test_overlap": 0}) + return manifests + + +if __name__ == "__main__": + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--output", type=Path, default=ROOT / "prepared") + p.add_argument("--train-source", type=Path) + p.add_argument("--test-source", type=Path) + a = p.parse_args() + result = prepare(a.output, {s: v for s, v in (("train", a.train_source), ("test", a.test_source)) if v}) + print(json.dumps({s: m["task_count"] for s, m in result.items()})) diff --git a/05-multi-harness-rl/project.yaml b/05-multi-harness-rl/project.yaml new file mode 100644 index 00000000..6491067d --- /dev/null +++ b/05-multi-harness-rl/project.yaml @@ -0,0 +1,32 @@ +name: multi-harness-rl +title: "SmolDataEnv: whitebox and multi-harness RL" +tagline: "One task split and reward across SETA, native OpenCode and Harbor; HF Jobs and Slurm recipes." +order: 5 +status: wip +hub: + org: FineEnvs + datasets: + - FineEnvs/SmolDataEnvs-harbor-train + - FineEnvs/SmolDataEnvs-harbor-test + spaces: + - FineEnvs/data-agent-seta-whitebox-env + - FineEnvs/data-agent-blackbox-opencode-env + - FineEnvs/data-agent-blackbox-harbor-env + trackio: + - FineEnvs/data-agent-training-comparison-trackio +envs: + - name: seta + summary: "Native bash/SETA tools driven by synchronous GRPO." + frameworks: + openenv: + verified: false + - name: native-opencode + summary: "Native OpenCode owns its loop; AsyncGRPO consumes captured tokens." + frameworks: + openenv: + verified: false + - name: harbor + summary: "Rotate four Harbor harnesses between task groups." + frameworks: + openenv: + verified: false diff --git a/05-multi-harness-rl/recipe.py b/05-multi-harness-rl/recipe.py new file mode 100644 index 00000000..94a926b0 --- /dev/null +++ b/05-multi-harness-rl/recipe.py @@ -0,0 +1,109 @@ +"""Shared configuration, task identity and reward policy.""" +from __future__ import annotations + +import hashlib +import json +import math +from pathlib import Path +import uuid + +ROOT = Path(__file__).resolve().parent +MODES = ("whitebox", "opencode", "multi-harness") + + +def write_json(path, value): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + try: + tmp.write_text(json.dumps(value, indent=2, allow_nan=False) + "\n") + tmp.replace(path) + finally: + tmp.unlink(missing_ok=True) + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def config(path=None, **overrides): + result = json.loads(Path(path or ROOT / "configs/default.json").read_text()) + result.update({k: v for k, v in overrides.items() if v is not None}) + if result["mode"] not in MODES or result["model"] not in result["models"]: + raise ValueError("Unknown mode or model") + for key in ("passes", "num_generations", "max_steps", "save_steps", "eval_steps", "eval_concurrency", + "batch_size", "gradient_accumulation_steps", "token_budget", "max_output_tokens"): + if type(result[key]) is not int or result[key] <= 0: + raise ValueError(f"{key} must be a positive integer") + if result["num_generations"] < 2: + raise ValueError("GRPO needs multiple rollouts of each task") + if result["max_inflight"] < result["num_generations"]: + raise ValueError("Inflight capacity must fit a complete group") + if result["max_outstanding_rollouts"] < 2 * result["num_generations"]: + raise ValueError("Outstanding capacity must fit two complete groups") + if result["eval_steps"] % result["save_steps"]: + raise ValueError("Evaluation cadence must be a multiple of checkpoint cadence") + if result["top_p"] != 1.0 or result["top_k"] != 0: + raise ValueError("Training requires full-vocabulary sampling for logprob consistency") + if not 0 <= result["efficiency_weight"] <= 0.1: + raise ValueError("Efficiency weight must be between zero and 0.1") + result["profile"] = result["models"][result["model"]] + return result + + +def task_rows(split): + rows = json.loads((ROOT / "data" / f"{split}.json").read_text()) + if len({r["name"] for r in rows}) != len(rows): + raise ValueError("Duplicate task identity") + return rows + + +def schedule(cfg, rows): + catalog = {name: i for i, name in enumerate(sorted(r["name"] for r in rows))} + harnesses = cfg["harnesses"] if cfg["mode"] == "multi-harness" else [cfg["mode"]] + groups = [] + for epoch in range(cfg["passes"]): + for i, row in enumerate(rows): + groups.append({"group_id": len(groups), "pass": epoch, "task_name": row["name"], + "task_index": catalog[row["name"]], + "harness": harnesses[(i + epoch) % len(harnesses)]}) + return groups + + +def resume_state(groups, state): + if state["execution_schedule_sha256"] != digest(groups): + raise ValueError("Cannot resume with a different task/harness schedule") + committed = set(state.get("admitted_rollouts", {}).values()) + if any(type(g) is not int or not 0 <= g < len(groups) for g in committed): + raise ValueError("Invalid committed group") + return {**state, "settled_groups": sorted(committed)} + + +def reward(correctness, calls, *, verified, weight=0.1, budget=15): + if correctness is None: + return None + if correctness not in (0, 1): + raise ValueError("Expected binary correctness") + if not 0 <= weight <= 0.1 or budget <= 0: + raise ValueError("Invalid reward parameters") + if not verified or calls is None: + return float(correctness) + if type(calls) is not int or calls < 0: + raise ValueError("Invalid verified tool count") + return float(correctness) * (1 + weight * budget / (budget + calls)) if calls else float(correctness) + + +def summary(records, expected): + graded = [r for r in records if r.get("correctness") in (0, 1) and not r.get("error")] + successes = sum(r["correctness"] for r in graded) + return {"expected": expected, "graded": len(graded), "correct": successes, + "pass_at_1": successes / len(graded) if graded else None, + "coverage": len(graded) / expected if expected else 0, + "complete": len(graded) == expected, + "mean_tool_calls": mean(r.get("tool_calls") for r in graded), + "mean_generated_tokens": mean(r.get("generated_tokens") for r in graded)} + + +def mean(values): + values = [v for v in values if isinstance(v, (int, float)) and math.isfinite(v)] + return sum(values) / len(values) if values else None diff --git a/05-multi-harness-rl/requirements.lock b/05-multi-harness-rl/requirements.lock new file mode 100644 index 00000000..75fec631 --- /dev/null +++ b/05-multi-harness-rl/requirements.lock @@ -0,0 +1,312 @@ +accelerate==1.14.0 +agent-detector==2.0.0 +aiofile==3.12.3 +aiofiles==25.1.0 +aiohappyeyeballs==2.7.1 +aiohttp==3.14.3 +aiohttp-retry==2.9.1 +aiosignal==1.4.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anthropic==1.9.0 +antlr4-python3-runtime==4.13.2 +anyio==4.15.1 +apache-tvm-ffi==0.1.9 +astor==0.8.1 +attrs==26.1.0 +authlib==1.8.0 +beartype==0.22.9 +bidict==0.24.1 +bitsandbytes==0.49.2 +blake3==1.0.10 +boto3==1.43.105 +botocore==1.43.105 +bracex==3.0.1 +brotli==1.2.0 +cachetools==7.2.0 +caio==0.12.9 +cbor2==6.1.4 +certifi==2026.7.22 +cffi==2.1.1 +charset-normalizer==3.5.2 +click==8.5.0 +cloudpickle==3.1.2 +compressed-tensors==0.17.0 +connectrpc==0.11.1 +cryptography==50.0.1 +cuda-bindings==13.4.3 +cuda-core==1.2.1 +cuda-pathfinder==1.8.2 +cuda-python==13.4.1 +cuda-tile==1.3.0 +cuda-toolkit==13.0.2 +cyclopts==5.1.0 +datasets==5.0.0 +daytona==0.203.0 +daytona-analytics-api-client==0.203.0 +daytona-analytics-api-client-async==0.203.0 +daytona-api-client==0.203.0 +daytona-api-client-async==0.203.0 +daytona-toolbox-api-client==0.203.0 +daytona-toolbox-api-client-async==0.203.0 +deprecated==1.3.1 +deprecation==2.1.0 +depyf==0.20.0 +detect-installer==0.2.1 +dill==0.4.1 +dirhash==0.5.0 +diskcache==5.6.3 +distro==1.9.0 +dnspython==2.8.0 +dockerfile-parse==2.0.1 +docstring-parser==0.18.0 +e2b==2.36.0 +einops==0.8.2 +email-validator==2.3.0 +exceptiongroup==1.3.1 +fastapi==0.136.3 +fastapi-cli==0.0.32 +fastapi-cloud-cli==0.26.0 +fastar==0.12.0 +fastmcp==3.4.4 +fastmcp-slim==3.4.4 +fastsafetensors==0.4.0 +fastuuid==0.14.0 +filelock==4.0.7 +fla-core==0.5.2 +flash-linear-attention==0.5.2 +flashinfer-cubin==0.6.13 +flashinfer-python==0.6.13 +frozenlist==1.8.0 +fsspec==2026.4.0 +googleapis-common-protos==1.75.5 +gradio==6.20.0 +gradio-client==2.5.0 +griffelib==2.3.0 +groovy==0.1.2 +grpcio==1.84.0 +h11==0.16.0 +h2==4.4.1 +harbor==0.22.0 +hf-gradio==0.4.1 +hf-xet==1.6.0 +hpack==4.2.0 +httpcore==1.0.9 +httpcore2==2.13.1 +httptools==0.8.0 +httpx==0.28.1 +httpx-sse==0.4.3 +httpx-ws==0.9.0 +httpx2==2.13.1 +huggingface-hub==1.24.0 +humming-kernels==0.1.10 +hyperframe==6.1.0 +idna==3.20 +ijson==3.5.1 +importlib-metadata==8.9.0 +iniconfig==2.3.0 +interegular==0.3.3 +jaraco-classes==3.4.0 +jaraco-context==6.1.2 +jaraco-functools==4.6.0 +jeepney==0.9.0 +jinja2==3.1.6 +jiter==0.17.0 +jmespath==1.1.0 +joserfc==1.7.5 +jsonref==1.1.0 +jsonschema==4.26.0 +jsonschema-path==0.5.0 +jsonschema-specifications==2025.9.1 +kernels==0.15.2 +kernels-data==0.16.2 +keyring==25.7.0 +lark==1.2.2 +latex2sympy2-extended==1.11.0 +litellm==1.103.1 +llguidance==1.7.6 +llvmlite==0.47.0 +lm-format-enforcer==0.11.3 +loguru==0.7.3 +markdown-it-py==4.2.0 +markupsafe==3.0.3 +math-verify==0.9.0 +mcp==1.28.1 +mdurl==0.1.2 +mistral-common==1.12.0 +ml-dtypes==0.6.0 +model-hosting-container-standards==0.1.16 +more-itertools==11.1.0 +mpmath==1.3.0 +msgspec==0.22.0 +multidict==6.9.1 +multiprocess==0.70.19 +networkx==3.7 +ninja==1.13.0 +numba==0.65.0 +numpy==2.3.5 +nvidia-cublas==13.1.0.3 +nvidia-cuda-cccl==13.3.4.3.1 +nvidia-cuda-crt==13.4.92 +nvidia-cuda-cupti==13.0.85 +nvidia-cuda-nvcc==13.2.86 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-tileiras==13.2.86 +nvidia-cudnn-cu13==9.19.0.56 +nvidia-cudnn-frontend==1.28.0 +nvidia-cufft==12.0.0.61 +nvidia-cufile==1.15.1.6 +nvidia-curand==10.4.0.35 +nvidia-cusolver==12.0.4.66 +nvidia-cusparse==12.6.3.3 +nvidia-cusparselt-cu13==0.8.0 +nvidia-cutlass-dsl==4.5.2 +nvidia-cutlass-dsl-libs-base==4.5.2 +nvidia-cutlass-dsl-libs-cu13==4.5.2 +nvidia-ml-py==13.615.71 +nvidia-nccl-cu13==2.28.9 +nvidia-nvjitlink==13.0.88 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvtx==13.0.85 +nvidia-nvvm==13.2.86 +nvtx==0.2.15 +obstore==0.11.1 +openai==2.52.0 +openai-harmony==0.0.8 +openapi-pydantic==0.5.1 +opencv-python-headless==5.0.0.93 +opentelemetry-api==1.45.0 +opentelemetry-exporter-http-transport==0.66b0 +opentelemetry-exporter-otlp==1.45.0 +opentelemetry-exporter-otlp-common==0.66b0 +opentelemetry-exporter-otlp-proto-common==1.45.0 +opentelemetry-exporter-otlp-proto-grpc==1.45.0 +opentelemetry-exporter-otlp-proto-http==1.45.0 +opentelemetry-instrumentation==0.66b0 +opentelemetry-instrumentation-aiohttp-client==0.66b0 +opentelemetry-proto==1.45.0 +opentelemetry-sdk==1.45.0 +opentelemetry-semantic-conventions==0.66b0 +opentelemetry-semantic-conventions-ai==0.5.1 +opentelemetry-util-http==0.66b0 +orjson==3.12.0 +outlines-core==0.2.14 +packaging==26.2 +pandas==3.0.6 +partial-json-parser==0.2.1.1.post7 +pathable==0.6.0 +pathspec==1.1.1 +peft==0.19.1 +pillow==12.3.0 +platformdirs==4.12.2 +pluggy==1.6.0 +postgrest==2.31.0 +prometheus-client==0.26.0 +prometheus-fastapi-instrumentator==8.1.0 +propcache==0.5.4 +protobuf==7.36.2 +protobuf-py==0.1.1 +protobuf-py-ext==0.1.1 +psutil==7.2.2 +py-cpuinfo==9.0.0 +py-key-value-aio==0.4.6 +pyarrow==25.0.1 +pybase64==1.5.0 +pycountry==26.2.16 +pycparser==3.0 +pydantic==2.13.4 +pydantic-core==2.46.4 +pydantic-extra-types==2.11.1 +pydantic-settings==2.15.0 +pydub==0.25.1 +pyelftools==0.33 +pygments==2.21.0 +pyjwt==2.15.1 +pynvvideocodec==2.0.4 +pyperclip==1.11.0 +pyqwest==0.7.0 +pytest==9.1.1 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.2 +python-engineio==4.14.0 +python-json-logger==4.2.0 +python-multipart==0.0.32 +python-socketio==5.17.0 +pytz==2026.4 +pyyaml==6.0.3 +pyzmq==27.2.0 +quack-kernels==0.5.0 +realtime==2.31.0 +referencing==0.37.0 +regex==2026.9.29 +requests==2.34.2 +rich==15.0.0 +rich-rst==2.2.0 +rich-toolkit==0.20.5 +rignore==0.8.1 +rpds-py==2026.6.3 +s3transfer==0.19.2 +safehttpx==0.1.7 +safetensors==0.8.0 +scantree==0.0.4 +secretstorage==3.5.0 +semantic-version==2.10.0 +sentencepiece==0.2.2 +sentry-sdk==2.71.0 +setproctitle==1.3.7 +setuptools==80.10.2 +shellingham==1.5.4 +shortuuid==1.0.13 +simple-websocket==1.1.0 +six==1.17.0 +sniffio==1.3.1 +sse-starlette==3.5.0 +starlette==1.7.0 +storage3==2.31.0 +strenum==0.4.15 +supabase==2.31.0 +supabase-auth==2.31.0 +supabase-functions==2.31.0 +supervisor==4.3.0 +sympy==1.14.0 +tabulate==0.10.0 +tenacity==9.1.4 +tiktoken==0.14.0 +tilelang==0.1.9 +tokenizers==0.22.2 +tokenspeed-mla==0.1.2 +tokenspeed-triton==3.8.10.post20260920 +toml==0.10.2 +tomli==2.4.1 +tomli-w==1.2.0 +tomlkit==0.14.0 +torch==2.11.0 +torch-c-dlpack-ext==0.1.5 +torchaudio==2.11.0 +torchcodec==0.16.0 +torchvision==0.26.0 +tqdm==4.70.1 +trackio==0.33.0 +transformers==5.14.1 +triton==3.6.0 +truststore==0.10.4 +typer==0.27.0 +typing-extensions==4.16.0 +typing-inspection==0.4.4 +uncalled-for==0.4.0 +urllib3==2.8.0 +uvicorn==0.51.0 +uvloop==0.22.1 +vllm==0.25.1 +watchfiles==1.3.0 +wcmatch==10.2.1 +websocket-client==1.9.2 +websockets==15.0.1 +wrapt==2.5.0 +wsproto==1.3.2 +xgrammar==0.2.3 +xxhash==4.0.1 +yarl==1.25.1 +z3-solver==4.15.4.0 +zipp==4.1.0 diff --git a/05-multi-harness-rl/requirements.txt b/05-multi-harness-rl/requirements.txt new file mode 100644 index 00000000..9248d1fc --- /dev/null +++ b/05-multi-harness-rl/requirements.txt @@ -0,0 +1,40 @@ +# Direct pins from the frozen comparison trainer environment. Vendored TRL/OpenEnv are supplied separately. +torch==2.11.0 +vllm==0.25.1 +transformers==5.14.1 +datasets==5.0.0 +huggingface_hub==1.24.0 +accelerate==1.14.0 +bitsandbytes==0.49.2 +kernels==0.15.2 +trackio==0.33.0 +gradio==6.20.0 +httpx==0.28.1 +# Harbor -> Supabase requires websockets <16; matches the validated environment service. +websockets==15.0.1 +harbor==0.22.0 +daytona==0.203.0 +pydantic==2.13.4 +fastapi==0.136.3 +uvicorn==0.51.0 +mcp==1.28.1 +fastmcp==3.4.4 +tomli==2.4.1 +tomli-w==1.2.0 +ninja==1.13.0 +peft==0.19.1 +# vLLM's Numba requires <2.5; mistral-common on Python 3.12 requires <2.4. +numpy>=2.2,<2.4 +safetensors==0.8.0 +rich==15.0.0 +typer==0.27.0 +python-dotenv==1.2.2 +packaging==26.2 +jinja2==3.1.6 + +openai==2.52.0 +e2b==2.36.0 +math-verify==0.9.0 +pytest==9.1.1 +pyyaml==6.0.3 +flash-linear-attention==0.5.2 diff --git a/05-multi-harness-rl/run.py b/05-multi-harness-rl/run.py new file mode 100644 index 00000000..7697c10e --- /dev/null +++ b/05-multi-harness-rl/run.py @@ -0,0 +1,182 @@ +"""Plan, serve, train or evaluate the three SmolDataEnv modes.""" +import argparse +import json +import os +from pathlib import Path +import signal +import socket +import subprocess +import sys +import time +from urllib.parse import urlparse + +from recipe import ROOT, MODES, config, write_json + + +def parser(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("action", choices=["plan", "train", "eval", "smoke", "environment", "_train", "_eval"]) + p.add_argument("--config", type=Path) + p.add_argument("--model", choices=["lfm", "qwen"]) + p.add_argument("--mode", choices=MODES) + p.add_argument("--data", type=Path, default=ROOT / "prepared") + p.add_argument("--output", type=Path) + p.add_argument("--run-name") + p.add_argument("--checkpoint", type=Path) + p.add_argument("--resume", type=Path) + p.add_argument("--concurrency", type=int) + p.add_argument("--limit", type=int, help="Eval tasks per harness; omit for all 250") + p.add_argument("--server", help="External URL, or a local bind URL; owned services default to unused ports") + p.add_argument("--vllm-url") + p.add_argument("--capture-port", type=int) + p.add_argument("--external-services", action="store_true", help="Use the supplied server URLs without starting services") + return p + + +def unused_port(): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + return listener.getsockname()[1] + + +def wait_ready(url, process, timeout=1200): + import httpx + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError(f"Service exited with {process.returncode}; inspect its log") + try: + if httpx.get(url + "/health", timeout=5).status_code == 200: + return + except httpx.HTTPError: + pass + time.sleep(2) + raise TimeoutError(f"Service readiness exceeded {timeout}s") + + +def launch_local(args, cfg): + from runtime.bootstrap import activate + activate() + from runtime.models import tokenizer_for, serving_command, check_visible_response + from runtime.environment import credentials + + credentials() + if not (args.data / "ready.json").exists(): + raise ValueError("Run prepare.py before allocating GPUs") + output = Path(cfg["output"]) + output.mkdir(parents=True, exist_ok=True) + if args.action in {"train", "smoke"} and not args.resume and (output / "metrics.jsonl").exists(): + raise ValueError("Existing training output; choose a new run name or pass --resume") + training = args.action in {"train", "smoke"} + if args.checkpoint: + from runtime.checkpoints import verify + info = verify(args.checkpoint) + if info["base_model"] != cfg["profile"]["id"]: + raise ValueError("Checkpoint model differs from configured model") + if args.resume: + from runtime.checkpoints import verify + info = verify(args.resume, resume=True) + if info["base_model"] != cfg["profile"]["id"] or info["base_revision"] != cfg["profile"]["revision"]: + raise ValueError("Resume model differs from configured model") + model = str(args.resume or args.checkpoint or cfg["profile"]["id"]) + template = output / "chat_template.jinja" + tokenizer_for(cfg, template) + config_path = output / "config.json" + write_json(config_path, cfg) + write_json(output / "services.json", {"environment": args.server, "vllm": args.vllm_url, + "capture_port": args.capture_port}) + env = dict(os.environ, PYTHONUNBUFFERED="1", VLLM_SERVER_DEV_MODE="1", + VLLM_USE_DEEP_GEMM="0", VLLM_DEEP_GEMM_WARMUP="skip", VLLM_USE_FLASHINFER_SAMPLER="0", + TRACKIO_DIR=str(output / "trackio")) + processes = [] + def spawn(command, name, child_env=None): + with (output / f"{name}.log").open("a") as stream: + proc = subprocess.Popen(command, env=child_env or env, stdout=stream, stderr=subprocess.STDOUT, + start_new_session=True, cwd=ROOT) + processes.append(proc) + return proc + devices = os.environ.get("CUDA_VISIBLE_DEVICES", "0,1").split(",") + if len(devices) < 2: + raise ValueError("Allocate two GPUs: inference + trainer, or two evaluation replicas") + try: + inference_env = dict(env, CUDA_VISIBLE_DEVICES=devices[0] if training else ",".join(devices[:2])) + port = urlparse(args.vllm_url).port or 8000 + engine = spawn(serving_command(cfg, model, training=training, template_path=template, port=port), "vllm", inference_env) + wait_ready(args.vllm_url, engine) + write_json(output / "inference-check.json", check_visible_response(cfg, args.vllm_url)) + base = [sys.executable, str(ROOT / "run.py")] + common = ["--config", str(config_path), "--data", str(args.data.resolve()), "--output", str(output), + "--server", args.server, "--vllm-url", args.vllm_url, + "--capture-port", str(args.capture_port)] + service_mode = cfg["mode"] if training or cfg["mode"] == "whitebox" else "multi-harness" + service = spawn(base + ["environment", *common, "--mode", service_mode], "environment") + wait_ready(args.server, service) + action = "_train" if training else "_eval" + command = base + [action, *common] + if args.resume: + command += ["--resume", str(args.resume.resolve())] + if args.checkpoint: + command += ["--checkpoint", str(args.checkpoint.resolve())] + if args.limit: + command += ["--limit", str(args.limit)] + worker = spawn(command, "train" if training else "eval", dict(env, CUDA_VISIBLE_DEVICES=devices[1]) if training else env) + while worker.poll() is None: + if engine.poll() is not None or service.poll() is not None: + raise RuntimeError("Inference or environment service died; stopping worker") + time.sleep(2) + if worker.returncode: + raise RuntimeError(f"Worker failed ({worker.returncode}); inspect {output}") + finally: + for proc in reversed(processes): + if proc.poll() is None: + os.killpg(proc.pid, signal.SIGTERM) + for proc in reversed(processes): + try: + proc.wait(timeout=30) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + + +def main(): + args = parser().parse_args() + if args.external_services and (not args.server or not args.vllm_url): + raise ValueError("External services require --server and --vllm-url") + args.server = args.server or f"http://127.0.0.1:{unused_port()}" + args.vllm_url = args.vllm_url or f"http://127.0.0.1:{unused_port()}" + args.capture_port = args.capture_port or unused_port() + cfg = config(args.config, model=args.model, mode=args.mode, eval_concurrency=args.concurrency) + name = args.run_name or cfg.get("run_name") or f'{cfg["model"]}-{cfg["mode"]}-{time.strftime("%Y%m%d-%H%M%S")}' + cfg.update(run_name=name, output=str((args.output or Path(cfg.get("output", ROOT / "runs" / name))).resolve())) + if args.action == "smoke": + cfg.update(max_steps=2, save_steps=1, eval_steps=1, num_generations=2, + max_inflight=4, max_outstanding_rollouts=4, batch_size=1, gradient_accumulation_steps=2) + if args.action == "plan": + print(json.dumps(cfg, indent=2)) + return + if args.action in {"train", "eval", "smoke"}: + if not args.external_services: + return launch_local(args, cfg) + args.action = "_eval" if args.action == "eval" else "_train" + from runtime.bootstrap import activate + activate() + if args.action == "environment": + from runtime.environment import serve + serve(cfg, args.data, args.vllm_url, port=urlparse(args.server).port or 8200, + capture_port=args.capture_port) + elif args.action == "_train": + if cfg["mode"] == "whitebox": + from train.whitebox import train + else: + from train.blackbox import train + train(cfg, args.data, args.server, args.vllm_url, args.resume) + else: + from eval.evaluate import evaluate + report = evaluate(cfg, args.data, args.server, args.vllm_url, + limit=args.limit, checkpoint=args.checkpoint or "baseline") + if not report["complete"]: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/05-multi-harness-rl/runtime/bootstrap.py b/05-multi-harness-rl/runtime/bootstrap.py new file mode 100644 index 00000000..e90f38db --- /dev/null +++ b/05-multi-harness-rl/runtime/bootstrap.py @@ -0,0 +1,88 @@ +"""Fetch immutable runtime dependencies without copying the experiment archive into Git.""" +from __future__ import annotations + +import argparse +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +RUNTIME = ROOT / ".runtime" + + +def run(*args): + subprocess.run(list(map(str, args)), check=True) + + +def bootstrap(local_archive=None, local_trl=None, local_openenv=None): + lock = json.loads((ROOT / "configs/runtime-lock.json").read_text()) + archive = lock["archive"] + destination = RUNTIME / "archive" + + def fetch(item): + name, expected = item + path = destination / name + if path.exists() and hashlib.sha256(path.read_bytes()).hexdigest() == expected: + return + if local_archive: + data = (Path(local_archive) / "04-data-agent" / name).read_bytes() + else: + url = f'https://raw.githubusercontent.com/{archive["repo"]}/{archive["revision"]}/04-data-agent/{name}' + with urllib.request.urlopen(url, timeout=60) as stream: + data = stream.read() + if hashlib.sha256(data).hexdigest() != expected: + raise ValueError(f"Runtime source hash mismatch: {name}") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + with ThreadPoolExecutor(max_workers=8) as pool: + list(pool.map(fetch, archive["files"].items())) + for name, local in (("trl", local_trl), ("openenv", local_openenv)): + entry = lock[name] + target = RUNTIME / name + if not target.exists(): + run("git", "clone", "--no-checkout", "--filter=blob:none", local or f'https://github.com/{entry["repo"]}.git', target) + try: + run("git", "-C", target, "cat-file", "-e", entry["revision"]) + except subprocess.CalledProcessError: + run("git", "-C", target, "fetch", "origin", entry["revision"]) + run("git", "-C", target, "checkout", "--detach", entry["revision"]) + packages = RUNTIME / "packages" + packages.mkdir(exist_ok=True) + for name, folder in (("data_agent_env", "blackbox-opencode"), ("whitebox_bash", "whitebox-bash")): + link = packages / name + if link.is_symlink(): + link.unlink() + shutil.copytree(destination / "envs" / folder, link, dirs_exist_ok=True) + sys.path.insert(0, str(ROOT)) + from runtime.patches import apply + apply(packages) + print(f"Verified {len(archive['files'])} runtime files; pinned TRL and OpenEnv ready.") + + +def activate(): + paths = [ROOT, RUNTIME / "packages", RUNTIME / "archive/train", + RUNTIME / "archive/hf/runtime", RUNTIME / "archive/model_runs/lfm25", + RUNTIME / "openenv/src", RUNTIME / "openenv/envs", RUNTIME / "trl"] + if not (RUNTIME / "trl/trl").is_dir(): + raise RuntimeError("Run python runtime/bootstrap.py first") + sys.path[:0] = list(map(str, paths)) + os.environ["PYTHONPATH"] = os.pathsep.join(map(str, paths)) + os.environ.setdefault("TRL_EXPERIMENTAL_SILENCE", "1") + os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") + os.environ.setdefault("OMP_NUM_THREADS", "1") + + +if __name__ == "__main__": + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--local-archive", type=Path) + p.add_argument("--local-trl", type=Path) + p.add_argument("--local-openenv", type=Path) + a = p.parse_args() + bootstrap(a.local_archive, a.local_trl, a.local_openenv) diff --git a/05-multi-harness-rl/runtime/checkpoints.py b/05-multi-harness-rl/runtime/checkpoints.py new file mode 100644 index 00000000..2e3a963e --- /dev/null +++ b/05-multi-harness-rl/runtime/checkpoints.py @@ -0,0 +1,57 @@ +"""Portable checkpoint markers. The evaluator verifies bytes before loading weights.""" +import hashlib +import json +from pathlib import Path + +from recipe import write_json + +REQUIRED = ("config.json", "trainer_state.json", "tokenizer_config.json", "tokenizer.json", + "training_args.bin", "optimizer.pt", "scheduler.pt", "rng_state.pth") + + +def sha256(path): + value = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(8 * 1024 * 1024), b""): + value.update(block) + return value.hexdigest() + + +def saved(path, cfg, step): + path = Path(path) + for name in REQUIRED: + if not (path / name).is_file() or (path / name).stat().st_size == 0: + raise ValueError(f"Incomplete checkpoint: {name}") + if not list(path.glob("*.safetensors")): + raise ValueError("Checkpoint has no model weights") + if json.loads((path / "trainer_state.json").read_text())["global_step"] != step: + raise ValueError("Checkpoint step mismatch") + files = {p.name: p.stat().st_size for p in path.iterdir() if p.is_file() and not p.name.startswith(".") and + p.name not in {"checkpoint.saved.json", "checkpoint.ready.json", "eval.request.json", "recipe.json"}} + write_json(path / "checkpoint.saved.json", {"step": step, "base_model": cfg["profile"]["id"], + "base_revision": cfg["profile"]["revision"], "files": files}) + + +def make_ready(path): + path = Path(path) + marker = json.loads((path / "checkpoint.saved.json").read_text()) + hashes = {} + for name, size in marker["files"].items(): + if Path(name).name != name or (path / name).stat().st_size != size: + raise ValueError("Checkpoint incomplete or changed after save") + hashes[name] = sha256(path / name) + if (path / name).stat().st_size != size: + raise ValueError("Checkpoint changed during hashing") + write_json(path / "checkpoint.ready.json", {**marker, "sha256": hashes}) + + +def verify(path, *, resume=False): + path = Path(path) + marker = json.loads((path / "checkpoint.ready.json").read_text()) + names = marker["sha256"] + for name, expected in names.items(): + if Path(name).name != name or sha256(path / name) != expected: + raise ValueError(f"Checkpoint integrity check failed: {name}") + if resume and any(n not in names for n in REQUIRED): + raise ValueError("Resume requires a complete optimizer checkpoint") + return marker diff --git a/05-multi-harness-rl/runtime/environment.py b/05-multi-harness-rl/runtime/environment.py new file mode 100644 index 00000000..7bfb4b19 --- /dev/null +++ b/05-multi-harness-rl/runtime/environment.py @@ -0,0 +1,46 @@ +"""A job-local OpenEnv service. Sandboxes remain isolated on the selected provider.""" +import os +import json +from pathlib import Path + + +def credentials(): + from huggingface_hub import get_token + token = get_token() + if not token: + raise ValueError("Log in with hf auth login; Harbor task staging requires HF_TOKEN") + os.environ["HF_TOKEN"] = token + + +def serve(cfg, data, vllm, port=8200, capture_port=8300): + import uvicorn + + credentials() + data = Path(data).resolve() + output = Path(cfg["output"]).resolve() + os.environ.update(OPENENV_LLM_URL=vllm, OPENENV_MODEL=cfg["profile"]["id"], + OPENENV_MAX_OUTPUT_TOKENS=str(cfg["max_output_tokens"]), ENABLE_WEB_INTERFACE="false", RUN_OWNER=cfg["run_name"], + MAX_CONCURRENT_ENVS=str(max(cfg["max_inflight"], cfg["eval_concurrency"]) + 8), + OPENENV_HARBOR_AGENT_VERSIONS=json.dumps(cfg["harness_versions"]), + OPENENV_CAPTURE_PORT=str(capture_port), OPENENV_EXPOSE="gradio") + if cfg["mode"] == "opencode": + os.environ.update(DATA_AGENT_SPLITS="train,test", DATA_AGENT_SANDBOX=cfg["sandbox"], + DATA_AGENT_OPENCODE_VERSION=cfg["harness_versions"]["opencode"], + DATA_AGENT_FROZEN_TASKS_DIR=str(data / "datasets"), DATA_AGENT_CAPTURE_EXPOSE="gradio", + DATA_AGENT_CAPTURE_PORT=str(capture_port), + DATA_AGENT_MAX_CONCURRENT=str(max(cfg["max_inflight"], cfg["eval_concurrency"]))) + from data_agent_env.server.app import app + elif cfg["mode"] == "whitebox": + if cfg["sandbox"] != "daytona": + raise ValueError("The SETA adapter currently requires Daytona") + os.environ.update(DAYTONA_COMPARISON_RUN=str(data), WHITE_BOX_BASH_TASK_SOURCE="harbor-frozen", + DAYTONA_WHITEBOX_TRIALS=str(output / "trials"), WHITEBOX_RECORDS=str(output / "whitebox-records"), + EFFICIENCY_WEIGHT=str(cfg["efficiency_weight"]), TOOL_BUDGET=str(cfg["tool_budget"]), + WHITE_BOX_BASH_MAX_SESSIONS=str(max(cfg["max_inflight"], cfg["eval_concurrency"]) + 8), + WHITE_BOX_BASH_MAX_CONCURRENT_ENVS=str(max(cfg["max_inflight"], cfg["eval_concurrency"]) + 8)) + from whitebox_bash.server.app import app + else: + os.environ.update(OPENENV_DATASETS=",".join(str(data / "datasets" / s) for s in ("train", "test")), + OPENENV_HARBOR_TRIALS_DIR=str(output / "trials")) + from harbor_env.server.app import app + uvicorn.run(app, host="127.0.0.1", port=port, ws_ping_timeout=None, timeout_keep_alive=120) diff --git a/05-multi-harness-rl/runtime/job.sh b/05-multi-harness-rl/runtime/job.sh new file mode 100644 index 00000000..dea84c88 --- /dev/null +++ b/05-multi-harness-rl/runtime/job.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail +mkdir -p /workspace/recipe +cp -a /recipe-source/. /workspace/recipe/ +cd /workspace/recipe +uv venv --python 3.12 .venv +export PATH="$PWD/.venv/bin:$PATH" +if [[ "$1" == "watch" ]]; then + shift + uv pip install --python .venv/bin/python huggingface_hub==1.24.0 + exec python3 eval/watch.py "$@" +fi +python3 runtime/bootstrap.py +uv pip install --python .venv/bin/python -r requirements.lock +python3 prepare.py +exec python3 run.py "$@" diff --git a/05-multi-harness-rl/runtime/launch.py b/05-multi-harness-rl/runtime/launch.py new file mode 100644 index 00000000..9f325eea --- /dev/null +++ b/05-multi-harness-rl/runtime/launch.py @@ -0,0 +1,122 @@ +"""Submit the same job-local recipe to HF Jobs or Slurm; dry-run by default.""" +import argparse +import json +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import sys +import tempfile + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from recipe import MODES, write_json + + +def stage(destination): + destination = Path(destination) + allowed = {"run.py", "recipe.py", "prepare.py", "requirements.txt", "requirements.lock", "README.md", "REPRODUCE.md"} + for folder in ("configs", "data", "train", "eval", "runtime"): + allowed.update(str(p.relative_to(ROOT)) for p in (ROOT / folder).rglob("*") + if p.is_file() and p.suffix in {".py", ".sh", ".json"} and "__pycache__" not in p.parts) + for name in sorted(allowed): + if (ROOT / name).is_file(): + target = destination / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ROOT / name, target) + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("backend", choices=["hf", "slurm"]) + p.add_argument("action", choices=["train", "eval", "smoke", "watch"]) + p.add_argument("--model", choices=["lfm", "qwen"], default="lfm") + p.add_argument("--config", type=Path) + p.add_argument("--mode", choices=MODES, default="multi-harness") + p.add_argument("--run-name", required=True) + p.add_argument("--namespace", default="FineEnvs") + p.add_argument("--bucket", help="Existing HF bucket, e.g. FineEnvs/smoldataenv-runs") + p.add_argument("--image", default="huggingface/trl@sha256:8433cde7eaf3b289f3daffdaefcab6bb71499ca1ff59dc5d05abba7a71603a58") + p.add_argument("--flavor", default="h200x2") + p.add_argument("--timeout", default="24h") + p.add_argument("--partition") + p.add_argument("--data", type=Path, default=ROOT / "prepared") + p.add_argument("--output-root", type=Path, default=ROOT / "runs") + p.add_argument("--checkpoint") + p.add_argument("--resume") + p.add_argument("--concurrency", type=int, default=35) + p.add_argument("--limit", type=int) + p.add_argument("--max-active-evals", type=int, default=1) + p.add_argument("--submit", action="store_true") + args = p.parse_args() + if Path(args.run_name).name != args.run_name or args.run_name in {".", ".."}: + p.error("run-name must be a single directory name") + output = str(Path("/outputs" if args.backend == "hf" else args.output_root.resolve()) / args.run_name) + flags = [args.action, "--model", args.model, "--mode", args.mode, "--run-name", args.run_name, + "--output", output, "--concurrency", str(args.concurrency)] + if args.config: + path = args.config + if args.backend == "hf" and not str(path).startswith("/outputs/"): + try: + relative = path.resolve().relative_to(ROOT) + except ValueError: + p.error("Put a custom HF config inside this recipe's configs directory") + if relative.parts[0] != "configs": + p.error("Custom configs must be in configs/") + path = Path("/workspace/recipe") / relative + flags += ["--config", str(path)] + for name in ("checkpoint", "resume", "limit"): + value = getattr(args, name) + if value is not None: + flags += ["--" + name, str(value)] + if args.backend == "hf": + if not args.bucket: + p.error("HF Jobs require --bucket for durable checkpoints and logs") + if args.action == "watch": + flags = ["watch", "--root", "/outputs", "--backend", "hf", "--bucket", args.bucket, + "--namespace", args.namespace, "--flavor", args.flavor, + "--concurrency", str(args.concurrency), "--max-active", str(args.max_active_evals), "--submit"] + specification = {"image": args.image, "command": ["bash", "/recipe-source/runtime/job.sh", *flags], + "flavor": "cpu-basic" if args.action == "watch" else args.flavor, "timeout": args.timeout, "namespace": args.namespace, + "name": args.run_name, "labels": {"recipe": "smoldataenv-multi-harness", "role": args.action}, + "env": {"PYTHONUNBUFFERED": "1", **{k: os.environ[k] for k in + ("DAYTONA_API_URL", "DAYTONA_TARGET") if os.environ.get(k)}}} + print(json.dumps({**specification, "output_bucket": args.bucket, "secret_names": ["HF_TOKEN", "DAYTONA_API_KEY"]}, indent=2)) + if args.submit: + from huggingface_hub import get_token, run_job, sync_job_volume, Volume + token = get_token() + if not token or not os.environ.get("DAYTONA_API_KEY"): + p.error("HF login and DAYTONA_API_KEY are required") + with tempfile.TemporaryDirectory(prefix="smoldataenv-recipe-") as temp: + stage(temp) + source = sync_job_volume(temp, "/recipe-source", namespace=args.namespace) + volume = Volume(type="bucket", source=args.bucket, mount_path="/outputs", read_only=False) + job = run_job(**specification, volumes=[source, volume], + secrets={"HF_TOKEN": token, "DAYTONA_API_KEY": os.environ["DAYTONA_API_KEY"]}) + record = {"id": job.id, "url": job.url, "run_name": args.run_name, "namespace": args.namespace} + write_json(ROOT / "runs/submissions" / f"{args.run_name}.json", record) + print(json.dumps(record)) + else: + if not args.partition: + p.error("Choose an available Slurm partition explicitly") + args.output_root.mkdir(parents=True, exist_ok=True) + command = [sys.executable, str(ROOT / "run.py"), *flags, "--data", str(args.data.resolve())] + if args.action == "watch": + command = [sys.executable, str(ROOT / "eval/watch.py"), "--root", str(args.output_root.resolve()), + "--backend", "slurm", "--partition", args.partition, "--data", str(args.data.resolve()), + "--concurrency", str(args.concurrency), "--max-active", str(args.max_active_evals), "--submit"] + submit = ["sbatch", "--parsable", "--partition", args.partition, "--nodes=1", "--gpus=2", + "--cpus-per-task=16", "--mem=192G", "--time=24:00:00", "--job-name", args.run_name, + "--output", str(args.output_root / (args.run_name + "-%j.log")), "--wrap", shlex.join(command)] + if args.action == "watch": + submit.remove("--gpus=2") + submit[submit.index("--cpus-per-task=16")] = "--cpus-per-task=1" + submit[submit.index("--mem=192G")] = "--mem=4G" + print(shlex.join(submit), flush=True) + if args.submit: + subprocess.run(submit, check=True) + + +if __name__ == "__main__": + main() diff --git a/05-multi-harness-rl/runtime/models.py b/05-multi-harness-rl/runtime/models.py new file mode 100644 index 00000000..877822f5 --- /dev/null +++ b/05-multi-harness-rl/runtime/models.py @@ -0,0 +1,73 @@ +"""Render the same non-thinking prompt in the inference engine and trainer.""" +from pathlib import Path + + +def tokenizer_for(cfg, template_path=None): + from copy import deepcopy + from transformers import AutoTokenizer + + profile = cfg["profile"] + tokenizer = AutoTokenizer.from_pretrained(profile["id"], revision=profile["revision"]) + if cfg["model"] == "lfm": + template = tokenizer.chat_template + old = '{{- "<|im_start|>assistant\\n" -}}' + if template.count(old) != 1: + raise ValueError("LFM template changed; requalify the non-thinking prefix") + tokenizer.chat_template = template.replace(old, '{{- "<|im_start|>assistant\\n" -}}') + from trl.chat_template_utils import lfm2_2_5_template + tokenizer.response_template = deepcopy(lfm2_2_5_template) + else: + from trl.chat_template_utils import qwen3_5_template + tokenizer.response_template = deepcopy(qwen3_5_template) + tokenizer.chat_template = "{%- set enable_thinking = false -%}\n" + tokenizer.chat_template + rendered = tokenizer.apply_chat_template([{"role": "user", "content": "Say OK."}], + tokenize=False, add_generation_prompt=True, enable_thinking=False, preserve_thinking=True) + tail = rendered.rsplit("<|im_start|>assistant", 1)[-1] + if "" not in tail or tail.count("") != tail.count(""): + raise ValueError("Generation prompt still opens a thinking block") + if template_path: + Path(template_path).write_text(tokenizer.chat_template) + return tokenizer + + +def serving_command(cfg, model, *, training, template_path, port=8000): + p = cfg["profile"] + cmd = ["vllm", "serve", model, "--host", "127.0.0.1", "--port", str(port), + "--served-model-name", p["id"], "--dtype", "bfloat16", "--tensor-parallel-size", "1", + "--data-parallel-size", "1" if training else str(cfg["eval_dp"]), + "--max-model-len", str(cfg["max_model_len"]), "--gpu-memory-utilization", "0.85", + "--enable-auto-tool-choice", "--tool-call-parser", p["tool_parser"], + "--chat-template", str(template_path), + "--default-chat-template-kwargs", '{"enable_thinking":false,"preserve_thinking":true}', + "--generation-config", "vllm", "--override-generation-config", + '{"temperature":0.8,"top_p":1.0,"top_k":-1}', + "--logprobs-mode", "processed_logprobs", "--return-tokens-as-token-ids", + "--enforce-eager", "--no-enable-prefix-caching"] + if model == p["id"]: + cmd += ["--revision", p["revision"]] + if training: + cmd += ["--weight-transfer-config", '{"backend":"nccl"}'] + if cfg["model"] == "qwen": + cmd += ["--limit-mm-per-prompt", '{"image":0,"video":0}', "--gdn-prefill-backend", "triton"] + return cmd + + +def check_visible_response(cfg, url): + """Catch non-thinking outputs being swallowed by a reasoning parser.""" + from openai import OpenAI + + checks = [] + with OpenAI(base_url=url.rstrip("/") + "/v1", api_key="unused", timeout=120) as client: + for stream in (False, True): + response = client.chat.completions.create(model=cfg["profile"]["id"], + messages=[{"role": "user", "content": "Reply with exactly OK."}], + temperature=0, max_tokens=32, stream=stream) + messages = ([c.choices[0].delta for c in response if c.choices] if stream else + [response.choices[0].message]) + content = "".join(m.content or "" for m in messages) + reasoning = "".join(str((m.model_extra or {}).get(key) or "") + for m in messages for key in ("reasoning", "reasoning_content")) + if not content.strip() or reasoning: + raise ValueError(f"Non-thinking response was empty or hidden in reasoning (stream={stream})") + checks.append({"stream": stream, "visible_chars": len(content), "reasoning_chars": len(reasoning)}) + return checks diff --git a/05-multi-harness-rl/runtime/patches.py b/05-multi-harness-rl/runtime/patches.py new file mode 100644 index 00000000..555d70a1 --- /dev/null +++ b/05-multi-harness-rl/runtime/patches.py @@ -0,0 +1,55 @@ +"""Small, checked adapters to the immutable native environment dependencies.""" + + +def replace(text, old, new): + if text.count(old) != 1: + raise ValueError(f"Pinned runtime adapter no longer applies: {old[:80]}") + return text.replace(old, new, 1) + + +def apply(packages): + path = packages / "data_agent_env/server/capture.py" + path.write_text(replace(path.read_text(), "max_output_tokens=16384,", + 'max_output_tokens=int(os.environ["OPENENV_MAX_OUTPUT_TOKENS"]),')) + path = packages / "data_agent_env/server/rollout.py" + text = replace(path.read_text(), 'opencode run --print-logs "$(cat {config.home}/workdir/task.md)"', + 'opencode run --format json --print-logs "$(cat {config.home}/workdir/task.md)" > /tmp/native-events.jsonl') + text = replace(text, " timed_out = exit_code != 0", """ timed_out = exit_code != 0 + native_actions = None + if not timed_out: + from train.adapters import opencode_count + try: + native_actions = opencode_count(sandbox.read_text('/tmp/native-events.jsonl')) + except (ValueError, KeyError, TypeError, OSError): + pass""") + text = replace(text, ' **metadata_for(task, grade, n_tool_calls),', + ' **metadata_for(task, grade, n_tool_calls),\n "verified_native_actions": native_actions,') + path.write_text(text) + path = packages / "whitebox_bash/server/environment.py" + text = replace(path.read_text(), "from .sandbox import WORKDIR, ExecResult, Sandbox", + "from .sandbox import ExecResult\nfrom daytona_whitebox_backend import DaytonaSandbox as Sandbox\nWORKDIR = '/workdir'") + text = replace(text, "sb = Sandbox.start(timeout_s=SANDBOX_TIMEOUT_S,", "sb = Sandbox.start(task=task, timeout_s=SANDBOX_TIMEOUT_S,") + start = text.index(" check_passed: bool | None = None") + end = text.index(" return verdict.as_dict()", start) + len(" return verdict.as_dict()") + text = replace(text, text[start:end], """ from runtime.whitebox_reward import grade + try: + return grade(s) + finally: + _release(session_id)""") + path.write_text(text) + path = packages / "whitebox_bash/tasks.py" + text = replace(path.read_text(), ' if TASK_SOURCE == "data-agent" and split not in ("demo",):', + " if TASK_SOURCE == 'harbor-frozen':\n from daytona_whitebox_backend import load_frozen_tasks\n" + " tasks = load_frozen_tasks(split)\n _CACHE[split] = tasks\n return tasks\n" + ' if TASK_SOURCE == "data-agent" and split not in ("demo",):') + text = replace(text, ' d.pop("check", None)', ' d.pop("check", None)\n d.pop("metadata", None)\n d.pop("setup", None)') + path.write_text(text) + path = packages / "whitebox_bash/client.py" + text = path.read_text() + start = text.index(" # TRL's reward column is a float") + end = text.index(" if verdict.ungraded:", start) + text = replace(text, text[start:end], " # TRL ignores NaN rewards from ungraded episodes.\n") + text = replace(text, "reporting 0.0 to the trainer", "excluding reward from the trainer") + text = replace(text, "self._reward = 0.0 if verdict.reward is None else verdict.reward", + 'self._reward = float("nan") if verdict.reward is None else verdict.reward') + path.write_text(text) diff --git a/05-multi-harness-rl/runtime/whitebox_reward.py b/05-multi-harness-rl/runtime/whitebox_reward.py new file mode 100644 index 00000000..6387d7fd --- /dev/null +++ b/05-multi-harness-rl/runtime/whitebox_reward.py @@ -0,0 +1,18 @@ +"""Keep native correctness and verified tool count separate from shaping.""" +import os +from pathlib import Path + +from recipe import reward, write_json + + +def grade(session): + verdict = session.sandbox.grade(session.submitted) + correctness = verdict.get("reward") + calls = sum(c["tool"] != "submit_solution" for c in session.calls) + weight = float(os.environ["EFFICIENCY_WEIGHT"]) if session.split == "train" else 0.0 + value = reward(correctness, calls, verified=True, weight=weight, + budget=float(os.environ["TOOL_BUDGET"])) + record = {"correctness": correctness, "tool_calls": calls, "reward": value, + "task_index": session.index, "split": session.split, "tool_count_verified": True} + write_json(Path(os.environ["WHITEBOX_RECORDS"]) / f"{session.session_id}.json", record) + return {**verdict, "reward": value, "n_tool_calls": calls} diff --git a/05-multi-harness-rl/tests/test_contracts.py b/05-multi-harness-rl/tests/test_contracts.py new file mode 100644 index 00000000..ebce8859 --- /dev/null +++ b/05-multi-harness-rl/tests/test_contracts.py @@ -0,0 +1,160 @@ +import json +from collections import Counter +from pathlib import Path +import shutil +import sys +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from recipe import config, digest, reward, resume_state, schedule, summary, task_rows, write_json +from train.adapters import atif_count, opencode_count +from train.generation import group_size, protect_generation +from eval.evaluate import capture_metrics +from runtime.checkpoints import make_ready, saved, verify +from runtime.launch import stage +from runtime.models import check_visible_response + + +def test_frozen_data_and_rotation(): + train, test = task_rows("train"), task_rows("test") + assert len(train) == 1000 and len(test) == 250 + assert Counter(r["difficulty"] for r in train) == {"medium": 400, "hard": 600} + assert Counter(r["difficulty"] for r in test) == {"easy": 33, "medium": 118, "hard": 99} + for key in ("source", "notebook", "instruction_sha256"): + assert not {r[key] for r in train} & {r[key] for r in test} + groups = schedule(config(), train) + assert len(groups) == 2000 + assert Counter(g["harness"] for g in groups[:1000]) == {h: 250 for h in config()["harnesses"]} + for a, b in zip(groups[:1000], groups[1000:], strict=True): + assert a["task_name"] == b["task_name"] and a["harness"] != b["harness"] + + +def test_native_means_native(): + assert {g["harness"] for g in schedule(config(mode="opencode"), task_rows("train"))} == {"opencode"} + + +def test_nonthinking_preflight_checks_streaming_content(monkeypatch): + hidden = False + def create(*, stream, **kwargs): + bad = stream and hidden + message = SimpleNamespace(content="" if bad else "OK", model_extra={"reasoning": "OK" if bad else None}) + choice = SimpleNamespace(delta=message, message=message) + response = SimpleNamespace(choices=[choice]) + return [response] if stream else response + class Client: + def __init__(self, **kwargs): + self.chat = SimpleNamespace(completions=SimpleNamespace(create=create)) + def __enter__(self): return self + def __exit__(self, *args): pass + monkeypatch.setitem(sys.modules, "openai", SimpleNamespace(OpenAI=Client)) + assert len(check_visible_response(config(), "http://localhost")) == 2 + hidden = True + with pytest.raises(ValueError, match="stream=True"): + check_visible_response(config(), "http://localhost") + + +@pytest.mark.parametrize("kwargs", [{"eval_steps": 75}, {"num_generations": 1}, {"top_p": .95}, + {"max_outstanding_rollouts": 8}, {"eval_concurrency": 0}]) +def test_reject_invalid_config(kwargs): + with pytest.raises(ValueError): + config(**kwargs) + + +def test_resume_skips_committed_groups_and_retries_uncommitted(): + groups = schedule(config(), task_rows("train")) + state = {"execution_schedule_sha256": digest(groups), "settled_groups": [0, 1, 2], + "admitted_rollouts": {"a": 0, "b": 2}} + assert resume_state(groups, state)["settled_groups"] == [0, 2] + with pytest.raises(ValueError): + resume_state(groups[::-1], state) + + +def test_reward_does_not_pay_for_failure_or_unknown_counts(): + assert reward(None, 2, verified=True) is None + assert reward(0, 1, verified=True) == 0 + assert reward(1, None, verified=False) == 1 + assert reward(1, 0, verified=True) == 1 + assert reward(1, 15, verified=True) == 1.05 + assert reward(1, 2, verified=True) > reward(1, 20, verified=True) + + +def test_native_counts_require_unambiguous_actions(tmp_path): + path = tmp_path / "trajectory.json" + write_json(path, {"schema_version": "ATIF-v1.6", "steps": [ + {"source": "agent", "tool_calls": [{"tool_call_id": "a"}, {"tool_call_id": "b"}]}, + {"source": "environment", "tool_calls": [{"tool_call_id": "a"}]}]}) + assert atif_count(path) == 2 + event = {"type": "tool_use", "part": {"callID": "a", "state": {"status": "completed"}}} + stream = json.dumps(event) + '\n{"type":"step_finish"}' + assert opencode_count(stream) == 1 + with pytest.raises(ValueError): + opencode_count(stream + "\n" + json.dumps(event)) + with pytest.raises(ValueError): + opencode_count(json.dumps(event)) + + +def test_diverged_tool_histories_are_never_collapsed(): + assert group_size([[1], [1], [2], [2]], 2) == 2 + assert group_size([[1], [3], [2], [2]], 2) == 1 + assert group_size([[1], [1], [2]], 2) == 1 + + +def test_whitebox_cache_reset_accepts_empty_success_and_rejects_failure(): + import requests + response = requests.Response() + response.status_code = 200 + response._content = b"" + client = SimpleNamespace(base_url="http://localhost", session=SimpleNamespace(post=lambda url: response)) + trainer = SimpleNamespace(vllm_generation=SimpleNamespace(vllm_client=client, generate=lambda **kw: None)) + protect_generation(trainer) + client.reset_prefix_cache() + response.status_code = 500 + with pytest.raises(requests.HTTPError): + client.reset_prefix_cache() + + +def test_capture_uses_engine_tokens_and_masks(): + turn = SimpleNamespace(trainable=True, prompt_token_ids=[10, 20], completion_token_ids=[30], + per_token_logps=[-.5], loss_mask=[0, 0, 1]) + result = SimpleNamespace(turns=[turn]) + assert capture_metrics(result)["generated_tokens"] == 1 + turn.loss_mask = [1, 0, 1] + with pytest.raises(ValueError): + capture_metrics(result) + turn.loss_mask = [0, 0, 1] + turn.per_token_logps = [float("nan")] + with pytest.raises(ValueError): + capture_metrics(result) + + +def test_ungraded_is_not_incorrect(): + result = summary([{"correctness": 1}, {"correctness": 0}, {"correctness": None, "error": "transport"}], 3) + assert result["pass_at_1"] == .5 and result["graded"] == 2 and not result["complete"] + assert summary([], 1)["pass_at_1"] is None + + +def test_checkpoint_transfer_is_portable_and_detects_tampering(tmp_path): + checkpoint = tmp_path / "original" + checkpoint.mkdir() + for name in ("config.json", "tokenizer_config.json", "tokenizer.json", "training_args.bin", + "optimizer.pt", "scheduler.pt", "rng_state.pth", "model.safetensors"): + (checkpoint / name).write_bytes(b"fixture") + write_json(checkpoint / "trainer_state.json", {"global_step": 50}) + saved(checkpoint, config(), 50) + make_ready(checkpoint) + moved = tmp_path / "different-root" / "checkpoint-50" + shutil.copytree(checkpoint, moved) + assert verify(moved, resume=True)["step"] == 50 + (moved / "model.safetensors").write_bytes(b"changed") + with pytest.raises(ValueError): + verify(moved) + + +def test_hf_source_bundle_omits_local_state(tmp_path): + stage(tmp_path) + paths = [str(p.relative_to(tmp_path)) for p in tmp_path.rglob("*")] + assert "configs/runtime-lock.json" in paths + assert not any(p.startswith((".env", ".runtime", "runs", "prepared")) for p in paths) diff --git a/05-multi-harness-rl/tests/test_runtime.py b/05-multi-harness-rl/tests/test_runtime.py new file mode 100644 index 00000000..692c733c --- /dev/null +++ b/05-multi-harness-rl/tests/test_runtime.py @@ -0,0 +1,78 @@ +"""CPU integration checks against the fetched runtime and prepared task files.""" +import contextlib +import io +import json +import os +from pathlib import Path +import runpy +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +pytestmark = pytest.mark.skipif(not (ROOT / ".runtime/trl/trl").exists() or not (ROOT / "prepared/ready.json").exists(), + reason="Run bootstrap.py and prepare.py for runtime checks") + + +def test_corrected_grader_returns_only_numeric_fields(monkeypatch): + from prepare import GRADER_SHA256 + import hashlib + path = next((ROOT / "prepared/datasets/test/tasks").glob("*/tests/grader.py")) + assert hashlib.sha256(path.read_bytes()).hexdigest() == GRADER_SHA256 + module = runpy.run_path(str(path)) + monkeypatch.setenv("EXPECTED_ANSWER", "42") + monkeypatch.setenv("REWARD_MODE", "numeric") + for answer, expected in [("42", 1.0), ("17", 0.0), ("", 0.0)]: + monkeypatch.setattr(sys, "stdin", io.StringIO(answer)) + stream = io.StringIO() + with contextlib.redirect_stdout(stream): + module["main_json"]() + result = json.loads(stream.getvalue()) + assert result["correctness"] == expected + assert all(type(v) in {float, int} for v in result.values()) + + +def test_native_task_apis_share_ids_and_do_not_expose_answers(monkeypatch): + from runtime.bootstrap import activate + activate() + monkeypatch.setenv("DATA_AGENT_FROZEN_TASKS_DIR", str(ROOT / "prepared/datasets")) + monkeypatch.setenv("DAYTONA_COMPARISON_RUN", str(ROOT / "prepared")) + monkeypatch.setenv("WHITE_BOX_BASH_TASK_SOURCE", "harbor-frozen") + from data_agent_env.tasks import rows_for + from whitebox_bash.tasks import _load + for split, count in (("train", 1000), ("test", 250)): + native, whitebox = rows_for(split), _load(split) + assert len(native) == len(whitebox) == count + assert [t["task_id"] for t in native] == [t.metadata["source_name"] for t in whitebox] + assert all("answer" not in t.public() and "metadata" not in t.public() for t in whitebox) + + +def test_worker_options_match_pinned_runtime(): + import ast + import inspect + from runtime.bootstrap import activate + activate() + from trl.experimental.async_grpo.async_rollout_worker import _AsyncRolloutLoop + from trl.experimental.async_grpo.openenv_harness import _HarnessRolloutLoop + from hard_curriculum_train import FiniteLoop + names = set() + for cls in (_AsyncRolloutLoop, _HarnessRolloutLoop, FiniteLoop): + names.update(inspect.signature(cls.__init__).parameters) + names.add("max_outstanding_rollouts") + tree = ast.parse((ROOT / "train/blackbox.py").read_text()) + for call in ast.walk(tree): + if isinstance(call, ast.Call) and isinstance(call.func, ast.Name) and call.func.id == "Worker": + assert not {kw.arg for kw in call.keywords} - names + + +def test_native_openenv_partial_mask_reaches_trl(): + from runtime.bootstrap import activate + activate() + from data_agent_env.models import DataAgentRolloutResult, DataAgentTurn + from data_agent_env.harness import to_trace_entries + from trl.experimental.async_grpo.openenv_harness import _turns_from_trace + result = DataAgentRolloutResult(rollout_type="train", turns=[DataAgentTurn( + trainable=True, prompt_token_ids=[1, 2], completion_token_ids=[3, 4], + per_token_logps=[-.2, -.4], loss_mask=[0, 0, 1, 0])]) + assert _turns_from_trace(to_trace_entries(result))[0].output_mask == [1, 0] diff --git a/05-multi-harness-rl/train/adapters.py b/05-multi-harness-rl/train/adapters.py new file mode 100644 index 00000000..c95368ae --- /dev/null +++ b/05-multi-harness-rl/train/adapters.py @@ -0,0 +1,129 @@ +"""One task identity and reward boundary for both blackbox implementations.""" +from __future__ import annotations + +import json +from pathlib import Path + +from recipe import reward, write_json + + +def atif_count(path): + data = json.loads(Path(path).read_text()) + if not str(data.get("schema_version", "")).startswith("ATIF-"): + raise ValueError("Unknown native trajectory format") + calls = [c for s in data["steps"] if s.get("source") == "agent" for c in s.get("tool_calls", []) or []] + ids = [c.get("tool_call_id") for c in calls] + if any(not x for x in ids) or len(set(ids)) != len(ids): + raise ValueError("Missing or repeated native action IDs") + return len(ids) + + +def opencode_count(text): + rows = [json.loads(line) for line in text.splitlines() if line.strip()] + if not rows or not any(r.get("type") == "step_finish" for r in rows): + raise ValueError("Incomplete native OpenCode event stream") + parts = [r["part"] for r in rows if r.get("type") == "tool_use"] + ids = [p["callID"] for p in parts] + if len(set(ids)) != len(ids) or any(not i for i in ids): + raise ValueError("Ambiguous native tool actions") + if any(p.get("state", {}).get("status") not in {"completed", "error"} for p in parts): + raise ValueError("Unfinished tool action") + return len(ids) + + +class Session: + def __init__(self, inner, cfg, evidence, trial_root): + self.inner, self.cfg = inner, cfg + self.evidence, self.trial_root = evidence, Path(trial_root) + + def __getattr__(self, name): + inner = self.__dict__.get("inner") + if inner is None: + raise AttributeError(name) + return getattr(inner, name) + + @property + def result(self): + return getattr(self.inner, "result", getattr(self.inner, "_result", None)) + + def fetch_proxy_trace(self): + self.trace = self.inner.fetch_proxy_trace() + return self.trace + + def verify(self, transcript, final_state=None): + from openenv.core.harness import VerifyResult + + original = self.inner.verify(transcript, final_state) + result = self.result + correctness = original.env_reward + calls, error = None, None + if result is not None: + if self.cfg["mode"] == "opencode": + value = result.correctness + correctness = None if value is None else float(value >= 1) + calls = result.metadata.get("verified_native_actions") + else: + try: + calls = atif_count(self.trial_root / result.trial_name / "agent/trajectory.json") + except (OSError, ValueError, KeyError, TypeError) as exc: + error = type(exc).__name__ + shaped = reward(correctness, calls, verified=calls is not None, + weight=self.cfg["efficiency_weight"], budget=self.cfg["tool_budget"]) + record = {**self.evidence, "correctness": correctness, "tool_calls": calls, + "tool_count_verified": calls is not None, "tool_count_error": error, + "reward": shaped, "bonus": None if shaped is None else shaped - correctness} + if getattr(self, "trace", None): + self.trace[0]["recipe_reward"] = record + write_json(Path(self.cfg["output"]) / "rollouts" / f'{self.evidence["episode_id"]}.json', record) + return VerifyResult(env_reward=correctness, done=original.done, metrics={**original.metrics, **record}, + artifacts=original.artifacts) + + +def rollout_reward(outcome): + if not outcome.trace: + return None + return outcome.trace[0]["recipe_reward"]["reward"] + + +class Factory: + def __init__(self, cfg, data, server, vllm, groups, trial_root, split="train"): + self.cfg, self.data, self.server, self.vllm = cfg, str(data), server, vllm + self.groups, self.trial_root, self.split = groups, str(trial_root), split + manifest = json.loads((Path(data) / f"{split}_manifest.json").read_text()) + self.tasks = sorted(manifest["tasks"], key=lambda row: row["name"]) + self.instructions = [(Path(data) / "datasets" / split / "tasks" / r["name"] / "instruction.md").read_text() + for r in self.tasks] + + def rows(self): + return [{"prompt": [{"role": "user", "content": self.instructions[g["task_index"]]}], + "task_name": g["task_name"], "harness": g["harness"]} for g in self.groups] + + def create(self, task, seed=None, episode_id=None): + from harbor_env.harness import HarborSession, _instruction_of + + group_id = 0 if seed is None else seed + group = self.groups[group_id] + index = group["task_index"] + instruction = self.instructions[index] + if _instruction_of(task).strip() != instruction.strip(): + raise ValueError("Prompt and frozen task schedule disagree") + sampling = {k: self.cfg[k] for k in ("temperature", "top_p", "top_k")} + if self.cfg["mode"] == "opencode": + from data_agent_env import DataAgentEnv + from data_agent_env.harness import DataAgentSession + + inner = DataAgentSession(DataAgentEnv(self.server, message_timeout_s=1800), self.split, + index, instruction, llm_url=self.vllm, model=self.cfg["profile"]["id"], + sandbox=self.cfg["sandbox"], sampling=sampling, + agent_step_limit=self.cfg["agent_step_limit"], agent_timeout_s=self.cfg["agent_timeout_sec"]) + else: + from harbor_env import HarborEnv + + inner = HarborSession(env=HarborEnv(self.server, message_timeout_s=1800), owns_env=True, + split=str(Path(self.data) / "datasets" / self.split), task_index=index, instruction=instruction, + harness=group["harness"], sandbox=self.cfg["sandbox"], llm_url=self.vllm, + model=self.cfg["profile"]["id"], sampling=sampling, reward_key="correctness,reward", + agent_step_limit=self.cfg["agent_step_limit"], agent_timeout_sec=self.cfg["agent_timeout_sec"]) + evidence = {"group_id": group_id, "task_name": group["task_name"], "harness": group["harness"], + "episode_id": episode_id} + return Session(inner, self.cfg, evidence, self.trial_root) diff --git a/05-multi-harness-rl/train/blackbox.py b/05-multi-harness-rl/train/blackbox.py new file mode 100644 index 00000000..7c51a76e --- /dev/null +++ b/05-multi-harness-rl/train/blackbox.py @@ -0,0 +1,82 @@ +"""AsyncGRPO with atomic rollout admission and a finite, resumable task schedule.""" +import json +import os +from pathlib import Path + +from datasets import Dataset +from hard_curriculum_train import FiniteLoop, FiniteTrainer, FiniteWorker +from trl.experimental.async_grpo import AsyncGRPOConfig + +from recipe import digest, resume_state, schedule, task_rows, write_json +from runtime.models import tokenizer_for +from train.adapters import Factory, rollout_reward +from train.logging_utils import Checkpoints, Logs, publish_checkpoint + + +class GuardedLoop(FiniteLoop): + async def _score_group(self, group): + bundles = await super()._score_group(group) + self._ungraded_streak = 0 if bundles else getattr(self, "_ungraded_streak", 0) + 1 + if self._ungraded_streak >= 3: + raise RuntimeError("Three consecutive groups produced no trainable rollouts; inspect capture and environment logs") + return bundles + + +class Worker(FiniteWorker): + _loop_cls = GuardedLoop + + +def train(cfg, data, server, vllm, resume=None): + output = Path(cfg["output"]) + output.mkdir(parents=True, exist_ok=True) + groups = schedule(cfg, task_rows("train")) + write_json(output / "schedule.json", groups) + os.environ.update(CURRICULUM_GROUP_LIMIT=str(len(groups)), CURRICULUM_SCHEDULE_SHA256=digest(groups), + CURRICULUM_AUDIT=str(output / "audit")) + if resume: + state = json.loads((Path(resume) / "curriculum_state.json").read_text()) + state = resume_state(groups, state) + write_json(output / "resume-curriculum.json", state) + os.environ["CURRICULUM_RESUME_STATE"] = str(output / "resume-curriculum.json") + else: + os.environ.pop("CURRICULUM_RESUME_STATE", None) + if cfg["model"] == "lfm": + from packing import install + install() + factory = Factory(cfg, data, server, vllm, groups, output / "trials") + dataset = Dataset.from_list(factory.rows()) + tokenizer = tokenizer_for(cfg) + selector = None + if cfg["mode"] == "opencode": + from data_agent_env import opencode_agent_turns + selector = opencode_agent_turns + worker = Worker(harness_session_factory=factory, harness_adapter=None, + train_turn_fn=None, agent_turn_fn=selector, fork_threshold_tokens=0, rollout_reward_fn=rollout_reward, + model_name=cfg["profile"]["id"], processing_class=tokenizer, dataset=dataset, reward_funcs=[], + num_generations=cfg["num_generations"], max_inflight_tasks=cfg["max_inflight"], + max_outstanding_rollouts=cfg["max_outstanding_rollouts"], vllm_server_url=vllm, + max_tokens=cfg["max_completion_length"], temperature=cfg["temperature"], top_p=1.0, top_k=0, + log_completions=False) + args = AsyncGRPOConfig(output_dir=str(output), learning_rate=cfg["learning_rate"], + lr_scheduler_type="constant", warmup_steps=0, max_steps=cfg["max_steps"], + per_device_train_batch_size=cfg["batch_size"], gradient_accumulation_steps=cfg["gradient_accumulation_steps"], + num_generations=cfg["num_generations"], max_completion_length=cfg["max_completion_length"], + max_inflight_tasks=cfg["max_inflight"], max_staleness=cfg["max_staleness"], fork_threshold_tokens=0, + token_budget=cfg["token_budget"], temperature=cfg["temperature"], top_p=1.0, top_k=0, + optim="paged_adamw_8bit", bf16=True, dtype="bfloat16", gradient_checkpointing=True, + gradient_checkpointing_kwargs={"use_reentrant": False}, + model_init_kwargs={"revision": cfg["profile"]["revision"]}, + vllm_server_base_url=vllm, heartbeat_stale_after_s=900, + save_strategy="steps", save_steps=cfg["save_steps"], save_total_limit=None, + logging_steps=1, report_to="trackio", project=cfg["project"], run_name=cfg["run_name"], + trackio_space_id=cfg["trackio_space_id"], seed=cfg["seed"]) + trainer = FiniteTrainer(model=cfg["profile"]["id"], args=args, train_dataset=dataset, + rollout_worker=worker, max_row_tokens=cfg["max_model_len"], admission_dir=output / "audit") + trainer.add_callback(Logs()) + trainer.add_callback(Checkpoints(cfg)) + write_json(output / "trainer-config.json", args.to_dict()) + trainer.train(resume_from_checkpoint=str(resume) if resume else None) + checkpoint = output / f"checkpoint-{trainer.state.global_step}" + if not (checkpoint / "checkpoint.saved.json").exists(): + trainer._save_checkpoint(trainer.model, None) + publish_checkpoint(checkpoint, cfg, trainer.state.global_step, final=True) diff --git a/05-multi-harness-rl/train/generation.py b/05-multi-harness-rl/train/generation.py new file mode 100644 index 00000000..203c1321 --- /dev/null +++ b/05-multi-harness-rl/train/generation.py @@ -0,0 +1,34 @@ +"""Avoid grouping diverged tool histories as repeated initial prompts.""" + + +def group_size(prompts, requested): + if requested < 1: + raise ValueError("Group size must be positive") + if len(prompts) % requested: + return 1 + for start in range(0, len(prompts), requested): + if any(p != prompts[start] for p in prompts[start:start + requested]): + return 1 + return requested + + +def protect_generation(trainer): + client = trainer.vllm_generation.vllm_client + def reset_prefix_cache(): + # vLLM acknowledges cache resets with an empty HTTP 200 body. + response = client.session.post(f"{client.base_url}/reset_prefix_cache") + response.raise_for_status() + client.reset_prefix_cache = reset_prefix_cache + generate = trainer.vllm_generation.generate + def checked(*args, **kwargs): + prompts = kwargs.get("prompts", args[0] if args else None) + kwargs = {**kwargs, "num_generations": group_size(prompts, kwargs.get("num_generations", 1))} + output = generate(*args, **kwargs) + returned, completions, logprobs, ids = output + if returned != prompts: + raise ValueError("vLLM changed the supplied token prompts") + for completion, lps, token_ids in zip(completions, logprobs, ids, strict=True): + if len(completion) != len(lps) or token_ids != [[token] for token in completion]: + raise ValueError("Sampled token/logprob IDs do not agree") + return output + trainer.vllm_generation.generate = checked diff --git a/05-multi-harness-rl/train/logging_utils.py b/05-multi-harness-rl/train/logging_utils.py new file mode 100644 index 00000000..dee18bc3 --- /dev/null +++ b/05-multi-harness-rl/train/logging_utils.py @@ -0,0 +1,38 @@ +"""Durable scalar logs and checkpoint handoffs for a separate evaluation watcher.""" +import json +import math +from pathlib import Path +import time + +from transformers import TrainerCallback +from recipe import write_json + + +class Logs(TrainerCallback): + def on_log(self, args, state, control, logs=None, **kwargs): + values = {k: v for k, v in (logs or {}).items() + if isinstance(v, (int, float)) and math.isfinite(v)} + path = Path(args.output_dir) / "metrics.jsonl" + with path.open("a") as stream: + stream.write(json.dumps({"step": state.global_step, "time": time.time(), **values}) + "\n") + stream.flush() + + +def publish_checkpoint(path, cfg, step, *, final=False): + from runtime.checkpoints import saved + + path = Path(path) + saved(path, cfg, step) + write_json(path / "recipe.json", cfg) + write_json(path / "eval.request.json", {"step": step, "final": final, + "evaluate": final or step % cfg["eval_steps"] == 0, "created_at": time.time()}) + + +class Checkpoints(TrainerCallback): + def __init__(self, cfg): + self.cfg = cfg + + def on_save(self, args, state, control, **kwargs): + if state.is_world_process_zero: + publish_checkpoint(Path(args.output_dir) / f"checkpoint-{state.global_step}", self.cfg, + state.global_step, final=control.should_training_stop) diff --git a/05-multi-harness-rl/train/whitebox.py b/05-multi-harness-rl/train/whitebox.py new file mode 100644 index 00000000..95a7655c --- /dev/null +++ b/05-multi-harness-rl/train/whitebox.py @@ -0,0 +1,84 @@ +"""Synchronous GRPO: TRL owns generation and invokes native bash/SETA tools.""" +from pathlib import Path +import json +import math + +from datasets import Dataset +from trl import GRPOConfig, GRPOTrainer +from whitebox_bash import white_box_bash_env + +from recipe import schedule, task_rows, write_json +from runtime.models import tokenizer_for +from train.logging_utils import Checkpoints, Logs, publish_checkpoint +from train.generation import protect_generation + +SYSTEM = ("You are a terminal agent working in a sandbox. Use the available tools to inspect the " + "files and solve the task. Call submit_solution with the final answer itself, " + "not a command that would produce it.") + + +class TokenCheckedTrainer(GRPOTrainer): + def _generate_single_turn(self, prompt_ids, *args, **kwargs): + completions, logprobs = super()._generate_single_turn(prompt_ids, *args, **kwargs) + for prompt, completion, lps in zip(prompt_ids, completions, logprobs, strict=True): + if len(completion) != len(lps) or not all(math.isfinite(p) for p in lps): + raise ValueError("Invalid whitebox engine token/logprob capture") + self._captured_calls.append({"prompt_ids": prompt.copy(), "completion_ids": completion.copy(), + "logprobs": lps.copy()}) + return completions, logprobs + + def _generate(self, *args, **kwargs): + self._captured_calls = [] + return super()._generate(*args, **kwargs) + + def _tool_call_loop(self, prompts, prompt_ids, *args, **kwargs): + from whitebox_tito import audit_rows + result = super()._tool_call_loop(prompts, prompt_ids, *args, **kwargs) + masks, _, completions, logps, _, _, _ = result + checks = audit_rows(prompt_ids, completions, masks, logps, self._captured_calls) + with (Path(self.args.output_dir) / "token-audit.jsonl").open("a") as stream: + stream.write(json.dumps({"step": self.state.global_step, "rows": checks, + "engine_calls": len(self._captured_calls)}) + "\n") + return result + + +def train(cfg, data, server, vllm, resume=None): + from run import unused_port + output = Path(cfg["output"]) + output.mkdir(parents=True, exist_ok=True) + groups = schedule(cfg, task_rows("train")) + write_json(output / "schedule.json", groups) + rows = [{"prompt": [{"role": "system", "content": SYSTEM}, {"role": "user", "content": "Solve the task."}], + "split": "train", "index": g["task_index"]} for g in groups] + args = GRPOConfig(output_dir=str(output), learning_rate=cfg["learning_rate"], + lr_scheduler_type="constant", warmup_steps=0, beta=0.0, loss_type="dapo", + max_steps=cfg["max_steps"], num_train_epochs=1, shuffle_dataset=False, + num_generations=cfg["num_generations"], per_device_train_batch_size=1, + gradient_accumulation_steps=cfg["batch_size"] * cfg["gradient_accumulation_steps"], + max_completion_length=cfg["max_completion_length"], max_tool_calling_iterations=cfg["agent_step_limit"] - 1, + temperature=cfg["temperature"], top_p=1.0, top_k=0, + chat_template_kwargs={"enable_thinking": False, "preserve_thinking": True}, + generation_kwargs={"max_tokens": cfg["max_output_tokens"]}, + model_init_kwargs={"revision": cfg["profile"]["revision"], "dtype": "bfloat16"}, + optim="paged_adamw_8bit", bf16=True, gradient_checkpointing=True, + gradient_checkpointing_kwargs={"use_reentrant": False}, + use_vllm=True, vllm_mode="server", vllm_server_base_url=vllm, + vllm_group_port=unused_port(), + vllm_server_timeout=900, vllm_max_model_length=cfg["max_model_len"], + save_strategy="steps", save_steps=cfg["save_steps"], save_total_limit=None, + logging_steps=1, report_to="trackio", project=cfg["project"], run_name=cfg["run_name"], + trackio_space_id=cfg["trackio_space_id"], seed=cfg["seed"]) + trainer = TokenCheckedTrainer(model=cfg["profile"]["id"], processing_class=tokenizer_for(cfg), args=args, + train_dataset=Dataset.from_list(rows), reward_funcs=[], + environment_factory=white_box_bash_env(server, split="train", toolsets="bash,seta", step_limit=cfg["agent_step_limit"]), + callbacks=[Logs(), Checkpoints(cfg)]) + # This TRL version reads the outer config for text-only tool rollouts. + text_config = trainer.model.config.get_text_config() + trainer.model.config.max_position_embeddings = min(text_config.max_position_embeddings, cfg["max_model_len"]) + write_json(output / "trainer-config.json", args.to_dict()) + protect_generation(trainer) + trainer.train(resume_from_checkpoint=str(resume) if resume else None) + checkpoint = output / f"checkpoint-{trainer.state.global_step}" + if not (checkpoint / "checkpoint.saved.json").exists(): + trainer._save_checkpoint(trainer.model, None) + publish_checkpoint(checkpoint, cfg, trainer.state.global_step, final=True) diff --git a/README.md b/README.md index 2ec0a419..da002e61 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ results and README, plus the Hub repos it owns. They read in order but stand alo | **01** | **[LaTeX OCR](./01-latex-ocr/)** | Train Qwen3-VL-2B to read math images into LaTeX, with a verifiable reward. | 1 | 1 | 1 | ✅ stable | | **02** | **[Watercolour](./02-watercolour/)** | Train Qwen3.5-35B-A3B to paint watercolours by writing p5.brush sketches, rewarded by an aesthetic preference model. | 1 | 1 | 0 | ✅ trained | | **03** | **[GeoGuesser](./03-geoguesser/)** | Drop a VLM at a random street corner on Earth and score it on kilometres of error. | 1 | 1 | 1 | ✅ stable | +| **05** | **[SmolDataEnv: whitebox and multi-harness RL](./05-multi-harness-rl/)** | One task split and reward across SETA, native OpenCode and Harbor; HF Jobs and Slurm recipes. | 3 | 1 | 0 | 🚧 wip | Generated from each project's `project.yaml` by `tools/build_index.py`. Adding a project means From 78bc6522c6f18e63cabd2eab1144366e2531848c Mon Sep 17 00:00:00 2001 From: adithya-s-k Date: Wed, 30 Sep 2026 08:11:32 +0000 Subject: [PATCH 02/48] Support Ampere attention for HF A100 smoke --- 05-multi-harness-rl/REPRODUCE.md | 2 ++ 05-multi-harness-rl/VALIDATION.md | 7 ++++--- 05-multi-harness-rl/runtime/bootstrap.py | 3 ++- 05-multi-harness-rl/runtime/models.py | 9 +++++++++ 05-multi-harness-rl/runtime/patches.py | 10 ++++++++++ 05-multi-harness-rl/tests/test_contracts.py | 19 +++++++++++++++++++ 05-multi-harness-rl/train/blackbox.py | 6 ++++-- 7 files changed, 50 insertions(+), 6 deletions(-) diff --git a/05-multi-harness-rl/REPRODUCE.md b/05-multi-harness-rl/REPRODUCE.md index d5184c7c..93c307e2 100644 --- a/05-multi-harness-rl/REPRODUCE.md +++ b/05-multi-harness-rl/REPRODUCE.md @@ -38,6 +38,8 @@ python runtime/launch.py hf watch --run-name eval-watcher \ Repeat the smoke for each model/mode before launching that combination. The default GPU flavor is `h200x2`: one trainer GPU and one serving GPU; evaluation uses both GPUs as DP2 replicas. HF Jobs currently has no two-A100 flavor. Override `--flavor` only with hardware that has enough GPUs and memory. The base image is pinned by digest; Python packages and source revisions are pinned separately. +The A100 alternative is `--flavor a100x4`; this recipe uses two of those four GPUs, but the full allocation is billed. Async training selects FlashAttention 2 on Ampere and FlashAttention 3 on Hopper, preserving packed-sequence boundaries. A checked adapter enables this choice in the pinned TRL runtime. See `VALIDATION.md` for hardware qualification status. + The bucket is mounted at `/outputs`. A run writes to `/outputs/RUN_NAME`; the watcher hashes completed checkpoints before launching evals. Keep the watcher alive until all final evaluations finish. Failures remain in `eval-watcher.json` for investigation rather than resubmitting indefinitely. Relaunch a failed evaluation explicitly with the same output name to retain already graded pairs. ```bash diff --git a/05-multi-harness-rl/VALIDATION.md b/05-multi-harness-rl/VALIDATION.md index 2ca0e354..80cb56f5 100644 --- a/05-multi-harness-rl/VALIDATION.md +++ b/05-multi-harness-rl/VALIDATION.md @@ -14,7 +14,7 @@ This is a new non-thinking recipe. Historical training results are not its basel | Check | Result | |---|---| -| CPU contracts and fetched-runtime integration | 21 passed | +| CPU contracts and fetched-runtime integration | 22 passed, including Ampere attention selection | | Clean, minimal CI environment | 17 passed; four runtime checks skipped without fetched dependencies | | Native OpenEnv to TRL partial mask | `[1, 0]` preserved | | Native LFM checkpoint handoff | Both initial smoke checkpoints saved; file hashes verified | @@ -26,7 +26,7 @@ This is a new non-thinking recipe. Historical training results are not its basel | Whitebox checkpoint reload eval | 2/2 tasks graded; one correct answer | | Async checkpoint resume | Resumed step 1 to 2; committed group 0 was not replayed | | HF Jobs setup | Fresh Python 3.12 environment, locked installation and all 1,250 tasks prepared successfully | -| HF Jobs GPU smoke | Corrected submission accepted; allocation pending | +| HF Jobs GPU smoke | A100 replacement submitted; pulling container image | | Model | Mode | Local smoke wall time | Nonzero gradient observed? | |---|---|---|---| @@ -55,6 +55,7 @@ Local smoke artifacts are ignored under `runs/qualification/`: configuration, se - Whitebox checkpoint eval: `qwen-whitebox-v5-eval`, job `92821`. - Async resume: `lfm-native-resume-v1`, job `92822`. Only optimizer step 2 was executed; the new admission record contains group 2, while the restored state retains group 0. - [HF setup check](https://huggingface.co/jobs/FineEnvs/6abcb82f031314b696343d7d): completed with the corrected Python 3.12 bootstrap. -- [HF GPU smoke](https://huggingface.co/jobs/FineEnvs/6abcb8c8031314b696343daf): current submission, bounded to 40 minutes of runtime. Superseded queued submissions were canceled before allocation. +- [H200 GPU smoke](https://huggingface.co/jobs/FineEnvs/6abcb8c8031314b696343daf): canceled after remaining queued for hardware. +- [A100 GPU smoke](https://huggingface.co/jobs/FineEnvs/6abcc3d9031314b6963440a0): replacement on A100×4, bounded to 40 minutes. At this check it was pulling the image. It uses two GPUs and tests the Ampere FlashAttention 2 path; successful GPU execution is still pending. Before a long HF run, finish the GPU smoke and verify bucket checkpoint handoff. Then collect a fresh non-thinking baseline on the fixed 250-task test set. A tiny smoke establishes execution, not benchmark performance or throughput at concurrency 35. diff --git a/05-multi-harness-rl/runtime/bootstrap.py b/05-multi-harness-rl/runtime/bootstrap.py index e90f38db..30e3dde7 100644 --- a/05-multi-harness-rl/runtime/bootstrap.py +++ b/05-multi-harness-rl/runtime/bootstrap.py @@ -61,8 +61,9 @@ def fetch(item): link.unlink() shutil.copytree(destination / "envs" / folder, link, dirs_exist_ok=True) sys.path.insert(0, str(ROOT)) - from runtime.patches import apply + from runtime.patches import apply, apply_trl apply(packages) + apply_trl(RUNTIME / "trl") print(f"Verified {len(archive['files'])} runtime files; pinned TRL and OpenEnv ready.") diff --git a/05-multi-harness-rl/runtime/models.py b/05-multi-harness-rl/runtime/models.py index 877822f5..ec02f12b 100644 --- a/05-multi-harness-rl/runtime/models.py +++ b/05-multi-harness-rl/runtime/models.py @@ -2,6 +2,15 @@ from pathlib import Path +def training_attention_backend(capability): + major, _ = capability + if major == 9: + return "kernels-community/flash-attn3" + if major == 8: + return "kernels-community/flash-attn2" + raise ValueError(f"Unqualified training GPU capability: {capability}") + + def tokenizer_for(cfg, template_path=None): from copy import deepcopy from transformers import AutoTokenizer diff --git a/05-multi-harness-rl/runtime/patches.py b/05-multi-harness-rl/runtime/patches.py index 555d70a1..e55343f0 100644 --- a/05-multi-harness-rl/runtime/patches.py +++ b/05-multi-harness-rl/runtime/patches.py @@ -7,6 +7,16 @@ def replace(text, old, new): return text.replace(old, new, 1) +def apply_trl(root): + path = root / "trl/experimental/async_grpo/async_grpo_trainer.py" + old = 'attn_implementation="kernels-community/flash-attn3",' + new = 'attn_implementation=model_init_kwargs.pop("attn_implementation", "kernels-community/flash-attn3"),' + text = path.read_text() + if text.count(new) == 1 and old not in text: + return + path.write_text(replace(text, old, new)) + + def apply(packages): path = packages / "data_agent_env/server/capture.py" path.write_text(replace(path.read_text(), "max_output_tokens=16384,", diff --git a/05-multi-harness-rl/tests/test_contracts.py b/05-multi-harness-rl/tests/test_contracts.py index ebce8859..e4ab21f3 100644 --- a/05-multi-harness-rl/tests/test_contracts.py +++ b/05-multi-harness-rl/tests/test_contracts.py @@ -18,6 +18,25 @@ from runtime.models import check_visible_response +def test_ampere_attention_keeps_packed_flash_attention(tmp_path): + from runtime.models import training_attention_backend + from runtime.patches import apply_trl + assert training_attention_backend((8, 0)) == "kernels-community/flash-attn2" + assert training_attention_backend((9, 0)) == "kernels-community/flash-attn3" + with pytest.raises(ValueError, match="Unqualified"): + training_attention_backend((7, 0)) + path = tmp_path / "trl/experimental/async_grpo/async_grpo_trainer.py" + path.parent.mkdir(parents=True) + path.write_text('model = create_model_from_path(model, attn_implementation="kernels-community/flash-attn3", **model_init_kwargs)') + apply_trl(tmp_path) + apply_trl(tmp_path) + calls = [] + kwargs = {"attn_implementation": training_attention_backend((8, 0)), "revision": "fixed"} + exec(path.read_text(), {"create_model_from_path": lambda *a, **k: calls.append(k), + "model": "test", "model_init_kwargs": kwargs}) + assert calls == [{"attn_implementation": "kernels-community/flash-attn2", "revision": "fixed"}] + + def test_frozen_data_and_rotation(): train, test = task_rows("train"), task_rows("test") assert len(train) == 1000 and len(test) == 250 diff --git a/05-multi-harness-rl/train/blackbox.py b/05-multi-harness-rl/train/blackbox.py index 7c51a76e..d6eeccc7 100644 --- a/05-multi-harness-rl/train/blackbox.py +++ b/05-multi-harness-rl/train/blackbox.py @@ -4,11 +4,12 @@ from pathlib import Path from datasets import Dataset +import torch from hard_curriculum_train import FiniteLoop, FiniteTrainer, FiniteWorker from trl.experimental.async_grpo import AsyncGRPOConfig from recipe import digest, resume_state, schedule, task_rows, write_json -from runtime.models import tokenizer_for +from runtime.models import tokenizer_for, training_attention_backend from train.adapters import Factory, rollout_reward from train.logging_utils import Checkpoints, Logs, publish_checkpoint @@ -65,7 +66,8 @@ def train(cfg, data, server, vllm, resume=None): token_budget=cfg["token_budget"], temperature=cfg["temperature"], top_p=1.0, top_k=0, optim="paged_adamw_8bit", bf16=True, dtype="bfloat16", gradient_checkpointing=True, gradient_checkpointing_kwargs={"use_reentrant": False}, - model_init_kwargs={"revision": cfg["profile"]["revision"]}, + model_init_kwargs={"revision": cfg["profile"]["revision"], + "attn_implementation": training_attention_backend(torch.cuda.get_device_capability())}, vllm_server_base_url=vllm, heartbeat_stale_after_s=900, save_strategy="steps", save_steps=cfg["save_steps"], save_total_limit=None, logging_steps=1, report_to="trackio", project=cfg["project"], run_name=cfg["run_name"], From 77a7457821f778583f4a6a4bcf4b3961a085ab5b Mon Sep 17 00:00:00 2001 From: adithya-s-k Date: Wed, 30 Sep 2026 08:25:17 +0000 Subject: [PATCH 03/48] Add Harbor smoke checkpoint reload evaluation --- 05-multi-harness-rl/REPRODUCE.md | 2 ++ 05-multi-harness-rl/VALIDATION.md | 8 +++++--- 05-multi-harness-rl/run.py | 16 +++++++++++++++- 05-multi-harness-rl/runtime/launch.py | 5 +++++ 05-multi-harness-rl/tests/test_contracts.py | 21 +++++++++++++++++++++ 5 files changed, 48 insertions(+), 4 deletions(-) diff --git a/05-multi-harness-rl/REPRODUCE.md b/05-multi-harness-rl/REPRODUCE.md index 93c307e2..5a5e910b 100644 --- a/05-multi-harness-rl/REPRODUCE.md +++ b/05-multi-harness-rl/REPRODUCE.md @@ -40,6 +40,8 @@ Repeat the smoke for each model/mode before launching that combination. The defa The A100 alternative is `--flavor a100x4`; this recipe uses two of those four GPUs, but the full allocation is billed. Async training selects FlashAttention 2 on Ampere and FlashAttention 3 on Hopper, preserving packed-sequence boundaries. A checked adapter enables this choice in the pinned TRL runtime. See `VALIDATION.md` for hardware qualification status. +For a combined training and reload check, add `--smoke-eval --concurrency 4` to the smoke command. After two updates, it hashes checkpoint 2, restarts the services and evaluates two test tasks per harness. Blackbox modes run all four Harbor harnesses, producing eight graded pairs when complete. The evaluation lives under `reload-eval/`; a training failure prevents it from starting. + The bucket is mounted at `/outputs`. A run writes to `/outputs/RUN_NAME`; the watcher hashes completed checkpoints before launching evals. Keep the watcher alive until all final evaluations finish. Failures remain in `eval-watcher.json` for investigation rather than resubmitting indefinitely. Relaunch a failed evaluation explicitly with the same output name to retain already graded pairs. ```bash diff --git a/05-multi-harness-rl/VALIDATION.md b/05-multi-harness-rl/VALIDATION.md index 80cb56f5..e45e30fa 100644 --- a/05-multi-harness-rl/VALIDATION.md +++ b/05-multi-harness-rl/VALIDATION.md @@ -26,7 +26,8 @@ This is a new non-thinking recipe. Historical training results are not its basel | Whitebox checkpoint reload eval | 2/2 tasks graded; one correct answer | | Async checkpoint resume | Resumed step 1 to 2; committed group 0 was not replayed | | HF Jobs setup | Fresh Python 3.12 environment, locked installation and all 1,250 tasks prepared successfully | -| HF Jobs GPU smoke | A100 replacement submitted; pulling container image | +| HF Jobs native OpenCode GPU smoke | A100 completed two steps; both saved checkpoint inventories match the bucket | +| HF Jobs Harbor smoke and reload eval | Running on A100; dependency installation completed | | Model | Mode | Local smoke wall time | Nonzero gradient observed? | |---|---|---|---| @@ -56,6 +57,7 @@ Local smoke artifacts are ignored under `runs/qualification/`: configuration, se - Async resume: `lfm-native-resume-v1`, job `92822`. Only optimizer step 2 was executed; the new admission record contains group 2, while the restored state retains group 0. - [HF setup check](https://huggingface.co/jobs/FineEnvs/6abcb82f031314b696343d7d): completed with the corrected Python 3.12 bootstrap. - [H200 GPU smoke](https://huggingface.co/jobs/FineEnvs/6abcb8c8031314b696343daf): canceled after remaining queued for hardware. -- [A100 GPU smoke](https://huggingface.co/jobs/FineEnvs/6abcc3d9031314b6963440a0): replacement on A100×4, bounded to 40 minutes. At this check it was pulling the image. It uses two GPUs and tests the Ampere FlashAttention 2 path; successful GPU execution is still pending. +- [A100 native OpenCode smoke](https://huggingface.co/jobs/FineEnvs/6abcc3d9031314b6963440a0): completed. Runtime from container start to finish was 10m 05s; the trainer reported 335.3s. Both optimizer steps had zero reward contrast and zero gradient, so this confirms execution rather than learning improvement. Six completed rollout records had verified native tool counts. Both checkpoints include model, optimizer and RNG state; all 13 recorded file sizes match the persistent bucket. Byte hashes and cross-job reload remain unverified for this checkpoint. Evidence downloaded to `runs/qualification/hf-a100-native-v1/`. +- [A100 Harbor smoke](https://huggingface.co/jobs/FineEnvs/6abcc72d031314b696344167): two-step multi-harness training followed by checkpoint-2 reload evaluation on two tasks × four harnesses, concurrency four, 45-minute cap. Submitted with `--smoke-eval`; running, with dependency installation completed at this check. This tests all four evaluation adapters, while two training steps alone need not consume all four harnesses. The new orchestration has 19 passing CPU contract checks, including aborting reload after training failure. -Before a long HF run, finish the GPU smoke and verify bucket checkpoint handoff. Then collect a fresh non-thinking baseline on the fixed 250-task test set. A tiny smoke establishes execution, not benchmark performance or throughput at concurrency 35. +Before a long HF run, finish the Harbor smoke and verify checkpoint reload. A same-job reload does not qualify a separate evaluation job reading the bucket; that handoff remains a separate check. Then collect a fresh non-thinking baseline on the fixed 250-task test set. A tiny smoke establishes execution, not benchmark performance or throughput at concurrency 35. diff --git a/05-multi-harness-rl/run.py b/05-multi-harness-rl/run.py index 7697c10e..fdf11c78 100644 --- a/05-multi-harness-rl/run.py +++ b/05-multi-harness-rl/run.py @@ -26,6 +26,7 @@ def parser(): p.add_argument("--resume", type=Path) p.add_argument("--concurrency", type=int) p.add_argument("--limit", type=int, help="Eval tasks per harness; omit for all 250") + p.add_argument("--smoke-eval", action="store_true", help="Reload the smoke checkpoint for two tasks per harness") p.add_argument("--server", help="External URL, or a local bind URL; owned services default to unused ports") p.add_argument("--vllm-url") p.add_argument("--capture-port", type=int) @@ -140,6 +141,8 @@ def spawn(command, name, child_env=None): def main(): args = parser().parse_args() + if args.smoke_eval and (args.action != "smoke" or args.external_services): + raise ValueError("--smoke-eval requires a smoke with job-local services") if args.external_services and (not args.server or not args.vllm_url): raise ValueError("External services require --server and --vllm-url") args.server = args.server or f"http://127.0.0.1:{unused_port()}" @@ -156,7 +159,18 @@ def main(): return if args.action in {"train", "eval", "smoke"}: if not args.external_services: - return launch_local(args, cfg) + launch_local(args, cfg) + if args.smoke_eval: + from runtime.checkpoints import make_ready + checkpoint = Path(cfg["output"]) / f'checkpoint-{cfg["max_steps"]}' + make_ready(checkpoint) + eval_args = argparse.Namespace(**vars(args)) + eval_args.action, eval_args.checkpoint, eval_args.resume = "eval", checkpoint, None + eval_args.limit = args.limit or 2 + eval_cfg = {**cfg, "output": str(Path(cfg["output"]) / "reload-eval"), + "run_name": cfg["run_name"] + "-reload", "eval_concurrency": args.concurrency or 4} + launch_local(eval_args, eval_cfg) + return args.action = "_eval" if args.action == "eval" else "_train" from runtime.bootstrap import activate activate() diff --git a/05-multi-harness-rl/runtime/launch.py b/05-multi-harness-rl/runtime/launch.py index 9f325eea..78bf66e9 100644 --- a/05-multi-harness-rl/runtime/launch.py +++ b/05-multi-harness-rl/runtime/launch.py @@ -47,14 +47,19 @@ def main(): p.add_argument("--resume") p.add_argument("--concurrency", type=int, default=35) p.add_argument("--limit", type=int) + p.add_argument("--smoke-eval", action="store_true") p.add_argument("--max-active-evals", type=int, default=1) p.add_argument("--submit", action="store_true") args = p.parse_args() + if args.smoke_eval and args.action != "smoke": + p.error("--smoke-eval requires the smoke action") if Path(args.run_name).name != args.run_name or args.run_name in {".", ".."}: p.error("run-name must be a single directory name") output = str(Path("/outputs" if args.backend == "hf" else args.output_root.resolve()) / args.run_name) flags = [args.action, "--model", args.model, "--mode", args.mode, "--run-name", args.run_name, "--output", output, "--concurrency", str(args.concurrency)] + if args.smoke_eval: + flags.append("--smoke-eval") if args.config: path = args.config if args.backend == "hf" and not str(path).startswith("/outputs/"): diff --git a/05-multi-harness-rl/tests/test_contracts.py b/05-multi-harness-rl/tests/test_contracts.py index e4ab21f3..90ad6ef4 100644 --- a/05-multi-harness-rl/tests/test_contracts.py +++ b/05-multi-harness-rl/tests/test_contracts.py @@ -37,6 +37,27 @@ def test_ampere_attention_keeps_packed_flash_attention(tmp_path): assert calls == [{"attn_implementation": "kernels-community/flash-attn2", "revision": "fixed"}] +def test_smoke_reload_is_separate_and_training_failure_stops_eval(tmp_path, monkeypatch): + import run + import runtime.checkpoints + calls = [] + monkeypatch.setattr(sys, "argv", ["run.py", "smoke", "--model", "lfm", "--mode", "multi-harness", + "--output", str(tmp_path), "--smoke-eval", "--concurrency", "4"]) + monkeypatch.setattr(runtime.checkpoints, "make_ready", lambda p: calls.append(("ready", p))) + monkeypatch.setattr(run, "launch_local", lambda a, c: calls.append((a.action, a.checkpoint, c["output"], a.limit))) + run.main() + assert calls[0][0] == "smoke" + assert calls[1] == ("ready", tmp_path / "checkpoint-2") + assert calls[2] == ("eval", tmp_path / "checkpoint-2", str(tmp_path / "reload-eval"), 2) + calls.clear() + def fail(*args): + raise RuntimeError("training failed") + monkeypatch.setattr(run, "launch_local", fail) + with pytest.raises(RuntimeError, match="training failed"): + run.main() + assert not calls + + def test_frozen_data_and_rotation(): train, test = task_rows("train"), task_rows("test") assert len(train) == 1000 and len(test) == 250 From f55b603b694ba85b59146b88c19fd4c6469e5be3 Mon Sep 17 00:00:00 2001 From: adithya-s-k Date: Wed, 30 Sep 2026 09:02:52 +0000 Subject: [PATCH 04/48] Record successful HF Harbor smoke and reload evaluation --- 05-multi-harness-rl/VALIDATION.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/05-multi-harness-rl/VALIDATION.md b/05-multi-harness-rl/VALIDATION.md index e45e30fa..37451e07 100644 --- a/05-multi-harness-rl/VALIDATION.md +++ b/05-multi-harness-rl/VALIDATION.md @@ -27,7 +27,7 @@ This is a new non-thinking recipe. Historical training results are not its basel | Async checkpoint resume | Resumed step 1 to 2; committed group 0 was not replayed | | HF Jobs setup | Fresh Python 3.12 environment, locked installation and all 1,250 tasks prepared successfully | | HF Jobs native OpenCode GPU smoke | A100 completed two steps; both saved checkpoint inventories match the bucket | -| HF Jobs Harbor smoke and reload eval | Running on A100; dependency installation completed | +| HF Jobs Harbor smoke and reload eval | Completed two training steps; checkpoint reload graded 8/8 pairs with TiTO and verified tool counts | | Model | Mode | Local smoke wall time | Nonzero gradient observed? | |---|---|---|---| @@ -58,6 +58,6 @@ Local smoke artifacts are ignored under `runs/qualification/`: configuration, se - [HF setup check](https://huggingface.co/jobs/FineEnvs/6abcb82f031314b696343d7d): completed with the corrected Python 3.12 bootstrap. - [H200 GPU smoke](https://huggingface.co/jobs/FineEnvs/6abcb8c8031314b696343daf): canceled after remaining queued for hardware. - [A100 native OpenCode smoke](https://huggingface.co/jobs/FineEnvs/6abcc3d9031314b6963440a0): completed. Runtime from container start to finish was 10m 05s; the trainer reported 335.3s. Both optimizer steps had zero reward contrast and zero gradient, so this confirms execution rather than learning improvement. Six completed rollout records had verified native tool counts. Both checkpoints include model, optimizer and RNG state; all 13 recorded file sizes match the persistent bucket. Byte hashes and cross-job reload remain unverified for this checkpoint. Evidence downloaded to `runs/qualification/hf-a100-native-v1/`. -- [A100 Harbor smoke](https://huggingface.co/jobs/FineEnvs/6abcc72d031314b696344167): two-step multi-harness training followed by checkpoint-2 reload evaluation on two tasks × four harnesses, concurrency four, 45-minute cap. Submitted with `--smoke-eval`; running, with dependency installation completed at this check. This tests all four evaluation adapters, while two training steps alone need not consume all four harnesses. The new orchestration has 19 passing CPU contract checks, including aborting reload after training failure. +- [A100 Harbor smoke](https://huggingface.co/jobs/FineEnvs/6abcc72d031314b696344167): two-step multi-harness training followed by checkpoint-2 reload evaluation on two tasks × four harnesses, concurrency four, 45-minute cap. Completed with `--smoke-eval` in 21m 17s overall; training took 627.9s. Both updates had zero reward contrast and zero gradient. Checkpoint 2 was hashed and reloaded with restarted services. All eight evaluation pairs graded on their first attempt, with TiTO and verified tool counts: OpenCode 1/2, Claude Code 1/2, Codex 1/2, Mini-SWE-Agent 2/2. These two medium tasks are a smoke, not a benchmark. Evidence is under `runs/qualification/hf-a100-harbor-v1/`. This tests all four evaluation adapters, while two training steps alone need not consume all four harnesses. The new orchestration has 19 passing CPU contract checks, including aborting reload after training failure. -Before a long HF run, finish the Harbor smoke and verify checkpoint reload. A same-job reload does not qualify a separate evaluation job reading the bucket; that handoff remains a separate check. Then collect a fresh non-thinking baseline on the fixed 250-task test set. A tiny smoke establishes execution, not benchmark performance or throughput at concurrency 35. +The native and Harbor HF execution smokes passed. A same-job reload does not qualify a separate evaluation job reading the bucket; that handoff remains a separate check. Neither HF training sample had reward contrast, so a positive-gradient HF check also remains before claiming learning validation. Then collect a fresh non-thinking baseline on the fixed 250-task test set. A tiny smoke establishes execution, not benchmark performance or throughput at concurrency 35. From 1d5b82121ebc12daf067dd28079838fa221933c0 Mon Sep 17 00:00:00 2001 From: adithya-s-k Date: Wed, 30 Sep 2026 09:14:59 +0000 Subject: [PATCH 05/48] Add 100-step baseline-to-checkpoint pilots for HF Jobs and Slurm --- 05-multi-harness-rl/README.md | 2 + 05-multi-harness-rl/REPRODUCE.md | 28 +++++- 05-multi-harness-rl/eval/evaluate.py | 13 ++- 05-multi-harness-rl/recipe.py | 22 +++++ 05-multi-harness-rl/run.py | 7 +- 05-multi-harness-rl/runtime/launch.py | 13 ++- 05-multi-harness-rl/runtime/pilot.py | 101 ++++++++++++++++++++ 05-multi-harness-rl/tests/test_contracts.py | 76 +++++++++++++++ 8 files changed, 253 insertions(+), 9 deletions(-) create mode 100644 05-multi-harness-rl/runtime/pilot.py diff --git a/05-multi-harness-rl/README.md b/05-multi-harness-rl/README.md index 474ee4eb..f29cade9 100644 --- a/05-multi-harness-rl/README.md +++ b/05-multi-harness-rl/README.md @@ -12,6 +12,8 @@ This is a focused successor to the archived `04-data-agent` experiment. Start wi The OpenEnv server runs **inside each training or evaluation job**. Daytona runs the isolated task containers. No environment Space is needed. Training and evaluation use separate GPU allocations, inference engines and environment processes; they still share your sandbox quota and storage bandwidth. +For initial qualification, the [100-step pilot](REPRODUCE.md#100-step-pilot) runs baseline evaluation, training and checkpoint evaluation sequentially in one allocation. It uses the same training configuration and a fixed evaluation subset, with a final comparison report. + ## Shared experiment | Setting | Value | diff --git a/05-multi-harness-rl/REPRODUCE.md b/05-multi-harness-rl/REPRODUCE.md index 5a5e910b..40206260 100644 --- a/05-multi-harness-rl/REPRODUCE.md +++ b/05-multi-harness-rl/REPRODUCE.md @@ -51,7 +51,33 @@ python runtime/launch.py hf eval --model lfm --mode multi-harness \ --bucket FineEnvs/YOUR_BUCKET --concurrency 35 ``` -For online Trackio, set `trackio_space_id` in `configs/default.json` before submission. Logs and offline Trackio data remain in the bucket regardless. Use `hf jobs ps` and `hf jobs logs JOB_ID` to monitor. The new HF container path still needs an actual GPU Job smoke; a successful local smoke alone does not qualify it. +For online Trackio, set `trackio_space_id` in `configs/default.json` before submission. Logs and offline Trackio data remain in the bucket regardless. Use `hf jobs ps` and `hf jobs logs JOB_ID` to monitor. Native OpenCode and Harbor passed HF A100 smokes; check `VALIDATION.md` for the remaining qualifications. + +## 100-step pilot + +The `pilot` action runs baseline eval, training to step 100, then checkpoint-100 eval in one allocation. Services restart between phases, so evaluation does not compete with training. It keeps the normal batch size, eight rollouts, LR `3e-6`, reward and task schedule; only the step ceiling and evaluation size change. Checkpoints save at 50 and 100. Do not start the separate eval watcher for these pilots. + +```bash +for mode in whitebox opencode multi-harness; do + python runtime/launch.py hf pilot --model lfm --mode "$mode" \ + --run-name "pilot-lfm-$mode-v1" --bucket FineEnvs/YOUR_BUCKET \ + --flavor a100x4 --timeout 12h --limit 25 --concurrency 35 +done +``` + +These commands are dry runs; add `--submit` when ready. Use `--model qwen` with distinct names for Qwen. `--limit 25` selects the same seeded subset for every mode and checkpoint: 3 easy, 12 medium, 10 hard. That gives 100 pairs per blackbox eval and 25 episodes per whitebox eval. Use `--limit 250` for the full test set. Small subsets are noisy; 100 training steps do not guarantee an improvement. + +Each output contains `pilot.json`, `baseline/`, `train/`, `checkpoint-100/` and a final `comparison.json`. The report includes per-harness pass@1, matched-success tool/token savings, first/last 20-update reward means and zero/nonzero gradient counts. Raw logs, task IDs and offline Trackio remain available. An incomplete baseline stops the pilot before training. To resume, use the same name/config plus `--resume /outputs/RUN_NAME/train/checkpoint-50`; completed phases are retained. The 12-hour cap is a limit, not an ETA. + +The Slurm equivalent uses the same phases and configuration, with two GPUs: + +```bash +python runtime/launch.py slurm pilot --partition YOUR_GPU_PARTITION \ + --model lfm --mode multi-harness --run-name pilot-lfm-multi-harness-v1 \ + --timeout 12h --limit 25 --concurrency 35 +``` + +Use a local checkpoint path for Slurm resume. Inspect the plan, then add `--submit`. No additional watcher, Space or Slurm job is required for the pilot. ## 3. Slurm or a local two-GPU machine diff --git a/05-multi-harness-rl/eval/evaluate.py b/05-multi-harness-rl/eval/evaluate.py index 01f59fc4..921644f0 100644 --- a/05-multi-harness-rl/eval/evaluate.py +++ b/05-multi-harness-rl/eval/evaluate.py @@ -6,7 +6,7 @@ import time import uuid -from recipe import digest, summary, task_rows, write_json +from recipe import digest, eval_rows, reward, summary, task_rows, write_json from train.adapters import Factory @@ -79,7 +79,9 @@ def whitebox_episode(cfg, server, vllm, index): correctness = env.get_reward() if not math.isfinite(correctness): raise ValueError("Ungraded whitebox episode") - return {"correctness": float(correctness >= 1), "tool_calls": calls, + correctness = float(correctness >= 1) + shaped = reward(correctness, calls, verified=True, weight=cfg["efficiency_weight"], budget=cfg["tool_budget"]) + return {"correctness": correctness, "tool_calls": calls, "tool_count_verified": True, "reward": shaped, "generated_tokens": generated, "prompt_tokens": inputs} finally: try: @@ -94,10 +96,11 @@ def evaluate(cfg, data, server, vllm, *, limit=None, checkpoint="baseline", max_ out = Path(cfg["output"]) / "eval" out.mkdir(parents=True, exist_ok=True) tasks = sorted(task_rows("test"), key=lambda row: row["name"]) - selected = tasks[:limit] if limit else tasks + selected = eval_rows(tasks, limit, stratified=cfg.get("eval_selection") == "stratified", seed=cfg["seed"]) + indices = {row["name"]: i for i, row in enumerate(tasks)} harnesses = ["whitebox"] if cfg["mode"] == "whitebox" else cfg["harnesses"] - groups = [{"task_name": row["name"], "task_index": i, "harness": h} - for i, row in enumerate(selected) for h in harnesses] + groups = [{"task_name": row["name"], "task_index": indices[row["name"]], "harness": h} + for row in selected for h in harnesses] identity = {"model": cfg["profile"], "checkpoint": str(checkpoint), "tasks": selected, "harnesses": harnesses, "template": cfg["profile"]["template"], "config": cfg} signature = digest(identity) diff --git a/05-multi-harness-rl/recipe.py b/05-multi-harness-rl/recipe.py index 94a926b0..40bb1873 100644 --- a/05-multi-harness-rl/recipe.py +++ b/05-multi-harness-rl/recipe.py @@ -4,6 +4,7 @@ import hashlib import json import math +import random from pathlib import Path import uuid @@ -58,6 +59,26 @@ def task_rows(split): return rows +def eval_rows(rows, limit=None, *, stratified=False, seed=0): + rows = sorted(rows, key=lambda row: row["name"]) + if limit is None: + return rows + if not 1 <= limit <= len(rows): + raise ValueError("Eval limit must fit the fixed test set") + if not stratified: + return rows[:limit] + strata = {d: [r for r in rows if r["difficulty"] == d] for d in sorted({r["difficulty"] for r in rows})} + exact = {d: limit * len(group) / len(rows) for d, group in strata.items()} + counts = {d: int(n) for d, n in exact.items()} + for d in sorted(strata, key=lambda d: (-(exact[d] - counts[d]), d))[:limit - sum(counts.values())]: + counts[d] += 1 + rng = random.Random(seed) + selected = [] + for d, group in strata.items(): + selected.extend(rng.sample(group, counts[d])) + return sorted(selected, key=lambda row: row["name"]) + + def schedule(cfg, rows): catalog = {name: i for i, name in enumerate(sorted(r["name"] for r in rows))} harnesses = cfg["harnesses"] if cfg["mode"] == "multi-harness" else [cfg["mode"]] @@ -100,6 +121,7 @@ def summary(records, expected): "pass_at_1": successes / len(graded) if graded else None, "coverage": len(graded) / expected if expected else 0, "complete": len(graded) == expected, + "mean_shaped_reward": mean(r.get("reward") for r in graded), "mean_tool_calls": mean(r.get("tool_calls") for r in graded), "mean_generated_tokens": mean(r.get("generated_tokens") for r in graded)} diff --git a/05-multi-harness-rl/run.py b/05-multi-harness-rl/run.py index fdf11c78..ca8c8739 100644 --- a/05-multi-harness-rl/run.py +++ b/05-multi-harness-rl/run.py @@ -15,7 +15,7 @@ def parser(): p = argparse.ArgumentParser(description=__doc__) - p.add_argument("action", choices=["plan", "train", "eval", "smoke", "environment", "_train", "_eval"]) + p.add_argument("action", choices=["plan", "train", "eval", "smoke", "pilot", "environment", "_train", "_eval"]) p.add_argument("--config", type=Path) p.add_argument("--model", choices=["lfm", "qwen"]) p.add_argument("--mode", choices=MODES) @@ -151,6 +151,11 @@ def main(): cfg = config(args.config, model=args.model, mode=args.mode, eval_concurrency=args.concurrency) name = args.run_name or cfg.get("run_name") or f'{cfg["model"]}-{cfg["mode"]}-{time.strftime("%Y%m%d-%H%M%S")}' cfg.update(run_name=name, output=str((args.output or Path(cfg.get("output", ROOT / "runs" / name))).resolve())) + if args.action == "pilot": + if args.external_services or args.checkpoint: + raise ValueError("Pilot owns its services; use --resume only for a saved pilot training checkpoint") + from runtime.pilot import pilot + return pilot(args, cfg) if args.action == "smoke": cfg.update(max_steps=2, save_steps=1, eval_steps=1, num_generations=2, max_inflight=4, max_outstanding_rollouts=4, batch_size=1, gradient_accumulation_steps=2) diff --git a/05-multi-harness-rl/runtime/launch.py b/05-multi-harness-rl/runtime/launch.py index 78bf66e9..ea23c5f7 100644 --- a/05-multi-harness-rl/runtime/launch.py +++ b/05-multi-harness-rl/runtime/launch.py @@ -3,6 +3,7 @@ import json import os from pathlib import Path +import re import shlex import shutil import subprocess @@ -14,6 +15,14 @@ from recipe import MODES, write_json +def slurm_time(value): + match = re.fullmatch(r"([1-9][0-9]*)([hms])", value) + if not match: + raise ValueError("Timeout must be a positive duration such as 12h or 40m") + seconds = int(match[1]) * {"h": 3600, "m": 60, "s": 1}[match[2]] + return f"{seconds // 3600:02}:{seconds // 60 % 60:02}:{seconds % 60:02}" + + def stage(destination): destination = Path(destination) allowed = {"run.py", "recipe.py", "prepare.py", "requirements.txt", "requirements.lock", "README.md", "REPRODUCE.md"} @@ -30,7 +39,7 @@ def stage(destination): def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument("backend", choices=["hf", "slurm"]) - p.add_argument("action", choices=["train", "eval", "smoke", "watch"]) + p.add_argument("action", choices=["train", "eval", "smoke", "pilot", "watch"]) p.add_argument("--model", choices=["lfm", "qwen"], default="lfm") p.add_argument("--config", type=Path) p.add_argument("--mode", choices=MODES, default="multi-harness") @@ -112,7 +121,7 @@ def main(): "--backend", "slurm", "--partition", args.partition, "--data", str(args.data.resolve()), "--concurrency", str(args.concurrency), "--max-active", str(args.max_active_evals), "--submit"] submit = ["sbatch", "--parsable", "--partition", args.partition, "--nodes=1", "--gpus=2", - "--cpus-per-task=16", "--mem=192G", "--time=24:00:00", "--job-name", args.run_name, + "--cpus-per-task=16", "--mem=192G", "--time=" + slurm_time(args.timeout), "--job-name", args.run_name, "--output", str(args.output_root / (args.run_name + "-%j.log")), "--wrap", shlex.join(command)] if args.action == "watch": submit.remove("--gpus=2") diff --git a/05-multi-harness-rl/runtime/pilot.py b/05-multi-harness-rl/runtime/pilot.py new file mode 100644 index 00000000..3c4ac63d --- /dev/null +++ b/05-multi-harness-rl/runtime/pilot.py @@ -0,0 +1,101 @@ +"""Baseline, 100 training updates and checkpoint evaluation in one allocation.""" +import argparse +import json +from pathlib import Path +import time + +from recipe import digest, eval_rows, mean, task_rows, write_json +from runtime.checkpoints import make_ready + + +def pilot(args, cfg): + from run import launch_local + + cfg = {**cfg, "max_steps": 100, "save_steps": 50, "eval_steps": 100, "eval_selection": "stratified"} + root = Path(cfg["output"]) + root.mkdir(parents=True, exist_ok=True) + limit = args.limit if args.limit is not None else 25 + selected = eval_rows(task_rows("test"), limit, stratified=True, seed=cfg["seed"]) + identity = {"config": cfg, "eval_tasks": selected} + signature = digest(identity) + manifest = root / "pilot.json" + state = json.loads(manifest.read_text()) if manifest.exists() else { + "sha256": signature, **identity, "phases": {}} + if state["sha256"] != signature: + raise ValueError("Pilot output belongs to a different configuration or test selection") + if args.resume and not state["phases"].get("baseline", {}).get("complete"): + raise ValueError("Resume requires this pilot's completed baseline") + + for phase in ("baseline", "train", "checkpoint-100"): + if state["phases"].get(phase, {}).get("complete"): + continue + entry = {"complete": False, "started_at": time.time()} + state["phases"][phase] = entry + write_json(manifest, state) + current = argparse.Namespace(**vars(args)) + current.action = "train" if phase == "train" else "eval" + current.limit = limit + current.checkpoint = None + current.resume = args.resume if phase == "train" else None + current.smoke_eval = False + phase_cfg = {**cfg, "output": str(root / phase), "run_name": cfg["run_name"] + "-" + phase} + try: + if current.resume: + make_ready(current.resume) + if phase == "checkpoint-100": + current.checkpoint = root / "train/checkpoint-100" + make_ready(current.checkpoint) + launch_local(current, phase_cfg) + if phase != "train": + report = json.loads((Path(phase_cfg["output"]) / "eval/summary.json").read_text()) + if not report["complete"]: + raise ValueError("Pilot requires complete evaluation coverage") + entry["evaluation"] = report + else: + marker = root / "train/checkpoint-100/checkpoint.saved.json" + if not marker.exists(): + raise ValueError("Training ended before checkpoint 100") + entry.update(complete=True, finished_at=time.time()) + write_json(manifest, state) + except BaseException as exc: + entry.update(error=type(exc).__name__, finished_at=time.time()) + write_json(manifest, state) + raise + write_json(root / "comparison.json", comparison(root, state)) + + +def comparison(root, state): + before = state["phases"]["baseline"]["evaluation"] + after = state["phases"]["checkpoint-100"]["evaluation"] + metrics = [json.loads(line) for line in (root / "train/metrics.jsonl").read_text().splitlines()] + committed = {row["step"]: row for row in metrics if "grad_norm" in row} + updates = [committed[step] for step in sorted(committed)] + result = {"baseline": before, "checkpoint_100": after, + "pass_at_1_change_pp": 100 * (after["pass_at_1"] - before["pass_at_1"]), + "updates_with_gradient_metrics": len(updates), + "nonzero_gradient_updates": sum(row["grad_norm"] > 0 for row in updates), + "zero_gradient_updates": sum(row["grad_norm"] == 0 for row in updates), + "note": "Mean tool usage includes different success sets; it is not matched-success efficiency."} + result["harnesses"] = {h: {"baseline": before["harnesses"][h], "checkpoint_100": values, + "pass_at_1_change_pp": 100 * (values["pass_at_1"] - before["harnesses"][h]["pass_at_1"])} + for h, values in after["harnesses"].items()} + for label, rows in (("first_20_updates", updates[:20]), ("last_20_updates", updates[-20:])): + result[label] = {"count": len(rows), "mean_reward": mean(row.get("reward") for row in rows), + "mean_grad_norm": mean(row["grad_norm"] for row in rows)} + base_pairs = {p.name: json.loads(p.read_text()) for p in (root / "baseline/eval/pairs").glob("*.json")} + matches = [] + for path in (root / "checkpoint-100/eval/pairs").glob("*.json"): + a, b = base_pairs[path.name], json.loads(path.read_text()) + if a.get("correctness") == b.get("correctness") == 1 and not a.get("error") and not b.get("error"): + matches.append((a, b)) + result["matched_successes"] = {} + for harness in ["all", *after["harnesses"]]: + pairs = [(a, b) for a, b in matches if harness == "all" or a["harness"] == harness] + values = {"pairs": len(pairs)} + for key in ("tool_calls", "generated_tokens"): + valid = [(a[key], b[key]) for a, b in pairs if a.get(key) is not None and b.get(key) is not None] + denominator = sum(a for a, _ in valid) + values[key + "_pairs"] = len(valid) + values[key + "_savings_pct"] = 100 * (1 - sum(b for _, b in valid) / denominator) if denominator else None + result["matched_successes"][harness] = values + return result diff --git a/05-multi-harness-rl/tests/test_contracts.py b/05-multi-harness-rl/tests/test_contracts.py index 90ad6ef4..deecd40f 100644 --- a/05-multi-harness-rl/tests/test_contracts.py +++ b/05-multi-harness-rl/tests/test_contracts.py @@ -72,6 +72,82 @@ def test_frozen_data_and_rotation(): assert a["task_name"] == b["task_name"] and a["harness"] != b["harness"] +def test_pilot_subset_is_fixed_and_covers_difficulty(): + from recipe import eval_rows + rows = task_rows("test") + chosen = eval_rows(rows, 25, stratified=True) + assert chosen == eval_rows(rows[::-1], 25, stratified=True) + assert Counter(r["difficulty"] for r in chosen) == {"easy": 3, "medium": 12, "hard": 10} + assert len({r["name"] for r in chosen}) == 25 + assert eval_rows(rows, 250, stratified=True) == eval_rows(rows) + with pytest.raises(ValueError): + eval_rows(rows, 0) + + +def test_slurm_uses_the_requested_allocation_limit(): + from runtime.launch import slurm_time + assert slurm_time("12h") == "12:00:00" + assert slurm_time("45m") == "00:45:00" + with pytest.raises(ValueError): + slurm_time("0h") + + +def test_eval_subset_retains_original_catalog_indices(tmp_path, monkeypatch): + import importlib + evaluator = importlib.import_module("eval.evaluate") + from recipe import eval_rows + rows = sorted(task_rows("test"), key=lambda r: r["name"]) + chosen = eval_rows(rows, 25, stratified=True) + seen = [] + monkeypatch.setattr(evaluator, "Factory", lambda cfg, data, server, vllm, groups, *a, **k: SimpleNamespace(groups=groups)) + def episode(factory, i): + group = factory.groups[i] + assert rows[group["task_index"]]["name"] == group["task_name"] + seen.append(group["task_name"]) + return {"correctness": 0} + monkeypatch.setattr(evaluator, "blackbox_episode", episode) + monkeypatch.setitem(sys.modules, "trackio", SimpleNamespace(init=lambda **kw: SimpleNamespace(log=lambda *a, **k: None, finish=lambda: None))) + cfg = {**config(), "output": str(tmp_path), "run_name": "subset-test", "eval_selection": "stratified"} + result = evaluator.evaluate(cfg, tmp_path, "unused", "unused", limit=25) + assert result["graded"] == 100 + assert set(seen) == {r["name"] for r in chosen} + + +def test_pilot_stops_on_incomplete_baseline_and_preserves_full_training_config(tmp_path, monkeypatch): + import run + import runtime.pilot + import runtime.checkpoints + calls = [] + args = SimpleNamespace(limit=25, resume=None) + cfg = {**config(mode="multi-harness"), "output": str(tmp_path), "run_name": "pilot-test"} + def fake_launch(a, c): + calls.append((a.action, c)) + output = Path(c["output"]) + if a.action == "train": + write_json(output / "checkpoint-100/checkpoint.saved.json", {"step": 100}) + else: + write_json(output / "eval/summary.json", {"complete": True}) + monkeypatch.setattr(run, "launch_local", fake_launch) + monkeypatch.setattr(runtime.pilot, "make_ready", lambda p: None) + monkeypatch.setattr(runtime.pilot, "comparison", lambda *a: {"complete": True}) + runtime.pilot.pilot(args, cfg) + assert [a for a, _ in calls] == ["eval", "train", "eval"] + train_cfg = calls[1][1] + assert (train_cfg["max_steps"], train_cfg["save_steps"], train_cfg["num_generations"], train_cfg["batch_size"]) == (100, 50, 8, 4) + calls.clear() + runtime.pilot.pilot(args, cfg) + assert not calls + cfg["output"] = str(tmp_path / "failed") + def incomplete(a, c): + write_json(Path(c["output"]) / "eval/summary.json", {"complete": False}) + monkeypatch.setattr(run, "launch_local", incomplete) + with pytest.raises(ValueError, match="complete evaluation"): + runtime.pilot.pilot(args, cfg) + state = json.loads((Path(cfg["output"]) / "pilot.json").read_text()) + assert not state["phases"]["baseline"]["complete"] + assert "train" not in state["phases"] + + def test_native_means_native(): assert {g["harness"] for g in schedule(config(mode="opencode"), task_rows("train"))} == {"opencode"} From 84bea5278485326858c8de6490fd364fc69cefcf Mon Sep 17 00:00:00 2001 From: adithya-s-k Date: Wed, 30 Sep 2026 09:16:22 +0000 Subject: [PATCH 06/48] Expose pilot phase and scalar progress in HF job logs --- 05-multi-harness-rl/run.py | 11 +++++++++++ 05-multi-harness-rl/runtime/pilot.py | 2 ++ 2 files changed, 13 insertions(+) diff --git a/05-multi-harness-rl/run.py b/05-multi-harness-rl/run.py index ca8c8739..52cb8b7e 100644 --- a/05-multi-harness-rl/run.py +++ b/05-multi-harness-rl/run.py @@ -121,9 +121,20 @@ def spawn(command, name, child_env=None): if args.limit: command += ["--limit", str(args.limit)] worker = spawn(command, "train" if training else "eval", dict(env, CUDA_VISIBLE_DEVICES=devices[1]) if training else env) + next_status = 0 while worker.poll() is None: if engine.poll() is not None or service.poll() is not None: raise RuntimeError("Inference or environment service died; stopping worker") + if time.monotonic() >= next_status: + progress_path = output / ("metrics.jsonl" if training else "eval/progress.json") + try: + content = progress_path.read_text() + progress = json.loads(content.splitlines()[-1] if training else content) + keys = ("step", "reward", "grad_norm", "loss") if training else ("expected", "graded", "correct", "coverage") + print(json.dumps({"run": cfg["run_name"], "progress": {k: progress[k] for k in keys if k in progress}}), flush=True) + except (OSError, ValueError, IndexError): + print(json.dumps({"run": cfg["run_name"], "status": "waiting_for_first_result"}), flush=True) + next_status = time.monotonic() + 60 time.sleep(2) if worker.returncode: raise RuntimeError(f"Worker failed ({worker.returncode}); inspect {output}") diff --git a/05-multi-harness-rl/runtime/pilot.py b/05-multi-harness-rl/runtime/pilot.py index 3c4ac63d..3c5af565 100644 --- a/05-multi-harness-rl/runtime/pilot.py +++ b/05-multi-harness-rl/runtime/pilot.py @@ -32,6 +32,7 @@ def pilot(args, cfg): entry = {"complete": False, "started_at": time.time()} state["phases"][phase] = entry write_json(manifest, state) + print(json.dumps({"pilot": cfg["run_name"], "phase": phase, "status": "started"}), flush=True) current = argparse.Namespace(**vars(args)) current.action = "train" if phase == "train" else "eval" current.limit = limit @@ -57,6 +58,7 @@ def pilot(args, cfg): raise ValueError("Training ended before checkpoint 100") entry.update(complete=True, finished_at=time.time()) write_json(manifest, state) + print(json.dumps({"pilot": cfg["run_name"], "phase": phase, "status": "complete"}), flush=True) except BaseException as exc: entry.update(error=type(exc).__name__, finished_at=time.time()) write_json(manifest, state) From c760ecf959df1b149db881229668aa43715910c1 Mon Sep 17 00:00:00 2001 From: adithya-s-k Date: Wed, 30 Sep 2026 09:17:55 +0000 Subject: [PATCH 07/48] Record LFM pilot scope and HF submissions --- 05-multi-harness-rl/VALIDATION.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/05-multi-harness-rl/VALIDATION.md b/05-multi-harness-rl/VALIDATION.md index 37451e07..74e94cdb 100644 --- a/05-multi-harness-rl/VALIDATION.md +++ b/05-multi-harness-rl/VALIDATION.md @@ -61,3 +61,15 @@ Local smoke artifacts are ignored under `runs/qualification/`: configuration, se - [A100 Harbor smoke](https://huggingface.co/jobs/FineEnvs/6abcc72d031314b696344167): two-step multi-harness training followed by checkpoint-2 reload evaluation on two tasks × four harnesses, concurrency four, 45-minute cap. Completed with `--smoke-eval` in 21m 17s overall; training took 627.9s. Both updates had zero reward contrast and zero gradient. Checkpoint 2 was hashed and reloaded with restarted services. All eight evaluation pairs graded on their first attempt, with TiTO and verified tool counts: OpenCode 1/2, Claude Code 1/2, Codex 1/2, Mini-SWE-Agent 2/2. These two medium tasks are a smoke, not a benchmark. Evidence is under `runs/qualification/hf-a100-harbor-v1/`. This tests all four evaluation adapters, while two training steps alone need not consume all four harnesses. The new orchestration has 19 passing CPU contract checks, including aborting reload after training failure. The native and Harbor HF execution smokes passed. A same-job reload does not qualify a separate evaluation job reading the bucket; that handoff remains a separate check. Neither HF training sample had reward contrast, so a positive-gradient HF check also remains before claiming learning validation. Then collect a fresh non-thinking baseline on the fixed 250-task test set. A tiny smoke establishes execution, not benchmark performance or throughput at concurrency 35. + +## 100-step pilot launch + +Selected scope: LFM only, all three modes, 25 fixed held-out tasks (3 easy, 12 medium, 10 hard). Baseline and checkpoint-100 evaluations use the same task IDs. Each blackbox evaluation has 100 pairs; whitebox has 25 episodes. Save at 50/100, eval concurrency 35, eight rollouts per group, unchanged full training batches and LR. Phases run sequentially inside one HF allocation, bounded to 12 hours. Slurm plans use the same command with two GPUs and the same time cap. + +- [Native OpenCode pilot](https://huggingface.co/jobs/FineEnvs/6abcd384031314b69634440b), `pilot-lfm-opencode-v2`: submitted. +- [Harbor multi-harness pilot](https://huggingface.co/jobs/FineEnvs/6abcd3874c46ef19870359af), `pilot-lfm-multi-harness-v2`: submitted. +- [Whitebox HF qualification](https://huggingface.co/jobs/FineEnvs/6abcd28f4c46ef1987035955): two updates plus reload eval, running before the whitebox pilot is launched. + +The first two pilot submissions were canceled while still scheduling, then replaced with copies that print phase and scalar progress in HF logs. No training progress was discarded. Configurations, scalar logs, local Trackio and phase reports live under each run prefix in `FineEnvs/data-agent-daytona-artifacts`. No Slurm pilot has been submitted. + +Pilot checks: 23 CPU contracts passed, plus the four fetched-runtime integration checks. Coverage includes fixed stratification, preserving original task catalog indices, stopping on incomplete baseline, retaining normal training batches, phase resume and Slurm time limits. A comparison fixture verifies repeated-step deduplication and matched-success tool/token savings. Actual baseline/100-step results are pending. From 1e7e2abcccca12584a791ad1df4c22a7d21682c8 Mon Sep 17 00:00:00 2001 From: adithya-s-k Date: Wed, 30 Sep 2026 09:20:26 +0000 Subject: [PATCH 08/48] Gate unqualified pilots on smoke and install pinned TRL metadata --- 05-multi-harness-rl/REPRODUCE.md | 3 +++ 05-multi-harness-rl/run.py | 3 +++ 05-multi-harness-rl/runtime/job.sh | 1 + 05-multi-harness-rl/runtime/launch.py | 5 +++++ 05-multi-harness-rl/runtime/pilot.py | 23 ++++++++++++++++++--- 05-multi-harness-rl/tests/test_contracts.py | 10 +++++++++ 6 files changed, 42 insertions(+), 3 deletions(-) diff --git a/05-multi-harness-rl/REPRODUCE.md b/05-multi-harness-rl/REPRODUCE.md index 40206260..6e311177 100644 --- a/05-multi-harness-rl/REPRODUCE.md +++ b/05-multi-harness-rl/REPRODUCE.md @@ -9,6 +9,7 @@ uv venv --python 3.12 source .venv/bin/activate uv pip install -r requirements.lock python runtime/bootstrap.py +uv pip install --no-deps --no-build-isolation -e .runtime/trl python prepare.py python -m pytest -q tests python run.py plan --model lfm --mode multi-harness @@ -67,6 +68,8 @@ done These commands are dry runs; add `--submit` when ready. Use `--model qwen` with distinct names for Qwen. `--limit 25` selects the same seeded subset for every mode and checkpoint: 3 easy, 12 medium, 10 hard. That gives 100 pairs per blackbox eval and 25 episodes per whitebox eval. Use `--limit 250` for the full test set. Small subsets are noisy; 100 training steps do not guarantee an improvement. +Add `--preflight-smoke` for a mode/hardware combination that has not passed its HF training and reload smoke. The pilot proceeds only if that two-update check passes. Keep the flag when resuming that pilot. The entrypoint installs the pinned TRL checkout without changing locked dependencies; this also supplies package metadata needed by whitebox checkpoint saving. + Each output contains `pilot.json`, `baseline/`, `train/`, `checkpoint-100/` and a final `comparison.json`. The report includes per-harness pass@1, matched-success tool/token savings, first/last 20-update reward means and zero/nonzero gradient counts. Raw logs, task IDs and offline Trackio remain available. An incomplete baseline stops the pilot before training. To resume, use the same name/config plus `--resume /outputs/RUN_NAME/train/checkpoint-50`; completed phases are retained. The 12-hour cap is a limit, not an ETA. The Slurm equivalent uses the same phases and configuration, with two GPUs: diff --git a/05-multi-harness-rl/run.py b/05-multi-harness-rl/run.py index 52cb8b7e..97db4538 100644 --- a/05-multi-harness-rl/run.py +++ b/05-multi-harness-rl/run.py @@ -27,6 +27,7 @@ def parser(): p.add_argument("--concurrency", type=int) p.add_argument("--limit", type=int, help="Eval tasks per harness; omit for all 250") p.add_argument("--smoke-eval", action="store_true", help="Reload the smoke checkpoint for two tasks per harness") + p.add_argument("--preflight-smoke", action="store_true", help="Gate a pilot on a two-update training and reload smoke") p.add_argument("--server", help="External URL, or a local bind URL; owned services default to unused ports") p.add_argument("--vllm-url") p.add_argument("--capture-port", type=int) @@ -152,6 +153,8 @@ def spawn(command, name, child_env=None): def main(): args = parser().parse_args() + if args.preflight_smoke and args.action != "pilot": + raise ValueError("--preflight-smoke requires the pilot action") if args.smoke_eval and (args.action != "smoke" or args.external_services): raise ValueError("--smoke-eval requires a smoke with job-local services") if args.external_services and (not args.server or not args.vllm_url): diff --git a/05-multi-harness-rl/runtime/job.sh b/05-multi-harness-rl/runtime/job.sh index dea84c88..023fadc6 100644 --- a/05-multi-harness-rl/runtime/job.sh +++ b/05-multi-harness-rl/runtime/job.sh @@ -12,5 +12,6 @@ if [[ "$1" == "watch" ]]; then fi python3 runtime/bootstrap.py uv pip install --python .venv/bin/python -r requirements.lock +uv pip install --python .venv/bin/python --no-deps --no-build-isolation -e .runtime/trl python3 prepare.py exec python3 run.py "$@" diff --git a/05-multi-harness-rl/runtime/launch.py b/05-multi-harness-rl/runtime/launch.py index ea23c5f7..d06cd97b 100644 --- a/05-multi-harness-rl/runtime/launch.py +++ b/05-multi-harness-rl/runtime/launch.py @@ -57,11 +57,14 @@ def main(): p.add_argument("--concurrency", type=int, default=35) p.add_argument("--limit", type=int) p.add_argument("--smoke-eval", action="store_true") + p.add_argument("--preflight-smoke", action="store_true") p.add_argument("--max-active-evals", type=int, default=1) p.add_argument("--submit", action="store_true") args = p.parse_args() if args.smoke_eval and args.action != "smoke": p.error("--smoke-eval requires the smoke action") + if args.preflight_smoke and args.action != "pilot": + p.error("--preflight-smoke requires the pilot action") if Path(args.run_name).name != args.run_name or args.run_name in {".", ".."}: p.error("run-name must be a single directory name") output = str(Path("/outputs" if args.backend == "hf" else args.output_root.resolve()) / args.run_name) @@ -69,6 +72,8 @@ def main(): "--output", output, "--concurrency", str(args.concurrency)] if args.smoke_eval: flags.append("--smoke-eval") + if args.preflight_smoke: + flags.append("--preflight-smoke") if args.config: path = args.config if args.backend == "hf" and not str(path).startswith("/outputs/"): diff --git a/05-multi-harness-rl/runtime/pilot.py b/05-multi-harness-rl/runtime/pilot.py index 3c5af565..5b684cd9 100644 --- a/05-multi-harness-rl/runtime/pilot.py +++ b/05-multi-harness-rl/runtime/pilot.py @@ -2,9 +2,11 @@ import argparse import json from pathlib import Path +import subprocess +import sys import time -from recipe import digest, eval_rows, mean, task_rows, write_json +from recipe import ROOT, digest, eval_rows, mean, task_rows, write_json from runtime.checkpoints import make_ready @@ -16,7 +18,8 @@ def pilot(args, cfg): root.mkdir(parents=True, exist_ok=True) limit = args.limit if args.limit is not None else 25 selected = eval_rows(task_rows("test"), limit, stratified=True, seed=cfg["seed"]) - identity = {"config": cfg, "eval_tasks": selected} + preflight = bool(getattr(args, "preflight_smoke", False)) + identity = {"config": cfg, "eval_tasks": selected, **({"preflight_smoke": True} if preflight else {})} signature = digest(identity) manifest = root / "pilot.json" state = json.loads(manifest.read_text()) if manifest.exists() else { @@ -26,7 +29,8 @@ def pilot(args, cfg): if args.resume and not state["phases"].get("baseline", {}).get("complete"): raise ValueError("Resume requires this pilot's completed baseline") - for phase in ("baseline", "train", "checkpoint-100"): + phases = (["preflight"] if preflight else []) + ["baseline", "train", "checkpoint-100"] + for phase in phases: if state["phases"].get(phase, {}).get("complete"): continue entry = {"complete": False, "started_at": time.time()} @@ -41,6 +45,19 @@ def pilot(args, cfg): current.smoke_eval = False phase_cfg = {**cfg, "output": str(root / phase), "run_name": cfg["run_name"] + "-" + phase} try: + if phase == "preflight": + path = root / "preflight-config.json" + write_json(path, phase_cfg) + subprocess.run([sys.executable, str(ROOT / "run.py"), "smoke", "--config", str(path), + "--data", str(args.data.resolve()), "--smoke-eval", "--limit", "2", + "--concurrency", "4"], check=True) + report = json.loads((root / "preflight/reload-eval/eval/summary.json").read_text()) + if not report["complete"]: + raise ValueError("Preflight reload evaluation is incomplete") + entry.update(complete=True, evaluation=report, finished_at=time.time()) + write_json(manifest, state) + print(json.dumps({"pilot": cfg["run_name"], "phase": phase, "status": "complete"}), flush=True) + continue if current.resume: make_ready(current.resume) if phase == "checkpoint-100": diff --git a/05-multi-harness-rl/tests/test_contracts.py b/05-multi-harness-rl/tests/test_contracts.py index deecd40f..af5a62bf 100644 --- a/05-multi-harness-rl/tests/test_contracts.py +++ b/05-multi-harness-rl/tests/test_contracts.py @@ -146,6 +146,16 @@ def incomplete(a, c): state = json.loads((Path(cfg["output"]) / "pilot.json").read_text()) assert not state["phases"]["baseline"]["complete"] assert "train" not in state["phases"] + cfg["output"] = str(tmp_path / "failed-preflight") + args.preflight_smoke, args.data = True, tmp_path + def failed_preflight(*a, **kw): + raise runtime.pilot.subprocess.CalledProcessError(1, "smoke") + monkeypatch.setattr(runtime.pilot.subprocess, "run", failed_preflight) + with pytest.raises(runtime.pilot.subprocess.CalledProcessError): + runtime.pilot.pilot(args, cfg) + state = json.loads((Path(cfg["output"]) / "pilot.json").read_text()) + assert not state["phases"]["preflight"]["complete"] + assert "baseline" not in state["phases"] def test_native_means_native(): From 209920b6a289cfba300cd72ab653fd1c94f56f0e Mon Sep 17 00:00:00 2001 From: adithya-s-k Date: Wed, 30 Sep 2026 09:21:45 +0000 Subject: [PATCH 09/48] Record whitebox metadata fix and gated pilot submission --- 05-multi-harness-rl/VALIDATION.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/05-multi-harness-rl/VALIDATION.md b/05-multi-harness-rl/VALIDATION.md index 74e94cdb..0ee6609b 100644 --- a/05-multi-harness-rl/VALIDATION.md +++ b/05-multi-harness-rl/VALIDATION.md @@ -68,7 +68,8 @@ Selected scope: LFM only, all three modes, 25 fixed held-out tasks (3 easy, 12 m - [Native OpenCode pilot](https://huggingface.co/jobs/FineEnvs/6abcd384031314b69634440b), `pilot-lfm-opencode-v2`: submitted. - [Harbor multi-harness pilot](https://huggingface.co/jobs/FineEnvs/6abcd3874c46ef19870359af), `pilot-lfm-multi-harness-v2`: submitted. -- [Whitebox HF qualification](https://huggingface.co/jobs/FineEnvs/6abcd28f4c46ef1987035955): two updates plus reload eval, running before the whitebox pilot is launched. +- [Whitebox HF qualification](https://huggingface.co/jobs/FineEnvs/6abcd28f4c46ef1987035955): failed saving step 1 because the fresh environment lacked TRL package metadata. Training and token auditing had reached the update. The entrypoint now installs the pinned checkout with `--no-deps --no-build-isolation`; package metadata was verified in a clean Python environment. +- [Whitebox pilot](https://huggingface.co/jobs/FineEnvs/6abcd472031314b696344457), `pilot-lfm-whitebox-v1`: submitted with `--preflight-smoke`. It must finish two training updates and reload eval before starting the baseline/100-step phases; any failed qualification stops the job. The first two pilot submissions were canceled while still scheduling, then replaced with copies that print phase and scalar progress in HF logs. No training progress was discarded. Configurations, scalar logs, local Trackio and phase reports live under each run prefix in `FineEnvs/data-agent-daytona-artifacts`. No Slurm pilot has been submitted. From 92f247d74cf051ab05641fb8516de0bc5b4f901a Mon Sep 17 00:00:00 2001 From: adithya-s-k Date: Wed, 30 Sep 2026 10:20:28 +0000 Subject: [PATCH 10/48] Make the multi-harness recipe reproducible for new users --- 05-multi-harness-rl/README.md | 121 ++++++----- 05-multi-harness-rl/REPRODUCE.md | 224 ++++++++++++++------ 05-multi-harness-rl/RESULTS.md | 58 +++++ 05-multi-harness-rl/VALIDATION.md | 8 + 05-multi-harness-rl/eval/watch.py | 4 +- 05-multi-harness-rl/recipe.py | 13 +- 05-multi-harness-rl/runtime/launch.py | 32 ++- 05-multi-harness-rl/tests/test_contracts.py | 27 +++ 8 files changed, 352 insertions(+), 135 deletions(-) create mode 100644 05-multi-harness-rl/RESULTS.md diff --git a/05-multi-harness-rl/README.md b/05-multi-harness-rl/README.md index f29cade9..0ca755cc 100644 --- a/05-multi-harness-rl/README.md +++ b/05-multi-harness-rl/README.md @@ -1,77 +1,88 @@ -# SmolDataEnv: whitebox and multi-harness RL +# SmolDataEnv: learn RL through three agent interfaces -Train the same data-analysis policy through three agent interfaces. Use **LFM2.5-2.6B** or **Qwen3.5-2B**, with thinking disabled, the same task split, and correctness plus a small tool-efficiency bonus. +Fine-tune a small language model to solve data-analysis tasks, then measure whether it answers more questions correctly and uses fewer tools. This example compares three ways of connecting an agent to a trainer, using the same tasks and reward. -This is a focused successor to the archived `04-data-agent` experiment. Start with [REPRODUCE.md](REPRODUCE.md) for HF Jobs and Slurm commands. The new recipe is under qualification; historical runs do not establish that this new non-thinking configuration is trained or validated. +**Start here:** [run the tutorial](REPRODUCE.md). You can submit an HF Job from a CPU laptop, or run on a local two-GPU machine or Slurm cluster. [Current results](RESULTS.md) distinguish completed checks from ongoing experiments; [validation notes](VALIDATION.md) contain the detailed evidence. -| Mode | Who drives the agent? | Training | Evaluation | -|---|---|---|---| -| `whitebox` | TRL calls native bash/SETA methods | Synchronous GRPO | 250 tasks with the same bash/SETA tools | -| `opencode` | Native OpenCode, without Harbor execution | AsyncGRPO | 250 tasks through each of four Harbor harnesses | -| `multi-harness` | Harbor runs OpenCode, Claude Code, Codex or Mini-SWE-Agent | AsyncGRPO | The same 250 tasks through all four harnesses | - -The OpenEnv server runs **inside each training or evaluation job**. Daytona runs the isolated task containers. No environment Space is needed. Training and evaluation use separate GPU allocations, inference engines and environment processes; they still share your sandbox quota and storage bandwidth. - -For initial qualification, the [100-step pilot](REPRODUCE.md#100-step-pilot) runs baseline evaluation, training and checkpoint evaluation sequentially in one allocation. It uses the same training configuration and a fixed evaluation subset, with a final comparison report. +## Choose an agent interface -## Shared experiment +A *harness* is the program that manages the model's conversation and tool calls. OpenEnv provides the environment interface. Harbor runs third-party harnesses and their task verifiers. -| Setting | Value | -|---|---| -| Training data | Fixed 1,000 tasks: 400 medium, 600 hard; no test task or notebook overlap | -| Schedule | Two passes; one harness per task group; rotate the harness on the second pass | -| Rollouts per group | 8, all using the same task and harness | -| Learning rate / sampling | `3e-6`; temperature `0.8`, top-p `1`, top-k disabled | -| Async limits | 32 inflight; 16 outstanding rollouts including queued work; staleness 4 | -| Optimizer | Full fine-tuning, paged AdamW 8-bit, BF16, gradient checkpointing | -| Async batching | Batch 4 × accumulation 4; whole rollouts admitted together; 40,960-token packing target | -| Synchronous batching | Microbatch 1 × accumulation 16 to limit logits memory | -| Episode limits | 17 model/tool iterations, 600 seconds; 4,096 output tokens per model call | -| Trainer limits | 16,384 completion tokens; 131,072 context; 1,000 optimizer-step ceiling | -| Save / evaluate | Save every 50 updates; evaluate every 100 and the final checkpoint | -| Evaluation | Pass@1; TP1/DP2; default concurrency 35, configurable | +| Mode | What happens | Trainer | Evaluation | +|---|---|---|---| +| `whitebox` | TRL generates responses and directly invokes bash/SETA tools | Synchronous GRPO | Native bash/SETA | +| `opencode` | Native OpenCode owns the agent loop; OpenEnv captures its model calls | AsyncGRPO | All four Harbor harnesses | +| `multi-harness` | Harbor rotates OpenCode, Claude Code, Codex and Mini-SWE-Agent between task groups | AsyncGRPO | All four Harbor harnesses | -Task groups and optimizer updates are different units. Async packing, discarded stale rollouts, and groups without reward contrast affect how many groups produce an update. Training stops at schedule exhaustion or the step ceiling, whichever comes first. Two scheduled passes do not guarantee two completed passes; checkpoints record the groups actually used. +**Native OpenCode training does not run through Harbor.** Both blackbox policies use the same Harbor evaluation interfaces so you can measure transfer to other harnesses. Whitebox evaluates its native tools, which is a different evaluation interface. -The reward is `correctness × (1 + 0.1 × 15 / (15 + tool_calls))`. Incorrect answers earn zero. A zero or unverified tool count receives no bonus. Infrastructure failures remain ungraded. Native action records provide the counts; the sandbox grader returns correctness only. Pass@1 reports binary correctness separately from shaping and always includes coverage. +Use `--model lfm` for **LiquidAI/LFM2.5-2.6B**, or `--model qwen` for **Qwen/Qwen3.5-2B**. Both use non-thinking prompts. The new runs need their own baseline; historical thinking-enabled results are not interchangeable. -Qwen uses `enable_thinking=False`. The pinned LFM template ignores that flag, so this recipe explicitly closes its generation-time thinking block. Serving omits reasoning parsing: a streaming reasoning parser can otherwise hide tool calls from the harness. Startup checks that both streaming and ordinary replies are visible. Earlier LFM RL results used thinking and are not a baseline for this setting. +## What happens in a run? -## Code comparison +1. Load the fixed task list and checked dataset revision. +2. Start vLLM and OpenEnv inside the job. Daytona supplies isolated task containers. +3. Generate eight rollouts for each task group. Each rollout tries to solve the task with tools. +4. Grade the answer and count verified native tool calls. +5. Train on the model's generated tokens, masking prompts and tool outputs. +6. Save checkpoints and evaluate them on held-out tasks. -The learning boundary is small: +TiTO means *tokens in, tokens out*: training uses the inference engine's actual token IDs and logprobs rather than reconstructing them from text. A rewritten conversation can produce several training rows from one rollout. A task, rollout, training row and optimizer step are therefore different units. -```python -# Whitebox: the trainer generates, calls tools and masks tool results. -GRPOTrainer(environment_factory=white_box_bash_env(...), ...) +Synchronous GRPO waits for its batch of tool rollouts. AsyncGRPO collects rollouts while training proceeds, with a bounded amount of policy staleness. A group whose rollouts all have the same reward supplies no relative learning signal. Successful execution alone does not establish useful learning. -# Native OpenCode: the native environment runs OpenCode and captures its model calls. -DataAgentSession(...) +## The shared experiment -# Multi-harness: Harbor owns the agent loop and returns captured model calls. -HarborSession(harness=group["harness"], ...) +| Setting | Default | +|---|---| +| Training set | 1,000 fixed tasks: 400 medium and 600 hard | +| Held-out set | 250 fixed tasks: 33 easy, 118 medium and 99 hard; no train/test task or notebook overlap | +| Learning rate / sampling | `3e-6`; temperature `0.8`; full-vocabulary sampling | +| Rollouts per task group | 8, using the same task and harness | +| Async limits | 32 inflight, 16 outstanding rollouts, maximum staleness 4 | +| Async batch | Batch 4 × accumulation 4; 40,960-token packing target | +| Whitebox batch | Microbatch 1 × accumulation 16 | +| Episode limits | 17 iterations, 600 seconds, 4,096 generated tokens per model call | +| Training limits | 16,384 completion tokens; 131,072 context; full BF16 fine-tuning | +| Checkpoints | Every 50 optimizer steps | +| Evaluation | Pass@1; default concurrency 35; TP1/DP2 inference | + +Correctness is binary. A correct answer with a verified, positive tool count earns: + +```text +reward = correctness × (1 + 0.1 × 15 / (15 + tool_calls)) ``` -Both blackbox modes use the same finite AsyncGRPO worker and whole-rollout admission. Engine token IDs, sampled logprobs and loss masks are preserved, including when a harness rewrites its history. Those rewrites can create several training rows from one rollout, so row-averaged training reward is not the same as rollout-level correctness. LFM's packed convolution states are isolated between sequences. +For example, a correct answer using 15 calls earns `1.05`; an incorrect answer earns `0`. Zero or unverified tool counts receive no bonus. Infrastructure failures remain ungraded. Evaluation reports correctness separately from shaped reward and tool/token usage. -| File | Purpose | -|---|---| -| [configs/default.json](configs/default.json) | Shared hyperparameters and model/harness pins | -| [prepare.py](prepare.py) | Fixed task selection and corrected Harbor grader checks | -| [train/whitebox.py](train/whitebox.py) | Synchronous tool-based training | -| [train/blackbox.py](train/blackbox.py), [train/adapters.py](train/adapters.py) | Async training and native/Harbor session boundary | -| [eval/evaluate.py](eval/evaluate.py) | Resumable pass@1 evaluation and usage metrics | -| [eval/watch.py](eval/watch.py) | Checkpoint verification and separate evaluation jobs | -| [runtime/launch.py](runtime/launch.py) | HF Jobs and Slurm submission; dry-run by default | +The full schedule contains two passes and stops at 1,000 updates or schedule exhaustion, whichever comes first. The 100-step pilot retains the same schedule and training settings but stops at 100 updates. Neither setting guarantees that every scheduled task is consumed. The committed-group audit records what async training actually used. -`runtime/bootstrap.py` fetches 45 existing environment/admission files and pinned TRL/OpenEnv checkouts into ignored `.runtime/`. Their source hashes are in [runtime-lock.json](configs/runtime-lock.json). Small native-environment adaptations are explicit in [runtime/patches.py](runtime/patches.py). The recipe does not require an old local experiment checkout. The pinned TRL integration is experimental; dependency upgrades require another smoke. +## Start small, then scale -## Logs and previous results +| Action | Purpose | +|---|---| +| `smoke --smoke-eval` | Two small training updates, checkpoint saving, then two-task reload evaluation | +| `pilot` | Baseline, 100 updates, checkpoint-100 evaluation on 25 fixed tasks by default | +| `train` + evaluation watcher | Longer training with separate checkpoint-evaluation jobs | -Each run keeps its resolved configuration, task schedule, scalar `metrics.jsonl`, local Trackio data, rollout reward/count evidence, service logs and full optimizer checkpoints. Each eval keeps a record per task/harness, coverage, missing pairs, pass@1, tool calls and generated/prompt tokens. Set `trackio_space_id` in the config to mirror to an online dashboard. +The pilot evaluates 3 easy, 12 medium and 10 hard tasks. That is **100 pairs per blackbox evaluation** or **25 whitebox episodes**. Its phases run sequentially in one allocation, restarting services between phases. No watcher is needed. Full training uses separate evaluation allocations; shared sandbox capacity and storage can still affect throughput. -[SmolDataEnv RL dashboard](https://huggingface.co/spaces/FineEnvs/data-agent-training-comparison-trackio) contains the historical experiments. Their full record remains on [the archived branch](https://github.com/adithya-s-k/FineEnvs/tree/archive/data-agent-experiments-20260930/04-data-agent). This recipe has no new published scores yet. +There is **no Space to deploy** for these runs. OpenEnv lives inside the GPU job; the task containers live in Daytona. A Trackio Space is optional for online charts. -The September 30 Harbor grader update is pinned here: null tool-efficiency fields must not discard a valid correctness reward. [SmolDataEnvs prompt updates](https://huggingface.co/datasets/FineEnvs/SmolDataEnvs/discussions/2) also change some single-program baselines. Do not combine those scores with an older protocol without identifying the revisions. +## Find the code -Historical impact depends on each run's frozen files. The corrected Hub tasks match this recipe's instruction hashes; 302 selected training tasks use a JSON reward wrapper where the old local copy used `reward.txt`. The recipe checks the corrected grader before serving either split. See [validation notes](VALIDATION.md) for smoke coverage and remaining checks. +| Path | Read it when you want to… | +|---|---| +| `configs/default.json` | Inspect model pins and shared hyperparameters | +| `run.py`, `runtime/launch.py` | Run locally, submit to HF Jobs or inspect a Slurm command | +| `runtime/pilot.py` | Follow the baseline → train → checkpoint-eval sequence | +| `train/whitebox.py` | Understand synchronous native tool training | +| `train/blackbox.py`, `train/adapters.py` | Understand async training and the native/Harbor boundary | +| `eval/evaluate.py` | Inspect pass@1, coverage and per-harness results | +| `eval/watch.py` | Submit separate evaluations for full training checkpoints | +| `prepare.py`, `data/` | Inspect the exact task selection and dataset checks | +| `configs/runtime-lock.json`, `runtime/patches.py` | Audit pinned dependencies and their small adaptations | + +`runtime/bootstrap.py` downloads the pinned TRL/OpenEnv sources and 45 existing runtime files into ignored `.runtime/`. It checks the archive file hashes. You do not need the original experiment workspace. Prepared tasks, checkpoints, caches and credentials are not committed. + +This is the condensed successor to the [archived experiment](https://github.com/adithya-s-k/FineEnvs/tree/archive/data-agent-experiments-20260930/04-data-agent). The [historical dashboard](https://huggingface.co/spaces/FineEnvs/data-agent-training-comparison-trackio) contains those earlier runs, not the live pilots unless explicitly configured. Corrected Harbor graders and dataset revisions are pinned here; see [VALIDATION.md](VALIDATION.md) before comparing historical scores. diff --git a/05-multi-harness-rl/REPRODUCE.md b/05-multi-harness-rl/REPRODUCE.md index 6e311177..07d6a734 100644 --- a/05-multi-harness-rl/REPRODUCE.md +++ b/05-multi-harness-rl/REPRODUCE.md @@ -1,122 +1,208 @@ -# Run the recipe +# Run SmolDataEnv RL -All commands below start in `05-multi-harness-rl/`. Python 3.12 is required. Choose `--mode whitebox`, `--mode opencode` or `--mode multi-harness`, and `--model lfm` or `--model qwen`. +Start from a checkout containing this example: -## 1. Prepare and inspect +```bash +cd 05-multi-harness-rl +``` + +Pick **HF Jobs** if you want the container to install and prepare everything. Pick **local/Slurm** if you already have a compatible GPU machine. Both use the same code, fixed tasks and model revisions. The current branch is local until published; the commands assume you already have its files. + +## 1. What you need + +- Python 3.12 and [uv](https://docs.astral.sh/uv/getting-started/installation/). +- An HF account with Jobs billing enabled, or access to a local GPU machine. +- A Daytona account and API key for task sandboxes. GPU and sandbox usage are billed separately. +- Two GPUs with enough memory: the qualified HF blackbox setup uses A100 80 GB; local smokes used H100s. The default HF request is H200×2. Other combinations need their own smoke. + +One GPU trains while the other serves rollouts. Evaluation uses both as independent inference replicas. HF offers A100×4 rather than A100×2; this recipe uses two GPUs but the entire allocation is billed. + +## 2. Submit from a CPU laptop with HF Jobs + +You do **not** need CUDA, PyTorch, the datasets or a local environment server for submission. ```bash uv venv --python 3.12 source .venv/bin/activate -uv pip install -r requirements.lock -python runtime/bootstrap.py -uv pip install --no-deps --no-build-isolation -e .runtime/trl -python prepare.py -python -m pytest -q tests -python run.py plan --model lfm --mode multi-harness -``` +uv pip install huggingface_hub==1.24.0 +hf auth login -`prepare.py` downloads immutable revisions of the corrected Harbor train/test datasets. It selects the exact IDs in `data/`, checks instruction and grader hashes, sets each task to one CPU/4 GB, and records file hashes. It never selects a fresh random test set. Data and downloaded runtime code are ignored by Git. +# Replace this with your own HF username or organization. +export HF_NAMESPACE="your-hf-username" +export HF_BUCKET="$HF_NAMESPACE/smoldataenv-runs" +hf buckets create "$HF_BUCKET" --private --exist-ok -Log in with `hf auth login` and set `DAYTONA_API_KEY` in your shell. Do not put credentials into a config or commit an `.env` file. The job-local service stages private task data; model-serving access is scoped to capture sessions. +# Enter your Daytona key without putting it in shell history. +read -rs -p "Daytona API key: " DAYTONA_API_KEY +export DAYTONA_API_KEY +``` + +Your HF login needs access to submit Jobs and write the bucket. The launcher uses the bucket owner as the job namespace; pass `--namespace` to choose another account you can use. Tokens are sent as HF Job secrets, not embedded in the uploaded source. Keep credentials out of config files. -## 2. HF Jobs +### First: verify one small run -Create a durable bucket you own. Supply its `owner/name` below. Launch commands print a plan unless you add `--submit`; neither Git push nor an environment Space is required. The launcher uploads only the allowlisted recipe source. +Commands print a submission plan by default. Read it, then add `--submit` to launch. ```bash -# Disposable two-update training smoke. python runtime/launch.py hf smoke --model lfm --mode opencode \ - --run-name smoke-lfm-native --bucket FineEnvs/YOUR_BUCKET --timeout 2h + --run-name smoke-lfm-opencode-v1 --bucket "$HF_BUCKET" \ + --flavor a100x4 --timeout 45m --smoke-eval --concurrency 4 +``` -# Inspect, then add --submit to launch. -python runtime/launch.py hf train --model lfm --mode multi-harness \ - --run-name lfm-multi-nonthinking-v1 --bucket FineEnvs/YOUR_BUCKET +The job creates a Python environment, installs locked dependencies, fetches the pinned runtime and prepares tasks. It then runs two small training updates, saves checkpoints 1 and 2, and reloads checkpoint 2 for evaluation. Blackbox modes evaluate two tasks through each of four Harbor harnesses; whitebox evaluates two native SETA episodes. Require complete grading and valid capture, not a particular score from this tiny sample. -# Separate CPU watcher submits evaluations on separate GPU Jobs. -python runtime/launch.py hf watch --run-name eval-watcher \ - --bucket FineEnvs/YOUR_BUCKET --concurrency 35 --max-active-evals 1 +### Next: a 100-step pilot + +```bash +python runtime/launch.py hf pilot --model lfm --mode multi-harness \ + --run-name pilot-lfm-multi-v1 --bucket "$HF_BUCKET" \ + --flavor a100x4 --timeout 12h --limit 25 --concurrency 35 ``` -Repeat the smoke for each model/mode before launching that combination. The default GPU flavor is `h200x2`: one trainer GPU and one serving GPU; evaluation uses both GPUs as DP2 replicas. HF Jobs currently has no two-A100 flavor. Override `--flavor` only with hardware that has enough GPUs and memory. The base image is pinned by digest; Python packages and source revisions are pinned separately. +Add `--submit` after checking the plan. The phases are: -The A100 alternative is `--flavor a100x4`; this recipe uses two of those four GPUs, but the full allocation is billed. Async training selects FlashAttention 2 on Ampere and FlashAttention 3 on Hopper, preserving packed-sequence boundaries. A checked adapter enables this choice in the pinned TRL runtime. See `VALIDATION.md` for hardware qualification status. +1. Baseline pass@1 on 25 fixed tasks: 3 easy, 12 medium and 10 hard. +2. Train with the normal eight-rollout configuration for 100 updates, saving at 50 and 100. +3. Hash and reload checkpoint 100; repeat the same evaluation. +4. Write `comparison.json` with quality, tool/token usage and gradient diagnostics. -For a combined training and reload check, add `--smoke-eval --concurrency 4` to the smoke command. After two updates, it hashes checkpoint 2, restarts the services and evaluates two test tasks per harness. Blackbox modes run all four Harbor harnesses, producing eight graded pairs when complete. The evaluation lives under `reload-eval/`; a training failure prevents it from starting. +Phases run sequentially in one allocation. Evaluation does not compete with training. An incomplete baseline stops the job before training. The 12-hour timeout is a spending/time cap, not a completion estimate. -The bucket is mounted at `/outputs`. A run writes to `/outputs/RUN_NAME`; the watcher hashes completed checkpoints before launching evals. Keep the watcher alive until all final evaluations finish. Failures remain in `eval-watcher.json` for investigation rather than resubmitting indefinitely. Relaunch a failed evaluation explicitly with the same output name to retain already graded pairs. +Change `--mode` to `opencode` or `whitebox` and choose a **new run name** for each comparison. Add `--preflight-smoke` if that mode/model/hardware combination has not passed the small training and reload test; the pilot proceeds only after it passes. Change `--model lfm` to `--model qwen` to use Qwen3.5-2B. Use `--limit 250` for the full held-out set. + +## 3. Monitor and read the results + +Submission prints a job URL and records it in `runs/submissions/RUN_NAME.json`. ```bash -python runtime/launch.py hf eval --model lfm --mode multi-harness \ - --run-name lfm-multi-nonthinking-v1-eval-checkpoint-100 \ - --checkpoint /outputs/lfm-multi-nonthinking-v1/checkpoint-100 \ - --bucket FineEnvs/YOUR_BUCKET --concurrency 35 +hf jobs inspect JOB_ID --namespace "$HF_NAMESPACE" +hf jobs logs JOB_ID --namespace "$HF_NAMESPACE" --tail 50 +hf jobs logs JOB_ID --namespace "$HF_NAMESPACE" --follow ``` -For online Trackio, set `trackio_space_id` in `configs/default.json` before submission. Logs and offline Trackio data remain in the bucket regardless. Use `hf jobs ps` and `hf jobs logs JOB_ID` to monitor. Native OpenCode and Harbor passed HF A100 smokes; check `VALIDATION.md` for the remaining qualifications. +Logs show phase transitions, training step/reward/gradient norm and evaluation coverage. A job marked `RUNNING` may still be installing packages, loading a model or running its baseline; it does not necessarily mean training has begun. Inspect the final job status as well as the logs. -## 100-step pilot - -The `pilot` action runs baseline eval, training to step 100, then checkpoint-100 eval in one allocation. Services restart between phases, so evaluation does not compete with training. It keeps the normal batch size, eight rollouts, LR `3e-6`, reward and task schedule; only the step ceiling and evaluation size change. Checkpoints save at 50 and 100. Do not start the separate eval watcher for these pilots. +Outputs are under `/outputs/RUN_NAME` inside the job, backed by your bucket. Afterward, download a run: ```bash -for mode in whitebox opencode multi-harness; do - python runtime/launch.py hf pilot --model lfm --mode "$mode" \ - --run-name "pilot-lfm-$mode-v1" --bucket FineEnvs/YOUR_BUCKET \ - --flavor a100x4 --timeout 12h --limit 25 --concurrency 35 -done +hf buckets sync "hf://buckets/$HF_BUCKET/RUN_NAME" runs/RUN_NAME ``` -These commands are dry runs; add `--submit` when ready. Use `--model qwen` with distinct names for Qwen. `--limit 25` selects the same seeded subset for every mode and checkpoint: 3 easy, 12 medium, 10 hard. That gives 100 pairs per blackbox eval and 25 episodes per whitebox eval. Use `--limit 250` for the full test set. Small subsets are noisy; 100 training steps do not guarantee an improvement. +```text +RUN_NAME/ +├── pilot.json configuration, exact test IDs and phase status +├── baseline/eval/ summary.json and one result per task/harness +├── train/ +│ ├── metrics.jsonl raw training scalars +│ ├── checkpoint-50/ +│ ├── checkpoint-100/ weights, optimizer, RNG and recipe +│ ├── rollouts/ blackbox correctness and tool-count evidence +│ ├── audit/ async group admission and consumed-rollout records +│ └── trackio/ local Trackio data +├── checkpoint-100/eval/ final held-out results +└── comparison.json baseline-to-checkpoint comparison +``` -Add `--preflight-smoke` for a mode/hardware combination that has not passed its HF training and reload smoke. The pilot proceeds only if that two-update check passes. Keep the flag when resuming that pilot. The entrypoint installs the pinned TRL checkout without changing locked dependencies; this also supplies package metadata needed by whitebox checkpoint saving. +Each phase also keeps its service logs and resolved `config.json`. Whitebox keeps `token-audit.jsonl` and `whitebox-records/`; Harbor keeps native trajectories in `trials/`. OpenEnv/vLLM startup URLs are in `services.json`. A preflight, when enabled, has its own directory. -Each output contains `pilot.json`, `baseline/`, `train/`, `checkpoint-100/` and a final `comparison.json`. The report includes per-harness pass@1, matched-success tool/token savings, first/last 20-update reward means and zero/nonzero gradient counts. Raw logs, task IDs and offline Trackio remain available. An incomplete baseline stops the pilot before training. To resume, use the same name/config plus `--resume /outputs/RUN_NAME/train/checkpoint-50`; completed phases are retained. The 12-hour cap is a limit, not an ETA. +Check these before calling a pilot successful: -The Slurm equivalent uses the same phases and configuration, with two GPUs: +| Evidence | Interpretation | +|---|---| +| Complete evaluation coverage | All intended pairs graded; missing infrastructure results are not incorrect answers | +| Nonzero gradient updates | At least some groups supplied learning signal; zero contrast yields zero policy gradient | +| Baseline vs checkpoint pass@1 | Held-out correctness, not shaped training reward | +| Matched-success tool/token savings | Usage changes on pairs solved by both models; report the matched-pair count | +| First/last 20-update reward means | A training diagnostic; async row weighting differs from rollout-level correctness | -```bash -python runtime/launch.py slurm pilot --partition YOUR_GPU_PARTITION \ - --model lfm --mode multi-harness --run-name pilot-lfm-multi-harness-v1 \ - --timeout 12h --limit 25 --concurrency 35 +A 25-task pilot is noisy. A higher reward or lower tool count alone does not establish better answer quality. Incorrect answers are retained as zero; only ungraded failures are retried. TiTO checks validate token/logprob/mask consistency, not benchmark performance. + +### Optional online Trackio + +Offline Trackio and scalar logs are always kept. To mirror charts, create a small config file under `configs/`: + +```json +{ + "project": "my-smoldataenv-pilot", + "trackio_space_id": "your-hf-username/smoldataenv-trackio" +} ``` -Use a local checkpoint path for Slurm resume. Inspect the plan, then add `--submit`. No additional watcher, Space or Slurm job is required for the pilot. +Pass `--config configs/my-run.json`. Your token needs permission to create/update that Space. Config files override [defaults](configs/default.json), so you only need the settings you want to change. Explicit CLI model, mode and concurrency options take precedence. Inspect the resolved settings with `python run.py plan --config configs/my-run.json`. -## 3. Slurm or a local two-GPU machine +## 4. Run locally or on Slurm -The same Python environment and prepared files must be visible on compute nodes. Choose a free GPU partition; no cluster name is hardcoded into the recipe. +On the GPU machine, install the full runtime. Keep the source, Python environment, task files and output directory accessible from the allocated node. ```bash -python runtime/launch.py slurm smoke --partition YOUR_GPU_PARTITION \ - --model lfm --mode opencode --run-name smoke-lfm-native +uv venv --python 3.12 +source .venv/bin/activate +uv pip install -r requirements.lock +python runtime/bootstrap.py +uv pip install --no-deps --no-build-isolation -e .runtime/trl +hf auth login +# Export DAYTONA_API_KEY as in the HF setup above. +python prepare.py +python -m pytest -q tests +``` -python runtime/launch.py slurm train --partition YOUR_GPU_PARTITION \ - --model qwen --mode multi-harness --run-name qwen-multi-v1 +The editable TRL installation supplies metadata needed for checkpoint model cards; it does not change the locked dependencies. `prepare.py` checks task instructions, corrected graders, train/test separation and dataset revisions. -# On a persistent CPU/login process, submit separate GPU eval allocations. -python eval/watch.py --root runs --backend slurm --partition YOUR_GPU_PARTITION \ - --concurrency 50 --max-active 1 --submit +On a local two-GPU machine: + +```bash +CUDA_VISIBLE_DEVICES=0,1 python run.py pilot --model lfm --mode multi-harness \ + --run-name pilot-lfm-multi-local --limit 25 --concurrency 35 ``` -Add `--submit` to the launch commands after inspecting them. They request one node, two GPUs, 16 CPUs and 192 GB RAM. Local execution without Slurm is `CUDA_VISIBLE_DEVICES=0,1 python run.py smoke --model lfm --mode opencode`. +On Slurm: -Each job starts its own vLLM and OpenEnv services on unused local ports, and stops its own process groups on exit. Their URLs are recorded in `services.json`. The capture proxy uses OpenEnv's Gradio tunnel so remote Daytona containers can reach it. Sandbox egress and tunnel reliability must be checked in the rollout smoke. +```bash +python runtime/launch.py slurm pilot --partition YOUR_GPU_PARTITION \ + --model lfm --mode multi-harness --run-name pilot-lfm-multi-slurm \ + --timeout 12h --limit 25 --concurrency 35 +``` + +Add `--submit` to submit the printed `sbatch` command. It requests one node, two GPUs, 16 CPUs, 192 GB RAM and the time limit you specified. Configure your cluster account/QoS as usual. Use `squeue -u "$USER"` and the printed log path to follow the job. No Slurm job is needed when you use HF Jobs. -## 4. Checkpoint and evaluation checks +## 5. Resume or scale up -The smoke saves at updates 1 and 2. Run the watcher with `--once` without `--submit` to verify checkpoint files and inspect eval commands. Then reload a smoke checkpoint for a two-task evaluation: +To resume a pilot, keep its name, configuration and evaluation limit. Point to the last complete training checkpoint: ```bash -python eval/watch.py --root runs --backend slurm --partition YOUR_GPU_PARTITION --once -python run.py eval --model lfm --mode opencode \ - --checkpoint runs/smoke-lfm-native/checkpoint-2 \ - --output runs/smoke-lfm-native-eval --limit 2 --concurrency 2 +python runtime/launch.py hf pilot --model lfm --mode multi-harness \ + --run-name pilot-lfm-multi-v1 --bucket "$HF_BUCKET" \ + --flavor a100x4 --timeout 12h --limit 25 --concurrency 35 \ + --resume /outputs/pilot-lfm-multi-v1/train/checkpoint-50 ``` -That is eight pass@1 pairs for a blackbox policy, or two SETA episodes for whitebox. Require a complete summary, valid captured tokens/logprobs/masks, verified tool counts, finite training metrics and a loadable checkpoint. Inspect `train.log`, `environment.log` and `vllm.log` when a check fails. +Keep `--preflight-smoke` if the original pilot used it. Completed phases and graded eval pairs are retained. For Slurm, use the checkpoint's absolute shared-filesystem path. Async resume restores committed group IDs; unconsumed tails of partially committed groups are recorded and abandoned. A pilot that fails before saving a checkpoint needs a fresh training attempt. + +For a longer run, use `train` instead of `pilot`. The default ceiling is 1,000 updates, with saves every 50 and evaluations every 100 plus the final checkpoint. A separate CPU watcher launches evaluation GPU jobs: + +```bash +python runtime/launch.py hf train --model lfm --mode multi-harness \ + --run-name lfm-multi-full-v1 --bucket "$HF_BUCKET" --timeout 24h +python runtime/launch.py hf watch --run-name eval-watcher \ + --bucket "$HF_BUCKET" --concurrency 35 --max-active-evals 1 --timeout 24h +``` + +Both are dry runs until `--submit` is added. Full evaluation uses all 250 tasks. Keep the watcher alive until final evals finish. The HF pilot qualifies same-job reload; separate-job live bucket handoff is still an additional qualification item in [VALIDATION.md](VALIDATION.md). Do not run the watcher for pilots. + +For Slurm, run `python eval/watch.py --root runs --backend slurm --partition YOUR_GPU_PARTITION --concurrency 35 --max-active 1 --submit` on a persistent process. Failed evaluations remain in its ledger for investigation; it does not resubmit forever. + +## Troubleshooting -Production saves every 50 updates; the watcher evaluates every 100 and the final checkpoint. Full blackbox eval is 1,000 pairs. An incorrect answer is retained as zero; only ungraded failures are retried. Incomplete evaluations are explicitly provisional. Do not retry completed incorrect answers and call the result pass@1. +| Symptom | First check | +|---|---| +| `SCHEDULING` | Hardware allocation or image pull; training has not started | +| Job exits during startup | HF logs, then phase `vllm.log` and `environment.log` | +| Rollouts are ungraded | Sandbox/provider limits, capture tunnel and verifier logs; keep errors separate from wrong answers | +| Reward and gradient remain zero | Per-rollout grades and group contrast; do not assume the optimizer is broken | +| Existing-output error | Use a new run name, or resume a complete checkpoint from the same run | +| Whitebox model-card save fails | Confirm the pinned TRL checkout was installed as above | +| New GPU/kernel combination fails | Run a small smoke first; avoid switching to attention that loses packed-sequence boundaries | -Resume with `--resume /path/to/checkpoint-N`. Blackbox checkpoints preserve committed group IDs, so groups already used by the optimizer are not replayed. Unconsumed tails of partially committed groups are recorded and abandoned; groups with no committed work can be regenerated. Keep the task schedule, model and reward configuration fixed across a resume. +The code fixes the data protocol and supports two pinned models and Daytona sandboxes. Adding a different dataset, model or provider requires adapting the task/template/backend boundary and repeating qualification. It is not a drop-in recipe for arbitrary models. -References: [HF Jobs](https://huggingface.co/docs/huggingface_hub/guides/jobs), [volume configuration](https://huggingface.co/docs/hub/jobs-configuration), [OpenCode CLI](https://dev.opencode.ai/docs/cli/). +References: [HF Jobs](https://huggingface.co/docs/huggingface_hub/guides/jobs), [bucket volumes](https://huggingface.co/docs/hub/jobs-configuration), [Daytona](https://www.daytona.io/docs/). diff --git a/05-multi-harness-rl/RESULTS.md b/05-multi-harness-rl/RESULTS.md new file mode 100644 index 00000000..13ad9d23 --- /dev/null +++ b/05-multi-harness-rl/RESULTS.md @@ -0,0 +1,58 @@ +# Current results + +These are qualification results for this pinned, non-thinking recipe. They do not establish that a 100-step run improves either model. Earlier experiment scores are kept on the [archived branch](https://github.com/adithya-s-k/FineEnvs/tree/archive/data-agent-experiments-20260930/04-data-agent). + +## Completed checks + +| Check | Result | Evidence | +|---|---|---| +| Local training, LFM and Qwen × all three modes | All six completed two updates and saved checkpoints | [Validation record](VALIDATION.md) | +| HF native OpenCode, LFM on A100 | Two updates; both checkpoints persisted | [HF job](https://huggingface.co/jobs/FineEnvs/6abcc3d9031314b6963440a0) | +| HF Harbor, LFM on A100 | Two updates; checkpoint reload; 8/8 pairs graded with TiTO and verified tool counts | [HF job](https://huggingface.co/jobs/FineEnvs/6abcc72d031314b696344167) | +| HF whitebox, LFM on A100 | Corrected preflight completed training and reload evaluation before starting its pilot | [HF job](https://huggingface.co/jobs/FineEnvs/6abcd472031314b696344457) | + +The Harbor reload smoke solved 5/8 pairs on two medium tasks. This is a functional check, not a benchmark. Both standalone HF blackbox training smokes had zero reward contrast; they validated execution and checkpointing, not learning. + +## LFM 100-step pilots + +Snapshot: **2026-09-30 10:17 UTC**. Each pilot uses the same 25 held-out tasks (3 easy, 12 medium, 10 hard), baseline evaluation, 100 training updates, saves at 50/100 and final evaluation. Evaluation concurrency is 35. Blackbox evaluations have 100 task/harness pairs; whitebox has 25 native episodes. + +| Mode | Baseline pass@1 | Optimizer step (metrics) | Learning signal observed? | Job | +|---|---|---:|---|---| +| Native OpenCode | 24/100 = 24% | 10 | Yes: step 9 reward 0.126 and gradient norm 3.50 | [OpenCode pilot](https://huggingface.co/jobs/FineEnvs/6abcd384031314b69634440b) | +| Harbor multi-harness | 17/100 = 17% | 5 | Yes: step 3 reward 0.212 and gradient norm 3.94 | [Harbor pilot](https://huggingface.co/jobs/FineEnvs/6abcd3874c46ef19870359af) | +| Whitebox SETA | 7/25 = 28% | 5 | No reward contrast in its first five updates | [Whitebox pilot](https://huggingface.co/jobs/FineEnvs/6abcd472031314b696344457) | + +All three jobs were running at this check. The whitebox save failure in its earlier smoke was traced to missing TRL package metadata and corrected before this pilot. The running jobs use immutable uploaded source snapshots; later tutorial edits do not modify them. + +Baseline scores above come from complete `baseline/eval/summary.json` files downloaded from the bucket, not minute-by-minute progress counts. The two blackbox pilots independently sampled the same untrained model at temperature 0.8; these are separate stochastic samples, so their difference cannot be attributed to training. Whitebox uses a different native evaluation interface. No checkpoint-100 result exists yet, and these early updates do not establish a rising reward curve. + +| Baseline evaluation harness | Native OpenCode pilot | Harbor pilot | +|---|---:|---:| +| OpenCode | 2/25 (8%) | 3/25 (12%) | +| Claude Code | 5/25 (20%) | 4/25 (16%) | +| Codex | 4/25 (16%) | 2/25 (8%) | +| Mini-SWE-Agent | 13/25 (52%) | 8/25 (32%) | + +The observed training records contain 2 correct answers among 51 native OpenCode rollouts, 4 among 32 Harbor rollouts, and 0 among 80 whitebox episodes. These include collected work and are not counts of optimizer-consumed samples. They explain why useful gradient updates are sparse; they are not held-out scores. + +## Locate an artifact + +Our qualification outputs use bucket `FineEnvs/data-agent-daytona-artifacts`, under these run prefixes: + +- `pilot-lfm-opencode-v2` +- `pilot-lfm-multi-harness-v2` +- `pilot-lfm-whitebox-v1` + +Each prefix contains `pilot.json`, `baseline/`, `train/`, `checkpoint-100/` and, after completion, `comparison.json`. Checkpoints contain model weights, optimizer and RNG state. Logs include local Trackio data, scalar metrics and rollout/audit evidence. Bucket access requires permission; job links may also require an HF login. No public trained-model artifact is claimed for these unfinished pilots. + +The [historical Trackio Space](https://huggingface.co/spaces/FineEnvs/data-agent-training-comparison-trackio) does not automatically contain these new pilots. Their current config keeps Trackio offline in the bucket. Follow the HF job logs for live progress. The tutorial explains how to enable your own online Trackio Space. + +## Remaining qualification + +- Finish the pilots and compare complete baseline/checkpoint-100 results. +- Confirm sustained learning signal, especially in native OpenCode and whitebox. +- Qualify separate evaluation jobs reading newly saved bucket checkpoints before enabling a long-run watcher. The completed smoke tested reload within one job. +- Treat Qwen HF execution and hardware combinations not tested here as needing their own smoke. + +Detailed pins, failure diagnoses, test counts and original smoke evidence are in [VALIDATION.md](VALIDATION.md). diff --git a/05-multi-harness-rl/VALIDATION.md b/05-multi-harness-rl/VALIDATION.md index 0ee6609b..deffa098 100644 --- a/05-multi-harness-rl/VALIDATION.md +++ b/05-multi-harness-rl/VALIDATION.md @@ -74,3 +74,11 @@ Selected scope: LFM only, all three modes, 25 fixed held-out tasks (3 easy, 12 m The first two pilot submissions were canceled while still scheduling, then replaced with copies that print phase and scalar progress in HF logs. No training progress was discarded. Configurations, scalar logs, local Trackio and phase reports live under each run prefix in `FineEnvs/data-agent-daytona-artifacts`. No Slurm pilot has been submitted. Pilot checks: 23 CPU contracts passed, plus the four fetched-runtime integration checks. Coverage includes fixed stratification, preserving original task catalog indices, stopping on incomplete baseline, retaining normal training batches, phase resume and Slurm time limits. A comparison fixture verifies repeated-step deduplication and matched-success tool/token savings. Actual baseline/100-step results are pending. + +## Tutorial audit, 30 September 2026 + +The beginner path now separates lightweight CPU-only HF submission from full local GPU installation. The launcher defaults to the bucket owner's namespace, preserves model/mode/concurrency settings from custom config files unless explicitly overridden, validates the config before submission, and rejects single-GPU allocations. Small config overlays inherit pinned defaults. No active job source or hyperparameter was changed by this audit. + +Checks: 29 tests passed against the fetched runtime. A clean copied checkout in a CPU-only client environment passed 25 contracts, with four runtime checks skipped as intended. HF submission plans and `run.py plan` work without PyTorch. Shell syntax, source compilation, the repository index and relative documentation links were checked. The original source archive and ignored experimental artifacts remain intact. + +The pilot baselines and current gradient evidence were downloaded from the bucket and summarized in [RESULTS.md](RESULTS.md). All three pilots were running; whitebox had passed its corrected preflight. Detailed audit snapshots remain ignored under `runs/qualification/tutorial-audit/`, `tutorial-audit-status.json` and `tutorial-audit-results.json`. No checkpoint-100 result is claimed. diff --git a/05-multi-harness-rl/eval/watch.py b/05-multi-harness-rl/eval/watch.py index 4f132842..692c9830 100644 --- a/05-multi-harness-rl/eval/watch.py +++ b/05-multi-harness-rl/eval/watch.py @@ -32,7 +32,7 @@ def main(): p.add_argument("--root", type=Path, required=True) p.add_argument("--backend", choices=["hf", "slurm"], required=True) p.add_argument("--bucket") - p.add_argument("--namespace", default="FineEnvs") + p.add_argument("--namespace", help="HF Jobs owner; defaults to the bucket owner") p.add_argument("--partition") p.add_argument("--flavor", default="h200x2") p.add_argument("--data", type=Path, default=ROOT / "prepared") @@ -44,6 +44,8 @@ def main(): args.root = args.root.resolve() if args.backend == "hf" and not args.bucket: p.error("--bucket is required for HF Jobs") + if args.backend == "hf": + args.namespace = args.namespace or args.bucket.split("/", 1)[0] if args.backend == "slurm" and not args.partition: p.error("--partition is required for Slurm") if args.max_active < 1 or args.concurrency < 1: diff --git a/05-multi-harness-rl/recipe.py b/05-multi-harness-rl/recipe.py index 40bb1873..2f28ae68 100644 --- a/05-multi-harness-rl/recipe.py +++ b/05-multi-harness-rl/recipe.py @@ -27,8 +27,19 @@ def digest(value): return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() +def merge_config(base, updates): + for key, value in updates.items(): + if isinstance(value, dict) and isinstance(base.get(key), dict): + merge_config(base[key], value) + else: + base[key] = value + return base + + def config(path=None, **overrides): - result = json.loads(Path(path or ROOT / "configs/default.json").read_text()) + result = json.loads((ROOT / "configs/default.json").read_text()) + if path: + merge_config(result, json.loads(Path(path).read_text())) result.update({k: v for k, v in overrides.items() if v is not None}) if result["mode"] not in MODES or result["model"] not in result["models"]: raise ValueError("Unknown mode or model") diff --git a/05-multi-harness-rl/runtime/launch.py b/05-multi-harness-rl/runtime/launch.py index d06cd97b..ed71715b 100644 --- a/05-multi-harness-rl/runtime/launch.py +++ b/05-multi-harness-rl/runtime/launch.py @@ -12,7 +12,7 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -from recipe import MODES, write_json +from recipe import MODES, config, write_json def slurm_time(value): @@ -40,11 +40,11 @@ def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument("backend", choices=["hf", "slurm"]) p.add_argument("action", choices=["train", "eval", "smoke", "pilot", "watch"]) - p.add_argument("--model", choices=["lfm", "qwen"], default="lfm") + p.add_argument("--model", choices=["lfm", "qwen"], help="Override config model (default: lfm)") p.add_argument("--config", type=Path) - p.add_argument("--mode", choices=MODES, default="multi-harness") + p.add_argument("--mode", choices=MODES, help="Override config mode (default: multi-harness)") p.add_argument("--run-name", required=True) - p.add_argument("--namespace", default="FineEnvs") + p.add_argument("--namespace", help="HF Jobs owner; defaults to the bucket owner") p.add_argument("--bucket", help="Existing HF bucket, e.g. FineEnvs/smoldataenv-runs") p.add_argument("--image", default="huggingface/trl@sha256:8433cde7eaf3b289f3daffdaefcab6bb71499ca1ff59dc5d05abba7a71603a58") p.add_argument("--flavor", default="h200x2") @@ -54,7 +54,7 @@ def main(): p.add_argument("--output-root", type=Path, default=ROOT / "runs") p.add_argument("--checkpoint") p.add_argument("--resume") - p.add_argument("--concurrency", type=int, default=35) + p.add_argument("--concurrency", type=int, help="Override eval concurrency (default: 35 from config)") p.add_argument("--limit", type=int) p.add_argument("--smoke-eval", action="store_true") p.add_argument("--preflight-smoke", action="store_true") @@ -67,9 +67,17 @@ def main(): p.error("--preflight-smoke requires the pilot action") if Path(args.run_name).name != args.run_name or args.run_name in {".", ".."}: p.error("run-name must be a single directory name") + if args.config is None or not str(args.config).startswith("/outputs/"): + config(args.config, model=args.model, mode=args.mode, eval_concurrency=args.concurrency) + if args.limit is not None and not 1 <= args.limit <= 250: + p.error("--limit must be between 1 and the 250 fixed test tasks") output = str(Path("/outputs" if args.backend == "hf" else args.output_root.resolve()) / args.run_name) - flags = [args.action, "--model", args.model, "--mode", args.mode, "--run-name", args.run_name, - "--output", output, "--concurrency", str(args.concurrency)] + flags = [args.action, "--run-name", args.run_name, "--output", output] + if args.concurrency is not None: + flags += ["--concurrency", str(args.concurrency)] + for key in ("model", "mode"): + if getattr(args, key) is not None: + flags += ["--" + key, getattr(args, key)] if args.smoke_eval: flags.append("--smoke-eval") if args.preflight_smoke: @@ -92,10 +100,16 @@ def main(): if args.backend == "hf": if not args.bucket: p.error("HF Jobs require --bucket for durable checkpoints and logs") + if len(args.bucket.split("/")) != 2 or not all(args.bucket.split("/")): + p.error("Use a bucket ID in owner/name form") + args.namespace = args.namespace or args.bucket.split("/", 1)[0] + gpu_count = re.search(r"x([0-9]+)$", args.flavor) + if args.action != "watch" and (gpu_count is None or int(gpu_count[1]) < 2): + p.error("This recipe needs at least two GPUs; use h200x2 or a100x4") if args.action == "watch": flags = ["watch", "--root", "/outputs", "--backend", "hf", "--bucket", args.bucket, "--namespace", args.namespace, "--flavor", args.flavor, - "--concurrency", str(args.concurrency), "--max-active", str(args.max_active_evals), "--submit"] + "--concurrency", str(args.concurrency if args.concurrency is not None else 35), "--max-active", str(args.max_active_evals), "--submit"] specification = {"image": args.image, "command": ["bash", "/recipe-source/runtime/job.sh", *flags], "flavor": "cpu-basic" if args.action == "watch" else args.flavor, "timeout": args.timeout, "namespace": args.namespace, "name": args.run_name, "labels": {"recipe": "smoldataenv-multi-harness", "role": args.action}, @@ -124,7 +138,7 @@ def main(): if args.action == "watch": command = [sys.executable, str(ROOT / "eval/watch.py"), "--root", str(args.output_root.resolve()), "--backend", "slurm", "--partition", args.partition, "--data", str(args.data.resolve()), - "--concurrency", str(args.concurrency), "--max-active", str(args.max_active_evals), "--submit"] + "--concurrency", str(args.concurrency if args.concurrency is not None else 35), "--max-active", str(args.max_active_evals), "--submit"] submit = ["sbatch", "--parsable", "--partition", args.partition, "--nodes=1", "--gpus=2", "--cpus-per-task=16", "--mem=192G", "--time=" + slurm_time(args.timeout), "--job-name", args.run_name, "--output", str(args.output_root / (args.run_name + "-%j.log")), "--wrap", shlex.join(command)] diff --git a/05-multi-harness-rl/tests/test_contracts.py b/05-multi-harness-rl/tests/test_contracts.py index af5a62bf..7eb081e0 100644 --- a/05-multi-harness-rl/tests/test_contracts.py +++ b/05-multi-harness-rl/tests/test_contracts.py @@ -162,6 +162,33 @@ def test_native_means_native(): assert {g["harness"] for g in schedule(config(mode="opencode"), task_rows("train"))} == {"opencode"} +def test_config_overlays_preserve_defaults_and_explicit_overrides(tmp_path): + path = tmp_path / "my-run.json" + write_json(path, {"model": "qwen", "mode": "whitebox", "eval_concurrency": 7, + "models": {"qwen": {"template": "enable_thinking_false"}}}) + cfg = config(path) + assert cfg["model"] == "qwen" and cfg["mode"] == "whitebox" and cfg["eval_concurrency"] == 7 + assert cfg["learning_rate"] == 3e-6 + assert cfg["profile"]["id"] == "Qwen/Qwen3.5-2B" + assert "lfm" in cfg["models"] + assert config(path, model="lfm")["profile"]["id"] == "LiquidAI/LFM2.5-2.6B" + + +def test_submission_uses_bucket_owner_without_overriding_config(monkeypatch, capsys): + from runtime import launch + monkeypatch.setattr(sys, "argv", ["launch.py", "hf", "pilot", "--bucket", "reader/my-runs", + "--run-name", "test", "--flavor", "a100x4"]) + launch.main() + plan = json.loads(capsys.readouterr().out) + assert plan["namespace"] == "reader" + assert "--mode" not in plan["command"] and "--model" not in plan["command"] + assert "--concurrency" not in plan["command"] + monkeypatch.setattr(sys, "argv", ["launch.py", "hf", "pilot", "--bucket", "reader/my-runs", + "--run-name", "test", "--flavor", "a100-large"]) + with pytest.raises(SystemExit): + launch.main() + + def test_nonthinking_preflight_checks_streaming_content(monkeypatch): hidden = False def create(*, stream, **kwargs): From 1e5ca01687baaed0f59bdc64cd8c3b40b6905e68 Mon Sep 17 00:00:00 2001 From: adithya-s-k Date: Wed, 30 Sep 2026 10:21:02 +0000 Subject: [PATCH 11/48] Refresh pilot monitoring snapshot --- 05-multi-harness-rl/RESULTS.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/05-multi-harness-rl/RESULTS.md b/05-multi-harness-rl/RESULTS.md index 13ad9d23..2e52b352 100644 --- a/05-multi-harness-rl/RESULTS.md +++ b/05-multi-harness-rl/RESULTS.md @@ -15,12 +15,12 @@ The Harbor reload smoke solved 5/8 pairs on two medium tasks. This is a function ## LFM 100-step pilots -Snapshot: **2026-09-30 10:17 UTC**. Each pilot uses the same 25 held-out tasks (3 easy, 12 medium, 10 hard), baseline evaluation, 100 training updates, saves at 50/100 and final evaluation. Evaluation concurrency is 35. Blackbox evaluations have 100 task/harness pairs; whitebox has 25 native episodes. +Snapshot: **2026-09-30 10:20 UTC**. Each pilot uses the same 25 held-out tasks (3 easy, 12 medium, 10 hard), baseline evaluation, 100 training updates, saves at 50/100 and final evaluation. Evaluation concurrency is 35. Blackbox evaluations have 100 task/harness pairs; whitebox has 25 native episodes. | Mode | Baseline pass@1 | Optimizer step (metrics) | Learning signal observed? | Job | |---|---|---:|---|---| | Native OpenCode | 24/100 = 24% | 10 | Yes: step 9 reward 0.126 and gradient norm 3.50 | [OpenCode pilot](https://huggingface.co/jobs/FineEnvs/6abcd384031314b69634440b) | -| Harbor multi-harness | 17/100 = 17% | 5 | Yes: step 3 reward 0.212 and gradient norm 3.94 | [Harbor pilot](https://huggingface.co/jobs/FineEnvs/6abcd3874c46ef19870359af) | +| Harbor multi-harness | 17/100 = 17% | 8 | Yes: step 8 reward 0.598 and gradient norm 2.70 | [Harbor pilot](https://huggingface.co/jobs/FineEnvs/6abcd3874c46ef19870359af) | | Whitebox SETA | 7/25 = 28% | 5 | No reward contrast in its first five updates | [Whitebox pilot](https://huggingface.co/jobs/FineEnvs/6abcd472031314b696344457) | All three jobs were running at this check. The whitebox save failure in its earlier smoke was traced to missing TRL package metadata and corrected before this pilot. The running jobs use immutable uploaded source snapshots; later tutorial edits do not modify them. @@ -34,7 +34,7 @@ Baseline scores above come from complete `baseline/eval/summary.json` files down | Codex | 4/25 (16%) | 2/25 (8%) | | Mini-SWE-Agent | 13/25 (52%) | 8/25 (32%) | -The observed training records contain 2 correct answers among 51 native OpenCode rollouts, 4 among 32 Harbor rollouts, and 0 among 80 whitebox episodes. These include collected work and are not counts of optimizer-consumed samples. They explain why useful gradient updates are sparse; they are not held-out scores. +The 10:17 UTC artifact sample contains 2 correct answers among 51 native OpenCode rollouts, 4 among 32 Harbor rollouts, and 0 among 80 whitebox episodes. These include collected work and are not counts of optimizer-consumed samples. They explain why useful gradient updates are sparse; they are not held-out scores. ## Locate an artifact From 3bdb81233c19f937807ec2b5ad184b83a1ba2b85 Mon Sep 17 00:00:00 2001 From: adithya-s-k Date: Wed, 30 Sep 2026 10:27:50 +0000 Subject: [PATCH 12/48] Simplify the tutorial and validation notes --- 05-multi-harness-rl/README.md | 4 +- 05-multi-harness-rl/REPRODUCE.md | 7 +- 05-multi-harness-rl/RESULTS.md | 8 +-- 05-multi-harness-rl/VALIDATION.md | 114 ++++++++++++++---------------- 4 files changed, 62 insertions(+), 71 deletions(-) diff --git a/05-multi-harness-rl/README.md b/05-multi-harness-rl/README.md index 0ca755cc..bb8e5209 100644 --- a/05-multi-harness-rl/README.md +++ b/05-multi-harness-rl/README.md @@ -71,7 +71,7 @@ There is **no Space to deploy** for these runs. OpenEnv lives inside the GPU job ## Find the code -| Path | Read it when you want to… | +| Path | What it does | |---|---| | `configs/default.json` | Inspect model pins and shared hyperparameters | | `run.py`, `runtime/launch.py` | Run locally, submit to HF Jobs or inspect a Slurm command | @@ -81,7 +81,7 @@ There is **no Space to deploy** for these runs. OpenEnv lives inside the GPU job | `eval/evaluate.py` | Inspect pass@1, coverage and per-harness results | | `eval/watch.py` | Submit separate evaluations for full training checkpoints | | `prepare.py`, `data/` | Inspect the exact task selection and dataset checks | -| `configs/runtime-lock.json`, `runtime/patches.py` | Audit pinned dependencies and their small adaptations | +| `configs/runtime-lock.json`, `runtime/patches.py` | Inspect pinned dependencies and compatibility fixes | `runtime/bootstrap.py` downloads the pinned TRL/OpenEnv sources and 45 existing runtime files into ignored `.runtime/`. It checks the archive file hashes. You do not need the original experiment workspace. Prepared tasks, checkpoints, caches and credentials are not committed. diff --git a/05-multi-harness-rl/REPRODUCE.md b/05-multi-harness-rl/REPRODUCE.md index 07d6a734..c4e5808a 100644 --- a/05-multi-harness-rl/REPRODUCE.md +++ b/05-multi-harness-rl/REPRODUCE.md @@ -1,12 +1,13 @@ # Run SmolDataEnv RL -Start from a checkout containing this example: +Check out the example branch: ```bash -cd 05-multi-harness-rl +git clone --branch 05-multi-harness-rl https://github.com/adithya-s-k/FineEnvs.git +cd FineEnvs/05-multi-harness-rl ``` -Pick **HF Jobs** if you want the container to install and prepare everything. Pick **local/Slurm** if you already have a compatible GPU machine. Both use the same code, fixed tasks and model revisions. The current branch is local until published; the commands assume you already have its files. +Pick **HF Jobs** if you want the container to install and prepare everything. Pick **local/Slurm** if you already have a compatible GPU machine. Both use the same code, fixed tasks and model revisions. ## 1. What you need diff --git a/05-multi-harness-rl/RESULTS.md b/05-multi-harness-rl/RESULTS.md index 2e52b352..58313259 100644 --- a/05-multi-harness-rl/RESULTS.md +++ b/05-multi-harness-rl/RESULTS.md @@ -1,6 +1,6 @@ # Current results -These are qualification results for this pinned, non-thinking recipe. They do not establish that a 100-step run improves either model. Earlier experiment scores are kept on the [archived branch](https://github.com/adithya-s-k/FineEnvs/tree/archive/data-agent-experiments-20260930/04-data-agent). +These are the checks and pilot results for the pinned, non-thinking recipe. The 100-step comparisons are still in progress. Earlier experiment scores are kept on the [archived branch](https://github.com/adithya-s-k/FineEnvs/tree/archive/data-agent-experiments-20260930/04-data-agent). ## Completed checks @@ -38,7 +38,7 @@ The 10:17 UTC artifact sample contains 2 correct answers among 51 native OpenCod ## Locate an artifact -Our qualification outputs use bucket `FineEnvs/data-agent-daytona-artifacts`, under these run prefixes: +Our test and pilot outputs use bucket `FineEnvs/data-agent-daytona-artifacts`, under these run prefixes: - `pilot-lfm-opencode-v2` - `pilot-lfm-multi-harness-v2` @@ -48,11 +48,11 @@ Each prefix contains `pilot.json`, `baseline/`, `train/`, `checkpoint-100/` and, The [historical Trackio Space](https://huggingface.co/spaces/FineEnvs/data-agent-training-comparison-trackio) does not automatically contain these new pilots. Their current config keeps Trackio offline in the bucket. Follow the HF job logs for live progress. The tutorial explains how to enable your own online Trackio Space. -## Remaining qualification +## Remaining checks - Finish the pilots and compare complete baseline/checkpoint-100 results. - Confirm sustained learning signal, especially in native OpenCode and whitebox. -- Qualify separate evaluation jobs reading newly saved bucket checkpoints before enabling a long-run watcher. The completed smoke tested reload within one job. +- Test separate evaluation jobs reading newly saved bucket checkpoints before enabling a long-run watcher. The completed smoke tested reload within one job. - Treat Qwen HF execution and hardware combinations not tested here as needing their own smoke. Detailed pins, failure diagnoses, test counts and original smoke evidence are in [VALIDATION.md](VALIDATION.md). diff --git a/05-multi-harness-rl/VALIDATION.md b/05-multi-harness-rl/VALIDATION.md index deffa098..fe1ff1c7 100644 --- a/05-multi-harness-rl/VALIDATION.md +++ b/05-multi-harness-rl/VALIDATION.md @@ -1,84 +1,74 @@ -# Qualification notes +# Validation -This is a new non-thinking recipe. Historical training results are not its baseline. +This page records what we tested for the pinned, non-thinking recipe. [RESULTS.md](RESULTS.md) has the pilot scores and progress; [REPRODUCE.md](REPRODUCE.md) has the commands. -## Data and runtime +## Automated checks -- Public Hub preparation passed for all 1,000 training tasks and 250 test tasks. Instruction hashes match the fixed selection; the corrected numeric-only grader is checked for every task. -- Train revision: `4719635555de1666f374d847baebb500d368e493`. Test revision: `b130595b579fc6026ad5762925a9a3b40e72f5bb`. -- Compared with the old staged files, 302 training tasks now use the JSON reward wrapper. Task metadata also changed. The test instructions and grader did not change. -- OpenEnv is pinned to public commit `4f4c85fb9038f43efc2f51858a27638277f16355`, including partial-mask validation. No unpublished OpenEnv checkout is required. -- TRL, native environment sources and file hashes are in `configs/runtime-lock.json`. The Python dependency set resolves successfully; `requirements.lock` records the resolved versions. +- 29 tests passed with the downloaded runtime and prepared tasks. +- A clean CPU-only checkout passed 25 tests. Four integration tests skip until the runtime and tasks are downloaded. +- HF submission plans and `run.py plan` work without PyTorch installed. +- Shell syntax, Python compilation, documentation links and the repository index passed. -## Checks on 30 September 2026 +Tests cover task identity, train/test separation, reward calculation, token/logprob alignment, partial masks, checkpoint integrity, resume, config overrides and pilot failure handling. They also check that the uploaded source excludes credentials and local run artifacts. -| Check | Result | -|---|---| -| CPU contracts and fetched-runtime integration | 22 passed, including Ampere attention selection | -| Clean, minimal CI environment | 17 passed; four runtime checks skipped without fetched dependencies | -| Native OpenEnv to TRL partial mask | `[1, 0]` preserved | -| Native LFM checkpoint handoff | Both initial smoke checkpoints saved; file hashes verified | -| Corrected four-harness reload eval | 8/8 pairs graded; all token checks passed; four correct answers | -| Qwen whitebox training | Two updates and both checkpoints saved; second update had reward contrast and nonzero gradient | -| Qwen whitebox token audit | Both batches passed; 68 captured calls in total | -| Corrected non-thinking HTTP and streaming replies | Passed for LFM and Qwen | -| Both models, all three training modes | All six completed two updates and saved both checkpoints | -| Whitebox checkpoint reload eval | 2/2 tasks graded; one correct answer | -| Async checkpoint resume | Resumed step 1 to 2; committed group 0 was not replayed | -| HF Jobs setup | Fresh Python 3.12 environment, locked installation and all 1,250 tasks prepared successfully | -| HF Jobs native OpenCode GPU smoke | A100 completed two steps; both saved checkpoint inventories match the bucket | -| HF Jobs Harbor smoke and reload eval | Completed two training steps; checkpoint reload graded 8/8 pairs with TiTO and verified tool counts | +## Local GPU tests -| Model | Mode | Local smoke wall time | Nonzero gradient observed? | -|---|---|---|---| -| Qwen3.5-2B | Whitebox | 10m 26s | Yes | -| Qwen3.5-2B | Native OpenCode | 8m 35s | No reward contrast in this sample | -| Qwen3.5-2B | Harbor multi-harness | 9m 07s | Yes | -| LFM2.5-2.6B | Whitebox | 6m 37s | No reward contrast in this sample | -| LFM2.5-2.6B | Native OpenCode | 9m 30s | No reward contrast in this sample | -| LFM2.5-2.6B | Harbor multi-harness | 13m 01s | Yes | +Both models completed two optimizer updates and saved both checkpoints in all three modes. -Wall time includes model loading, sandbox setup, two updates, saving and cleanup. Smoke batches use two rollouts and low concurrency, not the full training batch or eval concurrency 35. Local smokes use the existing cluster runtime; the separate HF check qualifies a fresh installation from the lockfile. +| Model | Mode | Total time | Nonzero gradient observed? | Slurm job | +|---|---|---|---|---| +| Qwen3.5-2B | Whitebox | 10m 26s | Yes | `92750` | +| Qwen3.5-2B | Native OpenCode | 8m 35s | No reward contrast in this sample | `92753` | +| Qwen3.5-2B | Harbor | 9m 07s | Yes | `92760` | +| LFM2.5-2.6B | Whitebox | 6m 37s | No reward contrast in this sample | `92759` | +| LFM2.5-2.6B | Native OpenCode | 9m 30s | No reward contrast in this sample | `92752` | +| LFM2.5-2.6B | Harbor | 13m 01s | Yes | `92751` | -The initial LFM smoke exposed a serving error: the reasoning parser classified tool-call text as reasoning, leaving Claude Code with empty assistant replies. Non-thinking serving now omits that parser. Claude Code subsequently emitted native tool calls, and a Harbor OpenCode rollout produced correctness `1` with nine calls and shaped reward `1.0625`. Initial LFM smoke scores must not be reused as a baseline. +These times include startup, sandbox work, training, saving and cleanup. Smokes use two rollouts per group and low concurrency; they do not measure full-run throughput. -Other fixes found by the smokes: job-specific ports on shared Slurm nodes, explicit HF login forwarding to Harbor task staging, the SDK's typed bucket mount, explicit tool-response templates, the whitebox cache-reset response, and the text context limit on Qwen's outer model config. The pinned base image has Python 3.11; the HF entrypoint now creates its own Python 3.12 environment before installing the lockfile. +Additional checks passed: -## Evidence +- Whitebox token audits matched 68 engine calls per model. +- LFM checkpoint evaluation graded 8/8 task/harness pairs with valid token capture (job `92757`). +- Qwen whitebox checkpoint evaluation graded 2/2 tasks (job `92821`). +- Async resume continued from step 1 to step 2 without replaying committed group 0 (job `92822`). +- The OpenEnv-to-TRL adapter preserved the partial completion mask `[1, 0]`. -Local smoke artifacts are ignored under `runs/qualification/`: configuration, service logs, rollout reward records, `token-audit.jsonl`, local Trackio data and checkpoints. CPU logs are `cpu-tests-final.log` and `ci-tests.log`; public data preparation is `prepare-hub.log` and `prepared-hub/`. +Local evidence is kept under ignored `runs/qualification/`, including service logs, reward records, token audits and checkpoints. -- Qwen whitebox: `qwen-whitebox-v5`, Slurm job `92750`. -- Corrected LFM Harbor: `lfm-multi-harness-v4`, job `92751`. -- Corrected native OpenCode: `lfm-opencode-v4` / `qwen-opencode-v1`, jobs `92752` / `92753`. -- Corrected LFM reload evaluation: `lfm-nonthinking-eval-v2`, job `92757`. -- LFM whitebox / Qwen Harbor: jobs `92759` / `92760`. -- Whitebox checkpoint eval: `qwen-whitebox-v5-eval`, job `92821`. -- Async resume: `lfm-native-resume-v1`, job `92822`. Only optimizer step 2 was executed; the new admission record contains group 2, while the restored state retains group 0. -- [HF setup check](https://huggingface.co/jobs/FineEnvs/6abcb82f031314b696343d7d): completed with the corrected Python 3.12 bootstrap. -- [H200 GPU smoke](https://huggingface.co/jobs/FineEnvs/6abcb8c8031314b696343daf): canceled after remaining queued for hardware. -- [A100 native OpenCode smoke](https://huggingface.co/jobs/FineEnvs/6abcc3d9031314b6963440a0): completed. Runtime from container start to finish was 10m 05s; the trainer reported 335.3s. Both optimizer steps had zero reward contrast and zero gradient, so this confirms execution rather than learning improvement. Six completed rollout records had verified native tool counts. Both checkpoints include model, optimizer and RNG state; all 13 recorded file sizes match the persistent bucket. Byte hashes and cross-job reload remain unverified for this checkpoint. Evidence downloaded to `runs/qualification/hf-a100-native-v1/`. -- [A100 Harbor smoke](https://huggingface.co/jobs/FineEnvs/6abcc72d031314b696344167): two-step multi-harness training followed by checkpoint-2 reload evaluation on two tasks × four harnesses, concurrency four, 45-minute cap. Completed with `--smoke-eval` in 21m 17s overall; training took 627.9s. Both updates had zero reward contrast and zero gradient. Checkpoint 2 was hashed and reloaded with restarted services. All eight evaluation pairs graded on their first attempt, with TiTO and verified tool counts: OpenCode 1/2, Claude Code 1/2, Codex 1/2, Mini-SWE-Agent 2/2. These two medium tasks are a smoke, not a benchmark. Evidence is under `runs/qualification/hf-a100-harbor-v1/`. This tests all four evaluation adapters, while two training steps alone need not consume all four harnesses. The new orchestration has 19 passing CPU contract checks, including aborting reload after training failure. +## HF Jobs tests -The native and Harbor HF execution smokes passed. A same-job reload does not qualify a separate evaluation job reading the bucket; that handoff remains a separate check. Neither HF training sample had reward contrast, so a positive-gradient HF check also remains before claiming learning validation. Then collect a fresh non-thinking baseline on the fixed 250-task test set. A tiny smoke establishes execution, not benchmark performance or throughput at concurrency 35. +| Test | Result | Evidence | +|---|---|---| +| Fresh installation | Python 3.12, locked dependencies and all 1,250 tasks prepared | [Setup job](https://huggingface.co/jobs/FineEnvs/6abcb82f031314b696343d7d) | +| Native OpenCode on A100 | Two steps; both checkpoint file inventories match the bucket | [Native smoke](https://huggingface.co/jobs/FineEnvs/6abcc3d9031314b6963440a0) | +| Harbor on A100 | Two steps; checkpoint 2 reloaded; 8/8 eval pairs passed capture checks and graded on the first attempt | [Harbor smoke](https://huggingface.co/jobs/FineEnvs/6abcc72d031314b696344167) | +| Whitebox on A100 | Two-step preflight and reload eval passed before its pilot started | [Whitebox pilot](https://huggingface.co/jobs/FineEnvs/6abcd472031314b696344457) | -## 100-step pilot launch +Native OpenCode took 10m 05s overall, including 335.3s in the trainer. Harbor took 21m 17s overall, including 627.9s in the trainer. Both standalone blackbox smokes had zero reward contrast. The later pilots produced nonzero gradients; their current progress is in [RESULTS.md](RESULTS.md). -Selected scope: LFM only, all three modes, 25 fixed held-out tasks (3 easy, 12 medium, 10 hard). Baseline and checkpoint-100 evaluations use the same task IDs. Each blackbox evaluation has 100 pairs; whitebox has 25 episodes. Save at 50/100, eval concurrency 35, eight rollouts per group, unchanged full training batches and LR. Phases run sequentially inside one HF allocation, bounded to 12 hours. Slurm plans use the same command with two GPUs and the same time cap. +## Fixes found during testing -- [Native OpenCode pilot](https://huggingface.co/jobs/FineEnvs/6abcd384031314b69634440b), `pilot-lfm-opencode-v2`: submitted. -- [Harbor multi-harness pilot](https://huggingface.co/jobs/FineEnvs/6abcd3874c46ef19870359af), `pilot-lfm-multi-harness-v2`: submitted. -- [Whitebox HF qualification](https://huggingface.co/jobs/FineEnvs/6abcd28f4c46ef1987035955): failed saving step 1 because the fresh environment lacked TRL package metadata. Training and token auditing had reached the update. The entrypoint now installs the pinned checkout with `--no-deps --no-build-isolation`; package metadata was verified in a clean Python environment. -- [Whitebox pilot](https://huggingface.co/jobs/FineEnvs/6abcd472031314b696344457), `pilot-lfm-whitebox-v1`: submitted with `--preflight-smoke`. It must finish two training updates and reload eval before starting the baseline/100-step phases; any failed qualification stops the job. +- **Hidden tool calls:** the LFM reasoning parser swallowed tool-call text. Non-thinking serving now omits that parser and checks visible streaming and ordinary replies. Earlier smoke scores from that setup are superseded. +- **A100 attention:** the pinned async trainer hardcoded FlashAttention 3. A checked adapter selects FlashAttention 2 on Ampere while preserving packed-sequence boundaries. +- **Fresh HF installation:** the image contains Python 3.11. The entrypoint creates Python 3.12 and installs the pinned TRL checkout, including metadata needed for whitebox checkpoint saving. +- **Whitebox compatibility:** explicit response templates, Qwen's text context limit and an empty successful cache-reset response needed handling. +- **Shared-node startup:** job-specific ports, HF login forwarding and the typed bucket mount were tested in the live smokes. -The first two pilot submissions were canceled while still scheduling, then replaced with copies that print phase and scalar progress in HF logs. No training progress was discarded. Configurations, scalar logs, local Trackio and phase reports live under each run prefix in `FineEnvs/data-agent-daytona-artifacts`. No Slurm pilot has been submitted. +## Data and source versions -Pilot checks: 23 CPU contracts passed, plus the four fetched-runtime integration checks. Coverage includes fixed stratification, preserving original task catalog indices, stopping on incomplete baseline, retaining normal training batches, phase resume and Slurm time limits. A comparison fixture verifies repeated-step deduplication and matched-success tool/token savings. Actual baseline/100-step results are pending. +The task lists contain 1,000 training tasks and 250 test tasks, with no task, notebook or instruction-hash overlap. Preparation verifies every instruction and the corrected numeric-only grader. -## Tutorial audit, 30 September 2026 +- Train revision: `4719635555de1666f374d847baebb500d368e493`. +- Test revision: `b130595b579fc6026ad5762925a9a3b40e72f5bb`. +- OpenEnv: `4f4c85fb9038f43efc2f51858a27638277f16355`, including partial-mask validation. +- Other source revisions and archive hashes: [runtime-lock.json](configs/runtime-lock.json). +- Python dependencies: [requirements.lock](requirements.lock). -The beginner path now separates lightweight CPU-only HF submission from full local GPU installation. The launcher defaults to the bucket owner's namespace, preserves model/mode/concurrency settings from custom config files unless explicitly overridden, validates the config before submission, and rejects single-GPU allocations. Small config overlays inherit pinned defaults. No active job source or hyperparameter was changed by this audit. +Compared with the old staged task files, 302 selected training tasks now use the JSON reward wrapper. Instructions and the corrected grader match; metadata changed. Historical results need to be checked against each run's actual data and serving configuration. -Checks: 29 tests passed against the fetched runtime. A clean copied checkout in a CPU-only client environment passed 25 contracts, with four runtime checks skipped as intended. HF submission plans and `run.py plan` work without PyTorch. Shell syntax, source compilation, the repository index and relative documentation links were checked. The original source archive and ignored experimental artifacts remain intact. +## Still to check -The pilot baselines and current gradient evidence were downloaded from the bucket and summarized in [RESULTS.md](RESULTS.md). All three pilots were running; whitebox had passed its corrected preflight. Detailed audit snapshots remain ignored under `runs/qualification/tutorial-audit/`, `tutorial-audit-status.json` and `tutorial-audit-results.json`. No checkpoint-100 result is claimed. +The three LFM 100-step pilots are in progress. Checkpoint-100 scores and sustained learning remain unverified. HF same-job reload has passed, but a separate evaluation job loading newly saved bucket checkpoints still needs testing before relying on the long-run watcher. Qwen HF execution and other hardware combinations need their own smoke. + +This PR contains the recipe, tests and evidence links. Large artifacts remain in the HF bucket or ignored local run directories. From 0db10f400553eba0a8dab10b549e78549029e930 Mon Sep 17 00:00:00 2001 From: adithya-s-k Date: Wed, 30 Sep 2026 10:30:34 +0000 Subject: [PATCH 13/48] Remove job links from the pilot summary table --- 05-multi-harness-rl/RESULTS.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/05-multi-harness-rl/RESULTS.md b/05-multi-harness-rl/RESULTS.md index 58313259..fb5e246c 100644 --- a/05-multi-harness-rl/RESULTS.md +++ b/05-multi-harness-rl/RESULTS.md @@ -17,11 +17,11 @@ The Harbor reload smoke solved 5/8 pairs on two medium tasks. This is a function Snapshot: **2026-09-30 10:20 UTC**. Each pilot uses the same 25 held-out tasks (3 easy, 12 medium, 10 hard), baseline evaluation, 100 training updates, saves at 50/100 and final evaluation. Evaluation concurrency is 35. Blackbox evaluations have 100 task/harness pairs; whitebox has 25 native episodes. -| Mode | Baseline pass@1 | Optimizer step (metrics) | Learning signal observed? | Job | -|---|---|---:|---|---| -| Native OpenCode | 24/100 = 24% | 10 | Yes: step 9 reward 0.126 and gradient norm 3.50 | [OpenCode pilot](https://huggingface.co/jobs/FineEnvs/6abcd384031314b69634440b) | -| Harbor multi-harness | 17/100 = 17% | 8 | Yes: step 8 reward 0.598 and gradient norm 2.70 | [Harbor pilot](https://huggingface.co/jobs/FineEnvs/6abcd3874c46ef19870359af) | -| Whitebox SETA | 7/25 = 28% | 5 | No reward contrast in its first five updates | [Whitebox pilot](https://huggingface.co/jobs/FineEnvs/6abcd472031314b696344457) | +| Mode | Baseline pass@1 | Optimizer step (metrics) | Learning signal observed? | +|---|---|---:|---| +| Native OpenCode | 24/100 = 24% | 10 | Yes: step 9 reward 0.126 and gradient norm 3.50 | +| Harbor multi-harness | 17/100 = 17% | 8 | Yes: step 8 reward 0.598 and gradient norm 2.70 | +| Whitebox SETA | 7/25 = 28% | 5 | No reward contrast in its first five updates | All three jobs were running at this check. The whitebox save failure in its earlier smoke was traced to missing TRL package metadata and corrected before this pilot. The running jobs use immutable uploaded source snapshots; later tutorial edits do not modify them. From 7b11118432504701d79fed85f70e067d891b7fc3 Mon Sep 17 00:00:00 2001 From: adithya-s-k Date: Wed, 30 Sep 2026 10:36:49 +0000 Subject: [PATCH 14/48] Link SmolDataEnvs, article curves and the quick smoke --- 04-smoldataenvs/project.yaml | 23 + 05-multi-harness-rl/README.md | 24 +- 05-multi-harness-rl/REPRODUCE.md | 2 +- 05-multi-harness-rl/RESULTS.md | 23 + .../assets/historical-lfm-curves.json | 1 + .../assets/historical-lfm-curves.svg | 4381 +++++++++++++++++ 05-multi-harness-rl/assets/plot_history.py | 65 + 05-multi-harness-rl/project.yaml | 4 +- README.md | 3 +- 9 files changed, 4520 insertions(+), 6 deletions(-) create mode 100644 04-smoldataenvs/project.yaml create mode 100644 05-multi-harness-rl/assets/historical-lfm-curves.json create mode 100644 05-multi-harness-rl/assets/historical-lfm-curves.svg create mode 100644 05-multi-harness-rl/assets/plot_history.py diff --git a/04-smoldataenvs/project.yaml b/04-smoldataenvs/project.yaml new file mode 100644 index 00000000..74d765f9 --- /dev/null +++ b/04-smoldataenvs/project.yaml @@ -0,0 +1,23 @@ +name: smoldataenvs +title: "SmolDataEnvs" +tagline: "Data-analysis tasks, verified SFT traces and a single-turn GRPO tutorial on HF Jobs." +order: 4 +status: wip +hub: + org: FineEnvs + collection: FineEnvs/smoldataenvs + datasets: + - FineEnvs/SmolDataEnvs + - FineEnvs/SmolDataEnvs-sft + - FineEnvs/SmolDataEnvs-harbor-train + - FineEnvs/SmolDataEnvs-harbor-test + - FineEnvs/SmolDataEnvs-harbor-eval + trackio: + - AdithyaSK/smoldataenvs-trackio +envs: + - name: python-sandbox + summary: "Run one generated Python program against task data and grade its answer." + backend: hf-jobs + frameworks: + trl: + verified: false diff --git a/05-multi-harness-rl/README.md b/05-multi-harness-rl/README.md index bb8e5209..8b5f9e2b 100644 --- a/05-multi-harness-rl/README.md +++ b/05-multi-harness-rl/README.md @@ -1,7 +1,9 @@ -# SmolDataEnv: learn RL through three agent interfaces +# SmolDataEnvs: learn RL through three agent interfaces Fine-tune a small language model to solve data-analysis tasks, then measure whether it answers more questions correctly and uses fewer tools. This example compares three ways of connecting an agent to a trainer, using the same tasks and reward. +Start with [04: SmolDataEnvs](../04-smoldataenvs/) for the dataset, SFT and single-turn RL. This chapter takes those tasks into multi-turn agent harnesses. The companion article, [The ultimate guide to multi-harness RL](https://huggingface.co/spaces/AdithyaSK/multi-harness-rl), explains the approach and earlier experiments. + **Start here:** [run the tutorial](REPRODUCE.md). You can submit an HF Job from a CPU laptop, or run on a local two-GPU machine or Slurm cluster. [Current results](RESULTS.md) distinguish completed checks from ongoing experiments; [validation notes](VALIDATION.md) contain the detailed evidence. ## Choose an agent interface @@ -59,6 +61,16 @@ The full schedule contains two passes and stops at 1,000 updates or schedule exh ## Start small, then scale +After the [HF account, bucket and sandbox setup](REPRODUCE.md#2-submit-from-a-cpu-laptop-with-hf-jobs), preview a quick training and evaluation smoke: + +```bash +python runtime/launch.py hf smoke --model lfm --mode opencode \ + --run-name smoke-lfm-opencode-v1 --bucket "$HF_BUCKET" \ + --flavor a100x4 --timeout 45m --smoke-eval --concurrency 4 +``` + +Add `--submit` to launch. This runs two small updates, saves both checkpoints and reloads checkpoint 2 for evaluation on two tasks through all four harnesses. For the other modes, use `--mode multi-harness` or `--mode whitebox` and a new run name. Whitebox evaluates two native SETA episodes. The [tutorial](REPRODUCE.md) also covers local and Slurm execution. + | Action | Purpose | |---|---| | `smoke --smoke-eval` | Two small training updates, checkpoint saving, then two-task reload evaluation | @@ -85,4 +97,12 @@ There is **no Space to deploy** for these runs. OpenEnv lives inside the GPU job `runtime/bootstrap.py` downloads the pinned TRL/OpenEnv sources and 45 existing runtime files into ignored `.runtime/`. It checks the archive file hashes. You do not need the original experiment workspace. Prepared tasks, checkpoints, caches and credentials are not committed. -This is the condensed successor to the [archived experiment](https://github.com/adithya-s-k/FineEnvs/tree/archive/data-agent-experiments-20260930/04-data-agent). The [historical dashboard](https://huggingface.co/spaces/FineEnvs/data-agent-training-comparison-trackio) contains those earlier runs, not the live pilots unless explicitly configured. Corrected Harbor graders and dataset revisions are pinned here; see [VALIDATION.md](VALIDATION.md) before comparing historical scores. +## Earlier training results + +![Historical LFM training reward, held-out pass@1 and tool-call savings](assets/historical-lfm-curves.svg) + +These curves come from the [article](https://huggingface.co/spaces/AdithyaSK/multi-harness-rl), using 250 test tasks across four harnesses. Both historical LFM runs used Harbor, including the OpenCode-only run. This tutorial's `opencode` mode uses native OpenCode. The curves are historical context, not results from the new non-thinking pilots. + +Training reward uses a trailing 50-update mean. Evaluation is unsmoothed; hollow markers indicate incomplete coverage. Tool-call savings compare pairs solved by both baseline and checkpoint. See [the data source and final scores](RESULTS.md#historical-article-curves), or explore the [Trackio dashboard](https://huggingface.co/spaces/FineEnvs/data-agent-training-comparison-trackio). + +The full experiment remains on the [archived branch](https://github.com/adithya-s-k/FineEnvs/tree/archive/data-agent-experiments-20260930/04-data-agent). Corrected Harbor graders and dataset revisions are pinned here; [VALIDATION.md](VALIDATION.md) records what has been checked for this recipe. diff --git a/05-multi-harness-rl/REPRODUCE.md b/05-multi-harness-rl/REPRODUCE.md index c4e5808a..818c268d 100644 --- a/05-multi-harness-rl/REPRODUCE.md +++ b/05-multi-harness-rl/REPRODUCE.md @@ -1,4 +1,4 @@ -# Run SmolDataEnv RL +# Run SmolDataEnvs RL Check out the example branch: diff --git a/05-multi-harness-rl/RESULTS.md b/05-multi-harness-rl/RESULTS.md index fb5e246c..dab1102f 100644 --- a/05-multi-harness-rl/RESULTS.md +++ b/05-multi-harness-rl/RESULTS.md @@ -56,3 +56,26 @@ The [historical Trackio Space](https://huggingface.co/spaces/FineEnvs/data-agent - Treat Qwen HF execution and hardware combinations not tested here as needing their own smoke. Detailed pins, failure diagnoses, test counts and original smoke evidence are in [VALIDATION.md](VALIDATION.md). + +## Historical article curves + +The [multi-harness RL article](https://huggingface.co/spaces/AdithyaSK/multi-harness-rl) covers the earlier LFM and Qwen experiments on [SmolDataEnvs](../04-smoldataenvs/). The README figure shows its two LFM runs, both trained with correctness plus tool efficiency and evaluated on 250 tasks across four harnesses. + +| Historical LFM run | Step-1,000 pass@1 | Graded pairs | Tool-call savings on matched successes | +|---|---:|---:|---:| +| Harbor, OpenCode only | 52.3% | 1,000/1,000 | 11.39% | +| Harbor, multi-harness | 54.2% | 1,000/1,000 | 31.07% | + +The recorded baseline is 421/998 graded pairs (42.18%); two pairs were ungraded. Some early checkpoints also have incomplete coverage, marked with hollow points in the figure. These are observed pass@1 scores, not confidence intervals. The two final scores alone do not establish a statistically significant difference between runs. + +Both historical runs used Harbor. The current tutorial instead uses native OpenCode for its `opencode` mode and explicitly non-thinking serving for both models. These curves do not validate the new pilot configuration. The article also includes Qwen results, whose historical reward was correctness only. + +The figure uses a [pinned copy of the article data](https://huggingface.co/spaces/AdithyaSK/multi-harness-rl/blob/a1e03a6c7d8fd0d51d0cf7361ba51a469cc6defc/app/src/content/assets/data/training-results.json). The [local excerpt](assets/historical-lfm-curves.json) keeps the plotted values, evaluation coverage, source URL and source SHA-256. The [Trackio dashboard](https://huggingface.co/spaces/FineEnvs/data-agent-training-comparison-trackio) provides the historical run charts. + +To redraw the figure from this folder, without downloading data or starting a job: + +```bash +uv run --no-project --with matplotlib==3.11.1 python assets/plot_history.py +``` + +Training reward is a trailing 50-update mean, with raw values shown faintly. Evaluation is unsmoothed. Tool savings use task/harness pairs solved by both baseline and checkpoint; positive values mean fewer calls. diff --git a/05-multi-harness-rl/assets/historical-lfm-curves.json b/05-multi-harness-rl/assets/historical-lfm-curves.json new file mode 100644 index 00000000..d1a376e1 --- /dev/null +++ b/05-multi-harness-rl/assets/historical-lfm-curves.json @@ -0,0 +1 @@ +{"source_url":"https://huggingface.co/spaces/AdithyaSK/multi-harness-rl/blob/a1e03a6c7d8fd0d51d0cf7361ba51a469cc6defc/app/src/content/assets/data/training-results.json","source_sha256":"50f7a07f51c1feaffaeb4d27fba5e64698a93d06fdedec0a332c8aa7e057f749","source_note":"2026-09-24 run audit: data/evaluations.csv, lfm_evaluation_all_metrics.csv, lfm_eval_tokens.json, lfm_eval_efficiency.json","runs":{"lfm_harbor_opencode":{"label":"Harbor OpenCode only","train":{"step":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000],"reward":[0.0,0.0,0.662,0.0,0.5376,0.2679,0.0,0.3604,1.0811,0.5396,0.8005,0.0,0.0,0.0,0.0,0.8056,0.6726,0.802,0.5357,0.0,0.0,0.43,0.8002,0.8063,0.0,0.0,0.1339,0.797,1.0627,0.0,0.0,0.0,0.5344,1.0833,1.0812,0.0,0.0,0.8062,0.5395,1.0802,0.4325,1.072,0.86,0.0,0.0,0.0,0.0,0.2708,0.2656,0.4018,0.2697,1.0824,0.5354,1.0791,0.8114,1.0802,0.65,0.8082,1.0811,0.8082,0.5385,0.0,0.2708,0.5395,0.2708,0.0,0.0,0.2663,0.2656,0.3533,0.0,0.0,0.2143,1.0771,1.0679,1.0687,0.7966,0.8035,0.8082,0.8072,0.0,0.2708,0.2621,0.7205,0.8533,0.43,0.0,0.2136,0.0,0.2697,0.0,0.0,0.0,0.0,0.5395,0.8065,0.0,0.0,0.4294,0.5351,0.3963,0.8077,1.0786,1.0811,0.7702,0.9273,0.8613,0.2679,0.2656,0.0,0.0,0.2644,0.0,0.0,0.4002,0.795,0.0,0.267,0.5395,0.2697,0.0,0.8125,1.0822,0.9397,0.538,0.5344,0.8054,1.0745,1.0671,0.8022,0.5376,0.8072,0.8037,0.0,0.0,0.2697,0.0,0.0,0.6717,0.2708,0.2679,0.8474,0.7952,0.267,0.5313,0.0,0.0,0.0,0.0,0.0,0.8088,1.0791,0.5375,0.5375,0.5395,0.2697,0.0,0.8087,1.08,0.5387,0.0,0.4233,0.641,0.4261,0.0,0.2679,0.4031,0.8616,0.5385,0.8026,0.5385,0.2656,0.5417,0.8114,0.8093,1.0741,1.0728,0.6381,1.0626,1.0691,0.5342,0.0,0.0,0.0,0.0,1.0734,1.0757,1.0767,0.8617,0.7222,0.0,0.5395,1.0798,0.6734,0.2677,1.0724,1.0742,1.0753,0.5395,0.2679,0.5385,1.0748,0.8114,0.0,0.0,0.2697,0.805,1.0777,0.6755,0.0,0.1349,0.2697,0.0,0.0,0.0,0.5342,0.2687,0.0,0.0,0.0,0.0,0.0,0.0,0.5385,0.2708,1.0811,0.8125,0.0,0.5315,0.8045,0.1344,0.8046,0.5395,0.0,0.0,0.65,0.5417,0.0,1.0774,1.0761,0.7193,0.0,0.0,0.6411,0.2697,0.0,0.2167,1.0792,0.9463,1.08,1.0711,1.0703,1.0804,0.8084,1.0791,0.4012,0.5312,0.803,0.8093,0.8054,0.2679,0.0,0.2697,1.0811,0.5385,0.0,0.1349,0.6754,0.5406,0.5396,0.0,0.2697,1.0702,1.0772,0.213,0.4046,0.8077,0.0,0.2644,0.0,0.0,0.0,0.0,0.0,0.0,0.5375,0.267,0.8082,0.5385,0.267,0.8072,1.08,0.2697,0.0,0.2687,0.2687,0.0,0.0,0.0,0.0,0.5417,0.0,0.8632,1.0781,0.0,0.0,0.6411,0.2708,0.8105,0.8103,1.0822,1.0822,0.0,0.0,1.0782,1.0789,0.807,0.5396,0.8092,1.0833,1.0811,0.9468,0.8087,1.0771,1.0683,0.5375,0.863,0.2721,0.2708,0.0,0.0,0.5395,0.0,0.0,0.8042,0.8125,0.2708,0.0,0.6698,0.1344,0.5376,0.0,0.5291,0.2663,0.2687,0.0,0.1514,0.267,0.2154,0.9458,1.0802,0.5406,0.6743,0.8114,0.0,0.5289,0.2103,0.2656,0.0,1.0546,0.7017,0.0,0.0,0.0,0.2158,1.0771,0.8568,1.0789,0.5385,0.2697,0.0,0.0,0.2167,0.8115,0.0,0.1344,0.9393,1.0702,0.7962,1.0644,1.0789,1.0751,1.0825,1.0802,0.7176,0.4293,0.6755,1.0789,1.0802,1.0816,1.0667,1.0811,1.0796,1.0811,1.08,1.0822,1.0709,0.9319,0.8125,0.8114,0.8054,0.8045,0.0,1.0781,1.0822,1.0822,0.8103,0.6474,0.2687,1.0791,1.077,0.8065,0.5366,0.5395,0.2697,0.5376,0.6444,1.0814,0.0,0.0,1.0806,0.0,0.5401,1.0751,0.5385,0.267,0.6765,1.0811,0.9345,0.0,0.0,1.0822,1.0781,0.2697,1.0833,0.8125,0.2708,0.0,0.5417,0.5406,1.0833,1.0822,1.0833,1.0833,0.9383,0.5375,0.8092,0.5417,0.8104,1.0811,1.0833,0.4261,0.0,0.0,0.0,0.0,1.0756,0.8125,1.0812,1.0822,0.5406,0.5417,0.2708,0.0,0.0,0.0,0.0,0.8037,1.0734,0.8114,1.0802,0.5376,1.0804,0.902,0.2687,1.0737,1.0793,1.0811,0.2708,0.0,0.0,0.0,0.8104,1.0833,0.8005,0.0,0.0,0.2143,0.0,0.2097,0.0,0.0,0.0,0.4052,0.619,0.5357,0.6388,0.0,0.6419,1.0675,0.2708,0.2708,1.0774,0.8114,1.0825,1.0826,0.8126,0.8125,0.0,0.0,0.7222,1.081,1.08,1.0789,0.0,0.0,0.5357,0.2679,0.5342,0.8082,1.0712,1.078,1.0742,1.0811,0.5366,0.8037,0.0,0.5312,1.0757,0.5332,0.2687,0.0,0.0,0.8093,0.2692,0.8125,1.0833,1.0833,1.0833,0.1354,0.2167,0.8063,0.8125,1.0833,1.0725,0.5366,0.8026,0.8635,0.0,0.2708,0.5396,1.0772,1.0833,1.083,0.6735,1.0812,1.0801,1.0751,0.5376,0.4286,0.8055,0.5375,0.4301,1.0741,0.8037,0.0,0.0,0.2708,0.0,0.2708,0.2708,0.5406,0.0,0.2708,1.0825,1.0833,1.0791,0.5406,0.0,0.2143,0.8019,0.8038,0.0,0.2687,0.5313,0.2663,0.8095,0.4325,0.0,0.0,0.0,0.0,0.7991,0.1806,0.8125,0.0,0.6474,0.3548,0.6474,0.0,0.267,1.0687,1.0613,0.7938,0.4286,0.43,0.5387,0.8065,0.0,0.5358,0.2663,0.5341,0.5406,0.8076,0.8125,0.2697,0.0,0.5406,1.0822,1.0752,1.0777,0.5385,0.0,0.267,0.0,0.5395,0.2679,0.0,0.0,0.0,0.0,0.2687,1.0781,0.0,0.0,0.0,0.2708,0.5375,0.8092,0.5417,1.0833,1.0822,1.0811,0.5406,0.0,0.0,0.0,0.0,0.2663,0.5313,0.8063,0.4333,0.8092,0.8114,1.0833,1.0833,1.0833,1.0833,1.0833,0.9457,1.0806,1.0811,1.0812,1.0811,0.5417,0.2656,0.0,0.0,0.0,0.2708,0.6429,0.8095,1.0833,1.0806,0.5406,0.0,0.65,1.0833,0.5396,0.8062,0.0,0.2679,0.2697,0.5323,0.267,0.0,0.0,0.2167,0.8114,0.5406,0.0,0.5395,1.0782,0.5385,0.2697,0.2708,0.5417,1.0794,1.0744,0.6482,1.0833,0.8125,1.0833,0.5417,0.0,0.5429,0.0,0.4326,0.6397,1.0745,1.0833,1.0758,1.0723,0.5385,1.0755,0.8982,1.0732,0.9891,0.8037,0.5362,0.2629,0.8082,1.0811,0.4325,0.0,0.0,0.0,0.0,0.0,0.5396,1.07,0.5366,0.8114,1.0833,0.8093,0.4301,0.6771,0.1354,0.0,0.0,0.0,0.2697,0.0,0.5406,0.5385,0.8058,0.8125,1.0833,1.0833,1.0781,0.5352,1.0741,1.0833,0.5417,0.0,0.2125,0.5366,0.2687,1.08,0.5395,0.536,0.0,0.0,0.5406,1.0791,0.7553,0.2697,0.8059,0.5406,1.08,0.6452,0.8055,0.2687,0.2708,0.5395,1.0789,0.8632,1.078,0.8114,1.0822,0.5417,0.0,1.08,1.0807,1.0833,1.0833,0.9385,0.3584,0.2697,1.0811,0.5406,0.0,0.2687,1.0774,0.6476,1.0714,0.8082,0.0,0.0,0.5417,0.5417,1.0735,0.9393,0.2703,0.2687,0.2687,1.0833,1.0792,0.0,0.0,0.0,0.532,1.0717,0.8076,0.2708,0.0,0.8093,1.0811,0.0,0.0,0.8125,1.0833,0.3611,0.0,0.4308,0.8114,1.076,1.0817,1.0797,1.0821,1.0816,1.0833,0.1354,0.6419,0.8014,0.0,0.0,0.8098,0.1354,0.0,0.5406,0.8114,1.0763,1.0751,0.8114,1.0811,0.6467,0.8072,0.6716,0.53,1.062,0.2708,1.0827,0.2629,0.8558,0.1798,0.2167,0.267,0.2692,0.2697,0.0,0.0,0.0,0.5357,0.2679,0.7875,1.0822,1.0833,1.0822,1.0731,0.8082,0.0,0.0,0.5406,0.8125,1.0811,1.0833,0.0,0.0,0.801,0.802,0.1344,0.7208,1.0822,1.08,0.0,0.0,0.1332,0.2708,0.1806,0.8125,1.0833,1.0833,1.0833,0.8634,1.0817,0.6401,0.267,0.0,0.807,0.8093,1.0807,0.8667,1.0812,0.4301,0.0,0.2687,0.8032,0.8093,0.5406,0.5344,0.4308,1.0802,1.0736,1.0781,0.2708,0.0,0.0,0.4333,1.0833,0.9028,1.0833,0.5406,0.5406,0.8103,0.0,0.267,0.2656,0.0,0.5396,1.0793,0.8022,0.2708,0.5385,0.0,0.0,0.0,0.8073,1.0833,1.0796,1.073,0.0,1.0833,1.0822,0.8985,0.8039,0.536,0.5333,1.0745,0.8093,0.8082,0.8093,0.8088,0.9453,0.8028,1.0729,0.2687,0.0,0.5417,0.5417,0.5417,1.0833,0.2708,0.0,0.2708,0.0,0.0,0.0,0.0,0.9431,0.9371,0.2691,0.0,0.5417,0.8125,0.5406,0.8114,0.2708,0.0,0.8072,0.8632,0.2697,0.2697,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.4333,0.5387,0.6409,0.2125,0.0,0.0,0.0,0.804,0.0,0.2679]},"eval":[{"step":0,"overall":42.18,"correct":421,"graded":998,"complete":false},{"step":100,"overall":47.97,"correct":473,"graded":986,"complete":false},{"step":200,"overall":51.17,"correct":501,"graded":979,"complete":false},{"step":300,"overall":50.15,"correct":496,"graded":989,"complete":false},{"step":400,"overall":50.65,"correct":506,"graded":999,"complete":false},{"step":500,"overall":50.1,"correct":501,"graded":1000,"complete":true},{"step":600,"overall":51.8,"correct":518,"graded":1000,"complete":true},{"step":700,"overall":50.8,"correct":508,"graded":1000,"complete":true},{"step":800,"overall":51.2,"correct":512,"graded":1000,"complete":true},{"step":900,"overall":52.4,"correct":524,"graded":1000,"complete":true},{"step":1000,"overall":52.3,"correct":523,"graded":1000,"complete":true}],"efficiency":[{"step":0,"tool_savings":-0.0},{"step":100,"tool_savings":3.74},{"step":200,"tool_savings":3.26},{"step":300,"tool_savings":6.24},{"step":400,"tool_savings":5.91},{"step":500,"tool_savings":7.33},{"step":600,"tool_savings":12.48},{"step":700,"tool_savings":10.32},{"step":800,"tool_savings":12.97},{"step":900,"tool_savings":13.19},{"step":1000,"tool_savings":11.39}]},"lfm_harbor_multi":{"label":"Harbor multi-harness","train":{"step":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000],"reward":[0.0,0.0,0.0889,0.5513,0.3526,0.8329,0.9746,0.5831,0.1765,0.0,0.7613,0.163,0.1339,0.9289,0.0,0.0,0.0,0.9342,0.7246,0.1155,0.0,0.0182,0.5307,0.2656,0.2663,0.0,0.6655,0.9355,0.0,0.9836,0.235,0.0,0.0,0.4265,0.1902,0.0,0.1366,0.1937,0.0,0.0,0.0,0.0,0.9359,0.463,0.9293,0.8042,0.8083,0.5355,0.9311,0.4446,0.2634,0.9347,0.267,0.0,0.1984,0.022,0.4251,0.0,0.0,0.6676,1.0636,0.8048,1.0741,0.5417,0.8103,0.0,0.0,0.4016,1.0696,0.3473,0.5342,0.0,0.0,0.0,0.3982,0.3999,0.3978,0.2639,0.3956,0.7654,1.0715,1.0676,0.5385,0.9415,0.2669,0.0,0.0,0.0,0.0353,0.0,0.25,0.3417,0.0,0.5238,0.5345,0.4006,0.2158,1.0833,1.0752,0.5362,0.6694,0.2687,1.0646,0.88,0.2644,0.2687,0.0,0.0,0.0,0.4917,0.4019,0.0,0.2107,0.2679,0.5101,0.0,0.0,0.0,0.0,0.5385,1.0752,0.4862,0.2744,0.0,0.0,0.0,0.9323,1.0648,1.0691,0.0,0.0,0.5323,0.1332,1.0661,0.8864,0.3957,0.4062,0.8093,0.5368,1.0722,1.0527,1.0626,1.0608,0.6071,0.0,0.0,0.1339,0.6162,1.0671,0.5757,1.0778,1.025,0.4691,0.1923,0.0,1.0607,0.8055,1.0707,0.7871,0.2652,0.7563,0.7864,0.0,0.0,0.0443,0.9368,0.6754,0.2687,0.6686,0.0,0.0,0.0,0.0211,0.1335,0.0,0.0,0.0,0.0,0.0,0.0,0.0791,0.5349,0.0,0.9275,0.2663,0.3969,0.1918,0.0,0.1784,0.0889,0.2374,0.5798,0.8991,0.0345,0.0,0.0,0.0,1.0733,1.0711,0.4556,0.8728,0.4568,1.0669,1.0678,0.6418,1.0741,1.0675,0.0,0.0536,0.2117,0.0,0.0,0.674,0.2687,0.3995,0.9318,0.265,0.2136,0.6679,1.0735,0.0,0.0,0.0,0.0,0.0,0.8312,0.224,0.7568,1.0698,1.0719,0.1962,0.0,0.2346,1.0246,0.0,0.5276,0.1339,0.2657,0.5376,1.0791,0.672,0.1344,0.0,0.0,0.4562,0.0,0.5402,0.8103,0.0,0.0,0.6707,0.8495,0.2653,1.0812,0.8125,1.0707,1.0695,0.2687,0.5395,0.0,0.7554,0.0,0.0,0.1328,0.0,0.0,1.073,0.8162,0.1891,0.0,0.0,0.2697,0.0,0.2629,0.0,0.0742,0.0,0.0,0.265,0.5839,0.0,0.2378,1.0756,0.8092,1.074,0.6735,0.1325,0.0357,0.0,0.0656,0.5308,1.0704,0.2125,0.2679,0.0,0.0,0.3644,0.188,1.0733,0.0,0.0,0.1322,0.8032,0.2162,0.0,0.795,1.0627,1.0656,1.0746,0.6735,0.1339,0.9935,1.0785,1.0761,0.8103,0.6725,0.943,0.8066,0.5385,0.8037,0.1069,0.0,0.279,0.0,0.0,0.3533,0.6844,1.0654,1.0776,1.078,0.9354,0.9382,1.0802,0.9379,0.3999,0.0,0.72,0.2663,0.0,1.0754,0.0,1.0775,0.0,0.2675,1.0771,1.0695,0.0,0.0,0.0,1.0624,0.5741,0.7486,0.2889,1.08,0.8074,0.2693,0.0,0.0,0.5321,1.0745,1.0779,1.0742,0.9377,1.0649,0.3395,1.0614,0.4165,0.9372,0.3506,0.1322,0.1325,0.8308,0.8987,1.0586,1.078,0.8015,0.1332,0.1322,0.5294,0.9277,0.9343,1.0692,1.0688,1.0576,1.0692,1.0825,0.8635,0.0,0.0,0.2639,0.5333,0.8092,1.0766,0.8092,0.1344,0.0,0.0,0.0,0.0,0.0,0.0,0.5406,0.0,0.0,0.4031,0.2136,0.5755,0.0,0.0,0.0,0.1939,1.0634,0.5317,0.5417,0.2167,0.0,0.323,0.7608,0.3215,0.0,0.43,0.476,0.707,0.8,0.2122,0.7973,0.5334,1.0622,0.9376,0.3988,1.0749,0.2571,0.0,0.1332,0.0,0.0,0.9355,0.4012,0.8926,1.0691,1.0744,0.8763,0.4018,0.0,0.8041,0.9381,0.2653,0.0896,0.5782,0.6693,0.769,1.0738,1.0743,0.8049,0.5349,0.5366,0.428,0.2136,0.7998,0.9378,1.053,1.0664,0.8958,0.0,0.0,0.1335,0.5407,1.076,0.5374,0.0,0.3571,0.8613,0.25,0.0,0.0203,0.8019,0.4042,0.0,0.0,0.0,0.0765,0.5366,1.0767,0.9108,0.7719,0.3561,1.0618,0.0,0.0,1.0608,1.0543,0.6719,0.2158,1.073,0.0,0.0,0.5309,0.8037,0.1531,0.0,0.0,0.5174,0.0,0.0,0.8059,1.0703,1.0735,1.074,0.8586,0.1798,0.2685,0.8063,0.5342,0.1514,0.0,0.2397,0.0,0.1654,1.0767,0.9877,0.0,0.215,0.3552,0.6696,1.0775,1.0759,0.0,0.0,0.0,0.2697,0.0,0.5332,0.6569,0.8072,1.0761,1.0711,1.0758,0.5417,0.2687,1.0822,0.716,1.0655,1.0776,1.0705,0.9307,0.8546,0.0,0.3068,0.2337,0.1011,0.258,1.0767,0.0,0.6614,1.0701,0.8125,1.0811,0.4042,0.4397,0.0,0.0,0.5368,0.0,0.2629,0.6692,0.6182,1.0713,0.0,0.0,0.9416,0.4029,0.5011,0.1704,1.0616,1.0695,1.0649,0.6743,0.0,0.6172,0.5406,0.1354,0.1325,0.9328,0.9299,0.2997,0.4286,0.9368,0.7996,0.2634,1.0665,0.5337,0.6669,1.0767,0.0371,0.0,0.0,0.0,0.0,0.1065,0.0,0.8123,0.3409,0.7441,0.2992,1.0772,0.0,0.0,0.0,0.1344,0.936,0.6579,0.0,0.0,0.6761,1.0811,1.0833,0.3331,0.0,0.5396,0.8072,1.0756,0.515,0.0,1.0671,0.0,0.0,0.0,0.0,0.5385,0.0,0.0921,0.1344,1.078,1.076,0.0,0.0,0.5319,0.0,0.1339,1.0694,1.067,0.8014,1.0791,1.0744,1.0695,1.0755,0.0,0.7906,1.0654,0.8243,1.0833,0.9422,0.301,0.0,1.0679,0.9417,0.0,0.0,0.0,0.0,1.0723,0.6745,0.2683,0.5278,0.0,0.0,1.0707,0.0,0.1332,0.7151,0.8104,0.1349,1.0776,0.1349,0.0,0.6227,0.0,0.0,0.6685,1.0653,0.5524,1.0698,1.0616,1.0677,0.0,0.1344,0.802,0.0,0.0,0.5363,1.078,0.997,1.0729,1.0704,0.9363,0.8017,0.1326,0.4122,0.0331,0.7977,0.4301,0.4316,0.67,0.4256,0.1637,0.0,0.0,0.2667,0.5314,0.806,0.6707,1.0791,1.0822,0.3072,1.061,0.9013,0.5345,0.9384,0.086,0.0,0.0411,0.1515,0.0,0.1786,0.8729,0.673,1.0119,0.3044,0.0341,0.2663,0.1199,0.0,0.4979,0.2708,0.6743,0.5406,1.0613,0.1322,0.6635,0.8068,0.6478,0.6056,0.6735,0.7982,0.8062,0.0,0.5387,0.8464,0.94,0.8095,0.2697,0.0,1.08,1.0746,0.8114,0.0,0.0,0.6714,1.0682,0.7491,0.5277,1.0706,0.2675,1.0754,0.886,0.2158,0.0,1.0683,1.0741,0.8034,0.0487,0.0,0.1469,0.466,0.3583,1.0746,1.078,1.0731,0.5385,1.0697,1.0706,0.2688,0.2481,0.6706,1.0746,0.3316,0.0,0.0561,0.4052,0.0,0.2679,0.0,0.9352,1.0765,1.0709,0.1339,0.8062,1.0705,0.8417,0.0,0.2656,0.267,0.0,0.0,0.0,0.0,0.6595,1.067,0.6655,0.0166,0.0,0.2857,1.0716,1.0578,0.6081,1.0675,0.2693,0.0,0.8112,0.2675,0.8125,0.5385,0.2708,0.2657,1.0501,0.8969,0.6239,0.864,0.464,1.0589,1.0607,1.0072,0.2675,0.4014,0.1339,0.8062,1.0705,0.8326,0.8011,0.1659,0.9458,1.0822,0.9388,0.8014,0.3829,0.0,0.0,0.7443,0.207,0.0,1.0595,0.2657,0.0356,0.0,0.1806,0.1349,0.0,0.1071,0.9771,1.0738,1.0771,1.0549,0.5695,1.053,0.0822,0.34,0.2668,0.803,0.5362,0.5349,0.6656,0.1325,0.1339,0.2683,0.9301,0.8033,0.0,0.0,0.0,0.4333,0.4001,1.0768,0.4891,0.0,0.6729,0.9378,0.938,0.5362,0.4027,1.0742,0.6694,0.9336,0.5746,0.4972,0.0,0.0,0.3993,0.0,0.0,0.103,0.1959,0.0,0.0,0.6676,0.2337,1.0602,0.7995,0.2675,0.2656,0.2043,0.0,0.0,0.5797,1.0757,1.0789,1.0753,1.0791,0.8056,0.0,0.2679,1.0745,0.9431,0.659,0.0,1.0044,1.0719,0.1783,0.2679,0.9423,1.0794,0.9019,1.0833,0.2708,0.0,0.8082,0.2697,1.0761,0.5358,1.0786,0.1354,0.0,0.1433,0.8015,0.0,0.0,1.0307,0.8707,1.0731,0.8894,0.9757,0.3949,0.0,0.0,0.2167,0.7222,1.0798,0.8092,0.0,0.213,0.5327,0.0376,0.0,0.0,1.0743,1.0737,1.0686,0.3328,0.2143,0.2885,0.6418,1.073,0.8041,0.0,0.0,0.8483,0.1349,0.8032,0.2708,0.0,1.0806,0.0,0.0,0.0,0.0,0.0,0.0,1.0737,1.0705,1.0648,1.0738,1.0748,0.0,0.0,1.0822,0.8095,0.4904,1.078,1.0767,0.8464,0.0]},"eval":[{"step":0,"overall":42.18,"correct":421,"graded":998,"complete":false},{"step":100,"overall":47.06,"correct":465,"graded":988,"complete":false},{"step":200,"overall":52.63,"correct":520,"graded":988,"complete":false},{"step":300,"overall":50.7,"correct":507,"graded":1000,"complete":true},{"step":400,"overall":51.0,"correct":510,"graded":1000,"complete":true},{"step":500,"overall":52.1,"correct":521,"graded":1000,"complete":true},{"step":600,"overall":50.7,"correct":507,"graded":1000,"complete":true},{"step":700,"overall":54.6,"correct":546,"graded":1000,"complete":true},{"step":800,"overall":54.0,"correct":540,"graded":1000,"complete":true},{"step":850,"overall":54.5,"correct":545,"graded":1000,"complete":true},{"step":900,"overall":52.0,"correct":520,"graded":1000,"complete":true},{"step":1000,"overall":54.2,"correct":542,"graded":1000,"complete":true}],"efficiency":[{"step":0,"tool_savings":-0.0},{"step":100,"tool_savings":10.11},{"step":200,"tool_savings":15.01},{"step":300,"tool_savings":18.38},{"step":400,"tool_savings":23.02},{"step":500,"tool_savings":23.74},{"step":600,"tool_savings":23.71},{"step":700,"tool_savings":26.69},{"step":800,"tool_savings":24.17},{"step":850,"tool_savings":25.79},{"step":900,"tool_savings":28.56},{"step":1000,"tool_savings":31.07}]}}} diff --git a/05-multi-harness-rl/assets/historical-lfm-curves.svg b/05-multi-harness-rl/assets/historical-lfm-curves.svg new file mode 100644 index 00000000..a7afcb3e --- /dev/null +++ b/05-multi-harness-rl/assets/historical-lfm-curves.svg @@ -0,0 +1,4381 @@ + + + + + + + + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 200 + + + + + + + + + + + + + 400 + + + + + + + + + + + + + 600 + + + + + + + + + + + + + 800 + + + + + + + + + + + + + 1000 + + + + Optimizer step / checkpoint + + + + + + + + + + + + + + + + + 0.0 + + + + + + + + + + + + + 0.2 + + + + + + + + + + + + + 0.4 + + + + + + + + + + + + + 0.6 + + + + + + + + + + + + + 0.8 + + + + + + + + + + + + + 1.0 + + + + Correctness + efficiency bonus + + + + + + + + + + + + + + + + + + + + + + Training reward + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 200 + + + + + + + + + + + + + 400 + + + + + + + + + + + + + 600 + + + + + + + + + + + + + 800 + + + + + + + + + + + + + 1000 + + + + Optimizer step / checkpoint + + + + + + + + + + + + + + 35 + + + + + + + + + + + + + 40 + + + + + + + + + + + + + 45 + + + + + + + + + + + + + 50 + + + + + + + + + + + + + 55 + + + + + + + + + + + + + 60 + + + + Pass@1 on graded pairs (%) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 52.3% + + + 54.2% + + + Held-out answer quality + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 200 + + + + + + + + + + + + + 400 + + + + + + + + + + + + + 600 + + + + + + + + + + + + + 800 + + + + + + + + + + + + + 1000 + + + + Optimizer step / checkpoint + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 5 + + + + + + + + + + + + + 10 + + + + + + + + + + + + + 15 + + + + + + + + + + + + + 20 + + + + + + + + + + + + + 25 + + + + + + + + + + + + + 30 + + + + + + + + + + + + + 35 + + + + Savings on matched successes (%) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 11.4% + + + 31.1% + + + Tool-call savings + + + + LFM2.5-2.6B: earlier multi-harness RL results + + + Historical article runs | 250 test tasks x 4 harnesses | Pass@1 + + + Training: trailing 50-update mean; faint lines are raw rewards. Evaluation: measured checkpoints, no smoothing. + + + Hollow markers: incomplete coverage. Savings use pairs solved by baseline and checkpoint. Both runs used Harbor. + + + Source: multi-harness RL article, pinned snapshot a1e03a6. These are not the new non-thinking pilot results. + + + + + + + Harbor OpenCode only + + + + + + Harbor multi-harness + + + + + + + + + + + + + + + diff --git a/05-multi-harness-rl/assets/plot_history.py b/05-multi-harness-rl/assets/plot_history.py new file mode 100644 index 00000000..8ddcff7b --- /dev/null +++ b/05-multi-harness-rl/assets/plot_history.py @@ -0,0 +1,65 @@ +"""Rebuild the article comparison from its pinned data excerpt.""" + +import json +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt + + +def main(): + root = Path(__file__).resolve().parent + data = json.loads((root / "historical-lfm-curves.json").read_text()) + plt.rcParams.update({ + "font.family": "DejaVu Sans", "font.size": 11, + "axes.spines.top": False, "axes.spines.right": False, + "axes.edgecolor": "#cbd5e1", "text.color": "#172033", + "axes.labelcolor": "#475569", "svg.fonttype": "none", + "svg.hashsalt": "smoldataenvs-history", + }) + fig, axes = plt.subplots(1, 3, figsize=(15, 5.8)) + fig.subplots_adjust(left=.065, right=.975, top=.69, bottom=.26, wspace=.30) + fig.text(.065, .94, "LFM2.5-2.6B: earlier multi-harness RL results", fontsize=21, weight="bold") + fig.text(.065, .885, "Historical article runs | 250 test tasks x 4 harnesses | Pass@1", color="#475569") + for (key, run), color in zip(data["runs"].items(), ["#2563eb", "#d97706"]): + steps, rewards = run["train"]["step"], run["train"]["reward"] + smooth = [sum(rewards[max(0, i - 49):i + 1]) / min(i + 1, 50) for i in range(len(rewards))] + axes[0].plot(steps, rewards, color=color, alpha=.08, linewidth=.6) + axes[0].plot(steps, smooth, color=color, linewidth=2, label=run["label"]) + points = run["eval"] + axes[1].plot([p["step"] for p in points], [p["overall"] for p in points], color=color, linewidth=2) + for p in points: + axes[1].plot(p["step"], p["overall"], "o", color=color, markerfacecolor=color if p["complete"] else "white", markersize=5) + p = points[-1] + axes[1].annotate(f'{p["overall"]:.1f}%', (p["step"], p["overall"]), xytext=(0, 10 if key.endswith("multi") else -18), textcoords="offset points", ha="right", color=color, weight="bold") + savings = run["efficiency"] + axes[2].plot([p["step"] for p in savings], [p["tool_savings"] for p in savings], color=color, linewidth=2) + coverage = {p["step"]: p["complete"] for p in points} + for p in savings: + axes[2].plot(p["step"], p["tool_savings"], "o", color=color, markerfacecolor=color if coverage[p["step"]] else "white", markersize=5) + p = savings[-1] + axes[2].annotate(f'{p["tool_savings"]:.1f}%', (p["step"], p["tool_savings"]), xytext=(0, 10), textcoords="offset points", ha="right", color=color, weight="bold") + for ax, title, ylabel in zip(axes, ["Training reward", "Held-out answer quality", "Tool-call savings"], ["Correctness + efficiency bonus", "Pass@1 on graded pairs (%)", "Savings on matched successes (%)"]): + ax.set_title(title, loc="left", weight="bold", pad=14) + ax.set_xlabel("Optimizer step / checkpoint") + ax.set_ylabel(ylabel) + ax.grid(alpha=.18) + ax.set_axisbelow(True) + ax.set_xlim(-20, 1030) + axes[0].set_ylim(0, 1.15) + axes[1].set_ylim(35, 60) + axes[2].set_ylim(-2, 36) + fig.legend(*axes[0].get_legend_handles_labels(), loc="upper left", bbox_to_anchor=(.06, .845), ncol=2, frameon=False) + fig.text(.065, .15, "Training: trailing 50-update mean; faint lines are raw rewards. Evaluation: measured checkpoints, no smoothing.", fontsize=10) + fig.text(.065, .105, "Hollow markers: incomplete coverage. Savings use pairs solved by baseline and checkpoint. Both runs used Harbor.", fontsize=10) + fig.text(.065, .06, "Source: multi-harness RL article, pinned snapshot a1e03a6. These are not the new non-thinking pilot results.", fontsize=10, color="#64748b") + output = root / "historical-lfm-curves.svg" + fig.savefig(output, metadata={"Date": None}) + output.write_text("\n".join(line.rstrip() for line in output.read_text().splitlines()) + "\n") + plt.close(fig) + + +if __name__ == "__main__": + main() diff --git a/05-multi-harness-rl/project.yaml b/05-multi-harness-rl/project.yaml index 6491067d..5575ee6d 100644 --- a/05-multi-harness-rl/project.yaml +++ b/05-multi-harness-rl/project.yaml @@ -1,6 +1,6 @@ name: multi-harness-rl -title: "SmolDataEnv: whitebox and multi-harness RL" -tagline: "One task split and reward across SETA, native OpenCode and Harbor; HF Jobs and Slurm recipes." +title: "SmolDataEnvs: multi-harness RL" +tagline: "Train on SmolDataEnvs with SETA, native OpenCode or Harbor, using HF Jobs or Slurm." order: 5 status: wip hub: diff --git a/README.md b/README.md index da002e61..4e8cb8ee 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,8 @@ results and README, plus the Hub repos it owns. They read in order but stand alo | **01** | **[LaTeX OCR](./01-latex-ocr/)** | Train Qwen3-VL-2B to read math images into LaTeX, with a verifiable reward. | 1 | 1 | 1 | ✅ stable | | **02** | **[Watercolour](./02-watercolour/)** | Train Qwen3.5-35B-A3B to paint watercolours by writing p5.brush sketches, rewarded by an aesthetic preference model. | 1 | 1 | 0 | ✅ trained | | **03** | **[GeoGuesser](./03-geoguesser/)** | Drop a VLM at a random street corner on Earth and score it on kilometres of error. | 1 | 1 | 1 | ✅ stable | -| **05** | **[SmolDataEnv: whitebox and multi-harness RL](./05-multi-harness-rl/)** | One task split and reward across SETA, native OpenCode and Harbor; HF Jobs and Slurm recipes. | 3 | 1 | 0 | 🚧 wip | +| **04** | **[SmolDataEnvs](./04-smoldataenvs/)** | Data-analysis tasks, verified SFT traces and a single-turn GRPO tutorial on HF Jobs. | 1 | 1 | 0 | 🚧 wip | +| **05** | **[SmolDataEnvs: multi-harness RL](./05-multi-harness-rl/)** | Train on SmolDataEnvs with SETA, native OpenCode or Harbor, using HF Jobs or Slurm. | 3 | 1 | 0 | 🚧 wip | Generated from each project's `project.yaml` by `tools/build_index.py`. Adding a project means From 1cfce10442ab07e4b8293a0a232f432dd9518609 Mon Sep 17 00:00:00 2001 From: adithya-s-k Date: Wed, 30 Sep 2026 17:45:35 +0000 Subject: [PATCH 15/48] Migrate RL recipe to merged OpenEnv training contract --- 05-multi-harness-rl/README.md | 4 +- 05-multi-harness-rl/RESULTS.md | 22 ++-- 05-multi-harness-rl/VALIDATION.md | 17 ++- 05-multi-harness-rl/configs/runtime-lock.json | 4 +- 05-multi-harness-rl/eval/evaluate.py | 3 +- 05-multi-harness-rl/recipe.py | 11 ++ 05-multi-harness-rl/requirements.lock | 2 +- 05-multi-harness-rl/requirements.txt | 2 +- 05-multi-harness-rl/runtime/bootstrap.py | 4 + 05-multi-harness-rl/runtime/job.sh | 2 +- 05-multi-harness-rl/runtime/native_capture.py | 22 ++++ 05-multi-harness-rl/runtime/patches.py | 26 +++++ 05-multi-harness-rl/tests/test_contracts.py | 24 ++++ 05-multi-harness-rl/tests/test_runtime.py | 106 +++++++++++++++++- 05-multi-harness-rl/train/adapters.py | 53 +++++---- 05-multi-harness-rl/train/blackbox.py | 10 +- 05-multi-harness-rl/train/whitebox.py | 4 +- 17 files changed, 262 insertions(+), 54 deletions(-) create mode 100644 05-multi-harness-rl/runtime/native_capture.py diff --git a/05-multi-harness-rl/README.md b/05-multi-harness-rl/README.md index 8b5f9e2b..a831a7a2 100644 --- a/05-multi-harness-rl/README.md +++ b/05-multi-harness-rl/README.md @@ -31,6 +31,8 @@ Use `--model lfm` for **LiquidAI/LFM2.5-2.6B**, or `--model qwen` for **Qwen/Qwe TiTO means *tokens in, tokens out*: training uses the inference engine's actual token IDs and logprobs rather than reconstructing them from text. A rewritten conversation can produce several training rows from one rollout. A task, rollout, training row and optimizer step are therefore different units. +Both blackbox modes implement OpenEnv's `fetch_training_trace()` contract. The producer selects agent calls and supplies explicit masks; TRL receives the engine IDs unchanged. Partial masks and zero-masked agent turns survive. The worker supplies sampling before the factory starts a session. Native tool counts are carried as structured reward metadata, separately from token eligibility. + Synchronous GRPO waits for its batch of tool rollouts. AsyncGRPO collects rollouts while training proceeds, with a bounded amount of policy staleness. A group whose rollouts all have the same reward supplies no relative learning signal. Successful execution alone does not establish useful learning. ## The shared experiment @@ -57,7 +59,7 @@ reward = correctness × (1 + 0.1 × 15 / (15 + tool_calls)) For example, a correct answer using 15 calls earns `1.05`; an incorrect answer earns `0`. Zero or unverified tool counts receive no bonus. Infrastructure failures remain ungraded. Evaluation reports correctness separately from shaped reward and tool/token usage. -The full schedule contains two passes and stops at 1,000 updates or schedule exhaustion, whichever comes first. The 100-step pilot retains the same schedule and training settings but stops at 100 updates. Neither setting guarantees that every scheduled task is consumed. The committed-group audit records what async training actually used. +The full schedule contains two passes and stops at 1,000 updates or schedule exhaustion, whichever comes first. The 100-step pilot retains the same schedule and training settings but stops at 100 updates. Neither setting guarantees that every scheduled task is consumed. The committed-group audit records what async training actually used. For whitebox, 16 rollouts per update divided by 8 rollouts per task means 2 unique prompts per update. Its 2,000 scheduled prompts therefore allow 1,000 updates; custom step limits are capped at schedule exhaustion. ## Start small, then scale diff --git a/05-multi-harness-rl/RESULTS.md b/05-multi-harness-rl/RESULTS.md index dab1102f..d7470839 100644 --- a/05-multi-harness-rl/RESULTS.md +++ b/05-multi-harness-rl/RESULTS.md @@ -1,6 +1,6 @@ # Current results -These are the checks and pilot results for the pinned, non-thinking recipe. The 100-step comparisons are still in progress. Earlier experiment scores are kept on the [archived branch](https://github.com/adithya-s-k/FineEnvs/tree/archive/data-agent-experiments-20260930/04-data-agent). +These are the checks and pilot results for the non-thinking recipe. Two blackbox pilots completed 100 updates and evaluation; the whitebox pilot stopped at step 31 after an HTTP read timeout. Those pilots used the previous runtime pins. The merged-contract smoke qualification is recorded separately in [VALIDATION.md](VALIDATION.md). Earlier experiment scores are kept on the [archived branch](https://github.com/adithya-s-k/FineEnvs/tree/archive/data-agent-experiments-20260930/04-data-agent). ## Completed checks @@ -15,17 +15,17 @@ The Harbor reload smoke solved 5/8 pairs on two medium tasks. This is a function ## LFM 100-step pilots -Snapshot: **2026-09-30 10:20 UTC**. Each pilot uses the same 25 held-out tasks (3 easy, 12 medium, 10 hard), baseline evaluation, 100 training updates, saves at 50/100 and final evaluation. Evaluation concurrency is 35. Blackbox evaluations have 100 task/harness pairs; whitebox has 25 native episodes. +Snapshot: **2026-09-30 17:45 UTC**, read from the saved bucket summaries. Each pilot uses the same 25 held-out tasks (3 easy, 12 medium, 10 hard), baseline evaluation, 100 training updates, saves at 50/100 and final evaluation. Evaluation concurrency is 35. Blackbox evaluations have 100 task/harness pairs; whitebox has 25 native episodes. -| Mode | Baseline pass@1 | Optimizer step (metrics) | Learning signal observed? | -|---|---|---:|---| -| Native OpenCode | 24/100 = 24% | 10 | Yes: step 9 reward 0.126 and gradient norm 3.50 | -| Harbor multi-harness | 17/100 = 17% | 8 | Yes: step 8 reward 0.598 and gradient norm 2.70 | -| Whitebox SETA | 7/25 = 28% | 5 | No reward contrast in its first five updates | +| Mode | Baseline pass@1 | Checkpoint-100 pass@1 | Training status | +|---|---|---|---| +| Native OpenCode | 24/100 = 24% | 20/100 = 20% | Completed 100 updates | +| Harbor multi-harness | 17/100 = 17% | 32/100 = 32% | Completed 100 updates | +| Whitebox SETA | 7/25 = 28% | Not available | Stopped at step 31; HTTP read timeout | -All three jobs were running at this check. The whitebox save failure in its earlier smoke was traced to missing TRL package metadata and corrected before this pilot. The running jobs use immutable uploaded source snapshots; later tutorial edits do not modify them. +Scores come from complete `baseline/eval/summary.json` and `checkpoint-100/eval/summary.json` files. These are separate stochastic baseline samples at temperature 0.8, so the initial 24% versus 17% difference is not a training effect. Whitebox uses a different native evaluation interface. With only 25 tasks per harness, treat changes as pilot observations, not evidence of a reliable model ranking. -Baseline scores above come from complete `baseline/eval/summary.json` files downloaded from the bucket, not minute-by-minute progress counts. The two blackbox pilots independently sampled the same untrained model at temperature 0.8; these are separate stochastic samples, so their difference cannot be attributed to training. Whitebox uses a different native evaluation interface. No checkpoint-100 result exists yet, and these early updates do not establish a rising reward curve. +The jobs used immutable uploaded source snapshots; the new contract migration does not modify these results. The whitebox timeout remains a limitation of that earlier pilot, not a completed 100-step comparison. | Baseline evaluation harness | Native OpenCode pilot | Harbor pilot | |---|---:|---:| @@ -44,13 +44,13 @@ Our test and pilot outputs use bucket `FineEnvs/data-agent-daytona-artifacts`, u - `pilot-lfm-multi-harness-v2` - `pilot-lfm-whitebox-v1` -Each prefix contains `pilot.json`, `baseline/`, `train/`, `checkpoint-100/` and, after completion, `comparison.json`. Checkpoints contain model weights, optimizer and RNG state. Logs include local Trackio data, scalar metrics and rollout/audit evidence. Bucket access requires permission; job links may also require an HF login. No public trained-model artifact is claimed for these unfinished pilots. +Each prefix contains `pilot.json`, `baseline/`, `train/`, `checkpoint-100/` and, after completion, `comparison.json`. Checkpoints contain model weights, optimizer and RNG state. Logs include local Trackio data, scalar metrics and rollout/audit evidence. Bucket access requires permission; job links may also require an HF login. No public trained-model artifact is claimed for these pilots. The [historical Trackio Space](https://huggingface.co/spaces/FineEnvs/data-agent-training-comparison-trackio) does not automatically contain these new pilots. Their current config keeps Trackio offline in the bucket. Follow the HF job logs for live progress. The tutorial explains how to enable your own online Trackio Space. ## Remaining checks -- Finish the pilots and compare complete baseline/checkpoint-100 results. +- Complete a whitebox 100-step pilot with the updated runtime. The earlier pilot failed before checkpoint 50. - Confirm sustained learning signal, especially in native OpenCode and whitebox. - Test separate evaluation jobs reading newly saved bucket checkpoints before enabling a long-run watcher. The completed smoke tested reload within one job. - Treat Qwen HF execution and hardware combinations not tested here as needing their own smoke. diff --git a/05-multi-harness-rl/VALIDATION.md b/05-multi-harness-rl/VALIDATION.md index fe1ff1c7..ec9bca28 100644 --- a/05-multi-harness-rl/VALIDATION.md +++ b/05-multi-harness-rl/VALIDATION.md @@ -2,7 +2,18 @@ This page records what we tested for the pinned, non-thinking recipe. [RESULTS.md](RESULTS.md) has the pilot scores and progress; [REPRODUCE.md](REPRODUCE.md) has the commands. -## Automated checks +## Merged contract update (30 September) + +The recipe now pins merged [OpenEnv #1280](https://github.com/huggingface/OpenEnv/pull/1280) and the updated [TRL #6947](https://github.com/huggingface/trl/pull/6947). The previous GPU results below used the earlier pins and are retained as historical evidence. + +- 43 CPU checks cover the updated recipe, including native partial/zero masks, invalid captures, correctness plus efficiency through the typed TRL worker, and the whitebox update budget against TRL's sampler. +- The TRL integration passed 51 contract/reconciliation checks against the merged producer. +- Fresh-clone bootstrap and repeat-bootstrap handling are tested. User edits are preserved; only the recipe's known attention patch is removed before updating TRL. +- HF A100 training and checkpoint-reload smokes are running for all three modes. Results will be recorded here after completion. + +Execution status, typed execution limits and structured tool outcomes in OpenEnv remain follow-up API work from the TRL review. The recipe uses explicit per-session turn/time budgets and verified native tool counts; it does not claim the broader session API is complete. + +## Earlier automated checks - 29 tests passed with the downloaded runtime and prepared tasks. - A clean CPU-only checkout passed 25 tests. Four integration tests skip until the runtime and tasks are downloaded. @@ -61,7 +72,7 @@ The task lists contain 1,000 training tasks and 250 test tasks, with no task, no - Train revision: `4719635555de1666f374d847baebb500d368e493`. - Test revision: `b130595b579fc6026ad5762925a9a3b40e72f5bb`. -- OpenEnv: `4f4c85fb9038f43efc2f51858a27638277f16355`, including partial-mask validation. +- Current OpenEnv: `86a180ede21e044f7929b9a7783ad83aa67d83a3`. Earlier GPU qualification used `4f4c85fb9038f43efc2f51858a27638277f16355`. - Other source revisions and archive hashes: [runtime-lock.json](configs/runtime-lock.json). - Python dependencies: [requirements.lock](requirements.lock). @@ -69,6 +80,6 @@ Compared with the old staged task files, 302 selected training tasks now use the ## Still to check -The three LFM 100-step pilots are in progress. Checkpoint-100 scores and sustained learning remain unverified. HF same-job reload has passed, but a separate evaluation job loading newly saved bucket checkpoints still needs testing before relying on the long-run watcher. Qwen HF execution and other hardware combinations need their own smoke. +The earlier blackbox pilots completed: Harbor 17% to 32% and native OpenCode 24% to 20%, each over 100 held-out pairs. Whitebox stopped at step 31 after an HTTP read timeout. See [RESULTS.md](RESULTS.md); these results do not establish sustained learning or qualify the new pins. HF same-job reload has passed, but a separate evaluation job loading newly saved bucket checkpoints still needs testing before relying on the long-run watcher. Qwen HF execution and other hardware combinations need their own smoke. This PR contains the recipe, tests and evidence links. Large artifacts remain in the HF bucket or ignored local run directories. diff --git a/05-multi-harness-rl/configs/runtime-lock.json b/05-multi-harness-rl/configs/runtime-lock.json index 0d8f3db6..06d1dab4 100644 --- a/05-multi-harness-rl/configs/runtime-lock.json +++ b/05-multi-harness-rl/configs/runtime-lock.json @@ -52,10 +52,10 @@ }, "trl": { "repo": "adithya-s-k/trl", - "revision": "3cb8efdd8586f1eb1b4d4f9b346f9bdad7e86bb4" + "revision": "173f37e9dca9a8d7aecf75c9076f63dbfc853899" }, "openenv": { "repo": "huggingface/OpenEnv", - "revision": "4f4c85fb9038f43efc2f51858a27638277f16355" + "revision": "86a180ede21e044f7929b9a7783ad83aa67d83a3" } } diff --git a/05-multi-harness-rl/eval/evaluate.py b/05-multi-harness-rl/eval/evaluate.py index 921644f0..8bc9b0ff 100644 --- a/05-multi-harness-rl/eval/evaluate.py +++ b/05-multi-harness-rl/eval/evaluate.py @@ -110,7 +110,8 @@ def evaluate(cfg, data, server, vllm, *, limit=None, checkpoint="baseline", max_ write_json(identity_path, {"sha256": signature, **identity}) # Both blackbox policies use the same four Harbor evaluation harnesses. factory = Factory({**cfg, "mode": "multi-harness"}, data, server, vllm, groups, - Path(cfg["output"]) / "trials", split="test") if cfg["mode"] != "whitebox" else None + Path(cfg["output"]) / "trials", split="test", + sampling={k: cfg[k] for k in ("temperature", "top_p", "top_k")}) if cfg["mode"] != "whitebox" else None def one(item): group_id, group = item diff --git a/05-multi-harness-rl/recipe.py b/05-multi-harness-rl/recipe.py index 2f28ae68..877b80f6 100644 --- a/05-multi-harness-rl/recipe.py +++ b/05-multi-harness-rl/recipe.py @@ -63,6 +63,17 @@ def config(path=None, **overrides): return result +def whitebox_step_limit(cfg, task_count): + rollouts_per_update = cfg["batch_size"] * cfg["gradient_accumulation_steps"] + if rollouts_per_update % cfg["num_generations"]: + raise ValueError("Whitebox update must contain complete GRPO groups") + tasks_per_update = rollouts_per_update // cfg["num_generations"] + available = task_count // tasks_per_update + if not available: + raise ValueError("Whitebox schedule is smaller than one update") + return min(cfg["max_steps"], available) + + def task_rows(split): rows = json.loads((ROOT / "data" / f"{split}.json").read_text()) if len({r["name"] for r in rows}) != len(rows): diff --git a/05-multi-harness-rl/requirements.lock b/05-multi-harness-rl/requirements.lock index 75fec631..d534f663 100644 --- a/05-multi-harness-rl/requirements.lock +++ b/05-multi-harness-rl/requirements.lock @@ -98,7 +98,7 @@ httpx==0.28.1 httpx-sse==0.4.3 httpx-ws==0.9.0 httpx2==2.13.1 -huggingface-hub==1.24.0 +huggingface-hub==1.29.0 humming-kernels==0.1.10 hyperframe==6.1.0 idna==3.20 diff --git a/05-multi-harness-rl/requirements.txt b/05-multi-harness-rl/requirements.txt index 9248d1fc..46f7231f 100644 --- a/05-multi-harness-rl/requirements.txt +++ b/05-multi-harness-rl/requirements.txt @@ -3,7 +3,7 @@ torch==2.11.0 vllm==0.25.1 transformers==5.14.1 datasets==5.0.0 -huggingface_hub==1.24.0 +huggingface_hub==1.29.0 accelerate==1.14.0 bitsandbytes==0.49.2 kernels==0.15.2 diff --git a/05-multi-harness-rl/runtime/bootstrap.py b/05-multi-harness-rl/runtime/bootstrap.py index 30e3dde7..f8dd0d7b 100644 --- a/05-multi-harness-rl/runtime/bootstrap.py +++ b/05-multi-harness-rl/runtime/bootstrap.py @@ -52,6 +52,10 @@ def fetch(item): run("git", "-C", target, "cat-file", "-e", entry["revision"]) except subprocess.CalledProcessError: run("git", "-C", target, "fetch", "origin", entry["revision"]) + if name == "trl": + sys.path.insert(0, str(ROOT)) + from runtime.patches import restore_trl + restore_trl(target) run("git", "-C", target, "checkout", "--detach", entry["revision"]) packages = RUNTIME / "packages" packages.mkdir(exist_ok=True) diff --git a/05-multi-harness-rl/runtime/job.sh b/05-multi-harness-rl/runtime/job.sh index 023fadc6..070b0872 100644 --- a/05-multi-harness-rl/runtime/job.sh +++ b/05-multi-harness-rl/runtime/job.sh @@ -7,7 +7,7 @@ uv venv --python 3.12 .venv export PATH="$PWD/.venv/bin:$PATH" if [[ "$1" == "watch" ]]; then shift - uv pip install --python .venv/bin/python huggingface_hub==1.24.0 + uv pip install --python .venv/bin/python huggingface_hub==1.29.0 exec python3 eval/watch.py "$@" fi python3 runtime/bootstrap.py diff --git a/05-multi-harness-rl/runtime/native_capture.py b/05-multi-harness-rl/runtime/native_capture.py new file mode 100644 index 00000000..20456afc --- /dev/null +++ b/05-multi-harness-rl/runtime/native_capture.py @@ -0,0 +1,22 @@ +"""Typed producer boundary for the archived native OpenCode wire format.""" +from openenv.core.harness import TrainingTrace, TrainingTurn + + +def training_trace(result): + if result is None: + raise RuntimeError("No completed native rollout") + if result.rollout_type != "train": + raise ValueError("Native rollout has no training capture") + if any(f.startswith("[FATAL]") for f in result.metadata.get("capture_findings", [])): + raise ValueError("Native capture has fatal validation findings") + return TrainingTrace(turns=[TrainingTurn( + node_id=turn.capture_metadata["node_id"], + prompt_token_ids=turn.prompt_token_ids, + completion_token_ids=turn.completion_token_ids, + per_token_logps=turn.per_token_logps, + loss_mask=turn.loss_mask, + request={"messages": turn.request_messages, "tools": turn.request_tools}, + response={"choices": [{"message": {"role": "assistant", "content": turn.text, + "tool_calls": turn.tool_calls or None}, "finish_reason": turn.finish_reason}]}, + metadata=turn.capture_metadata, + ) for turn in result.turns]) diff --git a/05-multi-harness-rl/runtime/patches.py b/05-multi-harness-rl/runtime/patches.py index e55343f0..6226a6ca 100644 --- a/05-multi-harness-rl/runtime/patches.py +++ b/05-multi-harness-rl/runtime/patches.py @@ -7,6 +7,19 @@ def replace(text, old, new): return text.replace(old, new, 1) +def restore_trl(root): + import subprocess + relative = "trl/experimental/async_grpo/async_grpo_trainer.py" + path = root / relative + if not path.exists(): + return + original = subprocess.check_output(["git", "-C", str(root), "show", f"HEAD:{relative}"], text=True) + adapted = original.replace('attn_implementation="kernels-community/flash-attn3",', + 'attn_implementation=model_init_kwargs.pop("attn_implementation", "kernels-community/flash-attn3"),', 1) + if path.read_text() == adapted: + path.write_text(original) + + def apply_trl(root): path = root / "trl/experimental/async_grpo/async_grpo_trainer.py" old = 'attn_implementation="kernels-community/flash-attn3",' @@ -21,6 +34,19 @@ def apply(packages): path = packages / "data_agent_env/server/capture.py" path.write_text(replace(path.read_text(), "max_output_tokens=16384,", 'max_output_tokens=int(os.environ["OPENENV_MAX_OUTPUT_TOKENS"]),')) + text = path.read_text() + text = replace(text, "from openenv.core.harness.capture import CaptureServer, to_trace_entries", + "from openenv.core.harness.capture import CaptureServer\nfrom openenv.core.harness.capture.contract import to_training_trace") + text = replace(text, "turns_from_capture(to_trace_entries(session.graph, document))", + "turns_from_capture(to_training_trace(session.graph, document).to_trace_entries())") + path.write_text(text) + path = packages / "data_agent_env/harness.py" + text = replace(path.read_text(), " def fetch_proxy_trace(self) -> list[TraceEntry]:", + " def fetch_training_trace(self):\n" + " from runtime.native_capture import training_trace\n" + " return training_trace(self._result)\n\n" + " def fetch_proxy_trace(self) -> list[TraceEntry]:") + path.write_text(text) path = packages / "data_agent_env/server/rollout.py" text = replace(path.read_text(), 'opencode run --print-logs "$(cat {config.home}/workdir/task.md)"', 'opencode run --format json --print-logs "$(cat {config.home}/workdir/task.md)" > /tmp/native-events.jsonl') diff --git a/05-multi-harness-rl/tests/test_contracts.py b/05-multi-harness-rl/tests/test_contracts.py index 7eb081e0..d5686806 100644 --- a/05-multi-harness-rl/tests/test_contracts.py +++ b/05-multi-harness-rl/tests/test_contracts.py @@ -311,3 +311,27 @@ def test_hf_source_bundle_omits_local_state(tmp_path): paths = [str(p.relative_to(tmp_path)) for p in tmp_path.rglob("*")] assert "configs/runtime-lock.json" in paths assert not any(p.startswith((".env", ".runtime", "runs", "prepared")) for p in paths) + + +def test_runtime_patch_restore_handles_fresh_clone_and_preserves_edits(tmp_path): + import subprocess + from runtime.patches import apply_trl, restore_trl + root = tmp_path / "checkout" + root.mkdir() + restore_trl(root) # A --no-checkout clone has no worktree files yet. + path = root / "trl/experimental/async_grpo/async_grpo_trainer.py" + path.parent.mkdir(parents=True) + original = 'attn_implementation="kernels-community/flash-attn3",\n' + path.write_text(original) + subprocess.run(["git", "init", "-q", str(root)], check=True) + subprocess.run(["git", "-C", str(root), "add", "."], check=True) + subprocess.run(["git", "-C", str(root), "-c", "user.name=Test", "-c", "user.email=test@example.org", + "commit", "-qm", "Initial"], check=True) + apply_trl(root) + restore_trl(root) + assert path.read_text() == original + apply_trl(root) + edited = path.read_text() + "# local user change\n" + path.write_text(edited) + restore_trl(root) + assert path.read_text() == edited diff --git a/05-multi-harness-rl/tests/test_runtime.py b/05-multi-harness-rl/tests/test_runtime.py index 692c733c..cd6087d7 100644 --- a/05-multi-harness-rl/tests/test_runtime.py +++ b/05-multi-harness-rl/tests/test_runtime.py @@ -66,13 +66,107 @@ def test_worker_options_match_pinned_runtime(): assert not {kw.arg for kw in call.keywords} - names -def test_native_openenv_partial_mask_reaches_trl(): +@pytest.mark.parametrize("mask", [[0, 0, 1, 0], [0, 0, 0, 0]]) +def test_native_openenv_partial_mask_reaches_trl(mask): from runtime.bootstrap import activate activate() from data_agent_env.models import DataAgentRolloutResult, DataAgentTurn - from data_agent_env.harness import to_trace_entries - from trl.experimental.async_grpo.openenv_harness import _turns_from_trace + from runtime.native_capture import training_trace + from trl.experimental.async_grpo.openenv_harness import _turns_from_training_trace result = DataAgentRolloutResult(rollout_type="train", turns=[DataAgentTurn( - trainable=True, prompt_token_ids=[1, 2], completion_token_ids=[3, 4], - per_token_logps=[-.2, -.4], loss_mask=[0, 0, 1, 0])]) - assert _turns_from_trace(to_trace_entries(result))[0].output_mask == [1, 0] + trainable=bool(any(mask)), prompt_token_ids=[1, 2], completion_token_ids=[3, 4], + per_token_logps=[-.2, -.4], loss_mask=mask, capture_metadata={"node_id": "agent-0"})]) + trace = training_trace(result) + assert len(trace.turns) == 1 + assert _turns_from_training_trace(trace)[0].output_mask == mask[2:] + + +@pytest.mark.parametrize("broken", ["eval", "mask", "duplicate", "fatal", "identity"]) +def test_native_training_capture_rejects_invalid_records(broken): + from runtime.bootstrap import activate + activate() + from data_agent_env.models import DataAgentRolloutResult, DataAgentTurn + from runtime.native_capture import training_trace + turn = DataAgentTurn(trainable=True, prompt_token_ids=[1], completion_token_ids=[2], + per_token_logps=[-.2], loss_mask=[0, 1], capture_metadata={"node_id": "a"}) + result = DataAgentRolloutResult(rollout_type="train", turns=[turn]) + if broken == "eval": + result.rollout_type = "eval" + elif broken == "mask": + turn.loss_mask = [] + elif broken == "identity": + turn.capture_metadata = {} + elif broken == "duplicate": + result.turns.append(turn) + else: + result.metadata["capture_findings"] = ["[FATAL] incomplete capture"] + with pytest.raises((ValueError, KeyError)): + training_trace(result) + + +@pytest.mark.parametrize("correctness,actions,expected", [(1.0, 5, 1.075), (0.0, 5, 0.0), (1.0, None, 1.0), (None, 5, None)]) +def test_native_session_worker_preserves_reward_after_typed_copy(tmp_path, correctness, actions, expected): + from runtime.bootstrap import activate + activate() + import queue + import threading + from types import SimpleNamespace + from unittest.mock import MagicMock + from accelerate import PartialState + from data_agent_env.models import DataAgentRolloutResult, DataAgentTurn + from data_agent_env.harness import DataAgentSession + from openenv.core.harness import TrainableSession + from trl.experimental.async_grpo.openenv_harness import _HarnessRolloutLoop + from train.adapters import Session, rollout_reward + + PartialState() + result = DataAgentRolloutResult(rollout_type="train", correctness=correctness, + reward=correctness, metadata={"verified_native_actions": actions}, turns=[DataAgentTurn( + trainable=True, prompt_token_ids=[1], completion_token_ids=[2, 3], + per_token_logps=[-.2, -.4], loss_mask=[0, 1, 0], capture_metadata={"node_id": "a"})]) + client = MagicMock() + client.run_rollout.return_value = result + cfg = {"mode": "opencode", "efficiency_weight": .1, "tool_budget": 15, "output": str(tmp_path)} + wrapped = Session(DataAgentSession(client, "train", 0, "task"), cfg, {"episode_id": "smoke"}, tmp_path) + assert isinstance(wrapped, TrainableSession) + policy = {} + def build(*, sampling): + policy.update(sampling) + return SimpleNamespace(create=lambda *args, **kwargs: wrapped) + loop = _HarnessRolloutLoop(harness_session_factory=build, rollout_reward_fn=rollout_reward, + model_name="test", dataset=[{"prompt": [{"role": "user", "content": "task"}]}], reward_funcs=[], + processing_class=MagicMock(eos_token_id=0, pad_token_id=0), rollout_buffer=queue.Queue(), + model_version_value=SimpleNamespace(value=0), heartbeat_value=SimpleNamespace(value=0.), + failed_event=threading.Event(), exception_info_queue=queue.Queue(), metrics_queue=queue.Queue(), + max_inflight_tasks=2, temperature=.8) + try: + actual, metrics = loop._run_session([]) + if expected is None: + assert actual[-1] is None + else: + assert actual[-1] == pytest.approx(expected) + assert actual[2][0].completion_mask == [0, 1, 0] + assert actual[2][0].old_log_probs == [0., -.2, -.4] + assert policy["temperature"] == .8 and policy["top_p"] == 1. + record = json.loads((tmp_path / "rollouts/smoke.json").read_text()) + assert record["reward"] == actual[-1] + client.close.assert_called_once() + finally: + loop._session_pool.shutdown(wait=True) + loop._loop.close() + + +@pytest.mark.parametrize("tasks,requested", [(2000, 1000), (2000, 2000), (5, 1000)]) +def test_whitebox_update_limit_matches_trl_sampler(tasks, requested): + from runtime.bootstrap import activate + activate() + from recipe import config, whitebox_step_limit + from trl.trainer.utils import RepeatSampler + cfg = config(mode="whitebox", max_steps=requested) + accumulation = cfg["batch_size"] * cfg["gradient_accumulation_steps"] + sampler = RepeatSampler(range(tasks), mini_repeat_count=cfg["num_generations"], + batch_size=accumulation // cfg["num_generations"], repeat_count=accumulation, shuffle=False) + available = len(list(sampler)) // (accumulation * accumulation) + assert whitebox_step_limit(cfg, tasks) == min(requested, available) + if tasks == 2000: + assert available == 1000 diff --git a/05-multi-harness-rl/train/adapters.py b/05-multi-harness-rl/train/adapters.py index c95368ae..2ac16bb9 100644 --- a/05-multi-harness-rl/train/adapters.py +++ b/05-multi-harness-rl/train/adapters.py @@ -46,9 +46,28 @@ def __getattr__(self, name): def result(self): return getattr(self.inner, "result", getattr(self.inner, "_result", None)) - def fetch_proxy_trace(self): - self.trace = self.inner.fetch_proxy_trace() - return self.trace + def wait_for_completion(self, timeout_s=None): + return self.inner.wait_for_completion(timeout_s) + + def fetch_training_trace(self): + trace = self.inner.fetch_training_trace() + calls, error = self.tool_usage() + inputs = {"tool_calls": calls, "tool_count_error": error, + "weight": self.cfg["efficiency_weight"], "budget": self.cfg["tool_budget"]} + for turn in trace.turns: + turn.metadata["recipe_reward_inputs"] = inputs + return trace + + def tool_usage(self): + result = self.result + if result is None: + return None, None + if self.cfg["mode"] == "opencode": + return result.metadata.get("verified_native_actions"), None + try: + return atif_count(self.trial_root / result.trial_name / "agent/trajectory.json"), None + except (OSError, ValueError, KeyError, TypeError) as exc: + return None, type(exc).__name__ def verify(self, transcript, final_state=None): from openenv.core.harness import VerifyResult @@ -56,24 +75,15 @@ def verify(self, transcript, final_state=None): original = self.inner.verify(transcript, final_state) result = self.result correctness = original.env_reward - calls, error = None, None - if result is not None: - if self.cfg["mode"] == "opencode": - value = result.correctness - correctness = None if value is None else float(value >= 1) - calls = result.metadata.get("verified_native_actions") - else: - try: - calls = atif_count(self.trial_root / result.trial_name / "agent/trajectory.json") - except (OSError, ValueError, KeyError, TypeError) as exc: - error = type(exc).__name__ + if self.cfg["mode"] == "opencode" and result is not None: + value = result.correctness + correctness = None if value is None else float(value >= 1) + calls, error = self.tool_usage() shaped = reward(correctness, calls, verified=calls is not None, weight=self.cfg["efficiency_weight"], budget=self.cfg["tool_budget"]) record = {**self.evidence, "correctness": correctness, "tool_calls": calls, "tool_count_verified": calls is not None, "tool_count_error": error, "reward": shaped, "bonus": None if shaped is None else shaped - correctness} - if getattr(self, "trace", None): - self.trace[0]["recipe_reward"] = record write_json(Path(self.cfg["output"]) / "rollouts" / f'{self.evidence["episode_id"]}.json', record) return VerifyResult(env_reward=correctness, done=original.done, metrics={**original.metrics, **record}, artifacts=original.artifacts) @@ -82,13 +92,16 @@ def verify(self, transcript, final_state=None): def rollout_reward(outcome): if not outcome.trace: return None - return outcome.trace[0]["recipe_reward"]["reward"] + inputs = outcome.trace[0]["metadata"]["recipe_reward_inputs"] + return reward(outcome.env_reward, inputs["tool_calls"], verified=inputs["tool_calls"] is not None, + weight=inputs["weight"], budget=inputs["budget"]) class Factory: - def __init__(self, cfg, data, server, vllm, groups, trial_root, split="train"): + def __init__(self, cfg, data, server, vllm, groups, trial_root, split="train", *, sampling=None): self.cfg, self.data, self.server, self.vllm = cfg, str(data), server, vllm self.groups, self.trial_root, self.split = groups, str(trial_root), split + self.sampling = sampling manifest = json.loads((Path(data) / f"{split}_manifest.json").read_text()) self.tasks = sorted(manifest["tasks"], key=lambda row: row["name"]) self.instructions = [(Path(data) / "datasets" / split / "tasks" / r["name"] / "instruction.md").read_text() @@ -107,7 +120,9 @@ def create(self, task, seed=None, episode_id=None): instruction = self.instructions[index] if _instruction_of(task).strip() != instruction.strip(): raise ValueError("Prompt and frozen task schedule disagree") - sampling = {k: self.cfg[k] for k in ("temperature", "top_p", "top_k")} + sampling = self.sampling + if sampling is None: + raise ValueError("Training factory requires the worker sampling policy") if self.cfg["mode"] == "opencode": from data_agent_env import DataAgentEnv from data_agent_env.harness import DataAgentSession diff --git a/05-multi-harness-rl/train/blackbox.py b/05-multi-harness-rl/train/blackbox.py index d6eeccc7..deca21ed 100644 --- a/05-multi-harness-rl/train/blackbox.py +++ b/05-multi-harness-rl/train/blackbox.py @@ -1,5 +1,6 @@ """AsyncGRPO with atomic rollout admission and a finite, resumable task schedule.""" import json +from functools import partial import os from pathlib import Path @@ -47,12 +48,9 @@ def train(cfg, data, server, vllm, resume=None): factory = Factory(cfg, data, server, vllm, groups, output / "trials") dataset = Dataset.from_list(factory.rows()) tokenizer = tokenizer_for(cfg) - selector = None - if cfg["mode"] == "opencode": - from data_agent_env import opencode_agent_turns - selector = opencode_agent_turns - worker = Worker(harness_session_factory=factory, harness_adapter=None, - train_turn_fn=None, agent_turn_fn=selector, fork_threshold_tokens=0, rollout_reward_fn=rollout_reward, + factory_builder = partial(Factory, cfg, data, server, vllm, groups, output / "trials") + worker = Worker(harness_session_factory=factory_builder, harness_adapter=None, + lossless_capture=True, rollout_reward_fn=rollout_reward, model_name=cfg["profile"]["id"], processing_class=tokenizer, dataset=dataset, reward_funcs=[], num_generations=cfg["num_generations"], max_inflight_tasks=cfg["max_inflight"], max_outstanding_rollouts=cfg["max_outstanding_rollouts"], vllm_server_url=vllm, diff --git a/05-multi-harness-rl/train/whitebox.py b/05-multi-harness-rl/train/whitebox.py index 95a7655c..e03310fb 100644 --- a/05-multi-harness-rl/train/whitebox.py +++ b/05-multi-harness-rl/train/whitebox.py @@ -7,7 +7,7 @@ from trl import GRPOConfig, GRPOTrainer from whitebox_bash import white_box_bash_env -from recipe import schedule, task_rows, write_json +from recipe import schedule, task_rows, whitebox_step_limit, write_json from runtime.models import tokenizer_for from train.logging_utils import Checkpoints, Logs, publish_checkpoint from train.generation import protect_generation @@ -52,7 +52,7 @@ def train(cfg, data, server, vllm, resume=None): "split": "train", "index": g["task_index"]} for g in groups] args = GRPOConfig(output_dir=str(output), learning_rate=cfg["learning_rate"], lr_scheduler_type="constant", warmup_steps=0, beta=0.0, loss_type="dapo", - max_steps=cfg["max_steps"], num_train_epochs=1, shuffle_dataset=False, + max_steps=whitebox_step_limit(cfg, len(groups)), num_train_epochs=1, shuffle_dataset=False, num_generations=cfg["num_generations"], per_device_train_batch_size=1, gradient_accumulation_steps=cfg["batch_size"] * cfg["gradient_accumulation_steps"], max_completion_length=cfg["max_completion_length"], max_tool_calling_iterations=cfg["agent_step_limit"] - 1, From 4d9c040a28695c484de75bc23d6a70a99beda258 Mon Sep 17 00:00:00 2001 From: adithya-s-k Date: Wed, 30 Sep 2026 17:54:26 +0000 Subject: [PATCH 16/48] Record merged-contract GPU smokes and sync TRL CI fix --- 05-multi-harness-rl/RESULTS.md | 2 ++ 05-multi-harness-rl/VALIDATION.md | 16 +++++++++++++++- 05-multi-harness-rl/configs/runtime-lock.json | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/05-multi-harness-rl/RESULTS.md b/05-multi-harness-rl/RESULTS.md index d7470839..50f8e4c5 100644 --- a/05-multi-harness-rl/RESULTS.md +++ b/05-multi-harness-rl/RESULTS.md @@ -4,6 +4,8 @@ These are the checks and pilot results for the non-thinking recipe. Two blackbox ## Completed checks +The merged-contract HF smokes completed two updates and checkpoint reload in all three modes. Reload evaluation graded native OpenCode 4/4 pairs, Harbor 4/4 pairs and whitebox 1/1 task. [Validation](VALIDATION.md#merged-contract-update-30-september) records pins and evidence. The qualification table below predates that migration. + | Check | Result | Evidence | |---|---|---| | Local training, LFM and Qwen × all three modes | All six completed two updates and saved checkpoints | [Validation record](VALIDATION.md) | diff --git a/05-multi-harness-rl/VALIDATION.md b/05-multi-harness-rl/VALIDATION.md index ec9bca28..c87d8dd6 100644 --- a/05-multi-harness-rl/VALIDATION.md +++ b/05-multi-harness-rl/VALIDATION.md @@ -9,7 +9,21 @@ The recipe now pins merged [OpenEnv #1280](https://github.com/huggingface/OpenEn - 43 CPU checks cover the updated recipe, including native partial/zero masks, invalid captures, correctness plus efficiency through the typed TRL worker, and the whitebox update budget against TRL's sampler. - The TRL integration passed 51 contract/reconciliation checks against the merged producer. - Fresh-clone bootstrap and repeat-bootstrap handling are tested. User edits are preserved; only the recipe's known attention patch is removed before updating TRL. -- HF A100 training and checkpoint-reload smokes are running for all three modes. Results will be recorded here after completion. +- Fresh HF A100 training and checkpoint-reload evaluation completed in all three modes, as recorded below. + +Each smoke used non-thinking LFM2.5-2.6B, two optimizer updates and two rollouts per group. Both checkpoints were saved; evaluation restarted inference from checkpoint 2. Blackbox evaluation used one fixed task across four harnesses; whitebox used one native task. + +| Mode | Updates / checkpoints | Reload eval graded | Correct | Nonzero gradient? | Evidence | +|---|---|---|---|---|---| +| Native OpenCode | 2 / 2 | 4/4 | 2/4 | No reward contrast | [HF smoke](https://huggingface.co/jobs/FineEnvs/6abd47c0fbc85ba68235c27c) | +| Harbor multi-harness | 2 / 2 | 4/4 | 1/4 | Yes, norm 5.47 at step 1 | [HF smoke](https://huggingface.co/jobs/FineEnvs/6abd47bdfbc85ba68235c27a) | +| Whitebox SETA | 2 / 2 | 1/1 | 1/1 | No reward contrast | [HF smoke](https://huggingface.co/jobs/FineEnvs/6abd47c3fbc85ba68235c27e) | + +These tiny samples validate execution, capture, saving and reload, not learning quality. Blackbox evaluations passed token/mask checks. A correct Harbor training rollout with six verified tool calls received 1.0714; failed rollouts received zero. Trainer reward averages are over training rows and must not be read as rollout pass@1. + +The jobs ran OpenEnv `86a180ed` and TRL `a39476c6`. The current TRL pin has identical experimental trainer code; later changes cover example warmup, optional tests, documentation and an upstream CI fix. The first fresh-container attempt exposed a bootstrap restore bug before training; it was fixed and regression-tested before these replacement jobs. + +Artifacts are in bucket `FineEnvs/data-agent-daytona-artifacts`, under `merged-contract-{opencode,multi-harness,whitebox}-v2`. Each has `metrics.jsonl`, checkpoints, local Trackio logs and `reload-eval/eval/summary.json`. Bucket access requires permission. Execution status, typed execution limits and structured tool outcomes in OpenEnv remain follow-up API work from the TRL review. The recipe uses explicit per-session turn/time budgets and verified native tool counts; it does not claim the broader session API is complete. diff --git a/05-multi-harness-rl/configs/runtime-lock.json b/05-multi-harness-rl/configs/runtime-lock.json index 06d1dab4..b6986be9 100644 --- a/05-multi-harness-rl/configs/runtime-lock.json +++ b/05-multi-harness-rl/configs/runtime-lock.json @@ -52,7 +52,7 @@ }, "trl": { "repo": "adithya-s-k/trl", - "revision": "173f37e9dca9a8d7aecf75c9076f63dbfc853899" + "revision": "2de2cea22b1a3da717c7904722109ca6e2ecb503" }, "openenv": { "repo": "huggingface/OpenEnv", From d3a845a9740b335560f6236f45b4b5b3087d702c Mon Sep 17 00:00:00 2001 From: adithya-s-k Date: Thu, 1 Oct 2026 06:10:09 +0000 Subject: [PATCH 17/48] Rewrite multi-harness example as three readable training tutorials --- .github/workflows/multi-harness-rl.yml | 2 +- 05-multi-harness-rl/.gitignore | 3 + 05-multi-harness-rl/README.md | 159 +++++---- 05-multi-harness-rl/REPRODUCE.md | 235 ++++-------- 05-multi-harness-rl/RESULTS.md | 6 +- 05-multi-harness-rl/VALIDATION.md | 110 ++---- 05-multi-harness-rl/check_setup.py | 50 +++ 05-multi-harness-rl/configs/default.json | 53 --- 05-multi-harness-rl/configs/runtime-lock.json | 61 ---- 05-multi-harness-rl/envs/__init__.py | 0 05-multi-harness-rl/envs/daytona.py | 144 ++++++++ 05-multi-harness-rl/envs/tasks.py | 75 ++++ 05-multi-harness-rl/eval/evaluate.py | 169 --------- 05-multi-harness-rl/eval/watch.py | 101 ------ 05-multi-harness-rl/evaluate.py | 283 +++++++++++++++ 05-multi-harness-rl/jobs/entrypoint.sh | 10 + 05-multi-harness-rl/jobs/hf_job.py | 134 +++++++ 05-multi-harness-rl/jobs/inference_proxy.py | 80 +++++ 05-multi-harness-rl/jobs/install.sh | 17 + 05-multi-harness-rl/jobs/run.py | 258 ++++++++++++++ 05-multi-harness-rl/jobs/serve_model.py | 80 +++++ 05-multi-harness-rl/jobs/train.slurm | 11 + 05-multi-harness-rl/prepare.py | 94 ++++- 05-multi-harness-rl/recipe.py | 153 -------- 05-multi-harness-rl/requirements.lock | 312 ---------------- 05-multi-harness-rl/requirements.txt | 52 +-- 05-multi-harness-rl/run.py | 215 ----------- 05-multi-harness-rl/runtime/bootstrap.py | 93 ----- 05-multi-harness-rl/runtime/checkpoints.py | 57 --- 05-multi-harness-rl/runtime/environment.py | 46 --- 05-multi-harness-rl/runtime/job.sh | 17 - 05-multi-harness-rl/runtime/launch.py | 155 -------- 05-multi-harness-rl/runtime/models.py | 82 ----- 05-multi-harness-rl/runtime/native_capture.py | 22 -- 05-multi-harness-rl/runtime/patches.py | 91 ----- 05-multi-harness-rl/runtime/pilot.py | 120 ------- .../runtime/whitebox_reward.py | 18 - 05-multi-harness-rl/tests/conftest.py | 26 ++ 05-multi-harness-rl/tests/test_contract.py | 244 +++++++++++++ 05-multi-harness-rl/tests/test_contracts.py | 337 ------------------ .../tests/test_data_and_eval.py | 61 ++++ 05-multi-harness-rl/tests/test_launch.py | 70 ++++ 05-multi-harness-rl/tests/test_runtime.py | 172 --------- 05-multi-harness-rl/train/adapters.py | 144 -------- 05-multi-harness-rl/train/blackbox.py | 82 ----- 05-multi-harness-rl/train/generation.py | 34 -- 05-multi-harness-rl/train/logging_utils.py | 38 -- 05-multi-harness-rl/train/whitebox.py | 84 ----- 05-multi-harness-rl/train_multi_harness.py | 255 +++++++++++++ 05-multi-harness-rl/train_opencode.py | 290 +++++++++++++++ 05-multi-harness-rl/train_whitebox.py | 302 ++++++++++++++++ 51 files changed, 2665 insertions(+), 3042 deletions(-) create mode 100644 05-multi-harness-rl/check_setup.py delete mode 100644 05-multi-harness-rl/configs/default.json delete mode 100644 05-multi-harness-rl/configs/runtime-lock.json create mode 100644 05-multi-harness-rl/envs/__init__.py create mode 100644 05-multi-harness-rl/envs/daytona.py create mode 100644 05-multi-harness-rl/envs/tasks.py delete mode 100644 05-multi-harness-rl/eval/evaluate.py delete mode 100644 05-multi-harness-rl/eval/watch.py create mode 100644 05-multi-harness-rl/evaluate.py create mode 100644 05-multi-harness-rl/jobs/entrypoint.sh create mode 100644 05-multi-harness-rl/jobs/hf_job.py create mode 100644 05-multi-harness-rl/jobs/inference_proxy.py create mode 100644 05-multi-harness-rl/jobs/install.sh create mode 100644 05-multi-harness-rl/jobs/run.py create mode 100644 05-multi-harness-rl/jobs/serve_model.py create mode 100644 05-multi-harness-rl/jobs/train.slurm delete mode 100644 05-multi-harness-rl/recipe.py delete mode 100644 05-multi-harness-rl/requirements.lock delete mode 100644 05-multi-harness-rl/run.py delete mode 100644 05-multi-harness-rl/runtime/bootstrap.py delete mode 100644 05-multi-harness-rl/runtime/checkpoints.py delete mode 100644 05-multi-harness-rl/runtime/environment.py delete mode 100644 05-multi-harness-rl/runtime/job.sh delete mode 100644 05-multi-harness-rl/runtime/launch.py delete mode 100644 05-multi-harness-rl/runtime/models.py delete mode 100644 05-multi-harness-rl/runtime/native_capture.py delete mode 100644 05-multi-harness-rl/runtime/patches.py delete mode 100644 05-multi-harness-rl/runtime/pilot.py delete mode 100644 05-multi-harness-rl/runtime/whitebox_reward.py create mode 100644 05-multi-harness-rl/tests/conftest.py create mode 100644 05-multi-harness-rl/tests/test_contract.py delete mode 100644 05-multi-harness-rl/tests/test_contracts.py create mode 100644 05-multi-harness-rl/tests/test_data_and_eval.py create mode 100644 05-multi-harness-rl/tests/test_launch.py delete mode 100644 05-multi-harness-rl/tests/test_runtime.py delete mode 100644 05-multi-harness-rl/train/adapters.py delete mode 100644 05-multi-harness-rl/train/blackbox.py delete mode 100644 05-multi-harness-rl/train/generation.py delete mode 100644 05-multi-harness-rl/train/logging_utils.py delete mode 100644 05-multi-harness-rl/train/whitebox.py create mode 100644 05-multi-harness-rl/train_multi_harness.py create mode 100644 05-multi-harness-rl/train_opencode.py create mode 100644 05-multi-harness-rl/train_whitebox.py diff --git a/.github/workflows/multi-harness-rl.yml b/.github/workflows/multi-harness-rl.yml index 2f7ef525..57e66572 100644 --- a/.github/workflows/multi-harness-rl.yml +++ b/.github/workflows/multi-harness-rl.yml @@ -16,6 +16,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.12' - - run: python -m pip install pytest==9.1.1 requests + - run: python -m pip install pytest==9.1.1 httpx - name: Check recipe contracts without GPUs or sandbox credentials run: python -m pytest 05-multi-harness-rl/tests -q diff --git a/05-multi-harness-rl/.gitignore b/05-multi-harness-rl/.gitignore index c1f1293c..ff8d02a0 100644 --- a/05-multi-harness-rl/.gitignore +++ b/05-multi-harness-rl/.gitignore @@ -6,3 +6,6 @@ runs/ prepared/ .env .gradio/ + +temp/ +.deps/ diff --git a/05-multi-harness-rl/README.md b/05-multi-harness-rl/README.md index a831a7a2..9bd25c09 100644 --- a/05-multi-harness-rl/README.md +++ b/05-multi-harness-rl/README.md @@ -1,110 +1,111 @@ -# SmolDataEnvs: learn RL through three agent interfaces +# SmolDataEnvs: train an agent, then change its harness -Fine-tune a small language model to solve data-analysis tasks, then measure whether it answers more questions correctly and uses fewer tools. This example compares three ways of connecting an agent to a trainer, using the same tasks and reward. +A model can solve the same task through different agent programs. Does training through several programs help it transfer? This tutorial gives you three scripts to explore that question on data-analysis tasks. -Start with [04: SmolDataEnvs](../04-smoldataenvs/) for the dataset, SFT and single-turn RL. This chapter takes those tasks into multi-turn agent harnesses. The companion article, [The ultimate guide to multi-harness RL](https://huggingface.co/spaces/AdithyaSK/multi-harness-rl), explains the approach and earlier experiments. +Start with [04: SmolDataEnvs](../04-smoldataenvs/) for the dataset. The [multi-harness RL article](https://huggingface.co/spaces/AdithyaSK/multi-harness-rl) explains the earlier experiments. -**Start here:** [run the tutorial](REPRODUCE.md). You can submit an HF Job from a CPU laptop, or run on a local two-GPU machine or Slurm cluster. [Current results](RESULTS.md) distinguish completed checks from ongoing experiments; [validation notes](VALIDATION.md) contain the detailed evidence. +## Open a training script first -## Choose an agent interface +Each script reads like a notebook: settings → tokenizer → environment → reward → TRL config → training. The data loading, reward and trainer call are visible in that file. The numbered sections keep the setup and training flow easy to follow. To change the learning rate or batch size, edit the visible TRL config in that script. -A *harness* is the program that manages the model's conversation and tool calls. OpenEnv provides the environment interface. Harbor runs third-party harnesses and their task verifiers. +| Script | Who runs the tool loop? | Trainer | +|---|---|---| +| [train_whitebox.py](train_whitebox.py) | TRL calls Python bash/SETA tools | `GRPOTrainer` | +| [train_opencode.py](train_opencode.py) | Native OpenCode runs inside a sandbox | `AsyncGRPOTrainer` | +| [train_multi_harness.py](train_multi_harness.py) | Harbor runs OpenCode, Claude Code, Codex or Mini-SWE-Agent | `AsyncGRPOTrainer` | -| Mode | What happens | Trainer | Evaluation | -|---|---|---|---| -| `whitebox` | TRL generates responses and directly invokes bash/SETA tools | Synchronous GRPO | Native bash/SETA | -| `opencode` | Native OpenCode owns the agent loop; OpenEnv captures its model calls | AsyncGRPO | All four Harbor harnesses | -| `multi-harness` | Harbor rotates OpenCode, Claude Code, Codex and Mini-SWE-Agent between task groups | AsyncGRPO | All four Harbor harnesses | +Read whitebox first if you are new to tool-using RL. Its `BashEnvironment` shows exactly which tools the model can call and how an answer is graded. Then read OpenCode to see what changes when an existing agent owns the conversation. The multi-harness script assigns one harness to each task; its eight rollouts all use that assignment. -**Native OpenCode training does not run through Harbor.** Both blackbox policies use the same Harbor evaluation interfaces so you can measure transfer to other harnesses. Whitebox evaluates its native tools, which is a different evaluation interface. +**Current dependency status:** install TRL and OpenEnv from upstream main. The typed blackbox integration still needs [TRL #6947](https://github.com/huggingface/trl/pull/6947), which was open when this rewrite was checked. `check_setup.py` reports this before training starts. There is no automatic fork checkout or source patch. [Validation](VALIDATION.md) separates checks of this rewrite from older GPU results. -Use `--model lfm` for **LiquidAI/LFM2.5-2.6B**, or `--model qwen` for **Qwen/Qwen3.5-2B**. Both use non-thinking prompts. The new runs need their own baseline; historical thinking-enabled results are not interchangeable. +## What is being trained? -## What happens in a run? +The model receives a question and can inspect data in a Daytona sandbox. Its answer is compared with a held-out answer using the task's deterministic grader. A correct answer earns a small extra reward when it uses fewer tools: -1. Load the fixed task list and checked dataset revision. -2. Start vLLM and OpenEnv inside the job. Daytona supplies isolated task containers. -3. Generate eight rollouts for each task group. Each rollout tries to solve the task with tools. -4. Grade the answer and count verified native tool calls. -5. Train on the model's generated tokens, masking prompts and tool outputs. -6. Save checkpoints and evaluate them on held-out tasks. - -TiTO means *tokens in, tokens out*: training uses the inference engine's actual token IDs and logprobs rather than reconstructing them from text. A rewritten conversation can produce several training rows from one rollout. A task, rollout, training row and optimizer step are therefore different units. - -Both blackbox modes implement OpenEnv's `fetch_training_trace()` contract. The producer selects agent calls and supplies explicit masks; TRL receives the engine IDs unchanged. Partial masks and zero-masked agent turns survive. The worker supplies sampling before the factory starts a session. Native tool counts are carried as structured reward metadata, separately from token eligibility. - -Synchronous GRPO waits for its batch of tool rollouts. AsyncGRPO collects rollouts while training proceeds, with a bounded amount of policy staleness. A group whose rollouts all have the same reward supplies no relative learning signal. Successful execution alone does not establish useful learning. - -## The shared experiment +```python +bonus = 0.1 * 15 / (15 + tool_calls) if tool_calls > 0 else 0 +reward = correctness * (1 + bonus) +``` -| Setting | Default | -|---|---| -| Training set | 1,000 fixed tasks: 400 medium and 600 hard | -| Held-out set | 250 fixed tasks: 33 easy, 118 medium and 99 hard; no train/test task or notebook overlap | -| Learning rate / sampling | `3e-6`; temperature `0.8`; full-vocabulary sampling | -| Rollouts per task group | 8, using the same task and harness | -| Async limits | 32 inflight, 16 outstanding rollouts, maximum staleness 4 | -| Async batch | Batch 4 × accumulation 4; 40,960-token packing target | -| Whitebox batch | Microbatch 1 × accumulation 16 | -| Episode limits | 17 iterations, 600 seconds, 4,096 generated tokens per model call | -| Training limits | 16,384 completion tokens; 131,072 context; full BF16 fine-tuning | -| Checkpoints | Every 50 optimizer steps | -| Evaluation | Pass@1; default concurrency 35; TP1/DP2 inference | - -Correctness is binary. A correct answer with a verified, positive tool count earns: - -```text -reward = correctness × (1 + 0.1 × 15 / (15 + tool_calls)) +A correct answer with 15 tool calls earns `1.05`; a wrong answer earns `0`. Missing tool-count evidence gets no bonus. A failed environment without a grade is excluded, not scored as wrong. The code counts native tool actions, not model requests or training rows. + +Whitebox gives TRL the tools and lets it generate each turn. Blackbox gives TRL an OpenEnv session. OpenEnv returns the actual engine token IDs, behavior logprobs and loss masks through `fetch_training_trace()`. TRL trains on the eligible model tokens. Prompts and tool outputs are context. + +```python +# Whitebox: TRL owns generation and tool execution. +trainer = GRPOTrainer( + model=model, args=config, train_dataset=dataset, + reward_funcs=[], environment_factory=BashEnvironment, +) + +# Blackbox: the agent owns generation; OpenEnv captures its tokens. +worker = HarnessRolloutWorker( + harness_session_factory=factory, + harness_adapter=None, + rollout_reward_fn=reward, + # See either blackbox script for the model and sampling arguments. +) +trainer = AsyncGRPOTrainer( + model=model, args=config, train_dataset=dataset, rollout_worker=worker, +) ``` -For example, a correct answer using 15 calls earns `1.05`; an incorrect answer earns `0`. Zero or unverified tool counts receive no bonus. Infrastructure failures remain ungraded. Evaluation reports correctness separately from shaped reward and tool/token usage. +These are the boundaries to compare. The complete calls, including every training parameter, are in the scripts above. -The full schedule contains two passes and stops at 1,000 updates or schedule exhaustion, whichever comes first. The 100-step pilot retains the same schedule and training settings but stops at 100 updates. Neither setting guarantees that every scheduled task is consumed. The committed-group audit records what async training actually used. For whitebox, 16 rollouts per update divided by 8 rollouts per task means 2 unique prompts per update. Its 2,000 scheduled prompts therefore allow 1,000 updates; custom step limits are capped at schedule exhaustion. +## Run a small experiment -## Start small, then scale +Use Python 3.12 and two H100 or H200 GPUs: one for vLLM, one for training. HF Jobs can supply them; you do not need a GPU on your laptop. OpenEnv runs inside the job. Daytona supplies the task sandboxes. No environment Space is needed. -After the [HF account, bucket and sandbox setup](REPRODUCE.md#2-submit-from-a-cpu-laptop-with-hf-jobs), preview a quick training and evaluation smoke: +Follow [REPRODUCE.md](REPRODUCE.md) for installation, a two-step smoke, a 100-step comparison, and checkpoint evaluation. From a prepared local environment: ```bash -python runtime/launch.py hf smoke --model lfm --mode opencode \ - --run-name smoke-lfm-opencode-v1 --bucket "$HF_BUCKET" \ - --flavor a100x4 --timeout 45m --smoke-eval --concurrency 4 +# Two updates, saving both checkpoints. The normal eight-rollout group is retained. +python jobs/run.py train --mode multi_harness --steps 2 --save-steps 1 \ + --output runs/harbor-smoke + +# Restart inference from the saved weights and evaluate two tasks × four harnesses. +python jobs/run.py eval --mode multi_harness \ + --checkpoint runs/harbor-smoke/checkpoint-2 --step 2 \ + --tasks 2 --concurrency 4 --output runs/harbor-smoke-eval ``` -Add `--submit` to launch. This runs two small updates, saves both checkpoints and reloads checkpoint 2 for evaluation on two tasks through all four harnesses. For the other modes, use `--mode multi-harness` or `--mode whitebox` and a new run name. Whitebox evaluates two native SETA episodes. The [tutorial](REPRODUCE.md) also covers local and Slurm execution. +Use `--mode opencode` for native OpenCode, or `--mode whitebox` for bash/SETA. Blackbox evaluation uses all four Harbor harnesses for both policies. Whitebox evaluation uses its own bash/SETA tools. -| Action | Purpose | -|---|---| -| `smoke --smoke-eval` | Two small training updates, checkpoint saving, then two-task reload evaluation | -| `pilot` | Baseline, 100 updates, checkpoint-100 evaluation on 25 fixed tasks by default | -| `train` + evaluation watcher | Longer training with separate checkpoint-evaluation jobs | +## Shared settings -The pilot evaluates 3 easy, 12 medium and 10 hard tasks. That is **100 pairs per blackbox evaluation** or **25 whitebox episodes**. Its phases run sequentially in one allocation, restarting services between phases. No watcher is needed. Full training uses separate evaluation allocations; shared sandbox capacity and storage can still affect throughput. +| Setting | Value | +|---|---| +| Models | LFM2.5-2.6B or Qwen3.5-2B, non-thinking | +| Training tasks | 1,000 fixed tasks: 400 medium, 600 hard | +| Test tasks | 250 fixed tasks: 33 easy, 118 medium, 99 hard | +| Learning rate | `3e-6`, constant, no warmup | +| Sampling | Temperature `0.8`, full vocabulary | +| Rollouts per task group | 8 | +| Async limits | 32 inflight, maximum staleness 4 | +| Async batching | Batch 4, accumulation 4, packed row budget 40,960 tokens | +| Whitebox batching | Batch 1, accumulation 16 | +| Training length | Up to 1,000 optimizer updates | +| Save / evaluate | Save every 50; evaluate checkpoints 100, 200, … separately | +| Evaluation | Pass@1, default concurrency 35, TP1/DP2 | -There is **no Space to deploy** for these runs. OpenEnv lives inside the GPU job; the task containers live in Daytona. A Trackio Space is optional for online charts. +A step is an optimizer update, not a task. Async sampling and packing can use a variable number of prompts per step. This tutorial uses the upstream sampler and does not reproduce the archive's custom finite curriculum, exact committed-group resume, or whole-rollout weighting. Whitebox consumes 16 rollouts, or two groups of eight, per update. Blackbox histories can fork into multiple rows; long rows beyond the packing budget are dropped by upstream TRL. Inspect its drop and staleness metrics. -## Find the code +Both models use a 131,072-token serving context and up to 4,096 tokens per model call. Whitebox and Harbor cap the agent at 17 model turns. The public native OpenCode API has a 600-second timeout and per-call token cap, but no equivalent strict turn limit. These limits are not interchangeable. LFM's packed convolution boundary handling is explicit in each async script and checked on the GPU before training. -| Path | What it does | -|---|---| -| `configs/default.json` | Inspect model pins and shared hyperparameters | -| `run.py`, `runtime/launch.py` | Run locally, submit to HF Jobs or inspect a Slurm command | -| `runtime/pilot.py` | Follow the baseline → train → checkpoint-eval sequence | -| `train/whitebox.py` | Understand synchronous native tool training | -| `train/blackbox.py`, `train/adapters.py` | Understand async training and the native/Harbor boundary | -| `eval/evaluate.py` | Inspect pass@1, coverage and per-harness results | -| `eval/watch.py` | Submit separate evaluations for full training checkpoints | -| `prepare.py`, `data/` | Inspect the exact task selection and dataset checks | -| `configs/runtime-lock.json`, `runtime/patches.py` | Inspect pinned dependencies and compatibility fixes | +## Where to look next -`runtime/bootstrap.py` downloads the pinned TRL/OpenEnv sources and 45 existing runtime files into ignored `.runtime/`. It checks the archive file hashes. You do not need the original experiment workspace. Prepared tasks, checkpoints, caches and credentials are not committed. +- [envs/](envs/): Daytona operations, task data staging and host-side grading. No trainer subclasses. +- [evaluate.py](evaluate.py): individual pass@1 results, coverage, correctness, tool calls and token usage. Failed pairs remain visible and can be retried without rerunning graded pairs. +- [jobs/](jobs/): installation, service startup, HF submission and a Slurm template. No hidden training configuration. +- [prepare.py](prepare.py) and [data/](data/): fixed task selection and checked dataset revisions. +- [RESULTS.md](RESULTS.md): earlier experiment results and artifact locations. -## Earlier training results +Training keeps local Trackio data and checkpoints. Add `--space-id your-org/your-trackio-space` to also publish charts. It does not automatically write to the historical dashboard. -![Historical LFM training reward, held-out pass@1 and tool-call savings](assets/historical-lfm-curves.svg) +## Earlier experiment -These curves come from the [article](https://huggingface.co/spaces/AdithyaSK/multi-harness-rl), using 250 test tasks across four harnesses. Both historical LFM runs used Harbor, including the OpenCode-only run. This tutorial's `opencode` mode uses native OpenCode. The curves are historical context, not results from the new non-thinking pilots. +![Historical LFM training and evaluation curves](assets/historical-lfm-curves.svg) -Training reward uses a trailing 50-update mean. Evaluation is unsmoothed; hollow markers indicate incomplete coverage. Tool-call savings compare pairs solved by both baseline and checkpoint. See [the data source and final scores](RESULTS.md#historical-article-curves), or explore the [Trackio dashboard](https://huggingface.co/spaces/FineEnvs/data-agent-training-comparison-trackio). +These curves belong to the [article](https://huggingface.co/spaces/AdithyaSK/multi-harness-rl), not this rewrite. Both historical LFM policies used Harbor, including its OpenCode-only policy. The native OpenCode tutorial is a different interface. [RESULTS.md](RESULTS.md#historical-article-curves) links the plotted source data and the [historical dashboard](https://huggingface.co/spaces/FineEnvs/data-agent-training-comparison-trackio). -The full experiment remains on the [archived branch](https://github.com/adithya-s-k/FineEnvs/tree/archive/data-agent-experiments-20260930/04-data-agent). Corrected Harbor graders and dataset revisions are pinned here; [VALIDATION.md](VALIDATION.md) records what has been checked for this recipe. +The full experimental implementation remains on the [archived branch](https://github.com/adithya-s-k/FineEnvs/tree/archive/data-agent-experiments-20260930/04-data-agent). Earlier recipe code is also recoverable from this branch's git history. diff --git a/05-multi-harness-rl/REPRODUCE.md b/05-multi-harness-rl/REPRODUCE.md index 818c268d..71a5ed50 100644 --- a/05-multi-harness-rl/REPRODUCE.md +++ b/05-multi-harness-rl/REPRODUCE.md @@ -1,209 +1,122 @@ -# Run SmolDataEnvs RL +# Run the tutorial -Check out the example branch: +Start with one mode and a two-step smoke. Check that the checkpoint reloads and the evaluation grades every requested pair before spending time on a longer run. -```bash -git clone --branch 05-multi-harness-rl https://github.com/adithya-s-k/FineEnvs.git -cd FineEnvs/05-multi-harness-rl -``` - -Pick **HF Jobs** if you want the container to install and prepare everything. Pick **local/Slurm** if you already have a compatible GPU machine. Both use the same code, fixed tasks and model revisions. - -## 1. What you need - -- Python 3.12 and [uv](https://docs.astral.sh/uv/getting-started/installation/). -- An HF account with Jobs billing enabled, or access to a local GPU machine. -- A Daytona account and API key for task sandboxes. GPU and sandbox usage are billed separately. -- Two GPUs with enough memory: the qualified HF blackbox setup uses A100 80 GB; local smokes used H100s. The default HF request is H200×2. Other combinations need their own smoke. - -One GPU trains while the other serves rollouts. Evaluation uses both as independent inference replicas. HF offers A100×4 rather than A100×2; this recipe uses two GPUs but the entire allocation is billed. +## 1. Install and prepare the tasks -## 2. Submit from a CPU laptop with HF Jobs - -You do **not** need CUDA, PyTorch, the datasets or a local environment server for submission. +From the repository root: ```bash -uv venv --python 3.12 +cd 05-multi-harness-rl +uv venv --python 3.12 --seed .venv source .venv/bin/activate -uv pip install huggingface_hub==1.24.0 +bash jobs/install.sh hf auth login - -# Replace this with your own HF username or organization. -export HF_NAMESPACE="your-hf-username" -export HF_BUCKET="$HF_NAMESPACE/smoldataenv-runs" -hf buckets create "$HF_BUCKET" --private --exist-ok - -# Enter your Daytona key without putting it in shell history. -read -rs -p "Daytona API key: " DAYTONA_API_KEY -export DAYTONA_API_KEY +export HF_TOKEN="$(hf auth token)" +export DAYTONA_API_KEY="your-key" +python prepare.py ``` -Your HF login needs access to submit Jobs and write the bucket. The launcher uses the bucket owner as the job namespace; pass `--namespace` to choose another account you can use. Tokens are sent as HF Job secrets, not embedded in the uploaded source. Keep credentials out of config files. +The installer uses **TRL main**, **OpenEnv main** and Transformers main. OpenEnv's environment examples are loaded from the same checkout's `envs/` directory because they are outside its core wheel. No source files are modified. To update an existing checkout, run `git -C .deps/OpenEnv pull --ff-only` and rerun the installer. -### First: verify one small run +As of the rewrite, TRL #6947 is still open. Installation can finish while `check_setup.py` correctly stops on the missing typed consumer. Wait for that integration to merge before a main-only blackbox run. Do not treat the older pinned-runtime results as qualification of this installation. -Commands print a submission plan by default. Read it, then add `--submit` to launch. +Task preparation pins the corrected train/test dataset revisions and verifies instructions, graders and split separation. It downloads all 1,250 tasks into ignored `prepared/`. Credentials belong in the environment, never in source files. Create a [Daytona account](https://www.daytona.io/) and an HF bucket you can write to before using HF Jobs. -```bash -python runtime/launch.py hf smoke --model lfm --mode opencode \ - --run-name smoke-lfm-opencode-v1 --bucket "$HF_BUCKET" \ - --flavor a100x4 --timeout 45m --smoke-eval --concurrency 4 -``` +## 2. Submit from a laptop with HF Jobs -The job creates a Python environment, installs locked dependencies, fetches the pinned runtime and prepares tasks. It then runs two small training updates, saves checkpoints 1 and 2, and reloads checkpoint 2 for evaluation. Blackbox modes evaluate two tasks through each of four Harbor harnesses; whitebox evaluates two native SETA episodes. Require complete grading and valid capture, not a particular score from this tiny sample. - -### Next: a 100-step pilot +The submitter needs only `huggingface_hub`. It uploads the tutorial source, then installs dependencies and prepares tasks inside the job. It mounts an existing bucket at `/outputs` so checkpoints and logs survive the job. ```bash -python runtime/launch.py hf pilot --model lfm --mode multi-harness \ - --run-name pilot-lfm-multi-v1 --bucket "$HF_BUCKET" \ - --flavor a100x4 --timeout 12h --limit 25 --concurrency 35 -``` - -Add `--submit` after checking the plan. The phases are: - -1. Baseline pass@1 on 25 fixed tasks: 3 easy, 12 medium and 10 hard. -2. Train with the normal eight-rollout configuration for 100 updates, saving at 50 and 100. -3. Hash and reload checkpoint 100; repeat the same evaluation. -4. Write `comparison.json` with quality, tool/token usage and gradient diagnostics. - -Phases run sequentially in one allocation. Evaluation does not compete with training. An incomplete baseline stops the job before training. The 12-hour timeout is a spending/time cap, not a completion estimate. - -Change `--mode` to `opencode` or `whitebox` and choose a **new run name** for each comparison. Add `--preflight-smoke` if that mode/model/hardware combination has not passed the small training and reload test; the pilot proceeds only after it passes. Change `--model lfm` to `--model qwen` to use Qwen3.5-2B. Use `--limit 250` for the full held-out set. - -## 3. Monitor and read the results - -Submission prints a job URL and records it in `runs/submissions/RUN_NAME.json`. +pip install 'huggingface_hub>=1.29.0' +hf auth login +export DAYTONA_API_KEY="your-key" +export RUN_BUCKET="your-org/smoldataenv-runs" -```bash -hf jobs inspect JOB_ID --namespace "$HF_NAMESPACE" -hf jobs logs JOB_ID --namespace "$HF_NAMESPACE" --tail 50 -hf jobs logs JOB_ID --namespace "$HF_NAMESPACE" --follow +python jobs/hf_job.py train --mode multi_harness \ + --name lfm-harbor-smoke --bucket "$RUN_BUCKET" \ + --steps 2 --save-steps 1 --timeout 1h ``` -Logs show phase transitions, training step/reward/gradient norm and evaluation coverage. A job marked `RUNNING` may still be installing packages, loading a model or running its baseline; it does not necessarily mean training has begun. Inspect the final job status as well as the logs. +This prints the plan. Add `--submit` to launch. Use `h200x2` (default). Current upstream AsyncGRPO selects FlashAttention 3, so this tutorial does not patch in A100 support. -Outputs are under `/outputs/RUN_NAME` inside the job, backed by your bucket. Afterward, download a run: +After the smoke completes, load its checkpoint in a separate evaluation job: ```bash -hf buckets sync "hf://buckets/$HF_BUCKET/RUN_NAME" runs/RUN_NAME -``` - -```text -RUN_NAME/ -├── pilot.json configuration, exact test IDs and phase status -├── baseline/eval/ summary.json and one result per task/harness -├── train/ -│ ├── metrics.jsonl raw training scalars -│ ├── checkpoint-50/ -│ ├── checkpoint-100/ weights, optimizer, RNG and recipe -│ ├── rollouts/ blackbox correctness and tool-count evidence -│ ├── audit/ async group admission and consumed-rollout records -│ └── trackio/ local Trackio data -├── checkpoint-100/eval/ final held-out results -└── comparison.json baseline-to-checkpoint comparison -``` - -Each phase also keeps its service logs and resolved `config.json`. Whitebox keeps `token-audit.jsonl` and `whitebox-records/`; Harbor keeps native trajectories in `trials/`. OpenEnv/vLLM startup URLs are in `services.json`. A preflight, when enabled, has its own directory. - -Check these before calling a pilot successful: - -| Evidence | Interpretation | -|---|---| -| Complete evaluation coverage | All intended pairs graded; missing infrastructure results are not incorrect answers | -| Nonzero gradient updates | At least some groups supplied learning signal; zero contrast yields zero policy gradient | -| Baseline vs checkpoint pass@1 | Held-out correctness, not shaped training reward | -| Matched-success tool/token savings | Usage changes on pairs solved by both models; report the matched-pair count | -| First/last 20-update reward means | A training diagnostic; async row weighting differs from rollout-level correctness | - -A 25-task pilot is noisy. A higher reward or lower tool count alone does not establish better answer quality. Incorrect answers are retained as zero; only ungraded failures are retried. TiTO checks validate token/logprob/mask consistency, not benchmark performance. - -### Optional online Trackio - -Offline Trackio and scalar logs are always kept. To mirror charts, create a small config file under `configs/`: - -```json -{ - "project": "my-smoldataenv-pilot", - "trackio_space_id": "your-hf-username/smoldataenv-trackio" -} +python jobs/hf_job.py eval --mode multi_harness \ + --name lfm-harbor-smoke-eval --bucket "$RUN_BUCKET" \ + --checkpoint /outputs/lfm-harbor-smoke/checkpoint-2 --step 2 \ + --tasks 2 --concurrency 4 --timeout 1h --submit ``` -Pass `--config configs/my-run.json`. Your token needs permission to create/update that Space. Config files override [defaults](configs/default.json), so you only need the settings you want to change. Explicit CLI model, mode and concurrency options take precedence. Inspect the resolved settings with `python run.py plan --config configs/my-run.json`. +Repeat with `--mode opencode` and `--mode whitebox`, giving each job a fresh name. For Qwen, add `--model Qwen/Qwen3.5-2B` to **both** train and eval. A completed two-step smoke establishes execution, not learning. Check `reward`, `grad_norm`, checkpoint files and evaluation coverage. All-equal rewards can legitimately yield zero gradient. -## 4. Run locally or on Slurm +## 3. Run the same commands locally or with Slurm -On the GPU machine, install the full runtime. Keep the source, Python environment, task files and output directory accessible from the allocated node. +For a local two-GPU machine, activate the environment and run: ```bash -uv venv --python 3.12 -source .venv/bin/activate -uv pip install -r requirements.lock -python runtime/bootstrap.py -uv pip install --no-deps --no-build-isolation -e .runtime/trl -hf auth login -# Export DAYTONA_API_KEY as in the HF setup above. -python prepare.py -python -m pytest -q tests -``` - -The editable TRL installation supplies metadata needed for checkpoint model cards; it does not change the locked dependencies. `prepare.py` checks task instructions, corrected graders, train/test separation and dataset revisions. +CUDA_VISIBLE_DEVICES=0,1 python jobs/run.py train --mode opencode \ + --steps 2 --save-steps 1 --output runs/opencode-smoke -On a local two-GPU machine: - -```bash -CUDA_VISIBLE_DEVICES=0,1 python run.py pilot --model lfm --mode multi-harness \ - --run-name pilot-lfm-multi-local --limit 25 --concurrency 35 +CUDA_VISIBLE_DEVICES=0,1 python jobs/run.py eval --mode opencode \ + --checkpoint runs/opencode-smoke/checkpoint-2 --step 2 \ + --tasks 2 --concurrency 4 --output runs/opencode-smoke-eval ``` -On Slurm: +For Slurm, choose your cluster's H100/H200 partition. Submit from this folder after installing dependencies and preparing tasks: ```bash -python runtime/launch.py slurm pilot --partition YOUR_GPU_PARTITION \ - --model lfm --mode multi-harness --run-name pilot-lfm-multi-slurm \ - --timeout 12h --limit 25 --concurrency 35 +sbatch --partition=YOUR_PARTITION jobs/train.slurm train \ + --mode multi_harness --steps 100 --output runs/harbor-pilot + +# After checkpoint 100 has been saved: +sbatch --partition=YOUR_PARTITION jobs/train.slurm eval \ + --mode multi_harness --checkpoint runs/harbor-pilot/checkpoint-100 \ + --step 100 --tasks 25 --concurrency 35 --output runs/harbor-pilot-eval ``` -Add `--submit` to submit the printed `sbatch` command. It requests one node, two GPUs, 16 CPUs, 192 GB RAM and the time limit you specified. Configure your cluster account/QoS as usual. Use `squeue -u "$USER"` and the printed log path to follow the job. No Slurm job is needed when you use HF Jobs. +One allocation owns one trainer and one inference engine. Evaluation gets a **different allocation**, with two inference replicas and no trainer. Shared sandbox quotas and bucket bandwidth still need headroom. The launcher chooses unused local service ports and records them in `services.json`. Keep each allocation on its own assigned GPUs and use a different output directory. -## 5. Resume or scale up +## 4. Compare baseline with checkpoint 100 -To resume a pilot, keep its name, configuration and evaluation limit. Point to the last complete training checkpoint: +Use the same 25 fixed test tasks and interfaces for both measurements. Start an evaluation job without `--checkpoint` for the base model, then train for 100 updates, then evaluate `checkpoint-100`. ```bash -python runtime/launch.py hf pilot --model lfm --mode multi-harness \ - --run-name pilot-lfm-multi-v1 --bucket "$HF_BUCKET" \ - --flavor a100x4 --timeout 12h --limit 25 --concurrency 35 \ - --resume /outputs/pilot-lfm-multi-v1/train/checkpoint-50 -``` - -Keep `--preflight-smoke` if the original pilot used it. Completed phases and graded eval pairs are retained. For Slurm, use the checkpoint's absolute shared-filesystem path. Async resume restores committed group IDs; unconsumed tails of partially committed groups are recorded and abandoned. A pilot that fails before saving a checkpoint needs a fresh training attempt. +python jobs/hf_job.py eval --mode multi_harness --tasks 25 \ + --name lfm-baseline --bucket "$RUN_BUCKET" --submit -For a longer run, use `train` instead of `pilot`. The default ceiling is 1,000 updates, with saves every 50 and evaluations every 100 plus the final checkpoint. A separate CPU watcher launches evaluation GPU jobs: +python jobs/hf_job.py train --mode multi_harness --steps 100 \ + --name lfm-harbor-100 --bucket "$RUN_BUCKET" --submit -```bash -python runtime/launch.py hf train --model lfm --mode multi-harness \ - --run-name lfm-multi-full-v1 --bucket "$HF_BUCKET" --timeout 24h -python runtime/launch.py hf watch --run-name eval-watcher \ - --bucket "$HF_BUCKET" --concurrency 35 --max-active-evals 1 --timeout 24h +# Submit this only after checkpoint-100 has finished saving. +python jobs/hf_job.py eval --mode multi_harness --tasks 25 \ + --checkpoint /outputs/lfm-harbor-100/checkpoint-100 --step 100 \ + --name lfm-harbor-100-eval --bucket "$RUN_BUCKET" --submit ``` -Both are dry runs until `--submit` is added. Full evaluation uses all 250 tasks. Keep the watcher alive until final evals finish. The HF pilot qualifies same-job reload; separate-job live bucket handoff is still an additional qualification item in [VALIDATION.md](VALIDATION.md). Do not run the watcher for pilots. +Both blackbox policies use four evaluation harnesses: 25 tasks means 100 pairs. Whitebox uses its native tools: 25 tasks means 25 episodes. The subset is deterministic, taken from the sorted fixed test list. It is not the older pilot's stratified subset, so generate a new baseline. Omit `--tasks` for all 250 tasks. -For Slurm, run `python eval/watch.py --root runs --backend slurm --partition YOUR_GPU_PARTITION --concurrency 35 --max-active 1 --submit` on a persistent process. Failed evaluations remain in its ledger for investigation; it does not resubmit forever. +For a longer run, omit `--steps 100`. Checkpoints are saved every 50 updates. Submit independent evaluations at steps 100, 200, … using the corresponding path and `--step`. The trainer does not wait for them. This tutorial keeps submission explicit rather than claiming an untested automatic checkpoint watcher. -## Troubleshooting +## 5. Read the outputs -| Symptom | First check | +| Output | Contents | |---|---| -| `SCHEDULING` | Hardware allocation or image pull; training has not started | -| Job exits during startup | HF logs, then phase `vllm.log` and `environment.log` | -| Rollouts are ungraded | Sandbox/provider limits, capture tunnel and verifier logs; keep errors separate from wrong answers | -| Reward and gradient remain zero | Per-rollout grades and group contrast; do not assume the optimizer is broken | -| Existing-output error | Use a new run name, or resume a complete checkpoint from the same run | -| Whitebox model-card save fails | Confirm the pinned TRL checkout was installed as above | -| New GPU/kernel combination fails | Run a small smoke first; avoid switching to attention that loses packed-sequence boundaries | +| `train.log` / `eval.log` | Trainer or evaluator output | +| `vllm.log`, `harbor.log` | Inference and environment service logs | +| `training_config.json` | Resolved TRL hyperparameters | +| `task_names.json` | Selected training task identities | +| `checkpoint-50/`, `checkpoint-100/`, … | Standard TRL/Transformers training checkpoints | +| `final/` | Final model weights and tokenizer | +| `trackio/` | Local Trackio database | +| Eval `pairs/*.json` | One result per task/harness, including ungraded errors | +| Eval `summary.json` | Coverage, observed pass@1, reward, tool/token usage, harness and difficulty breakdowns | + +Add `--space-id your-org/your-trackio-space` for online Trackio too. Evaluation uses a separate named run and records the checkpoint identity. `pass_at_1_observed` excludes ungraded pairs; always report it together with coverage. A partial evaluation exits with status 2. Rerun the same eval command and output directory to retry missing/ungraded pairs. Graded successes **and graded failures** are retained. + +For direct experimentation, start the services once with the settings in [jobs/serve_model.py](jobs/serve_model.py) and [jobs/run.py](jobs/run.py), then run a training file directly. Native OpenCode additionally needs `SANDBOX_VLLM_URL` and `SANDBOX_VLLM_KEY`: the launcher creates an authenticated inference-only tunnel. Harbor uses OpenEnv's capture tunnel. Remote sandboxes cannot reach your machine's localhost directly. -The code fixes the data protocol and supports two pinned models and Daytona sandboxes. Adding a different dataset, model or provider requires adapting the task/template/backend boundary and repeating qualification. It is not a drop-in recipe for arbitrary models. +## What differs from the archive? -References: [HF Jobs](https://huggingface.co/docs/huggingface_hub/guides/jobs), [bucket volumes](https://huggingface.co/docs/hub/jobs-configuration), [Daytona](https://www.daytona.io/docs/). +The three training scripts use public trainers. They do not import custom trainer classes, download archived modules, replace installed source text, or assume that a worker seed is a task index. Training uses the same task lists, learning rate, reward formula, sampling and principal batch settings, but upstream async scheduling/row packing differs from the old custom curriculum. Exact resume, atomic rollout weighting and long-run automatic evaluation need separate qualification before claiming an identical experiment. diff --git a/05-multi-harness-rl/RESULTS.md b/05-multi-harness-rl/RESULTS.md index 50f8e4c5..5de9c042 100644 --- a/05-multi-harness-rl/RESULTS.md +++ b/05-multi-harness-rl/RESULTS.md @@ -1,10 +1,10 @@ -# Current results +# Historical results -These are the checks and pilot results for the non-thinking recipe. Two blackbox pilots completed 100 updates and evaluation; the whitebox pilot stopped at step 31 after an HTTP read timeout. Those pilots used the previous runtime pins. The merged-contract smoke qualification is recorded separately in [VALIDATION.md](VALIDATION.md). Earlier experiment scores are kept on the [archived branch](https://github.com/adithya-s-k/FineEnvs/tree/archive/data-agent-experiments-20260930/04-data-agent). +These are checks and pilot results from the earlier pinned recipe at `4d9c040`, before the tutorial rewrite. They do not qualify the current training scripts. Two blackbox pilots completed 100 updates and evaluation; the whitebox pilot stopped at step 31 after an HTTP read timeout. Those pilots used the previous runtime pins. The merged-contract smoke qualification is recorded separately in [VALIDATION.md](VALIDATION.md). Earlier experiment scores are kept on the [archived branch](https://github.com/adithya-s-k/FineEnvs/tree/archive/data-agent-experiments-20260930/04-data-agent). ## Completed checks -The merged-contract HF smokes completed two updates and checkpoint reload in all three modes. Reload evaluation graded native OpenCode 4/4 pairs, Harbor 4/4 pairs and whitebox 1/1 task. [Validation](VALIDATION.md#merged-contract-update-30-september) records pins and evidence. The qualification table below predates that migration. +The merged-contract HF smokes completed two updates and checkpoint reload in all three modes. Reload evaluation graded native OpenCode 4/4 pairs, Harbor 4/4 pairs and whitebox 1/1 task. [Earlier validation](https://github.com/adithya-s-k/FineEnvs/blob/4d9c040a28695c484de75bc23d6a70a99beda258/05-multi-harness-rl/VALIDATION.md) records pins and evidence. The qualification table below predates that migration. | Check | Result | Evidence | |---|---|---| diff --git a/05-multi-harness-rl/VALIDATION.md b/05-multi-harness-rl/VALIDATION.md index c87d8dd6..ac5e986f 100644 --- a/05-multi-harness-rl/VALIDATION.md +++ b/05-multi-harness-rl/VALIDATION.md @@ -1,99 +1,39 @@ -# Validation +# Validation of the tutorial rewrite -This page records what we tested for the pinned, non-thinking recipe. [RESULTS.md](RESULTS.md) has the pilot scores and progress; [REPRODUCE.md](REPRODUCE.md) has the commands. +The tutorial now uses public TRL trainers and OpenEnv producers. It does not download the archived experiment runtime or patch installed libraries. -## Merged contract update (30 September) +## Checked locally -The recipe now pins merged [OpenEnv #1280](https://github.com/huggingface/OpenEnv/pull/1280) and the updated [TRL #6947](https://github.com/huggingface/trl/pull/6947). The previous GPU results below used the earlier pins and are retained as historical evidence. +- 29 tests passed with the installed runtime: 10 data/launch checks and 19 contract/tokenizer checks. +- All 1,000 training and 250 test tasks were re-prepared from the local copies, checking instruction and grader hashes. +- A fresh dependency resolution succeeded with upstream TRL, OpenEnv and Transformers main. This was resolution, not a clean GPU installation. +- HF train/eval submission plans, Python compilation, shell syntax, Ruff, documentation links and the generated repository index passed. -- 43 CPU checks cover the updated recipe, including native partial/zero masks, invalid captures, correctness plus efficiency through the typed TRL worker, and the whitebox update budget against TRL's sampler. -- The TRL integration passed 51 contract/reconciliation checks against the merged producer. -- Fresh-clone bootstrap and repeat-bootstrap handling are tested. User edits are preserved; only the recipe's known attention patch is removed before updating TRL. -- Fresh HF A100 training and checkpoint-reload evaluation completed in all three modes, as recorded below. -Each smoke used non-thinking LFM2.5-2.6B, two optimizer updates and two rollouts per group. Both checkpoints were saved; evaluation restarted inference from checkpoint 2. Blackbox evaluation used one fixed task across four harnesses; whitebox used one native task. +The CPU checks exercise the fixed task split, reward gating, native tool counts, task identity independent of worker seeds, sandbox cleanup, tool schemas, and both non-thinking tokenizer templates. Contract checks pass partial and zero masks through the real OpenEnv producer and TRL consumer. -| Mode | Updates / checkpoints | Reload eval graded | Correct | Nonzero gradient? | Evidence | -|---|---|---|---|---|---| -| Native OpenCode | 2 / 2 | 4/4 | 2/4 | No reward contrast | [HF smoke](https://huggingface.co/jobs/FineEnvs/6abd47c0fbc85ba68235c27c) | -| Harbor multi-harness | 2 / 2 | 4/4 | 1/4 | Yes, norm 5.47 at step 1 | [HF smoke](https://huggingface.co/jobs/FineEnvs/6abd47bdfbc85ba68235c27a) | -| Whitebox SETA | 2 / 2 | 1/1 | 1/1 | No reward contrast | [HF smoke](https://huggingface.co/jobs/FineEnvs/6abd47c3fbc85ba68235c27e) | +```bash +python -m pytest tests -q # Lightweight checks, no model/runtime installation +python -m pytest tests -q --contract # Also exercise installed TRL/OpenEnv and tokenizers +python check_setup.py # API prerequisites +``` -These tiny samples validate execution, capture, saving and reload, not learning quality. Blackbox evaluations passed token/mask checks. A correct Harbor training rollout with six verified tool calls received 1.0714; failed rollouts received zero. Trainer reward averages are over training rows and must not be read as rollout pass@1. +Contract validation used merged OpenEnv `86a180ede21e044f7929b9a7783ad83aa67d83a3` and local TRL PR #6947 at `910138e9392dda75d3a9766e7c494d44f1d82723`. That TRL branch includes the public main trainer plus the pending typed consumer. This is not a claim that the consumer is already on main. The setup script fails clearly until main contains it. -The jobs ran OpenEnv `86a180ed` and TRL `a39476c6`. The current TRL pin has identical experimental trainer code; later changes cover example warmup, optional tests, documentation and an upstream CI fix. The first fresh-container attempt exposed a bootstrap restore bug before training; it was fixed and regression-tested before these replacement jobs. +## Still required before a long run -Artifacts are in bucket `FineEnvs/data-agent-daytona-artifacts`, under `merged-contract-{opencode,multi-harness,whitebox}-v2`. Each has `metrics.jsonl`, checkpoints, local Trackio logs and `reload-eval/eval/summary.json`. Bucket access requires permission. +- Resolve and install current upstream dependencies in a clean GPU job after TRL #6947 merges. +- For each model/mode, complete the documented two-update smoke and reload checkpoint 2 for evaluation. +- Verify LFM packed logits agree with separate sequences on the GPU. The async scripts perform this check and abort on a mismatch; a kernel that ignores sequence boundaries is not acceptable. +- Check nonzero learning signal over a larger sample, and complete the fresh baseline/checkpoint-100 comparison. +- Validate simultaneous training and evaluation against the available sandbox quota. Evaluation uses separate GPUs, but sandbox and storage capacity are shared. -Execution status, typed execution limits and structured tool outcomes in OpenEnv remain follow-up API work from the TRL review. The recipe uses explicit per-session turn/time budgets and verified native tool counts; it does not claim the broader session API is complete. +Native `opencode_env` is available on current OpenEnv main but is marked deprecated upstream. This tutorial retains it because the comparison explicitly calls for native OpenCode. Harbor-only OpenCode would be a different experiment. -## Earlier automated checks +The public native API bounds time and tokens per call; it does not expose Harbor's strict turn limit. Upstream async packing can drop rows longer than 40,960 tokens and can give one rollout multiple training rows. The tutorial does not claim the archive's custom scheduling, exact resume or rollout weighting guarantees. -- 29 tests passed with the downloaded runtime and prepared tasks. -- A clean CPU-only checkout passed 25 tests. Four integration tests skip until the runtime and tasks are downloaded. -- HF submission plans and `run.py plan` work without PyTorch installed. -- Shell syntax, Python compilation, documentation links and the repository index passed. +## Earlier evidence -Tests cover task identity, train/test separation, reward calculation, token/logprob alignment, partial masks, checkpoint integrity, resume, config overrides and pilot failure handling. They also check that the uploaded source excludes credentials and local run artifacts. +[RESULTS.md](RESULTS.md) retains the historical pilot scores and article curves. The [validation record before this rewrite](https://github.com/adithya-s-k/FineEnvs/blob/4d9c040a28695c484de75bc23d6a70a99beda258/05-multi-harness-rl/VALIDATION.md) contains the old GPU smokes, source pins and artifact paths. Those jobs used custom runtime changes, including A100 support, and do not qualify the rewritten scripts. -## Local GPU tests - -Both models completed two optimizer updates and saved both checkpoints in all three modes. - -| Model | Mode | Total time | Nonzero gradient observed? | Slurm job | -|---|---|---|---|---| -| Qwen3.5-2B | Whitebox | 10m 26s | Yes | `92750` | -| Qwen3.5-2B | Native OpenCode | 8m 35s | No reward contrast in this sample | `92753` | -| Qwen3.5-2B | Harbor | 9m 07s | Yes | `92760` | -| LFM2.5-2.6B | Whitebox | 6m 37s | No reward contrast in this sample | `92759` | -| LFM2.5-2.6B | Native OpenCode | 9m 30s | No reward contrast in this sample | `92752` | -| LFM2.5-2.6B | Harbor | 13m 01s | Yes | `92751` | - -These times include startup, sandbox work, training, saving and cleanup. Smokes use two rollouts per group and low concurrency; they do not measure full-run throughput. - -Additional checks passed: - -- Whitebox token audits matched 68 engine calls per model. -- LFM checkpoint evaluation graded 8/8 task/harness pairs with valid token capture (job `92757`). -- Qwen whitebox checkpoint evaluation graded 2/2 tasks (job `92821`). -- Async resume continued from step 1 to step 2 without replaying committed group 0 (job `92822`). -- The OpenEnv-to-TRL adapter preserved the partial completion mask `[1, 0]`. - -Local evidence is kept under ignored `runs/qualification/`, including service logs, reward records, token audits and checkpoints. - -## HF Jobs tests - -| Test | Result | Evidence | -|---|---|---| -| Fresh installation | Python 3.12, locked dependencies and all 1,250 tasks prepared | [Setup job](https://huggingface.co/jobs/FineEnvs/6abcb82f031314b696343d7d) | -| Native OpenCode on A100 | Two steps; both checkpoint file inventories match the bucket | [Native smoke](https://huggingface.co/jobs/FineEnvs/6abcc3d9031314b6963440a0) | -| Harbor on A100 | Two steps; checkpoint 2 reloaded; 8/8 eval pairs passed capture checks and graded on the first attempt | [Harbor smoke](https://huggingface.co/jobs/FineEnvs/6abcc72d031314b696344167) | -| Whitebox on A100 | Two-step preflight and reload eval passed before its pilot started | [Whitebox pilot](https://huggingface.co/jobs/FineEnvs/6abcd472031314b696344457) | - -Native OpenCode took 10m 05s overall, including 335.3s in the trainer. Harbor took 21m 17s overall, including 627.9s in the trainer. Both standalone blackbox smokes had zero reward contrast. The later pilots produced nonzero gradients; their current progress is in [RESULTS.md](RESULTS.md). - -## Fixes found during testing - -- **Hidden tool calls:** the LFM reasoning parser swallowed tool-call text. Non-thinking serving now omits that parser and checks visible streaming and ordinary replies. Earlier smoke scores from that setup are superseded. -- **A100 attention:** the pinned async trainer hardcoded FlashAttention 3. A checked adapter selects FlashAttention 2 on Ampere while preserving packed-sequence boundaries. -- **Fresh HF installation:** the image contains Python 3.11. The entrypoint creates Python 3.12 and installs the pinned TRL checkout, including metadata needed for whitebox checkpoint saving. -- **Whitebox compatibility:** explicit response templates, Qwen's text context limit and an empty successful cache-reset response needed handling. -- **Shared-node startup:** job-specific ports, HF login forwarding and the typed bucket mount were tested in the live smokes. - -## Data and source versions - -The task lists contain 1,000 training tasks and 250 test tasks, with no task, notebook or instruction-hash overlap. Preparation verifies every instruction and the corrected numeric-only grader. - -- Train revision: `4719635555de1666f374d847baebb500d368e493`. -- Test revision: `b130595b579fc6026ad5762925a9a3b40e72f5bb`. -- Current OpenEnv: `86a180ede21e044f7929b9a7783ad83aa67d83a3`. Earlier GPU qualification used `4f4c85fb9038f43efc2f51858a27638277f16355`. -- Other source revisions and archive hashes: [runtime-lock.json](configs/runtime-lock.json). -- Python dependencies: [requirements.lock](requirements.lock). - -Compared with the old staged task files, 302 selected training tasks now use the JSON reward wrapper. Instructions and the corrected grader match; metadata changed. Historical results need to be checked against each run's actual data and serving configuration. - -## Still to check - -The earlier blackbox pilots completed: Harbor 17% to 32% and native OpenCode 24% to 20%, each over 100 held-out pairs. Whitebox stopped at step 31 after an HTTP read timeout. See [RESULTS.md](RESULTS.md); these results do not establish sustained learning or qualify the new pins. HF same-job reload has passed, but a separate evaluation job loading newly saved bucket checkpoints still needs testing before relying on the long-run watcher. Qwen HF execution and other hardware combinations need their own smoke. - -This PR contains the recipe, tests and evidence links. Large artifacts remain in the HF bucket or ignored local run directories. +The previous files have been preserved locally under ignored `temp/pre-tutorial-rewrite/`; they also remain in git history. Prepared data, old experiments and credentials are not included in the PR. diff --git a/05-multi-harness-rl/check_setup.py b/05-multi-harness-rl/check_setup.py new file mode 100644 index 00000000..4c028667 --- /dev/null +++ b/05-multi-harness-rl/check_setup.py @@ -0,0 +1,50 @@ +"""Check the required public APIs before requesting GPUs or creating sandboxes.""" + +import inspect +import json +from importlib.metadata import version + + +def main(): + from harbor_env.harness import HarborSession + from opencode_env.harness import OpenCodeSessionFactory + from openenv.core.harness import TrainingTrace + from transformers.models.lfm2 import modeling_lfm2 + from trl import GRPOTrainer + from trl.experimental.async_grpo import AsyncGRPOTrainer, openenv_harness + + if "fetch_training_trace" not in inspect.getsource(openenv_harness): + raise RuntimeError( + "TRL main does not yet consume TrainingTrace. Wait for huggingface/trl#6947 to merge." + ) + if "seq_idx" not in inspect.getsource(modeling_lfm2.Lfm2ShortConv): + raise RuntimeError( + "This LFM implementation lacks sequence boundaries for packed training; install Transformers main." + ) + if "environment_factory" not in inspect.signature(GRPOTrainer).parameters: + raise RuntimeError("GRPOTrainer.environment_factory is required") + assert all( + callable(api) + for api in ( + TrainingTrace, + OpenCodeSessionFactory, + HarborSession, + AsyncGRPOTrainer, + ) + ) + print( + json.dumps( + { + name: version(name) + for name in ("trl", "openenv", "transformers", "vllm") + }, + indent=2, + ) + ) + print( + "Required APIs are present. GPU training and sandbox connectivity still need a smoke test." + ) + + +if __name__ == "__main__": + main() diff --git a/05-multi-harness-rl/configs/default.json b/05-multi-harness-rl/configs/default.json deleted file mode 100644 index a6cf2185..00000000 --- a/05-multi-harness-rl/configs/default.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "model": "lfm", - "mode": "multi-harness", - "seed": 0, - "passes": 2, - "max_steps": 1000, - "learning_rate": 0.000003, - "num_generations": 8, - "max_inflight": 32, - "max_outstanding_rollouts": 16, - "max_staleness": 4, - "batch_size": 4, - "gradient_accumulation_steps": 4, - "token_budget": 40960, - "max_model_len": 131072, - "max_completion_length": 16384, - "max_output_tokens": 4096, - "temperature": 0.8, - "top_p": 1.0, - "top_k": 0, - "agent_step_limit": 17, - "agent_timeout_sec": 600, - "save_steps": 50, - "eval_steps": 100, - "eval_concurrency": 35, - "eval_dp": 2, - "efficiency_weight": 0.1, - "tool_budget": 15, - "sandbox": "daytona", - "project": "smoldataenv-rl", - "trackio_space_id": null, - "models": { - "lfm": { - "id": "LiquidAI/LFM2.5-2.6B", - "revision": "654f9463ce32b05d0429d76fe1f580b27d4c1ac0", - "tool_parser": "lfm2", - "template": "lfm-closed-thinking" - }, - "qwen": { - "id": "Qwen/Qwen3.5-2B", - "revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", - "tool_parser": "qwen3_xml", - "template": "enable_thinking_false" - } - }, - "harnesses": ["opencode", "claude-code", "codex", "mini-swe-agent"], - "harness_versions": { - "opencode": "1.18.31", - "claude-code": "2.1.270", - "codex": "0.154.0", - "mini-swe-agent": "2.4.6" - } -} diff --git a/05-multi-harness-rl/configs/runtime-lock.json b/05-multi-harness-rl/configs/runtime-lock.json deleted file mode 100644 index b6986be9..00000000 --- a/05-multi-harness-rl/configs/runtime-lock.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "archive": { - "repo": "adithya-s-k/FineEnvs", - "revision": "bff1b69310db76bf048ae26f7ba1d70134d5f2e3", - "files": { - "envs/blackbox-opencode/tasks.py": "1a99bfb65008366993ce6794fe70a762985aadec01320cb9de9039bd762af802", - "envs/blackbox-opencode/config.py": "19914b42ea832893e52b1c8671e69aefd3908df76dc8bd97498507ecb64385ad", - "envs/blackbox-opencode/curriculum.py": "d3c2d9b47a4a517d8a671ca2a46c1e0682f3c2786b79ba2d6731022f279ec2c2", - "envs/blackbox-opencode/verifier.py": "02e2c9ab9d7340fa450f75e36f1418a8acfb6d6dc7593a77fb3d1cc3dbc797c3", - "envs/blackbox-opencode/__init__.py": "0eabe01dc2ba02aed86167f3c746d9a096379dfb26b0a917b59502c8affe8ebc", - "envs/blackbox-opencode/task.py": "8e492535a8bf6fd7b13e9e00a19a4ec6c03a801246143db842b185b9cc7d8d97", - "envs/blackbox-opencode/client.py": "18cee684f7f3ae377c565bdeeabcaa6e790b366fd859efce93938ed6ea7a52d7", - "envs/blackbox-opencode/reward.py": "fed11e459a0795f624ee9a18121890f0a6aa460b9e1b75763e9d66279fde5129", - "envs/blackbox-opencode/harness.py": "2037caa67cf1a8100f1acff3bbd6d510471261c402738c0634b97beac4ed5fbe", - "envs/blackbox-opencode/grader.py": "58220f93a7d78767f9152c7db6f33a67800d694b4f15eca8e53dbca02625f457", - "envs/blackbox-opencode/rollout.py": "be6b7c787ee187a74b5dfa9af7eef1e56efedaa14aa99dc3b72132dd15e6e6bc", - "envs/blackbox-opencode/models.py": "5fcc4a4b1e7d35b359d87463b83e4f35263733384768b7cc07c924227ae8d0fd", - "envs/blackbox-opencode/server/__init__.py": "0b63c98ec0d6e82d66fa0b13523d417d87dbba0e7aeea1d786aa13a8388e7184", - "envs/blackbox-opencode/server/app.py": "0e318a153dbb420e7a95e05a815466da8bdaf49f45bdeb1b9656ad2296f20a58", - "envs/blackbox-opencode/server/rollout.py": "eb961258c432798403b4bc41709b3ac8fa01c087041911022bac8adba283447b", - "envs/blackbox-opencode/server/capture.py": "97108b4a961e1c0217e04fd0f94b51de6f1a4356d232e49f1511a45bc911da49", - "envs/blackbox-opencode/server/environment.py": "95dfae62baa042742840efc67dad816e1deabcb62b568ad69204bf005ae01229", - "envs/blackbox-opencode/sandbox/hf.py": "8d2d1b3e0eba9819833f52ac714ff693bd1ae2cf467d12313a531837b5f2d795", - "envs/blackbox-opencode/sandbox/__init__.py": "399b02c8050f151072c32b59760ccc4c4c397cb129f406186cd153d974741556", - "envs/blackbox-opencode/sandbox/base.py": "2385954579f8061d34bcbc3e1a20b13658557d1dc8f872a8058aaf4670899d93", - "envs/blackbox-opencode/sandbox/e2b.py": "b30faa2628bc23a2ba3ebaa1b21ef467f3b5d8fac93a317d23bd5f4ce6c8bffa", - "envs/blackbox-opencode/sandbox/daytona.py": "9407386abc7ace49f6b6615ceb20ed8ab8e922840614d2cc863e5e211e079707", - "envs/whitebox-bash/grader.py": "4333d1526a3307a0b9b72a37199009831f7162f2d30cf3d8c7bbd656fdd7b859", - "envs/whitebox-bash/dataagent.py": "7d3b9d91e8f2f8c3679c86e10bba47c4ffcc234a9a065095fe5ee0a327d11943", - "envs/whitebox-bash/suite.py": "a0e162f042d6f6bf5efc4165e3e117d474df2723417e42abd4b3a1875502dc80", - "envs/whitebox-bash/client.py": "b3888637c79ae079c66e1a434090da89cfa5e2a90ec60b64a6c42ecf22782e7f", - "envs/whitebox-bash/__init__.py": "8f34adaf64792f79887b9a1cf72d9767db3a8425028e3566ee377050e1de6b08", - "envs/whitebox-bash/tasks.py": "465c5c721150a4f46929b8e4750c99565c165dbe25930b2ada226551b71c49ba", - "envs/whitebox-bash/models.py": "ea2a29f0c9dd0db0fc201069ed7183089b277cfd3b94bd562a7c19b6929291f8", - "envs/whitebox-bash/tools.py": "02eee5f7ce958ae48702f219859c59432929f7f7f4c99aeee8051472a2c6b35a", - "envs/whitebox-bash/server/__init__.py": "515cd1e723b755ac5281840119a4dd30fe6c63eb98b1b3a6d9b6b9b6c8978cd3", - "envs/whitebox-bash/server/environment.py": "f372126214413d7cddbc631ce6a388c23412effb5713931ee38c703a8c787836", - "envs/whitebox-bash/server/sandbox.py": "238d142a1f2ad114ddea12fb975670e5c1d39d361d80abd8983a326a58643b21", - "envs/whitebox-bash/server/app.py": "9d3cfae74f414a236a95a38844b8b2a78f28df6de2f872b06d1b6fd839d4cdd7", - "train/atomic_rollouts.py": "2c3a1918caff0d5c580f7dacaa914e6707ea7ec78365beb0411f92fb30c63616", - "train/training_audit.py": "2345477c719f15c3c8dd1b98bed0ca7d7c09c6eb3f7b00630dd8fb9de34bcb9b", - "train/checkpoint_artifacts.py": "9bc1055b9bdc1ec389fb73320c2519ab14dad6ace02e6e617b3c1d00365f06db", - "train/harness_schedule.py": "acca0746f9168c50d5e1ef1197cedba8c09126b28dc88fa34da01248cc36a0ba", - "train/multi_harness.py": "6609a24ee58ea1dfbf5ce10a7fea997e4028f82cf65cb4699d5a92a125673259", - "train/standalone_comparison.py": "bd0768a108b96129a2a5c8d80029b63a78fb8b578b96044eca19c90a50031d09", - "hf/runtime/hard_curriculum_train.py": "52c71aa1749536683901832817cd280a107ef7d66d9e7fa85efc2d0ff3dc8777", - "hf/runtime/daytona_whitebox_backend.py": "26677b370d16d8d9f8c4634d169d7ad9aff371d59c3b66583e028f85d2e8549c", - "hf/runtime/service_policy.py": "00c161987b6118c77dcde301fdd0eaeeba2708a4f386934bee1844af38c0701f", - "hf/runtime/whitebox_tito.py": "aee498fdb47f1b66e67118372e8641de79d8f01a8a6e63f9ca90cc6261f88be5", - "model_runs/lfm25/packing.py": "411c8be4ad0966f36ca5260b96d6c04fc20f4c8d7454b89cb64eda762a6df820" - } - }, - "trl": { - "repo": "adithya-s-k/trl", - "revision": "2de2cea22b1a3da717c7904722109ca6e2ecb503" - }, - "openenv": { - "repo": "huggingface/OpenEnv", - "revision": "86a180ede21e044f7929b9a7783ad83aa67d83a3" - } -} diff --git a/05-multi-harness-rl/envs/__init__.py b/05-multi-harness-rl/envs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/05-multi-harness-rl/envs/daytona.py b/05-multi-harness-rl/envs/daytona.py new file mode 100644 index 00000000..f0b36d96 --- /dev/null +++ b/05-multi-harness-rl/envs/daytona.py @@ -0,0 +1,144 @@ +"""Daytona implementation of the standalone OpenCode sandbox protocol.""" + +from __future__ import annotations + +import math +import os +import shlex +import time +import uuid +from functools import lru_cache + +from opencode_env.sandbox.base import ExecResult + + +@lru_cache(maxsize=1) +def client(): + from daytona import Daytona + + return Daytona() + + +class DaytonaBgJob: + def __init__(self, handle, directory, pid): + self.handle, self.directory, self._pid = handle, directory, pid + + @property + def pid(self): + return self._pid + + def wait(self, timeout=None): + deadline = time.monotonic() + timeout if timeout is not None else float("inf") + while time.monotonic() < deadline: + if self.handle.exists(self.directory + "/exit"): + return int(self.handle.read_text(self.directory + "/exit").strip()) + time.sleep(min(0.5, max(0, deadline - time.monotonic()))) + raise TimeoutError("Background command exceeded its deadline") + + def kill(self): + self.handle.exec(f"kill -TERM -- -{self.pid} 2>/dev/null || true", timeout=10) + + +class DaytonaSandboxHandle: + def __init__(self, sandbox): + self._sandbox = sandbox + self._deleted = False + + @property + def sandbox_id(self): + return self._sandbox.id + + def exec(self, cmd, *, envs=None, cwd=None, timeout=60): + # The SDK's exec is not a shell. Quote the complete script once and let bash + # interpret it; neither user text nor environment values are interpolated. + result = self._sandbox.process.exec( + "bash -lc " + shlex.quote(cmd), + cwd=cwd, + env=envs, + timeout=max(1, math.ceil(timeout)) if timeout is not None else None, + ) + code = result.exit_code + return ExecResult(124 if code is None else int(code), result.result or "", "") + + def write_text(self, path, content): + parent = path.rsplit("/", 1)[0] or "." + result = self.exec("mkdir -p " + shlex.quote(parent)) + if result.exit_code: + raise RuntimeError("Could not create sandbox directory") + self._sandbox.fs.upload_file(content.encode(), path) + + def read_text(self, path): + return self._sandbox.fs.download_file(path).decode() + + def exists(self, path): + return self.exec("test -e " + shlex.quote(path), timeout=15).exit_code == 0 + + def start_bg(self, cmd, *, envs=None, cwd=None): + directory = "/tmp/openenv-process-" + uuid.uuid4().hex + self.write_text( + directory + "/run.sh", + "#!/bin/bash\n(\n" + + cmd + + "\n)" + + "\ncode=$?\nprintf '%s' \"$code\" > " + + shlex.quote(directory + "/exit") + + "\n", + ) + result = self.exec( + "setsid bash " + + shlex.quote(directory + "/run.sh") + + " > " + + shlex.quote(directory + "/output") + + " 2>&1 < /dev/null & echo $!", + envs=envs, + cwd=cwd, + timeout=15, + ) + if result.exit_code: + raise RuntimeError("Could not launch background command") + return DaytonaBgJob( + self, directory, int(result.stdout.strip().splitlines()[-1]) + ) + + def kill(self): + if not self._deleted: + client().delete(self._sandbox, timeout=60, wait=True) + self._deleted = True + + +class DaytonaSandboxBackend: + def __init__(self, *, image, snapshot=None): + self.image, self.snapshot = ( + image, + snapshot or os.environ.get("DATA_AGENT_DAYTONA_SNAPSHOT"), + ) + + def create(self, *, timeout_s=900, envs=None, metadata=None): + from daytona import ( + CreateSandboxFromImageParams, + CreateSandboxFromSnapshotParams, + Resources, + ) + + options = { + "language": "python", + "os_user": "root", + "env_vars": envs, + "labels": { + **(metadata or {}), + "openenv_component": "blackbox-opencode", + "openenv_owner": os.environ.get("RUN_OWNER", "local"), + }, + "public": False, + "auto_stop_interval": max(30, math.ceil(timeout_s / 60)), + "auto_delete_interval": 0, + } + if self.snapshot: + params = CreateSandboxFromSnapshotParams(snapshot=self.snapshot, **options) + else: + params = CreateSandboxFromImageParams( + image=self.image, + resources=Resources(cpu=1, memory=4, disk=5), + **options, + ) + return DaytonaSandboxHandle(client().create(params, timeout=300)) diff --git a/05-multi-harness-rl/envs/tasks.py b/05-multi-harness-rl/envs/tasks.py new file mode 100644 index 00000000..217a4241 --- /dev/null +++ b/05-multi-harness-rl/envs/tasks.py @@ -0,0 +1,75 @@ +"""Stage public task files in a sandbox; run the held-out grader on the host.""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +import tomllib + + +def load_tasks(data, split): + root = Path(data).resolve() / "datasets" / split / "tasks" + tasks = [] + for folder in sorted(root.iterdir()): + spec = tomllib.loads((folder / "task.toml").read_text()) + tasks.append( + { + "name": folder.name, + "folder": str(folder), + "instruction": (folder / "instruction.md").read_text(), + "difficulty": spec["metadata"]["difficulty_tier"], + } + ) + if not tasks or len({t["instruction"] for t in tasks}) != len(tasks): + raise ValueError("Tasks must have unique instructions; run prepare.py first") + return tasks + + +def stage_task(sandbox, folder): + folder = Path(folder) + spec = tomllib.loads((folder / "task.toml").read_text()) + # Only data-download credentials enter the sandbox, never verifier answers. + env = { + key: os.path.expandvars(value) + for key, value in spec["environment"]["env"].items() + } + if any("${" in value for value in env.values()): + raise ValueError("Set HF_TOKEN before staging task data") + sandbox.write_text( + "/opt/pull_bucket.py", (folder / "environment/pull_bucket.py").read_text() + ) + result = sandbox.exec( + "mkdir -p /workdir /home/user/input && python3 /opt/pull_bucket.py " + '&& test -n "$(ls -A /home/user/input)"', + envs=env, + timeout=300, + ) + if result.exit_code: + raise RuntimeError("Task data download failed") + + +def grade_answer(sandbox, folder): + folder = Path(folder) + spec = tomllib.loads((folder / "task.toml").read_text()) + answer = ( + sandbox.read_text("/workdir/answer.txt") + if sandbox.exists("/workdir/answer.txt") + else "" + ) + result = subprocess.run( + [sys.executable, str(folder / "tests/grader.py"), "--json"], + input=answer, + text=True, + capture_output=True, + timeout=120, + check=False, + env={**os.environ, **spec["verifier"]["env"]}, + ) + if result.returncode: + raise RuntimeError("Task grader failed") + value = json.loads(result.stdout)["correctness"] + if value not in (0, 1): + raise ValueError("Expected a binary correctness grade") + return float(value) diff --git a/05-multi-harness-rl/eval/evaluate.py b/05-multi-harness-rl/eval/evaluate.py deleted file mode 100644 index 8bc9b0ff..00000000 --- a/05-multi-harness-rl/eval/evaluate.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Pass@1 with durable per-pair results and retries only for ungraded infrastructure failures.""" -from concurrent.futures import ThreadPoolExecutor, as_completed -import json -import math -from pathlib import Path -import time -import uuid - -from recipe import digest, eval_rows, reward, summary, task_rows, write_json -from train.adapters import Factory - - -def capture_metrics(result): - turns = [t for t in result.turns if t.trainable] - if not turns: - raise ValueError("No trainable capture from the token-enabled engine") - for turn in turns: - prompt, tokens, logps = turn.prompt_token_ids, turn.completion_token_ids, turn.per_token_logps - mask = turn.loss_mask - if not prompt or not tokens or len(tokens) != len(logps) or not all(math.isfinite(x) for x in logps): - raise ValueError("Invalid captured token/logprob alignment") - if mask is None or len(mask) != len(prompt) + len(tokens) or any(mask[:len(prompt)]): - raise ValueError("Invalid full-sequence loss mask") - if set(mask) - {0, 1} or not any(mask[len(prompt):]): - raise ValueError("No valid supervised completion span") - return {"tito_pass": True, "model_calls": len(turns), - "generated_tokens": sum(len(t.completion_token_ids) for t in turns), - "prompt_tokens": sum(len(t.prompt_token_ids) for t in turns)} - - -def blackbox_episode(factory, group_id): - group = factory.groups[group_id] - session = factory.create(factory.rows()[group_id]["prompt"], seed=group_id, episode_id=uuid.uuid4().hex) - try: - session.wait_for_completion(timeout_s=factory.cfg["agent_timeout_sec"] + 600) - session.verify([]) - record = json.loads((Path(factory.cfg["output"]) / "rollouts" / f'{session.evidence["episode_id"]}.json').read_text()) - return {**record, **capture_metrics(session.result)} - finally: - session.close() - - -def whitebox_episode(cfg, server, vllm, index): - from openai import OpenAI - from transformers.utils import get_json_schema - from whitebox_bash import white_box_bash_env - import inspect - from train.whitebox import SYSTEM - - env = white_box_bash_env(server, split="test", toolsets="bash,seta", step_limit=cfg["agent_step_limit"])() - client = OpenAI(base_url=vllm.rstrip("/") + "/v1", api_key="unused", timeout=cfg["agent_timeout_sec"]) - calls, generated, inputs = 0, 0, 0 - try: - text = env.reset(split="test", index=index) - tools = [get_json_schema(m) for name, m in inspect.getmembers(env, inspect.ismethod) - if not name.startswith("_") and name not in {"reset", "get_reward"}] - messages = [{"role": "system", "content": SYSTEM}, {"role": "user", "content": text}] - for _ in range(cfg["agent_step_limit"]): - response = client.chat.completions.create(model=cfg["profile"]["id"], messages=messages, tools=tools, - temperature=cfg["temperature"], top_p=1.0, max_tokens=cfg["max_output_tokens"], - extra_body={"chat_template_kwargs": {"enable_thinking": False, "preserve_thinking": True}}) - generated += response.usage.completion_tokens - inputs += response.usage.prompt_tokens - msg = response.choices[0].message - messages.append(msg.model_dump(exclude_none=True)) - if not msg.tool_calls: - break - submitted = False - for call in msg.tool_calls: - allowed = {t["function"]["name"] for t in tools} - if call.function.name not in allowed: - raise ValueError("Model called an unexposed tool") - value = str(getattr(env, call.function.name)(**json.loads(call.function.arguments))) - calls += call.function.name != "submit_solution" - submitted |= call.function.name == "submit_solution" - messages.append({"role": "tool", "tool_call_id": call.id, "content": value}) - if submitted: - break - correctness = env.get_reward() - if not math.isfinite(correctness): - raise ValueError("Ungraded whitebox episode") - correctness = float(correctness >= 1) - shaped = reward(correctness, calls, verified=True, weight=cfg["efficiency_weight"], budget=cfg["tool_budget"]) - return {"correctness": correctness, "tool_calls": calls, "tool_count_verified": True, "reward": shaped, - "generated_tokens": generated, "prompt_tokens": inputs} - finally: - try: - if env._session is not None: - env.get_reward() - finally: - env._mcp.close() - client.close() - - -def evaluate(cfg, data, server, vllm, *, limit=None, checkpoint="baseline", max_attempts=3): - out = Path(cfg["output"]) / "eval" - out.mkdir(parents=True, exist_ok=True) - tasks = sorted(task_rows("test"), key=lambda row: row["name"]) - selected = eval_rows(tasks, limit, stratified=cfg.get("eval_selection") == "stratified", seed=cfg["seed"]) - indices = {row["name"]: i for i, row in enumerate(tasks)} - harnesses = ["whitebox"] if cfg["mode"] == "whitebox" else cfg["harnesses"] - groups = [{"task_name": row["name"], "task_index": indices[row["name"]], "harness": h} - for row in selected for h in harnesses] - identity = {"model": cfg["profile"], "checkpoint": str(checkpoint), "tasks": selected, - "harnesses": harnesses, "template": cfg["profile"]["template"], "config": cfg} - signature = digest(identity) - identity_path = out / "identity.json" - if identity_path.exists() and json.loads(identity_path.read_text())["sha256"] != signature: - raise ValueError("Evaluation output belongs to a different checkpoint or protocol") - write_json(identity_path, {"sha256": signature, **identity}) - # Both blackbox policies use the same four Harbor evaluation harnesses. - factory = Factory({**cfg, "mode": "multi-harness"}, data, server, vllm, groups, - Path(cfg["output"]) / "trials", split="test", - sampling={k: cfg[k] for k in ("temperature", "top_p", "top_k")}) if cfg["mode"] != "whitebox" else None - - def one(item): - group_id, group = item - row = tasks[group["task_index"]] - path = out / "pairs" / f'{row["name"]}--{group["harness"]}.json' - if path.exists(): - prior = json.loads(path.read_text()) - if prior.get("correctness") in (0, 1) and not prior.get("error"): - return prior - for attempt in range(max_attempts): - started = time.time() - try: - result = (blackbox_episode(factory, group_id) if factory else - whitebox_episode(cfg, server, vllm, group["task_index"])) - if result.get("correctness") not in (0, 1): - raise ValueError("Verifier did not produce a binary grade") - record = {**group, "difficulty": row["difficulty"], **result, "attempt": attempt + 1, - "elapsed_sec": time.time() - started, "error": None} - except Exception as exc: - record = {**group, "difficulty": row["difficulty"], "correctness": None, - "error": type(exc).__name__, "attempt": attempt + 1} - write_json(path, record) - if record["correctness"] is not None: - return record - return record - - records = [] - with ThreadPoolExecutor(max_workers=cfg["eval_concurrency"]) as pool: - for future in as_completed([pool.submit(one, pair) for pair in enumerate(groups)]): - records.append(future.result()) - write_json(out / "progress.json", summary(records, len(groups))) - report = summary(records, len(groups)) - report["harnesses"] = {h: summary([r for r in records if r["harness"] == h], len(selected)) for h in harnesses} - report["difficulty"] = {d: summary([r for r in records if r["difficulty"] == d], - sum(t["difficulty"] == d for t in selected) * len(harnesses)) for d in ("easy", "medium", "hard")} - report["missing"] = [{"task_name": r["task_name"], "harness": r["harness"], "error": r["error"]} - for r in records if r["correctness"] is None] - write_json(out / "summary.json", report) - import trackio - run = trackio.init(project=cfg["project"], name=cfg["run_name"] + "-eval", - space_id=cfg["trackio_space_id"], config={"protocol_sha256": signature, "checkpoint": str(checkpoint)}) - step = 0 - if checkpoint != "baseline": - step = json.loads((Path(checkpoint) / "trainer_state.json").read_text())["global_step"] - metrics = {"eval/coverage": report["coverage"], "eval/graded": report["graded"]} - if report["pass_at_1"] is not None: - metrics["eval/pass_at_1_observed"] = report["pass_at_1"] - for harness, values in report["harnesses"].items(): - for key in ("pass_at_1", "coverage", "mean_tool_calls", "mean_generated_tokens"): - if values[key] is not None: - metrics[f"eval/{harness}/{key}"] = values[key] - run.log(metrics, step=step) - run.finish() - print(json.dumps(report, indent=2)) - return report diff --git a/05-multi-harness-rl/eval/watch.py b/05-multi-harness-rl/eval/watch.py deleted file mode 100644 index 692c9830..00000000 --- a/05-multi-harness-rl/eval/watch.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Hash finished checkpoints and submit evaluations on separate GPU allocations.""" -import argparse -import fcntl -import json -from pathlib import Path -import subprocess -import sys -import time - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT)) -from recipe import write_json -from runtime.checkpoints import make_ready - - -def scan(root): - for request in sorted(Path(root).glob("*/checkpoint-*/eval.request.json")): - checkpoint = request.parent - if not (checkpoint / "checkpoint.ready.json").exists(): - try: - make_ready(checkpoint) - except (OSError, ValueError) as exc: - print(json.dumps({"checkpoint": str(checkpoint), "status": "waiting_for_complete_files", - "error": type(exc).__name__}), flush=True) - continue - if json.loads(request.read_text())["evaluate"]: - yield checkpoint - - -def main(): - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--root", type=Path, required=True) - p.add_argument("--backend", choices=["hf", "slurm"], required=True) - p.add_argument("--bucket") - p.add_argument("--namespace", help="HF Jobs owner; defaults to the bucket owner") - p.add_argument("--partition") - p.add_argument("--flavor", default="h200x2") - p.add_argument("--data", type=Path, default=ROOT / "prepared") - p.add_argument("--concurrency", type=int, default=35) - p.add_argument("--max-active", type=int, default=1) - p.add_argument("--once", action="store_true") - p.add_argument("--submit", action="store_true") - args = p.parse_args() - args.root = args.root.resolve() - if args.backend == "hf" and not args.bucket: - p.error("--bucket is required for HF Jobs") - if args.backend == "hf": - args.namespace = args.namespace or args.bucket.split("/", 1)[0] - if args.backend == "slurm" and not args.partition: - p.error("--partition is required for Slurm") - if args.max_active < 1 or args.concurrency < 1: - p.error("Concurrency and max-active must be positive") - args.root.mkdir(parents=True, exist_ok=True) - state_path = args.root / "eval-watcher.json" - with (args.root / "eval-watcher.lock").open("w") as lock: - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - while True: - ledger = json.loads(state_path.read_text()) if state_path.exists() else {} - for key, entry in ledger.items(): - if entry["status"] != "active": - continue - if args.backend == "hf": - from huggingface_hub import inspect_job - stage = inspect_job(job_id=entry["id"], namespace=args.namespace).status.stage - if stage in {"COMPLETED", "ERROR", "CANCELED", "DELETED"}: - entry["status"] = "done" if stage == "COMPLETED" else "failed" - else: - status = subprocess.check_output(["sacct", "-j", entry["id"], "--noheader", "--format=State", "--parsable2"], text=True).splitlines() - if status and status[0] not in {"RUNNING", "PENDING", "CONFIGURING", "COMPLETING"}: - entry["status"] = "done" if status[0] == "COMPLETED" else "failed" - active = sum(e["status"] == "active" for e in ledger.values()) - for checkpoint in scan(args.root): - key = str(checkpoint.relative_to(args.root)) - if key in ledger or active >= args.max_active: - continue - cfg = json.loads((checkpoint / "recipe.json").read_text()) - name = f'{checkpoint.parent.name}-eval-{checkpoint.name}' - cmd = [sys.executable, str(ROOT / "runtime/launch.py"), args.backend, "eval", - "--mode", cfg["mode"], "--model", cfg["model"], "--run-name", name, - "--checkpoint", str(checkpoint), "--output-root", str(args.root), - "--config", str(checkpoint / "recipe.json"), - "--data", str(args.data), "--concurrency", str(args.concurrency)] - if args.backend == "hf": - cmd += ["--bucket", args.bucket, "--namespace", args.namespace, "--flavor", args.flavor] - else: - cmd += ["--partition", args.partition] - print(json.dumps({"checkpoint": key, "command": cmd}), flush=True) - if args.submit: - lines = subprocess.check_output(cmd + ["--submit"], text=True).strip().splitlines() - job_id = json.loads(lines[-1])["id"] if args.backend == "hf" else lines[-1].split(";")[0] - ledger[key] = {"id": job_id, "status": "active", "submitted_at": time.time()} - write_json(state_path, ledger) - active += 1 - write_json(state_path, ledger) - if args.once: - break - time.sleep(60) - - -if __name__ == "__main__": - main() diff --git a/05-multi-harness-rl/evaluate.py b/05-multi-harness-rl/evaluate.py new file mode 100644 index 00000000..7c6b53a2 --- /dev/null +++ b/05-multi-harness-rl/evaluate.py @@ -0,0 +1,283 @@ +"""Pass@1 on fixed held-out tasks; resume only missing or ungraded pairs.""" + +import argparse +import hashlib +import inspect +import json +import math +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + +from envs.tasks import load_tasks + +HARNESSES = ("opencode", "claude-code", "codex", "mini-swe-agent") + + +def write_json(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps(value, indent=2) + "\n") + temporary.replace(path) + + +def harbor_episode(args, index, task, harness): + from harbor_env import HarborEnv + from harbor_env.harness import HarborSession + + session = HarborSession( + env=HarborEnv(args.server, message_timeout_s=1800), + owns_env=True, + split=str(Path(args.data).resolve() / "datasets/test"), + task_index=index, + instruction=task["instruction"], + harness=harness, + sandbox="daytona", + llm_url=args.vllm_url, + model=args.model, + reward_key="correctness", + sampling={"temperature": 0.8, "top_p": 1.0, "top_k": -1}, + agent_step_limit=17, + agent_timeout_sec=600, + ) + try: + session.wait_for_completion(timeout_s=1200) + correctness = session.verify([]).env_reward + if correctness not in (0, 1): + raise RuntimeError("No verifier grade") + trace = ( + session.fetch_training_trace() + ) # Checks engine tokens, logprobs and masks. + path = Path(args.trials) / session.result.trial_name / "agent/trajectory.json" + try: + atif = json.loads(path.read_text()) + if not atif["schema_version"].startswith("ATIF-"): + raise ValueError("Unknown trajectory format") + ids = [ + c["tool_call_id"] + for s in atif["steps"] + if s.get("source") == "agent" + for c in s.get("tool_calls", []) or [] + ] + calls = len(ids) if all(ids) and len(set(ids)) == len(ids) else None + except (OSError, ValueError, KeyError, TypeError): + calls = None + return { + "correctness": correctness, + "tool_calls": calls, + "generated_tokens": sum(len(t.completion_token_ids) for t in trace.turns), + "prompt_tokens": sum(len(t.prompt_token_ids) for t in trace.turns), + "model_calls": len(trace.turns), + } + finally: + session.close() + + +def whitebox_episode(args, index, task, harness): + from openai import OpenAI + from transformers.utils import get_json_schema + + from train_whitebox import SYSTEM, BashEnvironment + + environment = BashEnvironment() + messages = [{"role": "system", "content": SYSTEM}] + generated, prompt_tokens = 0, 0 + try: + messages.append( + {"role": "user", "content": environment.reset(folder=task["folder"])} + ) + tools = [ + get_json_schema(method) + for name, method in inspect.getmembers(environment, inspect.ismethod) + if not name.startswith("_") and name not in {"reset", "get_reward"} + ] + allowed = {tool["function"]["name"] for tool in tools} + with OpenAI( + base_url=args.vllm_url.rstrip("/") + "/v1", api_key="unused", timeout=600 + ) as client: + for _ in range(17): + response = client.chat.completions.create( + model=args.model, + messages=messages, + tools=tools, + temperature=0.8, + top_p=1.0, + max_tokens=4096, + extra_body={"chat_template_kwargs": {"enable_thinking": False}}, + ) + message = response.choices[0].message + generated += response.usage.completion_tokens + prompt_tokens += response.usage.prompt_tokens + messages.append(message.model_dump(exclude_none=True)) + if not message.tool_calls: + break + for call in message.tool_calls: + if call.function.name not in allowed: + value = "Unknown tool" + else: + try: + value = getattr(environment, call.function.name)( + **json.loads(call.function.arguments) + ) + except (TypeError, ValueError, RuntimeError) as exc: + value = type(exc).__name__ + messages.append( + {"role": "tool", "tool_call_id": call.id, "content": str(value)} + ) + if environment._submitted: + break + value = environment.get_reward() + if not math.isfinite(value): + raise RuntimeError("No verifier grade") + return { + "correctness": environment._correctness, + "tool_calls": environment._calls, + "generated_tokens": generated, + "prompt_tokens": prompt_tokens, + } + finally: + environment._close() + + +def summarize(rows, expected): + graded = [row for row in rows if row.get("correctness") in (0, 1)] + + def mean(key): + values = [row[key] for row in graded if row.get(key) is not None] + return sum(values) / len(values) if values else None + + return { + "expected": expected, + "graded": len(graded), + "coverage": len(graded) / expected, + "pass_at_1_observed": mean("correctness"), + "combined_reward": mean("reward"), + "mean_tool_calls": mean("tool_calls"), + "tool_count_coverage": sum(r.get("tool_calls") is not None for r in graded), + "mean_generated_tokens": mean("generated_tokens"), + "mean_prompt_tokens": mean("prompt_tokens"), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--model", default="LiquidAI/LFM2.5-2.6B", help="The served model name" + ) + parser.add_argument( + "--checkpoint", + required=True, + help="Immutable model revision or checkpoint identity", + ) + parser.add_argument("--step", type=int, default=0) + parser.add_argument("--mode", choices=["blackbox", "whitebox"], default="blackbox") + parser.add_argument("--data", default="prepared") + parser.add_argument("--server", default="http://127.0.0.1:8200") + parser.add_argument("--vllm-url", default="http://127.0.0.1:8000") + parser.add_argument("--trials", default="runs/trials") + parser.add_argument("--output", required=True) + parser.add_argument("--tasks", type=int, default=250) + parser.add_argument("--concurrency", type=int, default=35) + parser.add_argument("--space-id", default=None) + args = parser.parse_args() + if not 1 <= args.tasks <= 250 or args.concurrency < 1: + parser.error("Use 1 to 250 tasks and positive concurrency") + tasks = load_tasks(args.data, "test") + # Stable subset: every model and checkpoint sees the same task/harness pairs. + selected = list(enumerate(tasks))[: args.tasks] + harnesses = ["whitebox"] if args.mode == "whitebox" else HARNESSES + output = Path(args.output) + identity = { + "protocol_source_sha256": hashlib.sha256( + Path(__file__).read_bytes() + ).hexdigest(), + "checkpoint": args.checkpoint, + "model": args.model, + "harnesses": harnesses, + "task_names": [t["name"] for _, t in selected], + "temperature": 0.8, + "instruction_hash": hashlib.sha256( + "".join(t["instruction"] for _, t in selected).encode() + ).hexdigest(), + } + identity_path = output / "identity.json" + if identity_path.exists() and json.loads(identity_path.read_text()) != json.loads( + json.dumps(identity) + ): + raise ValueError("Output already belongs to a different evaluation") + write_json(identity_path, identity) + pairs = [ + (index, task, harness) for index, task in selected for harness in harnesses + ] + + def one(pair): + index, task, harness = pair + path = output / "pairs" / f"{task['name']}--{harness}.json" + if path.exists(): + previous = json.loads(path.read_text()) + if previous.get("correctness") in (0, 1): + return previous + start = time.monotonic() + row = { + "task": task["name"], + "difficulty": task["difficulty"], + "harness": harness, + } + try: + episode = whitebox_episode if args.mode == "whitebox" else harbor_episode + row.update(episode(args, index, task, harness)) + calls = row["tool_calls"] + row["reward"] = row["correctness"] * ( + 1 + 1.5 / (15 + calls) if calls else 1 + ) + except Exception as exc: # noqa: BLE001 - persist an ungraded pair for retry + row.update(correctness=None, error=type(exc).__name__) + row["seconds"] = time.monotonic() - start + write_json(path, row) + return row + + rows = [] + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + for future in as_completed([pool.submit(one, pair) for pair in pairs]): + rows.append(future.result()) + write_json(output / "progress.json", summarize(rows, len(pairs))) + summary = summarize(rows, len(pairs)) + summary["harnesses"] = { + h: summarize([r for r in rows if r["harness"] == h], args.tasks) + for h in harnesses + } + summary["difficulty"] = { + d: summarize( + [r for r in rows if r["difficulty"] == d], + sum(t["difficulty"] == d for _, t in selected) * len(harnesses), + ) + for d in {t["difficulty"] for _, t in selected} + } + write_json(output / "summary.json", summary) + import os + + import trackio + + os.environ["TRACKIO_DIR"] = str(output.resolve() / "trackio") + run = trackio.init( + project="smoldataenv-rl", + name=output.name, + space_id=args.space_id, + config=identity, + ) + run.log( + { + f"eval/{key}": value + for key, value in summary.items() + if isinstance(value, (int, float)) + }, + step=args.step, + ) + run.finish() + print(json.dumps(summary, indent=2)) + if summary["graded"] != summary["expected"]: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/05-multi-harness-rl/jobs/entrypoint.sh b/05-multi-harness-rl/jobs/entrypoint.sh new file mode 100644 index 00000000..315f2419 --- /dev/null +++ b/05-multi-harness-rl/jobs/entrypoint.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail +mkdir -p /workspace/tutorial +cp -a /tutorial-source/. /workspace/tutorial/ +cd /workspace/tutorial +uv venv --python 3.12 --seed .venv +export PATH="$PWD/.venv/bin:$PATH" +bash jobs/install.sh +python prepare.py +exec python jobs/run.py "$@" diff --git a/05-multi-harness-rl/jobs/hf_job.py b/05-multi-harness-rl/jobs/hf_job.py new file mode 100644 index 00000000..9ae2c8d3 --- /dev/null +++ b/05-multi-harness-rl/jobs/hf_job.py @@ -0,0 +1,134 @@ +"""Submit a tutorial to HF Jobs, with source and persistent output volumes.""" + +import argparse +import json +import os +import shutil +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=["train", "eval"]) + parser.add_argument( + "--mode", + choices=["whitebox", "opencode", "multi_harness"], + default="multi_harness", + ) + parser.add_argument("--model", default="LiquidAI/LFM2.5-2.6B") + parser.add_argument("--name", required=True) + parser.add_argument( + "--bucket", + required=True, + help="An existing owner/bucket for checkpoints and logs", + ) + parser.add_argument("--flavor", default="h200x2") + parser.add_argument("--timeout", default="12h") + parser.add_argument("--steps", type=int, default=1000) + parser.add_argument("--save-steps", type=int, default=50) + parser.add_argument("--checkpoint", help="For eval: /outputs//checkpoint-100") + parser.add_argument("--step", type=int, default=0) + parser.add_argument("--tasks", type=int, default=250) + parser.add_argument("--concurrency", type=int, default=35) + parser.add_argument("--space-id") + parser.add_argument( + "--submit", action="store_true", help="Without this flag, print the job plan" + ) + args = parser.parse_args() + if Path(args.name).name != args.name or args.name in {".", ".."}: + parser.error("Use a single directory name for --name") + if args.flavor != "h200x2": + parser.error( + "Use h200x2 for this two-GPU HF recipe; local H100 pairs are also supported" + ) + namespace, separator, bucket = args.bucket.partition("/") + if not separator or not namespace or not bucket: + parser.error("Use owner/bucket for --bucket") + command = [ + "bash", + "/tutorial-source/jobs/entrypoint.sh", + args.action, + "--mode", + args.mode, + "--model", + args.model, + "--output", + "/outputs/" + args.name, + "--steps", + str(args.steps), + "--save-steps", + str(args.save_steps), + "--step", + str(args.step), + "--tasks", + str(args.tasks), + "--concurrency", + str(args.concurrency), + ] + if args.checkpoint: + command += ["--checkpoint", args.checkpoint] + if args.space_id: + command += ["--space-id", args.space_id] + specification = { + "image": "huggingface/trl@sha256:8433cde7eaf3b289f3daffdaefcab6bb71499ca1ff59dc5d05abba7a71603a58", + "command": command, + "namespace": namespace, + "flavor": args.flavor, + "timeout": args.timeout, + "name": args.name, + "env": {"PYTHONUNBUFFERED": "1"}, + } + print( + json.dumps( + { + **specification, + "bucket": args.bucket, + "secrets": ["HF_TOKEN", "DAYTONA_API_KEY"], + }, + indent=2, + ) + ) + if not args.submit: + return + from huggingface_hub import Volume, get_token, run_job, sync_job_volume + + token = get_token() + if not token or not os.environ.get("DAYTONA_API_KEY"): + parser.error("Log in to HF and set DAYTONA_API_KEY") + with tempfile.TemporaryDirectory() as directory: + # An allowlist keeps credentials, prepared tasks and old experiments out of the source upload. + paths = list(ROOT.glob("*.py")) + [ROOT / "requirements.txt"] + for folder in ("jobs", "envs", "data"): + paths += [ + p + for p in (ROOT / folder).rglob("*") + if p.suffix in {".py", ".sh", ".json"} + ] + for source in paths: + target = Path(directory) / source.relative_to(ROOT) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + source_volume = sync_job_volume( + directory, "/tutorial-source", namespace=namespace + ) + job = run_job( + **specification, + volumes=[ + source_volume, + Volume( + type="bucket", + source=args.bucket, + mount_path="/outputs", + read_only=False, + ), + ], + secrets={"HF_TOKEN": token, "DAYTONA_API_KEY": os.environ["DAYTONA_API_KEY"]}, + ) + print(job.url) + + +if __name__ == "__main__": + main() diff --git a/05-multi-harness-rl/jobs/inference_proxy.py b/05-multi-harness-rl/jobs/inference_proxy.py new file mode 100644 index 00000000..a022d679 --- /dev/null +++ b/05-multi-harness-rl/jobs/inference_proxy.py @@ -0,0 +1,80 @@ +"""Expose inference to native OpenCode sandboxes without exposing weight-update routes.""" + +import argparse +import os +import secrets +import threading +import time +from contextlib import asynccontextmanager +from pathlib import Path + +import httpx +import uvicorn +from fastapi import FastAPI, HTTPException, Request +from starlette.background import BackgroundTask +from starlette.responses import StreamingResponse + + +def create_app(key, upstream="http://127.0.0.1:8000"): + @asynccontextmanager + async def lifespan(app): + async with httpx.AsyncClient(timeout=660) as client: + app.state.client = client + yield + + app = FastAPI(lifespan=lifespan) + + @app.api_route("/v1/{path:path}", methods=["GET", "POST"]) + async def forward(path: str, request: Request): + if not secrets.compare_digest( + request.headers.get("authorization", ""), "Bearer " + key + ): + raise HTTPException(401) + if (request.method, path) not in { + ("GET", "models"), + ("POST", "chat/completions"), + }: + raise HTTPException(404) + client = app.state.client + message = client.build_request( + request.method, + upstream + "/v1/" + path, + content=await request.body(), + headers={"content-type": "application/json"}, + ) + response = await client.send(message, stream=True) + return StreamingResponse( + response.aiter_raw(), + status_code=response.status_code, + media_type=response.headers.get("content-type"), + background=BackgroundTask(response.aclose), + ) + + return app + + +def main(): + from gradio.networking import setup_tunnel + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", required=True) + parser.add_argument("--port", type=int, default=8400) + parser.add_argument("--upstream", default="http://127.0.0.1:8000") + args = parser.parse_args() + app = create_app(os.environ["SANDBOX_VLLM_KEY"], args.upstream) + server = uvicorn.Server( + uvicorn.Config(app, host="127.0.0.1", port=args.port, log_level="warning") + ) + thread = threading.Thread(target=server.run, daemon=True) + thread.start() + while not server.started: + if not thread.is_alive(): + raise RuntimeError("Inference proxy failed to start") + time.sleep(0.1) + url = setup_tunnel("127.0.0.1", args.port, secrets.token_urlsafe(24), None, None) + Path(args.output).write_text(url) + thread.join() + + +if __name__ == "__main__": + main() diff --git a/05-multi-harness-rl/jobs/install.sh b/05-multi-harness-rl/jobs/install.sh new file mode 100644 index 00000000..e9157793 --- /dev/null +++ b/05-multi-harness-rl/jobs/install.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail +# Run from 05-multi-harness-rl in a Python 3.12 virtual environment. +python -m pip install -r requirements.txt +python -m pip install 'trl @ git+https://github.com/huggingface/trl.git@main' +mkdir -p .deps +if [[ ! -d .deps/OpenEnv/.git ]]; then + git clone --depth 1 https://github.com/huggingface/OpenEnv.git .deps/OpenEnv +fi +python -m pip install -e .deps/OpenEnv +# The environment examples are source packages, outside OpenEnv's core wheel. +python - <<'PY' +from pathlib import Path +import site +Path(site.getsitepackages()[0], 'openenv_examples.pth').write_text(str(Path('.deps/OpenEnv/envs').resolve()) + '\n') +PY +python check_setup.py diff --git a/05-multi-harness-rl/jobs/run.py b/05-multi-harness-rl/jobs/run.py new file mode 100644 index 00000000..04c0ff67 --- /dev/null +++ b/05-multi-harness-rl/jobs/run.py @@ -0,0 +1,258 @@ +"""Launch services and one tutorial script on a two-GPU machine. Training logic lives in train_*.py.""" + +import argparse +import json +import os +import secrets +import signal +import socket +import subprocess +import sys +import time +from contextlib import ExitStack +from pathlib import Path + +import httpx + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=["train", "eval"]) + parser.add_argument( + "--mode", + choices=["whitebox", "opencode", "multi_harness"], + default="multi_harness", + ) + parser.add_argument("--model", default="LiquidAI/LFM2.5-2.6B") + parser.add_argument("--output", required=True) + parser.add_argument("--steps", type=int, default=1000) + parser.add_argument("--save-steps", type=int, default=50) + parser.add_argument("--checkpoint") + parser.add_argument("--step", type=int, default=0) + parser.add_argument("--tasks", type=int, default=250) + parser.add_argument("--concurrency", type=int, default=35) + parser.add_argument("--space-id") + args = parser.parse_args() + os.chdir(ROOT) + subprocess.run([sys.executable, "check_setup.py"], check=True) + if not Path("prepared/ready.json").exists(): + raise RuntimeError("Run prepare.py before allocating GPUs") + output = Path(args.output).resolve() + if output.exists() and args.action == "train": + raise ValueError("Use a fresh training output directory") + output.mkdir(parents=True, exist_ok=True) + with (output / "packages.txt").open("w") as packages: + subprocess.run( + [sys.executable, "-m", "pip", "freeze"], stdout=packages, check=True + ) + devices = os.environ.get("CUDA_VISIBLE_DEVICES", "0,1").split(",") + if len(devices) < 2: + raise RuntimeError("Allocate two GPUs") + env = dict( + os.environ, + PYTHONUNBUFFERED="1", + VLLM_USE_DEEP_GEMM="0", + VLLM_DEEP_GEMM_WARMUP="skip", + VLLM_USE_FLASHINFER_SAMPLER="0", + ) + with ExitStack() as stack: + sockets = [stack.enter_context(socket.socket()) for _ in range(4)] + for listener in sockets: + listener.bind(("127.0.0.1", 0)) + engine_port, server_port, capture_port, proxy_port = [ + s.getsockname()[1] for s in sockets + ] + engine_url = f"http://127.0.0.1:{engine_port}" + server_url = f"http://127.0.0.1:{server_port}" + (output / "services.json").write_text( + json.dumps({"vllm": engine_url, "openenv": server_url}) + ) + children = [] + + def spawn(command, name, extra=None): + with (output / (name + ".log")).open("a") as stream: + process = subprocess.Popen( + command, + env={**env, **(extra or {})}, + stdout=stream, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + children.append(process) + return process + + def healthy(): + if any(p.poll() is not None for p in children): + raise RuntimeError("A service exited; inspect logs in " + str(output)) + + def wait(url, timeout=1200): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + healthy() + try: + if httpx.get(url, timeout=5).status_code == 200: + return + except httpx.HTTPError: + pass + time.sleep(2) + raise TimeoutError(url) + + def stop(signum, frame): + raise KeyboardInterrupt + + signal.signal(signal.SIGTERM, stop) + try: + command = [ + sys.executable, + "jobs/serve_model.py", + "--model", + args.model, + "--output", + str(output), + "--port", + str(engine_port), + ] + if args.action == "eval": + command += ["--eval"] + if args.checkpoint: + command += ["--checkpoint", args.checkpoint] + spawn( + command, + "vllm", + { + "CUDA_VISIBLE_DEVICES": devices[0] + if args.action == "train" + else ",".join(devices[:2]) + }, + ) + wait(engine_url + "/health") + if args.mode != "whitebox" and ( + args.action == "eval" or args.mode == "multi_harness" + ): + spawn( + [ + sys.executable, + "-m", + "uvicorn", + "harbor_env.server.app:app", + "--host", + "127.0.0.1", + "--port", + str(server_port), + "--ws-ping-timeout", + "1800", + ], + "harbor", + { + "OPENENV_DATASETS": ",".join( + str(Path("prepared/datasets", s).resolve()) + for s in ("train", "test") + ), + "OPENENV_LLM_URL": engine_url, + "OPENENV_MODEL": args.model, + "OPENENV_HARBOR_TRIALS_DIR": str(output / "trials"), + "OPENENV_MAX_OUTPUT_TOKENS": "4096", + "OPENENV_CAPTURE_PORT": str(capture_port), + "OPENENV_EXPOSE": "gradio", + "MAX_CONCURRENT_ENVS": str(max(32, args.concurrency) + 8), + "ENABLE_WEB_INTERFACE": "false", + "OPENENV_HARBOR_AGENT_VERSIONS": '{"opencode":"1.18.31","claude-code":"2.1.270","codex":"0.154.0","mini-swe-agent":"2.4.6"}', + }, + ) + wait(server_url + "/health") + if args.action == "train" and args.mode == "opencode": + env["SANDBOX_VLLM_KEY"] = secrets.token_urlsafe(32) + location = output / "inference-url.txt" + spawn( + [ + sys.executable, + "jobs/inference_proxy.py", + "--output", + str(location), + "--port", + str(proxy_port), + "--upstream", + engine_url, + ], + "inference-proxy", + ) + deadline = time.monotonic() + 180 + while not location.exists(): + healthy() + if time.monotonic() > deadline: + raise TimeoutError("Public inference tunnel") + time.sleep(1) + env["SANDBOX_VLLM_URL"] = location.read_text() + # Confirm that the sandbox-facing endpoint has the expected model. + response = httpx.get( + env["SANDBOX_VLLM_URL"] + "/v1/models", + headers={"Authorization": "Bearer " + env["SANDBOX_VLLM_KEY"]}, + timeout=30, + ) + response.raise_for_status() + common = [ + "--model", + args.model, + "--output", + str(output), + "--vllm-url", + engine_url, + ] + if args.action == "eval" or args.mode == "multi_harness": + common += ["--trials", str(output / "trials"), "--server", server_url] + if args.space_id: + common += ["--space-id", args.space_id] + if args.action == "train": + command = [ + sys.executable, + f"train_{args.mode}.py", + *common, + "--steps", + str(args.steps), + "--save-steps", + str(args.save_steps), + ] + else: + command = [ + sys.executable, + "evaluate.py", + *common, + "--mode", + "whitebox" if args.mode == "whitebox" else "blackbox", + "--checkpoint", + args.checkpoint or args.model, + "--step", + str(args.step), + "--tasks", + str(args.tasks), + "--concurrency", + str(args.concurrency), + ] + worker = spawn(command, args.action, {"CUDA_VISIBLE_DEVICES": devices[1]}) + children.remove(worker) + try: + while worker.poll() is None: + healthy() + time.sleep(2) + if worker.returncode: + raise RuntimeError( + f"{args.action} exited {worker.returncode}; inspect {output}" + ) + finally: + children.append(worker) + finally: + for process in reversed(children): + if process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) + for process in reversed(children): + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + + +if __name__ == "__main__": + main() diff --git a/05-multi-harness-rl/jobs/serve_model.py b/05-multi-harness-rl/jobs/serve_model.py new file mode 100644 index 00000000..3897f06a --- /dev/null +++ b/05-multi-harness-rl/jobs/serve_model.py @@ -0,0 +1,80 @@ +"""Start the same non-thinking vLLM configuration for training or evaluation.""" + +import argparse +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from train_whitebox import MODEL_REVISIONS, tokenizer_for + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--model", default="LiquidAI/LFM2.5-2.6B", help="Base model identity" + ) + parser.add_argument( + "--checkpoint", help="Load these weights but retain the base served-model name" + ) + parser.add_argument("--output", required=True) + parser.add_argument("--port", type=int, default=8000) + parser.add_argument("--eval", action="store_true") + args = parser.parse_args() + output = Path(args.output) + output.mkdir(parents=True, exist_ok=True) + tokenizer = tokenizer_for(args.model) + template = output / "chat_template.jinja" + template.write_text(tokenizer.chat_template) + command = [ + "vllm", + "serve", + args.checkpoint or args.model, + "--host", + "127.0.0.1", + "--port", + str(args.port), + "--served-model-name", + args.model, + "--tensor-parallel-size", + "1", + "--data-parallel-size", + "2" if args.eval else "1", + "--dtype", + "bfloat16", + "--max-model-len", + "131072", + "--gpu-memory-utilization", + "0.85", + "--enable-auto-tool-choice", + "--tool-call-parser", + "lfm2" if "lfm" in args.model.lower() else "qwen3_xml", + "--chat-template", + str(template), + "--default-chat-template-kwargs", + '{"enable_thinking":false,"preserve_thinking":true}', + "--generation-config", + "vllm", + "--logprobs-mode", + "processed_logprobs", + "--return-tokens-as-token-ids", + "--enforce-eager", + "--no-enable-prefix-caching", + ] + if not args.checkpoint: + command += ["--revision", MODEL_REVISIONS[args.model]] + if not args.eval: + command += ["--weight-transfer-config", '{"backend":"nccl"}'] + os.environ["VLLM_SERVER_DEV_MODE"] = "1" + if "qwen" in args.model.lower(): + command += [ + "--limit-mm-per-prompt", + '{"image":0,"video":0}', + "--gdn-prefill-backend", + "triton", + ] + os.execvp(command[0], command) + + +if __name__ == "__main__": + main() diff --git a/05-multi-harness-rl/jobs/train.slurm b/05-multi-harness-rl/jobs/train.slurm new file mode 100644 index 00000000..c29e1a5c --- /dev/null +++ b/05-multi-harness-rl/jobs/train.slurm @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +#SBATCH --nodes=1 +#SBATCH --gpus=2 +#SBATCH --cpus-per-task=16 +#SBATCH --mem=192G +#SBATCH --time=12:00:00 +#SBATCH --output=smoldataenv-%j.log +set -euo pipefail +# Submit from 05-multi-harness-rl after installation and prepare.py. +# Activate your Python 3.12 environment before sbatch; it is inherited by the job. +python jobs/run.py "$@" diff --git a/05-multi-harness-rl/prepare.py b/05-multi-harness-rl/prepare.py index dac6a8b4..d559824e 100644 --- a/05-multi-harness-rl/prepare.py +++ b/05-multi-harness-rl/prepare.py @@ -1,18 +1,36 @@ """Materialize the fixed train/test tasks from corrected, pinned Harbor datasets.""" + from __future__ import annotations import argparse import hashlib import json -from pathlib import Path import shutil +from pathlib import Path + import tomllib -from recipe import ROOT, task_rows, write_json +ROOT = Path(__file__).resolve().parent + + +def task_rows(split): + return json.loads((ROOT / "data" / f"{split}.json").read_text()) + + +def write_json(path, data): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2) + "\n") + DATASETS = { - "train": ("FineEnvs/SmolDataEnvs-harbor-train", "4719635555de1666f374d847baebb500d368e493"), - "test": ("FineEnvs/SmolDataEnvs-harbor-test", "b130595b579fc6026ad5762925a9a3b40e72f5bb"), + "train": ( + "FineEnvs/SmolDataEnvs-harbor-train", + "4719635555de1666f374d847baebb500d368e493", + ), + "test": ( + "FineEnvs/SmolDataEnvs-harbor-test", + "b130595b579fc6026ad5762925a9a3b40e72f5bb", + ), } GRADER_SHA256 = "16f906e9248fa8b7e1141939e425e88e59d52d65e98e360bbacef3720a43629f" @@ -28,22 +46,37 @@ def prepare(destination, sources=None): repo, revision = DATASETS[split] source = (sources or {}).get(split) if source is None: - source = snapshot_download(repo, repo_type="dataset", revision=revision, - allow_patterns=[f'tasks/{r["name"]}/*' for r in rows], max_workers=16) + source = snapshot_download( + repo, + repo_type="dataset", + revision=revision, + allow_patterns=[f"tasks/{r['name']}/*" for r in rows], + max_workers=16, + ) source = Path(source) tasks = [] for row in rows: folder = source / "tasks" / row["name"] instruction = (folder / "instruction.md").read_bytes() if hashlib.sha256(instruction).hexdigest() != row["instruction_sha256"]: - raise ValueError(f"Historical task instruction changed: {split}/{row['name']}") + raise ValueError( + f"Historical task instruction changed: {split}/{row['name']}" + ) grader = (folder / "tests/grader.py").read_bytes() if hashlib.sha256(grader).hexdigest() != GRADER_SHA256: - raise ValueError(f"Expected corrected numeric-only Harbor grader: {folder}") + raise ValueError( + f"Expected corrected numeric-only Harbor grader: {folder}" + ) target = destination / "datasets" / split / "tasks" / row["name"] target.mkdir(parents=True, exist_ok=True) - for rel in ("instruction.md", "task.toml", "environment/Dockerfile", "environment/pull_bucket.py", - "tests/grader.py", "tests/test.sh"): + for rel in ( + "instruction.md", + "task.toml", + "environment/Dockerfile", + "environment/pull_bucket.py", + "tests/grader.py", + "tests/test.sh", + ): dst = target / rel dst.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(folder / rel, dst) @@ -51,22 +84,42 @@ def prepare(destination, sources=None): spec["environment"]["memory_mb"] = 4096 spec["environment"]["cpus"] = 1 (target / "task.toml").write_text(tomli_w.dumps(spec)) - hashes = {str(f.relative_to(destination / "datasets" / split)): hashlib.sha256(f.read_bytes()).hexdigest() - for f in sorted(target.rglob("*")) if f.is_file()} + hashes = { + str(f.relative_to(destination / "datasets" / split)): hashlib.sha256( + f.read_bytes() + ).hexdigest() + for f in sorted(target.rglob("*")) + if f.is_file() + } tasks.append({**row, "file_hashes": hashes}) local = (sources or {}).get(split) - manifest = {"source_dataset": repo, "source_revision": None if local else revision, - "expected_hub_revision": revision, "local_source": str(local) if local else None, - "task_count": len(tasks), "tasks": tasks} + manifest = { + "source_dataset": repo, + "source_revision": None if local else revision, + "expected_hub_revision": revision, + "local_source": str(local) if local else None, + "task_count": len(tasks), + "tasks": tasks, + } write_json(destination / f"{split}_manifest.json", manifest) manifests[split] = manifest train = {r["source"] for r in manifests["train"]["tasks"]} test = {r["source"] for r in manifests["test"]["tasks"]} - notebooks = [{r.get("notebook") for r in manifests[s]["tasks"]} - {None} for s in ("train", "test")] + notebooks = [ + {r.get("notebook") for r in manifests[s]["tasks"]} - {None} + for s in ("train", "test") + ] if train & test or notebooks[0] & notebooks[1]: raise ValueError("Train/test overlap") - write_json(destination / "ready.json", {"train": len(train), "test": len(test), - "grader_sha256": GRADER_SHA256, "train_test_overlap": 0}) + write_json( + destination / "ready.json", + { + "train": len(train), + "test": len(test), + "grader_sha256": GRADER_SHA256, + "train_test_overlap": 0, + }, + ) return manifests @@ -76,5 +129,8 @@ def prepare(destination, sources=None): p.add_argument("--train-source", type=Path) p.add_argument("--test-source", type=Path) a = p.parse_args() - result = prepare(a.output, {s: v for s, v in (("train", a.train_source), ("test", a.test_source)) if v}) + result = prepare( + a.output, + {s: v for s, v in (("train", a.train_source), ("test", a.test_source)) if v}, + ) print(json.dumps({s: m["task_count"] for s, m in result.items()})) diff --git a/05-multi-harness-rl/recipe.py b/05-multi-harness-rl/recipe.py deleted file mode 100644 index 877b80f6..00000000 --- a/05-multi-harness-rl/recipe.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Shared configuration, task identity and reward policy.""" -from __future__ import annotations - -import hashlib -import json -import math -import random -from pathlib import Path -import uuid - -ROOT = Path(__file__).resolve().parent -MODES = ("whitebox", "opencode", "multi-harness") - - -def write_json(path, value): - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - tmp = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - try: - tmp.write_text(json.dumps(value, indent=2, allow_nan=False) + "\n") - tmp.replace(path) - finally: - tmp.unlink(missing_ok=True) - - -def digest(value): - return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() - - -def merge_config(base, updates): - for key, value in updates.items(): - if isinstance(value, dict) and isinstance(base.get(key), dict): - merge_config(base[key], value) - else: - base[key] = value - return base - - -def config(path=None, **overrides): - result = json.loads((ROOT / "configs/default.json").read_text()) - if path: - merge_config(result, json.loads(Path(path).read_text())) - result.update({k: v for k, v in overrides.items() if v is not None}) - if result["mode"] not in MODES or result["model"] not in result["models"]: - raise ValueError("Unknown mode or model") - for key in ("passes", "num_generations", "max_steps", "save_steps", "eval_steps", "eval_concurrency", - "batch_size", "gradient_accumulation_steps", "token_budget", "max_output_tokens"): - if type(result[key]) is not int or result[key] <= 0: - raise ValueError(f"{key} must be a positive integer") - if result["num_generations"] < 2: - raise ValueError("GRPO needs multiple rollouts of each task") - if result["max_inflight"] < result["num_generations"]: - raise ValueError("Inflight capacity must fit a complete group") - if result["max_outstanding_rollouts"] < 2 * result["num_generations"]: - raise ValueError("Outstanding capacity must fit two complete groups") - if result["eval_steps"] % result["save_steps"]: - raise ValueError("Evaluation cadence must be a multiple of checkpoint cadence") - if result["top_p"] != 1.0 or result["top_k"] != 0: - raise ValueError("Training requires full-vocabulary sampling for logprob consistency") - if not 0 <= result["efficiency_weight"] <= 0.1: - raise ValueError("Efficiency weight must be between zero and 0.1") - result["profile"] = result["models"][result["model"]] - return result - - -def whitebox_step_limit(cfg, task_count): - rollouts_per_update = cfg["batch_size"] * cfg["gradient_accumulation_steps"] - if rollouts_per_update % cfg["num_generations"]: - raise ValueError("Whitebox update must contain complete GRPO groups") - tasks_per_update = rollouts_per_update // cfg["num_generations"] - available = task_count // tasks_per_update - if not available: - raise ValueError("Whitebox schedule is smaller than one update") - return min(cfg["max_steps"], available) - - -def task_rows(split): - rows = json.loads((ROOT / "data" / f"{split}.json").read_text()) - if len({r["name"] for r in rows}) != len(rows): - raise ValueError("Duplicate task identity") - return rows - - -def eval_rows(rows, limit=None, *, stratified=False, seed=0): - rows = sorted(rows, key=lambda row: row["name"]) - if limit is None: - return rows - if not 1 <= limit <= len(rows): - raise ValueError("Eval limit must fit the fixed test set") - if not stratified: - return rows[:limit] - strata = {d: [r for r in rows if r["difficulty"] == d] for d in sorted({r["difficulty"] for r in rows})} - exact = {d: limit * len(group) / len(rows) for d, group in strata.items()} - counts = {d: int(n) for d, n in exact.items()} - for d in sorted(strata, key=lambda d: (-(exact[d] - counts[d]), d))[:limit - sum(counts.values())]: - counts[d] += 1 - rng = random.Random(seed) - selected = [] - for d, group in strata.items(): - selected.extend(rng.sample(group, counts[d])) - return sorted(selected, key=lambda row: row["name"]) - - -def schedule(cfg, rows): - catalog = {name: i for i, name in enumerate(sorted(r["name"] for r in rows))} - harnesses = cfg["harnesses"] if cfg["mode"] == "multi-harness" else [cfg["mode"]] - groups = [] - for epoch in range(cfg["passes"]): - for i, row in enumerate(rows): - groups.append({"group_id": len(groups), "pass": epoch, "task_name": row["name"], - "task_index": catalog[row["name"]], - "harness": harnesses[(i + epoch) % len(harnesses)]}) - return groups - - -def resume_state(groups, state): - if state["execution_schedule_sha256"] != digest(groups): - raise ValueError("Cannot resume with a different task/harness schedule") - committed = set(state.get("admitted_rollouts", {}).values()) - if any(type(g) is not int or not 0 <= g < len(groups) for g in committed): - raise ValueError("Invalid committed group") - return {**state, "settled_groups": sorted(committed)} - - -def reward(correctness, calls, *, verified, weight=0.1, budget=15): - if correctness is None: - return None - if correctness not in (0, 1): - raise ValueError("Expected binary correctness") - if not 0 <= weight <= 0.1 or budget <= 0: - raise ValueError("Invalid reward parameters") - if not verified or calls is None: - return float(correctness) - if type(calls) is not int or calls < 0: - raise ValueError("Invalid verified tool count") - return float(correctness) * (1 + weight * budget / (budget + calls)) if calls else float(correctness) - - -def summary(records, expected): - graded = [r for r in records if r.get("correctness") in (0, 1) and not r.get("error")] - successes = sum(r["correctness"] for r in graded) - return {"expected": expected, "graded": len(graded), "correct": successes, - "pass_at_1": successes / len(graded) if graded else None, - "coverage": len(graded) / expected if expected else 0, - "complete": len(graded) == expected, - "mean_shaped_reward": mean(r.get("reward") for r in graded), - "mean_tool_calls": mean(r.get("tool_calls") for r in graded), - "mean_generated_tokens": mean(r.get("generated_tokens") for r in graded)} - - -def mean(values): - values = [v for v in values if isinstance(v, (int, float)) and math.isfinite(v)] - return sum(values) / len(values) if values else None diff --git a/05-multi-harness-rl/requirements.lock b/05-multi-harness-rl/requirements.lock deleted file mode 100644 index d534f663..00000000 --- a/05-multi-harness-rl/requirements.lock +++ /dev/null @@ -1,312 +0,0 @@ -accelerate==1.14.0 -agent-detector==2.0.0 -aiofile==3.12.3 -aiofiles==25.1.0 -aiohappyeyeballs==2.7.1 -aiohttp==3.14.3 -aiohttp-retry==2.9.1 -aiosignal==1.4.0 -annotated-doc==0.0.5 -annotated-types==0.8.0 -anthropic==1.9.0 -antlr4-python3-runtime==4.13.2 -anyio==4.15.1 -apache-tvm-ffi==0.1.9 -astor==0.8.1 -attrs==26.1.0 -authlib==1.8.0 -beartype==0.22.9 -bidict==0.24.1 -bitsandbytes==0.49.2 -blake3==1.0.10 -boto3==1.43.105 -botocore==1.43.105 -bracex==3.0.1 -brotli==1.2.0 -cachetools==7.2.0 -caio==0.12.9 -cbor2==6.1.4 -certifi==2026.7.22 -cffi==2.1.1 -charset-normalizer==3.5.2 -click==8.5.0 -cloudpickle==3.1.2 -compressed-tensors==0.17.0 -connectrpc==0.11.1 -cryptography==50.0.1 -cuda-bindings==13.4.3 -cuda-core==1.2.1 -cuda-pathfinder==1.8.2 -cuda-python==13.4.1 -cuda-tile==1.3.0 -cuda-toolkit==13.0.2 -cyclopts==5.1.0 -datasets==5.0.0 -daytona==0.203.0 -daytona-analytics-api-client==0.203.0 -daytona-analytics-api-client-async==0.203.0 -daytona-api-client==0.203.0 -daytona-api-client-async==0.203.0 -daytona-toolbox-api-client==0.203.0 -daytona-toolbox-api-client-async==0.203.0 -deprecated==1.3.1 -deprecation==2.1.0 -depyf==0.20.0 -detect-installer==0.2.1 -dill==0.4.1 -dirhash==0.5.0 -diskcache==5.6.3 -distro==1.9.0 -dnspython==2.8.0 -dockerfile-parse==2.0.1 -docstring-parser==0.18.0 -e2b==2.36.0 -einops==0.8.2 -email-validator==2.3.0 -exceptiongroup==1.3.1 -fastapi==0.136.3 -fastapi-cli==0.0.32 -fastapi-cloud-cli==0.26.0 -fastar==0.12.0 -fastmcp==3.4.4 -fastmcp-slim==3.4.4 -fastsafetensors==0.4.0 -fastuuid==0.14.0 -filelock==4.0.7 -fla-core==0.5.2 -flash-linear-attention==0.5.2 -flashinfer-cubin==0.6.13 -flashinfer-python==0.6.13 -frozenlist==1.8.0 -fsspec==2026.4.0 -googleapis-common-protos==1.75.5 -gradio==6.20.0 -gradio-client==2.5.0 -griffelib==2.3.0 -groovy==0.1.2 -grpcio==1.84.0 -h11==0.16.0 -h2==4.4.1 -harbor==0.22.0 -hf-gradio==0.4.1 -hf-xet==1.6.0 -hpack==4.2.0 -httpcore==1.0.9 -httpcore2==2.13.1 -httptools==0.8.0 -httpx==0.28.1 -httpx-sse==0.4.3 -httpx-ws==0.9.0 -httpx2==2.13.1 -huggingface-hub==1.29.0 -humming-kernels==0.1.10 -hyperframe==6.1.0 -idna==3.20 -ijson==3.5.1 -importlib-metadata==8.9.0 -iniconfig==2.3.0 -interegular==0.3.3 -jaraco-classes==3.4.0 -jaraco-context==6.1.2 -jaraco-functools==4.6.0 -jeepney==0.9.0 -jinja2==3.1.6 -jiter==0.17.0 -jmespath==1.1.0 -joserfc==1.7.5 -jsonref==1.1.0 -jsonschema==4.26.0 -jsonschema-path==0.5.0 -jsonschema-specifications==2025.9.1 -kernels==0.15.2 -kernels-data==0.16.2 -keyring==25.7.0 -lark==1.2.2 -latex2sympy2-extended==1.11.0 -litellm==1.103.1 -llguidance==1.7.6 -llvmlite==0.47.0 -lm-format-enforcer==0.11.3 -loguru==0.7.3 -markdown-it-py==4.2.0 -markupsafe==3.0.3 -math-verify==0.9.0 -mcp==1.28.1 -mdurl==0.1.2 -mistral-common==1.12.0 -ml-dtypes==0.6.0 -model-hosting-container-standards==0.1.16 -more-itertools==11.1.0 -mpmath==1.3.0 -msgspec==0.22.0 -multidict==6.9.1 -multiprocess==0.70.19 -networkx==3.7 -ninja==1.13.0 -numba==0.65.0 -numpy==2.3.5 -nvidia-cublas==13.1.0.3 -nvidia-cuda-cccl==13.3.4.3.1 -nvidia-cuda-crt==13.4.92 -nvidia-cuda-cupti==13.0.85 -nvidia-cuda-nvcc==13.2.86 -nvidia-cuda-nvrtc==13.0.88 -nvidia-cuda-runtime==13.0.96 -nvidia-cuda-tileiras==13.2.86 -nvidia-cudnn-cu13==9.19.0.56 -nvidia-cudnn-frontend==1.28.0 -nvidia-cufft==12.0.0.61 -nvidia-cufile==1.15.1.6 -nvidia-curand==10.4.0.35 -nvidia-cusolver==12.0.4.66 -nvidia-cusparse==12.6.3.3 -nvidia-cusparselt-cu13==0.8.0 -nvidia-cutlass-dsl==4.5.2 -nvidia-cutlass-dsl-libs-base==4.5.2 -nvidia-cutlass-dsl-libs-cu13==4.5.2 -nvidia-ml-py==13.615.71 -nvidia-nccl-cu13==2.28.9 -nvidia-nvjitlink==13.0.88 -nvidia-nvshmem-cu13==3.4.5 -nvidia-nvtx==13.0.85 -nvidia-nvvm==13.2.86 -nvtx==0.2.15 -obstore==0.11.1 -openai==2.52.0 -openai-harmony==0.0.8 -openapi-pydantic==0.5.1 -opencv-python-headless==5.0.0.93 -opentelemetry-api==1.45.0 -opentelemetry-exporter-http-transport==0.66b0 -opentelemetry-exporter-otlp==1.45.0 -opentelemetry-exporter-otlp-common==0.66b0 -opentelemetry-exporter-otlp-proto-common==1.45.0 -opentelemetry-exporter-otlp-proto-grpc==1.45.0 -opentelemetry-exporter-otlp-proto-http==1.45.0 -opentelemetry-instrumentation==0.66b0 -opentelemetry-instrumentation-aiohttp-client==0.66b0 -opentelemetry-proto==1.45.0 -opentelemetry-sdk==1.45.0 -opentelemetry-semantic-conventions==0.66b0 -opentelemetry-semantic-conventions-ai==0.5.1 -opentelemetry-util-http==0.66b0 -orjson==3.12.0 -outlines-core==0.2.14 -packaging==26.2 -pandas==3.0.6 -partial-json-parser==0.2.1.1.post7 -pathable==0.6.0 -pathspec==1.1.1 -peft==0.19.1 -pillow==12.3.0 -platformdirs==4.12.2 -pluggy==1.6.0 -postgrest==2.31.0 -prometheus-client==0.26.0 -prometheus-fastapi-instrumentator==8.1.0 -propcache==0.5.4 -protobuf==7.36.2 -protobuf-py==0.1.1 -protobuf-py-ext==0.1.1 -psutil==7.2.2 -py-cpuinfo==9.0.0 -py-key-value-aio==0.4.6 -pyarrow==25.0.1 -pybase64==1.5.0 -pycountry==26.2.16 -pycparser==3.0 -pydantic==2.13.4 -pydantic-core==2.46.4 -pydantic-extra-types==2.11.1 -pydantic-settings==2.15.0 -pydub==0.25.1 -pyelftools==0.33 -pygments==2.21.0 -pyjwt==2.15.1 -pynvvideocodec==2.0.4 -pyperclip==1.11.0 -pyqwest==0.7.0 -pytest==9.1.1 -python-dateutil==2.9.0.post0 -python-dotenv==1.2.2 -python-engineio==4.14.0 -python-json-logger==4.2.0 -python-multipart==0.0.32 -python-socketio==5.17.0 -pytz==2026.4 -pyyaml==6.0.3 -pyzmq==27.2.0 -quack-kernels==0.5.0 -realtime==2.31.0 -referencing==0.37.0 -regex==2026.9.29 -requests==2.34.2 -rich==15.0.0 -rich-rst==2.2.0 -rich-toolkit==0.20.5 -rignore==0.8.1 -rpds-py==2026.6.3 -s3transfer==0.19.2 -safehttpx==0.1.7 -safetensors==0.8.0 -scantree==0.0.4 -secretstorage==3.5.0 -semantic-version==2.10.0 -sentencepiece==0.2.2 -sentry-sdk==2.71.0 -setproctitle==1.3.7 -setuptools==80.10.2 -shellingham==1.5.4 -shortuuid==1.0.13 -simple-websocket==1.1.0 -six==1.17.0 -sniffio==1.3.1 -sse-starlette==3.5.0 -starlette==1.7.0 -storage3==2.31.0 -strenum==0.4.15 -supabase==2.31.0 -supabase-auth==2.31.0 -supabase-functions==2.31.0 -supervisor==4.3.0 -sympy==1.14.0 -tabulate==0.10.0 -tenacity==9.1.4 -tiktoken==0.14.0 -tilelang==0.1.9 -tokenizers==0.22.2 -tokenspeed-mla==0.1.2 -tokenspeed-triton==3.8.10.post20260920 -toml==0.10.2 -tomli==2.4.1 -tomli-w==1.2.0 -tomlkit==0.14.0 -torch==2.11.0 -torch-c-dlpack-ext==0.1.5 -torchaudio==2.11.0 -torchcodec==0.16.0 -torchvision==0.26.0 -tqdm==4.70.1 -trackio==0.33.0 -transformers==5.14.1 -triton==3.6.0 -truststore==0.10.4 -typer==0.27.0 -typing-extensions==4.16.0 -typing-inspection==0.4.4 -uncalled-for==0.4.0 -urllib3==2.8.0 -uvicorn==0.51.0 -uvloop==0.22.1 -vllm==0.25.1 -watchfiles==1.3.0 -wcmatch==10.2.1 -websocket-client==1.9.2 -websockets==15.0.1 -wrapt==2.5.0 -wsproto==1.3.2 -xgrammar==0.2.3 -xxhash==4.0.1 -yarl==1.25.1 -z3-solver==4.15.4.0 -zipp==4.1.0 diff --git a/05-multi-harness-rl/requirements.txt b/05-multi-harness-rl/requirements.txt index 46f7231f..3a32a1e3 100644 --- a/05-multi-harness-rl/requirements.txt +++ b/05-multi-harness-rl/requirements.txt @@ -1,40 +1,16 @@ -# Direct pins from the frozen comparison trainer environment. Vendored TRL/OpenEnv are supplied separately. -torch==2.11.0 +# TRL and OpenEnv are installed from upstream main by jobs/install.sh. vllm==0.25.1 -transformers==5.14.1 -datasets==5.0.0 -huggingface_hub==1.29.0 -accelerate==1.14.0 -bitsandbytes==0.49.2 -kernels==0.15.2 -trackio==0.33.0 -gradio==6.20.0 -httpx==0.28.1 -# Harbor -> Supabase requires websockets <16; matches the validated environment service. -websockets==15.0.1 -harbor==0.22.0 -daytona==0.203.0 -pydantic==2.13.4 -fastapi==0.136.3 -uvicorn==0.51.0 -mcp==1.28.1 -fastmcp==3.4.4 -tomli==2.4.1 -tomli-w==1.2.0 -ninja==1.13.0 -peft==0.19.1 -# vLLM's Numba requires <2.5; mistral-common on Python 3.12 requires <2.4. +transformers @ git+https://github.com/huggingface/transformers.git@main +datasets +accelerate +bitsandbytes +kernels +trackio +harbor[daytona]>=0.22.0 +daytona>=0.203.0 +math-verify +tomli-w +pytest +# Compatible with Harbor's websocket dependencies and vLLM's Numba. +websockets>=15,<16 numpy>=2.2,<2.4 -safetensors==0.8.0 -rich==15.0.0 -typer==0.27.0 -python-dotenv==1.2.2 -packaging==26.2 -jinja2==3.1.6 - -openai==2.52.0 -e2b==2.36.0 -math-verify==0.9.0 -pytest==9.1.1 -pyyaml==6.0.3 -flash-linear-attention==0.5.2 diff --git a/05-multi-harness-rl/run.py b/05-multi-harness-rl/run.py deleted file mode 100644 index 97db4538..00000000 --- a/05-multi-harness-rl/run.py +++ /dev/null @@ -1,215 +0,0 @@ -"""Plan, serve, train or evaluate the three SmolDataEnv modes.""" -import argparse -import json -import os -from pathlib import Path -import signal -import socket -import subprocess -import sys -import time -from urllib.parse import urlparse - -from recipe import ROOT, MODES, config, write_json - - -def parser(): - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("action", choices=["plan", "train", "eval", "smoke", "pilot", "environment", "_train", "_eval"]) - p.add_argument("--config", type=Path) - p.add_argument("--model", choices=["lfm", "qwen"]) - p.add_argument("--mode", choices=MODES) - p.add_argument("--data", type=Path, default=ROOT / "prepared") - p.add_argument("--output", type=Path) - p.add_argument("--run-name") - p.add_argument("--checkpoint", type=Path) - p.add_argument("--resume", type=Path) - p.add_argument("--concurrency", type=int) - p.add_argument("--limit", type=int, help="Eval tasks per harness; omit for all 250") - p.add_argument("--smoke-eval", action="store_true", help="Reload the smoke checkpoint for two tasks per harness") - p.add_argument("--preflight-smoke", action="store_true", help="Gate a pilot on a two-update training and reload smoke") - p.add_argument("--server", help="External URL, or a local bind URL; owned services default to unused ports") - p.add_argument("--vllm-url") - p.add_argument("--capture-port", type=int) - p.add_argument("--external-services", action="store_true", help="Use the supplied server URLs without starting services") - return p - - -def unused_port(): - with socket.socket() as listener: - listener.bind(("127.0.0.1", 0)) - return listener.getsockname()[1] - - -def wait_ready(url, process, timeout=1200): - import httpx - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if process.poll() is not None: - raise RuntimeError(f"Service exited with {process.returncode}; inspect its log") - try: - if httpx.get(url + "/health", timeout=5).status_code == 200: - return - except httpx.HTTPError: - pass - time.sleep(2) - raise TimeoutError(f"Service readiness exceeded {timeout}s") - - -def launch_local(args, cfg): - from runtime.bootstrap import activate - activate() - from runtime.models import tokenizer_for, serving_command, check_visible_response - from runtime.environment import credentials - - credentials() - if not (args.data / "ready.json").exists(): - raise ValueError("Run prepare.py before allocating GPUs") - output = Path(cfg["output"]) - output.mkdir(parents=True, exist_ok=True) - if args.action in {"train", "smoke"} and not args.resume and (output / "metrics.jsonl").exists(): - raise ValueError("Existing training output; choose a new run name or pass --resume") - training = args.action in {"train", "smoke"} - if args.checkpoint: - from runtime.checkpoints import verify - info = verify(args.checkpoint) - if info["base_model"] != cfg["profile"]["id"]: - raise ValueError("Checkpoint model differs from configured model") - if args.resume: - from runtime.checkpoints import verify - info = verify(args.resume, resume=True) - if info["base_model"] != cfg["profile"]["id"] or info["base_revision"] != cfg["profile"]["revision"]: - raise ValueError("Resume model differs from configured model") - model = str(args.resume or args.checkpoint or cfg["profile"]["id"]) - template = output / "chat_template.jinja" - tokenizer_for(cfg, template) - config_path = output / "config.json" - write_json(config_path, cfg) - write_json(output / "services.json", {"environment": args.server, "vllm": args.vllm_url, - "capture_port": args.capture_port}) - env = dict(os.environ, PYTHONUNBUFFERED="1", VLLM_SERVER_DEV_MODE="1", - VLLM_USE_DEEP_GEMM="0", VLLM_DEEP_GEMM_WARMUP="skip", VLLM_USE_FLASHINFER_SAMPLER="0", - TRACKIO_DIR=str(output / "trackio")) - processes = [] - def spawn(command, name, child_env=None): - with (output / f"{name}.log").open("a") as stream: - proc = subprocess.Popen(command, env=child_env or env, stdout=stream, stderr=subprocess.STDOUT, - start_new_session=True, cwd=ROOT) - processes.append(proc) - return proc - devices = os.environ.get("CUDA_VISIBLE_DEVICES", "0,1").split(",") - if len(devices) < 2: - raise ValueError("Allocate two GPUs: inference + trainer, or two evaluation replicas") - try: - inference_env = dict(env, CUDA_VISIBLE_DEVICES=devices[0] if training else ",".join(devices[:2])) - port = urlparse(args.vllm_url).port or 8000 - engine = spawn(serving_command(cfg, model, training=training, template_path=template, port=port), "vllm", inference_env) - wait_ready(args.vllm_url, engine) - write_json(output / "inference-check.json", check_visible_response(cfg, args.vllm_url)) - base = [sys.executable, str(ROOT / "run.py")] - common = ["--config", str(config_path), "--data", str(args.data.resolve()), "--output", str(output), - "--server", args.server, "--vllm-url", args.vllm_url, - "--capture-port", str(args.capture_port)] - service_mode = cfg["mode"] if training or cfg["mode"] == "whitebox" else "multi-harness" - service = spawn(base + ["environment", *common, "--mode", service_mode], "environment") - wait_ready(args.server, service) - action = "_train" if training else "_eval" - command = base + [action, *common] - if args.resume: - command += ["--resume", str(args.resume.resolve())] - if args.checkpoint: - command += ["--checkpoint", str(args.checkpoint.resolve())] - if args.limit: - command += ["--limit", str(args.limit)] - worker = spawn(command, "train" if training else "eval", dict(env, CUDA_VISIBLE_DEVICES=devices[1]) if training else env) - next_status = 0 - while worker.poll() is None: - if engine.poll() is not None or service.poll() is not None: - raise RuntimeError("Inference or environment service died; stopping worker") - if time.monotonic() >= next_status: - progress_path = output / ("metrics.jsonl" if training else "eval/progress.json") - try: - content = progress_path.read_text() - progress = json.loads(content.splitlines()[-1] if training else content) - keys = ("step", "reward", "grad_norm", "loss") if training else ("expected", "graded", "correct", "coverage") - print(json.dumps({"run": cfg["run_name"], "progress": {k: progress[k] for k in keys if k in progress}}), flush=True) - except (OSError, ValueError, IndexError): - print(json.dumps({"run": cfg["run_name"], "status": "waiting_for_first_result"}), flush=True) - next_status = time.monotonic() + 60 - time.sleep(2) - if worker.returncode: - raise RuntimeError(f"Worker failed ({worker.returncode}); inspect {output}") - finally: - for proc in reversed(processes): - if proc.poll() is None: - os.killpg(proc.pid, signal.SIGTERM) - for proc in reversed(processes): - try: - proc.wait(timeout=30) - except subprocess.TimeoutExpired: - os.killpg(proc.pid, signal.SIGKILL) - proc.wait() - - -def main(): - args = parser().parse_args() - if args.preflight_smoke and args.action != "pilot": - raise ValueError("--preflight-smoke requires the pilot action") - if args.smoke_eval and (args.action != "smoke" or args.external_services): - raise ValueError("--smoke-eval requires a smoke with job-local services") - if args.external_services and (not args.server or not args.vllm_url): - raise ValueError("External services require --server and --vllm-url") - args.server = args.server or f"http://127.0.0.1:{unused_port()}" - args.vllm_url = args.vllm_url or f"http://127.0.0.1:{unused_port()}" - args.capture_port = args.capture_port or unused_port() - cfg = config(args.config, model=args.model, mode=args.mode, eval_concurrency=args.concurrency) - name = args.run_name or cfg.get("run_name") or f'{cfg["model"]}-{cfg["mode"]}-{time.strftime("%Y%m%d-%H%M%S")}' - cfg.update(run_name=name, output=str((args.output or Path(cfg.get("output", ROOT / "runs" / name))).resolve())) - if args.action == "pilot": - if args.external_services or args.checkpoint: - raise ValueError("Pilot owns its services; use --resume only for a saved pilot training checkpoint") - from runtime.pilot import pilot - return pilot(args, cfg) - if args.action == "smoke": - cfg.update(max_steps=2, save_steps=1, eval_steps=1, num_generations=2, - max_inflight=4, max_outstanding_rollouts=4, batch_size=1, gradient_accumulation_steps=2) - if args.action == "plan": - print(json.dumps(cfg, indent=2)) - return - if args.action in {"train", "eval", "smoke"}: - if not args.external_services: - launch_local(args, cfg) - if args.smoke_eval: - from runtime.checkpoints import make_ready - checkpoint = Path(cfg["output"]) / f'checkpoint-{cfg["max_steps"]}' - make_ready(checkpoint) - eval_args = argparse.Namespace(**vars(args)) - eval_args.action, eval_args.checkpoint, eval_args.resume = "eval", checkpoint, None - eval_args.limit = args.limit or 2 - eval_cfg = {**cfg, "output": str(Path(cfg["output"]) / "reload-eval"), - "run_name": cfg["run_name"] + "-reload", "eval_concurrency": args.concurrency or 4} - launch_local(eval_args, eval_cfg) - return - args.action = "_eval" if args.action == "eval" else "_train" - from runtime.bootstrap import activate - activate() - if args.action == "environment": - from runtime.environment import serve - serve(cfg, args.data, args.vllm_url, port=urlparse(args.server).port or 8200, - capture_port=args.capture_port) - elif args.action == "_train": - if cfg["mode"] == "whitebox": - from train.whitebox import train - else: - from train.blackbox import train - train(cfg, args.data, args.server, args.vllm_url, args.resume) - else: - from eval.evaluate import evaluate - report = evaluate(cfg, args.data, args.server, args.vllm_url, - limit=args.limit, checkpoint=args.checkpoint or "baseline") - if not report["complete"]: - raise SystemExit(2) - - -if __name__ == "__main__": - main() diff --git a/05-multi-harness-rl/runtime/bootstrap.py b/05-multi-harness-rl/runtime/bootstrap.py deleted file mode 100644 index f8dd0d7b..00000000 --- a/05-multi-harness-rl/runtime/bootstrap.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Fetch immutable runtime dependencies without copying the experiment archive into Git.""" -from __future__ import annotations - -import argparse -from concurrent.futures import ThreadPoolExecutor -import hashlib -import json -import os -from pathlib import Path -import shutil -import subprocess -import sys -import urllib.request - -ROOT = Path(__file__).resolve().parents[1] -RUNTIME = ROOT / ".runtime" - - -def run(*args): - subprocess.run(list(map(str, args)), check=True) - - -def bootstrap(local_archive=None, local_trl=None, local_openenv=None): - lock = json.loads((ROOT / "configs/runtime-lock.json").read_text()) - archive = lock["archive"] - destination = RUNTIME / "archive" - - def fetch(item): - name, expected = item - path = destination / name - if path.exists() and hashlib.sha256(path.read_bytes()).hexdigest() == expected: - return - if local_archive: - data = (Path(local_archive) / "04-data-agent" / name).read_bytes() - else: - url = f'https://raw.githubusercontent.com/{archive["repo"]}/{archive["revision"]}/04-data-agent/{name}' - with urllib.request.urlopen(url, timeout=60) as stream: - data = stream.read() - if hashlib.sha256(data).hexdigest() != expected: - raise ValueError(f"Runtime source hash mismatch: {name}") - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - - with ThreadPoolExecutor(max_workers=8) as pool: - list(pool.map(fetch, archive["files"].items())) - for name, local in (("trl", local_trl), ("openenv", local_openenv)): - entry = lock[name] - target = RUNTIME / name - if not target.exists(): - run("git", "clone", "--no-checkout", "--filter=blob:none", local or f'https://github.com/{entry["repo"]}.git', target) - try: - run("git", "-C", target, "cat-file", "-e", entry["revision"]) - except subprocess.CalledProcessError: - run("git", "-C", target, "fetch", "origin", entry["revision"]) - if name == "trl": - sys.path.insert(0, str(ROOT)) - from runtime.patches import restore_trl - restore_trl(target) - run("git", "-C", target, "checkout", "--detach", entry["revision"]) - packages = RUNTIME / "packages" - packages.mkdir(exist_ok=True) - for name, folder in (("data_agent_env", "blackbox-opencode"), ("whitebox_bash", "whitebox-bash")): - link = packages / name - if link.is_symlink(): - link.unlink() - shutil.copytree(destination / "envs" / folder, link, dirs_exist_ok=True) - sys.path.insert(0, str(ROOT)) - from runtime.patches import apply, apply_trl - apply(packages) - apply_trl(RUNTIME / "trl") - print(f"Verified {len(archive['files'])} runtime files; pinned TRL and OpenEnv ready.") - - -def activate(): - paths = [ROOT, RUNTIME / "packages", RUNTIME / "archive/train", - RUNTIME / "archive/hf/runtime", RUNTIME / "archive/model_runs/lfm25", - RUNTIME / "openenv/src", RUNTIME / "openenv/envs", RUNTIME / "trl"] - if not (RUNTIME / "trl/trl").is_dir(): - raise RuntimeError("Run python runtime/bootstrap.py first") - sys.path[:0] = list(map(str, paths)) - os.environ["PYTHONPATH"] = os.pathsep.join(map(str, paths)) - os.environ.setdefault("TRL_EXPERIMENTAL_SILENCE", "1") - os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") - os.environ.setdefault("OMP_NUM_THREADS", "1") - - -if __name__ == "__main__": - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--local-archive", type=Path) - p.add_argument("--local-trl", type=Path) - p.add_argument("--local-openenv", type=Path) - a = p.parse_args() - bootstrap(a.local_archive, a.local_trl, a.local_openenv) diff --git a/05-multi-harness-rl/runtime/checkpoints.py b/05-multi-harness-rl/runtime/checkpoints.py deleted file mode 100644 index 2e3a963e..00000000 --- a/05-multi-harness-rl/runtime/checkpoints.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Portable checkpoint markers. The evaluator verifies bytes before loading weights.""" -import hashlib -import json -from pathlib import Path - -from recipe import write_json - -REQUIRED = ("config.json", "trainer_state.json", "tokenizer_config.json", "tokenizer.json", - "training_args.bin", "optimizer.pt", "scheduler.pt", "rng_state.pth") - - -def sha256(path): - value = hashlib.sha256() - with Path(path).open("rb") as stream: - for block in iter(lambda: stream.read(8 * 1024 * 1024), b""): - value.update(block) - return value.hexdigest() - - -def saved(path, cfg, step): - path = Path(path) - for name in REQUIRED: - if not (path / name).is_file() or (path / name).stat().st_size == 0: - raise ValueError(f"Incomplete checkpoint: {name}") - if not list(path.glob("*.safetensors")): - raise ValueError("Checkpoint has no model weights") - if json.loads((path / "trainer_state.json").read_text())["global_step"] != step: - raise ValueError("Checkpoint step mismatch") - files = {p.name: p.stat().st_size for p in path.iterdir() if p.is_file() and not p.name.startswith(".") and - p.name not in {"checkpoint.saved.json", "checkpoint.ready.json", "eval.request.json", "recipe.json"}} - write_json(path / "checkpoint.saved.json", {"step": step, "base_model": cfg["profile"]["id"], - "base_revision": cfg["profile"]["revision"], "files": files}) - - -def make_ready(path): - path = Path(path) - marker = json.loads((path / "checkpoint.saved.json").read_text()) - hashes = {} - for name, size in marker["files"].items(): - if Path(name).name != name or (path / name).stat().st_size != size: - raise ValueError("Checkpoint incomplete or changed after save") - hashes[name] = sha256(path / name) - if (path / name).stat().st_size != size: - raise ValueError("Checkpoint changed during hashing") - write_json(path / "checkpoint.ready.json", {**marker, "sha256": hashes}) - - -def verify(path, *, resume=False): - path = Path(path) - marker = json.loads((path / "checkpoint.ready.json").read_text()) - names = marker["sha256"] - for name, expected in names.items(): - if Path(name).name != name or sha256(path / name) != expected: - raise ValueError(f"Checkpoint integrity check failed: {name}") - if resume and any(n not in names for n in REQUIRED): - raise ValueError("Resume requires a complete optimizer checkpoint") - return marker diff --git a/05-multi-harness-rl/runtime/environment.py b/05-multi-harness-rl/runtime/environment.py deleted file mode 100644 index 7bfb4b19..00000000 --- a/05-multi-harness-rl/runtime/environment.py +++ /dev/null @@ -1,46 +0,0 @@ -"""A job-local OpenEnv service. Sandboxes remain isolated on the selected provider.""" -import os -import json -from pathlib import Path - - -def credentials(): - from huggingface_hub import get_token - token = get_token() - if not token: - raise ValueError("Log in with hf auth login; Harbor task staging requires HF_TOKEN") - os.environ["HF_TOKEN"] = token - - -def serve(cfg, data, vllm, port=8200, capture_port=8300): - import uvicorn - - credentials() - data = Path(data).resolve() - output = Path(cfg["output"]).resolve() - os.environ.update(OPENENV_LLM_URL=vllm, OPENENV_MODEL=cfg["profile"]["id"], - OPENENV_MAX_OUTPUT_TOKENS=str(cfg["max_output_tokens"]), ENABLE_WEB_INTERFACE="false", RUN_OWNER=cfg["run_name"], - MAX_CONCURRENT_ENVS=str(max(cfg["max_inflight"], cfg["eval_concurrency"]) + 8), - OPENENV_HARBOR_AGENT_VERSIONS=json.dumps(cfg["harness_versions"]), - OPENENV_CAPTURE_PORT=str(capture_port), OPENENV_EXPOSE="gradio") - if cfg["mode"] == "opencode": - os.environ.update(DATA_AGENT_SPLITS="train,test", DATA_AGENT_SANDBOX=cfg["sandbox"], - DATA_AGENT_OPENCODE_VERSION=cfg["harness_versions"]["opencode"], - DATA_AGENT_FROZEN_TASKS_DIR=str(data / "datasets"), DATA_AGENT_CAPTURE_EXPOSE="gradio", - DATA_AGENT_CAPTURE_PORT=str(capture_port), - DATA_AGENT_MAX_CONCURRENT=str(max(cfg["max_inflight"], cfg["eval_concurrency"]))) - from data_agent_env.server.app import app - elif cfg["mode"] == "whitebox": - if cfg["sandbox"] != "daytona": - raise ValueError("The SETA adapter currently requires Daytona") - os.environ.update(DAYTONA_COMPARISON_RUN=str(data), WHITE_BOX_BASH_TASK_SOURCE="harbor-frozen", - DAYTONA_WHITEBOX_TRIALS=str(output / "trials"), WHITEBOX_RECORDS=str(output / "whitebox-records"), - EFFICIENCY_WEIGHT=str(cfg["efficiency_weight"]), TOOL_BUDGET=str(cfg["tool_budget"]), - WHITE_BOX_BASH_MAX_SESSIONS=str(max(cfg["max_inflight"], cfg["eval_concurrency"]) + 8), - WHITE_BOX_BASH_MAX_CONCURRENT_ENVS=str(max(cfg["max_inflight"], cfg["eval_concurrency"]) + 8)) - from whitebox_bash.server.app import app - else: - os.environ.update(OPENENV_DATASETS=",".join(str(data / "datasets" / s) for s in ("train", "test")), - OPENENV_HARBOR_TRIALS_DIR=str(output / "trials")) - from harbor_env.server.app import app - uvicorn.run(app, host="127.0.0.1", port=port, ws_ping_timeout=None, timeout_keep_alive=120) diff --git a/05-multi-harness-rl/runtime/job.sh b/05-multi-harness-rl/runtime/job.sh deleted file mode 100644 index 070b0872..00000000 --- a/05-multi-harness-rl/runtime/job.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -mkdir -p /workspace/recipe -cp -a /recipe-source/. /workspace/recipe/ -cd /workspace/recipe -uv venv --python 3.12 .venv -export PATH="$PWD/.venv/bin:$PATH" -if [[ "$1" == "watch" ]]; then - shift - uv pip install --python .venv/bin/python huggingface_hub==1.29.0 - exec python3 eval/watch.py "$@" -fi -python3 runtime/bootstrap.py -uv pip install --python .venv/bin/python -r requirements.lock -uv pip install --python .venv/bin/python --no-deps --no-build-isolation -e .runtime/trl -python3 prepare.py -exec python3 run.py "$@" diff --git a/05-multi-harness-rl/runtime/launch.py b/05-multi-harness-rl/runtime/launch.py deleted file mode 100644 index ed71715b..00000000 --- a/05-multi-harness-rl/runtime/launch.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Submit the same job-local recipe to HF Jobs or Slurm; dry-run by default.""" -import argparse -import json -import os -from pathlib import Path -import re -import shlex -import shutil -import subprocess -import sys -import tempfile - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT)) -from recipe import MODES, config, write_json - - -def slurm_time(value): - match = re.fullmatch(r"([1-9][0-9]*)([hms])", value) - if not match: - raise ValueError("Timeout must be a positive duration such as 12h or 40m") - seconds = int(match[1]) * {"h": 3600, "m": 60, "s": 1}[match[2]] - return f"{seconds // 3600:02}:{seconds // 60 % 60:02}:{seconds % 60:02}" - - -def stage(destination): - destination = Path(destination) - allowed = {"run.py", "recipe.py", "prepare.py", "requirements.txt", "requirements.lock", "README.md", "REPRODUCE.md"} - for folder in ("configs", "data", "train", "eval", "runtime"): - allowed.update(str(p.relative_to(ROOT)) for p in (ROOT / folder).rglob("*") - if p.is_file() and p.suffix in {".py", ".sh", ".json"} and "__pycache__" not in p.parts) - for name in sorted(allowed): - if (ROOT / name).is_file(): - target = destination / name - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(ROOT / name, target) - - -def main(): - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("backend", choices=["hf", "slurm"]) - p.add_argument("action", choices=["train", "eval", "smoke", "pilot", "watch"]) - p.add_argument("--model", choices=["lfm", "qwen"], help="Override config model (default: lfm)") - p.add_argument("--config", type=Path) - p.add_argument("--mode", choices=MODES, help="Override config mode (default: multi-harness)") - p.add_argument("--run-name", required=True) - p.add_argument("--namespace", help="HF Jobs owner; defaults to the bucket owner") - p.add_argument("--bucket", help="Existing HF bucket, e.g. FineEnvs/smoldataenv-runs") - p.add_argument("--image", default="huggingface/trl@sha256:8433cde7eaf3b289f3daffdaefcab6bb71499ca1ff59dc5d05abba7a71603a58") - p.add_argument("--flavor", default="h200x2") - p.add_argument("--timeout", default="24h") - p.add_argument("--partition") - p.add_argument("--data", type=Path, default=ROOT / "prepared") - p.add_argument("--output-root", type=Path, default=ROOT / "runs") - p.add_argument("--checkpoint") - p.add_argument("--resume") - p.add_argument("--concurrency", type=int, help="Override eval concurrency (default: 35 from config)") - p.add_argument("--limit", type=int) - p.add_argument("--smoke-eval", action="store_true") - p.add_argument("--preflight-smoke", action="store_true") - p.add_argument("--max-active-evals", type=int, default=1) - p.add_argument("--submit", action="store_true") - args = p.parse_args() - if args.smoke_eval and args.action != "smoke": - p.error("--smoke-eval requires the smoke action") - if args.preflight_smoke and args.action != "pilot": - p.error("--preflight-smoke requires the pilot action") - if Path(args.run_name).name != args.run_name or args.run_name in {".", ".."}: - p.error("run-name must be a single directory name") - if args.config is None or not str(args.config).startswith("/outputs/"): - config(args.config, model=args.model, mode=args.mode, eval_concurrency=args.concurrency) - if args.limit is not None and not 1 <= args.limit <= 250: - p.error("--limit must be between 1 and the 250 fixed test tasks") - output = str(Path("/outputs" if args.backend == "hf" else args.output_root.resolve()) / args.run_name) - flags = [args.action, "--run-name", args.run_name, "--output", output] - if args.concurrency is not None: - flags += ["--concurrency", str(args.concurrency)] - for key in ("model", "mode"): - if getattr(args, key) is not None: - flags += ["--" + key, getattr(args, key)] - if args.smoke_eval: - flags.append("--smoke-eval") - if args.preflight_smoke: - flags.append("--preflight-smoke") - if args.config: - path = args.config - if args.backend == "hf" and not str(path).startswith("/outputs/"): - try: - relative = path.resolve().relative_to(ROOT) - except ValueError: - p.error("Put a custom HF config inside this recipe's configs directory") - if relative.parts[0] != "configs": - p.error("Custom configs must be in configs/") - path = Path("/workspace/recipe") / relative - flags += ["--config", str(path)] - for name in ("checkpoint", "resume", "limit"): - value = getattr(args, name) - if value is not None: - flags += ["--" + name, str(value)] - if args.backend == "hf": - if not args.bucket: - p.error("HF Jobs require --bucket for durable checkpoints and logs") - if len(args.bucket.split("/")) != 2 or not all(args.bucket.split("/")): - p.error("Use a bucket ID in owner/name form") - args.namespace = args.namespace or args.bucket.split("/", 1)[0] - gpu_count = re.search(r"x([0-9]+)$", args.flavor) - if args.action != "watch" and (gpu_count is None or int(gpu_count[1]) < 2): - p.error("This recipe needs at least two GPUs; use h200x2 or a100x4") - if args.action == "watch": - flags = ["watch", "--root", "/outputs", "--backend", "hf", "--bucket", args.bucket, - "--namespace", args.namespace, "--flavor", args.flavor, - "--concurrency", str(args.concurrency if args.concurrency is not None else 35), "--max-active", str(args.max_active_evals), "--submit"] - specification = {"image": args.image, "command": ["bash", "/recipe-source/runtime/job.sh", *flags], - "flavor": "cpu-basic" if args.action == "watch" else args.flavor, "timeout": args.timeout, "namespace": args.namespace, - "name": args.run_name, "labels": {"recipe": "smoldataenv-multi-harness", "role": args.action}, - "env": {"PYTHONUNBUFFERED": "1", **{k: os.environ[k] for k in - ("DAYTONA_API_URL", "DAYTONA_TARGET") if os.environ.get(k)}}} - print(json.dumps({**specification, "output_bucket": args.bucket, "secret_names": ["HF_TOKEN", "DAYTONA_API_KEY"]}, indent=2)) - if args.submit: - from huggingface_hub import get_token, run_job, sync_job_volume, Volume - token = get_token() - if not token or not os.environ.get("DAYTONA_API_KEY"): - p.error("HF login and DAYTONA_API_KEY are required") - with tempfile.TemporaryDirectory(prefix="smoldataenv-recipe-") as temp: - stage(temp) - source = sync_job_volume(temp, "/recipe-source", namespace=args.namespace) - volume = Volume(type="bucket", source=args.bucket, mount_path="/outputs", read_only=False) - job = run_job(**specification, volumes=[source, volume], - secrets={"HF_TOKEN": token, "DAYTONA_API_KEY": os.environ["DAYTONA_API_KEY"]}) - record = {"id": job.id, "url": job.url, "run_name": args.run_name, "namespace": args.namespace} - write_json(ROOT / "runs/submissions" / f"{args.run_name}.json", record) - print(json.dumps(record)) - else: - if not args.partition: - p.error("Choose an available Slurm partition explicitly") - args.output_root.mkdir(parents=True, exist_ok=True) - command = [sys.executable, str(ROOT / "run.py"), *flags, "--data", str(args.data.resolve())] - if args.action == "watch": - command = [sys.executable, str(ROOT / "eval/watch.py"), "--root", str(args.output_root.resolve()), - "--backend", "slurm", "--partition", args.partition, "--data", str(args.data.resolve()), - "--concurrency", str(args.concurrency if args.concurrency is not None else 35), "--max-active", str(args.max_active_evals), "--submit"] - submit = ["sbatch", "--parsable", "--partition", args.partition, "--nodes=1", "--gpus=2", - "--cpus-per-task=16", "--mem=192G", "--time=" + slurm_time(args.timeout), "--job-name", args.run_name, - "--output", str(args.output_root / (args.run_name + "-%j.log")), "--wrap", shlex.join(command)] - if args.action == "watch": - submit.remove("--gpus=2") - submit[submit.index("--cpus-per-task=16")] = "--cpus-per-task=1" - submit[submit.index("--mem=192G")] = "--mem=4G" - print(shlex.join(submit), flush=True) - if args.submit: - subprocess.run(submit, check=True) - - -if __name__ == "__main__": - main() diff --git a/05-multi-harness-rl/runtime/models.py b/05-multi-harness-rl/runtime/models.py deleted file mode 100644 index ec02f12b..00000000 --- a/05-multi-harness-rl/runtime/models.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Render the same non-thinking prompt in the inference engine and trainer.""" -from pathlib import Path - - -def training_attention_backend(capability): - major, _ = capability - if major == 9: - return "kernels-community/flash-attn3" - if major == 8: - return "kernels-community/flash-attn2" - raise ValueError(f"Unqualified training GPU capability: {capability}") - - -def tokenizer_for(cfg, template_path=None): - from copy import deepcopy - from transformers import AutoTokenizer - - profile = cfg["profile"] - tokenizer = AutoTokenizer.from_pretrained(profile["id"], revision=profile["revision"]) - if cfg["model"] == "lfm": - template = tokenizer.chat_template - old = '{{- "<|im_start|>assistant\\n" -}}' - if template.count(old) != 1: - raise ValueError("LFM template changed; requalify the non-thinking prefix") - tokenizer.chat_template = template.replace(old, '{{- "<|im_start|>assistant\\n" -}}') - from trl.chat_template_utils import lfm2_2_5_template - tokenizer.response_template = deepcopy(lfm2_2_5_template) - else: - from trl.chat_template_utils import qwen3_5_template - tokenizer.response_template = deepcopy(qwen3_5_template) - tokenizer.chat_template = "{%- set enable_thinking = false -%}\n" + tokenizer.chat_template - rendered = tokenizer.apply_chat_template([{"role": "user", "content": "Say OK."}], - tokenize=False, add_generation_prompt=True, enable_thinking=False, preserve_thinking=True) - tail = rendered.rsplit("<|im_start|>assistant", 1)[-1] - if "" not in tail or tail.count("") != tail.count(""): - raise ValueError("Generation prompt still opens a thinking block") - if template_path: - Path(template_path).write_text(tokenizer.chat_template) - return tokenizer - - -def serving_command(cfg, model, *, training, template_path, port=8000): - p = cfg["profile"] - cmd = ["vllm", "serve", model, "--host", "127.0.0.1", "--port", str(port), - "--served-model-name", p["id"], "--dtype", "bfloat16", "--tensor-parallel-size", "1", - "--data-parallel-size", "1" if training else str(cfg["eval_dp"]), - "--max-model-len", str(cfg["max_model_len"]), "--gpu-memory-utilization", "0.85", - "--enable-auto-tool-choice", "--tool-call-parser", p["tool_parser"], - "--chat-template", str(template_path), - "--default-chat-template-kwargs", '{"enable_thinking":false,"preserve_thinking":true}', - "--generation-config", "vllm", "--override-generation-config", - '{"temperature":0.8,"top_p":1.0,"top_k":-1}', - "--logprobs-mode", "processed_logprobs", "--return-tokens-as-token-ids", - "--enforce-eager", "--no-enable-prefix-caching"] - if model == p["id"]: - cmd += ["--revision", p["revision"]] - if training: - cmd += ["--weight-transfer-config", '{"backend":"nccl"}'] - if cfg["model"] == "qwen": - cmd += ["--limit-mm-per-prompt", '{"image":0,"video":0}', "--gdn-prefill-backend", "triton"] - return cmd - - -def check_visible_response(cfg, url): - """Catch non-thinking outputs being swallowed by a reasoning parser.""" - from openai import OpenAI - - checks = [] - with OpenAI(base_url=url.rstrip("/") + "/v1", api_key="unused", timeout=120) as client: - for stream in (False, True): - response = client.chat.completions.create(model=cfg["profile"]["id"], - messages=[{"role": "user", "content": "Reply with exactly OK."}], - temperature=0, max_tokens=32, stream=stream) - messages = ([c.choices[0].delta for c in response if c.choices] if stream else - [response.choices[0].message]) - content = "".join(m.content or "" for m in messages) - reasoning = "".join(str((m.model_extra or {}).get(key) or "") - for m in messages for key in ("reasoning", "reasoning_content")) - if not content.strip() or reasoning: - raise ValueError(f"Non-thinking response was empty or hidden in reasoning (stream={stream})") - checks.append({"stream": stream, "visible_chars": len(content), "reasoning_chars": len(reasoning)}) - return checks diff --git a/05-multi-harness-rl/runtime/native_capture.py b/05-multi-harness-rl/runtime/native_capture.py deleted file mode 100644 index 20456afc..00000000 --- a/05-multi-harness-rl/runtime/native_capture.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Typed producer boundary for the archived native OpenCode wire format.""" -from openenv.core.harness import TrainingTrace, TrainingTurn - - -def training_trace(result): - if result is None: - raise RuntimeError("No completed native rollout") - if result.rollout_type != "train": - raise ValueError("Native rollout has no training capture") - if any(f.startswith("[FATAL]") for f in result.metadata.get("capture_findings", [])): - raise ValueError("Native capture has fatal validation findings") - return TrainingTrace(turns=[TrainingTurn( - node_id=turn.capture_metadata["node_id"], - prompt_token_ids=turn.prompt_token_ids, - completion_token_ids=turn.completion_token_ids, - per_token_logps=turn.per_token_logps, - loss_mask=turn.loss_mask, - request={"messages": turn.request_messages, "tools": turn.request_tools}, - response={"choices": [{"message": {"role": "assistant", "content": turn.text, - "tool_calls": turn.tool_calls or None}, "finish_reason": turn.finish_reason}]}, - metadata=turn.capture_metadata, - ) for turn in result.turns]) diff --git a/05-multi-harness-rl/runtime/patches.py b/05-multi-harness-rl/runtime/patches.py deleted file mode 100644 index 6226a6ca..00000000 --- a/05-multi-harness-rl/runtime/patches.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Small, checked adapters to the immutable native environment dependencies.""" - - -def replace(text, old, new): - if text.count(old) != 1: - raise ValueError(f"Pinned runtime adapter no longer applies: {old[:80]}") - return text.replace(old, new, 1) - - -def restore_trl(root): - import subprocess - relative = "trl/experimental/async_grpo/async_grpo_trainer.py" - path = root / relative - if not path.exists(): - return - original = subprocess.check_output(["git", "-C", str(root), "show", f"HEAD:{relative}"], text=True) - adapted = original.replace('attn_implementation="kernels-community/flash-attn3",', - 'attn_implementation=model_init_kwargs.pop("attn_implementation", "kernels-community/flash-attn3"),', 1) - if path.read_text() == adapted: - path.write_text(original) - - -def apply_trl(root): - path = root / "trl/experimental/async_grpo/async_grpo_trainer.py" - old = 'attn_implementation="kernels-community/flash-attn3",' - new = 'attn_implementation=model_init_kwargs.pop("attn_implementation", "kernels-community/flash-attn3"),' - text = path.read_text() - if text.count(new) == 1 and old not in text: - return - path.write_text(replace(text, old, new)) - - -def apply(packages): - path = packages / "data_agent_env/server/capture.py" - path.write_text(replace(path.read_text(), "max_output_tokens=16384,", - 'max_output_tokens=int(os.environ["OPENENV_MAX_OUTPUT_TOKENS"]),')) - text = path.read_text() - text = replace(text, "from openenv.core.harness.capture import CaptureServer, to_trace_entries", - "from openenv.core.harness.capture import CaptureServer\nfrom openenv.core.harness.capture.contract import to_training_trace") - text = replace(text, "turns_from_capture(to_trace_entries(session.graph, document))", - "turns_from_capture(to_training_trace(session.graph, document).to_trace_entries())") - path.write_text(text) - path = packages / "data_agent_env/harness.py" - text = replace(path.read_text(), " def fetch_proxy_trace(self) -> list[TraceEntry]:", - " def fetch_training_trace(self):\n" - " from runtime.native_capture import training_trace\n" - " return training_trace(self._result)\n\n" - " def fetch_proxy_trace(self) -> list[TraceEntry]:") - path.write_text(text) - path = packages / "data_agent_env/server/rollout.py" - text = replace(path.read_text(), 'opencode run --print-logs "$(cat {config.home}/workdir/task.md)"', - 'opencode run --format json --print-logs "$(cat {config.home}/workdir/task.md)" > /tmp/native-events.jsonl') - text = replace(text, " timed_out = exit_code != 0", """ timed_out = exit_code != 0 - native_actions = None - if not timed_out: - from train.adapters import opencode_count - try: - native_actions = opencode_count(sandbox.read_text('/tmp/native-events.jsonl')) - except (ValueError, KeyError, TypeError, OSError): - pass""") - text = replace(text, ' **metadata_for(task, grade, n_tool_calls),', - ' **metadata_for(task, grade, n_tool_calls),\n "verified_native_actions": native_actions,') - path.write_text(text) - path = packages / "whitebox_bash/server/environment.py" - text = replace(path.read_text(), "from .sandbox import WORKDIR, ExecResult, Sandbox", - "from .sandbox import ExecResult\nfrom daytona_whitebox_backend import DaytonaSandbox as Sandbox\nWORKDIR = '/workdir'") - text = replace(text, "sb = Sandbox.start(timeout_s=SANDBOX_TIMEOUT_S,", "sb = Sandbox.start(task=task, timeout_s=SANDBOX_TIMEOUT_S,") - start = text.index(" check_passed: bool | None = None") - end = text.index(" return verdict.as_dict()", start) + len(" return verdict.as_dict()") - text = replace(text, text[start:end], """ from runtime.whitebox_reward import grade - try: - return grade(s) - finally: - _release(session_id)""") - path.write_text(text) - path = packages / "whitebox_bash/tasks.py" - text = replace(path.read_text(), ' if TASK_SOURCE == "data-agent" and split not in ("demo",):', - " if TASK_SOURCE == 'harbor-frozen':\n from daytona_whitebox_backend import load_frozen_tasks\n" - " tasks = load_frozen_tasks(split)\n _CACHE[split] = tasks\n return tasks\n" - ' if TASK_SOURCE == "data-agent" and split not in ("demo",):') - text = replace(text, ' d.pop("check", None)', ' d.pop("check", None)\n d.pop("metadata", None)\n d.pop("setup", None)') - path.write_text(text) - path = packages / "whitebox_bash/client.py" - text = path.read_text() - start = text.index(" # TRL's reward column is a float") - end = text.index(" if verdict.ungraded:", start) - text = replace(text, text[start:end], " # TRL ignores NaN rewards from ungraded episodes.\n") - text = replace(text, "reporting 0.0 to the trainer", "excluding reward from the trainer") - text = replace(text, "self._reward = 0.0 if verdict.reward is None else verdict.reward", - 'self._reward = float("nan") if verdict.reward is None else verdict.reward') - path.write_text(text) diff --git a/05-multi-harness-rl/runtime/pilot.py b/05-multi-harness-rl/runtime/pilot.py deleted file mode 100644 index 5b684cd9..00000000 --- a/05-multi-harness-rl/runtime/pilot.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Baseline, 100 training updates and checkpoint evaluation in one allocation.""" -import argparse -import json -from pathlib import Path -import subprocess -import sys -import time - -from recipe import ROOT, digest, eval_rows, mean, task_rows, write_json -from runtime.checkpoints import make_ready - - -def pilot(args, cfg): - from run import launch_local - - cfg = {**cfg, "max_steps": 100, "save_steps": 50, "eval_steps": 100, "eval_selection": "stratified"} - root = Path(cfg["output"]) - root.mkdir(parents=True, exist_ok=True) - limit = args.limit if args.limit is not None else 25 - selected = eval_rows(task_rows("test"), limit, stratified=True, seed=cfg["seed"]) - preflight = bool(getattr(args, "preflight_smoke", False)) - identity = {"config": cfg, "eval_tasks": selected, **({"preflight_smoke": True} if preflight else {})} - signature = digest(identity) - manifest = root / "pilot.json" - state = json.loads(manifest.read_text()) if manifest.exists() else { - "sha256": signature, **identity, "phases": {}} - if state["sha256"] != signature: - raise ValueError("Pilot output belongs to a different configuration or test selection") - if args.resume and not state["phases"].get("baseline", {}).get("complete"): - raise ValueError("Resume requires this pilot's completed baseline") - - phases = (["preflight"] if preflight else []) + ["baseline", "train", "checkpoint-100"] - for phase in phases: - if state["phases"].get(phase, {}).get("complete"): - continue - entry = {"complete": False, "started_at": time.time()} - state["phases"][phase] = entry - write_json(manifest, state) - print(json.dumps({"pilot": cfg["run_name"], "phase": phase, "status": "started"}), flush=True) - current = argparse.Namespace(**vars(args)) - current.action = "train" if phase == "train" else "eval" - current.limit = limit - current.checkpoint = None - current.resume = args.resume if phase == "train" else None - current.smoke_eval = False - phase_cfg = {**cfg, "output": str(root / phase), "run_name": cfg["run_name"] + "-" + phase} - try: - if phase == "preflight": - path = root / "preflight-config.json" - write_json(path, phase_cfg) - subprocess.run([sys.executable, str(ROOT / "run.py"), "smoke", "--config", str(path), - "--data", str(args.data.resolve()), "--smoke-eval", "--limit", "2", - "--concurrency", "4"], check=True) - report = json.loads((root / "preflight/reload-eval/eval/summary.json").read_text()) - if not report["complete"]: - raise ValueError("Preflight reload evaluation is incomplete") - entry.update(complete=True, evaluation=report, finished_at=time.time()) - write_json(manifest, state) - print(json.dumps({"pilot": cfg["run_name"], "phase": phase, "status": "complete"}), flush=True) - continue - if current.resume: - make_ready(current.resume) - if phase == "checkpoint-100": - current.checkpoint = root / "train/checkpoint-100" - make_ready(current.checkpoint) - launch_local(current, phase_cfg) - if phase != "train": - report = json.loads((Path(phase_cfg["output"]) / "eval/summary.json").read_text()) - if not report["complete"]: - raise ValueError("Pilot requires complete evaluation coverage") - entry["evaluation"] = report - else: - marker = root / "train/checkpoint-100/checkpoint.saved.json" - if not marker.exists(): - raise ValueError("Training ended before checkpoint 100") - entry.update(complete=True, finished_at=time.time()) - write_json(manifest, state) - print(json.dumps({"pilot": cfg["run_name"], "phase": phase, "status": "complete"}), flush=True) - except BaseException as exc: - entry.update(error=type(exc).__name__, finished_at=time.time()) - write_json(manifest, state) - raise - write_json(root / "comparison.json", comparison(root, state)) - - -def comparison(root, state): - before = state["phases"]["baseline"]["evaluation"] - after = state["phases"]["checkpoint-100"]["evaluation"] - metrics = [json.loads(line) for line in (root / "train/metrics.jsonl").read_text().splitlines()] - committed = {row["step"]: row for row in metrics if "grad_norm" in row} - updates = [committed[step] for step in sorted(committed)] - result = {"baseline": before, "checkpoint_100": after, - "pass_at_1_change_pp": 100 * (after["pass_at_1"] - before["pass_at_1"]), - "updates_with_gradient_metrics": len(updates), - "nonzero_gradient_updates": sum(row["grad_norm"] > 0 for row in updates), - "zero_gradient_updates": sum(row["grad_norm"] == 0 for row in updates), - "note": "Mean tool usage includes different success sets; it is not matched-success efficiency."} - result["harnesses"] = {h: {"baseline": before["harnesses"][h], "checkpoint_100": values, - "pass_at_1_change_pp": 100 * (values["pass_at_1"] - before["harnesses"][h]["pass_at_1"])} - for h, values in after["harnesses"].items()} - for label, rows in (("first_20_updates", updates[:20]), ("last_20_updates", updates[-20:])): - result[label] = {"count": len(rows), "mean_reward": mean(row.get("reward") for row in rows), - "mean_grad_norm": mean(row["grad_norm"] for row in rows)} - base_pairs = {p.name: json.loads(p.read_text()) for p in (root / "baseline/eval/pairs").glob("*.json")} - matches = [] - for path in (root / "checkpoint-100/eval/pairs").glob("*.json"): - a, b = base_pairs[path.name], json.loads(path.read_text()) - if a.get("correctness") == b.get("correctness") == 1 and not a.get("error") and not b.get("error"): - matches.append((a, b)) - result["matched_successes"] = {} - for harness in ["all", *after["harnesses"]]: - pairs = [(a, b) for a, b in matches if harness == "all" or a["harness"] == harness] - values = {"pairs": len(pairs)} - for key in ("tool_calls", "generated_tokens"): - valid = [(a[key], b[key]) for a, b in pairs if a.get(key) is not None and b.get(key) is not None] - denominator = sum(a for a, _ in valid) - values[key + "_pairs"] = len(valid) - values[key + "_savings_pct"] = 100 * (1 - sum(b for _, b in valid) / denominator) if denominator else None - result["matched_successes"][harness] = values - return result diff --git a/05-multi-harness-rl/runtime/whitebox_reward.py b/05-multi-harness-rl/runtime/whitebox_reward.py deleted file mode 100644 index 6387d7fd..00000000 --- a/05-multi-harness-rl/runtime/whitebox_reward.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Keep native correctness and verified tool count separate from shaping.""" -import os -from pathlib import Path - -from recipe import reward, write_json - - -def grade(session): - verdict = session.sandbox.grade(session.submitted) - correctness = verdict.get("reward") - calls = sum(c["tool"] != "submit_solution" for c in session.calls) - weight = float(os.environ["EFFICIENCY_WEIGHT"]) if session.split == "train" else 0.0 - value = reward(correctness, calls, verified=True, weight=weight, - budget=float(os.environ["TOOL_BUDGET"])) - record = {"correctness": correctness, "tool_calls": calls, "reward": value, - "task_index": session.index, "split": session.split, "tool_count_verified": True} - write_json(Path(os.environ["WHITEBOX_RECORDS"]) / f"{session.session_id}.json", record) - return {**verdict, "reward": value, "n_tool_calls": calls} diff --git a/05-multi-harness-rl/tests/conftest.py b/05-multi-harness-rl/tests/conftest.py new file mode 100644 index 00000000..e3883ff7 --- /dev/null +++ b/05-multi-harness-rl/tests/conftest.py @@ -0,0 +1,26 @@ +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--contract", + action="store_true", + help="Run integration checks against installed TRL and OpenEnv", + ) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", "contract: requires the tutorial's installed runtime" + ) + + +def pytest_collection_modifyitems(config, items): + if not config.getoption("--contract"): + for item in items: + if "contract" in item.keywords: + item.add_marker( + pytest.mark.skip( + reason="Use --contract after installing TRL/OpenEnv" + ) + ) diff --git a/05-multi-harness-rl/tests/test_contract.py b/05-multi-harness-rl/tests/test_contract.py new file mode 100644 index 00000000..2d669c09 --- /dev/null +++ b/05-multi-harness-rl/tests/test_contract.py @@ -0,0 +1,244 @@ +"""Run with --contract after installing the tutorial dependencies.""" + +import json +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +pytestmark = pytest.mark.contract + + +@pytest.fixture +def trace(): + from openenv.core.harness import TrainingTrace, TrainingTurn + + return TrainingTrace( + turns=[ + TrainingTurn( + node_id="agent", + prompt_token_ids=[1, 2], + completion_token_ids=[3, 4, 5], + per_token_logps=[-0.1, -0.2, -0.3], + loss_mask=[0, 0, 1, 0, 1], + ), + TrainingTurn( + node_id="context", + prompt_token_ids=[1], + completion_token_ids=[2], + per_token_logps=[-0.3], + loss_mask=[0, 0], + ), + ] + ) + + +def test_native_trace_preserves_masks_and_passes_native_count_to_trl(trace): + from trl.experimental.async_grpo.openenv_harness import _turns_from_training_trace + + from train_opencode import TaskSession, reward + + events = [{"type": "step_finish"}] + [ + { + "type": "tool_use", + "part": {"callID": str(i), "state": {"status": "completed"}}, + } + for i in range(6) + ] + native = Mock() + native.fetch_training_trace.return_value = trace + native.fetch_trace.return_value = "\n".join(json.dumps(e) for e in events) + result = TaskSession(native).fetch_training_trace() + assert [t.loss_mask for t in result.turns] == [[0, 0, 1, 0, 1], [0, 0]] + # Exercise the real typed consumer, including diagnostic metadata used by reward(). + records = _turns_from_training_trace(result) + entries = result.to_trace_entries() + assert records[0].output_mask == [1, 0, 1] + assert records[1].output_mask == [0] + assert reward(SimpleNamespace(env_reward=1.0, trace=entries)) == pytest.approx( + 1 + 1.5 / 21 + ) + + +@pytest.mark.parametrize( + "grade,calls,expected", + [(None, 4, None), (0, 4, 0), (1, None, 1), (1, 0, 1), (1, 15, 1.05)], +) +def test_both_async_rewards_match(grade, calls, expected): + from train_multi_harness import reward as harbor + from train_opencode import reward as native + + outcome = SimpleNamespace( + env_reward=grade, trace=[{"metadata": {"native_tool_calls": calls}}] + ) + assert native(outcome) == expected + assert harbor(outcome) == expected + + +def test_harbor_assignment_depends_on_prompt_not_seed(monkeypatch): + import train_multi_harness as tutorial + + rows = [{"instruction": str(i)} for i in range(8)] + args = SimpleNamespace( + server="unused", + data="prepared", + vllm_url="unused", + model="test", + trials="trials", + ) + monkeypatch.setattr(tutorial, "HarborEnv", Mock()) + constructor = Mock() + monkeypatch.setattr(tutorial, "TaskSession", constructor) + factory = tutorial.TaskFactory(args, rows, sampling={"temperature": 0.8}) + for seed in (0, 17, 9000): + factory.create([{"role": "user", "content": "5"}], seed=seed) + assert constructor.call_args.kwargs["task_index"] == 5 + assert constructor.call_args.kwargs["harness"] == "claude-code" + assert constructor.call_args.kwargs["sampling"] == {"temperature": 0.8} + + +def test_native_setup_failure_closes_sandbox(monkeypatch): + import train_opencode as tutorial + + monkeypatch.setenv("SANDBOX_VLLM_URL", "https://example.test") + monkeypatch.setenv("SANDBOX_VLLM_KEY", "test-key") + factory = tutorial.TaskFactory( + SimpleNamespace(model="test"), + [{"instruction": "task", "folder": "unused"}], + sampling={"temperature": 0.8}, + ) + session = Mock() + factory.factory = Mock() + factory.factory.create.return_value = session + monkeypatch.setattr( + tutorial, "stage_task", Mock(side_effect=RuntimeError("staging")) + ) + with pytest.raises(RuntimeError, match="staging"): + factory.create([{"role": "user", "content": "task"}], seed=999) + session.close.assert_called_once() + session.start_agent.assert_not_called() + + +def test_whitebox_reward_and_cleanup(monkeypatch): + import train_whitebox as tutorial + + env = tutorial.BashEnvironment() + sandbox = Mock() + env._sandbox, env._folder, env._calls = sandbox, Path("unused"), 15 + monkeypatch.setattr(tutorial, "grade_answer", lambda *args: 1.0) + assert env.get_reward() == 1.05 + sandbox.kill.assert_called_once() + assert env._sandbox is None + + +@pytest.mark.parametrize( + "module", ["train_opencode", "train_multi_harness", "train_whitebox"] +) +@pytest.mark.parametrize("model", ["LiquidAI/LFM2.5-2.6B", "Qwen/Qwen3.5-2B"]) +def test_nonthinking_prompt(module, model): + import importlib + + tokenizer = importlib.import_module(module).tokenizer_for(model) + prompt = tokenizer.apply_chat_template( + [{"role": "user", "content": "Say OK"}], + tokenize=False, + add_generation_prompt=True, + enable_thinking=False, + ) + tail = prompt.rsplit("<|im_start|>assistant", 1)[-1] + assert "" in tail and tail.count("") == tail.count("") + + +def test_whitebox_public_tool_schemas_are_valid(): + import inspect + + from transformers.utils import get_json_schema + + from train_whitebox import BashEnvironment + + env = BashEnvironment() + tools = [ + get_json_schema(method)["function"]["name"] + for name, method in inspect.getmembers(env, inspect.ismethod) + if not name.startswith("_") and name not in {"reset", "get_reward"} + ] + assert set(tools) == { + "bash", + "read", + "write", + "edit", + "ls", + "grep", + "glob", + "submit_solution", + } + + +def test_inference_proxy_blocks_unauthorized_and_admin_routes(): + from fastapi.testclient import TestClient + + from jobs.inference_proxy import create_app + + with TestClient(create_app("test-secret")) as client: + assert client.get("/v1/models").status_code == 401 + assert ( + client.get( + "/v1/models", headers={"Authorization": "Bearer wrong"} + ).status_code + == 401 + ) + assert ( + client.post( + "/v1/update_weights", headers={"Authorization": "Bearer test-secret"} + ).status_code + == 404 + ) + assert ( + client.post( + "/update_weights", headers={"Authorization": "Bearer test-secret"} + ).status_code + == 404 + ) + + +def test_public_config_signatures_accept_tutorial_keywords(): + import ast + import inspect + + from trl import GRPOConfig + from trl.experimental.async_grpo import AsyncGRPOConfig + + for name, cls in [ + ("whitebox", GRPOConfig), + ("opencode", AsyncGRPOConfig), + ("multi_harness", AsyncGRPOConfig), + ]: + tree = ast.parse((ROOT / f"train_{name}.py").read_text()) + calls = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == cls.__name__ + ] + assert len(calls) == 1 + assert {kw.arg for kw in calls[0].keywords} <= set( + inspect.signature(cls).parameters + ) + + +def test_whitebox_does_not_count_submission(monkeypatch): + import time + + from train_whitebox import BashEnvironment + + env = BashEnvironment() + env._sandbox, env._deadline, env._calls = Mock(), time.monotonic() + 60, 3 + env.submit_solution("42") + assert env._calls == 3 + with pytest.raises(RuntimeError, match="finished"): + env.bash("echo retry") diff --git a/05-multi-harness-rl/tests/test_contracts.py b/05-multi-harness-rl/tests/test_contracts.py deleted file mode 100644 index d5686806..00000000 --- a/05-multi-harness-rl/tests/test_contracts.py +++ /dev/null @@ -1,337 +0,0 @@ -import json -from collections import Counter -from pathlib import Path -import shutil -import sys -from types import SimpleNamespace - -import pytest - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT)) -from recipe import config, digest, reward, resume_state, schedule, summary, task_rows, write_json -from train.adapters import atif_count, opencode_count -from train.generation import group_size, protect_generation -from eval.evaluate import capture_metrics -from runtime.checkpoints import make_ready, saved, verify -from runtime.launch import stage -from runtime.models import check_visible_response - - -def test_ampere_attention_keeps_packed_flash_attention(tmp_path): - from runtime.models import training_attention_backend - from runtime.patches import apply_trl - assert training_attention_backend((8, 0)) == "kernels-community/flash-attn2" - assert training_attention_backend((9, 0)) == "kernels-community/flash-attn3" - with pytest.raises(ValueError, match="Unqualified"): - training_attention_backend((7, 0)) - path = tmp_path / "trl/experimental/async_grpo/async_grpo_trainer.py" - path.parent.mkdir(parents=True) - path.write_text('model = create_model_from_path(model, attn_implementation="kernels-community/flash-attn3", **model_init_kwargs)') - apply_trl(tmp_path) - apply_trl(tmp_path) - calls = [] - kwargs = {"attn_implementation": training_attention_backend((8, 0)), "revision": "fixed"} - exec(path.read_text(), {"create_model_from_path": lambda *a, **k: calls.append(k), - "model": "test", "model_init_kwargs": kwargs}) - assert calls == [{"attn_implementation": "kernels-community/flash-attn2", "revision": "fixed"}] - - -def test_smoke_reload_is_separate_and_training_failure_stops_eval(tmp_path, monkeypatch): - import run - import runtime.checkpoints - calls = [] - monkeypatch.setattr(sys, "argv", ["run.py", "smoke", "--model", "lfm", "--mode", "multi-harness", - "--output", str(tmp_path), "--smoke-eval", "--concurrency", "4"]) - monkeypatch.setattr(runtime.checkpoints, "make_ready", lambda p: calls.append(("ready", p))) - monkeypatch.setattr(run, "launch_local", lambda a, c: calls.append((a.action, a.checkpoint, c["output"], a.limit))) - run.main() - assert calls[0][0] == "smoke" - assert calls[1] == ("ready", tmp_path / "checkpoint-2") - assert calls[2] == ("eval", tmp_path / "checkpoint-2", str(tmp_path / "reload-eval"), 2) - calls.clear() - def fail(*args): - raise RuntimeError("training failed") - monkeypatch.setattr(run, "launch_local", fail) - with pytest.raises(RuntimeError, match="training failed"): - run.main() - assert not calls - - -def test_frozen_data_and_rotation(): - train, test = task_rows("train"), task_rows("test") - assert len(train) == 1000 and len(test) == 250 - assert Counter(r["difficulty"] for r in train) == {"medium": 400, "hard": 600} - assert Counter(r["difficulty"] for r in test) == {"easy": 33, "medium": 118, "hard": 99} - for key in ("source", "notebook", "instruction_sha256"): - assert not {r[key] for r in train} & {r[key] for r in test} - groups = schedule(config(), train) - assert len(groups) == 2000 - assert Counter(g["harness"] for g in groups[:1000]) == {h: 250 for h in config()["harnesses"]} - for a, b in zip(groups[:1000], groups[1000:], strict=True): - assert a["task_name"] == b["task_name"] and a["harness"] != b["harness"] - - -def test_pilot_subset_is_fixed_and_covers_difficulty(): - from recipe import eval_rows - rows = task_rows("test") - chosen = eval_rows(rows, 25, stratified=True) - assert chosen == eval_rows(rows[::-1], 25, stratified=True) - assert Counter(r["difficulty"] for r in chosen) == {"easy": 3, "medium": 12, "hard": 10} - assert len({r["name"] for r in chosen}) == 25 - assert eval_rows(rows, 250, stratified=True) == eval_rows(rows) - with pytest.raises(ValueError): - eval_rows(rows, 0) - - -def test_slurm_uses_the_requested_allocation_limit(): - from runtime.launch import slurm_time - assert slurm_time("12h") == "12:00:00" - assert slurm_time("45m") == "00:45:00" - with pytest.raises(ValueError): - slurm_time("0h") - - -def test_eval_subset_retains_original_catalog_indices(tmp_path, monkeypatch): - import importlib - evaluator = importlib.import_module("eval.evaluate") - from recipe import eval_rows - rows = sorted(task_rows("test"), key=lambda r: r["name"]) - chosen = eval_rows(rows, 25, stratified=True) - seen = [] - monkeypatch.setattr(evaluator, "Factory", lambda cfg, data, server, vllm, groups, *a, **k: SimpleNamespace(groups=groups)) - def episode(factory, i): - group = factory.groups[i] - assert rows[group["task_index"]]["name"] == group["task_name"] - seen.append(group["task_name"]) - return {"correctness": 0} - monkeypatch.setattr(evaluator, "blackbox_episode", episode) - monkeypatch.setitem(sys.modules, "trackio", SimpleNamespace(init=lambda **kw: SimpleNamespace(log=lambda *a, **k: None, finish=lambda: None))) - cfg = {**config(), "output": str(tmp_path), "run_name": "subset-test", "eval_selection": "stratified"} - result = evaluator.evaluate(cfg, tmp_path, "unused", "unused", limit=25) - assert result["graded"] == 100 - assert set(seen) == {r["name"] for r in chosen} - - -def test_pilot_stops_on_incomplete_baseline_and_preserves_full_training_config(tmp_path, monkeypatch): - import run - import runtime.pilot - import runtime.checkpoints - calls = [] - args = SimpleNamespace(limit=25, resume=None) - cfg = {**config(mode="multi-harness"), "output": str(tmp_path), "run_name": "pilot-test"} - def fake_launch(a, c): - calls.append((a.action, c)) - output = Path(c["output"]) - if a.action == "train": - write_json(output / "checkpoint-100/checkpoint.saved.json", {"step": 100}) - else: - write_json(output / "eval/summary.json", {"complete": True}) - monkeypatch.setattr(run, "launch_local", fake_launch) - monkeypatch.setattr(runtime.pilot, "make_ready", lambda p: None) - monkeypatch.setattr(runtime.pilot, "comparison", lambda *a: {"complete": True}) - runtime.pilot.pilot(args, cfg) - assert [a for a, _ in calls] == ["eval", "train", "eval"] - train_cfg = calls[1][1] - assert (train_cfg["max_steps"], train_cfg["save_steps"], train_cfg["num_generations"], train_cfg["batch_size"]) == (100, 50, 8, 4) - calls.clear() - runtime.pilot.pilot(args, cfg) - assert not calls - cfg["output"] = str(tmp_path / "failed") - def incomplete(a, c): - write_json(Path(c["output"]) / "eval/summary.json", {"complete": False}) - monkeypatch.setattr(run, "launch_local", incomplete) - with pytest.raises(ValueError, match="complete evaluation"): - runtime.pilot.pilot(args, cfg) - state = json.loads((Path(cfg["output"]) / "pilot.json").read_text()) - assert not state["phases"]["baseline"]["complete"] - assert "train" not in state["phases"] - cfg["output"] = str(tmp_path / "failed-preflight") - args.preflight_smoke, args.data = True, tmp_path - def failed_preflight(*a, **kw): - raise runtime.pilot.subprocess.CalledProcessError(1, "smoke") - monkeypatch.setattr(runtime.pilot.subprocess, "run", failed_preflight) - with pytest.raises(runtime.pilot.subprocess.CalledProcessError): - runtime.pilot.pilot(args, cfg) - state = json.loads((Path(cfg["output"]) / "pilot.json").read_text()) - assert not state["phases"]["preflight"]["complete"] - assert "baseline" not in state["phases"] - - -def test_native_means_native(): - assert {g["harness"] for g in schedule(config(mode="opencode"), task_rows("train"))} == {"opencode"} - - -def test_config_overlays_preserve_defaults_and_explicit_overrides(tmp_path): - path = tmp_path / "my-run.json" - write_json(path, {"model": "qwen", "mode": "whitebox", "eval_concurrency": 7, - "models": {"qwen": {"template": "enable_thinking_false"}}}) - cfg = config(path) - assert cfg["model"] == "qwen" and cfg["mode"] == "whitebox" and cfg["eval_concurrency"] == 7 - assert cfg["learning_rate"] == 3e-6 - assert cfg["profile"]["id"] == "Qwen/Qwen3.5-2B" - assert "lfm" in cfg["models"] - assert config(path, model="lfm")["profile"]["id"] == "LiquidAI/LFM2.5-2.6B" - - -def test_submission_uses_bucket_owner_without_overriding_config(monkeypatch, capsys): - from runtime import launch - monkeypatch.setattr(sys, "argv", ["launch.py", "hf", "pilot", "--bucket", "reader/my-runs", - "--run-name", "test", "--flavor", "a100x4"]) - launch.main() - plan = json.loads(capsys.readouterr().out) - assert plan["namespace"] == "reader" - assert "--mode" not in plan["command"] and "--model" not in plan["command"] - assert "--concurrency" not in plan["command"] - monkeypatch.setattr(sys, "argv", ["launch.py", "hf", "pilot", "--bucket", "reader/my-runs", - "--run-name", "test", "--flavor", "a100-large"]) - with pytest.raises(SystemExit): - launch.main() - - -def test_nonthinking_preflight_checks_streaming_content(monkeypatch): - hidden = False - def create(*, stream, **kwargs): - bad = stream and hidden - message = SimpleNamespace(content="" if bad else "OK", model_extra={"reasoning": "OK" if bad else None}) - choice = SimpleNamespace(delta=message, message=message) - response = SimpleNamespace(choices=[choice]) - return [response] if stream else response - class Client: - def __init__(self, **kwargs): - self.chat = SimpleNamespace(completions=SimpleNamespace(create=create)) - def __enter__(self): return self - def __exit__(self, *args): pass - monkeypatch.setitem(sys.modules, "openai", SimpleNamespace(OpenAI=Client)) - assert len(check_visible_response(config(), "http://localhost")) == 2 - hidden = True - with pytest.raises(ValueError, match="stream=True"): - check_visible_response(config(), "http://localhost") - - -@pytest.mark.parametrize("kwargs", [{"eval_steps": 75}, {"num_generations": 1}, {"top_p": .95}, - {"max_outstanding_rollouts": 8}, {"eval_concurrency": 0}]) -def test_reject_invalid_config(kwargs): - with pytest.raises(ValueError): - config(**kwargs) - - -def test_resume_skips_committed_groups_and_retries_uncommitted(): - groups = schedule(config(), task_rows("train")) - state = {"execution_schedule_sha256": digest(groups), "settled_groups": [0, 1, 2], - "admitted_rollouts": {"a": 0, "b": 2}} - assert resume_state(groups, state)["settled_groups"] == [0, 2] - with pytest.raises(ValueError): - resume_state(groups[::-1], state) - - -def test_reward_does_not_pay_for_failure_or_unknown_counts(): - assert reward(None, 2, verified=True) is None - assert reward(0, 1, verified=True) == 0 - assert reward(1, None, verified=False) == 1 - assert reward(1, 0, verified=True) == 1 - assert reward(1, 15, verified=True) == 1.05 - assert reward(1, 2, verified=True) > reward(1, 20, verified=True) - - -def test_native_counts_require_unambiguous_actions(tmp_path): - path = tmp_path / "trajectory.json" - write_json(path, {"schema_version": "ATIF-v1.6", "steps": [ - {"source": "agent", "tool_calls": [{"tool_call_id": "a"}, {"tool_call_id": "b"}]}, - {"source": "environment", "tool_calls": [{"tool_call_id": "a"}]}]}) - assert atif_count(path) == 2 - event = {"type": "tool_use", "part": {"callID": "a", "state": {"status": "completed"}}} - stream = json.dumps(event) + '\n{"type":"step_finish"}' - assert opencode_count(stream) == 1 - with pytest.raises(ValueError): - opencode_count(stream + "\n" + json.dumps(event)) - with pytest.raises(ValueError): - opencode_count(json.dumps(event)) - - -def test_diverged_tool_histories_are_never_collapsed(): - assert group_size([[1], [1], [2], [2]], 2) == 2 - assert group_size([[1], [3], [2], [2]], 2) == 1 - assert group_size([[1], [1], [2]], 2) == 1 - - -def test_whitebox_cache_reset_accepts_empty_success_and_rejects_failure(): - import requests - response = requests.Response() - response.status_code = 200 - response._content = b"" - client = SimpleNamespace(base_url="http://localhost", session=SimpleNamespace(post=lambda url: response)) - trainer = SimpleNamespace(vllm_generation=SimpleNamespace(vllm_client=client, generate=lambda **kw: None)) - protect_generation(trainer) - client.reset_prefix_cache() - response.status_code = 500 - with pytest.raises(requests.HTTPError): - client.reset_prefix_cache() - - -def test_capture_uses_engine_tokens_and_masks(): - turn = SimpleNamespace(trainable=True, prompt_token_ids=[10, 20], completion_token_ids=[30], - per_token_logps=[-.5], loss_mask=[0, 0, 1]) - result = SimpleNamespace(turns=[turn]) - assert capture_metrics(result)["generated_tokens"] == 1 - turn.loss_mask = [1, 0, 1] - with pytest.raises(ValueError): - capture_metrics(result) - turn.loss_mask = [0, 0, 1] - turn.per_token_logps = [float("nan")] - with pytest.raises(ValueError): - capture_metrics(result) - - -def test_ungraded_is_not_incorrect(): - result = summary([{"correctness": 1}, {"correctness": 0}, {"correctness": None, "error": "transport"}], 3) - assert result["pass_at_1"] == .5 and result["graded"] == 2 and not result["complete"] - assert summary([], 1)["pass_at_1"] is None - - -def test_checkpoint_transfer_is_portable_and_detects_tampering(tmp_path): - checkpoint = tmp_path / "original" - checkpoint.mkdir() - for name in ("config.json", "tokenizer_config.json", "tokenizer.json", "training_args.bin", - "optimizer.pt", "scheduler.pt", "rng_state.pth", "model.safetensors"): - (checkpoint / name).write_bytes(b"fixture") - write_json(checkpoint / "trainer_state.json", {"global_step": 50}) - saved(checkpoint, config(), 50) - make_ready(checkpoint) - moved = tmp_path / "different-root" / "checkpoint-50" - shutil.copytree(checkpoint, moved) - assert verify(moved, resume=True)["step"] == 50 - (moved / "model.safetensors").write_bytes(b"changed") - with pytest.raises(ValueError): - verify(moved) - - -def test_hf_source_bundle_omits_local_state(tmp_path): - stage(tmp_path) - paths = [str(p.relative_to(tmp_path)) for p in tmp_path.rglob("*")] - assert "configs/runtime-lock.json" in paths - assert not any(p.startswith((".env", ".runtime", "runs", "prepared")) for p in paths) - - -def test_runtime_patch_restore_handles_fresh_clone_and_preserves_edits(tmp_path): - import subprocess - from runtime.patches import apply_trl, restore_trl - root = tmp_path / "checkout" - root.mkdir() - restore_trl(root) # A --no-checkout clone has no worktree files yet. - path = root / "trl/experimental/async_grpo/async_grpo_trainer.py" - path.parent.mkdir(parents=True) - original = 'attn_implementation="kernels-community/flash-attn3",\n' - path.write_text(original) - subprocess.run(["git", "init", "-q", str(root)], check=True) - subprocess.run(["git", "-C", str(root), "add", "."], check=True) - subprocess.run(["git", "-C", str(root), "-c", "user.name=Test", "-c", "user.email=test@example.org", - "commit", "-qm", "Initial"], check=True) - apply_trl(root) - restore_trl(root) - assert path.read_text() == original - apply_trl(root) - edited = path.read_text() + "# local user change\n" - path.write_text(edited) - restore_trl(root) - assert path.read_text() == edited diff --git a/05-multi-harness-rl/tests/test_data_and_eval.py b/05-multi-harness-rl/tests/test_data_and_eval.py new file mode 100644 index 00000000..d5dd6953 --- /dev/null +++ b/05-multi-harness-rl/tests/test_data_and_eval.py @@ -0,0 +1,61 @@ +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from envs.tasks import stage_task +from evaluate import summarize, write_json + + +def test_fixed_split_is_disjoint_and_keeps_the_training_mix(): + train = json.loads((ROOT / "data/train.json").read_text()) + test = json.loads((ROOT / "data/test.json").read_text()) + assert len(train) == 1000 and len(test) == 250 + assert sum(row["difficulty"] == "medium" for row in train) == 400 + assert sum(row["difficulty"] == "hard" for row in train) == 600 + for key in ("source", "notebook", "instruction_sha256"): + assert not {t[key] for t in train} & {t[key] for t in test} + + +def test_ungraded_is_excluded_from_pass_at_one_but_visible_in_coverage(): + result = summarize( + [ + {"correctness": 1, "tool_calls": 6}, + {"correctness": 0}, + {"correctness": None}, + ], + 4, + ) + assert result["pass_at_1_observed"] == 0.5 + assert result["coverage"] == 0.5 + assert result["mean_tool_calls"] == 6 + assert result["tool_count_coverage"] == 1 + assert summarize([], 4)["pass_at_1_observed"] is None + + +def test_staging_does_not_upload_gold_or_leave_tokens_in_files(tmp_path, monkeypatch): + monkeypatch.setenv("HF_TOKEN", "test-download-token") + (tmp_path / "environment").mkdir() + (tmp_path / "environment/pull_bucket.py").write_text("print('download')") + (tmp_path / "task.toml").write_text( + '[environment.env]\nHF_TOKEN="${HF_TOKEN}"\n[verifier.env]\nEXPECTED_ANSWER="hidden-gold"\n' + ) + writes, commands = [], [] + sandbox = SimpleNamespace( + write_text=lambda *x: writes.append(x), + exec=lambda *x, **kw: commands.append((x, kw)) or SimpleNamespace(exit_code=0), + ) + stage_task(sandbox, tmp_path) + assert "hidden-gold" not in str(writes) + str(commands) + assert "test-download-token" not in str(writes) + assert commands[0][1]["envs"] == {"HF_TOKEN": "test-download-token"} + + +def test_atomic_results_are_readable_after_replacement(tmp_path): + target = tmp_path / "pairs" / "a.json" + write_json(target, {"correctness": None}) + write_json(target, {"correctness": 1}) + assert json.loads(target.read_text())["correctness"] == 1 + assert not target.with_suffix(".tmp").exists() diff --git a/05-multi-harness-rl/tests/test_launch.py b/05-multi-harness-rl/tests/test_launch.py new file mode 100644 index 00000000..d0ae4e0d --- /dev/null +++ b/05-multi-harness-rl/tests/test_launch.py @@ -0,0 +1,70 @@ +"""Exercise service wiring without allocating GPUs or starting real subprocesses.""" + +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + + +@pytest.mark.parametrize("action", ["train", "eval"]) +@pytest.mark.parametrize("mode", ["whitebox", "opencode", "multi_harness"]) +def test_each_mode_gets_the_right_services_and_gpu(tmp_path, monkeypatch, action, mode): + pytest.importorskip("httpx") + from jobs import run + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(run, "ROOT", tmp_path) + (tmp_path / "prepared").mkdir() + (tmp_path / "prepared/ready.json").write_text("{}") + monkeypatch.setattr( + sys, "argv", ["run.py", action, "--mode", mode, "--output", "outputs"] + ) + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "2,3") + monkeypatch.setattr(run.subprocess, "run", Mock()) + monkeypatch.setattr(run.signal, "signal", Mock()) + monkeypatch.setattr(run.os, "killpg", Mock()) + monkeypatch.setattr( + run.httpx, + "get", + Mock( + return_value=SimpleNamespace(status_code=200, raise_for_status=lambda: None) + ), + ) + calls = [] + + def spawn(command, **kwargs): + calls.append((command, kwargs["env"])) + is_worker = command[1].startswith("train_") or command[1] == "evaluate.py" + if command[1] == "jobs/inference_proxy.py": + Path(command[command.index("--output") + 1]).write_text( + "https://sandbox-inference.test" + ) + return SimpleNamespace( + poll=lambda: 0 if is_worker else None, + returncode=0, + pid=1234, + wait=lambda **kw: 0, + ) + + monkeypatch.setattr(run.subprocess, "Popen", spawn) + run.main() + engine, worker = calls[0], calls[-1] + assert engine[1]["CUDA_VISIBLE_DEVICES"] == ("2" if action == "train" else "2,3") + assert worker[1]["CUDA_VISIBLE_DEVICES"] == "3" + engine_port = engine[0][engine[0].index("--port") + 1] + assert ( + worker[0][worker[0].index("--vllm-url") + 1] + == f"http://127.0.0.1:{engine_port}" + ) + has_harbor = any("harbor_env.server.app:app" in command for command, _ in calls) + assert has_harbor == ( + mode != "whitebox" and (mode == "multi_harness" or action == "eval") + ) + assert any("jobs/inference_proxy.py" in command for command, _ in calls) == ( + action == "train" and mode == "opencode" + ) + assert worker[0][1] == (f"train_{mode}.py" if action == "train" else "evaluate.py") diff --git a/05-multi-harness-rl/tests/test_runtime.py b/05-multi-harness-rl/tests/test_runtime.py deleted file mode 100644 index cd6087d7..00000000 --- a/05-multi-harness-rl/tests/test_runtime.py +++ /dev/null @@ -1,172 +0,0 @@ -"""CPU integration checks against the fetched runtime and prepared task files.""" -import contextlib -import io -import json -import os -from pathlib import Path -import runpy -import sys - -import pytest - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT)) -pytestmark = pytest.mark.skipif(not (ROOT / ".runtime/trl/trl").exists() or not (ROOT / "prepared/ready.json").exists(), - reason="Run bootstrap.py and prepare.py for runtime checks") - - -def test_corrected_grader_returns_only_numeric_fields(monkeypatch): - from prepare import GRADER_SHA256 - import hashlib - path = next((ROOT / "prepared/datasets/test/tasks").glob("*/tests/grader.py")) - assert hashlib.sha256(path.read_bytes()).hexdigest() == GRADER_SHA256 - module = runpy.run_path(str(path)) - monkeypatch.setenv("EXPECTED_ANSWER", "42") - monkeypatch.setenv("REWARD_MODE", "numeric") - for answer, expected in [("42", 1.0), ("17", 0.0), ("", 0.0)]: - monkeypatch.setattr(sys, "stdin", io.StringIO(answer)) - stream = io.StringIO() - with contextlib.redirect_stdout(stream): - module["main_json"]() - result = json.loads(stream.getvalue()) - assert result["correctness"] == expected - assert all(type(v) in {float, int} for v in result.values()) - - -def test_native_task_apis_share_ids_and_do_not_expose_answers(monkeypatch): - from runtime.bootstrap import activate - activate() - monkeypatch.setenv("DATA_AGENT_FROZEN_TASKS_DIR", str(ROOT / "prepared/datasets")) - monkeypatch.setenv("DAYTONA_COMPARISON_RUN", str(ROOT / "prepared")) - monkeypatch.setenv("WHITE_BOX_BASH_TASK_SOURCE", "harbor-frozen") - from data_agent_env.tasks import rows_for - from whitebox_bash.tasks import _load - for split, count in (("train", 1000), ("test", 250)): - native, whitebox = rows_for(split), _load(split) - assert len(native) == len(whitebox) == count - assert [t["task_id"] for t in native] == [t.metadata["source_name"] for t in whitebox] - assert all("answer" not in t.public() and "metadata" not in t.public() for t in whitebox) - - -def test_worker_options_match_pinned_runtime(): - import ast - import inspect - from runtime.bootstrap import activate - activate() - from trl.experimental.async_grpo.async_rollout_worker import _AsyncRolloutLoop - from trl.experimental.async_grpo.openenv_harness import _HarnessRolloutLoop - from hard_curriculum_train import FiniteLoop - names = set() - for cls in (_AsyncRolloutLoop, _HarnessRolloutLoop, FiniteLoop): - names.update(inspect.signature(cls.__init__).parameters) - names.add("max_outstanding_rollouts") - tree = ast.parse((ROOT / "train/blackbox.py").read_text()) - for call in ast.walk(tree): - if isinstance(call, ast.Call) and isinstance(call.func, ast.Name) and call.func.id == "Worker": - assert not {kw.arg for kw in call.keywords} - names - - -@pytest.mark.parametrize("mask", [[0, 0, 1, 0], [0, 0, 0, 0]]) -def test_native_openenv_partial_mask_reaches_trl(mask): - from runtime.bootstrap import activate - activate() - from data_agent_env.models import DataAgentRolloutResult, DataAgentTurn - from runtime.native_capture import training_trace - from trl.experimental.async_grpo.openenv_harness import _turns_from_training_trace - result = DataAgentRolloutResult(rollout_type="train", turns=[DataAgentTurn( - trainable=bool(any(mask)), prompt_token_ids=[1, 2], completion_token_ids=[3, 4], - per_token_logps=[-.2, -.4], loss_mask=mask, capture_metadata={"node_id": "agent-0"})]) - trace = training_trace(result) - assert len(trace.turns) == 1 - assert _turns_from_training_trace(trace)[0].output_mask == mask[2:] - - -@pytest.mark.parametrize("broken", ["eval", "mask", "duplicate", "fatal", "identity"]) -def test_native_training_capture_rejects_invalid_records(broken): - from runtime.bootstrap import activate - activate() - from data_agent_env.models import DataAgentRolloutResult, DataAgentTurn - from runtime.native_capture import training_trace - turn = DataAgentTurn(trainable=True, prompt_token_ids=[1], completion_token_ids=[2], - per_token_logps=[-.2], loss_mask=[0, 1], capture_metadata={"node_id": "a"}) - result = DataAgentRolloutResult(rollout_type="train", turns=[turn]) - if broken == "eval": - result.rollout_type = "eval" - elif broken == "mask": - turn.loss_mask = [] - elif broken == "identity": - turn.capture_metadata = {} - elif broken == "duplicate": - result.turns.append(turn) - else: - result.metadata["capture_findings"] = ["[FATAL] incomplete capture"] - with pytest.raises((ValueError, KeyError)): - training_trace(result) - - -@pytest.mark.parametrize("correctness,actions,expected", [(1.0, 5, 1.075), (0.0, 5, 0.0), (1.0, None, 1.0), (None, 5, None)]) -def test_native_session_worker_preserves_reward_after_typed_copy(tmp_path, correctness, actions, expected): - from runtime.bootstrap import activate - activate() - import queue - import threading - from types import SimpleNamespace - from unittest.mock import MagicMock - from accelerate import PartialState - from data_agent_env.models import DataAgentRolloutResult, DataAgentTurn - from data_agent_env.harness import DataAgentSession - from openenv.core.harness import TrainableSession - from trl.experimental.async_grpo.openenv_harness import _HarnessRolloutLoop - from train.adapters import Session, rollout_reward - - PartialState() - result = DataAgentRolloutResult(rollout_type="train", correctness=correctness, - reward=correctness, metadata={"verified_native_actions": actions}, turns=[DataAgentTurn( - trainable=True, prompt_token_ids=[1], completion_token_ids=[2, 3], - per_token_logps=[-.2, -.4], loss_mask=[0, 1, 0], capture_metadata={"node_id": "a"})]) - client = MagicMock() - client.run_rollout.return_value = result - cfg = {"mode": "opencode", "efficiency_weight": .1, "tool_budget": 15, "output": str(tmp_path)} - wrapped = Session(DataAgentSession(client, "train", 0, "task"), cfg, {"episode_id": "smoke"}, tmp_path) - assert isinstance(wrapped, TrainableSession) - policy = {} - def build(*, sampling): - policy.update(sampling) - return SimpleNamespace(create=lambda *args, **kwargs: wrapped) - loop = _HarnessRolloutLoop(harness_session_factory=build, rollout_reward_fn=rollout_reward, - model_name="test", dataset=[{"prompt": [{"role": "user", "content": "task"}]}], reward_funcs=[], - processing_class=MagicMock(eos_token_id=0, pad_token_id=0), rollout_buffer=queue.Queue(), - model_version_value=SimpleNamespace(value=0), heartbeat_value=SimpleNamespace(value=0.), - failed_event=threading.Event(), exception_info_queue=queue.Queue(), metrics_queue=queue.Queue(), - max_inflight_tasks=2, temperature=.8) - try: - actual, metrics = loop._run_session([]) - if expected is None: - assert actual[-1] is None - else: - assert actual[-1] == pytest.approx(expected) - assert actual[2][0].completion_mask == [0, 1, 0] - assert actual[2][0].old_log_probs == [0., -.2, -.4] - assert policy["temperature"] == .8 and policy["top_p"] == 1. - record = json.loads((tmp_path / "rollouts/smoke.json").read_text()) - assert record["reward"] == actual[-1] - client.close.assert_called_once() - finally: - loop._session_pool.shutdown(wait=True) - loop._loop.close() - - -@pytest.mark.parametrize("tasks,requested", [(2000, 1000), (2000, 2000), (5, 1000)]) -def test_whitebox_update_limit_matches_trl_sampler(tasks, requested): - from runtime.bootstrap import activate - activate() - from recipe import config, whitebox_step_limit - from trl.trainer.utils import RepeatSampler - cfg = config(mode="whitebox", max_steps=requested) - accumulation = cfg["batch_size"] * cfg["gradient_accumulation_steps"] - sampler = RepeatSampler(range(tasks), mini_repeat_count=cfg["num_generations"], - batch_size=accumulation // cfg["num_generations"], repeat_count=accumulation, shuffle=False) - available = len(list(sampler)) // (accumulation * accumulation) - assert whitebox_step_limit(cfg, tasks) == min(requested, available) - if tasks == 2000: - assert available == 1000 diff --git a/05-multi-harness-rl/train/adapters.py b/05-multi-harness-rl/train/adapters.py deleted file mode 100644 index 2ac16bb9..00000000 --- a/05-multi-harness-rl/train/adapters.py +++ /dev/null @@ -1,144 +0,0 @@ -"""One task identity and reward boundary for both blackbox implementations.""" -from __future__ import annotations - -import json -from pathlib import Path - -from recipe import reward, write_json - - -def atif_count(path): - data = json.loads(Path(path).read_text()) - if not str(data.get("schema_version", "")).startswith("ATIF-"): - raise ValueError("Unknown native trajectory format") - calls = [c for s in data["steps"] if s.get("source") == "agent" for c in s.get("tool_calls", []) or []] - ids = [c.get("tool_call_id") for c in calls] - if any(not x for x in ids) or len(set(ids)) != len(ids): - raise ValueError("Missing or repeated native action IDs") - return len(ids) - - -def opencode_count(text): - rows = [json.loads(line) for line in text.splitlines() if line.strip()] - if not rows or not any(r.get("type") == "step_finish" for r in rows): - raise ValueError("Incomplete native OpenCode event stream") - parts = [r["part"] for r in rows if r.get("type") == "tool_use"] - ids = [p["callID"] for p in parts] - if len(set(ids)) != len(ids) or any(not i for i in ids): - raise ValueError("Ambiguous native tool actions") - if any(p.get("state", {}).get("status") not in {"completed", "error"} for p in parts): - raise ValueError("Unfinished tool action") - return len(ids) - - -class Session: - def __init__(self, inner, cfg, evidence, trial_root): - self.inner, self.cfg = inner, cfg - self.evidence, self.trial_root = evidence, Path(trial_root) - - def __getattr__(self, name): - inner = self.__dict__.get("inner") - if inner is None: - raise AttributeError(name) - return getattr(inner, name) - - @property - def result(self): - return getattr(self.inner, "result", getattr(self.inner, "_result", None)) - - def wait_for_completion(self, timeout_s=None): - return self.inner.wait_for_completion(timeout_s) - - def fetch_training_trace(self): - trace = self.inner.fetch_training_trace() - calls, error = self.tool_usage() - inputs = {"tool_calls": calls, "tool_count_error": error, - "weight": self.cfg["efficiency_weight"], "budget": self.cfg["tool_budget"]} - for turn in trace.turns: - turn.metadata["recipe_reward_inputs"] = inputs - return trace - - def tool_usage(self): - result = self.result - if result is None: - return None, None - if self.cfg["mode"] == "opencode": - return result.metadata.get("verified_native_actions"), None - try: - return atif_count(self.trial_root / result.trial_name / "agent/trajectory.json"), None - except (OSError, ValueError, KeyError, TypeError) as exc: - return None, type(exc).__name__ - - def verify(self, transcript, final_state=None): - from openenv.core.harness import VerifyResult - - original = self.inner.verify(transcript, final_state) - result = self.result - correctness = original.env_reward - if self.cfg["mode"] == "opencode" and result is not None: - value = result.correctness - correctness = None if value is None else float(value >= 1) - calls, error = self.tool_usage() - shaped = reward(correctness, calls, verified=calls is not None, - weight=self.cfg["efficiency_weight"], budget=self.cfg["tool_budget"]) - record = {**self.evidence, "correctness": correctness, "tool_calls": calls, - "tool_count_verified": calls is not None, "tool_count_error": error, - "reward": shaped, "bonus": None if shaped is None else shaped - correctness} - write_json(Path(self.cfg["output"]) / "rollouts" / f'{self.evidence["episode_id"]}.json', record) - return VerifyResult(env_reward=correctness, done=original.done, metrics={**original.metrics, **record}, - artifacts=original.artifacts) - - -def rollout_reward(outcome): - if not outcome.trace: - return None - inputs = outcome.trace[0]["metadata"]["recipe_reward_inputs"] - return reward(outcome.env_reward, inputs["tool_calls"], verified=inputs["tool_calls"] is not None, - weight=inputs["weight"], budget=inputs["budget"]) - - -class Factory: - def __init__(self, cfg, data, server, vllm, groups, trial_root, split="train", *, sampling=None): - self.cfg, self.data, self.server, self.vllm = cfg, str(data), server, vllm - self.groups, self.trial_root, self.split = groups, str(trial_root), split - self.sampling = sampling - manifest = json.loads((Path(data) / f"{split}_manifest.json").read_text()) - self.tasks = sorted(manifest["tasks"], key=lambda row: row["name"]) - self.instructions = [(Path(data) / "datasets" / split / "tasks" / r["name"] / "instruction.md").read_text() - for r in self.tasks] - - def rows(self): - return [{"prompt": [{"role": "user", "content": self.instructions[g["task_index"]]}], - "task_name": g["task_name"], "harness": g["harness"]} for g in self.groups] - - def create(self, task, seed=None, episode_id=None): - from harbor_env.harness import HarborSession, _instruction_of - - group_id = 0 if seed is None else seed - group = self.groups[group_id] - index = group["task_index"] - instruction = self.instructions[index] - if _instruction_of(task).strip() != instruction.strip(): - raise ValueError("Prompt and frozen task schedule disagree") - sampling = self.sampling - if sampling is None: - raise ValueError("Training factory requires the worker sampling policy") - if self.cfg["mode"] == "opencode": - from data_agent_env import DataAgentEnv - from data_agent_env.harness import DataAgentSession - - inner = DataAgentSession(DataAgentEnv(self.server, message_timeout_s=1800), self.split, - index, instruction, llm_url=self.vllm, model=self.cfg["profile"]["id"], - sandbox=self.cfg["sandbox"], sampling=sampling, - agent_step_limit=self.cfg["agent_step_limit"], agent_timeout_s=self.cfg["agent_timeout_sec"]) - else: - from harbor_env import HarborEnv - - inner = HarborSession(env=HarborEnv(self.server, message_timeout_s=1800), owns_env=True, - split=str(Path(self.data) / "datasets" / self.split), task_index=index, instruction=instruction, - harness=group["harness"], sandbox=self.cfg["sandbox"], llm_url=self.vllm, - model=self.cfg["profile"]["id"], sampling=sampling, reward_key="correctness,reward", - agent_step_limit=self.cfg["agent_step_limit"], agent_timeout_sec=self.cfg["agent_timeout_sec"]) - evidence = {"group_id": group_id, "task_name": group["task_name"], "harness": group["harness"], - "episode_id": episode_id} - return Session(inner, self.cfg, evidence, self.trial_root) diff --git a/05-multi-harness-rl/train/blackbox.py b/05-multi-harness-rl/train/blackbox.py deleted file mode 100644 index deca21ed..00000000 --- a/05-multi-harness-rl/train/blackbox.py +++ /dev/null @@ -1,82 +0,0 @@ -"""AsyncGRPO with atomic rollout admission and a finite, resumable task schedule.""" -import json -from functools import partial -import os -from pathlib import Path - -from datasets import Dataset -import torch -from hard_curriculum_train import FiniteLoop, FiniteTrainer, FiniteWorker -from trl.experimental.async_grpo import AsyncGRPOConfig - -from recipe import digest, resume_state, schedule, task_rows, write_json -from runtime.models import tokenizer_for, training_attention_backend -from train.adapters import Factory, rollout_reward -from train.logging_utils import Checkpoints, Logs, publish_checkpoint - - -class GuardedLoop(FiniteLoop): - async def _score_group(self, group): - bundles = await super()._score_group(group) - self._ungraded_streak = 0 if bundles else getattr(self, "_ungraded_streak", 0) + 1 - if self._ungraded_streak >= 3: - raise RuntimeError("Three consecutive groups produced no trainable rollouts; inspect capture and environment logs") - return bundles - - -class Worker(FiniteWorker): - _loop_cls = GuardedLoop - - -def train(cfg, data, server, vllm, resume=None): - output = Path(cfg["output"]) - output.mkdir(parents=True, exist_ok=True) - groups = schedule(cfg, task_rows("train")) - write_json(output / "schedule.json", groups) - os.environ.update(CURRICULUM_GROUP_LIMIT=str(len(groups)), CURRICULUM_SCHEDULE_SHA256=digest(groups), - CURRICULUM_AUDIT=str(output / "audit")) - if resume: - state = json.loads((Path(resume) / "curriculum_state.json").read_text()) - state = resume_state(groups, state) - write_json(output / "resume-curriculum.json", state) - os.environ["CURRICULUM_RESUME_STATE"] = str(output / "resume-curriculum.json") - else: - os.environ.pop("CURRICULUM_RESUME_STATE", None) - if cfg["model"] == "lfm": - from packing import install - install() - factory = Factory(cfg, data, server, vllm, groups, output / "trials") - dataset = Dataset.from_list(factory.rows()) - tokenizer = tokenizer_for(cfg) - factory_builder = partial(Factory, cfg, data, server, vllm, groups, output / "trials") - worker = Worker(harness_session_factory=factory_builder, harness_adapter=None, - lossless_capture=True, rollout_reward_fn=rollout_reward, - model_name=cfg["profile"]["id"], processing_class=tokenizer, dataset=dataset, reward_funcs=[], - num_generations=cfg["num_generations"], max_inflight_tasks=cfg["max_inflight"], - max_outstanding_rollouts=cfg["max_outstanding_rollouts"], vllm_server_url=vllm, - max_tokens=cfg["max_completion_length"], temperature=cfg["temperature"], top_p=1.0, top_k=0, - log_completions=False) - args = AsyncGRPOConfig(output_dir=str(output), learning_rate=cfg["learning_rate"], - lr_scheduler_type="constant", warmup_steps=0, max_steps=cfg["max_steps"], - per_device_train_batch_size=cfg["batch_size"], gradient_accumulation_steps=cfg["gradient_accumulation_steps"], - num_generations=cfg["num_generations"], max_completion_length=cfg["max_completion_length"], - max_inflight_tasks=cfg["max_inflight"], max_staleness=cfg["max_staleness"], fork_threshold_tokens=0, - token_budget=cfg["token_budget"], temperature=cfg["temperature"], top_p=1.0, top_k=0, - optim="paged_adamw_8bit", bf16=True, dtype="bfloat16", gradient_checkpointing=True, - gradient_checkpointing_kwargs={"use_reentrant": False}, - model_init_kwargs={"revision": cfg["profile"]["revision"], - "attn_implementation": training_attention_backend(torch.cuda.get_device_capability())}, - vllm_server_base_url=vllm, heartbeat_stale_after_s=900, - save_strategy="steps", save_steps=cfg["save_steps"], save_total_limit=None, - logging_steps=1, report_to="trackio", project=cfg["project"], run_name=cfg["run_name"], - trackio_space_id=cfg["trackio_space_id"], seed=cfg["seed"]) - trainer = FiniteTrainer(model=cfg["profile"]["id"], args=args, train_dataset=dataset, - rollout_worker=worker, max_row_tokens=cfg["max_model_len"], admission_dir=output / "audit") - trainer.add_callback(Logs()) - trainer.add_callback(Checkpoints(cfg)) - write_json(output / "trainer-config.json", args.to_dict()) - trainer.train(resume_from_checkpoint=str(resume) if resume else None) - checkpoint = output / f"checkpoint-{trainer.state.global_step}" - if not (checkpoint / "checkpoint.saved.json").exists(): - trainer._save_checkpoint(trainer.model, None) - publish_checkpoint(checkpoint, cfg, trainer.state.global_step, final=True) diff --git a/05-multi-harness-rl/train/generation.py b/05-multi-harness-rl/train/generation.py deleted file mode 100644 index 203c1321..00000000 --- a/05-multi-harness-rl/train/generation.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Avoid grouping diverged tool histories as repeated initial prompts.""" - - -def group_size(prompts, requested): - if requested < 1: - raise ValueError("Group size must be positive") - if len(prompts) % requested: - return 1 - for start in range(0, len(prompts), requested): - if any(p != prompts[start] for p in prompts[start:start + requested]): - return 1 - return requested - - -def protect_generation(trainer): - client = trainer.vllm_generation.vllm_client - def reset_prefix_cache(): - # vLLM acknowledges cache resets with an empty HTTP 200 body. - response = client.session.post(f"{client.base_url}/reset_prefix_cache") - response.raise_for_status() - client.reset_prefix_cache = reset_prefix_cache - generate = trainer.vllm_generation.generate - def checked(*args, **kwargs): - prompts = kwargs.get("prompts", args[0] if args else None) - kwargs = {**kwargs, "num_generations": group_size(prompts, kwargs.get("num_generations", 1))} - output = generate(*args, **kwargs) - returned, completions, logprobs, ids = output - if returned != prompts: - raise ValueError("vLLM changed the supplied token prompts") - for completion, lps, token_ids in zip(completions, logprobs, ids, strict=True): - if len(completion) != len(lps) or token_ids != [[token] for token in completion]: - raise ValueError("Sampled token/logprob IDs do not agree") - return output - trainer.vllm_generation.generate = checked diff --git a/05-multi-harness-rl/train/logging_utils.py b/05-multi-harness-rl/train/logging_utils.py deleted file mode 100644 index dee18bc3..00000000 --- a/05-multi-harness-rl/train/logging_utils.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Durable scalar logs and checkpoint handoffs for a separate evaluation watcher.""" -import json -import math -from pathlib import Path -import time - -from transformers import TrainerCallback -from recipe import write_json - - -class Logs(TrainerCallback): - def on_log(self, args, state, control, logs=None, **kwargs): - values = {k: v for k, v in (logs or {}).items() - if isinstance(v, (int, float)) and math.isfinite(v)} - path = Path(args.output_dir) / "metrics.jsonl" - with path.open("a") as stream: - stream.write(json.dumps({"step": state.global_step, "time": time.time(), **values}) + "\n") - stream.flush() - - -def publish_checkpoint(path, cfg, step, *, final=False): - from runtime.checkpoints import saved - - path = Path(path) - saved(path, cfg, step) - write_json(path / "recipe.json", cfg) - write_json(path / "eval.request.json", {"step": step, "final": final, - "evaluate": final or step % cfg["eval_steps"] == 0, "created_at": time.time()}) - - -class Checkpoints(TrainerCallback): - def __init__(self, cfg): - self.cfg = cfg - - def on_save(self, args, state, control, **kwargs): - if state.is_world_process_zero: - publish_checkpoint(Path(args.output_dir) / f"checkpoint-{state.global_step}", self.cfg, - state.global_step, final=control.should_training_stop) diff --git a/05-multi-harness-rl/train/whitebox.py b/05-multi-harness-rl/train/whitebox.py deleted file mode 100644 index e03310fb..00000000 --- a/05-multi-harness-rl/train/whitebox.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Synchronous GRPO: TRL owns generation and invokes native bash/SETA tools.""" -from pathlib import Path -import json -import math - -from datasets import Dataset -from trl import GRPOConfig, GRPOTrainer -from whitebox_bash import white_box_bash_env - -from recipe import schedule, task_rows, whitebox_step_limit, write_json -from runtime.models import tokenizer_for -from train.logging_utils import Checkpoints, Logs, publish_checkpoint -from train.generation import protect_generation - -SYSTEM = ("You are a terminal agent working in a sandbox. Use the available tools to inspect the " - "files and solve the task. Call submit_solution with the final answer itself, " - "not a command that would produce it.") - - -class TokenCheckedTrainer(GRPOTrainer): - def _generate_single_turn(self, prompt_ids, *args, **kwargs): - completions, logprobs = super()._generate_single_turn(prompt_ids, *args, **kwargs) - for prompt, completion, lps in zip(prompt_ids, completions, logprobs, strict=True): - if len(completion) != len(lps) or not all(math.isfinite(p) for p in lps): - raise ValueError("Invalid whitebox engine token/logprob capture") - self._captured_calls.append({"prompt_ids": prompt.copy(), "completion_ids": completion.copy(), - "logprobs": lps.copy()}) - return completions, logprobs - - def _generate(self, *args, **kwargs): - self._captured_calls = [] - return super()._generate(*args, **kwargs) - - def _tool_call_loop(self, prompts, prompt_ids, *args, **kwargs): - from whitebox_tito import audit_rows - result = super()._tool_call_loop(prompts, prompt_ids, *args, **kwargs) - masks, _, completions, logps, _, _, _ = result - checks = audit_rows(prompt_ids, completions, masks, logps, self._captured_calls) - with (Path(self.args.output_dir) / "token-audit.jsonl").open("a") as stream: - stream.write(json.dumps({"step": self.state.global_step, "rows": checks, - "engine_calls": len(self._captured_calls)}) + "\n") - return result - - -def train(cfg, data, server, vllm, resume=None): - from run import unused_port - output = Path(cfg["output"]) - output.mkdir(parents=True, exist_ok=True) - groups = schedule(cfg, task_rows("train")) - write_json(output / "schedule.json", groups) - rows = [{"prompt": [{"role": "system", "content": SYSTEM}, {"role": "user", "content": "Solve the task."}], - "split": "train", "index": g["task_index"]} for g in groups] - args = GRPOConfig(output_dir=str(output), learning_rate=cfg["learning_rate"], - lr_scheduler_type="constant", warmup_steps=0, beta=0.0, loss_type="dapo", - max_steps=whitebox_step_limit(cfg, len(groups)), num_train_epochs=1, shuffle_dataset=False, - num_generations=cfg["num_generations"], per_device_train_batch_size=1, - gradient_accumulation_steps=cfg["batch_size"] * cfg["gradient_accumulation_steps"], - max_completion_length=cfg["max_completion_length"], max_tool_calling_iterations=cfg["agent_step_limit"] - 1, - temperature=cfg["temperature"], top_p=1.0, top_k=0, - chat_template_kwargs={"enable_thinking": False, "preserve_thinking": True}, - generation_kwargs={"max_tokens": cfg["max_output_tokens"]}, - model_init_kwargs={"revision": cfg["profile"]["revision"], "dtype": "bfloat16"}, - optim="paged_adamw_8bit", bf16=True, gradient_checkpointing=True, - gradient_checkpointing_kwargs={"use_reentrant": False}, - use_vllm=True, vllm_mode="server", vllm_server_base_url=vllm, - vllm_group_port=unused_port(), - vllm_server_timeout=900, vllm_max_model_length=cfg["max_model_len"], - save_strategy="steps", save_steps=cfg["save_steps"], save_total_limit=None, - logging_steps=1, report_to="trackio", project=cfg["project"], run_name=cfg["run_name"], - trackio_space_id=cfg["trackio_space_id"], seed=cfg["seed"]) - trainer = TokenCheckedTrainer(model=cfg["profile"]["id"], processing_class=tokenizer_for(cfg), args=args, - train_dataset=Dataset.from_list(rows), reward_funcs=[], - environment_factory=white_box_bash_env(server, split="train", toolsets="bash,seta", step_limit=cfg["agent_step_limit"]), - callbacks=[Logs(), Checkpoints(cfg)]) - # This TRL version reads the outer config for text-only tool rollouts. - text_config = trainer.model.config.get_text_config() - trainer.model.config.max_position_embeddings = min(text_config.max_position_embeddings, cfg["max_model_len"]) - write_json(output / "trainer-config.json", args.to_dict()) - protect_generation(trainer) - trainer.train(resume_from_checkpoint=str(resume) if resume else None) - checkpoint = output / f"checkpoint-{trainer.state.global_step}" - if not (checkpoint / "checkpoint.saved.json").exists(): - trainer._save_checkpoint(trainer.model, None) - publish_checkpoint(checkpoint, cfg, trainer.state.global_step, final=True) diff --git a/05-multi-harness-rl/train_multi_harness.py b/05-multi-harness-rl/train_multi_harness.py new file mode 100644 index 00000000..889fcd8d --- /dev/null +++ b/05-multi-harness-rl/train_multi_harness.py @@ -0,0 +1,255 @@ +"""Train through four Harbor harnesses with AsyncGRPO. Read the numbered sections in order.""" + +# %% 1. Choose the model and run settings. +import argparse +import json +import os +from functools import partial +from pathlib import Path + +from datasets import Dataset +from transformers import AutoTokenizer + +from envs.tasks import load_tasks + +MODEL_REVISIONS = { + "LiquidAI/LFM2.5-2.6B": "654f9463ce32b05d0429d76fe1f580b27d4c1ac0", + "Qwen/Qwen3.5-2B": "15852e8c16360a2fea060d615a32b45270f8a8fc", +} + + +def arguments(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", default="LiquidAI/LFM2.5-2.6B") + parser.add_argument("--data", default="prepared") + parser.add_argument("--vllm-url", default="http://127.0.0.1:8000") + parser.add_argument("--server", default="http://127.0.0.1:8200") + parser.add_argument("--trials", default="runs/trials") + parser.add_argument("--output", default="runs/multi_harness") + parser.add_argument("--steps", type=int, default=1000) + parser.add_argument("--save-steps", type=int, default=50) + parser.add_argument("--inflight", type=int, default=32) + parser.add_argument( + "--space-id", + default=None, + help="Optional Trackio Space; local logs are always kept", + ) + return parser.parse_args() + + +# %% 2. Load the tokenizer in non-thinking mode. +def tokenizer_for(model): + from copy import deepcopy + + from trl.chat_template_utils import lfm2_2_5_template, qwen3_5_template + + tokenizer = AutoTokenizer.from_pretrained(model, revision=MODEL_REVISIONS[model]) + if "lfm" in model.lower(): + # LFM's template opens even when enable_thinking=False. + start = r'{{- "<|im_start|>assistant\n" -}}' + if tokenizer.chat_template.count(start) != 1: + raise ValueError( + "LFM template changed; check its non-thinking generation prefix" + ) + tokenizer.chat_template = tokenizer.chat_template.replace( + start, r'{{- "<|im_start|>assistant\n" -}}' + ) + tokenizer.response_template = deepcopy(lfm2_2_5_template) + else: + tokenizer.chat_template = ( + "{%- set enable_thinking = false -%}\n" + tokenizer.chat_template + ) + tokenizer.response_template = deepcopy(qwen3_5_template) + return tokenizer + + +# %% 3. Select one harness per task. All eight rollouts use the same assignment. +from harbor_env import HarborEnv +from harbor_env.harness import HarborSession + +HARNESSES = ("opencode", "claude-code", "codex", "mini-swe-agent") + + +class TaskSession(HarborSession): + def fetch_training_trace(self): + trace = super().fetch_training_trace() + # ATIF counts actual tool actions, not model requests or training rows. + path = self.trial_root / self.result.trial_name / "agent/trajectory.json" + try: + trajectory = json.loads(path.read_text()) + if not trajectory["schema_version"].startswith("ATIF-"): + raise ValueError("Unknown trajectory format") + ids = [ + call["tool_call_id"] + for step in trajectory["steps"] + if step.get("source") == "agent" + for call in step.get("tool_calls", []) or [] + ] + if any(not value for value in ids) or len(set(ids)) != len(ids): + raise ValueError("Missing or repeated tool action IDs") + calls = len(ids) + except (OSError, ValueError, KeyError, TypeError): + calls = None + for turn in trace.turns: + turn.metadata["native_tool_calls"] = calls + return trace + + +class TaskFactory: + def __init__(self, args, tasks, *, sampling): + self.args, self.sampling = args, sampling + self.tasks = {task["instruction"]: (i, task) for i, task in enumerate(tasks)} + + def create(self, task, seed=None, episode_id=None): + # Match by prompt, not by the worker seed: seeds can change after a restart. + index, row = self.tasks[task[-1]["content"]] + session = TaskSession( + env=HarborEnv(self.args.server, message_timeout_s=1800), + owns_env=True, + split=str(Path(self.args.data).resolve() / "datasets/train"), + task_index=index, + instruction=row["instruction"], + harness=HARNESSES[index % len(HARNESSES)], + sandbox="daytona", + llm_url=self.args.vllm_url, + model=self.args.model, + sampling=self.sampling, + reward_key="correctness", + agent_step_limit=17, + agent_timeout_sec=600, + ) + session.trial_root = Path(self.args.trials) + return session + + +# %% 4. Reward correctness, with a small bonus for fewer verified tool calls. +def reward(outcome): + if outcome.env_reward is None: + return None # Infrastructure failures do not become wrong answers. + calls = ( + outcome.trace[0]["metadata"].get("native_tool_calls") if outcome.trace else None + ) + bonus = 0.1 * 15 / (15 + calls) if calls is not None and calls > 0 else 0.0 + return outcome.env_reward * (1 + bonus) + + +# %% 5. Configure public AsyncGRPO. These are the experiment's hyperparameters. +def main(): + import torch + from trl.experimental.async_grpo import AsyncGRPOConfig, AsyncGRPOTrainer + from trl.experimental.async_grpo.openenv_harness import HarnessRolloutWorker + + args = arguments() + output = Path(args.output) + output.mkdir(parents=True, exist_ok=True) + os.environ["TRACKIO_DIR"] = str(output.resolve() / "trackio") + tasks = load_tasks(args.data, "train") + dataset = Dataset.from_list( + [ + {"prompt": [{"role": "user", "content": task["instruction"]}]} + for task in tasks + ] + ).shuffle(seed=0) + tokenizer = tokenizer_for(args.model) + config = AsyncGRPOConfig( + output_dir=str(output), + learning_rate=3e-6, + lr_scheduler_type="constant", + warmup_steps=0, + max_steps=args.steps, + per_device_train_batch_size=4, + gradient_accumulation_steps=4, + num_generations=8, + max_completion_length=16384, + temperature=0.8, + top_p=1.0, + top_k=-1, + max_inflight_tasks=args.inflight, + max_staleness=4, + token_budget=40960, + fork_threshold_tokens=0, + optim="paged_adamw_8bit", + bf16=True, + dtype="bfloat16", + model_init_kwargs={ + "revision": MODEL_REVISIONS[args.model], + "use_kernels": "lfm" in args.model.lower(), + }, + gradient_checkpointing=True, + gradient_checkpointing_kwargs={"use_reentrant": False}, + vllm_server_base_url=args.vllm_url, + heartbeat_stale_after_s=900, + save_strategy="steps", + save_steps=args.save_steps, + save_total_limit=None, + logging_steps=1, + report_to="trackio", + project="smoldataenv-rl", + run_name=output.name, + trackio_space_id=args.space_id, + seed=0, + ) + worker = HarnessRolloutWorker( + harness_session_factory=partial(TaskFactory, args, tasks), + harness_adapter=None, # OpenEnv captures a CLI agent's own model calls. + rollout_reward_fn=reward, + lossless_capture=True, + model_name=args.model, + processing_class=tokenizer, + dataset=dataset, + reward_funcs=[], + num_generations=8, + max_inflight_tasks=args.inflight, + vllm_server_url=args.vllm_url, + max_tokens=16384, + temperature=0.8, + top_p=1.0, + top_k=-1, + chat_template_kwargs={"enable_thinking": False}, + ) + trainer = AsyncGRPOTrainer( + model=args.model, + args=config, + processing_class=tokenizer, + train_dataset=dataset, + rollout_worker=worker, + ) + if trainer.model.config.model_type == "lfm2": + # Packed samples reset attention positions. LFM's convolutions also need sequence IDs. + def reset_convolutions(module, positional, keyword): + keyword["seq_idx"] = ((keyword["position_ids"] == 0).cumsum(-1) - 1).to( + torch.int32 + ) + return positional, keyword + + trainer.model.register_forward_pre_hook(reset_convolutions, with_kwargs=True) + # Abort if the installed convolution kernel ignores packed sequence boundaries. + trainer.model.eval() + with torch.no_grad(): + ids = torch.tensor( + [[10, 20, 30, 40, 50, 60, 70, 80]], device=trainer.model.device + ) + separate = trainer.model( + input_ids=ids[:, 4:], + position_ids=torch.arange(4, device=ids.device)[None], + use_cache=False, + ).logits + packed = trainer.model( + input_ids=ids, + position_ids=torch.arange(4, device=ids.device).repeat(2)[None], + use_cache=False, + ).logits[:, 4:] + torch.testing.assert_close(packed, separate, atol=0.05, rtol=0.01) + trainer.model.train() + + # 6. Train, then save weights and tokenizer for evaluation. + (output / "training_config.json").write_text(json.dumps(config.to_dict(), indent=2)) + (output / "task_names.json").write_text(json.dumps([t["name"] for t in tasks])) + trainer.train() + trainer.save_model(str(output / "final")) + tokenizer.save_pretrained(output / "final") + trainer.save_state() + + +if __name__ == "__main__": + main() diff --git a/05-multi-harness-rl/train_opencode.py b/05-multi-harness-rl/train_opencode.py new file mode 100644 index 00000000..d51832d9 --- /dev/null +++ b/05-multi-harness-rl/train_opencode.py @@ -0,0 +1,290 @@ +"""Train native OpenCode with AsyncGRPO. Read the numbered sections in order.""" + +# %% 1. Choose the model and run settings. +import argparse +import json +import os +from functools import partial +from pathlib import Path + +from datasets import Dataset +from transformers import AutoTokenizer + +from envs.tasks import load_tasks + +MODEL_REVISIONS = { + "LiquidAI/LFM2.5-2.6B": "654f9463ce32b05d0429d76fe1f580b27d4c1ac0", + "Qwen/Qwen3.5-2B": "15852e8c16360a2fea060d615a32b45270f8a8fc", +} + + +def arguments(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", default="LiquidAI/LFM2.5-2.6B") + parser.add_argument("--data", default="prepared") + parser.add_argument("--vllm-url", default="http://127.0.0.1:8000") + parser.add_argument("--output", default="runs/opencode") + parser.add_argument("--steps", type=int, default=1000) + parser.add_argument("--save-steps", type=int, default=50) + parser.add_argument("--inflight", type=int, default=32) + parser.add_argument( + "--space-id", + default=None, + help="Optional Trackio Space; local logs are always kept", + ) + return parser.parse_args() + + +# %% 2. Load the tokenizer in non-thinking mode. +def tokenizer_for(model): + from copy import deepcopy + + from trl.chat_template_utils import lfm2_2_5_template, qwen3_5_template + + tokenizer = AutoTokenizer.from_pretrained(model, revision=MODEL_REVISIONS[model]) + if "lfm" in model.lower(): + # LFM's template opens even when enable_thinking=False. + start = r'{{- "<|im_start|>assistant\n" -}}' + if tokenizer.chat_template.count(start) != 1: + raise ValueError( + "LFM template changed; check its non-thinking generation prefix" + ) + tokenizer.chat_template = tokenizer.chat_template.replace( + start, r'{{- "<|im_start|>assistant\n" -}}' + ) + tokenizer.response_template = deepcopy(lfm2_2_5_template) + else: + tokenizer.chat_template = ( + "{%- set enable_thinking = false -%}\n" + tokenizer.chat_template + ) + tokenizer.response_template = deepcopy(qwen3_5_template) + return tokenizer + + +# %% 3. Run native OpenCode in Daytona. Harbor is not involved in training. +from opencode_env.config import OpenCodeConfig +from opencode_env.harness import OpenCodeSessionFactory +from opencode_env.task import OpenCodeTask +from openenv.core.harness import VerifyResult + +from envs.daytona import DaytonaSandboxBackend +from envs.tasks import grade_answer, stage_task + + +class TaskSession: + def __init__(self, session): + self.session = session + + def __getattr__(self, name): + return getattr(self.session, name) + + def fetch_training_trace(self): + trace = self.session.fetch_training_trace() + try: + events = [ + json.loads(line) + for line in self.session.fetch_trace().splitlines() + if line.strip() + ] + parts = [ + event["part"] for event in events if event.get("type") == "tool_use" + ] + ids = [part["callID"] for part in parts] + if not any(event.get("type") == "step_finish" for event in events): + raise ValueError("Incomplete OpenCode event stream") + if any(not value for value in ids) or len(set(ids)) != len(ids): + raise ValueError("Missing or repeated tool action IDs") + if any( + part.get("state", {}).get("status") not in {"completed", "error"} + for part in parts + ): + raise ValueError("Unfinished tool action") + calls = len(ids) + except (OSError, ValueError, KeyError, TypeError): + calls = None + for turn in trace.turns: + turn.metadata["native_tool_calls"] = calls + return trace + + +def verify(sandbox, task): + return VerifyResult( + env_reward=grade_answer(sandbox, task.metadata["folder"]), done=True + ) + + +class TaskFactory: + def __init__(self, args, tasks, *, sampling): + self.tasks = {task["instruction"]: task for task in tasks} + config = OpenCodeConfig( + base_url=os.environ["SANDBOX_VLLM_URL"].rstrip("/") + "/v1", + api_key=os.environ["SANDBOX_VLLM_KEY"], + model=args.model, + opencode_version="1.18.31", + sandbox_home="/root", + proxy_disable_thinking=True, + proxy_max_tokens_cap=4096, + agent_timeout_s=600, + run_format="json", + extra_setup_shell="mkdir -p /workdir && rmdir /root/workdir && ln -s /workdir /root/workdir", + disabled_tools=["webfetch", "question", "task"], + extra_opencode_json={"permission": {"*": "allow"}}, + ) + self.factory = OpenCodeSessionFactory( + config=config, + sampling=sampling, + verifier=verify, + mode="transparent_proxy", + sandbox_backend=DaytonaSandboxBackend( + image="docker.io/savatar101/env-data-agent-train:base" + ), + ) + + def create(self, task, seed=None, episode_id=None): + row = self.tasks[task[-1]["content"]] + task = OpenCodeTask( + instruction=row["instruction"], metadata={"folder": row["folder"]} + ) + session = self.factory.create( + task, seed=seed, episode_id=episode_id, start_agent=False + ) + try: + stage_task(session.sandbox, row["folder"]) + session.start_agent() + return TaskSession(session) + except BaseException: + session.close() + raise + + +# %% 4. Reward correctness, with a small bonus for fewer verified tool calls. +def reward(outcome): + if outcome.env_reward is None: + return None # Infrastructure failures do not become wrong answers. + calls = ( + outcome.trace[0]["metadata"].get("native_tool_calls") if outcome.trace else None + ) + bonus = 0.1 * 15 / (15 + calls) if calls is not None and calls > 0 else 0.0 + return outcome.env_reward * (1 + bonus) + + +# %% 5. Configure public AsyncGRPO. These are the experiment's hyperparameters. +def main(): + import torch + from trl.experimental.async_grpo import AsyncGRPOConfig, AsyncGRPOTrainer + from trl.experimental.async_grpo.openenv_harness import HarnessRolloutWorker + + args = arguments() + output = Path(args.output) + output.mkdir(parents=True, exist_ok=True) + os.environ["TRACKIO_DIR"] = str(output.resolve() / "trackio") + tasks = load_tasks(args.data, "train") + dataset = Dataset.from_list( + [ + {"prompt": [{"role": "user", "content": task["instruction"]}]} + for task in tasks + ] + ).shuffle(seed=0) + tokenizer = tokenizer_for(args.model) + config = AsyncGRPOConfig( + output_dir=str(output), + learning_rate=3e-6, + lr_scheduler_type="constant", + warmup_steps=0, + max_steps=args.steps, + per_device_train_batch_size=4, + gradient_accumulation_steps=4, + num_generations=8, + max_completion_length=16384, + temperature=0.8, + top_p=1.0, + top_k=-1, + max_inflight_tasks=args.inflight, + max_staleness=4, + token_budget=40960, + fork_threshold_tokens=0, + optim="paged_adamw_8bit", + bf16=True, + dtype="bfloat16", + model_init_kwargs={ + "revision": MODEL_REVISIONS[args.model], + "use_kernels": "lfm" in args.model.lower(), + }, + gradient_checkpointing=True, + gradient_checkpointing_kwargs={"use_reentrant": False}, + vllm_server_base_url=args.vllm_url, + heartbeat_stale_after_s=900, + save_strategy="steps", + save_steps=args.save_steps, + save_total_limit=None, + logging_steps=1, + report_to="trackio", + project="smoldataenv-rl", + run_name=output.name, + trackio_space_id=args.space_id, + seed=0, + ) + worker = HarnessRolloutWorker( + harness_session_factory=partial(TaskFactory, args, tasks), + harness_adapter=None, # OpenEnv captures a CLI agent's own model calls. + rollout_reward_fn=reward, + lossless_capture=True, + model_name=args.model, + processing_class=tokenizer, + dataset=dataset, + reward_funcs=[], + num_generations=8, + max_inflight_tasks=args.inflight, + vllm_server_url=args.vllm_url, + max_tokens=16384, + temperature=0.8, + top_p=1.0, + top_k=-1, + chat_template_kwargs={"enable_thinking": False}, + ) + trainer = AsyncGRPOTrainer( + model=args.model, + args=config, + processing_class=tokenizer, + train_dataset=dataset, + rollout_worker=worker, + ) + if trainer.model.config.model_type == "lfm2": + # Packed samples reset attention positions. LFM's convolutions also need sequence IDs. + def reset_convolutions(module, positional, keyword): + keyword["seq_idx"] = ((keyword["position_ids"] == 0).cumsum(-1) - 1).to( + torch.int32 + ) + return positional, keyword + + trainer.model.register_forward_pre_hook(reset_convolutions, with_kwargs=True) + # Abort if the installed convolution kernel ignores packed sequence boundaries. + trainer.model.eval() + with torch.no_grad(): + ids = torch.tensor( + [[10, 20, 30, 40, 50, 60, 70, 80]], device=trainer.model.device + ) + separate = trainer.model( + input_ids=ids[:, 4:], + position_ids=torch.arange(4, device=ids.device)[None], + use_cache=False, + ).logits + packed = trainer.model( + input_ids=ids, + position_ids=torch.arange(4, device=ids.device).repeat(2)[None], + use_cache=False, + ).logits[:, 4:] + torch.testing.assert_close(packed, separate, atol=0.05, rtol=0.01) + trainer.model.train() + + # 6. Train, then save weights and tokenizer for evaluation. + (output / "training_config.json").write_text(json.dumps(config.to_dict(), indent=2)) + (output / "task_names.json").write_text(json.dumps([t["name"] for t in tasks])) + trainer.train() + trainer.save_model(str(output / "final")) + tokenizer.save_pretrained(output / "final") + trainer.save_state() + + +if __name__ == "__main__": + main() diff --git a/05-multi-harness-rl/train_whitebox.py b/05-multi-harness-rl/train_whitebox.py new file mode 100644 index 00000000..64736aa9 --- /dev/null +++ b/05-multi-harness-rl/train_whitebox.py @@ -0,0 +1,302 @@ +"""Train a bash/SETA agent with synchronous GRPO. Read the numbered sections in order.""" + +# %% 1. Choose the model and run settings. +import argparse +import json +import os +from pathlib import Path + +from datasets import Dataset +from transformers import AutoTokenizer + +from envs.tasks import load_tasks + +MODEL_REVISIONS = { + "LiquidAI/LFM2.5-2.6B": "654f9463ce32b05d0429d76fe1f580b27d4c1ac0", + "Qwen/Qwen3.5-2B": "15852e8c16360a2fea060d615a32b45270f8a8fc", +} + + +def arguments(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", default="LiquidAI/LFM2.5-2.6B") + parser.add_argument("--data", default="prepared") + parser.add_argument("--vllm-url", default="http://127.0.0.1:8000") + parser.add_argument("--output", default="runs/whitebox") + parser.add_argument("--steps", type=int, default=1000) + parser.add_argument("--save-steps", type=int, default=50) + parser.add_argument( + "--space-id", + default=None, + help="Optional Trackio Space; local logs are always kept", + ) + return parser.parse_args() + + +# %% 2. Load the tokenizer in non-thinking mode. +def tokenizer_for(model): + from copy import deepcopy + + from trl.chat_template_utils import lfm2_2_5_template, qwen3_5_template + + tokenizer = AutoTokenizer.from_pretrained(model, revision=MODEL_REVISIONS[model]) + if "lfm" in model.lower(): + # LFM's template opens even when enable_thinking=False. + start = r'{{- "<|im_start|>assistant\n" -}}' + if tokenizer.chat_template.count(start) != 1: + raise ValueError( + "LFM template changed; check its non-thinking generation prefix" + ) + tokenizer.chat_template = tokenizer.chat_template.replace( + start, r'{{- "<|im_start|>assistant\n" -}}' + ) + tokenizer.response_template = deepcopy(lfm2_2_5_template) + else: + tokenizer.chat_template = ( + "{%- set enable_thinking = false -%}\n" + tokenizer.chat_template + ) + tokenizer.response_template = deepcopy(qwen3_5_template) + return tokenizer + + +# %% 3. Give TRL a Python environment. Public methods become model tools. +import math +import shlex +import socket +import time + +from envs.daytona import DaytonaSandboxBackend +from envs.tasks import grade_answer, stage_task + +SYSTEM = "Use the sandbox tools to solve the task. Call submit_solution with the final answer itself." + + +class BashEnvironment: + def __init__(self): + self._sandbox = None + self._calls = 0 + self._deadline = 0 + self._submitted = False + + def reset(self, folder, **kwargs): + self._close() + self._folder = Path(folder) + self._calls, self._submitted = 0, False + self._sandbox = DaytonaSandboxBackend( + image="docker.io/savatar101/env-data-agent-train:base" + ).create(timeout_s=900) + try: + stage_task(self._sandbox, self._folder) + except BaseException: + self._close() + raise + self._deadline = time.monotonic() + 600 + return (self._folder / "instruction.md").read_text() + + def _check(self): + if self._submitted or time.monotonic() >= self._deadline: + raise RuntimeError("Episode is finished") + + def _execute(self, command): + self._check() + self._calls += 1 + result = self._sandbox.exec( + command, + cwd="/workdir", + timeout=min(60, max(1, self._deadline - time.monotonic())), + ) + return (result.stdout + result.stderr)[-16000:] + + def bash(self, command: str) -> str: + """Run a shell command in /workdir. + + Args: + command: The shell command to execute. + """ + return self._execute(command) + + def read(self, path: str) -> str: + """Read a text file. + + Args: + path: Absolute path of the file. + """ + return self._execute("cat -- " + shlex.quote(path)) + + def write(self, path: str, content: str) -> str: + """Write a text file, creating parent directories. + + Args: + path: Absolute path of the file. + content: Text to write. + """ + self._check() + self._calls += 1 + self._sandbox.write_text(path, content) + return "Saved." + + def edit(self, path: str, old: str, new: str) -> str: + """Replace one exact occurrence in a file. + + Args: + path: Absolute path of the file. + old: Text that must occur exactly once. + new: Replacement text. + """ + self._check() + self._calls += 1 + text = self._sandbox.read_text(path) + if text.count(old) != 1: + return "The old text must occur exactly once." + self._sandbox.write_text(path, text.replace(old, new, 1)) + return "Saved." + + def ls(self, path: str = "/workdir") -> str: + """List a directory. + + Args: + path: Directory to list. + """ + return self._execute("ls -la -- " + shlex.quote(path)) + + def grep(self, pattern: str, path: str) -> str: + """Search text files recursively. + + Args: + pattern: Regular expression to search for. + path: File or directory to search. + """ + return self._execute( + "grep -rn -- " + shlex.quote(pattern) + " " + shlex.quote(path) + ) + + def glob(self, pattern: str, path: str = "/workdir") -> str: + """Find files by name. + + Args: + pattern: Filename pattern, for example *.csv. + path: Directory to search. + """ + return self._execute( + "find " + shlex.quote(path) + " -name " + shlex.quote(pattern) + ) + + def submit_solution(self, answer: str) -> str: + """Save your final answer and finish the episode. + + Args: + answer: The answer itself, not a command. + """ + self._check() + self._sandbox.write_text("/workdir/answer.txt", answer) + self._submitted = True + return "Answer saved. End your response now." + + # 4. Grade the answer. Submitting it does not count as a tool action. + def get_reward(self): + try: + self._correctness = grade_answer(self._sandbox, self._folder) + bonus = 0.1 * 15 / (15 + self._calls) if self._calls > 0 else 0.0 + return self._correctness * (1 + bonus) + except (OSError, RuntimeError, ValueError, TimeoutError): + return math.nan # No grade is different from a wrong answer. + finally: + self._close() + + def _close(self): + if self._sandbox is not None: + self._sandbox.kill() + self._sandbox = None + + +# %% 5. Configure GRPO. TRL generates responses and invokes the tools above. +def main(): + from trl import GRPOConfig, GRPOTrainer + + args = arguments() + output = Path(args.output) + output.mkdir(parents=True, exist_ok=True) + os.environ["TRACKIO_DIR"] = str(output.resolve() / "trackio") + tasks = load_tasks(args.data, "train") + dataset = Dataset.from_list( + [ + { + "prompt": [ + {"role": "system", "content": SYSTEM}, + {"role": "user", "content": "Solve the task."}, + ], + "folder": task["folder"], + } + for task in tasks + ] + ).shuffle(seed=0) + tokenizer = tokenizer_for(args.model) + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + weight_sync_port = listener.getsockname()[1] + config = GRPOConfig( + output_dir=str(output), + learning_rate=3e-6, + lr_scheduler_type="constant", + warmup_steps=0, + max_steps=args.steps, + per_device_train_batch_size=1, + gradient_accumulation_steps=16, + num_generations=8, + max_completion_length=16384, + max_tool_calling_iterations=16, + temperature=0.8, + top_p=1.0, + top_k=0, + beta=0.0, + loss_type="dapo", + chat_template_kwargs={"enable_thinking": False, "preserve_thinking": True}, + generation_kwargs={"max_tokens": 4096}, + optim="paged_adamw_8bit", + bf16=True, + model_init_kwargs={ + "dtype": "bfloat16", + "revision": MODEL_REVISIONS[args.model], + }, + gradient_checkpointing=True, + gradient_checkpointing_kwargs={"use_reentrant": False}, + use_vllm=True, + vllm_mode="server", + vllm_server_base_url=args.vllm_url, + vllm_server_timeout=900, + vllm_group_port=weight_sync_port, + vllm_max_model_length=131072, + save_strategy="steps", + save_steps=args.save_steps, + save_total_limit=None, + logging_steps=1, + report_to="trackio", + project="smoldataenv-rl", + run_name=output.name, + trackio_space_id=args.space_id, + seed=0, + ) + trainer = GRPOTrainer( + model=args.model, + processing_class=tokenizer, + args=config, + train_dataset=dataset, + reward_funcs=[], + environment_factory=BashEnvironment, + ) + # 6. Train, then save weights and tokenizer for evaluation. + (output / "training_config.json").write_text(json.dumps(config.to_dict(), indent=2)) + (output / "task_names.json").write_text(json.dumps([t["name"] for t in tasks])) + try: + trainer.train() + trainer.save_model(str(output / "final")) + tokenizer.save_pretrained(output / "final") + trainer.save_state() + finally: + # Environments are also closed after grading; this covers interrupted rollouts. + for environment in trainer.environments or []: + environment._close() + + +if __name__ == "__main__": + main() From 6bd0f04da9aea2778c746e2eb8df930512af364d Mon Sep 17 00:00:00 2001 From: adithya-s-k Date: Thu, 1 Oct 2026 12:15:20 +0530 Subject: [PATCH 18/48] article(mhrl): bring the multi-harness RL article over from 04-data-agent The article was drafted on 04-data-agent, which is archived. This adds it as it stood there (19b727e), with its row in content/README.md, so the review rounds that follow apply on top of 05-multi-harness-rl. Co-Authored-By: Claude Opus 5.5 --- content/README.md | 1 + .../.ai/skills/article-frontmatter/SKILL.md | 116 + .../.ai/skills/create-html-embed/SKILL.md | 133 + .../skills/create-html-embed/directives.md | 504 + .../.ai/skills/deploy-to-hf-spaces/SKILL.md | 121 + .../articles/multi-harness-rl/.cursor/skills | 1 + .../articles/multi-harness-rl/.dockerignore | 36 + .../articles/multi-harness-rl/.gitattributes | 35 + content/articles/multi-harness-rl/.gitignore | 52 + content/articles/multi-harness-rl/AGENTS.md | 80 + content/articles/multi-harness-rl/CLAUDE.md | 11 + content/articles/multi-harness-rl/Dockerfile | 123 + content/articles/multi-harness-rl/GEMINI.md | 1 + content/articles/multi-harness-rl/LICENSE | 33 + content/articles/multi-harness-rl/README.md | 57 + .../multi-harness-rl/app/astro.config.mjs | 91 + .../multi-harness-rl/app/package-lock.json | 12351 ++++++++++++++++ .../multi-harness-rl/app/package.json | 90 + .../app/plugins/astro/generate-llms-txt.mjs | 212 + .../app/plugins/rehype/code-copy.mjs | 94 + .../app/plugins/rehype/post-citation.mjs | 494 + .../app/plugins/rehype/restore-at-in-code.mjs | 22 + .../app/plugins/rehype/wrap-outputs.mjs | 38 + .../app/plugins/rehype/wrap-tables.mjs | 43 + .../remark/ignore-citations-in-code.mjs | 21 + .../app/plugins/remark/output-container.mjs | 23 + .../plugins/remark/unwrap-citation-links.mjs | 57 + .../multi-harness-rl/app/postcss.config.mjs | 14 + .../articles/multi-harness-rl/app/public/data | 1 + .../app/public/hf-space-parent-listener.js | 55 + .../app/public/logos/README.md | 26 + .../app/public/logos/claude-code.svg | 3 + .../app/public/logos/codex.png | Bin 0 -> 4001 bytes .../app/public/logos/gemini-cli.png | Bin 0 -> 12861 bytes .../app/public/logos/goose.png | Bin 0 -> 4093 bytes .../app/public/logos/hermes.png | Bin 0 -> 11456 bytes .../app/public/logos/kimi-cli.png | Bin 0 -> 17113 bytes .../app/public/logos/opencode.svg | 7 + .../app/public/logos/openhands-sdk.svg | 1 + .../app/public/logos/qwen-coder.png | Bin 0 -> 12800 bytes .../app/public/logos/swe-agent.svg | 184 + .../app/public/logos/terminus-2.png | Bin 0 -> 2417 bytes .../app/public/logos/trae-agent.png | Bin 0 -> 1939 bytes .../app/public/papers/README.md | 14 + .../app/public/papers/agentlt.png | Bin 0 -> 52852 bytes .../app/public/papers/katcoder.png | Bin 0 -> 39087 bytes .../app/public/papers/laguna.png | Bin 0 -> 88769 bytes .../app/public/papers/openforge.png | Bin 0 -> 32659 bytes .../app/public/papers/orchard.png | Bin 0 -> 100867 bytes .../multi-harness-rl/app/public/robots.txt | 5 + .../app/public/scripts/color-palettes.js | 274 + .../app/scripts/EXPORT-PDF-BOOK.md | 311 + .../app/scripts/README-PDF-BOOK.md | 309 + .../app/scripts/export-latex.mjs | 358 + .../app/scripts/export-pdf-book-simple.mjs | 626 + .../app/scripts/export-pdf-book.mjs | 612 + .../app/scripts/export-pdf.mjs | 742 + .../app/scripts/extract-used-assets.mjs | 397 + .../app/scripts/generate-trackio-data.mjs | 196 + .../app/scripts/jitter-trackio-data.mjs | 129 + .../app/scripts/latex-importer/README.md | 169 + .../scripts/latex-importer/bib-cleaner.mjs | 104 + .../latex-importer/filters/equation-ids.lua | 134 + .../app/scripts/latex-importer/index.mjs | 138 + .../latex-importer/latex-converter.mjs | 330 + .../scripts/latex-importer/mdx-converter.mjs | 896 ++ .../latex-importer/metadata-extractor.mjs | 170 + .../scripts/latex-importer/package-lock.json | 1272 ++ .../app/scripts/latex-importer/package.json | 33 + .../scripts/latex-importer/post-processor.mjs | 439 + .../latex-importer/reference-preprocessor.mjs | 239 + .../app/scripts/notion-importer/.cursorignore | 1 + .../app/scripts/notion-importer/README.md | 334 + .../app/scripts/notion-importer/env.example | 2 + .../app/scripts/notion-importer/index.mjs | 494 + .../scripts/notion-importer/mdx-converter.mjs | 863 ++ .../notion-importer/notion-converter.mjs | 266 + .../scripts/notion-importer/package-lock.json | 2032 +++ .../app/scripts/notion-importer/package.json | 38 + .../notion-importer/post-processor.mjs | 837 ++ .../notion-importer/static/bibliography.bib | 1711 +++ .../notion-importer/static/frontmatter.mdx | 59 + .../test-comment-fix-processed.md | 30 + .../notion-importer/test-component-spacing.md | 23 + .../notion-importer/test-external-images.md | 15 + .../notion-importer/test-social-media.md | 9 + .../app/scripts/notion-importer/yarn.lock | 1118 ++ .../app/scripts/screenshot-elements.mjs | 905 ++ .../app/scripts/sync-template.mjs | 447 + .../app/src/components/Accordion.astro | 196 + .../app/src/components/Footer.astro | 440 + .../app/src/components/FullWidth.astro | 9 + .../app/src/components/Glossary.astro | 337 + .../app/src/components/HeroArticle.astro | 845 ++ .../app/src/components/HeroPaper.astro | 210 + .../app/src/components/HfUser.astro | 96 + .../app/src/components/HtmlEmbed.astro | 927 ++ .../app/src/components/Image.astro | 629 + .../app/src/components/Note.astro | 124 + .../app/src/components/Quote.astro | 140 + .../app/src/components/RawHtml.astro | 6 + .../app/src/components/Reference.astro | 74 + .../app/src/components/Seo.astro | 81 + .../app/src/components/Sidenote.astro | 270 + .../app/src/components/Stack.astro | 141 + .../app/src/components/TableOfContents.astro | 1152 ++ .../app/src/components/ThemeToggle.astro | 126 + .../app/src/components/Wide.astro | 9 + .../app/src/components/heroUtils.ts | 87 + .../app/src/content/article.mdx | 43 + .../app/src/content/assets/data/.gitkeep | 0 .../src/content/assets/data/harbor-task.json | 140 + .../assets/data/harness-disclosure.json | 77 + .../assets/data/harness-ecosystem.json | 96 + .../assets/data/scaffold-reversal.json | 50 + .../src/content/assets/image/placeholder.png | Bin 0 -> 67 bytes .../app/src/content/bibliography.bib | 213 + .../app/src/content/chapters/async-grpo.mdx | 28 + .../app/src/content/chapters/conclusions.mdx | 43 + .../app/src/content/chapters/data-agent.mdx | 259 + .../app/src/content/chapters/evals.mdx | 19 + .../app/src/content/chapters/introduction.mdx | 152 + .../src/content/chapters/openenv-harbor.mdx | 274 + .../src/content/chapters/rl-environments.mdx | 273 + .../content/chapters/training-ablations.mdx | 35 + .../content/chapters/what-is-a-harness.mdx | 121 + .../src/content/embeds/ablation-matrix.html | 65 + .../app/src/content/embeds/banner-opaque.html | 730 + .../app/src/content/embeds/banner-seam.html | 676 + .../app/src/content/embeds/banner.html | 258 + .../src/content/embeds/d3-anatomy-hero.html | 409 + .../src/content/embeds/d3-capture-levels.html | 54 + .../src/content/embeds/d3-capture-proxy.html | 279 + .../content/embeds/d3-dataset-pipeline.html | 64 + .../content/embeds/d3-dialects-funnel.html | 184 + .../content/embeds/d3-eval-comparison.html | 65 + .../content/embeds/d3-format-mismatch.html | 61 + .../app/src/content/embeds/d3-full-stack.html | 65 + .../src/content/embeds/d3-harbor-task.html | 112 + .../content/embeds/d3-harness-anatomy.html | 250 + .../src/content/embeds/d3-harness-lockin.html | 77 + .../src/content/embeds/d3-llm-rl-coding.html | 707 + .../app/src/content/embeds/d3-nm-hub.html | 64 + .../src/content/embeds/d3-overfit-modes.html | 63 + .../embeds/d3-posttraining-timeline.html | 67 + .../src/content/embeds/d3-rollout-graph.html | 191 + .../content/embeds/d3-scaffold-reversal.html | 163 + .../app/src/content/embeds/d3-serve-flow.html | 62 + .../src/content/embeds/d3-token-contract.html | 124 + .../src/content/embeds/d3-token-drift.html | 145 + .../src/content/embeds/d3-trainer-record.html | 163 + .../content/embeds/d3-training-curves.html | 65 + .../app/src/content/embeds/d3-who-drives.html | 253 + .../app/src/content/embeds/harbor-matrix.html | 60 + .../embeds/harness-ecosystem-matrix.html | 64 + .../src/content/embeds/paper-receipts.html | 73 + .../multi-harness-rl/app/src/env.d.ts | 7 + .../app/src/pages/dataviz.astro | 612 + .../app/src/pages/index.astro | 443 + .../app/src/scripts/README.md | 65 + .../app/src/scripts/mermaid-zoom-optimized.js | 294 + .../app/src/scripts/mermaid-zoom.js | 858 ++ .../multi-harness-rl/app/src/styles/_base.css | 168 + .../app/src/styles/_layout.css | 384 + .../app/src/styles/_print-book.css | 1002 ++ .../app/src/styles/_print.css | 128 + .../app/src/styles/_reset.css | 13 + .../app/src/styles/_variables.css | 126 + .../app/src/styles/components/_button.css | 58 + .../app/src/styles/components/_card.css | 138 + .../app/src/styles/components/_code.css | 357 + .../app/src/styles/components/_form.css | 243 + .../app/src/styles/components/_mermaid.css | 250 + .../app/src/styles/components/_table.css | 121 + .../app/src/styles/components/_tag.css | 14 + .../app/src/styles/global.css | 54 + .../app/src/utils/extract-embeds.mjs | 1089 ++ .../articles/multi-harness-rl/app/yarn.lock | 5857 ++++++++ .../create-research-article/cli.js | 635 + .../create-research-article/package.json | 35 + .../articles/multi-harness-rl/entrypoint.sh | 46 + content/articles/multi-harness-rl/nginx.conf | 61 + .../multi-harness-rl/scripts/release.mjs | 95 + 183 files changed, 58700 insertions(+) create mode 100644 content/articles/multi-harness-rl/.ai/skills/article-frontmatter/SKILL.md create mode 100644 content/articles/multi-harness-rl/.ai/skills/create-html-embed/SKILL.md create mode 100644 content/articles/multi-harness-rl/.ai/skills/create-html-embed/directives.md create mode 100644 content/articles/multi-harness-rl/.ai/skills/deploy-to-hf-spaces/SKILL.md create mode 120000 content/articles/multi-harness-rl/.cursor/skills create mode 100644 content/articles/multi-harness-rl/.dockerignore create mode 100644 content/articles/multi-harness-rl/.gitattributes create mode 100644 content/articles/multi-harness-rl/.gitignore create mode 100644 content/articles/multi-harness-rl/AGENTS.md create mode 100644 content/articles/multi-harness-rl/CLAUDE.md create mode 100644 content/articles/multi-harness-rl/Dockerfile create mode 120000 content/articles/multi-harness-rl/GEMINI.md create mode 100644 content/articles/multi-harness-rl/LICENSE create mode 100644 content/articles/multi-harness-rl/README.md create mode 100644 content/articles/multi-harness-rl/app/astro.config.mjs create mode 100644 content/articles/multi-harness-rl/app/package-lock.json create mode 100644 content/articles/multi-harness-rl/app/package.json create mode 100644 content/articles/multi-harness-rl/app/plugins/astro/generate-llms-txt.mjs create mode 100644 content/articles/multi-harness-rl/app/plugins/rehype/code-copy.mjs create mode 100644 content/articles/multi-harness-rl/app/plugins/rehype/post-citation.mjs create mode 100644 content/articles/multi-harness-rl/app/plugins/rehype/restore-at-in-code.mjs create mode 100644 content/articles/multi-harness-rl/app/plugins/rehype/wrap-outputs.mjs create mode 100644 content/articles/multi-harness-rl/app/plugins/rehype/wrap-tables.mjs create mode 100644 content/articles/multi-harness-rl/app/plugins/remark/ignore-citations-in-code.mjs create mode 100644 content/articles/multi-harness-rl/app/plugins/remark/output-container.mjs create mode 100644 content/articles/multi-harness-rl/app/plugins/remark/unwrap-citation-links.mjs create mode 100644 content/articles/multi-harness-rl/app/postcss.config.mjs create mode 120000 content/articles/multi-harness-rl/app/public/data create mode 100644 content/articles/multi-harness-rl/app/public/hf-space-parent-listener.js create mode 100644 content/articles/multi-harness-rl/app/public/logos/README.md create mode 100644 content/articles/multi-harness-rl/app/public/logos/claude-code.svg create mode 100644 content/articles/multi-harness-rl/app/public/logos/codex.png create mode 100644 content/articles/multi-harness-rl/app/public/logos/gemini-cli.png create mode 100644 content/articles/multi-harness-rl/app/public/logos/goose.png create mode 100644 content/articles/multi-harness-rl/app/public/logos/hermes.png create mode 100644 content/articles/multi-harness-rl/app/public/logos/kimi-cli.png create mode 100644 content/articles/multi-harness-rl/app/public/logos/opencode.svg create mode 100644 content/articles/multi-harness-rl/app/public/logos/openhands-sdk.svg create mode 100644 content/articles/multi-harness-rl/app/public/logos/qwen-coder.png create mode 100644 content/articles/multi-harness-rl/app/public/logos/swe-agent.svg create mode 100644 content/articles/multi-harness-rl/app/public/logos/terminus-2.png create mode 100644 content/articles/multi-harness-rl/app/public/logos/trae-agent.png create mode 100644 content/articles/multi-harness-rl/app/public/papers/README.md create mode 100644 content/articles/multi-harness-rl/app/public/papers/agentlt.png create mode 100644 content/articles/multi-harness-rl/app/public/papers/katcoder.png create mode 100644 content/articles/multi-harness-rl/app/public/papers/laguna.png create mode 100644 content/articles/multi-harness-rl/app/public/papers/openforge.png create mode 100644 content/articles/multi-harness-rl/app/public/papers/orchard.png create mode 100644 content/articles/multi-harness-rl/app/public/robots.txt create mode 100644 content/articles/multi-harness-rl/app/public/scripts/color-palettes.js create mode 100644 content/articles/multi-harness-rl/app/scripts/EXPORT-PDF-BOOK.md create mode 100644 content/articles/multi-harness-rl/app/scripts/README-PDF-BOOK.md create mode 100755 content/articles/multi-harness-rl/app/scripts/export-latex.mjs create mode 100755 content/articles/multi-harness-rl/app/scripts/export-pdf-book-simple.mjs create mode 100755 content/articles/multi-harness-rl/app/scripts/export-pdf-book.mjs create mode 100644 content/articles/multi-harness-rl/app/scripts/export-pdf.mjs create mode 100644 content/articles/multi-harness-rl/app/scripts/extract-used-assets.mjs create mode 100644 content/articles/multi-harness-rl/app/scripts/generate-trackio-data.mjs create mode 100644 content/articles/multi-harness-rl/app/scripts/jitter-trackio-data.mjs create mode 100644 content/articles/multi-harness-rl/app/scripts/latex-importer/README.md create mode 100644 content/articles/multi-harness-rl/app/scripts/latex-importer/bib-cleaner.mjs create mode 100644 content/articles/multi-harness-rl/app/scripts/latex-importer/filters/equation-ids.lua create mode 100644 content/articles/multi-harness-rl/app/scripts/latex-importer/index.mjs create mode 100644 content/articles/multi-harness-rl/app/scripts/latex-importer/latex-converter.mjs create mode 100644 content/articles/multi-harness-rl/app/scripts/latex-importer/mdx-converter.mjs create mode 100644 content/articles/multi-harness-rl/app/scripts/latex-importer/metadata-extractor.mjs create mode 100644 content/articles/multi-harness-rl/app/scripts/latex-importer/package-lock.json create mode 100644 content/articles/multi-harness-rl/app/scripts/latex-importer/package.json create mode 100644 content/articles/multi-harness-rl/app/scripts/latex-importer/post-processor.mjs create mode 100644 content/articles/multi-harness-rl/app/scripts/latex-importer/reference-preprocessor.mjs create mode 100644 content/articles/multi-harness-rl/app/scripts/notion-importer/.cursorignore create mode 100644 content/articles/multi-harness-rl/app/scripts/notion-importer/README.md create mode 100644 content/articles/multi-harness-rl/app/scripts/notion-importer/env.example create mode 100644 content/articles/multi-harness-rl/app/scripts/notion-importer/index.mjs create mode 100644 content/articles/multi-harness-rl/app/scripts/notion-importer/mdx-converter.mjs create mode 100644 content/articles/multi-harness-rl/app/scripts/notion-importer/notion-converter.mjs create mode 100644 content/articles/multi-harness-rl/app/scripts/notion-importer/package-lock.json create mode 100644 content/articles/multi-harness-rl/app/scripts/notion-importer/package.json create mode 100644 content/articles/multi-harness-rl/app/scripts/notion-importer/post-processor.mjs create mode 100644 content/articles/multi-harness-rl/app/scripts/notion-importer/static/bibliography.bib create mode 100644 content/articles/multi-harness-rl/app/scripts/notion-importer/static/frontmatter.mdx create mode 100644 content/articles/multi-harness-rl/app/scripts/notion-importer/test-comment-fix-processed.md create mode 100644 content/articles/multi-harness-rl/app/scripts/notion-importer/test-component-spacing.md create mode 100644 content/articles/multi-harness-rl/app/scripts/notion-importer/test-external-images.md create mode 100644 content/articles/multi-harness-rl/app/scripts/notion-importer/test-social-media.md create mode 100644 content/articles/multi-harness-rl/app/scripts/notion-importer/yarn.lock create mode 100644 content/articles/multi-harness-rl/app/scripts/screenshot-elements.mjs create mode 100644 content/articles/multi-harness-rl/app/scripts/sync-template.mjs create mode 100644 content/articles/multi-harness-rl/app/src/components/Accordion.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/Footer.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/FullWidth.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/Glossary.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/HeroArticle.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/HeroPaper.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/HfUser.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/HtmlEmbed.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/Image.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/Note.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/Quote.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/RawHtml.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/Reference.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/Seo.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/Sidenote.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/Stack.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/TableOfContents.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/ThemeToggle.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/Wide.astro create mode 100644 content/articles/multi-harness-rl/app/src/components/heroUtils.ts create mode 100644 content/articles/multi-harness-rl/app/src/content/article.mdx create mode 100644 content/articles/multi-harness-rl/app/src/content/assets/data/.gitkeep create mode 100644 content/articles/multi-harness-rl/app/src/content/assets/data/harbor-task.json create mode 100644 content/articles/multi-harness-rl/app/src/content/assets/data/harness-disclosure.json create mode 100644 content/articles/multi-harness-rl/app/src/content/assets/data/harness-ecosystem.json create mode 100644 content/articles/multi-harness-rl/app/src/content/assets/data/scaffold-reversal.json create mode 100644 content/articles/multi-harness-rl/app/src/content/assets/image/placeholder.png create mode 100644 content/articles/multi-harness-rl/app/src/content/bibliography.bib create mode 100644 content/articles/multi-harness-rl/app/src/content/chapters/async-grpo.mdx create mode 100644 content/articles/multi-harness-rl/app/src/content/chapters/conclusions.mdx create mode 100644 content/articles/multi-harness-rl/app/src/content/chapters/data-agent.mdx create mode 100644 content/articles/multi-harness-rl/app/src/content/chapters/evals.mdx create mode 100644 content/articles/multi-harness-rl/app/src/content/chapters/introduction.mdx create mode 100644 content/articles/multi-harness-rl/app/src/content/chapters/openenv-harbor.mdx create mode 100644 content/articles/multi-harness-rl/app/src/content/chapters/rl-environments.mdx create mode 100644 content/articles/multi-harness-rl/app/src/content/chapters/training-ablations.mdx create mode 100644 content/articles/multi-harness-rl/app/src/content/chapters/what-is-a-harness.mdx create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/ablation-matrix.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/banner-opaque.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/banner-seam.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/banner.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-anatomy-hero.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-capture-levels.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-capture-proxy.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-dataset-pipeline.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-dialects-funnel.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-eval-comparison.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-format-mismatch.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-full-stack.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-harbor-task.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-harness-anatomy.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-harness-lockin.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-llm-rl-coding.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-nm-hub.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-overfit-modes.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-posttraining-timeline.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-rollout-graph.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-scaffold-reversal.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-serve-flow.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-token-contract.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-token-drift.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-trainer-record.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-training-curves.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/d3-who-drives.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/harbor-matrix.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/harness-ecosystem-matrix.html create mode 100644 content/articles/multi-harness-rl/app/src/content/embeds/paper-receipts.html create mode 100644 content/articles/multi-harness-rl/app/src/env.d.ts create mode 100644 content/articles/multi-harness-rl/app/src/pages/dataviz.astro create mode 100644 content/articles/multi-harness-rl/app/src/pages/index.astro create mode 100644 content/articles/multi-harness-rl/app/src/scripts/README.md create mode 100644 content/articles/multi-harness-rl/app/src/scripts/mermaid-zoom-optimized.js create mode 100644 content/articles/multi-harness-rl/app/src/scripts/mermaid-zoom.js create mode 100644 content/articles/multi-harness-rl/app/src/styles/_base.css create mode 100644 content/articles/multi-harness-rl/app/src/styles/_layout.css create mode 100644 content/articles/multi-harness-rl/app/src/styles/_print-book.css create mode 100644 content/articles/multi-harness-rl/app/src/styles/_print.css create mode 100644 content/articles/multi-harness-rl/app/src/styles/_reset.css create mode 100644 content/articles/multi-harness-rl/app/src/styles/_variables.css create mode 100644 content/articles/multi-harness-rl/app/src/styles/components/_button.css create mode 100644 content/articles/multi-harness-rl/app/src/styles/components/_card.css create mode 100644 content/articles/multi-harness-rl/app/src/styles/components/_code.css create mode 100644 content/articles/multi-harness-rl/app/src/styles/components/_form.css create mode 100644 content/articles/multi-harness-rl/app/src/styles/components/_mermaid.css create mode 100644 content/articles/multi-harness-rl/app/src/styles/components/_table.css create mode 100644 content/articles/multi-harness-rl/app/src/styles/components/_tag.css create mode 100644 content/articles/multi-harness-rl/app/src/styles/global.css create mode 100644 content/articles/multi-harness-rl/app/src/utils/extract-embeds.mjs create mode 100644 content/articles/multi-harness-rl/app/yarn.lock create mode 100755 content/articles/multi-harness-rl/create-research-article/cli.js create mode 100644 content/articles/multi-harness-rl/create-research-article/package.json create mode 100644 content/articles/multi-harness-rl/entrypoint.sh create mode 100644 content/articles/multi-harness-rl/nginx.conf create mode 100644 content/articles/multi-harness-rl/scripts/release.mjs diff --git a/content/README.md b/content/README.md index c1cf2b21..4d5da759 100644 --- a/content/README.md +++ b/content/README.md @@ -8,6 +8,7 @@ Each item ships to the Hub as a Space; the source of truth is here. | Article | Source | Live | |---|---|---| | **The ultimate guide to RL environments** | [`articles/rl-environments-guide/`](./articles/rl-environments-guide/) | [▶️ Space](https://huggingface.co/spaces/AdithyaSK/rl-environments-guide) | +| **The ultimate guide to multi-harness RL** | [`articles/multi-harness-rl/`](./articles/multi-harness-rl/) | [▶️ Space](https://huggingface.co/spaces/AdithyaSK/multi-harness-rl) | Built with [research-article-template](https://huggingface.co/spaces/tfrere/research-article-template) (Astro), served as a Docker Space. diff --git a/content/articles/multi-harness-rl/.ai/skills/article-frontmatter/SKILL.md b/content/articles/multi-harness-rl/.ai/skills/article-frontmatter/SKILL.md new file mode 100644 index 00000000..0d7c3144 --- /dev/null +++ b/content/articles/multi-harness-rl/.ai/skills/article-frontmatter/SKILL.md @@ -0,0 +1,116 @@ +--- +name: article-frontmatter +description: Configure article metadata via MDX frontmatter. Use when the user asks about titles, authors, affiliations, template variants, banner, DOI, PDF export, or any article.mdx frontmatter field. +--- + +# Article Frontmatter Reference + +All metadata lives in `app/src/content/article.mdx` frontmatter (YAML block). + +## Frontmatter fields + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `title` | string | required | Article title. Supports `\n` for line breaks (rendered as `
`). | +| `description` | string | `""` | Short description / subtitle shown below the title. | +| `authors` | array | `[]` | List of authors (see below). | +| `affiliations` | array | `[]` | List of affiliations (see below). | +| `published` | string | — | Publication date, e.g. `"Apr. 04, 2026"`. | +| `template` | `"article"` or `"paper"` | `"article"` | Layout variant (see below). | +| `banner` | string | `"banner.html"` | Banner embed filename in `embeds/`. | +| `doi` | string | — | DOI identifier, shown in footer. | +| `showPdf` | boolean | `true` | Show PDF download button in metadata bar. | +| `tableOfContentsAutoCollapse` | boolean | `false` | Auto-collapse TOC sections on scroll. | +| `licence` | string | — | Licence text (HTML allowed), shown in footer. | +| `pdfProOnly` | boolean | `false` | Gate PDF download behind HF Pro badge. | +| `seoThumbImage` | string | — | Custom OG image URL for social sharing. | +| `links` | array | `[]` | External links shown in paper template hero (see below). | + +## Title line breaks + +Long titles are automatically balanced across lines (`text-wrap: balance`). Titles longer than 60 characters are automatically downsized for readability (>100 chars: even smaller). + +To force a manual line break, use `\n` inside the title string: + +```yaml +title: "Why Open-Source LLMs\nAre Reshaping the AI Landscape" +``` + +This renders as two lines in the Hero section. The plain-text version (for SEO / PDF) strips the break automatically. + +## Template variants + +| Value | Layout | Features | +|-------|--------|----------| +| `article` (default) | Full layout | Banner, sidebar TOC, figure numbering, citation block, DOI, PDF export | +| `paper` | Single centered column | No TOC sidebar, no figure numbering, no citation/DOI block, lighter footer | + +## Authors and affiliations + +```yaml +authors: + - name: "Alice Martin" + url: "https://example.com/alice" + affiliations: [1] + - name: "Bob Chen" + affiliations: [1, 2] +affiliations: + - name: "Hugging Face" + url: "https://huggingface.co" + - name: "MIT" + url: "https://mit.edu" +``` + +Affiliation indices are 1-based and rendered as superscript numbers next to author names. + +## External links (paper template) + +The `links` field adds buttons below the author/affiliation line in the `paper` template hero. Each link has a `label` and a `url`: + +```yaml +links: + - label: "Paper" + url: "https://arxiv.org/abs/..." + - label: "Code" + url: "https://github.com/..." + - label: "Demo" + url: "https://huggingface.co/spaces/..." + - label: "Data" + url: "https://huggingface.co/datasets/..." +``` + +Links are rendered as pill-shaped buttons and only visible in the `paper` template. They are hidden in the `article` template. + +## README tag (critical) + +The project `README.md` contains a YAML frontmatter block with a `tags` field: + +```yaml +tags: + - research-article-template +``` + +**NEVER remove the `research-article-template` tag from the README.** This tag is used by the [Research Article Gallery](https://huggingface.co/spaces/tfrere/research-article-gallery) to discover and list all articles built with this template. Removing it will make the Space invisible in the gallery. + +## Complete example + +```yaml +--- +title: "Scaling Laws for\nNeural Language Models" +description: "An empirical study of scaling behavior across model size, data, and compute" +authors: + - name: "Alice Martin" + url: "https://example.com/alice" + affiliations: [1] +affiliations: + - name: "Hugging Face" + url: "https://huggingface.co" +published: "Apr. 04, 2026" +template: "article" +banner: "banner.html" +doi: "10.1234/example.2026" +showPdf: true +tableOfContentsAutoCollapse: true +licence: "This work is licensed under CC BY 4.0." +--- +``` diff --git a/content/articles/multi-harness-rl/.ai/skills/create-html-embed/SKILL.md b/content/articles/multi-harness-rl/.ai/skills/create-html-embed/SKILL.md new file mode 100644 index 00000000..436aeb88 --- /dev/null +++ b/content/articles/multi-harness-rl/.ai/skills/create-html-embed/SKILL.md @@ -0,0 +1,133 @@ +--- +name: create-html-embed +description: Create self-contained D3 HTML embed charts for the research article template. Use when the user asks to create a chart, visualization, embed, D3 chart, line chart, bar chart, scatter plot, sankey diagram, or any data visualization as an HTML embed file. +--- + +# Create HTML Embed + +Create self-contained D3.js chart embeds for the research article template. + +## Before you start + +**Read the full directives file** for all conventions, patterns, and checklists: + +- [directives.md](directives.md) — single source of truth for embed authoring rules + +This covers: colors & palettes, layout, SVG scope, mounting, theming, controls, tooltips, data loading, responsiveness, legends, accessibility, performance, error handling, printing, and the full agent checklist. + +## Workflow + +### Step 1: Understand the request + +Clarify with the user: +- What type of chart? (line, bar, scatter, sankey, waffle, heatmap, custom) +- What data source? (CSV path, JSON, inline data) +- Interactive controls needed? (metric selector, filters) +- Any specific design requirements? + +### Step 2: Create the HTML file + +- Location: `app/src/content/embeds/` +- Naming: `d3-.html` (e.g., `d3-training-loss.html`) +- Root class: `.d3-` (must match filename) + +### Step 3: Follow the mandatory structure + +Every embed must have this structure: + +```html +
+ + +``` + +### Step 4: Integrate in MDX + +Import and use the `HtmlEmbed` component: + +```mdx +import HtmlEmbed from '../../components/HtmlEmbed.astro'; + + +``` + +#### HtmlEmbed props + +| Prop | Type | Description | +|------|------|-------------| +| `src` | string | Path to HTML file in `embeds/` (required) | +| `title` | string | Title above the card | +| `desc` | string | Description below (supports HTML) | +| `frameless` | boolean | Removes card background/border | +| `wide` | boolean | Wide layout (~1100px) | +| `data` | string or string[] | Path(s) to data files | +| `config` | object | JSON config passed via `data-config` attribute | + +#### Usage examples + +```mdx + + + + + + + + + + + + + + +``` + +## Key conventions (quick reference) + +Full details in the directives file. The critical ones: + +1. **Colors**: Use `window.ColorPalettes.getColors('categorical', n)` — never hardcode palettes +2. **CSS variables**: `--text-color`, `--surface-bg`, `--border-color`, `--axis-color`, `--tick-color`, `--grid-color` +3. **Dark mode**: Check `document.documentElement.getAttribute('data-theme') === 'dark'` +4. **Mount guard**: Always set `container.dataset.mounted = 'true'` +5. **Data loading**: Try `/data/` first, then `./assets/data/` — use `fetchFirstAvailable()` +6. **Responsiveness**: `ResizeObserver` on container, recompute on resize +7. **Legend**: HTML-based, title "Legend", swatch 14x14px +8. **Controls**: HTML only (no SVG UI), selects labeled "Metric" when applicable +9. **Tooltip**: Single `.d3-tooltip` absolutely positioned inside container +10. **No globals**: Everything in IIFE, nothing on `window` + +## Data files + +- Store data in: `app/src/content/assets/data/` +- Served from: `/data/` (public) at build time +- Formats: CSV (preferred for tabular), JSON (for nested/hierarchical) + +## Post-creation checklist + +After creating the embed, verify against the **Agent Checklist** (section 14.1) and **Definition of Done** (section 14.2) in [directives.md](directives.md). diff --git a/content/articles/multi-harness-rl/.ai/skills/create-html-embed/directives.md b/content/articles/multi-harness-rl/.ai/skills/create-html-embed/directives.md new file mode 100644 index 00000000..329833c8 --- /dev/null +++ b/content/articles/multi-harness-rl/.ai/skills/create-html-embed/directives.md @@ -0,0 +1,504 @@ +## Embed Chart Authoring Guidelines + +### Quickstart (TL;DR) +- Create a single self-contained HTML fragment: root div + scoped style + IIFE script. +- Draw marks/axes in SVG; render UI (legend and controls) in HTML. +- Place legend and controls BELOW the chart (header appended after the chart). Include a legend title "Legend" and a select labeled "Metric" when relevant. +- Load data from public `/data` first, then fall back to `assets/data`. +- Use `window.ColorPalettes` for colors; stick to CSS variables for theming. + +Minimal header markup: +```html +
+
Legend
+
+ +
+
+
+ + +
+ +
+``` + +See also: `d3-line-simple.html`, `d3-line-quad.html`, `d3-benchmark.html`. + +Authoring rules for creating a new interactive chart as a single self-contained `.html` file under `src/content/embeds/`. These conventions are derived from `d3-bar.html`, `d3-comparison.html`, `d3-neural.html`, `d3-line.html`, and `d3-pie.html`. + +### A) Colors & palettes (MANDATORY) +- Always obtain color arrays from `window.ColorPalettes`; do not hardcode palettes. +- Use the categorical/sequential/diverging helpers and the current primary color. +- If you change `--primary-color` dynamically, call `window.ColorPalettes.refresh()` so listeners update. + +Usage: +```js +// Usage (with explicit counts) +const cat = window.ColorPalettes.getColors('categorical', 8); +const seq = window.ColorPalettes.getColors('sequential', 8); +const div = window.ColorPalettes.getColors('diverging', 7); + +// For current primary color string +const primaryHex = window.ColorPalettes.getPrimary(); + +// If you change --primary-color dynamically, call refresh to notify listeners +document.documentElement.style.setProperty('--primary-color', '#6D4AFF'); +window.ColorPalettes.refresh(); +``` + +Notes: +- Keep chart accents (lines, markers, selection) aligned with `--primary-color`. +- Prefer CSS variables for fills/strokes when possible; derive series colors via `ColorPalettes`. +- Provide a graceful fallback to CSS variables if `window.ColorPalettes` is unavailable. + +### B) Layout & form elements (HTML-only) +- All UI controls (labels, selects, sliders, buttons, toggles) must be plain HTML inside the root container. +- Do not draw controls with SVG; style them consistently (rounded 8px, custom caret, focus ring). +- Use `