diff --git a/.gitmodules b/.gitmodules index 9f0724cd8e..b937d9f4d5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "evals/deep-swe"] path = evals/deep-swe url = https://github.com/datacurve-ai/deep-swe +[submodule "evals/vendor/pier"] + path = evals/vendor/pier + url = https://github.com/lavaman131/pier diff --git a/DEV_SETUP.md b/DEV_SETUP.md index 3195307bd7..c9842ad580 100644 --- a/DEV_SETUP.md +++ b/DEV_SETUP.md @@ -7,6 +7,8 @@ This document covers setup, the local dev loop, testing patterns, and project la ## Prerequisites - **[Bun](https://bun.sh) ≥ 1.3.14** — runtime, package manager, and test runner +- **[uv](https://docs.astral.sh/uv/)** — Python package/environment manager for the `evals/` harness +- **Docker** — required for local Pier/DeepSWE sandbox runs This repo uses **Bun** for all development, scripts, and testing. The `@bastani/workflows` workspace package ships raw `.ts` files with no build step; Atomic bundles it into `@bastani/atomic` during the coding-agent build. @@ -15,11 +17,53 @@ This repo uses **Bun** for all development, scripts, and testing. The `@bastani/ ## Setup ```bash -git clone git@github.com:bastani-inc/atomic.git +git clone --recurse-submodules git@github.com:bastani-inc/atomic.git cd atomic bun install ``` +If you cloned without submodules, initialize them before running evals or touching vendored benchmark harnesses: + +```bash +git submodule update --init --recursive +``` + +Current submodules include `evals/deep-swe` and `evals/vendor/pier`; the evals package points `datacurve-pier` at the local editable `evals/vendor/pier` checkout. + +The eval harness is Python-based and uses [`uv`](https://docs.astral.sh/uv/) from the `evals/` directory: + +```bash +cd evals +uv sync +uv run python -c 'import pier, pathlib; print(pathlib.Path(pier.__file__).resolve())' +uv run pier --help +``` + +The `pier` import should resolve to `evals/vendor/pier/src/pier/__init__.py`. After pulling submodule pointer changes or local Pier edits, refresh the editable install with: + +```bash +cd evals +uv sync --reinstall-package datacurve-pier +``` + +Example single-task DeepSWE run with Atomic and the local Pier checkout: + +```bash +cd evals +export OPENROUTER_API_KEY=... +uv run pier run \ + -p deep-swe/tasks \ + --agent-import-path atomic_pier:Atomic \ + --model openrouter/openai/gpt-5.5 \ + --agent-kwarg thinking=xhigh \ + --agent-kwarg version=next \ + --agent-env 'OPENROUTER_API_KEY=${OPENROUTER_API_KEY}' \ + --n-tasks 1 \ + --sample-seed 0 \ + --n-concurrent 1 \ + --force-build +``` + `bun install` runs the root `prepare` script, which installs Git hooks with [`prek`](https://prek.j178.dev/) from [`prek.toml`](./prek.toml). The hook shims installed by default come from `default_install_hook_types`; currently that is `pre-commit`. To reinstall hooks manually, run `bun run hooks:install`. Set `PREK_DISABLE_INSTALL=1` to skip hook installation for a local install; CI skips it automatically. The root `package.json` is a private workspace package named `atomic-monorepo`. The only publishable package is `packages/coding-agent` (`@bastani/atomic`); other `packages/*` workspaces are bundled or internal. diff --git a/evals/pyproject.toml b/evals/pyproject.toml index 81c1e1ab26..d592138705 100644 --- a/evals/pyproject.toml +++ b/evals/pyproject.toml @@ -9,4 +9,4 @@ dependencies = [ ] [tool.uv.sources] -datacurve-pier = { git = "https://github.com/datacurve-ai/pier" } +datacurve-pier = { path = "vendor/pier", editable = true } diff --git a/evals/uv.lock b/evals/uv.lock index 1fba0afb2c..669e8ed2a1 100644 --- a/evals/uv.lock +++ b/evals/uv.lock @@ -378,7 +378,7 @@ wheels = [ [[package]] name = "datacurve-pier" version = "0.3.0" -source = { git = "https://github.com/datacurve-ai/pier#3f688a1bd0d15f77f7191609e9f8ecdd3a69db33" } +source = { editable = "vendor/pier" } dependencies = [ { name = "anthropic" }, { name = "anyio" }, @@ -403,6 +403,38 @@ dependencies = [ { name = "uvicorn" }, ] +[package.metadata] +requires-dist = [ + { name = "anthropic", specifier = ">=0.97.0" }, + { name = "anyio", specifier = ">=4.13.0" }, + { name = "botocore", specifier = ">=1.43.11" }, + { name = "claude-agent-sdk", specifier = ">=0.1.17" }, + { name = "daytona", specifier = ">=0.121.0" }, + { name = "fastapi", specifier = ">=0.128.0" }, + { name = "harbor", specifier = ">=0.5.0" }, + { name = "jinja2", specifier = ">=3.1.0" }, + { name = "litellm", specifier = ">=1.83.0" }, + { name = "loguru", specifier = ">=0.7.3" }, + { name = "mcp", specifier = ">=1.27.0" }, + { name = "modal", specifier = ">=1.4.2" }, + { name = "pydantic", specifier = ">=2.13.3" }, + { name = "python-dotenv", specifier = ">=1.2.2" }, + { name = "pyyaml", specifier = ">=6.0.3" }, + { name = "rich", specifier = ">=14.3.4" }, + { name = "shortuuid", specifier = ">=1.0.13" }, + { name = "tenacity", specifier = ">=9.1.4" }, + { name = "toml", specifier = ">=0.10.2" }, + { name = "typer", specifier = ">=0.12" }, + { name = "uvicorn", specifier = ">=0.38.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=9.0.3" }, + { name = "pytest-asyncio", specifier = ">=1.4.0" }, + { name = "ruff", specifier = ">=0.5" }, +] + [[package]] name = "datasets" version = "5.0.0" @@ -593,7 +625,7 @@ dependencies = [ ] [package.metadata] -requires-dist = [{ name = "datacurve-pier", git = "https://github.com/datacurve-ai/pier" }] +requires-dist = [{ name = "datacurve-pier", editable = "vendor/pier" }] [[package]] name = "fastapi" diff --git a/evals/vendor/pier b/evals/vendor/pier new file mode 160000 index 0000000000..bbc6857eaa --- /dev/null +++ b/evals/vendor/pier @@ -0,0 +1 @@ +Subproject commit bbc6857eaab414f14633e1e9d697d1bf974266fc