diff --git a/.github/workflows/api-parity-audit.yml b/.github/workflows/api-parity-audit.yml index af6014c..28f10c6 100644 --- a/.github/workflows/api-parity-audit.yml +++ b/.github/workflows/api-parity-audit.yml @@ -45,6 +45,163 @@ jobs: - name: Test the audit run: python -m unittest scripts/test_audit_api_coverage.py + - name: Verify rolling PR credential + env: + REGEN_PR_TOKEN: ${{ secrets.REGEN_PR_TOKEN }} + run: | + set -euo pipefail + + if [ -z "$REGEN_PR_TOKEN" ]; then + echo "::error::REGEN_PR_TOKEN is missing or empty. Replace it with a token that has Contents and Pull requests read/write access to coval-ai/cli, then rerun this workflow. A secret name in repository settings does not prove that Actions receives a value." + exit 1 + fi + + api_headers=( + --header "Authorization: Bearer $REGEN_PR_TOKEN" + --header "Accept: application/vnd.github+json" + --header "X-GitHub-Api-Version: 2022-11-28" + ) + curl_options=( + --silent + --show-error + --connect-timeout 10 + --max-time 30 + ) + + github_api_status() { + local label="$1" + local response_headers="$2" + local response_body="$3" + local status + shift 3 + + if ! status="$( + curl "${curl_options[@]}" \ + --dump-header "$response_headers" \ + --output "$response_body" \ + --write-out '%{http_code}' \ + "${api_headers[@]}" \ + "$@" + )"; then + echo "::error::$label could not reach GitHub within 30 seconds. Check GitHub availability and rerun the workflow." >&2 + return 1 + fi + printf '%s' "$status" + } + + response_header() { + local response_headers="$1" + local header_name="$2" + awk -v header_name="$header_name" ' + index(tolower($0), tolower(header_name) ":") == 1 { + sub(/^[^:]*:[[:space:]]*/, "") + sub(/\r$/, "") + value = $0 + } + END { print value } + ' "$response_headers" + } + + report_api_failure() { + local label="$1" + local required_access="$2" + local status="$3" + local response_headers="$4" + local response_body="$5" + local message remaining reset retry_after + + message="$( + jq -r '(.message // "GitHub API request failed") | tostring | gsub("[\\r\\n]"; " ") | .[0:300]' \ + "$response_body" 2>/dev/null || true + )" + message="${message:-GitHub API request failed}" + remaining="$(response_header "$response_headers" x-ratelimit-remaining)" + reset="$(response_header "$response_headers" x-ratelimit-reset)" + retry_after="$(response_header "$response_headers" retry-after)" + + if [ "$status" = "429" ] || { [ "$status" = "403" ] && [ "$remaining" = "0" ]; }; then + echo "::error::$label was rate-limited by GitHub (HTTP $status; retry-after=${retry_after:-not supplied}; rate-limit-reset=${reset:-not supplied}). Rerun after the indicated limit resets. GitHub: $message" + elif [ "$status" = "401" ]; then + echo "::error::$label could not authenticate (HTTP 401). Replace the expired, revoked, or invalid REGEN_PR_TOKEN. GitHub: $message" + elif [ "$status" = "403" ]; then + echo "::error::$label was denied by repository or organization policy, or REGEN_PR_TOKEN lacks effective $required_access (HTTP 403). GitHub: $message" + elif [ "$status" = "404" ]; then + echo "::error::$label could not access coval-ai/cli (HTTP 404). Confirm the token is scoped to this repository and its owner can access it. GitHub: $message" + else + echo "::error::$label received an unexpected GitHub response (HTTP $status). GitHub: $message" + fi + } + + repo_response="$RUNNER_TEMP/regen-pr-repo.json" + repo_headers="$RUNNER_TEMP/regen-pr-repo.headers" + if ! repo_status="$( + github_api_status \ + "REGEN_PR_TOKEN repository check" \ + "$repo_headers" \ + "$repo_response" \ + https://api.github.com/repos/coval-ai/cli + )"; then + exit 1 + fi + if [ "$repo_status" != "200" ]; then + report_api_failure \ + "REGEN_PR_TOKEN repository check" \ + "repository read access" \ + "$repo_status" \ + "$repo_headers" \ + "$repo_response" + exit 1 + fi + + # These deliberately invalid requests cannot create a ref or PR. GitHub + # returns 422 only after authorizing each write endpoint. + ref_response="$RUNNER_TEMP/regen-pr-ref-probe.json" + ref_headers="$RUNNER_TEMP/regen-pr-ref-probe.headers" + if ! ref_status="$( + github_api_status \ + "REGEN_PR_TOKEN Contents permission check" \ + "$ref_headers" \ + "$ref_response" \ + --request POST \ + --data '{"ref":"refs/heads/__coval_cli_permission_probe__","sha":"0000000000000000000000000000000000000000"}' \ + https://api.github.com/repos/coval-ai/cli/git/refs + )"; then + exit 1 + fi + if [ "$ref_status" != "422" ]; then + report_api_failure \ + "REGEN_PR_TOKEN Contents permission check" \ + "Contents read/write access" \ + "$ref_status" \ + "$ref_headers" \ + "$ref_response" + exit 1 + fi + + probe_suffix="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + pr_response="$RUNNER_TEMP/regen-pr-pull-probe.json" + pr_headers="$RUNNER_TEMP/regen-pr-pull-probe.headers" + if ! pr_status="$( + github_api_status \ + "REGEN_PR_TOKEN Pull requests permission check" \ + "$pr_headers" \ + "$pr_response" \ + --request POST \ + --data "{\"title\":\"permission probe\",\"head\":\"__permission_probe_head_${probe_suffix}__\",\"base\":\"__permission_probe_base_${probe_suffix}__\"}" \ + https://api.github.com/repos/coval-ai/cli/pulls + )"; then + exit 1 + fi + if [ "$pr_status" != "422" ]; then + report_api_failure \ + "REGEN_PR_TOKEN Pull requests permission check" \ + "Pull requests read/write access" \ + "$pr_status" \ + "$pr_headers" \ + "$pr_response" + exit 1 + fi + - name: Refresh the deterministic parity report run: >- python scripts/audit_api_coverage.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 47035ef..863d615 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,6 +43,24 @@ jobs: with: python-version: "3.12" + - name: Verify Homebrew publishing credential + env: + GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} + run: | + set -euo pipefail + if [ -z "$GH_TOKEN" ]; then + echo "::error::HOMEBREW_TAP_TOKEN is missing or empty." + exit 1 + fi + if ! can_push="$(gh api repos/coval-ai/homebrew-tap --jq '.permissions.push // false')"; then + echo "::error::HOMEBREW_TAP_TOKEN could not access coval-ai/homebrew-tap." + exit 1 + fi + if [ "$can_push" != "true" ]; then + echo "::error::HOMEBREW_TAP_TOKEN cannot push to coval-ai/homebrew-tap. Replace it with a token that has Contents read/write access." + exit 1 + fi + - name: Match Cargo manifests and release tag run: python scripts/release_version.py --expected-tag "$RELEASE_TAG" diff --git a/Cargo.lock b/Cargo.lock index 5ac8e54..b6e5f5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -278,7 +278,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "coval" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index 842aa48..d9973e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "coval" -version = "0.6.0" +version = "0.7.0" edition = "2021" description = "CLI for Coval AI agent evaluation platform" license = "MIT" diff --git a/README.md b/README.md index fc1cd66..d7f5760 100644 --- a/README.md +++ b/README.md @@ -10,12 +10,15 @@ Command-line interface for the [Coval](https://coval.dev) AI evaluation platform brew install coval-ai/tap/coval ``` -### Cargo +### Build from source ```bash -cargo install coval +cargo install --git https://github.com/coval-ai/cli --locked ``` +The CLI is not currently published to crates.io. Official versioned artifacts +are the GitHub release binaries and the `coval-ai/tap/coval` Homebrew formula. + ### Binary Download pre-built binaries from [Releases](https://github.com/coval-ai/cli/releases). @@ -221,6 +224,47 @@ until the command implementation or an explicitly reviewed manifest exception reconciles the drift. A GitHub issue is used only if the automation itself fails before it can create or update that PR. +The schedule is Monday 2:00 AM PST (10:00 UTC; 3:00 AM during daylight saving +time). GitHub Actions schedules can start later during busy periods. The audit +compares the live public OpenAPI catalog with the checked-in manifest and +first-class Rust command surface. It does not synthesize command UX or publish +a CLI release. A maintainer must implement newly reported commands, regenerate +the report, include the appropriate version bump, and merge the green PR. + +To run or recover the workflow: + +```bash +# Run the same audit locally. +python3 -m venv .venv +.venv/bin/python -m pip install --requirement scripts/requirements-audit.txt +.venv/bin/python -m unittest scripts/test_audit_api_coverage.py +.venv/bin/python scripts/audit_api_coverage.py \ + --write-markdown api-coverage-report.md + +# Confirm the secret record and recent runs. This cannot verify the secret value. +gh secret list --repo coval-ai/cli +gh run list --repo coval-ai/cli --workflow api-parity-audit.yml --limit 5 + +# Replace a missing, empty, expired, or revoked token without putting it in shell history. +gh secret set REGEN_PR_TOKEN --repo coval-ai/cli + +# Prove the replacement by dispatching and watching that exact workflow run. +run_id="$( + gh api repos/coval-ai/cli/actions/workflows/api-parity-audit.yml/dispatches \ + --method POST \ + -H 'X-GitHub-Api-Version: 2026-03-10' \ + -F ref=main \ + -F return_run_details=true \ + --jq '.workflow_run_id' +)" +gh run watch "$run_id" --repo coval-ai/cli --exit-status +``` + +`REGEN_PR_TOKEN` must be a fine-grained token limited to `coval-ai/cli` with +Contents and Pull requests read/write access. A visible secret name is not +proof that its stored value is non-empty or usable; only a successful workflow +run proves that. Do not print the token or pass it as a command-line argument. + The SDK regeneration workflow can open deterministic codegen PRs because its published clients are generated from OpenAPI. The CLI command surface is still hand-written, so this repository does not present an automated audit as command @@ -245,6 +289,52 @@ python3 scripts/bump_version.py minor python3 scripts/bump_version.py patch ``` +Publishing a new version: + +1. Include the version bump in the implementation PR. Use `minor` for new + first-class commands and `patch` for backward-compatible fixes. +2. Run `cargo fmt --check`, `cargo clippy --all-targets -- -D warnings`, + `cargo test`, and the API coverage commands above. Confirm that + `Cargo.toml`, `Cargo.lock`, and `api-coverage-report.md` are all updated. +3. Merge only after current-head CI and review are green. The successful CI run + for that exact `main` commit triggers `Release on version bump`. +4. Verify the `v*` tag, all five binary artifacts, `SHA256SUMS`, and the GitHub + release. Then verify that `coval-ai/homebrew-tap` contains the same version + and that `brew update && brew upgrade coval-ai/tap/coval` installs it. + +If the automatic release needs a retry, dispatch `Release on version bump` from +the `main` branch. It reuses an existing matching tag safely. Do not hand-create +a tag unless intentionally using the lower-level `Release` workflow; a tag must +exactly match the Cargo version, and the lower-level workflow still requires +all release credentials. A successful GitHub release is not complete publishing +proof until the Homebrew formula reports the same version. + +To recover Homebrew publishing, replace the token interactively, dispatch the +retry from `main`, and verify both destinations: + +```bash +gh secret set HOMEBREW_TAP_TOKEN --repo coval-ai/cli +expected_tag="$(python3 scripts/release_version.py)" +expected_version="${expected_tag#v}" +run_id="$( + gh api repos/coval-ai/cli/actions/workflows/release-on-version-bump.yml/dispatches \ + --method POST \ + -H 'X-GitHub-Api-Version: 2026-03-10' \ + -F ref=main \ + -F return_run_details=true \ + --jq '.workflow_run_id' +)" +gh run watch "$run_id" --repo coval-ai/cli --exit-status +gh release view "$expected_tag" --repo coval-ai/cli +gh api -H 'Accept: application/vnd.github.raw+json' \ + repos/coval-ai/homebrew-tap/contents/Formula/coval.rb | \ + grep -F "version \"$expected_version\"" +``` + +The release validation checks that `HOMEBREW_TAP_TOKEN` can push to +`coval-ai/homebrew-tap` before building artifacts. As with `REGEN_PR_TOKEN`, a +secret record alone does not prove that the credential is present or authorized. + The release workflow validates tag/version consistency, builds all five target binaries, creates or updates the GitHub release, and updates `coval-ai/homebrew-tap`. A manual `Release on version bump` dispatch safely diff --git a/api-coverage-report.md b/api-coverage-report.md index 31ee249..9723a34 100644 --- a/api-coverage-report.md +++ b/api-coverage-report.md @@ -11,10 +11,10 @@ opens or updates a PR only when coverage actually changes. | Metric | Value | | --- | ---: | | Reconciliation status | PASS | -| Published operations | 174 | -| First-class CLI operations | 118 | +| Published operations | 175 | +| First-class CLI operations | 119 | | Reviewed gaps | 56 | -| Client operations | 126 | +| Client operations | 127 | Catalog: https://api.coval.dev/v1/openapi diff --git a/api-coverage.toml b/api-coverage.toml index c38a886..edb941f 100644 --- a/api-coverage.toml +++ b/api-coverage.toml @@ -3,9 +3,9 @@ [snapshot] catalog_url = "https://api.coval.dev/v1/openapi" -reviewed_at = "2026-08-04" -published_operations = 174 -cli_supported_operations = 118 +reviewed_at = "2026-08-12" +published_operations = 175 +cli_supported_operations = 119 [[allowed_extra]] operation = "POST /test-cases/{test_case_id}/media:upload-url" diff --git a/src/client/mod.rs b/src/client/mod.rs index 6ddfedb..1baeeb6 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -704,6 +704,12 @@ impl MetricsClient<'_> { Ok(resp.metric) } + pub async fn duplicate(&self, id: &str) -> Result { + let url = self.0.url(&format!("/v1/metrics/{id}/duplicate")); + let resp: models::GetMetricResponse = self.0.post_empty(url).await?; + Ok(resp.metric) + } + pub async fn update( &self, id: &str, diff --git a/src/commands/metrics.rs b/src/commands/metrics.rs index b89a8d6..71036bf 100644 --- a/src/commands/metrics.rs +++ b/src/commands/metrics.rs @@ -17,6 +17,7 @@ pub enum MetricCommands { List(ListArgs), Get(GetArgs), Create(Box), + Duplicate(DuplicateArgs), Update(Box), Delete(DeleteArgs), Test(TestArgs), @@ -60,6 +61,7 @@ impl MetricCommands { Self::List(_) => "list", Self::Get(_) => "get", Self::Create(_) => "create", + Self::Duplicate(_) => "duplicate", Self::Update(_) => "update", Self::Delete(_) => "delete", Self::Test(_) => "test", @@ -103,6 +105,12 @@ pub struct GetArgs { metric_id: String, } +#[derive(Args)] +pub struct DuplicateArgs { + /// ID of the metric to duplicate + metric_id: String, +} + #[derive(Args)] #[command( after_help = "Required fields by metric type:\n llm-binary --prompt\n categorical --prompt --categories\n numerical --prompt --min-value --max-value\n audio-binary --prompt\n audio-categorical --prompt --categories\n audio-numerical --prompt --min-value --max-value\n toolcall --prompt\n metadata --metadata-field-type --metadata-field-key\n regex --regex-pattern\n pause --min-pause-duration\n composite --criteria-source test_case --criteria-path expected_behaviors\n or --criteria-source metric_metadata --criteria \"a,b,c\"" @@ -432,6 +440,16 @@ pub async fn execute(cmd: MetricCommands, client: &CovalClient, ctx: &OutputCont next_actions::item_result("metrics", &metric.id), ); } + MetricCommands::Duplicate(args) => { + let metric = client.metrics().duplicate(&args.metric_id).await?; + emit_one_with_actions( + ctx, + "metrics", + operation, + &metric, + next_actions::item_result("metrics", &metric.id), + ); + } MetricCommands::Update(args) => { let args = *args; let mut input = args.input_json.object()?; diff --git a/tests/cli_tests.rs b/tests/cli_tests.rs index 02a83a9..45753be 100644 --- a/tests/cli_tests.rs +++ b/tests/cli_tests.rs @@ -3480,6 +3480,39 @@ async fn test_metrics_create_composite_test_case() { .stdout(predicate::str::contains("comp1")); } +#[tokio::test] +async fn test_metrics_duplicate() { + let mock_server = MockServer::start().await; + + Mock::given(method("POST")) + .and(path("/v1/metrics/met_source/duplicate")) + .and(header("X-API-Key", "test_key")) + .respond_with(ResponseTemplate::new(201).set_body_json(json!({ + "metric": { + "name": "metrics/met_copy", + "id": "met_copy", + "metric_name": "Copied metric", + "description": "A duplicated metric", + "metric_type": "METRIC_LLM", + "create_time": "2026-08-12T12:00:00Z" + } + }))) + .mount(&mock_server) + .await; + + coval() + .arg("--api-key") + .arg("test_key") + .arg("--api-url") + .arg(mock_server.uri()) + .arg("metrics") + .arg("duplicate") + .arg("met_source") + .assert() + .success() + .stdout(predicate::str::contains("met_copy")); +} + #[test] fn test_metrics_create_composite_test_case_requires_criteria_path() { coval()