Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 46 additions & 49 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1333,68 +1333,65 @@ jobs:
PY

dogfood-build:
# One sticky comment per app pull request with a link to a tagged dev build
# of this exact head, so anyone can try the change in one click. The build
# controller treats this job's success as the build request and verifies the
# pull request itself before building, so the job name is part of that
# contract. Not required and not in ci-status: a comment failure never
# blocks a merge.
# Opt-in: only pull requests labelled dev-build get it, on each push and
# when the label is added; app-free changes (docs, web) never do. The
# controller builds the newest head each time a worker frees up.
# Build one exact app pull-request head for trusted org members and hand
# the ZIP to the default-branch publisher. That publisher writes the
# private R2 object and keeps one short-lived signed link on the PR while
# the fleet controller is tailnet-only.
name: "Dogfood build #${{ github.event.pull_request.number }}"
needs: changes
if: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref != 'feat-cmux-next' && contains(github.event.pull_request.labels.*.name, 'dev-build') && github.event.action != 'unlabeled' && (github.event.action != 'labeled' || github.event.label.name == 'dev-build') && github.repository_owner == 'manaflow-ai' && github.event.pull_request.head.repo.full_name == github.repository && needs.changes.result == 'success' && needs.changes.outputs.macos != 'false' }}
runs-on: ubuntu-24.04 # github-hosted-required: trusted pull-request-write token
timeout-minutes: 5
# A failed comment must not turn the run red; the controller acts only on success.
continue-on-error: true
if: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref != 'feat-cmux-next' && contains(github.event.pull_request.labels.*.name, 'dev-build') && github.event.action != 'unlabeled' && (github.event.action != 'labeled' || github.event.label.name == 'dev-build') && github.repository_owner == 'manaflow-ai' && github.event.pull_request.head.repo.full_name == github.repository && (github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'OWNER') && needs.changes.result == 'success' && needs.changes.outputs.macos != 'false' }}
runs-on: ${{ vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-26' }}
timeout-minutes: 45
# This job is intentionally outside ci-status, so a build or publication
# failure is visible without becoming a merge requirement.
permissions:
contents: read
pull-requests: write
steps:
- name: Post the dogfood link
- name: Checkout the exact pull-request head
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 1
submodules: recursive
persist-credentials: false

- name: Download the pinned GhosttyKit
run: ./scripts/download-prebuilt-ghosttykit.sh

- name: Build and package the production-authenticated app
id: artifact
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
CMUX_GHOSTTYKIT_PREPROVISIONED: '1'
CMUX_SKIP_ZIG_BUILD: '1'
CMUX_SWIFT_FRONTEND_WORKAROUND: '1'
CMUX_DEV_CLOUD_ENABLED: '0'
PR: ${{ github.event.pull_request.number }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
set -euo pipefail
tag="pr-$PR-${HEAD_SHA:0:8}"
marker='<!-- cmux:dogfood-build -->'
# One comment per pull request: edit ours in place on every push.
existing=$(gh api "repos/$GH_REPO/issues/$PR/comments" --paginate \
--jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | contains(\"$marker\"))) | .id")
existing=${existing%%$'\n'*}
# The head the previous link named, so a regression can be bisected
# across the pushes the controller skipped (it builds the newest head).
previous=""
if [ -n "$existing" ]; then
previous=$(gh api "repos/$GH_REPO/issues/comments/$existing" --jq '.body' | grep -oE '^\*\*Dogfood build\*\* of `[0-9a-f]{40}`' | grep -oE '[0-9a-f]{40}' || true)
fi
range=""
if [ -n "$previous" ] && [ "$previous" != "$HEAD_SHA" ]; then
commits=$(gh api "repos/$GH_REPO/compare/$previous...$HEAD_SHA" --jq '.total_commits' 2>/dev/null) || true
[[ "$commits" =~ ^[0-9]+$ ]] || commits="?"
range="Covers \`${previous:0:8}..${HEAD_SHA:0:8}\` ([commits: $commits](https://github.com/$GH_REPO/compare/$previous...$HEAD_SHA)) since the previous link, [cmux DEV pr-$PR-${previous:0:8}.app](http://127.0.0.1:17320/pr-$PR-${previous:0:8}); if that push was skipped, its page names the newer build. To build a commit in between: \`cmux-ci build cmux --ref <sha> --tag bisect-<sha8> --workspace https://github.com/$GH_REPO/pull/$PR\`."
fi
cat > body.md <<EOF
$marker
**Dogfood build** of \`$HEAD_SHA\`

[cmux DEV $tag.app](http://127.0.0.1:17320/$tag)

The link opens this exact commit in the cmux dev menu bar app; the page waits until the build is ready. Builds run only while this PR has the \`dev-build\` label. Under load the fleet builds the newest push each time a worker frees up, so some pushes are skipped. It signs in against production, so Cloud or backend changes still need a tagged build with a development backend.
./scripts/reload.sh --tag "$tag" --prod-auth --no-global-cli-links
app="$(find "$HOME/Library/Developer/Xcode/DerivedData" -type d -path "*/Build/Products/Debug/cmux DEV $tag.app" -print -quit)"
test -d "$app"
archive="$RUNNER_TEMP/cmux-$tag.app.zip"
/usr/bin/ditto -c -k --keepParent "$app" "$archive"
digest="$(shasum -a 256 "$archive" | awk '{print $1}')"
{
echo "archive=$archive"
echo "digest=$digest"
echo "tag=$tag"
echo "size=$(stat -f '%z' "$archive")"
} >> "$GITHUB_OUTPUT"

$range
EOF
if [ -n "$existing" ]; then
gh api -X PATCH "repos/$GH_REPO/issues/comments/$existing" -F body=@body.md >/dev/null
else
gh api -X POST "repos/$GH_REPO/issues/$PR/comments" -F body=@body.md >/dev/null
fi
echo "Dogfood link for $tag posted on #$PR."
- name: Upload the exact app for the trusted publisher
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0 # v7.0.1
with:
name: dogfood-app-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
path: ${{ steps.artifact.outputs.archive }}
if-no-files-found: error
retention-days: 2
compression-level: 0

ci-status:
needs:
Expand Down
170 changes: 170 additions & 0 deletions .github/workflows/dogfood-artifact-publish.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
name: Dogfood artifact publish

on:
workflow_run:
workflows: [CI]
types: [completed]

permissions: {}

env:
SOURCE_WORKFLOW_PATHS: .github/workflows/ci.yml

jobs:
publish:
if: ${{ github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request' }}
runs-on: ubuntu-24.04
environment: artifacts
permissions:
actions: read
contents: read
pull-requests: write
steps:
- name: Find the trusted org-member dogfood artifact
id: pr
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
RUN_ID: ${{ github.event.workflow_run.id }}
HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
run: |
set -euo pipefail
pr="$(gh api "repos/$GH_REPO/actions/runs/$RUN_ID" --jq '.pull_requests[0].number // empty')"
if [ -z "$pr" ]; then
echo 'publish=false' >> "$GITHUB_OUTPUT"
exit 0
fi
payload="$(gh api "repos/$GH_REPO/pulls/$pr")"
PR_JSON="$payload" PR_NUMBER="$pr" HEAD_SHA="$HEAD_SHA" python3 - <<'PY'
import json
import os
import sys

pr = json.loads(os.environ["PR_JSON"])
expected_repo = os.environ["GH_REPO"]
head = pr.get("head") or {}
repo = (head.get("repo") or {}).get("full_name")
labels = {label.get("name") for label in pr.get("labels") or []}
trusted = (
pr.get("state") == "open"
and repo == expected_repo
and head.get("sha") == os.environ["HEAD_SHA"]
and pr.get("author_association") in {"MEMBER", "OWNER"}
and "dev-build" in labels
)
if not trusted:
print("workflow run is not a current trusted org-member dev build; skipping")
with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output:
output.write("publish=false\n")
sys.exit(0)
with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output:
output.write("publish=true\n")
output.write(f"pr={os.environ['PR_NUMBER']}\n")
output.write(f"sha={os.environ['HEAD_SHA']}\n")
PY

- name: Checkout trusted publisher scripts
if: steps.pr.outputs.publish == 'true'
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ github.event.repository.default_branch }}
fetch-depth: 1
persist-credentials: false

- name: Download the exact PR app archive
id: artifact
if: steps.pr.outputs.publish == 'true'
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
RUN_ID: ${{ github.event.workflow_run.id }}
PR: ${{ steps.pr.outputs.pr }}
HEAD_SHA: ${{ steps.pr.outputs.sha }}
run: |
set -euo pipefail
mkdir -p "$RUNNER_TEMP/dogfood"
gh run download "$RUN_ID" --repo "$GH_REPO" \
--name "dogfood-app-$PR-$HEAD_SHA" \
--dir "$RUNNER_TEMP/dogfood"
archive="$(find "$RUNNER_TEMP/dogfood" -type f -name '*.zip' -print -quit)"
test -f "$archive"
digest="$(sha256sum "$archive" | awk '{print $1}')"
echo "archive=$archive" >> "$GITHUB_OUTPUT"
echo "digest=$digest" >> "$GITHUB_OUTPUT"

- name: Upload the immutable app to the current R2 artifact store
if: steps.pr.outputs.publish == 'true'
env:
AWS_ACCESS_KEY_ID: ${{ secrets.CMUX_CEF_R2_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.CMUX_CEF_R2_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: auto
R2_ENDPOINT: https://${{ secrets.CMUX_CEF_R2_ACCOUNT_ID }}.r2.cloudflarestorage.com
R2_BUCKET: cmux-fleet-artifacts
ARCHIVE: ${{ steps.artifact.outputs.archive }}
DIGEST: ${{ steps.artifact.outputs.digest }}
run: |
set -euo pipefail
python3 scripts/ci/upload-r2-object.py \
--file "$ARCHIVE" \
--endpoint-url "$R2_ENDPOINT" \
--bucket "$R2_BUCKET" \
--key "artifacts/$DIGEST" \
--content-type application/zip \
--cache-control 'private, no-store' \
--write-once

- name: Sign and verify the download URL
id: signed-url
if: steps.pr.outputs.publish == 'true'
env:
AWS_ACCESS_KEY_ID: ${{ secrets.CMUX_CEF_R2_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.CMUX_CEF_R2_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: auto
R2_ENDPOINT: https://${{ secrets.CMUX_CEF_R2_ACCOUNT_ID }}.r2.cloudflarestorage.com
R2_BUCKET: cmux-fleet-artifacts
DIGEST: ${{ steps.artifact.outputs.digest }}
run: |
set -euo pipefail
url="$(python3 scripts/ci/presign-r2-url.py \
--endpoint-url "$R2_ENDPOINT" \
--bucket "$R2_BUCKET" \
--key "artifacts/$DIGEST" \
--expires-in 900)"
curl --fail --silent --show-error --head "$url" >/dev/null
printf 'url=%s\n' "$url" >> "$GITHUB_OUTPUT"

- name: Post the signed dogfood link
if: steps.pr.outputs.publish == 'true'
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
PR: ${{ steps.pr.outputs.pr }}
HEAD_SHA: ${{ steps.pr.outputs.sha }}
TAG: pr-${{ steps.pr.outputs.pr }}-${{ steps.pr.outputs.sha }}
DIGEST: ${{ steps.artifact.outputs.digest }}
DOWNLOAD_URL: ${{ steps.signed-url.outputs.url }}
MARKER: '<!-- cmux-fleet-dogfood-artifact -->'
run: |
set -euo pipefail
existing="$(gh api "repos/$GH_REPO/issues/$PR/comments" --paginate \
--jq ".[] | select(.body | contains(\\\"$MARKER\\\")) | .id" | head -n1)"
BODY_PATH="$RUNNER_TEMP/dogfood-comment.md" \
python3 - <<'PY'
import os
from pathlib import Path
body = (
f"{os.environ['MARKER']}\n"
f"**cmux dogfood artifact** for `{os.environ['HEAD_SHA']}`\n\n"
f"[Download cmux DEV artifact]({os.environ['DOWNLOAD_URL']})\n\n"
f"SHA-256: `{os.environ['DIGEST']}`\n"
f"Tag: `{os.environ['TAG']}`\n\n"
"The download URL expires after 15 minutes."
)
Path(os.environ['BODY_PATH']).write_text(body + "\n", encoding="utf-8")
PY
if [ -n "$existing" ]; then
gh api -X PATCH "repos/$GH_REPO/issues/comments/$existing" -F body=@"$RUNNER_TEMP/dogfood-comment.md" >/dev/null
else
gh api -X POST "repos/$GH_REPO/issues/$PR/comments" -F body=@"$RUNNER_TEMP/dogfood-comment.md" >/dev/null
fi
echo "Dogfood artifact link posted on #$PR."
103 changes: 103 additions & 0 deletions scripts/ci/presign-r2-url.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
#!/usr/bin/env python3
"""Create a short-lived AWS SigV4 GET URL for one private R2 object."""

from __future__ import annotations

import argparse
import datetime as dt
import hashlib
import hmac
import os
import urllib.parse


def _sign(key: bytes, message: str) -> bytes:
return hmac.new(key, message.encode("utf-8"), hashlib.sha256).digest()


def _signing_key(secret: str, date_stamp: str, region: str) -> bytes:
date_key = _sign(("AWS4" + secret).encode("utf-8"), date_stamp)
region_key = _sign(date_key, region)
service_key = _sign(region_key, "s3")
return _sign(service_key, "aws4_request")


def _canonical_path(bucket: str, key: str) -> str:
parts = [bucket] + [part for part in key.split("/") if part]
return "/" + "/".join(urllib.parse.quote(part, safe="~") for part in parts)


def _canonical_query(values: dict[str, str]) -> str:
return "&".join(
f"{urllib.parse.quote(name, safe='~')}={urllib.parse.quote(value, safe='~')}"
for name, value in sorted(values.items())
)


def presign(args: argparse.Namespace) -> str:
access_key = os.environ.get("AWS_ACCESS_KEY_ID", "")
secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY", "")
if not access_key or not secret_key:
raise SystemExit("AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are required")
parsed = urllib.parse.urlsplit(args.endpoint_url.rstrip("/"))
if parsed.scheme != "https" or not parsed.netloc:
raise SystemExit("R2 endpoint URL must use HTTPS and include a host")
if not args.bucket or not args.key or any(part in {".", ".."} for part in args.key.split("/")):
raise SystemExit("bucket and key are required and key path traversal is refused")
if not 1 <= args.expires_in <= 900:
raise SystemExit("--expires-in must be between 1 and 900 seconds")

amz_date = os.environ.get("CMUX_R2_PRESIGN_AMZ_DATE")
if not amz_date:
amz_date = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
if len(amz_date) != 16 or not amz_date.endswith("Z"):
raise SystemExit("CMUX_R2_PRESIGN_AMZ_DATE must be an AWS timestamp")
date_stamp = amz_date[:8]
region = os.environ.get("AWS_DEFAULT_REGION", "auto")
scope = f"{date_stamp}/{region}/s3/aws4_request"
query = {
"X-Amz-Algorithm": "AWS4-HMAC-SHA256",
"X-Amz-Credential": f"{access_key}/{scope}",
"X-Amz-Date": amz_date,
"X-Amz-Expires": str(args.expires_in),
"X-Amz-SignedHeaders": "host",
}
session_token = os.environ.get("AWS_SESSION_TOKEN")
if session_token:
query["X-Amz-Security-Token"] = session_token
canonical_uri = _canonical_path(args.bucket, args.key)
canonical_query = _canonical_query(query)
canonical_request = "\n".join(
["GET", canonical_uri, canonical_query, f"host:{parsed.netloc}\n", "host", "UNSIGNED-PAYLOAD"]
)
string_to_sign = "\n".join(
[
"AWS4-HMAC-SHA256",
amz_date,
scope,
hashlib.sha256(canonical_request.encode("utf-8")).hexdigest(),
]
)
signature = hmac.new(
_signing_key(secret_key, date_stamp, region),
string_to_sign.encode("utf-8"),
hashlib.sha256,
).hexdigest()
query["X-Amz-Signature"] = signature
return urllib.parse.urlunsplit(
(parsed.scheme, parsed.netloc, canonical_uri, _canonical_query(query), "")
)


def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--endpoint-url", required=True)
parser.add_argument("--bucket", required=True)
parser.add_argument("--key", required=True)
parser.add_argument("--expires-in", type=int, default=900)
print(presign(parser.parse_args()))
return 0


if __name__ == "__main__":
raise SystemExit(main())
4 changes: 4 additions & 0 deletions tests/test-execution.toml
Original file line number Diff line number Diff line change
Expand Up @@ -971,3 +971,7 @@ lane = "linux-guard"
[[test]]
path = "tests/test_nightly_next_guard.py"
lane = "linux-guard"

[[test]]
path = "tests/test_r2_presign.py"
lane = "linux-guard"
Loading
Loading