diff --git a/.github/workflows/dcm-production-deploy.yml b/.github/workflows/dcm-production-deploy.yml new file mode 100644 index 00000000000..e7747fd5f43 --- /dev/null +++ b/.github/workflows/dcm-production-deploy.yml @@ -0,0 +1,187 @@ +name: DCM production deploy + +on: + pull_request: + branches: [dcm-production] + paths: + - ".github/workflows/dcm-production-deploy.yml" + - "deploy/dcm/**" + - "AGENTS.md" + - "docs/DCM_PRODUCTION_DEPLOYMENT.md" + - "docs/DIVINE_CREATIVE_FORK_WORKFLOW.md" + push: + branches: [dcm-production] + workflow_dispatch: + +concurrency: + group: dcm-production-deploy + cancel-in-progress: false + +permissions: {} + +env: + IMAGE_NAME: ghcr.io/divine-creative-ministries/buzz + +jobs: + validate: + name: Validate DCM deployment files + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + + - name: Validate scripts + run: deploy/dcm/tests/test-deploy-scripts.sh + + build: + name: Build immutable production image + if: github.event_name != 'pull_request' + needs: validate + runs-on: ubuntu-24.04 + timeout-minutes: 90 + permissions: + contents: read + packages: write + id-token: write + attestations: write + outputs: + digest: ${{ steps.build.outputs.digest }} + steps: + - name: Require the production branch + env: + SOURCE_REF: ${{ github.ref }} + run: | + if [[ "$SOURCE_REF" != "refs/heads/dcm-production" ]]; then + echo "::error::Production builds must use dcm-production; got $SOURCE_REF" + exit 1 + fi + + - name: Checkout exact production commit + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Verify checked-out commit + run: test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + with: + buildkitd-config-inline: | + [worker.oci] + max-parallelism = 2 + + - name: Log in to GHCR + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and publish exact commit + id: build + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + with: + context: . + file: ./Dockerfile + target: runtime + platforms: linux/amd64 + push: true + tags: ${{ env.IMAGE_NAME }}:dcm-${{ github.sha }} + labels: | + org.opencontainers.image.title=Divine Creative Buzz + org.opencontainers.image.description=Divine Creative Ministries production Buzz relay + org.opencontainers.image.licenses=Apache-2.0 + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.source=https://github.com/${{ github.repository }} + cache-from: type=registry,ref=${{ env.IMAGE_NAME }}-buildcache:amd64 + cache-to: type=registry,ref=${{ env.IMAGE_NAME }}-buildcache:amd64,mode=max,compression=zstd + + - name: Attest image provenance + uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 + with: + subject-name: ${{ env.IMAGE_NAME }} + subject-digest: ${{ steps.build.outputs.digest }} + push-to-registry: true + + - name: Verify published provenance + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + IMAGE_DIGEST: ${{ steps.build.outputs.digest }} + run: gh attestation verify "oci://${IMAGE_NAME}@${IMAGE_DIGEST}" --repo "$GITHUB_REPOSITORY" + + - name: Record immutable image + env: + IMAGE_DIGEST: ${{ steps.build.outputs.digest }} + run: | + { + echo "### DCM production image" + echo + echo "Commit: \`$GITHUB_SHA\`" + echo "Image: \`${IMAGE_NAME}@${IMAGE_DIGEST}\`" + } >> "$GITHUB_STEP_SUMMARY" + + deploy: + name: Deploy to buzz.divinecreative.org + if: needs.build.result == 'success' && vars.DCM_DEPLOY_PAUSED != 'true' + needs: build + runs-on: ubuntu-24.04 + timeout-minutes: 30 + environment: + name: production + url: https://buzz.divinecreative.org + permissions: + contents: read + steps: + - name: Configure restricted SSH identity + env: + DEPLOY_KEY: ${{ secrets.DCM_DEPLOY_SSH_KEY }} + KNOWN_HOSTS: ${{ vars.DCM_DEPLOY_KNOWN_HOSTS }} + run: | + install -m 700 -d "$HOME/.ssh" + printf '%s\n' "$DEPLOY_KEY" > "$HOME/.ssh/dcm_deploy" + chmod 600 "$HOME/.ssh/dcm_deploy" + printf '%s\n' "$KNOWN_HOSTS" > "$HOME/.ssh/known_hosts" + chmod 600 "$HOME/.ssh/known_hosts" + ssh-keygen -y -f "$HOME/.ssh/dcm_deploy" >/dev/null + ssh-keygen -l -f "$HOME/.ssh/known_hosts" + + - name: Request digest-pinned deployment + env: + DEPLOY_HOST: ${{ vars.DCM_DEPLOY_HOST }} + DEPLOY_PORT: ${{ vars.DCM_DEPLOY_PORT }} + DEPLOY_USER: ${{ vars.DCM_DEPLOY_USER }} + IMAGE_DIGEST: ${{ needs.build.outputs.digest }} + run: | + : "${DEPLOY_HOST:?missing DCM_DEPLOY_HOST}" + : "${DEPLOY_PORT:?missing DCM_DEPLOY_PORT}" + : "${DEPLOY_USER:?missing DCM_DEPLOY_USER}" + : "${IMAGE_DIGEST:?missing image digest}" + ssh \ + -i "$HOME/.ssh/dcm_deploy" \ + -p "$DEPLOY_PORT" \ + -o BatchMode=yes \ + -o IdentitiesOnly=yes \ + -o StrictHostKeyChecking=yes \ + -o UserKnownHostsFile="$HOME/.ssh/known_hosts" \ + -o ConnectTimeout=20 \ + "$DEPLOY_USER@$DEPLOY_HOST" \ + "deploy $GITHUB_SHA ${IMAGE_NAME}@${IMAGE_DIGEST}" + + - name: Deployment summary + env: + IMAGE_DIGEST: ${{ needs.build.outputs.digest }} + run: | + { + echo "### Production deployment passed" + echo + echo "URL: https://buzz.divinecreative.org" + echo "Commit: \`$GITHUB_SHA\`" + echo "Image: \`${IMAGE_NAME}@${IMAGE_DIGEST}\`" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/AGENTS.md b/AGENTS.md index 4f03b312bc3..2c8b38d7e01 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,6 +6,65 @@ code style, PR process, architecture), see [CONTRIBUTING.md](CONTRIBUTING.md). --- +## Divine Creative Fork Policy + +These rules apply when the working repository is +`Divine-Creative-Ministries/buzz`, or when work is based on that fork's +`dcm-production` branch. Before syncing upstream, changing branch strategy, +releasing, or deploying, read +[docs/DIVINE_CREATIVE_FORK_WORKFLOW.md](docs/DIVINE_CREATIVE_FORK_WORKFLOW.md). + +- Treat the fork's `main` branch as an exact, fast-forward-only mirror of + `block/buzz:main`. Never commit Divine Creative changes to it. +- Put Divine Creative changes on short-lived `agent/*` branches and open pull + requests into `dcm-production`. Do not push directly to `main` or + `dcm-production`, and never force-push either branch. +- Fetch both `origin` and `upstream` before selecting a base or syncing. Branch + general-purpose upstream contributions from `upstream/main`; do not include + private integrations, deployment configuration, or Divine Creative branding + in those branches. +- Before merging a new upstream version into `dcm-production`, create and push + a recovery tag for the current production commit. Perform the merge on a + short-lived sync branch and merge it through a reviewed pull request. +- Resolve conflicts by understanding both changes. Never use blanket + `ours`/`theirs`, "accept all," history rewriting, or change deletion merely + to make a merge pass. Explain intentional removals in the pull request. +- Run the repository quality gates and relevant client/integration tests after + every upstream sync. A textually clean merge can still have behavioral or + schema conflicts. +- Never develop by editing `/opt/buzz`, running containers, or production data + on the VPS. Build from a reviewed commit, deploy an immutable image, record + its source SHA and digest, back up state first, and retain a tested rollback + path. +- Treat every merge into `dcm-production` as authorization for an immediate + automatic deployment to `buzz.divinecreative.org`. Read + [docs/DCM_PRODUCTION_DEPLOYMENT.md](docs/DCM_PRODUCTION_DEPLOYMENT.md) before + changing workflows, deployment assets, Compose behavior, migrations, or + production configuration. Do not merge work that is not production-ready. +- Preserve the restricted deployment boundary: GitHub may request only a full + commit SHA and digest-pinned image through the forced-command `buzzdeploy` + key. Never grant that key a shell, broaden its sudo rule, use a floating + image tag, expose secrets, remove deployment serialization, or bypass image + verification, backup, health, migration, or rollback checks. +- Set `DCM_DEPLOY_PAUSED=true` before merging a change that requires a + maintenance window, uninstalled production configuration, or reviewed + migration recovery. Root-owned deployment tooling on the VPS is updated only + through a separate operator action; it never self-modifies from CI. +- Distribute the customized production iOS app as a **Private Custom App** in + App Store Connect, restricted to the Divine Creative organization in Apple + Business and delivered through Apple Business or MDM. TestFlight is for + temporary testing only; development, Ad Hoc, Enterprise, unlisted, and public + App Store distribution are not production substitutes unless an explicit, + documented exception is approved. See the iOS policy in the fork workflow. +- Never commit credentials, `.env` contents, private keys, backup archives, or + production data. A merge to `dcm-production` authorizes application image + deployment only; it does not authorize unrelated infrastructure changes or + replacement of the root-owned deployment trust boundary. +- If the requested action would violate these rules, or the repository/branch + state is ambiguous, stop and ask for direction instead of guessing. + +--- + ## Ecosystem Buzz spans five repos. This one (`block/buzz`) is the OSS source for the relay, desktop, mobile, and CLI. The others handle internal builds and deployment: diff --git a/deploy/dcm/README.md b/deploy/dcm/README.md new file mode 100644 index 00000000000..8f3e6e1dda5 --- /dev/null +++ b/deploy/dcm/README.md @@ -0,0 +1,46 @@ +# DCM Production Deployment Assets + +These files implement the production boundary documented in +[`docs/DCM_PRODUCTION_DEPLOYMENT.md`](../../docs/DCM_PRODUCTION_DEPLOYMENT.md). +They are maintained in Git, but installation or replacement of root-owned VPS +deployment tooling is always a separate operator action. + +## Installed Layout + +| Repository file | VPS destination | +| --- | --- | +| `bin/buzz-deploy` | `/usr/local/sbin/buzz-deploy` | +| `bin/buzz-deploy-gate` | `/usr/local/libexec/buzz-deploy-gate` | +| `bin/buzzctl` | `/usr/local/sbin/buzzctl` | +| `compose/compose.pairing.yml` | `/etc/buzz/compose.pairing.yml` | +| `compose/Caddyfile` | `/etc/buzz/Caddyfile` | + +All installed files are root-owned and not writable by the `buzzdeploy` user. +The production environment stays at `/opt/buzz/deploy/compose/.env`, mode +`0600`, and is never copied to GitHub. + +The `buzzdeploy` SSH key must use a forced command: + +```text +restrict,command="/usr/local/libexec/buzz-deploy-gate" ssh-ed25519 dcm-production-deploy +``` + +The only permitted request is: + +```text +deploy <40-character-commit-sha> ghcr.io/divine-creative-ministries/buzz@sha256:<64-character-digest> +``` + +The gate validates the request and may invoke only the root-owned deployment +script through a narrow passwordless sudo rule. The GitHub key cannot obtain an +interactive shell, forward ports, choose another image registry, request a +rollback, or read production secrets. + +## Validation + +```bash +deploy/dcm/tests/test-deploy-scripts.sh +``` + +The test performs Bash syntax checks, ShellCheck, one accepted gate request, +and rejection tests for malformed or overprivileged requests. diff --git a/deploy/dcm/bin/buzz-deploy b/deploy/dcm/bin/buzz-deploy new file mode 100755 index 00000000000..b9caa52c605 --- /dev/null +++ b/deploy/dcm/bin/buzz-deploy @@ -0,0 +1,315 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +readonly COMPOSE_SOURCE_DIR="/opt/buzz/deploy/compose" +readonly DCM_CONFIG_DIR="/etc/buzz" +readonly STATE_DIR="/var/lib/buzz-deploy" +readonly BACKUP_ROOT="/var/backups/buzz" +readonly LOCK_FILE="/run/lock/buzz-deploy.lock" +readonly DEPLOY_LOG="/var/log/buzz-deploy.log" +readonly PUBLIC_URL="https://buzz.divinecreative.org" +readonly EXPECTED_IMAGE_PREFIX="ghcr.io/divine-creative-ministries/buzz@sha256:" +readonly BACKUP_KEEP="10" + +mkdir -p "$STATE_DIR" "$BACKUP_ROOT" +touch "$DEPLOY_LOG" +chmod 700 "$STATE_DIR" "$BACKUP_ROOT" +chmod 600 "$DEPLOY_LOG" + +exec 9>"$LOCK_FILE" +if ! flock -n 9; then + echo "Another Buzz deployment is already running." >&2 + exit 75 +fi + +log() { + local message="$*" + printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$message" | tee -a "$DEPLOY_LOG" >&2 + logger -t buzz-deploy -- "$message" || true +} + +die() { + log "ERROR: $*" + exit 1 +} + +require_file() { + [[ -f "$1" ]] || die "Missing required file: $1" +} + +require_layout() { + require_file "$COMPOSE_SOURCE_DIR/compose.yml" + require_file "$COMPOSE_SOURCE_DIR/compose.caddy.yml" + require_file "$COMPOSE_SOURCE_DIR/.env" + require_file "$DCM_CONFIG_DIR/compose.pairing.yml" + require_file "$DCM_CONFIG_DIR/Caddyfile" + require_file "$STATE_DIR/image.yml" +} + +compose() { + docker compose \ + --env-file "$COMPOSE_SOURCE_DIR/.env" \ + -f "$COMPOSE_SOURCE_DIR/compose.yml" \ + -f "$COMPOSE_SOURCE_DIR/compose.caddy.yml" \ + -f "$DCM_CONFIG_DIR/compose.pairing.yml" \ + -f "$STATE_DIR/image.yml" \ + "$@" +} + +write_image_overlay() { + local image="$1" + local destination="$2" + local temporary + temporary=$(mktemp "$STATE_DIR/image.XXXXXX") + cat >"$temporary" </dev/null || printf 'unknown\n' +} + +archive_volume() { + local volume="$1" + local archive="$2" + local backup_dir="$3" + docker volume inspect "$volume" >/dev/null + docker run --rm --network none \ + -v "$volume:/source:ro" \ + -v "$backup_dir:/backup" \ + alpine:3.22 \ + tar -C /source -czf "/backup/$archive" . +} + +prune_backups() { + local -a backups=() + local index + mapfile -t backups < <( + find "$BACKUP_ROOT" -mindepth 1 -maxdepth 1 -type d -name 'predeploy-*' \ + -printf '%T@ %p\n' | sort -rn | cut -d' ' -f2- + ) + for ((index = BACKUP_KEEP; index < ${#backups[@]}; index++)); do + [[ "${backups[$index]}" == "$BACKUP_ROOT"/predeploy-* ]] || die "Unsafe backup prune target" + rm -rf -- "${backups[$index]}" + done +} + +create_backup() { + local target_commit="$1" + local timestamp backup_dir + timestamp=$(date -u +%Y%m%dT%H%M%SZ) + backup_dir="$BACKUP_ROOT/predeploy-${timestamp}-${target_commit:0:12}" + install -d -m 700 "$backup_dir" + + log "Stopping relay write paths for a consistent pre-deploy backup" + compose stop relay pairing >/dev/null + + log "Creating pre-deploy backup at $backup_dir" + # Expansion is intentionally performed by the shell inside the container. + # shellcheck disable=SC2016 + if ! compose exec -T postgres sh -euc \ + 'PGPASSWORD="$POSTGRES_PASSWORD" pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc' \ + >"$backup_dir/postgres.dump"; then + log "PostgreSQL backup failed; restarting the previous release" + compose up -d --wait --wait-timeout 300 >/dev/null || true + return 1 + fi + + if ! archive_volume buzz-prod_buzz-minio-data minio.tar.gz "$backup_dir" || + ! archive_volume buzz-prod_buzz-git-data git.tar.gz "$backup_dir" || + ! archive_volume buzz-prod_buzz-redis-data redis.tar.gz "$backup_dir"; then + log "Persistent-volume backup failed; restarting the previous release" + compose up -d --wait --wait-timeout 300 >/dev/null || true + return 1 + fi + + if ! { + install -m 600 "$COMPOSE_SOURCE_DIR/.env" "$backup_dir/production.env" + install -m 600 "$STATE_DIR/image.yml" "$backup_dir/image.yml" + cp -a "$DCM_CONFIG_DIR/compose.pairing.yml" "$DCM_CONFIG_DIR/Caddyfile" "$backup_dir/" + ( + cd "$backup_dir" + sha256sum postgres.dump minio.tar.gz git.tar.gz redis.tar.gz production.env image.yml \ + compose.pairing.yml Caddyfile >SHA256SUMS + ) + printf '%s\n' "$(schema_version)" >"$backup_dir/schema.version" + chmod 600 "$backup_dir"/* + prune_backups + }; then + log "Backup finalization failed; restarting the previous release" + compose up -d --wait --wait-timeout 300 >/dev/null || true + return 1 + fi + printf '%s\n' "$backup_dir" +} + +healthcheck() { + local headers + headers=$(mktemp "$STATE_DIR/pairing-headers.XXXXXX") + trap 'rm -f "$headers"' RETURN + + for _ in {1..60}; do + if curl -fsS --max-time 10 "$PUBLIC_URL/_liveness" >/dev/null && + curl -fsS --max-time 10 "$PUBLIC_URL/_readiness" >/dev/null && + curl -fsS --max-time 10 -H 'Accept: application/nostr+json' "$PUBLIC_URL/" | grep -q '"name"'; then + : >"$headers" + curl --http1.1 -sS --max-time 5 -D "$headers" -o /dev/null \ + -H 'Connection: Upgrade' \ + -H 'Upgrade: websocket' \ + -H 'Sec-WebSocket-Version: 13' \ + -H 'Sec-WebSocket-Key: ZGNtLXByb2R1Y3Rpb24wMQ==' \ + "$PUBLIC_URL/pair" || true + if grep -Eq '^HTTP/[0-9.]+ 101([[:space:]]|$)' "$headers"; then + rm -f "$headers" + trap - RETURN + return 0 + fi + fi + sleep 5 + done + + rm -f "$headers" + trap - RETURN + return 1 +} + +verify_image() { + local commit="$1" + local image="$2" + local revision + docker pull "$image" >/dev/null + revision=$(docker image inspect "$image" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}') + [[ "$revision" == "$commit" ]] || die "Image revision label does not match requested commit" +} + +record_success() { + local commit="$1" + local image="$2" + local schema="$3" + + if [[ -f "$STATE_DIR/current.commit" ]]; then + cp -f "$STATE_DIR/current.commit" "$STATE_DIR/previous.commit" + cp -f "$STATE_DIR/current.image" "$STATE_DIR/previous.image" + cp -f "$STATE_DIR/current.schema" "$STATE_DIR/previous.schema" + fi + printf '%s\n' "$commit" >"$STATE_DIR/current.commit" + printf '%s\n' "$image" >"$STATE_DIR/current.image" + printf '%s\n' "$schema" >"$STATE_DIR/current.schema" + chmod 600 "$STATE_DIR"/* +} + +restore_previous_release() { + local previous_overlay="$1" + cp -f "$previous_overlay" "$STATE_DIR/image.yml" + chmod 600 "$STATE_DIR/image.yml" + compose up -d --wait --wait-timeout 300 + healthcheck +} + +deploy_release() { + local commit="$1" + local image="$2" + local previous_overlay schema_before schema_after backup_dir + + [[ "$commit" =~ ^[0-9a-f]{40}$ ]] || die "Commit must be a full lowercase SHA" + [[ "$image" =~ ^${EXPECTED_IMAGE_PREFIX}[0-9a-f]{64}$ ]] || die "Unexpected image repository or digest" + + require_layout + previous_overlay=$(mktemp "$STATE_DIR/previous-image.XXXXXX") + cp -f "$STATE_DIR/image.yml" "$previous_overlay" + trap 'rm -f "$previous_overlay"' RETURN + + log "Verifying immutable image for commit $commit" + verify_image "$commit" "$image" + docker pull alpine:3.22 >/dev/null + schema_before=$(schema_version) + if ! backup_dir=$(create_backup "$commit"); then + die "Pre-deploy backup failed; the previous release was restarted" + fi + log "Pre-deploy backup completed: $backup_dir" + + write_image_overlay "$image" "$STATE_DIR/image.yml" + log "Starting candidate image $image" + if compose up -d --wait --wait-timeout 300 && healthcheck; then + schema_after=$(schema_version) + record_success "$commit" "$image" "$schema_after" + log "Deployment succeeded for commit $commit at schema $schema_after" + rm -f "$previous_overlay" + trap - RETURN + return 0 + fi + + schema_after=$(schema_version) + if [[ "$schema_before" != "unknown" && "$schema_after" == "$schema_before" ]]; then + log "Candidate failed health checks without a schema change; rolling back automatically" + if restore_previous_release "$previous_overlay"; then + log "Automatic rollback succeeded" + else + die "Candidate and rollback health checks failed; operator recovery is required" + fi + die "Candidate failed health checks and was rolled back" + fi + + die "Candidate failed after schema changed from $schema_before to $schema_after; automatic binary rollback is unsafe. Use backup $backup_dir for reviewed recovery" +} + +manual_backup() { + require_layout + local commit + commit=$(cat "$STATE_DIR/current.commit" 2>/dev/null || printf 'manual-backup') + create_backup "$commit" + compose up -d --wait --wait-timeout 300 >/dev/null + healthcheck || die "Services failed health checks after backup" +} + +manual_rollback() { + require_layout + require_file "$STATE_DIR/previous.commit" + require_file "$STATE_DIR/previous.image" + require_file "$STATE_DIR/previous.schema" + require_file "$STATE_DIR/current.schema" + + local previous_commit previous_image previous_schema current_schema + previous_commit=$(cat "$STATE_DIR/previous.commit") + previous_image=$(cat "$STATE_DIR/previous.image") + previous_schema=$(cat "$STATE_DIR/previous.schema") + current_schema=$(schema_version) + + [[ "$current_schema" == "$previous_schema" ]] || + die "Rollback blocked: current schema $current_schema does not match previous release schema $previous_schema" + + create_backup "$previous_commit" >/dev/null + write_image_overlay "$previous_image" "$STATE_DIR/image.yml" + compose up -d --wait --wait-timeout 300 + healthcheck || die "Manual rollback failed health checks" + record_success "$previous_commit" "$previous_image" "$current_schema" + log "Manual rollback succeeded to commit $previous_commit" +} + +case "${1:-}" in + deploy) + deploy_release "${2:?commit required}" "${3:?digest-pinned image required}" + ;; + backup) + manual_backup + ;; + rollback) + manual_rollback + ;; + *) + echo "Usage: buzz-deploy deploy | backup | rollback" >&2 + exit 64 + ;; +esac diff --git a/deploy/dcm/bin/buzz-deploy-gate b/deploy/dcm/bin/buzz-deploy-gate new file mode 100755 index 00000000000..ac0a1992623 --- /dev/null +++ b/deploy/dcm/bin/buzz-deploy-gate @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly EXPECTED_IMAGE_PREFIX="ghcr.io/divine-creative-ministries/buzz@sha256:" +readonly ORIGINAL_COMMAND="${SSH_ORIGINAL_COMMAND:-}" + +reject() { + echo "Rejected production deployment request." >&2 + exit 64 +} + +IFS=' ' read -r action commit image extra <<<"${ORIGINAL_COMMAND}" + +[[ "${action:-}" == "deploy" ]] || reject +[[ "${commit:-}" =~ ^[0-9a-f]{40}$ ]] || reject +[[ "${image:-}" =~ ^${EXPECTED_IMAGE_PREFIX}[0-9a-f]{64}$ ]] || reject +[[ -z "${extra:-}" ]] || reject + +if [[ "${BUZZ_DEPLOY_GATE_TEST_ONLY:-false}" == "true" ]]; then + printf 'deploy %s %s\n' "$commit" "$image" + exit 0 +fi + +exec sudo -n /usr/local/sbin/buzz-deploy deploy "$commit" "$image" diff --git a/deploy/dcm/bin/buzzctl b/deploy/dcm/bin/buzzctl new file mode 100755 index 00000000000..2781c569f43 --- /dev/null +++ b/deploy/dcm/bin/buzzctl @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly COMPOSE_SOURCE_DIR="/opt/buzz/deploy/compose" +readonly DCM_CONFIG_DIR="/etc/buzz" +readonly STATE_DIR="/var/lib/buzz-deploy" + +compose() { + docker compose \ + --env-file "$COMPOSE_SOURCE_DIR/.env" \ + -f "$COMPOSE_SOURCE_DIR/compose.yml" \ + -f "$COMPOSE_SOURCE_DIR/compose.caddy.yml" \ + -f "$DCM_CONFIG_DIR/compose.pairing.yml" \ + -f "$STATE_DIR/image.yml" \ + "$@" +} + +case "${1:-help}" in + status|ps) + compose ps + ;; + config) + compose config + ;; + logs) + shift + compose logs -f "${@:-relay}" + ;; + start|up) + compose up -d --wait --wait-timeout 300 + ;; + restart) + compose up -d --wait --wait-timeout 300 --force-recreate relay pairing + ;; + stop|down) + compose down + ;; + backup) + exec /usr/local/sbin/buzz-deploy backup + ;; + rollback) + exec /usr/local/sbin/buzz-deploy rollback + ;; + deploy) + exec /usr/local/sbin/buzz-deploy deploy "${2:?commit required}" "${3:?digest-pinned image required}" + ;; + add-member) + exec docker compose \ + --env-file "$COMPOSE_SOURCE_DIR/.env" \ + -f "$COMPOSE_SOURCE_DIR/compose.yml" \ + exec relay /usr/local/bin/buzz-admin add-member \ + --pubkey "${2:?npub or hex pubkey required}" "${@:3}" + ;; + remove-member) + exec docker compose \ + --env-file "$COMPOSE_SOURCE_DIR/.env" \ + -f "$COMPOSE_SOURCE_DIR/compose.yml" \ + exec relay /usr/local/bin/buzz-admin remove-member \ + --pubkey "${2:?npub or hex pubkey required}" "${@:3}" + ;; + list-members) + exec docker compose \ + --env-file "$COMPOSE_SOURCE_DIR/.env" \ + -f "$COMPOSE_SOURCE_DIR/compose.yml" \ + exec relay /usr/local/bin/buzz-admin list-members + ;; + help|-h|--help) + cat <<'EOF' +Usage: buzzctl + +Commands: + status Show production containers + config Validate the rendered Compose configuration + logs [service] Follow logs; relay is the default + start Start the pinned production release + restart Recreate the relay and pairing services + stop Stop the production stack without deleting volumes + backup Create a consistent pre-deploy-style backup + rollback Return to the previous image when schema-compatible + deploy Manually deploy an approved digest-pinned image + add-member [--role member|admin] + Add a relay member + remove-member [--role member|admin] + Remove a relay member + list-members List relay members +EOF + ;; + *) + echo "Unknown command: $1" >&2 + exit 64 + ;; +esac diff --git a/deploy/dcm/compose/Caddyfile b/deploy/dcm/compose/Caddyfile new file mode 100644 index 00000000000..f622863074a --- /dev/null +++ b/deploy/dcm/compose/Caddyfile @@ -0,0 +1,8 @@ +{$BUZZ_DOMAIN} { + encode zstd gzip + + @pairing path /pair + reverse_proxy @pairing pairing:5000 + + reverse_proxy relay:3000 +} diff --git a/deploy/dcm/compose/compose.pairing.yml b/deploy/dcm/compose/compose.pairing.yml new file mode 100644 index 00000000000..d0d31cb6fc7 --- /dev/null +++ b/deploy/dcm/compose/compose.pairing.yml @@ -0,0 +1,30 @@ +services: + relay: + environment: + BUZZ_PAIRING_RELAY_URL: wss://buzz.divinecreative.org/pair + + pairing: + image: ${BUZZ_IMAGE:-ghcr.io/block/buzz:main} + entrypoint: ["/usr/local/bin/buzz-pair-relay"] + environment: + BUZZ_PAIR_RELAY_BIND_ADDR: 0.0.0.0:5000 + expose: + - "5000" + healthcheck: + test: ["CMD-SHELL", "bash -ec 'exec 3<>/dev/tcp/127.0.0.1/5000'"] + interval: 10s + timeout: 3s + retries: 6 + start_period: 5s + restart: unless-stopped + networks: + - buzz-net + + caddy: + depends_on: + pairing: + condition: service_healthy + relay: + condition: service_healthy + volumes: + - /etc/buzz/Caddyfile:/etc/caddy/Caddyfile:ro diff --git a/deploy/dcm/tests/test-deploy-scripts.sh b/deploy/dcm/tests/test-deploy-scripts.sh new file mode 100755 index 00000000000..89eba9c54b1 --- /dev/null +++ b/deploy/dcm/tests/test-deploy-scripts.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +GATE="$ROOT/deploy/dcm/bin/buzz-deploy-gate" + +bash -n "$ROOT/deploy/dcm/bin/buzz-deploy" "$ROOT/deploy/dcm/bin/buzz-deploy-gate" "$ROOT/deploy/dcm/bin/buzzctl" +shellcheck "$ROOT/deploy/dcm/bin/buzz-deploy" "$ROOT/deploy/dcm/bin/buzz-deploy-gate" "$ROOT/deploy/dcm/bin/buzzctl" + +commit=0123456789abcdef0123456789abcdef01234567 +digest=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +image="ghcr.io/divine-creative-ministries/buzz@sha256:$digest" + +actual=$(SSH_ORIGINAL_COMMAND="deploy $commit $image" BUZZ_DEPLOY_GATE_TEST_ONLY=true "$GATE") +[[ "$actual" == "deploy $commit $image" ]] + +invalid_commands=( + "deploy short $image" + "deploy $commit ghcr.io/other/buzz@sha256:$digest" + "deploy $commit $image extra" + "rollback" + "" +) + +for command in "${invalid_commands[@]}"; do + if SSH_ORIGINAL_COMMAND="$command" BUZZ_DEPLOY_GATE_TEST_ONLY=true "$GATE" >/dev/null 2>&1; then + echo "Gate accepted invalid command: $command" >&2 + exit 1 + fi +done + +echo "DCM deployment script validation passed" diff --git a/docs/DCM_PRODUCTION_DEPLOYMENT.md b/docs/DCM_PRODUCTION_DEPLOYMENT.md new file mode 100644 index 00000000000..cdd05bb71ff --- /dev/null +++ b/docs/DCM_PRODUCTION_DEPLOYMENT.md @@ -0,0 +1,124 @@ +# Divine Creative Production Deployment + +This is the operating contract for `buzz.divinecreative.org`. It applies to +humans, coding agents, GitHub Actions, and VPS operators. + +## Deployment Authorization + +Merging a pull request into `dcm-production` authorizes and automatically +starts a production deployment. Do not merge work that is experimental, +partially tested, waiting on configuration, or not intended for immediate +production use. + +The fork's `main` branch never deploys and remains an exact upstream mirror. +Feature branches and upstream-sync branches target `dcm-production` through +pull requests. Direct pushes and force-pushes to either protected branch are +prohibited. + +Set the repository variable `DCM_DEPLOY_PAUSED=true` before merging when a +maintenance window, external dependency, unsafe migration, or production +incident makes automatic deployment inappropriate. While paused, the workflow +still builds and attests the immutable image but does not contact the VPS. +Unpausing and manually dispatching the workflow from `dcm-production` deploys +the current approved commit. + +## Trust Boundary + +The workflow `.github/workflows/dcm-production-deploy.yml`: + +1. Validates the DCM deployment scripts. +2. Builds the exact `dcm-production` commit for Linux AMD64. +3. Publishes `ghcr.io/divine-creative-ministries/buzz:dcm-`. +4. Records the immutable digest and creates GitHub build provenance. +5. Verifies that provenance before the deployment job starts. +6. Uses the GitHub `production` environment to request deployment over SSH. + +The environment contains only a dedicated SSH private key. Its matching public +key is installed for the VPS user `buzzdeploy` with a forced command and SSH +forwarding disabled. The key cannot open a shell. It can request only a deploy +of the expected GHCR repository using a full commit SHA and SHA-256 digest. + +The GitHub runner pins the VPS's Ed25519 host key with strict host-key checking. +The VPS pulls by digest, then verifies the image's +`org.opencontainers.image.revision` label matches the requested commit. + +## On-Host Deployment + +`/usr/local/sbin/buzz-deploy` is root-owned and serialized by +`/run/lock/buzz-deploy.lock`. A deployment performs these steps: + +1. Pull and verify the candidate image without changing running services. +2. Record the current SQLx migration version. +3. Stop the relay and pairing write paths for a consistent backup window. +4. Create a PostgreSQL custom-format dump and archives of MinIO media, Git + objects, and Redis state under `/var/backups/buzz`. +5. Copy the root-only environment and deployment configuration into that + root-only backup and generate SHA-256 checksums. +6. Retain the ten newest pre-deployment backups. +7. Pin both relay and pairing services to the candidate image digest. +8. Start the stack and wait for Compose health checks. +9. Verify public liveness, readiness, NIP-11, TLS routing, and the `/pair` + WebSocket upgrade. +10. Record the successful commit, digest, and migration version in + `/var/lib/buzz-deploy`. + +Deployment events are written to `/var/log/buzz-deploy.log`, the system journal, +and the GitHub Actions run. Secret values are never logged. + +Backups on the VPS are fast rollback safeguards, not disaster recovery for a +lost server. Copying encrypted backups to the Divine Creative NAS remains a +separate infrastructure task. + +## Failure and Rollback + +If the candidate fails and the SQLx migration version did not change, the +deployment script automatically restores the preceding image overlay, starts +the prior release, and verifies its health. + +If the migration version changed, automatic binary rollback is prohibited. An +older binary may not understand the new schema. The workflow fails and operator +review is required; recovery may require restoring the complete pre-deploy +backup. Never improvise a database downgrade. + +Changes under `migrations/` therefore require explicit migration and recovery +notes in the pull request. Pause automatic deployment before merge unless the +change is demonstrably backward-compatible and the recovery procedure has been +tested. + +## Operator Commands + +Run these through the existing `buzzops` account with `sudo`: + +```bash +sudo buzzctl status +sudo buzzctl config >/dev/null +sudo buzzctl logs relay +sudo buzzctl backup +sudo buzzctl rollback +``` + +`rollback` refuses to proceed when the current database migration version does +not match the previous release's recorded version. + +Root-owned deployment scripts and Compose overrides do not update themselves +from GitHub. Updating those security-sensitive files requires a separate, +reviewed operator installation and validation. Ordinary application code +deploys automatically through the immutable image. + +## Agent Rules + +- Treat every merge to `dcm-production` as an immediate production release. +- Never add arbitrary SSH commands, secret printing, floating image tags, or + mutable remote scripts to the workflow. +- Never broaden the `buzzdeploy` sudo or `authorized_keys` permissions. +- Never store the production environment, SSH private key, database dumps, or + application data in the repository or workflow artifacts. +- Preserve workflow concurrency and the VPS `flock`; deployments must remain + serial. +- Do not bypass build, attestation, backup, migration, health, or rollback + checks to make a deployment pass. +- Deployment-infrastructure changes require both repository review and a + separate root-owned VPS installation. A merge alone does not replace the + installed trust-boundary scripts. +- If production behavior, migration safety, or recovery compatibility is + uncertain, pause deployment before merge and ask the operator. diff --git a/docs/DIVINE_CREATIVE_FORK_WORKFLOW.md b/docs/DIVINE_CREATIVE_FORK_WORKFLOW.md new file mode 100644 index 00000000000..bcea4a1f25c --- /dev/null +++ b/docs/DIVINE_CREATIVE_FORK_WORKFLOW.md @@ -0,0 +1,178 @@ +# Divine Creative Fork Workflow + +This document defines how Divine Creative Ministries maintains and deploys its +Buzz fork without losing local work when the upstream project changes. + +## Repository and Branch Roles + +| Name | Role | +| --- | --- | +| `block/buzz` (`upstream`) | Official Buzz source and destination for generally useful contributions | +| `Divine-Creative-Ministries/buzz` (`origin`) | Divine Creative fork | +| `main` | Exact, fast-forward-only mirror of `upstream/main`; no Divine Creative commits | +| `dcm-production` | Long-lived integration branch for reviewed Divine Creative changes | +| `agent/*` | Short-lived implementation or upstream-sync branches targeting `dcm-production` | + +Keep the two lines of work separate: + +- Fixes and features useful to the wider Buzz project should be developed from + `upstream/main` and proposed to `block/buzz` under its contribution rules. +- Divine Creative branding, private integrations, environment-specific + behavior, and deployment configuration belong on branches targeting + `dcm-production`. +- A broadly useful experiment may begin downstream behind a feature flag, then + be cleaned up and submitted upstream as a separate change. + +## Normal Development + +1. Fetch current state from both remotes. +2. Create an `agent/` branch from `origin/dcm-production`. +3. Make focused commits that follow the upstream project's conventions, + including DCO sign-off where required. +4. Run the relevant quality gates and tests. +5. Open a pull request into `dcm-production`. Merge only after review and green + checks. + +Do not push feature commits directly to `main` or `dcm-production`. + +## Bringing in Upstream Updates + +The safe update sequence is: + +```bash +git fetch origin --prune +git fetch upstream --prune + +git switch main +git merge --ff-only upstream/main +git push origin main + +git tag -a "dcm-before-upstream-YYYYMMDD-HHMM" origin/dcm-production \ + -m "Divine Creative state before upstream sync" +git push origin "dcm-before-upstream-YYYYMMDD-HHMM" + +git switch -c "agent/sync-upstream-YYYYMMDD" origin/dcm-production +git merge --no-ff origin/main +``` + +Replace the timestamp placeholders with the actual UTC date and time. Then: + +1. Inspect every conflict in the context of both upstream intent and Divine + Creative's intended behavior. +2. Resolve overlaps manually. Do not use blanket `ours`, blanket `theirs`, or + "accept all" commands. +3. Review migrations, configuration defaults, external protocols, and client + behavior even when Git reports no textual conflict. +4. Run `just ci` plus any relevant relay, desktop, mobile, pairing, migration, + and end-to-end checks. +5. Open a pull request from the sync branch into `dcm-production`. Document + conflicts, decisions, test evidence, and any deployment or migration notes. + +Git normally stops on conflicting edits rather than silently choosing one. +The review and test steps protect against semantic conflicts, where both sets +of code merge cleanly but no longer behave correctly together. + +If upstream later implements the same general feature as the fork, remove the +downstream duplicate in a separate reviewed change after verifying that the +upstream behavior fully replaces it. + +## Customized iOS App Distribution + +The preferred and required production distribution method for Divine +Creative's customized Buzz iOS app is an Apple **Private Custom App**: + +1. Maintain the app under Divine Creative's active Apple Developer Program + organization account. +2. Create the app record in App Store Connect with **Private** distribution + selected from the outset. +3. Restrict availability to Divine Creative's verified Apple Business + organization ID. +4. Submit the app and every production update to Apple App Review. +5. Assign the approved app privately through Apple Business, preferably using + managed app assignment through the organization's MDM when device management + is available. + +This uses Apple's App Store distribution infrastructure without listing the +app publicly. Production releases do not have TestFlight's 90-day build +expiration, and installed App Store-distributed apps are not tied to an Ad Hoc +or development provisioning profile that must be periodically replaced. + +Keep the Apple Developer membership, Apple Business organization, bundle ID, +signing access, and App Store Connect ownership active and documented. This is +a durable deployment method, not a promise that a build will remain compatible +with every future iOS release or continue to be downloadable after accounts or +agreements lapse. + +The following methods have narrower purposes and are not the production path: + +- **TestFlight:** temporary internal or external beta testing only; each build + expires after 90 days. +- **Development or Ad Hoc signing:** device-limited testing only; these methods + depend on registered devices and provisioning profiles. +- **Apple Developer Enterprise Program:** not the default; it is restricted to + qualifying organizations and its in-house distribution still requires + certificate, profile, and membership lifecycle management. +- **Unlisted App Store distribution:** not private because anyone with the link + can potentially access it. +- **Public App Store distribution:** requires a separate documented decision. + Apple does not allow an approved app record to switch freely between private + and public distribution. + +Agents preparing an iOS release must verify the target is the Private Custom +App record and the Divine Creative Apple Business organization before upload. +Do not create a replacement public or unlisted record, change the bundle ID, or +rotate signing ownership without explicit approval and a migration plan. + +Official references: + +- [Set distribution methods](https://developer.apple.com/help/app-store-connect/manage-your-apps-availability/set-distribution-methods) +- [Learn about Custom Apps in Apple Business](https://support.apple.com/guide/business/learn-about-custom-apps-axm58ba3112a/web) +- [TestFlight overview](https://developer.apple.com/help/app-store-connect/test-a-beta-version/testflight-overview) +- [Apple Developer Program renewal](https://developer.apple.com/help/account/membership/renewal) + +## Deployment Rules + +A merge into `dcm-production` is permission to build and automatically deploy +that exact commit to `buzz.divinecreative.org`. Agents and reviewers must treat +the merge button as a production release action. The complete trust boundary, +backup, health-check, and rollback contract is defined in +[DCM_PRODUCTION_DEPLOYMENT.md](DCM_PRODUCTION_DEPLOYMENT.md). + +- Never use the live VPS as a development workspace. Do not hand-edit + `/opt/buzz`, files inside running containers, or production database state to + implement a feature. +- Build and publish an immutable image tag, such as + `ghcr.io/divine-creative-ministries/buzz:dcm-`. Record the complete + source commit and resulting image digest, and deploy by digest rather than by + the mutable tag. +- Before deployment, protect the root-only environment/configuration files and + create recoverable backups of PostgreSQL, object/media storage, and any + persistent Git data. Validate restoration in an isolated environment when + practical. +- Record migration expectations before starting. Do not assume a previous + binary can safely run after a one-way schema migration. +- After deployment, run service health checks and representative desktop and + mobile smoke tests. +- Keep the previous image, configuration, and recovery instructions available + until the new version is proven healthy. +- Pause automatic deployment before merging changes that require a maintenance + window, uninstalled root-owned configuration, or a migration whose recovery + path has not been proven. Deployment-security scripts on the VPS never update + themselves from a repository merge. + +Secrets, private keys, `.env` contents, backup archives, and production data +must never be committed to Git or attached to a public pull request. + +## Conflict and Recovery Rules + +- Never rewrite or force-push the protected branch histories to simplify a + sync. +- Never delete a local change only because it conflicts. Preserve it, adapt it, + or remove it intentionally with a documented reason. +- If a merge becomes unsafe or unclear, abort it and return to the pre-sync tag + rather than improvising on production. +- Keep changes small enough that a reviewer can identify which behavior is + upstream, which is Divine Creative-specific, and why both are needed. + +This workflow preserves three independent recovery points: the untouched +upstream mirror, the pre-sync production tag, and the previous deployed image.