diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2cc82976e..ca7c7e7b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,8 +70,12 @@ jobs: - name: Run ShellCheck run: | - if compgen -G "scripts/*.sh" > /dev/null; then - shellcheck --severity=warning scripts/*.sh + # Recurse so subdirectories like scripts/lib/ (sourceable libraries) + # are linted too. -x follows `# shellcheck source=...` directives. + shopt -s globstar nullglob + shell_files=(scripts/**/*.sh) + if [ ${#shell_files[@]} -gt 0 ]; then + shellcheck --severity=warning -x "${shell_files[@]}" else echo "No shell scripts found — skipping" fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..e01bc4c3b --- /dev/null +++ b/.gitignore @@ -0,0 +1,391 @@ +# ============================================================================ +# petry-projects baseline .gitignore — SECRETS ONLY +# ---------------------------------------------------------------------------- +# First layer of defense in the Push Protection Standard +# (standards/push-protection.md). Complements GitHub secret scanning + push +# protection, gitleaks pre-commit hooks, and the CI gitleaks job. +# +# Scope: SECRETS ONLY. This file is intentionally language-agnostic. Put +# build artifacts, OS cruft, and editor swap files in per-repo .gitignores or +# the matching language template from https://github.com/github/gitignore. +# +# Ordering note: keep `!` negations IMMEDIATELY AFTER the broad pattern they +# carve out of. Git evaluates rules top-to-bottom; a later ignore can re-hide +# a previously-negated file. A negation inside an already-ignored directory +# does NOT re-include it — always negate by file, never by directory. +# ============================================================================ + + +# --------------------------------------------------------------------------- +# 1. Dotenv family +# --------------------------------------------------------------------------- +# .env files are the single most common source of leaked credentials on GitHub. +.env +.env.* +.envrc +.env.local +.env.*.local +# Keep committed templates so contributors know which vars to set. +!.env.example +!.env.sample +!.env.template +!.env.*.example +!.env.*.sample +!.env.*.template +# direnv and dotenv-vault artifacts that can contain plaintext values +.direnv/ +.envrc.local +.env.vault.keys + + +# --------------------------------------------------------------------------- +# 2. Cloud provider credential files +# --------------------------------------------------------------------------- +# AWS shared credentials & SSO/CLI token caches (live AccessKey / SessionToken) +.aws/ +aws.credentials +aws_credentials +credentials.csv +# GCP service-account JSON keys and Application Default Credentials +gcp-key.json +gcp-service-account*.json +service-account*.json +service_account*.json +application_default_credentials.json +gha-creds-*.json +# Azure CLI profile / MSAL token cache +.azure/ +azureProfile.json +azureAuth.json +msal_token_cache.* +# DigitalOcean, Linode, Hetzner, Fly, Vercel, Netlify, Cloudflare +.doctl/ +.linode-cli +.hcloud/ +.fly/ +fly.toml.local +.vercel/ +.netlify/ +.wrangler/ +.cloudflared/ + + +# --------------------------------------------------------------------------- +# 3. Kubernetes / container / Helm secrets +# --------------------------------------------------------------------------- +# kubeconfigs embed bearer tokens and client certs +kubeconfig +kubeconfig.* +*.kubeconfig +.kube/ +# Docker registry auth (base64 basic auth for registries) +.docker/config.json +docker-config.json +.dockercfg +# Helm values files that conventionally hold secrets +secrets.yaml +secrets.yml +values.secret.yaml +values.secret.yml +values-secrets.yaml +values-secrets.yml +*.secrets.yaml +*.secrets.yml +# helm-secrets / sops-encrypted values are SAFE to commit — re-allow: +!*.enc.yaml +!*.enc.yml +!secrets.enc.yaml +!secrets.enc.yml +# Skaffold / Tilt local overrides +skaffold.local.yaml +tilt_config.json + + +# --------------------------------------------------------------------------- +# 4. SSH / TLS / GPG key material +# --------------------------------------------------------------------------- +# Private keys — any common format +*.pem +*.key +*.cer +*.der +*.p12 +*.pfx +*.jks +*.keystore +*.truststore +id_rsa +id_rsa.* +id_dsa +id_dsa.* +id_ecdsa +id_ecdsa.* +id_ed25519 +id_ed25519.* +*_rsa +*_dsa +*_ecdsa +*_ed25519 +known_hosts +authorized_keys +# GPG / PGP +*.gpg +*.pgp +*.asc +secring.* +# Re-allow common PUBLIC artifacts that legitimately live in repos +!*.pub +!*.pub.pem +!public.pem +!public_key.pem +!*.crt +!ca.crt +!*.cert +# NOTE: *.pem / *.key have HIGH false-positive rates (TLS test fixtures, JWT +# libraries, webpack dev certs). Per-repo .gitignores should `!` specific +# fixture FILES — never negate a whole directory. + + +# --------------------------------------------------------------------------- +# 5. Terraform / IaC state and variables +# --------------------------------------------------------------------------- +# State files contain plaintext secrets resolved at apply-time +*.tfstate +*.tfstate.* +*.tfstate.backup +crash.log +crash.*.log +# .tfvars routinely hold secrets — block all, allow only examples +*.tfvars +*.tfvars.json +!*.tfvars.example +!*.tfvars.sample +!example.tfvars +# Local .terraform cache +.terraform/ +.terraform.lock.hcl.bak +# Pulumi stack configs (often hold encrypted + plaintext values) +Pulumi.*.yaml +!Pulumi.yaml +# Ansible vault password files +vault-password +vault_password +.vault_pass +.vault-password-file + + +# --------------------------------------------------------------------------- +# 6. Secret manager local caches & key material +# --------------------------------------------------------------------------- +# SOPS / age — private keys are plaintext files +.sops/ +sops.agekey +age.key +age-key.txt +keys.txt +# HashiCorp Vault token files +.vault-token +.vault_token +vault-token +# Doppler CLI config (contains service tokens) +.doppler.yaml +.doppler/ +doppler.yaml +# 1Password CLI session tokens / service-account tokens +.op/ +op-session-* +.1password/ +# Infisical, Bitwarden CLI, chamber, envchain, teller +.infisical.json +.bw-session +.chamber +.teller.yml + + +# --------------------------------------------------------------------------- +# 7. Database dumps and DB client dotfiles +# --------------------------------------------------------------------------- +# Raw dumps frequently contain PII + embedded credentials in CREATE USER stmts +*.sql.gz +*.sql.bz2 +*.sql.xz +*.sql.zst +*.dump +*.bak +dump.rdb +mongodump/ +# Client config dotfiles that store passwords in plaintext +.pgpass +.my.cnf +.mylogin.cnf +.mongorc.js +.mongoshrc.js +.psqlrc.local +.pg_service.conf + + +# --------------------------------------------------------------------------- +# 8. Package registry / tooling credential dotfiles +# --------------------------------------------------------------------------- +# Recent supply-chain attacks specifically targeted these files. +.npmrc +.yarnrc +.yarnrc.yml +.pypirc +.gem/credentials +.gem/ +.cargo/credentials +.cargo/credentials.toml +.nuget/ +nuget.config +NuGet.Config +.m2/settings.xml +.gradle/gradle.properties +gradle-local.properties +.netrc +_netrc +.curlrc +.wgetrc +# GH CLI / Git Credential Manager +.gh-token +gh_token +.git-credentials +.config/gh/hosts.yml +# NOTE: .npmrc / .yarnrc.yml / nuget.config / gradle.properties also hold +# legitimate non-secret config. The safer pattern is to commit a `.npmrc.example` +# and keep the real file ignored. A per-repo override may re-allow a scrubbed +# root file via `!/.npmrc` if absolutely needed. + + +# --------------------------------------------------------------------------- +# 9. Cloud CLI session / token caches +# --------------------------------------------------------------------------- +# These rarely end up in repos, but when they do they grant live access — +# usually inside a Dockerfile COPY context that swept in a home dir. +.aws/sso/cache/ +.aws/cli/cache/ +.config/gcloud/ +.config/gcloud/application_default_credentials.json +.config/gcloud/legacy_credentials/ +.config/gcloud/access_tokens.db +.boto +.s3cfg +.rclone.conf +rclone.conf +.oci/ +.ibmcloud/ +.snowflake/ +.snowsql/config +.databricks/ +.databrickscfg + + +# --------------------------------------------------------------------------- +# 10. IDE files known to cache credentials +# --------------------------------------------------------------------------- +# JetBrains: workspace.xml can cache DB passwords; dataSources.local.xml +# holds per-user DB state. dataSources.xml (no .local) is intended for +# sharing and should not hold passwords — but review before committing. +.idea/workspace.xml +.idea/tasks.xml +.idea/usage.statistics.xml +.idea/shelf/ +.idea/dataSources.local.xml +.idea/dataSources/ +.idea/sqlDataSources.xml +.idea/dynamic.xml +.idea/**/aws.xml +# VS Code: sftp.json from the popular SFTP extension embeds passwords. +# Block local-variant settings; the shared settings.json stays committable. +.vscode/sftp.json +.vscode/ftp-sync.json +.vscode/settings.local.json +.vscode-server/ +# Eclipse secure storage +.metadata/.plugins/org.eclipse.core.runtime/.settings/org.eclipse.equinox.security* +# Cursor / Windsurf / Zed AI assistant caches (2025+) +.cursor/mcp.json +.cursor-server/ +.windsurf/ +.zed/settings.json +# NOTE: do NOT blanket-ignore .idea/ or .vscode/ here — that's a per-repo +# editor-cruft decision, not a secrets decision. + + +# --------------------------------------------------------------------------- +# 11. Generic "secret" / "credential" / "private" filename conventions +# --------------------------------------------------------------------------- +secrets.json +secrets.toml +secrets.env +secret.yaml +secret.yml +secret.json +credentials +credentials.yaml +credentials.yml +credentials.json +credentials.toml +private.json +private.yaml +private.yml +*.secret +*.secret.* +*.secrets +*.private +*.private.* +# Re-allow obviously-public template variants +!*.secret.example +!*.secrets.example +!secrets.example.* +!credentials.example.* +# Re-allow SOPS/age-encrypted variants (safe to commit) +!*.enc.yaml +!*.enc.yml +!*.enc.json +!*.sops.yaml +!*.sops.yml +!*.sops.json + + +# --------------------------------------------------------------------------- +# 12. Modern (2024-2026) credential-leak hotspots +# --------------------------------------------------------------------------- +# LLM / AI tooling config files where users paste API keys +.anthropic/ +.openai/ +.ollama/id_ed25519 +.continue/config.json +.aider.conf.yml +.aider.model.settings.yml +# GitHub Copilot local state +.github-copilot/ +# Supabase / PlanetScale / Neon / Turso CLI auth +.supabase/ +.pscale/ +.neon/ +.turso/ +# Railway / Render / Fly machine tokens +.railway/ +.render/ +# Stripe CLI, Twilio CLI, SendGrid +.stripe/ +.twilio-cli/ +# Temporal, Dagger, Earthly auth +.temporalio/ +.dagger/ +.earthly/config.yml + +# --------------------------------------------------------------------------- +# 13. Agent / local worktrees (org coding policy) +# --------------------------------------------------------------------------- +# Temporary worktrees created by Claude Code and other agents. Not strictly +# secret material, but the petry-projects coding guidelines require these +# paths to be ignored in every repo so an agent's scratch worktree cannot +# be committed accidentally. +.claude/worktrees/ +.worktrees/ + +# ============================================================================ +# End of petry-projects secrets baseline +# ============================================================================ diff --git a/AGENTS.md b/AGENTS.md index 59a29cd69..ea6a1efe1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,6 +20,7 @@ Read the relevant standard *before* making changes that touch CI, repo settings, | **Agent configuration** | [`standards/agent-standards.md`](https://github.com/petry-projects/.github/blob/main/standards/agent-standards.md) | CLAUDE.md / AGENTS.md / SKILL.md required structure, frontmatter rules, cross-references | | **Repo settings + labels** | [`standards/github-settings.md`](https://github.com/petry-projects/.github/blob/main/standards/github-settings.md) | Required settings, label set with exact colors, code-quality ruleset, branch protection | | **Dependabot config** | [`standards/dependabot-policy.md`](https://github.com/petry-projects/.github/blob/main/standards/dependabot-policy.md) and [`standards/dependabot/`](https://github.com/petry-projects/.github/tree/main/standards/dependabot) | Per-ecosystem dependabot.yml templates and policy | +| **Push protection** | [`standards/push-protection.md`](https://github.com/petry-projects/.github/blob/main/standards/push-protection.md) | Secret scanning + push protection, local gitleaks hooks, CI secret-scan job, incident response | **When fixing a compliance finding, the rule is: read the standard, then copy the template — do not generate from scratch.** Anything generated from scratch diff --git a/scripts/apply-repo-settings.sh b/scripts/apply-repo-settings.sh index 06449f708..746114398 100644 --- a/scripts/apply-repo-settings.sh +++ b/scripts/apply-repo-settings.sh @@ -35,6 +35,12 @@ ok() { echo "[OK] $*"; } err() { echo "[ERROR] $*" >&2; } skip() { echo "[SKIP] $*"; } +# Source the shared push-protection library — provides +# pp_apply_security_and_analysis() and the PP_REQUIRED_SA_SETTINGS list. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/push-protection.sh +. "$SCRIPT_DIR/lib/push-protection.sh" + usage() { echo "Usage: $0 " echo " $0 --all" @@ -187,6 +193,7 @@ if [ "$1" = "--all" ]; then for repo in $repos; do apply_settings "$repo" || failed=$((failed + 1)) apply_labels "$repo" + pp_apply_security_and_analysis "$repo" || failed=$((failed + 1)) done if [ "$failed" -gt 0 ]; then @@ -198,4 +205,5 @@ if [ "$1" = "--all" ]; then else apply_settings "$1" apply_labels "$1" + pp_apply_security_and_analysis "$1" fi diff --git a/scripts/compliance-audit.sh b/scripts/compliance-audit.sh index 82466e524..bba323770 100755 --- a/scripts/compliance-audit.sh +++ b/scripts/compliance-audit.sh @@ -98,6 +98,13 @@ gh_api() { return 1 } +# Source the shared push-protection library — provides pp_run_all_checks() +# and the PP_REQUIRED_SA_SETTINGS list. Sourced AFTER gh_api() and +# add_finding() are defined, since the lib's check functions call them. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/push-protection.sh +. "$SCRIPT_DIR/lib/push-protection.sh" + # --------------------------------------------------------------------------- # Ecosystem detection # --------------------------------------------------------------------------- @@ -1028,6 +1035,7 @@ main() { check_centralized_workflow_stubs "$repo" check_claude_md "$repo" check_agents_md "$repo" + pp_run_all_checks "$repo" log_end done diff --git a/scripts/lib/push-protection.sh b/scripts/lib/push-protection.sh new file mode 100644 index 000000000..4f340d599 --- /dev/null +++ b/scripts/lib/push-protection.sh @@ -0,0 +1,309 @@ +# shellcheck shell=bash +# scripts/lib/push-protection.sh — Shared push-protection logic +# +# Sourceable Bash library that implements the apply + check functions for +# the petry-projects Push Protection Standard: +# +# standards/push-protection.md +# +# Both scripts/apply-repo-settings.sh and scripts/compliance-audit.sh source +# this file so the org-required `security_and_analysis` settings, audit +# checks, and remediation logic live in exactly one place. Any future change +# to the standard's required state happens here, not in two parallel copies. +# +# ---------------------------------------------------------------------------- +# Caller contract +# ---------------------------------------------------------------------------- +# This library is `set -euo pipefail`-safe and designed to be sourced by a +# parent script. It does NOT call `set` itself. +# +# Required by ALL functions: +# - $ORG — GitHub org slug (e.g. "petry-projects") +# +# Required by `pp_apply_*` functions (used by apply-repo-settings.sh): +# - info(), ok(), err(), skip() — log helpers (string arg) +# - $DRY_RUN — "true" / "false" +# - `gh` CLI on PATH, GH_TOKEN with admin:repo scope +# +# Required by `pp_check_*` functions (used by compliance-audit.sh): +# - gh_api() — wrapper around `gh api` with retry +# - add_finding() — add_finding [] +# - `gh` CLI on PATH, GH_TOKEN with read:org + repo scope +# +# Functions are namespaced with the `pp_` prefix to avoid colliding with +# caller helpers. + +# --------------------------------------------------------------------------- +# Required state — single source of truth +# --------------------------------------------------------------------------- +# Each entry: "api_key:expected_value:severity:human_detail" +# Severity is one of: error, warning. The audit and the apply script both +# read from this list, so adding a new flag here automatically extends both. +PP_REQUIRED_SA_SETTINGS=( + "secret_scanning:enabled:error:Secret scanning must be enabled" + "secret_scanning_push_protection:enabled:error:Secret scanning push protection must be enabled" + "secret_scanning_ai_detection:enabled:warning:Secret scanning AI detection should be enabled" + "secret_scanning_non_provider_patterns:enabled:warning:Secret scanning non-provider patterns should be enabled" + "dependabot_security_updates:enabled:warning:Dependabot security updates should be enabled" +) + +# Minimum entries that every repo's .gitignore MUST contain. Every repo +# starting from the org baseline at /.gitignore satisfies these by default. +PP_REQUIRED_GITIGNORE_PATTERNS=( + ".env" + "*.pem" + "*.key" +) + +# Number of days back the bypass-recency check looks for unjustified +# push-protection bypasses. +PP_BYPASS_LOOKBACK_DAYS="${PP_BYPASS_LOOKBACK_DAYS:-30}" + +# Standard reference path used by every check finding. +PP_STANDARD_REF="standards/push-protection.md" + +# --------------------------------------------------------------------------- +# Apply: security_and_analysis (used by apply-repo-settings.sh) +# --------------------------------------------------------------------------- +# Idempotent: fetches current state, only PATCHes when at least one flag +# differs from the required value. Honors $DRY_RUN. +pp_apply_security_and_analysis() { + local repo="$1" + info "Applying push-protection security_and_analysis to $ORG/$repo ..." + + local current + current=$(gh api "repos/$ORG/$repo" --jq '.security_and_analysis // {}' 2>/dev/null || echo "{}") + + if [ "$current" = "{}" ] || [ -z "$current" ]; then + err "Could not fetch security_and_analysis for $ORG/$repo — check token has admin scope" + return 1 + fi + + local needs_patch=false + local payload="{}" + + local entry key expected actual + for entry in "${PP_REQUIRED_SA_SETTINGS[@]}"; do + IFS=':' read -r key expected _ _ <<< "$entry" + actual=$(echo "$current" | jq -r ".\"$key\".status // \"null\"") + + if [ "$actual" != "$expected" ]; then + info " $key: $actual → $expected" + needs_patch=true + payload=$(echo "$payload" | jq --arg k "$key" --arg v "$expected" '. + {($k): {status: $v}}') + else + ok " $key: already $actual" + fi + done + + if [ "$needs_patch" = false ]; then + ok "$ORG/$repo security_and_analysis already fully compliant — no changes needed" + return 0 + fi + + if [ "$DRY_RUN" = "true" ]; then + skip "DRY_RUN=true — skipping security_and_analysis PATCH for $ORG/$repo" + return 0 + fi + + # Wrap the per-key payload in a top-level security_and_analysis object and + # PATCH it via stdin (gh api -F doesn't accept nested JSON). + local full_payload + full_payload=$(echo "$payload" | jq '{security_and_analysis: .}') + + if echo "$full_payload" | gh api -X PATCH "repos/$ORG/$repo" --input - > /dev/null 2>&1; then + ok "$ORG/$repo security_and_analysis updated successfully" + else + err "Failed to PATCH security_and_analysis for $ORG/$repo — check admin scope and that the org plan supports these features" + return 1 + fi +} + +# --------------------------------------------------------------------------- +# Check: security_and_analysis (used by compliance-audit.sh) +# --------------------------------------------------------------------------- +# Reads the same PP_REQUIRED_SA_SETTINGS list the apply function uses, so +# the audit can never drift from what apply enforces. +pp_check_security_and_analysis() { + local repo="$1" + + local sa + sa=$(gh_api "repos/$ORG/$repo" --jq '.security_and_analysis // {}' 2>/dev/null || echo "{}") + + if [ "$sa" = "{}" ] || [ -z "$sa" ]; then + add_finding "$repo" "push-protection" "security_and_analysis_unavailable" "warning" \ + "Could not fetch security_and_analysis — token may lack admin scope, or the repo's plan does not expose these settings" \ + "$PP_STANDARD_REF#required-repo-level-settings" + return + fi + + local entry key expected severity detail actual + for entry in "${PP_REQUIRED_SA_SETTINGS[@]}"; do + IFS=':' read -r key expected severity detail <<< "$entry" + actual=$(echo "$sa" | jq -r ".\"$key\".status // \"null\"") + if [ "$actual" != "$expected" ]; then + add_finding "$repo" "push-protection" "$key" "$severity" \ + "$detail (current: \`$actual\`, expected: \`$expected\`)" \ + "$PP_STANDARD_REF#required-repo-level-settings" + fi + done +} + +# --------------------------------------------------------------------------- +# Check: open secret-scanning alerts +# --------------------------------------------------------------------------- +# Any open alert is a real leaked credential that needs rotation. This is an +# `error` finding — open alerts MUST be triaged within the SLA in the +# incident-response section of the standard. +pp_check_open_secret_alerts() { + local repo="$1" + + local alerts + alerts=$(gh_api "repos/$ORG/$repo/secret-scanning/alerts?state=open&per_page=100" 2>/dev/null || echo "[]") + + # 404 / disabled secret scanning produces a non-array response; coerce to [] + if ! echo "$alerts" | jq -e 'type == "array"' >/dev/null 2>&1; then + return + fi + + local count + count=$(echo "$alerts" | jq 'length') + + if [ "$count" -gt 0 ]; then + add_finding "$repo" "push-protection" "open_secret_alerts" "error" \ + "$count open secret-scanning alert(s) — rotate the leaked credentials before resolving" \ + "$PP_STANDARD_REF#incident-response" + fi +} + +# --------------------------------------------------------------------------- +# Check: ci.yml contains a secret-scan job using gitleaks +# --------------------------------------------------------------------------- +pp_check_secret_scan_ci_job() { + local repo="$1" + + local ci_b64 + ci_b64=$(gh_api "repos/$ORG/$repo/contents/.github/workflows/ci.yml" --jq '.content // ""' 2>/dev/null || echo "") + + if [ -z "$ci_b64" ]; then + add_finding "$repo" "push-protection" "secret_scan_ci_job_present" "error" \ + "No \`.github/workflows/ci.yml\` found — cannot verify the required \`secret-scan\` gitleaks job" \ + "$PP_STANDARD_REF#layer-3--ci-secret-scanning-secondary-defense" + return + fi + + local ci_content + # GitHub returns content base64-encoded, line-wrapped at 60 chars + ci_content=$(echo "$ci_b64" | tr -d '\n ' | base64 -d 2>/dev/null || echo "") + + if [ -z "$ci_content" ]; then + return + fi + + if ! echo "$ci_content" | grep -qiE 'gitleaks'; then + add_finding "$repo" "push-protection" "secret_scan_ci_job_present" "error" \ + "\`ci.yml\` does not contain a job using \`gitleaks\` — add the secret-scan job from the standard" \ + "$PP_STANDARD_REF#required-ci-job" + fi +} + +# --------------------------------------------------------------------------- +# Check: .gitignore contains the baseline secret-protection entries +# --------------------------------------------------------------------------- +pp_check_gitignore_secrets_block() { + local repo="$1" + + local gi_b64 + gi_b64=$(gh_api "repos/$ORG/$repo/contents/.gitignore" --jq '.content // ""' 2>/dev/null || echo "") + + if [ -z "$gi_b64" ]; then + add_finding "$repo" "push-protection" "gitignore_secrets_block" "warning" \ + "No \`.gitignore\` at repo root — start from the org baseline at /.gitignore" \ + "$PP_STANDARD_REF#required-gitignore-entries" + return + fi + + local gi_content + gi_content=$(echo "$gi_b64" | tr -d '\n ' | base64 -d 2>/dev/null || echo "") + + if [ -z "$gi_content" ]; then + return + fi + + local missing=() + local pattern + for pattern in "${PP_REQUIRED_GITIGNORE_PATTERNS[@]}"; do + # Use fixed-string match anchored to a line; ignore lines that start with `!` + # so a negation can't satisfy the requirement for the broad pattern. + if ! echo "$gi_content" | grep -vE '^[[:space:]]*!' | grep -qxF "$pattern"; then + missing+=("$pattern") + fi + done + + if [ ${#missing[@]} -gt 0 ]; then + add_finding "$repo" "push-protection" "gitignore_secrets_block" "warning" \ + "\`.gitignore\` is missing baseline secret patterns: $(IFS=', '; echo "${missing[*]}") — copy the org baseline at /.gitignore" \ + "$PP_STANDARD_REF#required-gitignore-entries" + fi +} + +# --------------------------------------------------------------------------- +# Check: recent push-protection bypasses +# --------------------------------------------------------------------------- +# Queries the secret-scanning alerts endpoint and looks for any alert that +# was bypassed via push protection within the lookback window. We can't tell +# from the alert payload whether a bypass had a documented justification, so +# this fires as a `warning` and the human reviewer must verify. +pp_check_push_protection_bypasses() { + local repo="$1" + + local alerts + alerts=$(gh_api "repos/$ORG/$repo/secret-scanning/alerts?per_page=100" 2>/dev/null || echo "[]") + + if ! echo "$alerts" | jq -e 'type == "array"' >/dev/null 2>&1; then + return + fi + + # Cutoff: now - PP_BYPASS_LOOKBACK_DAYS, in ISO-8601 + local cutoff + if date -u -d "$PP_BYPASS_LOOKBACK_DAYS days ago" "+%Y-%m-%dT%H:%M:%SZ" >/dev/null 2>&1; then + # GNU date (Linux / GitHub Actions runners) + cutoff=$(date -u -d "$PP_BYPASS_LOOKBACK_DAYS days ago" "+%Y-%m-%dT%H:%M:%SZ") + else + # BSD date (macOS) fallback + cutoff=$(date -u -v-"${PP_BYPASS_LOOKBACK_DAYS}d" "+%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo "") + fi + + if [ -z "$cutoff" ]; then + return + fi + + local recent_bypasses + recent_bypasses=$(echo "$alerts" | jq --arg cutoff "$cutoff" ' + [.[] | select( + .push_protection_bypassed == true and + (.push_protection_bypassed_at != null) and + (.push_protection_bypassed_at >= $cutoff) + )] | length + ' 2>/dev/null || echo "0") + + if [ "$recent_bypasses" -gt 0 ]; then + add_finding "$repo" "push-protection" "push_protection_bypasses_recent" "warning" \ + "$recent_bypasses push-protection bypass(es) in the last $PP_BYPASS_LOOKBACK_DAYS days — verify each had a documented justification" \ + "$PP_STANDARD_REF#bypass-policy" + fi +} + +# --------------------------------------------------------------------------- +# Convenience: run every push-protection check for one repo +# --------------------------------------------------------------------------- +# Callers may call individual checks if they want to interleave with other +# work, but the audit's per-repo loop just calls this single entry point. +pp_run_all_checks() { + local repo="$1" + pp_check_security_and_analysis "$repo" + pp_check_open_secret_alerts "$repo" + pp_check_secret_scan_ci_job "$repo" + pp_check_gitignore_secrets_block "$repo" + pp_check_push_protection_bypasses "$repo" +} diff --git a/standards/github-settings.md b/standards/github-settings.md index 8b42b3549..f7d058685 100644 --- a/standards/github-settings.md +++ b/standards/github-settings.md @@ -33,6 +33,22 @@ SHOULD be audited and brought into compliance. | **Has Wiki** | `false` | Disabled — documentation lives in the repo | | **Has Discussions** | `true` | **Required** — enables Discussions for ideation, feedback, and community engagement (see [Discussions Configuration](#discussions-configuration)) | +### Security & Analysis + +All repositories MUST have the following security features enabled. These are +the enforcement primitives behind the [Push Protection Standard](push-protection.md). + +| Setting | Standard Value | Rationale | +|---------|---------------|-----------| +| **Secret scanning** | `enabled` | Detect leaked credentials in history and new commits | +| **Secret scanning push protection** | `enabled` | Block pushes containing known secret patterns at the server side | +| **Secret scanning AI detection** | `enabled` | Catch generic secrets missed by regex patterns | +| **Secret scanning non-provider patterns** | `enabled` | Private keys, HTTP basic auth, high-entropy strings | +| **Dependabot security updates** | `enabled` | Automated patches for known-vulnerable dependencies | + +> See the full requirements, custom patterns, CI job, incident response flow, +> and compliance audit checks in [`push-protection.md`](push-protection.md). + ### Merge Settings | Setting | Standard Value | Rationale | diff --git a/standards/push-protection.md b/standards/push-protection.md new file mode 100644 index 000000000..4444f7290 --- /dev/null +++ b/standards/push-protection.md @@ -0,0 +1,431 @@ +# Push Protection Standard + +Standard for preventing secrets, API keys, credentials, and other sensitive +values from being accidentally committed or pushed to any repository in the +**petry-projects** organization. + +This standard is **defense in depth**: local hooks catch most leaks before a +commit lands, GitHub push protection blocks the push at the network boundary, +and CI scanning + secret scanning alerts catch anything that slips past the +first two layers. Any one layer failing is a warning; two layers failing is an +incident. + +--- + +## Scope + +This standard applies to **every repository** in `petry-projects`, regardless +of visibility (public or private) or language. It covers: + +- Source code, configuration files, and fixtures +- Workflow files (`.github/workflows/*.yml`) and reusable workflows +- Agent configuration (`CLAUDE.md`, `AGENTS.md`, `SKILL.md`, MCP configs) +- Documentation, issue/PR templates, and discussion posts +- Binary artifacts, screenshots, log files, and notebooks checked into git + +> **Private repos are in scope.** GitHub secret scanning is now free for +> private repos on the free plan, and a leaked credential in a private repo is +> still a credential that must be rotated. + +--- + +## What Counts as a Secret + +The following values MUST NEVER be committed to any repo, even temporarily, and +even in a branch that will be rebased or force-pushed: + +| Category | Examples | +|----------|----------| +| **Cloud provider credentials** | AWS access key / secret, GCP service account JSON, Azure connection string | +| **API tokens** | GitHub PAT / fine-grained token, SonarCloud token, Anthropic API key, OpenAI API key, Slack webhook / bot token, Stripe key | +| **Database credentials** | Postgres / MySQL / Mongo connection strings containing passwords, Redis AUTH strings | +| **Private keys** | SSH private keys, TLS private keys, GPG private keys, GitHub App private keys | +| **OAuth secrets** | Client secrets, refresh tokens, long-lived access tokens | +| **Signing keys** | Code signing certificates, Sigstore identities, npm publish tokens | +| **Internal URLs & identifiers** | Unpublished webhook URLs, internal hostnames, customer IDs, account IDs when not already public | +| **Personal data** | Real email addresses, phone numbers, or names in fixtures that are not explicitly test data | + +Low-entropy placeholder values (`sk-xxxx`, `AKIA...EXAMPLE`, `changeme`) are +permitted in documentation and tests **only when** they are obviously not real +(e.g., repeated characters, `EXAMPLE` suffix, or documented as placeholder). +When in doubt, use the GitHub-documented [dummy values](https://docs.github.com/en/code-security/secret-scanning/introduction/supported-secret-scanning-patterns). + +--- + +## Layer 1 — GitHub Push Protection (Primary Enforcement) + +GitHub's native [secret scanning push protection](https://docs.github.com/en/code-security/secret-scanning/push-protection-for-repositories-and-organizations) +is the **primary enforcement mechanism**. It blocks pushes that contain +detected secrets at the server side, before the commit is accepted, so nothing +ever lands in the repo history in the first place. + +### Required org-level settings + +Configured once at the organization level and inherited by all repos: + +| Setting | Value | Notes | +|---------|-------|-------| +| **Secret scanning** | **Enabled for all repos** | Public and private | +| **Push protection** | **Enabled for all repos** | Blocks pushes containing known secret patterns | +| **Push protection for contributors** | **Enabled** | Applies to forks and contributions | +| **Validity checks** | **Enabled** | Verifies leaked tokens against the provider so rotation can be prioritized | +| **Non-provider patterns** | **Enabled** | Adds generic patterns (private keys, HTTP basic auth, high-entropy strings) | +| **Custom patterns** | **Enabled (see below)** | Org-specific patterns live under Settings → Code security → Secret scanning | +| **Bypass privileges** | **Admin-only, with justification required** | Bypasses MUST include a reason and are audited | + +Apply these via: + +```bash +# Org-level (requires org admin) +gh api -X PATCH "orgs/petry-projects" \ + -f secret_scanning_enabled_for_new_repositories=true \ + -f secret_scanning_push_protection_enabled_for_new_repositories=true \ + -f secret_scanning_push_protection_custom_link_enabled=true \ + -f secret_scanning_push_protection_custom_link="https://github.com/petry-projects/.github/blob/main/standards/push-protection.md#what-to-do-when-push-protection-blocks-your-push" +``` + +### Required repo-level settings + +Every repository MUST have the following security features turned on. The +compliance audit checks these flags via `GET /repos/{owner}/{repo}`: + +| Setting | Path in API response | Required value | +|---------|----------------------|----------------| +| Secret scanning | `security_and_analysis.secret_scanning.status` | `enabled` | +| Secret scanning push protection | `security_and_analysis.secret_scanning_push_protection.status` | `enabled` | +| Secret scanning AI detection | `security_and_analysis.secret_scanning_ai_detection.status` | `enabled` | +| Secret scanning non-provider patterns | `security_and_analysis.secret_scanning_non_provider_patterns.status` | `enabled` | +| Dependabot security updates | `security_and_analysis.dependabot_security_updates.status` | `enabled` | + +Apply per repo via: + +```bash +gh api -X PATCH "repos/petry-projects/" \ + -F security_and_analysis='{ + "secret_scanning": {"status": "enabled"}, + "secret_scanning_push_protection": {"status": "enabled"}, + "secret_scanning_ai_detection": {"status": "enabled"}, + "secret_scanning_non_provider_patterns": {"status": "enabled"}, + "dependabot_security_updates": {"status": "enabled"} + }' +``` + +`scripts/apply-repo-settings.sh` MUST enforce these values alongside the +existing merge and label settings — it sources +[`scripts/lib/push-protection.sh`](../scripts/lib/push-protection.sh) and +calls `pp_apply_security_and_analysis` after the standard settings and +labels have been applied. The required `security_and_analysis` keys live +in the `PP_REQUIRED_SA_SETTINGS` array in that library — adding a new +required flag there automatically extends both the apply script and the +weekly audit. See [Application](#application-to-a-repository) below. + +### Custom secret scanning patterns + +The org MUST configure the following custom patterns in addition to the +provider-supplied ones: + +| Pattern name | Regex (illustrative) | Rationale | +|--------------|----------------------|-----------| +| `petry-internal-webhook` | `https://hooks\.petry-projects\.internal/[A-Za-z0-9/_-]{20,}` | Internal webhook URLs | +| `claude-oauth-token` | `sk-ant-oat01-[A-Za-z0-9_-]{40,}` | Anthropic OAuth tokens | +| `gha-pat-scoped` | `github_pat_[A-Za-z0-9_]{82}` | Fine-grained GitHub PATs (provider pattern supplements) | +| `generic-high-entropy` | High-entropy strings assigned to `*_TOKEN`, `*_SECRET`, `*_KEY` env vars | Catches untyped long strings in YAML and `.env` files | + +Custom patterns are configured at **Org settings → Code security → Secret +scanning → Custom patterns**. Each new pattern MUST be dry-run against all +repos before being enabled to estimate false-positive rate. + +--- + +## Layer 2 — Local Pre-Commit Prevention + +Local prevention catches leaks before they ever reach GitHub, which is both +faster for the developer and leaves no evidence in any remote history. Every +developer workstation and every Claude Code / agent environment SHOULD run +`gitleaks` (or an equivalent) as a pre-commit hook. + +### Recommended local tooling + +| Tool | Purpose | How it runs | +|------|---------|-------------| +| [`gitleaks`](https://github.com/gitleaks/gitleaks) | Fast, regex + entropy secret scanner | Pre-commit hook + CI | +| [`pre-commit`](https://pre-commit.com/) | Hook orchestrator | Manages the hook lifecycle | +| [`git-secrets`](https://github.com/awslabs/git-secrets) | AWS-focused secret scanner | Optional supplement | + +### Standard `.pre-commit-config.yaml` entry + +Repositories that adopt pre-commit SHOULD add this block: + +```yaml +repos: + - repo: https://github.com/gitleaks/gitleaks + rev: v8.24.0 + hooks: + - id: gitleaks + name: gitleaks (secret scan) + description: Detect hardcoded secrets before commit +``` + +Install locally with: + +```bash +pip install pre-commit +pre-commit install +pre-commit run gitleaks --all-files # one-off scan of tracked files in the working tree +# For a full git-history scan, use gitleaks directly: +gitleaks git --redact --exit-code 1 +``` + +> **Note:** `pre-commit run --all-files` only scans the current working tree, +> not the full git history. Use `gitleaks git` (or `gitleaks detect +> --log-opts="--all"` on older versions) to scan every commit reachable from +> every branch. + +### Agent workstation requirements + +Claude Code and other AI agents operating on petry-projects repos MUST: + +- Refuse to write real credentials to any file, even when asked. Use + placeholder values (``) in documentation and instruct the + user to source the value from an environment variable or secrets manager. +- Refuse to commit files containing strings that look like secrets, even when + explicitly instructed. Ask the user to confirm and route the value through a + secure store instead. +- When generating `.env.example` files, include key names only — never values. + +--- + +## Layer 3 — CI Secret Scanning (Secondary Defense) + +CI scanning is the last line of defense for code that has already made it into +a branch (e.g., historical commits, imported repositories, or pushes from +accounts with bypass privileges). + +### Required CI job + +Every repository's primary `ci.yml` workflow MUST include a `secret-scan` job +that runs `gitleaks` in full-history mode on every pull request and on every +push to `main`. + +```yaml +secret-scan: + name: Secret scan (gitleaks) + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + steps: + - name: Checkout (full history) + # Pin to SHA per Action Pinning Policy (ci-standards.md#action-pinning-policy). + # Look up current SHA: gh api repos/actions/checkout/git/refs/tags/v4 --jq '.object.sha' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + - name: Run gitleaks + # Pinned to SHA per Action Pinning Policy (ci-standards.md#action-pinning-policy). + # Refresh with: gh api repos/gitleaks/gitleaks-action/git/refs/tags/v2 --jq '.object.sha' + # then dereference if it points at an annotated tag. + uses: gitleaks/gitleaks-action@ff98106e4c7b2bc287b24eaf42907196329070c7 # v2.3.9 + with: + args: detect --source . --redact --verbose --exit-code 1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +``` + +The job MUST: + +- Use `fetch-depth: 0` so the full git history is scanned, not just the + PR diff +- Pass `--redact` so leaked values are NEVER written to workflow logs +- Fail the build (`--exit-code 1`) when any finding is detected +- Run as a **required check** via the `code-quality` ruleset + (see [`github-settings.md`](github-settings.md#code-quality--required-checks-ruleset-all-repositories)) + +### Coordination with AgentShield + +For agent-configuration files specifically, [`agent-shield.yml`](workflows/agent-shield.yml) +already runs 10 dedicated secret-detection rules across 14 patterns (see +[`agent-standards.md`](agent-standards.md#layer-1-agentshield-action-deep-security-scan)). +The `secret-scan` CI job is complementary — it covers non-agent files and +runs a broader entropy-based scan over the full history. Both MUST be green +for a PR to merge. + +--- + +## Developer Practices + +### Handling real secrets + +- Real credentials live in the [GitHub Actions secret store](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions), + in the developer's local `.env` files (gitignored), or in a dedicated secrets + manager. They never live in source files, documentation, or chat history. +- The standard org-level secrets (`APP_ID`, `APP_PRIVATE_KEY`, + `CLAUDE_CODE_OAUTH_TOKEN`, `SONAR_TOKEN`) are documented in + [`github-settings.md`](github-settings.md#organization-level-secrets-for-standard-ci). + Reference them via `${{ secrets. }}` in workflows. +- When a workflow needs a new secret, add it to the org-level store (if it is + reusable) or the repo-level store (if it is project-specific). Document the + purpose in the repo's `README.md` or `CONTRIBUTING.md`. + +### Required gitignore entries + +Every repository MUST start its `.gitignore` from the org baseline at +[`/.gitignore`](../.gitignore) in this repo. The baseline is **secrets-only** +and language-agnostic — it covers the dotenv family, cloud-provider credential +files, Kubernetes / Helm secrets, SSH/TLS/GPG key material, Terraform/IaC +state and `*.tfvars`, secret-manager local caches (sops, age, vault, doppler, +1password, infisical), database dumps and DB client dotfiles, package-registry +credential dotfiles (`.npmrc`, `.pypirc`, `.cargo/credentials`, etc.), cloud +CLI session caches, IDE files known to cache credentials (JetBrains +`workspace.xml`, VS Code `sftp.json`, Cursor `mcp.json`), and modern AI/LLM +tooling config files. + +> **Per-repo overrides are expected.** The baseline covers secrets only. +> Each repo MUST append its own language-specific entries (`node_modules/`, +> `target/`, `__pycache__/`, etc.) and OS cruft. Use the matching template +> from [github/gitignore](https://github.com/github/gitignore) and append +> below the baseline section. +> +> **Negation rules.** Several baseline patterns include `!` negations that +> re-allow legitimate files (e.g. `!.env.example`, `!*.crt`). Per-repo +> additions MUST NOT re-ignore those files. Always negate by **specific +> file path**, never by directory — a negation inside an ignored directory +> does not re-include the file. + +The minimum compliance audit check is that the file contains at least +`.env`, `*.pem`, `*.key`, and a `secrets`-style entry. Repos that copy the +org baseline verbatim satisfy this automatically. + +### Writing tests and fixtures + +- Use the dummy values listed in the [GitHub secret patterns documentation](https://docs.github.com/en/code-security/secret-scanning/introduction/supported-secret-scanning-patterns) + for test fixtures. These are whitelisted by GitHub push protection. +- Generate ephemeral test keys at runtime where possible (e.g., a freshly + created RSA keypair inside a `beforeAll` hook) rather than committing a + fixed test key. +- If a fixture MUST contain a realistic-looking value, prefix the filename + with `fixture-` and add a `.gitleaksignore` entry documenting the + justification. + +### Working in a branch that may contain a leaked secret + +If you suspect you have committed a secret locally but not yet pushed: + +1. **Stop.** Do not push. +2. Run `git log -p -- ` to find the commit(s) containing the secret. +3. If the secret is only in uncommitted working tree: remove it and + `git restore --staged `. +4. If the secret is in local commits not yet pushed: rewrite history with + `git rebase -i ` and amend the offending commit(s) to remove it. +5. Rotate the credential anyway — assume any value you typed into a terminal + has been logged somewhere. + +If you have already pushed: + +1. **Rotate the credential immediately** — assume it is compromised. +2. Follow the [Incident Response](#incident-response) procedure below. +3. Do NOT attempt to rewrite remote history as a first response — the value + is already in forks, caches, and CI logs. Rotation is faster and safer. + +--- + +## What to Do When Push Protection Blocks Your Push + +When `git push` fails with a push protection error, GitHub returns a URL +pointing to the blocked secret. The correct response is: + +1. **Do not bypass.** Bypassing push protection is an admin-only action, + requires a written justification, and is audited org-wide. +2. **Identify whether the value is a real secret or a false positive.** + - **Real secret:** remove it from the commit (see the rewrite procedure + above), rotate the credential, and force-push the rewritten branch. Open + an incident issue per the [Incident Response](#incident-response) + procedure. + - **False positive:** confirm with the org security owner, then add a + `.gitleaksignore` entry (for CI) and request a push protection bypass + with a `used_in_tests` or `false_positive` reason. +3. **Never** commit a modified version of the secret (e.g., adding a space, + splitting across lines, base64-encoding) to work around detection. This + is treated as the same severity as committing the original value. + +--- + +## Incident Response + +When a secret is confirmed leaked — whether caught by push protection bypass, +CI scanning, a secret scanning alert, or an external report — follow this +procedure: + +| Step | Action | Owner | Target | +|------|--------|-------|--------| +| 1 | **Rotate the credential** in the upstream provider (AWS, GitHub, Anthropic, etc.) | First responder | Immediately | +| 2 | **Revoke any derived tokens** (OAuth grants, downstream integrations) | First responder | Immediately | +| 3 | **Open a private security advisory** in the affected repo | First responder | Within 1 hour | +| 4 | **Audit access logs** for the credential to determine blast radius | Org admin | Within 24 hours | +| 5 | **Remove the secret from history** if appropriate (BFG, `git filter-repo`), recognizing that forks and caches may retain copies | Org admin | Within 24 hours, only after rotation | +| 6 | **Post-mortem** — document root cause, why existing layers did not catch it, and what changes prevent recurrence | Org admin | Within 7 days | +| 7 | **Update this standard** with any new patterns or lessons learned | Standards owner | Within 7 days | + +Rotation ALWAYS comes first. History rewriting is a cleanup step, not a +mitigation. + +--- + +## Application to a Repository + +When onboarding a repository to this standard: + +1. **Enable secret scanning + push protection** via the API call in + [Required repo-level settings](#required-repo-level-settings). `scripts/apply-repo-settings.sh` + enforces this on every run. +2. **Verify gitignore** contains the standard entries listed in + [Required gitignore entries](#required-gitignore-entries). +3. **Add the `secret-scan` job** to `ci.yml` per [Layer 3](#layer-3--ci-secret-scanning-secondary-defense). +4. **Add `secret-scan` as a required check** in the `code-quality` ruleset. + If provisioning is done via `scripts/apply-rulesets.sh`, first update + `detect_required_checks()` in that script to recognize and insert the + `secret-scan` check, then re-apply rulesets org-wide. Update + [`github-settings.md`](github-settings.md#code-quality--required-checks-ruleset-all-repositories) + if the ruleset template needs a new entry. +5. **Scan existing history** one time with `gitleaks git --redact --exit-code 1` + before enabling enforcement, to surface any pre-existing secrets across all + commits (not just the current working tree). +6. **Rotate anything found** during the initial scan — do not whitelist + existing findings without rotation. + +--- + +## Compliance Audit Checks + +The weekly compliance audit +([`scripts/compliance-audit.sh`](../scripts/compliance-audit.sh)) verifies the +following for every repository. The check logic lives in +[`scripts/lib/push-protection.sh`](../scripts/lib/push-protection.sh) and is +sourced by both the audit and the apply script so a single edit to +`PP_REQUIRED_SA_SETTINGS` (or the gitignore baseline list) propagates to +both at once: + +| Check | Severity | Detail | +|-------|----------|--------| +| `secret_scanning_enabled` | error | `security_and_analysis.secret_scanning.status == "enabled"` | +| `push_protection_enabled` | error | `security_and_analysis.secret_scanning_push_protection.status == "enabled"` | +| `ai_detection_enabled` | warning | `security_and_analysis.secret_scanning_ai_detection.status == "enabled"` | +| `non_provider_patterns_enabled` | warning | `security_and_analysis.secret_scanning_non_provider_patterns.status == "enabled"` | +| `dependabot_security_updates_enabled` | warning | `security_and_analysis.dependabot_security_updates.status == "enabled"` | +| `open_secret_alerts` | error | `GET /repos/{owner}/{repo}/secret-scanning/alerts?state=open` returns an empty array | +| `secret_scan_ci_job_present` | error | `.github/workflows/ci.yml` contains a job using `gitleaks` | +| `gitignore_secrets_block` | warning | `.gitignore` contains `.env`, `*.pem`, `*.key` entries | +| `push_protection_bypasses_recent` | warning | No bypasses in the last 30 days without a documented justification | + +Findings are reported as GitHub Issues labeled `security` + `compliance-audit` +per the existing audit flow. + +--- + +## Related Standards + +- [`github-settings.md`](github-settings.md) — repo settings, rulesets, org secrets +- [`agent-standards.md`](agent-standards.md) — AgentShield scanner and agent-config hygiene +- [`ci-standards.md`](ci-standards.md) — workflow templates and required checks +- [`dependabot-policy.md`](dependabot-policy.md) — dependency vulnerability updates