diff --git a/.gitignore b/.gitignore index ab9a610dc1..4b67afc78e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,9 @@ *.pub *.key *.decrypted~*.yaml -/age.key +# `*.key` misses suffixed backups like age.key.bak-x25519-only, which decrypt the +# tracked sops files in this PUBLIC repo. Supersedes the old `/age.key`. +age.key* /cloudflare-tunnel.json bootstrap/github-deploy.key bootstrap/github-deploy.key.pub diff --git a/kubernetes/apps/ai/hermes/app/kustomization.yaml b/kubernetes/apps/ai/hermes/app/kustomization.yaml index 9aac8de8f5..b06016f0be 100644 --- a/kubernetes/apps/ai/hermes/app/kustomization.yaml +++ b/kubernetes/apps/ai/hermes/app/kustomization.yaml @@ -5,4 +5,5 @@ kind: Kustomization resources: - helmrelease.yaml + - prometheusrule.yaml - secret.sops.yaml diff --git a/kubernetes/apps/ai/hermes/app/prometheusrule.yaml b/kubernetes/apps/ai/hermes/app/prometheusrule.yaml new file mode 100644 index 0000000000..83f2ba0be2 --- /dev/null +++ b/kubernetes/apps/ai/hermes/app/prometheusrule.yaml @@ -0,0 +1,79 @@ +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/monitoring.coreos.com/prometheusrule_v1.json +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: hermes-rules +spec: + groups: + - name: hermes.rules + # scripts/cron_health_export.py (on the hermes PVC, not in git) pushes + # cron/jobs.json state every 10m. Pushed, not scraped, so VM's 5m default + # staleness makes a bare instant selector return nothing between pushes: + # absent() would flap and the failure rules would go blind exactly half the + # time. last_over_time() pins every rule to the newest real sample instead of + # the staleness window. + rules: + # The scheduler already records per-job status in cron/jobs.json; nothing read + # it, so a dead cron stayed dark 40h. + - alert: HermesCronJobFailed + expr: |- + last_over_time(hermes_cron_last_status_ok[30m]) == 0 + for: 15m + annotations: + summary: >- + Hermes cron {{ $labels.name }} last run failed — it will keep failing on + schedule until the cause is fixed + labels: + severity: warning + + # Delivery is a separate failure from execution: the job can succeed and the + # Discord post still vanish, which reads green everywhere else. + - alert: HermesCronDeliveryFailed + expr: |- + last_over_time(hermes_cron_delivery_failed[30m]) == 1 + for: 15m + annotations: + summary: >- + Hermes cron {{ $labels.name }} ran but could not deliver its output — + the result is lost, not late + labels: + severity: warning + + # Already excludes disabled and paused jobs at export time. One hour of slack + # absorbs a long-running predecessor without flapping. + - alert: HermesCronOverdue + expr: |- + last_over_time(hermes_cron_overdue_seconds[30m]) > 3600 + for: 15m + annotations: + summary: >- + Hermes cron {{ $labels.name }} is {{ $value | humanizeDuration }} past its + scheduled run — the scheduler is not firing it + labels: + severity: warning + + # The exporter runs inside hermes, so a stale heartbeat means hermes itself is + # down. Without this the other three rules go quiet exactly when they matter. + - alert: HermesCronExporterStale + expr: |- + time() - last_over_time(hermes_cron_export_timestamp[2h]) > 1800 + for: 5m + annotations: + summary: >- + Hermes cron health metrics are {{ $value | humanizeDuration }} stale — + cron alerting is blind, check the hermes pod + labels: + severity: warning + + # No for:: absent() over a 2h-integrated window cannot flap, and any delay here + # reopens the gap where Stale has already resolved but Absent has not yet fired, + # which reports "recovered" in the middle of an outage. + - alert: HermesCronExporterAbsent + expr: |- + absent(last_over_time(hermes_cron_export_timestamp[2h])) + annotations: + summary: >- + No Hermes cron health metrics at all — the exporter never ran or was removed + labels: + severity: warning diff --git a/kubernetes/apps/flux-system/flux-operator/app/clusterrolebinding.yaml b/kubernetes/apps/flux-system/flux-operator/app/clusterrolebinding.yaml index dbd53a48e1..3f7b49b5f6 100644 --- a/kubernetes/apps/flux-system/flux-operator/app/clusterrolebinding.yaml +++ b/kubernetes/apps/flux-system/flux-operator/app/clusterrolebinding.yaml @@ -10,5 +10,10 @@ subjects: apiGroup: rbac.authorization.k8s.io roleRef: kind: ClusterRole - name: flux-web-admin + # The web UI authenticates as Anonymous, so this role is granted to anyone who can + # reach envoy-internal. flux-web-admin adds create on batch/jobs (arbitrary pod + # execution), patch/restart on workloads and delete on pods; flux-web-user is the + # chart's read-only variant. It still carries get/list/watch on *, which includes + # Secrets -- only real auth or dropping web.httpRoute closes that. + name: flux-web-user apiGroup: rbac.authorization.k8s.io diff --git a/kubernetes/apps/flux-system/flux-operator/app/helmrelease.yaml b/kubernetes/apps/flux-system/flux-operator/app/helmrelease.yaml index aed7da1108..f359795661 100644 --- a/kubernetes/apps/flux-system/flux-operator/app/helmrelease.yaml +++ b/kubernetes/apps/flux-system/flux-operator/app/helmrelease.yaml @@ -27,5 +27,8 @@ spec: parentRefs: - name: envoy-internal namespace: network + # Chart default is true. Caps pod ingress to 9080 (web) and 8080 (metrics, kept + # because serviceMonitor.create is on); `from` is namespaceSelector:{}, so reach + # to the anonymous UI is unchanged — the read-only roleRef is what limits it. networkPolicy: - create: false + create: true diff --git a/kubernetes/apps/kopiur-system/kopiur/app/kustomization.yaml b/kubernetes/apps/kopiur-system/kopiur/app/kustomization.yaml index 660b70723d..8be8c76c4c 100644 --- a/kubernetes/apps/kopiur-system/kopiur/app/kustomization.yaml +++ b/kubernetes/apps/kopiur-system/kopiur/app/kustomization.yaml @@ -5,3 +5,4 @@ kind: Kustomization resources: - helmrelease.yaml - ocirepository.yaml + - prometheusrule.yaml diff --git a/kubernetes/apps/kopiur-system/kopiur/app/prometheusrule.yaml b/kubernetes/apps/kopiur-system/kopiur/app/prometheusrule.yaml new file mode 100644 index 0000000000..b316c5e957 --- /dev/null +++ b/kubernetes/apps/kopiur-system/kopiur/app/prometheusrule.yaml @@ -0,0 +1,73 @@ +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/monitoring.coreos.com/prometheusrule_v1.json +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: kopiur-rules +spec: + groups: + # The chart ships its own `kopiur.rules` group (monitoring.prometheusRule.enabled + # in the HelmRelease); group and alert names here must not collide with it. + - name: kopiur.freshness.rules + rules: + # Backups were down 44h in July before anyone noticed: nothing watched + # freshness. Snapshots run hourly, so 6h is 6 missed runs, not a slow one. + # Named Overdue, not Stale: the chart's KopiurBackupStale fires off the same + # metric with the same severity, so a shared name is one alert fingerprint. + - alert: KopiurBackupOverdue + expr: |- + time() - kopiur_policy_last_backup_success_timestamp_seconds > 6 * 60 * 60 + for: 30m + annotations: + summary: >- + kopiur {{ $labels.namespace }}/{{ $labels.policy }} has no successful backup + for {{ $value | humanizeDuration }} — restore point is aging + labels: + severity: warning + + - alert: KopiurBackupCritical + expr: |- + time() - kopiur_policy_last_backup_success_timestamp_seconds > 24 * 60 * 60 + for: 30m + annotations: + summary: >- + kopiur {{ $labels.namespace }}/{{ $labels.policy }} has not backed up + successfully in {{ $value | humanizeDuration }} + labels: + severity: critical + + # The July outage removed the CSI snapshot CRDs, so policies stopped reporting + # rather than reporting failure. Absence is the failure mode to catch. + - alert: KopiurPolicyMetricsAbsent + expr: |- + absent(kopiur_policy_last_backup_success_timestamp_seconds) + for: 30m + annotations: + summary: >- + No kopiur policy backup metrics at all — the controller is down or every + policy stopped reporting; backup state is unknown, not healthy + labels: + severity: critical + + # A repo whose snapshot count only ever grows means retention is not pruning; + # a drop to zero means the repo was emptied under us. + - alert: KopiurRepoEmpty + expr: |- + kopiur_repo_snapshot_count == 0 + for: 1h + annotations: + summary: >- + kopiur repository {{ $labels.name }} reports zero snapshots — the repo is + empty or unreadable + labels: + severity: critical + + - alert: KopiurControllerNotLeader + expr: |- + max(kopiur_leader_is_leader) == 0 + for: 30m + annotations: + summary: >- + No kopiur controller holds the leader lease — nothing is scheduling backups + labels: + severity: critical diff --git a/kubernetes/components/kopiur/backup/snapshotpolicy.yaml b/kubernetes/components/kopiur/backup/snapshotpolicy.yaml index c59ab9ba3a..00fc0ffab3 100644 --- a/kubernetes/components/kopiur/backup/snapshotpolicy.yaml +++ b/kubernetes/components/kopiur/backup/snapshotpolicy.yaml @@ -34,6 +34,16 @@ spec: runAsGroup: ${KOPIUR_PGID:=568} capabilities: add: ${KOPIUR_MOVER_CAPS_ADD:=[]} + # Verification is opt-in and nobody had opted in: snapshots ran hourly for 18d with + # LAST-VERIFIED empty on all 18 policies. A repo that stops being restorable reads + # identical to a healthy one until the restore. Fixed hour + 3h jitter spreads the + # fleet wider than `H 3 * * 0` would (a single hashed minute inside one hour), which + # matters because quick verify walks the whole repo. + verification: + quick: + schedule: + cron: 0 3 * * 0 + jitter: 3h compression: compressor: zstd retention: