diff --git a/apps/dashboard/vite.config.ts b/apps/dashboard/vite.config.ts index aa66709..d3668a3 100644 --- a/apps/dashboard/vite.config.ts +++ b/apps/dashboard/vite.config.ts @@ -5,7 +5,7 @@ import react from '@vitejs/plugin-react' export default defineConfig({ plugins: [react()], server: { - port: 5174, + port: 5173, strictPort: true, }, }) diff --git a/argocd/app-of-apps.yaml b/argocd/app-of-apps.yaml index 54b9b83..c726c0e 100644 --- a/argocd/app-of-apps.yaml +++ b/argocd/app-of-apps.yaml @@ -1,16 +1,18 @@ -# Root Application — manages all child Applications via App-of-Apps pattern +# Legacy app-of-apps pointing to argocd/environments. +# The canonical root Application is k8s/argocd/app-of-apps.yaml (name: grainguard-apps). +# This file uses a distinct name to avoid last-write-wins identity collision. apiVersion: argoproj.io/v1alpha1 kind: Application metadata: - name: grainguard-apps + name: grainguard-apps-legacy namespace: argocd # Cascade delete: removing this app also removes all children finalizers: - resources-finalizer.argocd.argoproj.io spec: - project: default + project: grainguard source: - repoURL: https://github.com/your-org/grainguard + repoURL: https://github.com/pahuldeepp/GrainGuard-.git targetRevision: HEAD path: argocd/environments # folder containing per-env Application manifests destination: @@ -21,4 +23,4 @@ spec: prune: true selfHeal: true syncOptions: - - CreateNamespace=true \ No newline at end of file + - CreateNamespace=true diff --git a/argocd/environments/dev.yaml b/argocd/environments/dev.yaml index 8a50ba8..058817f 100644 --- a/argocd/environments/dev.yaml +++ b/argocd/environments/dev.yaml @@ -6,15 +6,15 @@ metadata: finalizers: - resources-finalizer.argocd.argoproj.io spec: - project: default + project: grainguard source: - repoURL: https://github.com/your-org/grainguard + repoURL: https://github.com/pahuldeepp/GrainGuard-.git targetRevision: HEAD - path: helm/grainguard + path: k8s/helm/grainguard helm: valueFiles: - values.yaml - - values.dev.yaml + - values-dev.yaml # Override image tags per deploy — CI will patch this parameters: - name: gateway.image.tag @@ -31,4 +31,4 @@ spec: syncOptions: - CreateNamespace=true - ServerSideApply=true - revisionHistoryLimit: 3 \ No newline at end of file + revisionHistoryLimit: 3 diff --git a/argocd/environments/prod.yaml b/argocd/environments/prod.yaml index 00a966d..be54055 100644 --- a/argocd/environments/prod.yaml +++ b/argocd/environments/prod.yaml @@ -6,15 +6,15 @@ metadata: finalizers: - resources-finalizer.argocd.argoproj.io spec: - project: default + project: grainguard source: - repoURL: https://github.com/your-org/grainguard + repoURL: https://github.com/pahuldeepp/GrainGuard-.git targetRevision: HEAD - path: helm/grainguard + path: k8s/helm/grainguard helm: valueFiles: - values.yaml - - values.prod.yaml + - values-prod.yaml parameters: - name: gateway.image.tag value: "latest" diff --git a/argocd/project.yaml b/argocd/project.yaml index 10302d1..2a2e66c 100644 --- a/argocd/project.yaml +++ b/argocd/project.yaml @@ -6,7 +6,7 @@ metadata: spec: description: GrainGuard platform project sourceRepos: - - https://github.com/your-org/grainguard + - https://github.com/pahuldeepp/GrainGuard-.git destinations: - namespace: grainguard-dev server: https://kubernetes.default.svc @@ -30,4 +30,4 @@ spec: description: Can sync prod (humans only, not CI) policies: - p, proj:grainguard:prod-deployer, applications, sync, grainguard/grainguard-prod, allow - - p, proj:grainguard:prod-deployer, applications, get, grainguard/grainguard-prod, allow \ No newline at end of file + - p, proj:grainguard:prod-deployer, applications, get, grainguard/grainguard-prod, allow diff --git a/docs/runbooks/postgres-backup-restore.md b/docs/runbooks/postgres-backup-restore.md new file mode 100644 index 0000000..4fb2a71 --- /dev/null +++ b/docs/runbooks/postgres-backup-restore.md @@ -0,0 +1,114 @@ +# Runbook: Postgres Backup and Restore + +**Purpose:** Capture and restore both the write-model and read-model Postgres databases for local drills and operator validation. +**Service affected:** telemetry-service, saga-orchestrator, bff, read-model-builder, asset-registry + +--- + +## Scope + +This runbook matches the local Docker stack: + +- write DB container: `grainguard-postgres` +- read DB container: `grainguard-postgres-read` +- scripts: + - [`backup-postgres.sh`](infra/scripts/backup-postgres.sh) + - [`restore-postgres.sh`](infra/scripts/restore-postgres.sh) + +These scripts create custom-format dumps for: + +- `grainguard` +- `grainguard_read` + +--- + +## Backup + +```bash +./infra/scripts/backup-postgres.sh +``` + +Optional custom target: + +```bash +BACKUP_ROOT=/tmp/grainguard-backups ./infra/scripts/backup-postgres.sh +``` + +Expected output: + +- `infra/backups/postgres//grainguard.dump` +- `infra/backups/postgres//grainguard_read.dump` +- `infra/backups/postgres//metadata.env` + +--- + +## Verify backup + +```bash +ls -lh infra/backups/postgres/ +``` + +Optional quick integrity check: + +```bash +docker exec -i grainguard-postgres pg_restore -l < infra/backups/postgres//grainguard.dump | head +docker exec -i grainguard-postgres-read pg_restore -l < infra/backups/postgres//grainguard_read.dump | head +``` + +--- + +## Restore + +Warning: this is destructive. The target database is dropped and recreated before restore. + +Restore both databases: + +```bash +./infra/scripts/restore-postgres.sh infra/backups/postgres/ +``` + +Restore only the write DB: + +```bash +./infra/scripts/restore-postgres.sh infra/backups/postgres/ write +``` + +Restore only the read DB: + +```bash +./infra/scripts/restore-postgres.sh infra/backups/postgres/ read +``` + +--- + +## Verify recovery + +```bash +docker exec grainguard-postgres psql -U postgres -d grainguard -c '\dt' +docker exec grainguard-postgres-read psql -U postgres -d grainguard_read -c '\dt' +curl -fsS http://localhost:8086/health +curl -fsS http://localhost:4000/graphql -H 'content-type: application/json' -d '{"query":"{ __typename }"}' +``` + +Check application logs if a service still has stale connections: + +```bash +docker logs --tail 100 grainguard-gateway +docker logs --tail 100 grainguard-bff +docker logs --tail 100 grainguard-telemetry +``` + +If needed, restart readers and API services: + +```bash +docker compose -f infra/docker/docker-compose.yml restart gateway bff telemetry-service read-model-builder saga-orchestrator +``` + +--- + +## Escalate if + +- `pg_restore` reports schema corruption +- backup files cannot be listed by `pg_restore -l` +- write DB restores successfully but read projections remain empty after service restart +- WAL/replication behavior is required rather than simple logical dump restore diff --git a/docs/runbooks/redis-backup-restore.md b/docs/runbooks/redis-backup-restore.md new file mode 100644 index 0000000..ff039db --- /dev/null +++ b/docs/runbooks/redis-backup-restore.md @@ -0,0 +1,93 @@ +# Runbook: Redis Backup and Restore + +**Purpose:** Capture and restore the standalone Redis instance used by the local Docker stack for cache and lock validation drills. +**Service affected:** bff, saga-orchestrator, jobs-worker, workflow-alerts + +--- + +## Scope + +This runbook targets the standalone Redis container in local Docker: + +- redis container: `grainguard-redis` +- scripts: + - [`backup-redis.sh`](infra/scripts/backup-redis.sh) + - [`restore-redis.sh`](infra/scripts/restore-redis.sh) + +It does **not** back up the six-node Redis cluster used for cluster-mode experiments. + +--- + +## Backup + +```bash +./infra/scripts/backup-redis.sh +``` + +Optional custom target: + +```bash +BACKUP_ROOT=/tmp/grainguard-backups ./infra/scripts/backup-redis.sh +``` + +Expected output: + +- `infra/backups/redis//dump.rdb` +- `infra/backups/redis//metadata.env` + +--- + +## Verify backup + +```bash +ls -lh infra/backups/redis/ +``` + +Sanity check Redis before restore work: + +```bash +docker exec grainguard-redis redis-cli PING +docker exec grainguard-redis redis-cli DBSIZE +``` + +--- + +## Restore + +Warning: this restarts the Redis container and can evict hot cache state and distributed locks. + +```bash +./infra/scripts/restore-redis.sh infra/backups/redis/ +``` + +--- + +## Verify recovery + +```bash +docker exec grainguard-redis redis-cli PING +docker exec grainguard-redis redis-cli DBSIZE +docker logs --tail 100 grainguard-bff +docker logs --tail 100 grainguard-saga-orchestrator +``` + +Optional application check: + +```bash +curl -fsS http://localhost:8086/health +``` + +If application logs still show stale lock/cache issues, restart the consumers: + +```bash +docker compose -f infra/docker/docker-compose.yml restart bff saga-orchestrator jobs-worker workflow-alerts +``` + +--- + +## Escalate if + +- Redis fails to start after replacing `dump.rdb` +- `PING` fails after restore +- cache recovers but saga lock keys remain permanently stale +- you need Redis Cluster restore, not standalone Redis restore diff --git a/go.mod b/go.mod index f24f375..38fee4a 100644 --- a/go.mod +++ b/go.mod @@ -18,9 +18,9 @@ require ( github.com/testcontainers/testcontainers-go v0.41.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.41.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 - go.opentelemetry.io/otel v1.41.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.41.0 - go.opentelemetry.io/otel/sdk v1.41.0 + go.opentelemetry.io/otel v1.42.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 + go.opentelemetry.io/otel/sdk v1.42.0 google.golang.org/grpc v1.79.3 google.golang.org/protobuf v1.36.11 ) @@ -85,9 +85,9 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.41.0 // indirect - go.opentelemetry.io/otel/metric v1.41.0 // indirect - go.opentelemetry.io/otel/trace v1.41.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 // indirect + go.opentelemetry.io/otel/metric v1.42.0 // indirect + go.opentelemetry.io/otel/trace v1.42.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect golang.org/x/crypto v0.49.0 // indirect golang.org/x/net v0.52.0 // indirect diff --git a/go.sum b/go.sum index d914c70..d0ae851 100644 --- a/go.sum +++ b/go.sum @@ -199,22 +199,22 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.6 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0/go.mod h1:KDgtbWKTQs4bM+VPUr6WlL9m/WXcmkCcBlIzqxPGzmI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= -go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.41.0 h1:ao6Oe+wSebTlQ1OEht7jlYTzQKE+pnx/iNywFvTbuuI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.41.0/go.mod h1:u3T6vz0gh/NVzgDgiwkgLxpsSF6PaPmo2il0apGJbls= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.41.0 h1:mq/Qcf28TWz719lE3/hMB4KkyDuLJIvgJnFGcd0kEUI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.41.0/go.mod h1:yk5LXEYhsL2htyDNJbEq7fWzNEigeEdV5xBF/Y+kAv0= +go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= +go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 h1:THuZiwpQZuHPul65w4WcwEnkX2QIuMT+UFoOrygtoJw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0/go.mod h1:J2pvYM5NGHofZ2/Ru6zw/TNWnEQp5crgyDeSrYpXkAw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 h1:zWWrB1U6nqhS/k6zYB74CjRpuiitRtLLi68VcgmOEto= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0/go.mod h1:2qXPNBX1OVRC0IwOnfo1ljoid+RD0QK3443EaqVlsOU= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0 h1:inYW9ZhgqiDqh6BioM7DVHHzEGVq76Db5897WLGZ5Go= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0/go.mod h1:Izur+Wt8gClgMJqO/cZ8wdeeMryJ/xxiOVgFSSfpDTY= -go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= -go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= -go.opentelemetry.io/otel/sdk v1.41.0 h1:YPIEXKmiAwkGl3Gu1huk1aYWwtpRLeskpV+wPisxBp8= -go.opentelemetry.io/otel/sdk v1.41.0/go.mod h1:ahFdU0G5y8IxglBf0QBJXgSe7agzjE4GiTJ6HT9ud90= -go.opentelemetry.io/otel/sdk/metric v1.41.0 h1:siZQIYBAUd1rlIWQT2uCxWJxcCO7q3TriaMlf08rXw8= -go.opentelemetry.io/otel/sdk/metric v1.41.0/go.mod h1:HNBuSvT7ROaGtGI50ArdRLUnvRTRGniSUZbxiWxSO8Y= -go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= -go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= +go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= +go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= +go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= +go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts= +go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA= +go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc= +go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= +go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= diff --git a/infra/docker/docker-compose.yml b/infra/docker/docker-compose.yml index 5c3d309..707943d 100644 --- a/infra/docker/docker-compose.yml +++ b/infra/docker/docker-compose.yml @@ -657,6 +657,7 @@ services: ports: - "4317:4317" - "4318:4318" + - "127.0.0.1:13133:13133" depends_on: - tempo networks: diff --git a/infra/docker/grafana/provisioning/dashboards/grainguard-overview.json b/infra/docker/grafana/provisioning/dashboards/grainguard-overview.json index 4252311..4c313a3 100644 --- a/infra/docker/grafana/provisioning/dashboards/grainguard-overview.json +++ b/infra/docker/grafana/provisioning/dashboards/grainguard-overview.json @@ -3,14 +3,14 @@ "editable":true, "graphTooltip":1, "panels":[ - {"title":"HTTP Request Rate","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":0},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(http_requests_total[5m])) by (status_code)","legendFormat":"{{status_code}}","refId":"A"}],"fieldConfig":{"defaults":{"unit":"reqps","custom":{"drawStyle":"line","fillOpacity":10}},"overrides":[]}}, - {"title":"HTTP Error Rate (5xx %)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":0},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(http_requests_total{status_code=~\"5..\"}[5m])) / sum(rate(http_requests_total[5m])) * 100","legendFormat":"5xx %","refId":"A"}],"fieldConfig":{"defaults":{"unit":"percent","thresholds":{"steps":[{"color":"green","value":null},{"color":"red","value":5}]},"custom":{"drawStyle":"line","fillOpacity":20}},"overrides":[]}}, - {"title":"P95 Latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95","refId":"A"}],"fieldConfig":{"defaults":{"unit":"s","custom":{"drawStyle":"line","fillOpacity":10}},"overrides":[]}}, - {"title":"Active Tenants","type":"stat","gridPos":{"h":8,"w":12,"x":12,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"count(count by (tenant_id) (http_requests_total))","legendFormat":"Tenants","refId":"A"}],"fieldConfig":{"defaults":{"thresholds":{"steps":[{"color":"green","value":null}]}},"overrides":[]}}, - {"title":"Kafka Consumer Lag","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":16},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"kafka_consumer_group_lag","legendFormat":"{{topic}} / {{group}}","refId":"A"}],"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"bars","fillOpacity":30}},"overrides":[]}}, - {"title":"RabbitMQ Queue Depth","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":16},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"rabbitmq_queue_messages_ready","legendFormat":"{{queue}}","refId":"A"}],"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"bars","fillOpacity":30}},"overrides":[]}}, - {"title":"Telemetry Ingest Rate","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":24},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"rate(telemetry_readings_total[5m])","legendFormat":"readings/s","refId":"A"}],"fieldConfig":{"defaults":{"unit":"ops","custom":{"drawStyle":"line","fillOpacity":15}},"overrides":[]}}, - {"title":"Alert Rate","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":24},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"rate(alerts_fired_total[5m])","legendFormat":"alerts/s","refId":"A"}],"fieldConfig":{"defaults":{"unit":"ops","custom":{"drawStyle":"line","fillOpacity":15}},"overrides":[]}} + {"title":"Gateway Request Rate","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":0},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(gateway_request_duration_seconds_count[5m])) or vector(0)","legendFormat":"req/s","refId":"A"}],"fieldConfig":{"defaults":{"unit":"reqps","custom":{"drawStyle":"line","fillOpacity":10}},"overrides":[]}}, + {"title":"Gateway Event Loop P99","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":0},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"gateway_nodejs_eventloop_lag_p99_seconds or vector(0)","legendFormat":"p99","refId":"A"}],"fieldConfig":{"defaults":{"unit":"s","thresholds":{"steps":[{"color":"green","value":null},{"color":"red","value":0.1}]},"custom":{"drawStyle":"line","fillOpacity":20}},"overrides":[]}}, + {"title":"Gateway P95 Latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(gateway_request_duration_seconds_bucket[5m])) by (le)) or vector(0)","legendFormat":"p95","refId":"A"}],"fieldConfig":{"defaults":{"unit":"s","custom":{"drawStyle":"line","fillOpacity":10}},"overrides":[]}}, + {"title":"Active Services","type":"stat","gridPos":{"h":8,"w":12,"x":12,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(up) or vector(0)","legendFormat":"Services","refId":"A"}],"fieldConfig":{"defaults":{"thresholds":{"steps":[{"color":"red","value":null},{"color":"yellow","value":1},{"color":"green","value":4}]}},"overrides":[]}}, + {"title":"Kafka Consumer Lag","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":16},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kafka_consumer_lag) or vector(0)","legendFormat":"lag","refId":"A"}],"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"bars","fillOpacity":30}},"overrides":[]}}, + {"title":"Publish Queue Depth","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":16},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(publish_queue_depth) or vector(0)","legendFormat":"publish","refId":"A"},{"expr":"sum(worker_queue_depth) or vector(0)","legendFormat":"worker","refId":"B"},{"expr":"sum(commit_queue_depth) or vector(0)","legendFormat":"commit","refId":"C"}],"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"bars","fillOpacity":30}},"overrides":[]}}, + {"title":"Gateway Cache Ops Rate","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":24},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(cache_hits_total[5m])) or vector(0)","legendFormat":"hits/s","refId":"A"},{"expr":"sum(rate(cache_misses_total[5m])) or vector(0)","legendFormat":"misses/s","refId":"B"},{"expr":"sum(rate(cache_errors_total[5m])) or vector(0)","legendFormat":"errors/s","refId":"C"}],"fieldConfig":{"defaults":{"unit":"ops","custom":{"drawStyle":"line","fillOpacity":15}},"overrides":[]}}, + {"title":"Gateway DB / Redis Avg Latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":24},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(gateway_db_duration_seconds_sum[5m])) / clamp_min(sum(rate(gateway_db_duration_seconds_count[5m])), 0.000001)","legendFormat":"db","refId":"A"},{"expr":"sum(rate(gateway_redis_duration_seconds_sum[5m])) / clamp_min(sum(rate(gateway_redis_duration_seconds_count[5m])), 0.000001)","legendFormat":"redis","refId":"B"}],"fieldConfig":{"defaults":{"unit":"s","custom":{"drawStyle":"line","fillOpacity":15}},"overrides":[]}} ], "refresh":"30s", "schemaVersion":39, diff --git a/infra/docker/grafana/provisioning/datasources/datasources.yaml b/infra/docker/grafana/provisioning/datasources/datasources.yaml index f29c8fc..f6c6506 100644 --- a/infra/docker/grafana/provisioning/datasources/datasources.yaml +++ b/infra/docker/grafana/provisioning/datasources/datasources.yaml @@ -1,7 +1,17 @@ apiVersion: 1 +prune: true + +deleteDatasources: + - name: Prometheus + orgId: 1 + - name: Loki + orgId: 1 + - name: Tempo + orgId: 1 datasources: - name: Prometheus + uid: prometheus type: prometheus access: proxy url: http://prometheus:9090 @@ -9,12 +19,14 @@ datasources: editable: false - name: Loki + uid: loki type: loki access: proxy url: http://loki:3100 editable: false - name: Tempo + uid: tempo type: tempo access: proxy url: http://tempo:3200 diff --git a/infra/docker/otel-collector-config.yaml b/infra/docker/otel-collector-config.yaml index a8c2465..625c4b5 100644 --- a/infra/docker/otel-collector-config.yaml +++ b/infra/docker/otel-collector-config.yaml @@ -1,4 +1,8 @@ -receivers: +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: otlp: protocols: grpc: @@ -6,14 +10,27 @@ http: endpoint: 0.0.0.0:4318 +processors: + batch: + timeout: 5s + send_batch_size: 512 + exporters: - otlp: - endpoint: grainguard-tempo:4317 - tls: - insecure: true + otlp_http: + endpoint: http://tempo:4318 + sending_queue: + enabled: true + queue_size: 2048 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s service: + extensions: [health_check] pipelines: traces: receivers: [otlp] - exporters: [otlp] + processors: [batch] + exporters: [otlp_http] diff --git a/infra/docker/prometheus.yml b/infra/docker/prometheus.yml index 13ec542..960248b 100644 --- a/infra/docker/prometheus.yml +++ b/infra/docker/prometheus.yml @@ -1,8 +1,20 @@ global: - scrape_interval: 5s + scrape_interval: 15s + evaluation_interval: 15s scrape_configs: + - job_name: "gateway" + static_configs: + - targets: ["gateway:3000"] + + - job_name: "bff" + static_configs: + - targets: ["bff:4000"] + - job_name: "read-model-builder" static_configs: - targets: ["read-model-builder:2112"] + - job_name: "tempo" + static_configs: + - targets: ["tempo:3200"] diff --git a/infra/scripts/backup-postgres.sh b/infra/scripts/backup-postgres.sh new file mode 100644 index 0000000..3fa605d --- /dev/null +++ b/infra/scripts/backup-postgres.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BACKUP_ROOT="${BACKUP_ROOT:-${ROOT_DIR}/backups}" +TIMESTAMP="${TIMESTAMP:-$(date -u +%Y%m%dT%H%M%SZ)}" +BACKUP_DIR="${BACKUP_ROOT}/postgres/${TIMESTAMP}" + +WRITE_CONTAINER="${WRITE_CONTAINER:-grainguard-postgres}" +READ_CONTAINER="${READ_CONTAINER:-grainguard-postgres-read}" +PGUSER="${PGUSER:-postgres}" +WRITE_DB="${WRITE_DB:-grainguard}" +READ_DB="${READ_DB:-grainguard_read}" + +mkdir -p "${BACKUP_DIR}" + +echo "[postgres-backup] writing backups into ${BACKUP_DIR}" + +docker exec "${WRITE_CONTAINER}" pg_dump \ + -U "${PGUSER}" \ + -d "${WRITE_DB}" \ + -Fc > "${BACKUP_DIR}/${WRITE_DB}.dump" + +docker exec "${READ_CONTAINER}" pg_dump \ + -U "${PGUSER}" \ + -d "${READ_DB}" \ + -Fc > "${BACKUP_DIR}/${READ_DB}.dump" + +cat > "${BACKUP_DIR}/metadata.env" </dev/null +docker cp "${REDIS_CONTAINER}:/data/dump.rdb" "${BACKUP_DIR}/dump.rdb" + +cat > "${BACKUP_DIR}/metadata.env" < [write|read|both]" >&2 + exit 1 +fi + +BACKUP_DIR="$1" +RESTORE_SCOPE="${2:-both}" + +if [[ ! -d "${BACKUP_DIR}" ]]; then + echo "[postgres-restore] backup dir not found: ${BACKUP_DIR}" >&2 + exit 1 +fi + +WRITE_CONTAINER="${WRITE_CONTAINER:-grainguard-postgres}" +READ_CONTAINER="${READ_CONTAINER:-grainguard-postgres-read}" +PGUSER="${PGUSER:-postgres}" +WRITE_DB="${WRITE_DB:-grainguard}" +READ_DB="${READ_DB:-grainguard_read}" + +restore_db() { + local container="$1" + local db="$2" + local dump_file="$3" + + if [[ ! -f "${dump_file}" ]]; then + echo "[postgres-restore] dump file missing: ${dump_file}" >&2 + exit 1 + fi + + echo "[postgres-restore] restoring ${db} into ${container}" + docker exec "${container}" psql -U "${PGUSER}" -d postgres -v ON_ERROR_STOP=1 \ + -c "DROP DATABASE IF EXISTS ${db};" + docker exec "${container}" psql -U "${PGUSER}" -d postgres -v ON_ERROR_STOP=1 \ + -c "CREATE DATABASE ${db};" + docker exec -i "${container}" pg_restore \ + -U "${PGUSER}" \ + -d "${db}" \ + --no-owner \ + --clean \ + --if-exists < "${dump_file}" +} + +case "${RESTORE_SCOPE}" in + write) + restore_db "${WRITE_CONTAINER}" "${WRITE_DB}" "${BACKUP_DIR}/${WRITE_DB}.dump" + ;; + read) + restore_db "${READ_CONTAINER}" "${READ_DB}" "${BACKUP_DIR}/${READ_DB}.dump" + ;; + both) + restore_db "${WRITE_CONTAINER}" "${WRITE_DB}" "${BACKUP_DIR}/${WRITE_DB}.dump" + restore_db "${READ_CONTAINER}" "${READ_DB}" "${BACKUP_DIR}/${READ_DB}.dump" + ;; + *) + echo "[postgres-restore] invalid scope: ${RESTORE_SCOPE}" >&2 + echo "expected one of: write, read, both" >&2 + exit 1 + ;; +esac + +echo "[postgres-restore] restore completed" diff --git a/infra/scripts/restore-redis.sh b/infra/scripts/restore-redis.sh new file mode 100644 index 0000000..eb17a99 --- /dev/null +++ b/infra/scripts/restore-redis.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "usage: $0 " >&2 + exit 1 +fi + +BACKUP_DIR="$1" +RDB_FILE="${BACKUP_DIR}/dump.rdb" +REDIS_CONTAINER="${REDIS_CONTAINER:-grainguard-redis}" + +if [[ ! -f "${RDB_FILE}" ]]; then + echo "[redis-restore] backup file not found: ${RDB_FILE}" >&2 + exit 1 +fi + +echo "[redis-restore] stopping ${REDIS_CONTAINER}" +docker stop "${REDIS_CONTAINER}" >/dev/null + +# Copy the RDB snapshot then clear any existing AOF files so Redis loads +# the restored dump on startup rather than replaying an AOF that would +# conflict with or overwrite the snapshot data. +docker cp "${RDB_FILE}" "${REDIS_CONTAINER}:/data/dump.rdb" +docker run --rm --volumes-from "${REDIS_CONTAINER}" alpine \ + sh -c 'rm -f /data/appendonly.aof /data/appendonly.aof.manifest /data/*.aof.bak 2>/dev/null; echo "[redis-restore] AOF cleared"' + +echo "[redis-restore] starting ${REDIS_CONTAINER}" +docker start "${REDIS_CONTAINER}" >/dev/null + +# Wait until Redis is ready to accept connections (up to 30 s) +echo "[redis-restore] waiting for Redis to be ready..." +for i in $(seq 1 30); do + if docker exec "${REDIS_CONTAINER}" redis-cli PING 2>/dev/null | grep -q PONG; then + break + fi + sleep 1 +done +docker exec "${REDIS_CONTAINER}" redis-cli PING | grep -q PONG \ + || { echo "[redis-restore] ERROR: Redis did not become ready after 30s" >&2; exit 1; } + +echo "[redis-restore] restore completed" diff --git a/k8s/argocd/app-of-apps.yaml b/k8s/argocd/app-of-apps.yaml index 67b370d..943f70e 100644 --- a/k8s/argocd/app-of-apps.yaml +++ b/k8s/argocd/app-of-apps.yaml @@ -8,7 +8,7 @@ metadata: finalizers: - resources-finalizer.argocd.argoproj.io spec: - project: default + project: grainguard source: repoURL: https://github.com/pahuldeepp/GrainGuard-.git targetRevision: HEAD diff --git a/k8s/argocd/apps/grainguard-dev.yaml b/k8s/argocd/apps/grainguard-dev.yaml index d281d4a..3f3cc2a 100644 --- a/k8s/argocd/apps/grainguard-dev.yaml +++ b/k8s/argocd/apps/grainguard-dev.yaml @@ -8,7 +8,7 @@ metadata: labels: environment: dev spec: - project: default + project: grainguard source: repoURL: https://github.com/pahuldeepp/GrainGuard-.git targetRevision: HEAD diff --git a/k8s/argocd/apps/grainguard-prod.yaml b/k8s/argocd/apps/grainguard-prod.yaml index bbf4e96..22c867f 100644 --- a/k8s/argocd/apps/grainguard-prod.yaml +++ b/k8s/argocd/apps/grainguard-prod.yaml @@ -8,7 +8,7 @@ metadata: labels: environment: prod spec: - project: default + project: grainguard source: repoURL: https://github.com/pahuldeepp/GrainGuard-.git targetRevision: HEAD diff --git a/k8s/argocd/install.sh b/k8s/argocd/install.sh index 67b8df2..324b0ce 100644 --- a/k8s/argocd/install.sh +++ b/k8s/argocd/install.sh @@ -14,6 +14,9 @@ kubectl apply -n "${ARGOCD_NAMESPACE}" \ echo "==> Waiting for ArgoCD to be ready..." kubectl rollout status deployment/argocd-server -n "${ARGOCD_NAMESPACE}" --timeout=120s +echo "==> Applying GrainGuard ArgoCD project..." +kubectl apply -f k8s/argocd/project.yaml + echo "==> Applying App of Apps..." kubectl apply -f k8s/argocd/app-of-apps.yaml diff --git a/k8s/argocd/project.yaml b/k8s/argocd/project.yaml new file mode 100644 index 0000000..2a2e66c --- /dev/null +++ b/k8s/argocd/project.yaml @@ -0,0 +1,33 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: grainguard + namespace: argocd +spec: + description: GrainGuard platform project + sourceRepos: + - https://github.com/pahuldeepp/GrainGuard-.git + destinations: + - namespace: grainguard-dev + server: https://kubernetes.default.svc + - namespace: grainguard-prod + server: https://kubernetes.default.svc + - namespace: argocd + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: "" + kind: Namespace + namespaceResourceWhitelist: + - group: "*" + kind: "*" + roles: + - name: dev-deployer + description: Can sync dev only + policies: + - p, proj:grainguard:dev-deployer, applications, sync, grainguard/grainguard-dev, allow + - p, proj:grainguard:dev-deployer, applications, get, grainguard/grainguard-dev, allow + - name: prod-deployer + description: Can sync prod (humans only, not CI) + policies: + - p, proj:grainguard:prod-deployer, applications, sync, grainguard/grainguard-prod, allow + - p, proj:grainguard:prod-deployer, applications, get, grainguard/grainguard-prod, allow diff --git a/k8s/helm/grainguard/templates/deployment.yaml b/k8s/helm/grainguard/templates/deployment.yaml index 6a88e08..c56124b 100644 --- a/k8s/helm/grainguard/templates/deployment.yaml +++ b/k8s/helm/grainguard/templates/deployment.yaml @@ -1,5 +1,6 @@ {{- range $name, $svc := .Values.services }} {{- if $svc.enabled }} +{{- $rollingUpdate := $svc.rollingUpdate | default dict }} --- apiVersion: apps/v1 kind: Deployment @@ -12,6 +13,11 @@ metadata: helm.sh/chart: {{ $.Chart.Name }}-{{ $.Chart.Version }} spec: replicas: {{ $svc.replicaCount | default 1 }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: {{ $rollingUpdate.maxUnavailable | default 0 }} + maxSurge: {{ $rollingUpdate.maxSurge | default 1 }} selector: matchLabels: app.kubernetes.io/name: {{ $name }} diff --git a/k8s/helm/grainguard/templates/networkpolicy.yaml b/k8s/helm/grainguard/templates/networkpolicy.yaml new file mode 100644 index 0000000..3fef8b4 --- /dev/null +++ b/k8s/helm/grainguard/templates/networkpolicy.yaml @@ -0,0 +1,69 @@ +{{- range $name, $svc := .Values.services }} +{{- $networkPolicy := $svc.networkPolicy | default dict }} +{{- if and $svc.enabled $networkPolicy.enabled }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $name }} + namespace: {{ $.Values.namespace }} + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Release.Name }} + helm.sh/chart: {{ $.Chart.Name }}-{{ $.Chart.Version }} +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Release.Name }} + policyTypes: + - Ingress + ingress: + {{- if $networkPolicy.allowExternal }} + - ports: + - protocol: TCP + port: {{ $svc.port }} + {{- if $svc.grpcPort }} + - protocol: TCP + port: {{ $svc.grpcPort }} + {{- end }} + {{- if and $svc.healthPort (ne $svc.healthPort $svc.port) }} + - protocol: TCP + port: {{ $svc.healthPort }} + {{- end }} + {{- end }} + {{- if $networkPolicy.allowFromSameNamespace }} + - from: + - podSelector: {} + ports: + - protocol: TCP + port: {{ $svc.port }} + {{- if $svc.grpcPort }} + - protocol: TCP + port: {{ $svc.grpcPort }} + {{- end }} + {{- if and $svc.healthPort (ne $svc.healthPort $svc.port) }} + - protocol: TCP + port: {{ $svc.healthPort }} + {{- end }} + {{- end }} + {{- range $source := $networkPolicy.allowFromServices }} + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: {{ $source }} + app.kubernetes.io/instance: {{ $.Release.Name }} + ports: + - protocol: TCP + port: {{ $svc.port }} + {{- if $svc.grpcPort }} + - protocol: TCP + port: {{ $svc.grpcPort }} + {{- end }} + {{- if and $svc.healthPort (ne $svc.healthPort $svc.port) }} + - protocol: TCP + port: {{ $svc.healthPort }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} diff --git a/k8s/helm/grainguard/templates/pdb.yaml b/k8s/helm/grainguard/templates/pdb.yaml index 064be03..b6b00fb 100644 --- a/k8s/helm/grainguard/templates/pdb.yaml +++ b/k8s/helm/grainguard/templates/pdb.yaml @@ -1,5 +1,8 @@ {{- range $name, $svc := .Values.services }} -{{- if and $svc.enabled (gt ($svc.replicaCount | default 1 | int) 1) }} +{{- $pdb := $svc.pdb | default dict }} +{{- $replicaCount := ($svc.replicaCount | default 1 | int) }} +{{- $pdbEnabled := $pdb.enabled | default (gt $replicaCount 1) }} +{{- if and $svc.enabled $pdbEnabled }} --- apiVersion: policy/v1 kind: PodDisruptionBudget @@ -10,7 +13,7 @@ metadata: app.kubernetes.io/name: {{ $name }} app.kubernetes.io/instance: {{ $.Release.Name }} spec: - minAvailable: 1 + minAvailable: {{ $pdb.minAvailable | default 1 }} selector: matchLabels: app.kubernetes.io/name: {{ $name }} diff --git a/k8s/helm/grainguard/values.yaml b/k8s/helm/grainguard/values.yaml index 85a5290..171f67d 100644 --- a/k8s/helm/grainguard/values.yaml +++ b/k8s/helm/grainguard/values.yaml @@ -46,6 +46,14 @@ services: maxReplicas: 10 targetCPUUtilizationPercentage: 70 targetMemoryUtilizationPercentage: 80 + pdb: + enabled: true + minAvailable: 1 + networkPolicy: + enabled: true + allowExternal: true + allowFromSameNamespace: true + allowFromServices: [] bff: enabled: true @@ -80,6 +88,15 @@ services: maxReplicas: 8 targetCPUUtilizationPercentage: 70 targetMemoryUtilizationPercentage: 80 + pdb: + enabled: true + minAvailable: 1 + networkPolicy: + enabled: true + allowExternal: false + allowFromSameNamespace: true + allowFromServices: + - gateway telemetry-service: enabled: true @@ -112,6 +129,15 @@ services: maxReplicas: 12 targetCPUUtilizationPercentage: 60 targetMemoryUtilizationPercentage: 75 + pdb: + enabled: true + minAvailable: 1 + networkPolicy: + enabled: true + allowExternal: false + allowFromSameNamespace: true + allowFromServices: + - gateway saga-orchestrator: enabled: true @@ -144,6 +170,14 @@ services: maxReplicas: 6 targetCPUUtilizationPercentage: 70 targetMemoryUtilizationPercentage: 80 + pdb: + enabled: true + minAvailable: 1 + networkPolicy: + enabled: true + allowExternal: false + allowFromSameNamespace: true + allowFromServices: [] read-model-builder: enabled: true @@ -174,6 +208,14 @@ services: maxReplicas: 8 targetCPUUtilizationPercentage: 70 targetMemoryUtilizationPercentage: 80 + pdb: + enabled: true + minAvailable: 1 + networkPolicy: + enabled: true + allowExternal: false + allowFromSameNamespace: true + allowFromServices: [] cdc-transformer: enabled: true @@ -253,6 +295,15 @@ services: maxReplicas: 6 targetCPUUtilizationPercentage: 70 targetMemoryUtilizationPercentage: 80 + pdb: + enabled: true + minAvailable: 1 + networkPolicy: + enabled: true + allowExternal: false + allowFromSameNamespace: true + allowFromServices: + - gateway risk-engine: enabled: true @@ -279,6 +330,14 @@ services: maxReplicas: 8 targetCPUUtilizationPercentage: 60 targetMemoryUtilizationPercentage: 75 + pdb: + enabled: true + minAvailable: 1 + networkPolicy: + enabled: true + allowExternal: false + allowFromSameNamespace: true + allowFromServices: [] search-indexer: enabled: true @@ -333,6 +392,14 @@ services: maxReplicas: 6 targetCPUUtilizationPercentage: 70 targetMemoryUtilizationPercentage: 80 + pdb: + enabled: true + minAvailable: 1 + networkPolicy: + enabled: true + allowExternal: false + allowFromSameNamespace: true + allowFromServices: [] jobs-worker: enabled: true @@ -360,6 +427,14 @@ services: maxReplicas: 6 targetCPUUtilizationPercentage: 70 targetMemoryUtilizationPercentage: 80 + pdb: + enabled: true + minAvailable: 1 + networkPolicy: + enabled: true + allowExternal: false + allowFromSameNamespace: true + allowFromServices: [] cassandra-writer: enabled: true @@ -387,3 +462,11 @@ services: maxReplicas: 8 targetCPUUtilizationPercentage: 60 targetMemoryUtilizationPercentage: 75 + pdb: + enabled: true + minAvailable: 1 + networkPolicy: + enabled: true + allowExternal: false + allowFromSameNamespace: true + allowFromServices: []