From c53892c15af93617156866c258de40938ffe405d Mon Sep 17 00:00:00 2001 From: Ho Phuoc Nghia Date: Thu, 9 Apr 2026 21:05:58 +0700 Subject: [PATCH 01/12] feat(postgrest-template): Remove obsolete files and enhance template structure --- .../examples/postgrest-template/README.md | 265 ------------------ .../examples/postgrest-template/template.yaml | 20 +- 2 files changed, 4 insertions(+), 281 deletions(-) delete mode 100644 apps/portal/examples/postgrest-template/README.md diff --git a/apps/portal/examples/postgrest-template/README.md b/apps/portal/examples/postgrest-template/README.md deleted file mode 100644 index 2eed44b..0000000 --- a/apps/portal/examples/postgrest-template/README.md +++ /dev/null @@ -1,265 +0,0 @@ -# PostgREST Backstage Template - -A complete Backstage software template for scaffolding PostgREST instant REST APIs with automatic database provisioning via the Helios Operator. - -## Overview - -This template creates: -- **Source Repository**: Dockerfile, SQL schema, PostgREST configuration -- **GitOps Repository**: Kubernetes manifests with HeliosApp CRD, Tekton CI/CD, ArgoCD integration - -## Key Features - -✅ **Auto-Generated REST API**: PostgREST creates CRUD endpoints from your database schema -✅ **Automatic Database Provisioning**: Helios Operator provision PostgreSQL on Kubernetes -✅ **CI/CD Pipeline**: Tekton builds custom Docker image with your schema -✅ **GitOps Deployment**: ArgoCD syncs from separate GitOps repository -✅ **JWT Authentication**: Built-in JWT support for securing your API -✅ **Role-Based Access Control**: Database-level permissions with Postgres roles - -## Architecture - -``` -Backstage Template - ↓ -Creates Two Repos: - ├── Source Repo (Dockerfile + schema + config) - │ ↓ - │ Tekton CI/CD - │ ↓ - │ Docker Image → docker.io/org/repo:latest - │ - └── GitOps Repo (Kubernetes manifests) - ↓ - ArgoCD - ↓ - Helios Operator - ├── PostgreSQL Database - ├── PostgREST Container - └── Ingress for REST API -``` - -## Requirements Met ✅ - -### Requirement 1: Scaffolding Only (No K8s Deployment) -- Template generates manifests but does NOT apply them -- Users manually deploy when ready via `kubectl apply -f gitops/` -- Reduces risk of accidental deployments in the wrong cluster - -### Requirement 2: Use Official PostgREST Image -- Dockerfile: `FROM postgrest/postgrest:v12.2.0` (official image) -- Users customize by copying their SQL schema into the image -- Results in: `Dockerfile` → `docker build` → `docker.io/org/repo:latest` custom image -- This custom image still uses official PostgREST as its base, but includes user's schema - -### Requirement 3: Parameterize Namespace -- Template includes `namespace` parameter (required in template.yaml) -- Dynamically injected into ALL Kubernetes manifests: - - `helios-app.yaml`: `metadata.namespace: ${{ parameters.namespace }}` - - `argocd-app.yaml`: `metadata.namespace: ${{ parameters.namespace }}` - - `kustomization.yaml`: All resources in same namespace - - `pipeline.yaml`: `metadata.namespace: ${{ parameters.namespace }}` - - `triggers.yaml`: `metadata.namespace: ${{ parameters.namespace }}` - -### Requirement 4: Support Custom Images with Registry -- Template includes Docker registry parameters: `dockerOrg`, `repoName` -- Dockerfile builds custom image FROM official postgrest/postgrest -- Tekton pipeline (in gitops repo) builds and pushes to registry -- GitOps manifests reference: `docker.io/${{ dockerOrg }}/${{ repoName }}:latest` -- Enables same flexibility as Node.js templates: users build their own images - -## Template Parameters - -### Component Information (Required) -| Parameter | Description | Example | -|-----------|-------------|---------| -| `name` | Service name | `my-api` | -| `port` | PostgREST listen port | `3000` | -| `dockerOrg` | Docker registry org/user | `mycompany` | -| `repoName` | Docker repository name | `my-api` | - -### PostgREST Configuration (Optional) -| Parameter | Description | Default | -|-----------|-------------|---------| -| `apiSchema` | PostgreSQL schema to expose | `public` | -| `jwtSecret` | JWT signing secret (32+ chars) | - | -| `jwtRole` | Default JWT role claim | `authenticated` | -| `anonRole` | Role for unauthenticated requests | `anon` | - -### Infrastructure (Optional) -| Parameter | Description | -|-----------|-------------| -| `namespace` | Kubernetes namespace | (required, input per cluster) | -| `databaseConfig` | Database type, name, etc. | (picker UI) | -| `repoUrl` | Git repository URL | (picker UI) | - -## What Gets Generated - -### Source Repository Structure -``` -source/ - Dockerfile # Builds custom image from postgrest/postgrest - postgrestrc.conf # PostgREST configuration - schema/ # User's database schema - README.md # Schema documentation - 01-tables.sql # Example tables - 02-permissions.sql # Example permissions - catalog-info.yaml # Backstage component metadata - README.md # User documentation -``` - -### GitOps Repository Structure -``` -gitops/ - helios-app.yaml # Main CRD: defines API + database requirement - argocd-app.yaml # Points ArgoCD to this repo - kustomization.yaml # Kubernetes bundle - pipeline.yaml # Tekton PipelineRun for CI/CD - triggers.yaml # Webhook EventListener + TriggerBinding - README.md # Deployment documentation -``` - -## Complete Workflow - -1. **Scaffold via Backstage UI** - - User fills in parameters (name, Docker org, etc.) - - Template generates source + gitops repositories - -2. **Customize Schema** (source repo) - - User edits `schema/01-tables.sql` with their tables - - Edits `schema/02-permissions.sql` for roles/access control - - Commits and pushes to source repo - -3. **Automatic CI/CD** (Webhook → Tekton) - - Webhook triggers Tekton pipeline - - Builds: `docker build -t docker.io/org/repo:latest .` - - Pushes: Docker image to registry - -4. **Manual GitOps Sync** (User → ArgoCD) - - User applies: `kubectl apply -f gitops/helios-app.yaml` - - ArgoCD watches gitops repo for changes - - ArgoCD deploys HeliosApp to cluster - -5. **Automatic Operator Deployment** (ArgoCD → Helios Operator) - - Helios Operator sees HeliosApp CRD - - Creates PostgreSQL database (with schema from Docker image) - - Starts PostgREST container - - Exposes REST API via Ingress - -6. **Use REST API** - - Users interact with auto-generated endpoints - - PostgREST serves REST CRUD operations - -## Deployment - -### Prerequisites -- Kubernetes 1.28+ -- Helios Operator 0.2.0+ installed -- ArgoCD 2.8+ installed -- Docker registry access (Docker Hub username) -- Backstage instance with scaffolder plugin - -### First-Time Setup - -```bash -# After template scaffolding -cd gitops/ - -# Review manifests -cat helios-app.yaml - -# Deploy to cluster -kubectl apply -f helios-app.yaml - -# Watch deployment progress -kubectl get heliosapp -w - -# List generated endpoints -kubectl get ingress -n your-namespace -``` - -### Update Deployment - -```bash -# From source repo: push schema changes -git add schema/ -git commit -m "Add new tables" -git push - -# Tekton auto-builds and pushes image - -# From gitops repo: review and commit any changes -cd ../gitops/ -git add . -git commit -m "Update configuration" -git push - -# ArgoCD auto-syncs -``` - -## Security Considerations - -1. **JWT Secrets**: Provide secure `jwtSecret` (32+ characters, random) -2. **Database Credentials**: Not stored in manifests; injected by Operator -3. **Role-Based Access**: Configure `anonRole` and `authenticated` roles in schema -4. **Docker Registry**: Use private registries for proprietary APIs -5. **Ingress TLS**: Ensure HTTPS is configured for production - -## Troubleshooting - -### API Not Responding -```bash -# Check PostgREST logs -kubectl logs -f deployment/my-api -c api - -# Verify database connection -kubectl exec -it deployment/my-api -- \ - sh -c 'curl -v http://localhost:3000/' -``` - -### Database Not Initialized -```bash -# Check Operator logs -kubectl logs -f deployment/helios-operator - -# Check database status -kubectl get database -n your-namespace - -# Check secrets injected -kubectl get secret -n your-namespace | grep db -``` - -### CI/CD Not Triggering -```bash -# Check webhook -kubectl get eventlisteners -n default - -# Check Tekton logs -kubectl logs -f tekton-triggers-controller -n tekton-pipelines - -# Verify webhook URL in Git repo settings -# Should point to: http://el-{repoName}-listener.default.svc.cluster.local:8080 -``` - -## Files in This Template - -- **template.yaml** - Backstage template definition -- **validate.sh** - Template validation script -- **content/source/** - Source repository template -- **content/gitops/** - GitOps repository template -- **README.md** (this file) - Template documentation - -## Next Steps - -1. Register this template in Backstage (`catalog-info.yaml`) -2. Users access via Backstage UI → Create Component → PostgREST API Template -3. Follow the scaffolding flow to generate their repositories -4. See generated `README.md` in source repo for customization guide - -## References - -- [PostgREST Documentation](https://postgrest.org) -- [Helios Operator Documentation](../../../../docs/OPERATOR.md) -- [Backstage Scaffolder Docs](https://backstage.io/docs/features/software-templates) -- [ArgoCD Documentation](https://argo-cd.readthedocs.io/) -- [Tekton Pipelines](https://tekton.dev) diff --git a/apps/portal/examples/postgrest-template/template.yaml b/apps/portal/examples/postgrest-template/template.yaml index d13e079..ef54a0d 100644 --- a/apps/portal/examples/postgrest-template/template.yaml +++ b/apps/portal/examples/postgrest-template/template.yaml @@ -17,8 +17,6 @@ spec: required: - name - port - - dockerOrg - - repoName properties: name: title: Name @@ -30,14 +28,6 @@ spec: type: number description: The port PostgREST listens on default: 3000 - dockerOrg: - title: Docker Registry Org/User - type: string - description: Your Docker Hub username or Organization - repoName: - title: Docker Repository Name - type: string - description: The name of the Docker repository (e.g. my-api) - title: PostgREST Configuration properties: @@ -105,11 +95,11 @@ spec: url: ./content/source targetPath: ./source values: - name: ${{ parameters.repoName }} + name: ${{ parameters.name }} owner: ${{ user.entity.metadata.name or 'guest' }} port: ${{ parameters.port }} description: "PostgREST API: ${{ parameters.name }}" - image: index.docker.io/${{ parameters.dockerOrg }}/${{ parameters.repoName }} + image: postgrest/postgrest apiSchema: ${{ parameters.apiSchema }} jwtSecret: ${{ parameters.jwtSecret }} jwtRole: ${{ parameters.jwtRole }} @@ -142,10 +132,8 @@ spec: url: ./content/gitops targetPath: ./gitops values: - name: ${{ parameters.repoName }} - image: index.docker.io/${{ parameters.dockerOrg }}/${{ parameters.repoName }} - dockerOrg: ${{ parameters.dockerOrg }} - repoName: ${{ parameters.repoName }} + name: ${{ parameters.name }} + image: postgrest/postgrest port: ${{ parameters.port }} namespace: ${{ parameters.namespace }} databaseType: ${{ parameters.databaseConfig.dbType }} From 9a91d2ee625095a5a1f33064de5fca9d40dc55e3 Mon Sep 17 00:00:00 2001 From: Ho Phuoc Nghia Date: Thu, 9 Apr 2026 21:57:06 +0700 Subject: [PATCH 02/12] feat(postgrest-template): Enhance PostgREST template with detailed README and schema files --- .../examples/postgrest-template/README.md | 265 ++++++++++++++++++ .../examples/postgrest-template/template.yaml | 23 +- 2 files changed, 280 insertions(+), 8 deletions(-) create mode 100644 apps/portal/examples/postgrest-template/README.md diff --git a/apps/portal/examples/postgrest-template/README.md b/apps/portal/examples/postgrest-template/README.md new file mode 100644 index 0000000..2eed44b --- /dev/null +++ b/apps/portal/examples/postgrest-template/README.md @@ -0,0 +1,265 @@ +# PostgREST Backstage Template + +A complete Backstage software template for scaffolding PostgREST instant REST APIs with automatic database provisioning via the Helios Operator. + +## Overview + +This template creates: +- **Source Repository**: Dockerfile, SQL schema, PostgREST configuration +- **GitOps Repository**: Kubernetes manifests with HeliosApp CRD, Tekton CI/CD, ArgoCD integration + +## Key Features + +✅ **Auto-Generated REST API**: PostgREST creates CRUD endpoints from your database schema +✅ **Automatic Database Provisioning**: Helios Operator provision PostgreSQL on Kubernetes +✅ **CI/CD Pipeline**: Tekton builds custom Docker image with your schema +✅ **GitOps Deployment**: ArgoCD syncs from separate GitOps repository +✅ **JWT Authentication**: Built-in JWT support for securing your API +✅ **Role-Based Access Control**: Database-level permissions with Postgres roles + +## Architecture + +``` +Backstage Template + ↓ +Creates Two Repos: + ├── Source Repo (Dockerfile + schema + config) + │ ↓ + │ Tekton CI/CD + │ ↓ + │ Docker Image → docker.io/org/repo:latest + │ + └── GitOps Repo (Kubernetes manifests) + ↓ + ArgoCD + ↓ + Helios Operator + ├── PostgreSQL Database + ├── PostgREST Container + └── Ingress for REST API +``` + +## Requirements Met ✅ + +### Requirement 1: Scaffolding Only (No K8s Deployment) +- Template generates manifests but does NOT apply them +- Users manually deploy when ready via `kubectl apply -f gitops/` +- Reduces risk of accidental deployments in the wrong cluster + +### Requirement 2: Use Official PostgREST Image +- Dockerfile: `FROM postgrest/postgrest:v12.2.0` (official image) +- Users customize by copying their SQL schema into the image +- Results in: `Dockerfile` → `docker build` → `docker.io/org/repo:latest` custom image +- This custom image still uses official PostgREST as its base, but includes user's schema + +### Requirement 3: Parameterize Namespace +- Template includes `namespace` parameter (required in template.yaml) +- Dynamically injected into ALL Kubernetes manifests: + - `helios-app.yaml`: `metadata.namespace: ${{ parameters.namespace }}` + - `argocd-app.yaml`: `metadata.namespace: ${{ parameters.namespace }}` + - `kustomization.yaml`: All resources in same namespace + - `pipeline.yaml`: `metadata.namespace: ${{ parameters.namespace }}` + - `triggers.yaml`: `metadata.namespace: ${{ parameters.namespace }}` + +### Requirement 4: Support Custom Images with Registry +- Template includes Docker registry parameters: `dockerOrg`, `repoName` +- Dockerfile builds custom image FROM official postgrest/postgrest +- Tekton pipeline (in gitops repo) builds and pushes to registry +- GitOps manifests reference: `docker.io/${{ dockerOrg }}/${{ repoName }}:latest` +- Enables same flexibility as Node.js templates: users build their own images + +## Template Parameters + +### Component Information (Required) +| Parameter | Description | Example | +|-----------|-------------|---------| +| `name` | Service name | `my-api` | +| `port` | PostgREST listen port | `3000` | +| `dockerOrg` | Docker registry org/user | `mycompany` | +| `repoName` | Docker repository name | `my-api` | + +### PostgREST Configuration (Optional) +| Parameter | Description | Default | +|-----------|-------------|---------| +| `apiSchema` | PostgreSQL schema to expose | `public` | +| `jwtSecret` | JWT signing secret (32+ chars) | - | +| `jwtRole` | Default JWT role claim | `authenticated` | +| `anonRole` | Role for unauthenticated requests | `anon` | + +### Infrastructure (Optional) +| Parameter | Description | +|-----------|-------------| +| `namespace` | Kubernetes namespace | (required, input per cluster) | +| `databaseConfig` | Database type, name, etc. | (picker UI) | +| `repoUrl` | Git repository URL | (picker UI) | + +## What Gets Generated + +### Source Repository Structure +``` +source/ + Dockerfile # Builds custom image from postgrest/postgrest + postgrestrc.conf # PostgREST configuration + schema/ # User's database schema + README.md # Schema documentation + 01-tables.sql # Example tables + 02-permissions.sql # Example permissions + catalog-info.yaml # Backstage component metadata + README.md # User documentation +``` + +### GitOps Repository Structure +``` +gitops/ + helios-app.yaml # Main CRD: defines API + database requirement + argocd-app.yaml # Points ArgoCD to this repo + kustomization.yaml # Kubernetes bundle + pipeline.yaml # Tekton PipelineRun for CI/CD + triggers.yaml # Webhook EventListener + TriggerBinding + README.md # Deployment documentation +``` + +## Complete Workflow + +1. **Scaffold via Backstage UI** + - User fills in parameters (name, Docker org, etc.) + - Template generates source + gitops repositories + +2. **Customize Schema** (source repo) + - User edits `schema/01-tables.sql` with their tables + - Edits `schema/02-permissions.sql` for roles/access control + - Commits and pushes to source repo + +3. **Automatic CI/CD** (Webhook → Tekton) + - Webhook triggers Tekton pipeline + - Builds: `docker build -t docker.io/org/repo:latest .` + - Pushes: Docker image to registry + +4. **Manual GitOps Sync** (User → ArgoCD) + - User applies: `kubectl apply -f gitops/helios-app.yaml` + - ArgoCD watches gitops repo for changes + - ArgoCD deploys HeliosApp to cluster + +5. **Automatic Operator Deployment** (ArgoCD → Helios Operator) + - Helios Operator sees HeliosApp CRD + - Creates PostgreSQL database (with schema from Docker image) + - Starts PostgREST container + - Exposes REST API via Ingress + +6. **Use REST API** + - Users interact with auto-generated endpoints + - PostgREST serves REST CRUD operations + +## Deployment + +### Prerequisites +- Kubernetes 1.28+ +- Helios Operator 0.2.0+ installed +- ArgoCD 2.8+ installed +- Docker registry access (Docker Hub username) +- Backstage instance with scaffolder plugin + +### First-Time Setup + +```bash +# After template scaffolding +cd gitops/ + +# Review manifests +cat helios-app.yaml + +# Deploy to cluster +kubectl apply -f helios-app.yaml + +# Watch deployment progress +kubectl get heliosapp -w + +# List generated endpoints +kubectl get ingress -n your-namespace +``` + +### Update Deployment + +```bash +# From source repo: push schema changes +git add schema/ +git commit -m "Add new tables" +git push + +# Tekton auto-builds and pushes image + +# From gitops repo: review and commit any changes +cd ../gitops/ +git add . +git commit -m "Update configuration" +git push + +# ArgoCD auto-syncs +``` + +## Security Considerations + +1. **JWT Secrets**: Provide secure `jwtSecret` (32+ characters, random) +2. **Database Credentials**: Not stored in manifests; injected by Operator +3. **Role-Based Access**: Configure `anonRole` and `authenticated` roles in schema +4. **Docker Registry**: Use private registries for proprietary APIs +5. **Ingress TLS**: Ensure HTTPS is configured for production + +## Troubleshooting + +### API Not Responding +```bash +# Check PostgREST logs +kubectl logs -f deployment/my-api -c api + +# Verify database connection +kubectl exec -it deployment/my-api -- \ + sh -c 'curl -v http://localhost:3000/' +``` + +### Database Not Initialized +```bash +# Check Operator logs +kubectl logs -f deployment/helios-operator + +# Check database status +kubectl get database -n your-namespace + +# Check secrets injected +kubectl get secret -n your-namespace | grep db +``` + +### CI/CD Not Triggering +```bash +# Check webhook +kubectl get eventlisteners -n default + +# Check Tekton logs +kubectl logs -f tekton-triggers-controller -n tekton-pipelines + +# Verify webhook URL in Git repo settings +# Should point to: http://el-{repoName}-listener.default.svc.cluster.local:8080 +``` + +## Files in This Template + +- **template.yaml** - Backstage template definition +- **validate.sh** - Template validation script +- **content/source/** - Source repository template +- **content/gitops/** - GitOps repository template +- **README.md** (this file) - Template documentation + +## Next Steps + +1. Register this template in Backstage (`catalog-info.yaml`) +2. Users access via Backstage UI → Create Component → PostgREST API Template +3. Follow the scaffolding flow to generate their repositories +4. See generated `README.md` in source repo for customization guide + +## References + +- [PostgREST Documentation](https://postgrest.org) +- [Helios Operator Documentation](../../../../docs/OPERATOR.md) +- [Backstage Scaffolder Docs](https://backstage.io/docs/features/software-templates) +- [ArgoCD Documentation](https://argo-cd.readthedocs.io/) +- [Tekton Pipelines](https://tekton.dev) diff --git a/apps/portal/examples/postgrest-template/template.yaml b/apps/portal/examples/postgrest-template/template.yaml index ef54a0d..f4adf66 100644 --- a/apps/portal/examples/postgrest-template/template.yaml +++ b/apps/portal/examples/postgrest-template/template.yaml @@ -17,6 +17,8 @@ spec: required: - name - port + - dockerOrg + - repoName properties: name: title: Name @@ -28,6 +30,14 @@ spec: type: number description: The port PostgREST listens on default: 3000 + dockerOrg: + title: Docker Registry Org/User + type: string + description: Your Docker Hub username or Organization + repoName: + title: Docker Repository Name + type: string + description: The name of the Docker repository (e.g. my-api) - title: PostgREST Configuration properties: @@ -95,12 +105,7 @@ spec: url: ./content/source targetPath: ./source values: - name: ${{ parameters.name }} - owner: ${{ user.entity.metadata.name or 'guest' }} - port: ${{ parameters.port }} - description: "PostgREST API: ${{ parameters.name }}" - image: postgrest/postgrest - apiSchema: ${{ parameters.apiSchema }} + name: ${{ parameters.repoName }}\n owner: ${{ user.entity.metadata.name or 'guest' }}\n port: ${{ parameters.port }}\n description: \"PostgREST API: ${{ parameters.name }}\"\n image: index.docker.io/${{ parameters.dockerOrg }}/${{ parameters.repoName }}\n apiSchema: ${{ parameters.apiSchema }} jwtSecret: ${{ parameters.jwtSecret }} jwtRole: ${{ parameters.jwtRole }} anonRole: ${{ parameters.anonRole }} @@ -132,8 +137,10 @@ spec: url: ./content/gitops targetPath: ./gitops values: - name: ${{ parameters.name }} - image: postgrest/postgrest + name: ${{ parameters.repoName }} + image: index.docker.io/${{ parameters.dockerOrg }}/${{ parameters.repoName }} + dockerOrg: ${{ parameters.dockerOrg }} + repoName: ${{ parameters.repoName }} port: ${{ parameters.port }} namespace: ${{ parameters.namespace }} databaseType: ${{ parameters.databaseConfig.dbType }} From 3ce52d2a1b7d56c7209845244e826d8f2f55d4ce Mon Sep 17 00:00:00 2001 From: Ho Phuoc Nghia Date: Sun, 12 Apr 2026 10:33:23 +0700 Subject: [PATCH 03/12] feat(postgrest-template): Remove obsolete GitOps files and update validation script --- .../postgrest-template/content/source/catalog-info.yaml | 2 +- apps/portal/examples/postgrest-template/template.yaml | 7 ++++++- apps/portal/examples/postgrest-template/validate.sh | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/apps/portal/examples/postgrest-template/content/source/catalog-info.yaml b/apps/portal/examples/postgrest-template/content/source/catalog-info.yaml index e1efe61..a4592ff 100644 --- a/apps/portal/examples/postgrest-template/content/source/catalog-info.yaml +++ b/apps/portal/examples/postgrest-template/content/source/catalog-info.yaml @@ -10,7 +10,7 @@ spec: type: service lifecycle: production owner: ${{ values.owner }} - provides: + providesApis: - name: REST API type: openapi uri: / diff --git a/apps/portal/examples/postgrest-template/template.yaml b/apps/portal/examples/postgrest-template/template.yaml index f4adf66..d13e079 100644 --- a/apps/portal/examples/postgrest-template/template.yaml +++ b/apps/portal/examples/postgrest-template/template.yaml @@ -105,7 +105,12 @@ spec: url: ./content/source targetPath: ./source values: - name: ${{ parameters.repoName }}\n owner: ${{ user.entity.metadata.name or 'guest' }}\n port: ${{ parameters.port }}\n description: \"PostgREST API: ${{ parameters.name }}\"\n image: index.docker.io/${{ parameters.dockerOrg }}/${{ parameters.repoName }}\n apiSchema: ${{ parameters.apiSchema }} + name: ${{ parameters.repoName }} + owner: ${{ user.entity.metadata.name or 'guest' }} + port: ${{ parameters.port }} + description: "PostgREST API: ${{ parameters.name }}" + image: index.docker.io/${{ parameters.dockerOrg }}/${{ parameters.repoName }} + apiSchema: ${{ parameters.apiSchema }} jwtSecret: ${{ parameters.jwtSecret }} jwtRole: ${{ parameters.jwtRole }} anonRole: ${{ parameters.anonRole }} diff --git a/apps/portal/examples/postgrest-template/validate.sh b/apps/portal/examples/postgrest-template/validate.sh index 1d335ff..83e9152 100755 --- a/apps/portal/examples/postgrest-template/validate.sh +++ b/apps/portal/examples/postgrest-template/validate.sh @@ -149,7 +149,7 @@ fi # Check that PGRST_DB_URI is referenced echo "" -echo "Test 7: Verifying PGRST_DB_URI integration..." +echo "Test 6: Verifying PGRST_DB_URI integration..." check_field "content/source/README.md" "PGRST_DB_URI" "PGRST_DB_URI documentation" check_field "content/gitops/helios-app.yaml" "database" "Database trait for credential injection" From b85b09404e5f4d3b9e2110f7e1091410db6cae13 Mon Sep 17 00:00:00 2001 From: Ho Phuoc Nghia Date: Sun, 12 Apr 2026 16:32:51 +0700 Subject: [PATCH 04/12] feat: Implement database migration trigger for PostgREST --- .../postgrest-template/content/source/catalog-info.yaml | 2 +- apps/portal/examples/postgrest-template/validate.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/portal/examples/postgrest-template/content/source/catalog-info.yaml b/apps/portal/examples/postgrest-template/content/source/catalog-info.yaml index a4592ff..e1efe61 100644 --- a/apps/portal/examples/postgrest-template/content/source/catalog-info.yaml +++ b/apps/portal/examples/postgrest-template/content/source/catalog-info.yaml @@ -10,7 +10,7 @@ spec: type: service lifecycle: production owner: ${{ values.owner }} - providesApis: + provides: - name: REST API type: openapi uri: / diff --git a/apps/portal/examples/postgrest-template/validate.sh b/apps/portal/examples/postgrest-template/validate.sh index 83e9152..1d335ff 100755 --- a/apps/portal/examples/postgrest-template/validate.sh +++ b/apps/portal/examples/postgrest-template/validate.sh @@ -149,7 +149,7 @@ fi # Check that PGRST_DB_URI is referenced echo "" -echo "Test 6: Verifying PGRST_DB_URI integration..." +echo "Test 7: Verifying PGRST_DB_URI integration..." check_field "content/source/README.md" "PGRST_DB_URI" "PGRST_DB_URI documentation" check_field "content/gitops/helios-app.yaml" "database" "Database trait for credential injection" From 1aedf7639eff0b66e0cb0d1898e1b65991e7ad9e Mon Sep 17 00:00:00 2001 From: Ho Phuoc Nghia Date: Wed, 15 Apr 2026 22:55:53 +0700 Subject: [PATCH 05/12] temp: enhance PostgREST template with default values and new actions for Kubernetes secret management --- .../internal/controller/tekton/pipelinerun.go | 13 ++- .../content/gitops/helios-app.yaml | 46 +++++----- .../content/source/postgrestrc.conf | 2 +- .../examples/postgrest-template/template.yaml | 23 ++++- .../examples/postgrest-template/validate.sh | 11 +-- .../backend/src/actions/kubernetes-apply.ts | 46 ++++++++-- .../src/actions/kubernetes-create-secret.ts | 84 +++++++++++++++++++ .../backend/src/extensions/scaffolder.ts | 2 + cue/definitions/tekton/tasks/git-update.cue | 38 ++++++--- 9 files changed, 207 insertions(+), 58 deletions(-) create mode 100644 apps/portal/packages/backend/src/actions/kubernetes-create-secret.ts diff --git a/apps/operator/internal/controller/tekton/pipelinerun.go b/apps/operator/internal/controller/tekton/pipelinerun.go index b3590ac..9d73af2 100644 --- a/apps/operator/internal/controller/tekton/pipelinerun.go +++ b/apps/operator/internal/controller/tekton/pipelinerun.go @@ -26,6 +26,15 @@ func GeneratePipelineRun(heliosApp *appv1alpha1.HeliosApp, pipelineName string) gitOpsSecretRef := cmp.Or(heliosApp.Spec.GitOpsSecretRef, "helios-gitops-bot") argoNS := cmp.Or(heliosApp.Spec.ArgoCDNamespace, "argocd") + replicas := heliosApp.Spec.Replicas + if replicas <= 0 { + replicas = 1 + } + port := heliosApp.Spec.Port + if port <= 0 || port > 65535 { + port = 8080 + } + params := make([]any, 0, 17) params = append(params, map[string]any{"name": "app-repo-url", "value": shared.RewriteGiteaURL(heliosApp.Spec.GitRepo)}, @@ -38,8 +47,8 @@ func GeneratePipelineRun(heliosApp *appv1alpha1.HeliosApp, pipelineName string) map[string]any{"name": "GITOPS_AUTHOR_NAME", "value": "Helios Bot"}, map[string]any{"name": "GITOPS_AUTHOR_EMAIL", "value": "helios-bot@helios.local"}, map[string]any{"name": "CONTEXT_SUBPATH", "value": contextSubpath}, - map[string]any{"name": "replicas", "value": fmt.Sprintf("%d", heliosApp.Spec.Replicas)}, - map[string]any{"name": "port", "value": fmt.Sprintf("%d", heliosApp.Spec.Port)}, + map[string]any{"name": "replicas", "value": fmt.Sprintf("%d", replicas)}, + map[string]any{"name": "port", "value": fmt.Sprintf("%d", port)}, map[string]any{"name": "test-command", "value": heliosApp.Spec.TestCommand}, map[string]any{"name": "argocd-namespace", "value": argoNS}, map[string]any{"name": "argocd-app-name", "value": heliosApp.Name + "-argocd"}, diff --git a/apps/portal/examples/postgrest-template/content/gitops/helios-app.yaml b/apps/portal/examples/postgrest-template/content/gitops/helios-app.yaml index b858168..0674fbc 100644 --- a/apps/portal/examples/postgrest-template/content/gitops/helios-app.yaml +++ b/apps/portal/examples/postgrest-template/content/gitops/helios-app.yaml @@ -1,34 +1,32 @@ -apiVersion: helios.io/v1alpha1 +apiVersion: app.helios.io/v1alpha1 kind: HeliosApp metadata: name: ${{ values.name }} namespace: ${{ values.namespace }} spec: - # Database migrations trigger: automatically triggers db-migrate pipeline - # when changes are made to db/migrations/ directory - triggerType: db-migrate - - git: - repo: ${{ values.sourceRepo }} - path: ./ - ref: main + # Owner and Git Configuration + owner: ${{ values.owner }} + gitRepo: ${{ values.sourceRepo }} + gitBranch: main + imageRepo: ${{ values.image }} + gitopsRepo: ${{ values.gitopsRepo }} + gitopsPath: ${{ values.name }} + webhookSecret: ${{ values.name }}-webhook-secret + port: ${{ values.port }} + replicas: 1 components: - name: api type: web-service - source: - image: ${{ values.image }}:latest - container: + properties: + # Use official PostgREST image from Docker Hub + image: index.docker.io/postgrest/postgrest:latest port: ${{ values.port }} env: # PostgREST Configuration # PGRST_DB_URI format: postgres://user:password@host:port/database - # Operator injects DB_USER, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME from database trait - - name: PGRST_DB_URI - valueFrom: - secretKeyRef: - name: ${{ values.name }}-db - key: PGRST_DB_URI + # The Operator automatically injects DB_HOST, DB_USER, DB_PASS, DB_PORT, DB_NAME from the database trait + # The native operator Phase 0.9 reconciliation builds the PGRST_DB_URI from these and injects it - name: PGRST_DB_SCHEMA value: "${{ values.apiSchema }}" - name: PGRST_DB_ANON_ROLE @@ -40,13 +38,15 @@ spec: - name: PGRST_MAX_ROWS value: "1000" - name: PGRST_LOG_LEVEL - value: notice + value: info traits: - type: database properties: dbType: postgres dbName: ${{ values.databaseName or values.name }}-db port: 5432 + version: "${{ values.databaseVersion or '16' }}" + storage: "1Gi" - type: service properties: port: ${{ values.port }} @@ -55,11 +55,3 @@ spec: enabled: true host: ${{ values.name }}.local path: / - - # ArgoCD will sync the deployment - argocd: - enabled: true - syncPolicy: - automated: - prune: true - selfHeal: true diff --git a/apps/portal/examples/postgrest-template/content/source/postgrestrc.conf b/apps/portal/examples/postgrest-template/content/source/postgrestrc.conf index f80523b..c7d0042 100644 --- a/apps/portal/examples/postgrest-template/content/source/postgrestrc.conf +++ b/apps/portal/examples/postgrest-template/content/source/postgrestrc.conf @@ -26,7 +26,7 @@ jwt-aud = "${{ values.jwtRole }}" max-rows = 1000 # Logging -log-level = notice +log-level = info # Connection pool settings (optional) db-pool = 10 diff --git a/apps/portal/examples/postgrest-template/template.yaml b/apps/portal/examples/postgrest-template/template.yaml index d13e079..489db41 100644 --- a/apps/portal/examples/postgrest-template/template.yaml +++ b/apps/portal/examples/postgrest-template/template.yaml @@ -150,6 +150,7 @@ spec: namespace: ${{ parameters.namespace }} databaseType: ${{ parameters.databaseConfig.dbType }} databaseName: ${{ parameters.databaseConfig.dbName }} + databaseVersion: ${{ parameters.databaseConfig.version or '16' }} apiSchema: ${{ parameters.apiSchema }} jwtSecret: ${{ parameters.jwtSecret }} jwtRole: ${{ parameters.jwtRole }} @@ -167,7 +168,27 @@ spec: sourcePath: ./gitops repoVisibility: public - # 3. Registration + # 3. Create webhook secret in cluster + - id: create-webhook-secret + name: Create Kubernetes Webhook Secret + action: kubernetes:create-secret + input: + namespace: ${{ parameters.namespace }} + secretName: ${{ parameters.repoName }}-webhook-secret + data: + # Tekton Triggers interceptor expects the shared secret under key `secret`. + secret: ${{ parameters.repoName }} + # Keep `secretToken` for compatibility with any legacy consumers. + secretToken: ${{ parameters.repoName }} + + # 4. Deploy HeliosApp to cluster + - id: apply-heliosapp + name: Apply HeliosApp to Cluster + action: kubernetes:apply + input: + manifestPath: ./gitops/helios-app.yaml + + # 5. Registration - id: register name: Register Component action: catalog:register diff --git a/apps/portal/examples/postgrest-template/validate.sh b/apps/portal/examples/postgrest-template/validate.sh index 1d335ff..1fb964d 100755 --- a/apps/portal/examples/postgrest-template/validate.sh +++ b/apps/portal/examples/postgrest-template/validate.sh @@ -49,17 +49,14 @@ if ! python3 -c "import yaml" 2>/dev/null; then python3 -m pip install -q pyyaml || { echo " ✗ Failed to install PyYAML"; exit 1; } fi -python3 << 'PYTHON_EOF' +TEMPLATE_DIR="$TEMPLATE_DIR" python3 << 'PYTHON_EOF' import yaml import os import sys -# Use the directory from which this script is run -template_dir = os.getcwd() -if not os.path.exists(os.path.join(template_dir, 'template.yaml')): - # If not in template dir, try to find it - script_dir = os.path.dirname(os.path.realpath(__file__)) - template_dir = script_dir +template_dir = os.environ.get('TEMPLATE_DIR') +if not template_dir: + template_dir = os.getcwd() yaml_files = [ 'template.yaml', diff --git a/apps/portal/packages/backend/src/actions/kubernetes-apply.ts b/apps/portal/packages/backend/src/actions/kubernetes-apply.ts index 94abe6c..4672b6c 100644 --- a/apps/portal/packages/backend/src/actions/kubernetes-apply.ts +++ b/apps/portal/packages/backend/src/actions/kubernetes-apply.ts @@ -10,11 +10,19 @@ export const createKubernetesApplyAction = () => { description: 'Applies a Kubernetes manifest file using kubectl', schema: { input: z => - z.object({ - manifestPath: z + z + .object({ + manifestPath: z + .string() + .optional() + .describe( + 'Path to the manifest file to apply, relative to the workspace', + ), + resource: z .string() + .optional() .describe( - 'Path to the manifest file to apply, relative to the workspace', + 'Alias for manifestPath (kept for backwards compatibility with older templates)', ), namespace: z .string() @@ -24,16 +32,33 @@ export const createKubernetesApplyAction = () => { .boolean() .optional() .describe('Whether the resources are namespaced'), - }), + values: z + .record(z.any()) + .optional() + .describe( + 'Optional values (ignored by this action; use fetch:template to render manifests)', + ), + }) + .refine(v => Boolean(v.manifestPath || v.resource), { + message: 'manifestPath (or resource) is required', + }), }, async handler(ctx) { - const { manifestPath, namespace } = ctx.input; + const { manifestPath, resource, namespace, values } = ctx.input; - if (!manifestPath) { - throw new InputError('manifestPath is required'); + const effectiveManifestPath = manifestPath ?? resource; + + if (!effectiveManifestPath) { + throw new InputError('manifestPath (or resource) is required'); + } + + if (values && Object.keys(values).length > 0) { + ctx.logger.warn( + 'kubernetes:apply received input.values but does not perform templating; ensure manifests are rendered by fetch:template before applying', + ); } - const args = ['apply', '-f', manifestPath]; + const args = ['apply', '-f', effectiveManifestPath]; if (namespace) { args.push('-n', namespace); } @@ -49,7 +74,10 @@ export const createKubernetesApplyAction = () => { }, }); - ctx.logger.info(`Successfully applied manifest: ${manifestPath}`); + ctx.logger.info( + `Successfully applied manifest: ${effectiveManifestPath}`, + ); }, }); }; + diff --git a/apps/portal/packages/backend/src/actions/kubernetes-create-secret.ts b/apps/portal/packages/backend/src/actions/kubernetes-create-secret.ts new file mode 100644 index 0000000..62104a9 --- /dev/null +++ b/apps/portal/packages/backend/src/actions/kubernetes-create-secret.ts @@ -0,0 +1,84 @@ +import { + createTemplateAction, + executeShellCommand, +} from '@backstage/plugin-scaffolder-node'; +import { InputError } from '@backstage/errors'; + +export const createKubernetesCreateSecretAction = () => { + return createTemplateAction({ + id: 'kubernetes:create-secret', + description: 'Creates (or replaces) a Kubernetes Secret using kubectl', + schema: { + input: z => + z.object({ + namespace: z + .string() + .optional() + .describe('Kubernetes namespace (defaults to "default")'), + secretName: z.string().describe('Name of the Secret to create'), + type: z + .string() + .optional() + .describe('Optional Secret type (defaults to Opaque)'), + data: z + .record(z.string()) + .describe( + 'Key/value pairs to populate the Secret (values are treated as stringData)', + ), + }), + }, + async handler(ctx) { + const { namespace = 'default', secretName, type, data } = ctx.input; + + if (!secretName) { + throw new InputError('secretName is required'); + } + if (!data || Object.keys(data).length === 0) { + throw new InputError('data must contain at least one key/value pair'); + } + + ctx.logger.info( + `Creating secret ${secretName} in namespace ${namespace} with ${Object.keys(data).length} keys`, + ); + + // Delete existing secret if it exists (ignore errors) + try { + await executeShellCommand({ + command: 'kubectl', + args: [ + 'delete', + 'secret', + secretName, + '-n', + namespace, + '--ignore-not-found', + ], + logger: ctx.logger, + }); + } catch { + // ignore + } + + const args = ['create', 'secret', 'generic', secretName, '-n', namespace]; + + if (type) { + args.push(`--type=${type}`); + } + + for (const [key, value] of Object.entries(data)) { + if (!key) { + throw new InputError('Secret data keys must be non-empty'); + } + args.push(`--from-literal=${key}=${value}`); + } + + await executeShellCommand({ + command: 'kubectl', + args, + logger: ctx.logger, + }); + + ctx.logger.info(`Successfully created secret: ${secretName}`); + }, + }); +}; diff --git a/apps/portal/packages/backend/src/extensions/scaffolder.ts b/apps/portal/packages/backend/src/extensions/scaffolder.ts index 84deaa3..8113ca8 100644 --- a/apps/portal/packages/backend/src/extensions/scaffolder.ts +++ b/apps/portal/packages/backend/src/extensions/scaffolder.ts @@ -4,6 +4,7 @@ import { coreServices } from '@backstage/backend-plugin-api'; import { createKubernetesApplyAction } from '../actions/kubernetes-apply'; import { createGitCredentialsSecretAction } from '../actions/create-git-credentials-secret'; import { createGiteaWebhookAction } from '../actions/create-gitea-webhook'; +import { createKubernetesCreateSecretAction } from '../actions/kubernetes-create-secret'; export const scaffolderModuleCustomActions = createBackendModule({ pluginId: 'scaffolder', @@ -16,6 +17,7 @@ export const scaffolderModuleCustomActions = createBackendModule({ }, async init({ scaffolder, config }) { scaffolder.addActions(createKubernetesApplyAction() as any); + scaffolder.addActions(createKubernetesCreateSecretAction() as any); scaffolder.addActions(createGitCredentialsSecretAction() as any); scaffolder.addActions(createGiteaWebhookAction({ config }) as any); }, diff --git a/cue/definitions/tekton/tasks/git-update.cue b/cue/definitions/tekton/tasks/git-update.cue index 91168c5..f8e2731 100644 --- a/cue/definitions/tekton/tasks/git-update.cue +++ b/cue/definitions/tekton/tasks/git-update.cue @@ -69,23 +69,38 @@ import "helios.io/cue/definitions/tekton" export IMAGE_URL="$(params.NEW_IMAGE_URL)" export REPLICAS="$(params.REPLICAS)" export PORT="$(params.PORT)" + + # Defensive defaults: avoid generating invalid manifests when inputs are empty/0 + if [ -z "${REPLICAS}" ] || [ "${REPLICAS}" = "0" ]; then + export REPLICAS="1" + fi + if [ -z "${PORT}" ] || [ "${PORT}" = "0" ]; then + export PORT="8080" + fi MANIFEST_PATH="$(params.MANIFEST_PATH)" # Logic tạo file tự động if echo "$MANIFEST_PATH" | grep -qvE '\\.ya?ml$'; then echo "Path '$MANIFEST_PATH' treated as DIRECTORY." mkdir -p "$MANIFEST_PATH" - - DEP_FILE="$MANIFEST_PATH/deployment.yaml" - SVC_FILE="$MANIFEST_PATH/service.yaml" - MANIFEST_FILES="$DEP_FILE $SVC_FILE" - APP_NAME=$(basename "$MANIFEST_PATH") - - if [ ! -f "$DEP_FILE" ]; then - echo "Creating default manifests..." - printf "apiVersion: apps/v1\\nkind: Deployment\\nmetadata:\\n name: ${APP_NAME}\\nspec:\\n replicas: ${REPLICAS}\\n selector:\\n matchLabels:\\n app: ${APP_NAME}\\n template:\\n metadata:\\n labels:\\n app: ${APP_NAME}\\n spec:\\n containers:\\n - name: app\\n image: ${IMAGE_URL}\\n ports:\\n - containerPort: ${PORT}\\n" > "$DEP_FILE" - - printf "apiVersion: v1\\nkind: Service\\nmetadata:\\n name: ${APP_NAME}\\nspec:\\n selector:\\n app: ${APP_NAME}\\n ports:\\n - protocol: TCP\\n port: ${PORT}\\n targetPort: ${PORT}\\n type: ClusterIP\\n" > "$SVC_FILE" + + # If the operator already renders a combined manifest.yaml in this directory, + # prefer updating that file rather than creating separate default manifests. + COMBINED_FILE="$MANIFEST_PATH/manifest.yaml" + if [ -f "$COMBINED_FILE" ]; then + MANIFEST_FILES="$COMBINED_FILE" + else + DEP_FILE="$MANIFEST_PATH/deployment.yaml" + SVC_FILE="$MANIFEST_PATH/service.yaml" + MANIFEST_FILES="$DEP_FILE $SVC_FILE" + APP_NAME=$(basename "$MANIFEST_PATH") + + if [ ! -f "$DEP_FILE" ]; then + echo "Creating default manifests..." + printf "apiVersion: apps/v1\\nkind: Deployment\\nmetadata:\\n name: ${APP_NAME}\\nspec:\\n replicas: ${REPLICAS}\\n selector:\\n matchLabels:\\n app: ${APP_NAME}\\n template:\\n metadata:\\n labels:\\n app: ${APP_NAME}\\n spec:\\n containers:\\n - name: app\\n image: ${IMAGE_URL}\\n ports:\\n - containerPort: ${PORT}\\n" > "$DEP_FILE" + + printf "apiVersion: v1\\nkind: Service\\nmetadata:\\n name: ${APP_NAME}\\nspec:\\n selector:\\n app: ${APP_NAME}\\n ports:\\n - protocol: TCP\\n port: ${PORT}\\n targetPort: ${PORT}\\n type: ClusterIP\\n" > "$SVC_FILE" + fi fi else echo "Path '$MANIFEST_PATH' treated as FILE." @@ -106,6 +121,7 @@ import "helios.io/cue/definitions/tekton" yq -i 'select(.kind == "Deployment") .spec.template.spec.containers[].image = env(IMAGE_URL)' "$FILE" yq -i 'select(.kind == "Deployment") .spec.replicas = env(REPLICAS)' "$FILE" yq -i 'select(.kind == "Deployment") .spec.template.spec.containers[].ports[0].containerPort = env(PORT)' "$FILE" + yq -i 'select(.kind == "Service") .spec.ports[0].port = env(PORT)' "$FILE" yq -i 'select(.kind == "Service") .spec.ports[0].targetPort = env(PORT)' "$FILE" fi done From a0db3a4c3be4a97ff17dc862b1dc801f9bba5498 Mon Sep 17 00:00:00 2001 From: Ho Phuoc Nghia Date: Thu, 16 Apr 2026 22:44:46 +0700 Subject: [PATCH 06/12] Add initial migration --- db/migration/001_initial.sql | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 db/migration/001_initial.sql diff --git a/db/migration/001_initial.sql b/db/migration/001_initial.sql new file mode 100644 index 0000000..f7e8490 --- /dev/null +++ b/db/migration/001_initial.sql @@ -0,0 +1,5 @@ +-- Initial schema +CREATE TABLE IF NOT EXISTS users ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL +); From edd5a5a949e431fa0f145b8a7a923d4dc6d17951 Mon Sep 17 00:00:00 2001 From: Ho Phuoc Nghia Date: Thu, 16 Apr 2026 23:46:25 +0700 Subject: [PATCH 07/12] feat: Enhance database migration support and improve webhook handling - Added `triggerType: db-migrate` to HeliosApp configuration for better migration handling. - Introduced a new Kubernetes namespace manifest for better resource management. - Updated PostgREST configuration to ensure log levels are consistently quoted. - Modified webhook creation to specify a dedicated listener for DB migration triggers. - Reorganized Tekton pipeline definitions to include a dedicated DB migration pipeline. - Updated Git clone task to transform localhost URLs for in-cluster access. - Enhanced database migration task to use Kubernetes secrets for database credentials. - Improved PostgREST reload task to utilize dynamic secret names for better flexibility. - Refined event listener to trigger DB migration pipelines based on specific directory changes. - Added new trigger template for DB migration to streamline pipeline execution on relevant events. --- .../internal/controller/database/resources.go | 4 +- .../controller/database/resources_test.go | 2 +- .../examples/postgrest-template/README.md | 698 ++++++++++++++---- .../content/gitops/helios-app.yaml | 3 +- .../content/gitops/namespace.yaml | 6 + .../content/source/postgrestrc.conf | 2 +- .../examples/postgrest-template/template.yaml | 17 +- .../tekton/pipelines/db-migrate.cue | 9 +- cue/definitions/tekton/tasks/db-migrate.cue | 126 ++-- cue/definitions/tekton/tasks/git-clone.cue | 17 +- .../tekton/tasks/postgrest-reload.cue | 99 +-- .../tekton/triggers/db-migrate-trigger.cue | 38 +- .../tekton/triggers/github-push.cue | 115 ++- cue/engine/tekton_builder.cue | 17 +- 14 files changed, 807 insertions(+), 346 deletions(-) create mode 100644 apps/portal/examples/postgrest-template/content/gitops/namespace.yaml diff --git a/apps/operator/internal/controller/database/resources.go b/apps/operator/internal/controller/database/resources.go index b36a1ce..8ac0a18 100644 --- a/apps/operator/internal/controller/database/resources.go +++ b/apps/operator/internal/controller/database/resources.go @@ -16,13 +16,13 @@ var requiredDatabaseSecretKeys = []string{"DB_USER", "DB_PASS", "DB_HOST"} // formatPostgresURI constructs a PostgreSQL connection URI from components. // It properly escapes the username and password for use in URLs. -// Format: postgres://username:password@host:port/dbname +// Format: postgres://username:password@host:port/dbname?sslmode=disable func formatPostgresURI(username, password, host, dbName string, port int32) string { // Escape username and password for use in URL enscodedUser := url.QueryEscape(username) enscodedPassword := url.QueryEscape(password) - return fmt.Sprintf("postgres://%s:%s@%s:%d/%s", + return fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=disable", enscodedUser, enscodedPassword, host, diff --git a/apps/operator/internal/controller/database/resources_test.go b/apps/operator/internal/controller/database/resources_test.go index a5f63c0..e0f3391 100644 --- a/apps/operator/internal/controller/database/resources_test.go +++ b/apps/operator/internal/controller/database/resources_test.go @@ -126,7 +126,7 @@ func TestGenerateDatabaseSecret(t *testing.T) { } // Check that PGRST_DB_URI is generated and properly escaped - expectedURI := "postgres://testuser:testpassword123@my-app-db:5432/my-app-db" + expectedURI := "postgres://testuser:testpassword123@my-app-db:5432/my-app-db?sslmode=disable" if string(secret.Data["PGRST_DB_URI"]) != expectedURI { t.Errorf("Expected PGRST_DB_URI %q, got %q", expectedURI, string(secret.Data["PGRST_DB_URI"])) } diff --git a/apps/portal/examples/postgrest-template/README.md b/apps/portal/examples/postgrest-template/README.md index 2eed44b..2c81199 100644 --- a/apps/portal/examples/postgrest-template/README.md +++ b/apps/portal/examples/postgrest-template/README.md @@ -1,244 +1,588 @@ # PostgREST Backstage Template -A complete Backstage software template for scaffolding PostgREST instant REST APIs with automatic database provisioning via the Helios Operator. +A complete Backstage software template for scaffolding PostgREST instant REST APIs with automatic database provisioning, zero-downtime migrations, and full CI/CD via the Helios Platform. ## Overview This template creates: -- **Source Repository**: Dockerfile, SQL schema, PostgREST configuration +- **Source Repository**: Dockerfile, SQL schema, PostgREST configuration, migration files - **GitOps Repository**: Kubernetes manifests with HeliosApp CRD, Tekton CI/CD, ArgoCD integration +- **Automatic Webhook Integration**: Triggers CI/CD pipelines on code changes ## Key Features -✅ **Auto-Generated REST API**: PostgREST creates CRUD endpoints from your database schema -✅ **Automatic Database Provisioning**: Helios Operator provision PostgreSQL on Kubernetes -✅ **CI/CD Pipeline**: Tekton builds custom Docker image with your schema -✅ **GitOps Deployment**: ArgoCD syncs from separate GitOps repository -✅ **JWT Authentication**: Built-in JWT support for securing your API +✅ **Auto-Generated REST API**: PostgREST creates CRUD endpoints directly from your database schema +✅ **Automatic Database Provisioning**: Helios Operator manages PostgreSQL databases on Kubernetes +✅ **Zero-Downtime Migrations**: Push migration files to `db/migration/` → automatic execution without redeployment +✅ **CI/CD Pipeline**: Tekton builds custom Docker image with your application code +✅ **GitOps Deployment**: ArgoCD syncs manifests from separate GitOps repository +✅ **JWT Authentication**: Built-in JWT token support for API security ✅ **Role-Based Access Control**: Database-level permissions with Postgres roles +✅ **Automated Webhooks**: Git webhooks trigger pipelines on every push ## Architecture ``` Backstage Template ↓ -Creates Two Repos: - ├── Source Repo (Dockerfile + schema + config) - │ ↓ - │ Tekton CI/CD - │ ↓ - │ Docker Image → docker.io/org/repo:latest +Creates Two Repos + Webhook Integration: + ├── Source Repo (Dockerfile + schema + migrations) + │ ↓ (on every push) + │ Tekton CI/CD Pipeline + │ ├─→ Build: Docker image with your code + │ ├─→ Test: Run any tests + │ └─→ Push: docker.io/org/repo:latest │ - └── GitOps Repo (Kubernetes manifests) - ↓ - ArgoCD - ↓ - Helios Operator - ├── PostgreSQL Database - ├── PostgREST Container - └── Ingress for REST API + ├── GitOps Repo (Kubernetes manifests) + │ ↓ (on every push) + │ ArgoCD Sync + │ └─→ Update deployments + │ + └── Database Migrations (automatic trigger) + ↓ (when db/migration/*.sql files change) + Tekton db-migrate Pipeline + ├─→ Clone repo at commit + ├─→ Run: golang-migrate up + └─→ Reload: PostgREST schema + (NO downtime, NO redeployment) ``` -## Requirements Met ✅ - -### Requirement 1: Scaffolding Only (No K8s Deployment) -- Template generates manifests but does NOT apply them -- Users manually deploy when ready via `kubectl apply -f gitops/` -- Reduces risk of accidental deployments in the wrong cluster - -### Requirement 2: Use Official PostgREST Image -- Dockerfile: `FROM postgrest/postgrest:v12.2.0` (official image) -- Users customize by copying their SQL schema into the image -- Results in: `Dockerfile` → `docker build` → `docker.io/org/repo:latest` custom image -- This custom image still uses official PostgREST as its base, but includes user's schema - -### Requirement 3: Parameterize Namespace -- Template includes `namespace` parameter (required in template.yaml) -- Dynamically injected into ALL Kubernetes manifests: - - `helios-app.yaml`: `metadata.namespace: ${{ parameters.namespace }}` - - `argocd-app.yaml`: `metadata.namespace: ${{ parameters.namespace }}` - - `kustomization.yaml`: All resources in same namespace - - `pipeline.yaml`: `metadata.namespace: ${{ parameters.namespace }}` - - `triggers.yaml`: `metadata.namespace: ${{ parameters.namespace }}` - -### Requirement 4: Support Custom Images with Registry -- Template includes Docker registry parameters: `dockerOrg`, `repoName` -- Dockerfile builds custom image FROM official postgrest/postgrest -- Tekton pipeline (in gitops repo) builds and pushes to registry -- GitOps manifests reference: `docker.io/${{ dockerOrg }}/${{ repoName }}:latest` -- Enables same flexibility as Node.js templates: users build their own images +## Database Migrations (The Game Changer) + +Instead of rebuilding and redeploying your container for database changes: + +**Old way** (❌ Slow, downtime): +``` +Edit schema → Commit → Build Docker image → Push → Redeploy → Downtime +``` + +**Helios way** (✅ Fast, zero downtime): +``` +Create migration file → Push → Automatic execution → Done +``` + +### How It Works + +1. Create migration files in `db/migration/` folder: + ```bash + db/migration/ + 001_initial.up.sql # Creates tables + 001_initial.down.sql # Rollback script + 002_add_indexes.up.sql # Add indexes (does not recreate data) + 002_add_indexes.down.sql # Rollback + ``` + +2. Push to source repository: + ```bash + git add db/migration/ + git commit -m "Add user profiles table" + git push origin main + ``` + +3. **Automatic Pipeline Trigger** 🚀 + - Helios Operator creates a webhook on your repo + - CEL filter triggers ONLY on migration path changes + - db-migrate pipeline: + - Clones repository + - Runs `golang-migrate up` + - Reloads PostgREST schema cache via `NOTIFY` command + - **API continues serving requests** ✨ + +4. Done! Your database is updated, PostgREST knows about it. + +### Benefits + +- **Zero downtime**: Migrations run in background, API never stops +- **Database-first development**: Evolve schema independently from code +- **Fast iterations**: Deploy schema changes in seconds, not minutes +- **Safe rollbacks**: Each migration has `down.sql` for instant rollback +- **Audit trail**: All migrations tracked in Git with timestamps + +## Design Philosophy + +This template follows three core principles: + +### 1. **GitOps-First** +- All infrastructure is version-controlled +- Changes are Pull Request → Review → Merge → Auto-deploy +- Rollback is just `git revert` + +### 2. **Zero-Downtime Operations** +- Database migrations don't require redeployment +- Code updates use rolling deployments +- API continues serving while schema evolves + +### 3. **Developer Experience** +- One command: `git push` → Everything happens automatically +- No manual kubectl commands needed after initial setup +- Clear error messages on failures +- Audit trail of all changes ## Template Parameters -### Component Information (Required) -| Parameter | Description | Example | -|-----------|-------------|---------| -| `name` | Service name | `my-api` | -| `port` | PostgREST listen port | `3000` | -| `dockerOrg` | Docker registry org/user | `mycompany` | -| `repoName` | Docker repository name | `my-api` | - -### PostgREST Configuration (Optional) -| Parameter | Description | Default | -|-----------|-------------|---------| -| `apiSchema` | PostgreSQL schema to expose | `public` | -| `jwtSecret` | JWT signing secret (32+ chars) | - | -| `jwtRole` | Default JWT role claim | `authenticated` | -| `anonRole` | Role for unauthenticated requests | `anon` | - -### Infrastructure (Optional) -| Parameter | Description | -|-----------|-------------| -| `namespace` | Kubernetes namespace | (required, input per cluster) | -| `databaseConfig` | Database type, name, etc. | (picker UI) | -| `repoUrl` | Git repository URL | (picker UI) | +When scaffolding via Backstage, you'll provide: + +| Parameter | Required? | Description | Example | +|-----------|-----------|-------------|---------| +| **Component Name** | ✅ | Service display name | `My Awesome API` | +| **Repository Name** | ✅ | Git repo name | `my-awesome-api` | +| **Docker Org** | ✅ | Docker Hub username/org | `mycompany` | +| **API Port** | ✅ | PostgREST listen port | `3000` | +| **Namespace** | ✅ | Kubernetes namespace | `production` | +| **Database Name** | ✅ | PostgreSQL database | `my-awesome-api-db` | +| **JWT Secret** | ⚠️ | For token signing (32+ chars) | (random) | +| **JWT Role** | Optional | Default role for tokens | `authenticated` | +| **Anon Role** | Optional | Unauthenticated role | `anon` | +| **API Schema** | Optional | PostgreSQL schema to expose | `public` | + +**Security tip:** Generate a random JWT secret: `openssl rand -base64 32` ## What Gets Generated -### Source Repository Structure +### Source Repository + +Your application code + schema: + +``` +source-repo/ + ├── Dockerfile # Builds custom image FROM postgrest/postgrest + ├── postgrestrc.conf # PostgREST configuration (port, logging, etc) + ├── schema/ + │ ├── 01-tables.sql # Table definitions (edit this!) + │ ├── 02-permissions.sql # Role permissions (edit this!) + │ └── README.md # Schema documentation + ├── db/ + │ └── migration/ # Database migrations (see below) + │ ├── 001_initial.up.sql + │ └── 001_initial.down.sql + ├── catalog-info.yaml # Backstage component metadata + ├── README.md # User documentation + └── .git/ # Git repository +``` + +**What you edit:** +- `schema/01-tables.sql` - Your database design +- `schema/02-permissions.sql` - Access control rules +- `db/migration/*.sql` - Zero-downtime schema updates +- `postgrestrc.conf` - PostgREST behavior +- `Dockerfile` - Build configuration (rarely needed) + +**How it works:** ``` -source/ - Dockerfile # Builds custom image from postgrest/postgrest - postgrestrc.conf # PostgREST configuration - schema/ # User's database schema - README.md # Schema documentation - 01-tables.sql # Example tables - 02-permissions.sql # Example permissions - catalog-info.yaml # Backstage component metadata - README.md # User documentation +Push to main branch + ↓ (webhook) +Tekton builds Docker image + ↓ +Image includes: postgrest + your schema + your migrations + ↓ +Push to docker.io//:latest + ↓ +ArgoCD detects new image + ↓ +Kubernetes restarts PostgREST pod (rolling deployment) + ↓ +Old requests drain, new requests use new schema ``` -### GitOps Repository Structure +### GitOps Repository + +Infrastructure declarations: + ``` -gitops/ - helios-app.yaml # Main CRD: defines API + database requirement - argocd-app.yaml # Points ArgoCD to this repo - kustomization.yaml # Kubernetes bundle - pipeline.yaml # Tekton PipelineRun for CI/CD - triggers.yaml # Webhook EventListener + TriggerBinding - README.md # Deployment documentation +gitops-repo/ + ├── helios-app.yaml # Main application definition + │ # (database + postgrest + webhook settings) + ├── namespace.yaml # Kubernetes namespace + ├── kustomization.yaml # Bundle all manifests + ├── tekton/ + │ ├── eventlistener.yaml # Webhook configuration + │ ├── triggerbinding.yaml # Extract params from webhook + │ └── triggertemplate.yaml # Define what to run + └── README.md # Deployment documentation ``` +**How webhooks work:** + +1. Source repo has webhook pointing to: `http://el--listener..svc.cluster.local:8080` +2. Every git push sends a JSON payload to that endpoint +3. EventListener validates signature + filters by branch/path +4. Matching commits trigger a PipelineRun +5. Pipeline executes in Kubernetes (build, test, push) + +**db-migrate webhook is separate:** + +1. db-migrate EventListener listens for changes to `db/migration/` folder only +2. Different trigger than the main CI/CD webhook +3. Only executes golang-migrate, doesn't rebuild image +4. Runs in <10 seconds (no container rebuild) + ## Complete Workflow +### Initial Setup (One-Time) + 1. **Scaffold via Backstage UI** - - User fills in parameters (name, Docker org, etc.) - - Template generates source + gitops repositories + ``` + Backstage → Create Component → PostgREST API Template + Fill in: name, registry, namespace, database settings + Template creates: source + gitops repos with webhooks + ``` + +2. **Deploy to Kubernetes** (Manual, one-time) + ```bash + cd gitops/ + kubectl apply -f helios-app.yaml + # Helios Operator creates PostgreSQL database + PostgREST container + ``` + +3. **Get your API endpoint** + ```bash + kubectl get ingress -n + # Your REST API is now live! + ``` + +### Day-to-Day Operations + +#### Add Features (Code Changes) +```bash +# 1. Edit your code in source repo +nano postgrestrc.conf +vim schema/01-tables.sql -2. **Customize Schema** (source repo) - - User edits `schema/01-tables.sql` with their tables - - Edits `schema/02-permissions.sql` for roles/access control - - Commits and pushes to source repo +# 2. Push changes +git add . +git commit -m "Add profile schema" +git push origin main + +# 3. Automatic CI/CD +# Webhook triggers Tekton: +# ✓ Builds Docker image +# ✓ Runs tests +# ✓ Pushes to registry +# +# GitOps repo auto-updates with new image version +# ArgoCD syncs → PostgREST pod restarts with new code +``` -3. **Automatic CI/CD** (Webhook → Tekton) - - Webhook triggers Tekton pipeline - - Builds: `docker build -t docker.io/org/repo:latest .` - - Pushes: Docker image to registry +#### Migrate Database (Zero Downtime) +```bash +# 1. Create migration files +mkdir -p db/migration +cat > db/migration/003_add_users.up.sql << 'EOF' +CREATE TABLE users ( + id SERIAL PRIMARY KEY, + email TEXT UNIQUE NOT NULL, + created_at TIMESTAMP DEFAULT NOW() +); +EOF + +cat > db/migration/003_add_users.down.sql << 'EOF' +DROP TABLE IF EXISTS users; +EOF + +# 2. Push migration files +git add db/migration/ +git commit -m "Add users table" +git push origin main + +# 3. Automatic db-migrate pipeline +# (You don't do anything else!) +# +# Webhook triggers db-migrate pipeline: +# ✓ Clones repository +# ✓ Runs golang-migrate +# ✓ Updates schema cache +# ✓ API continues working +# +# kubectl logs -f deployment/el-{name}-db-migrate-listener +# to watch the webhook trigger + +# Done! Your new endpoint is live: +# GET /users +# POST /users (insert) +# PATCH /users (update) +# DELETE /users (delete) +``` -4. **Manual GitOps Sync** (User → ArgoCD) - - User applies: `kubectl apply -f gitops/helios-app.yaml` - - ArgoCD watches gitops repo for changes - - ArgoCD deploys HeliosApp to cluster +#### Update GitOps Configuration +```bash +# 1. Edit manifests in gitops repo +nano helios-app.yaml # Change replicas, ports, etc -5. **Automatic Operator Deployment** (ArgoCD → Helios Operator) - - Helios Operator sees HeliosApp CRD - - Creates PostgreSQL database (with schema from Docker image) - - Starts PostgREST container - - Exposes REST API via Ingress +# 2. Push changes +git add . +git commit -m "Scale to 3 replicas" +git push origin main -6. **Use REST API** - - Users interact with auto-generated endpoints - - PostgREST serves REST CRUD operations +# 3. Automatic sync +# GitOps webhook triggers +# ArgoCD syncs → Kubernetes updates +``` -## Deployment +## Getting Started ### Prerequisites -- Kubernetes 1.28+ -- Helios Operator 0.2.0+ installed +- Kubernetes 1.28+ with Helios Operator 0.2.0+ - ArgoCD 2.8+ installed -- Docker registry access (Docker Hub username) - Backstage instance with scaffolder plugin +- Docker Hub account (for image registry) -### First-Time Setup +### Step 1: Create Your API via Backstage -```bash -# After template scaffolding -cd gitops/ +``` +Backstage → "Create Component" button → "PostgREST API" template + +Fill in: + Component Name: my-awesome-api + Docker Org: mycompany (or Docker Hub username) + Repository Name: my-awesome-api + API Port: 3000 + Namespace: production (or your target namespace) + JWT Secret: (random string, 32+ chars) + Database Name: my-awesome-api-db +``` + +**What gets created:** +- ✅ Source repository with Dockerfile, schema templates +- ✅ GitOps repository with deployed manifests +- ✅ Git webhooks (automatically registered) -# Review manifests -cat helios-app.yaml +### Step 2: Deploy to Your Cluster -# Deploy to cluster +```bash +# Clone the generated gitops repository +git clone https://your-gitea/mycompany/my-awesome-api-gitops.git +cd my-awesome-api-gitops + +# Deploy HeliosApp (this is what everything depends on) kubectl apply -f helios-app.yaml -# Watch deployment progress -kubectl get heliosapp -w +# Watch rollout +kubectl rollout status deployment/api -n production --timeout=5m -# List generated endpoints -kubectl get ingress -n your-namespace +# Get your API endpoint +kubectl get ingress -n production +# Result: my-awesome-api.company.internal or https://api.company.internal ``` -### Update Deployment +### Step 3: Design Your Database Schema + +Edit the source repository (`schema/` folder): ```bash -# From source repo: push schema changes +# Clone source repository +git clone https://your-gitea/mycompany/my-awesome-api.git +cd my-awesome-api + +# Edit schema files +cat > schema/01-tables.sql << 'EOF' +CREATE TABLE posts ( + id SERIAL PRIMARY KEY, + title TEXT NOT NULL, + body TEXT, + author_id INTEGER REFERENCES users(id), + created_at TIMESTAMP DEFAULT NOW() +); + +CREATE TABLE users ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + email TEXT UNIQUE NOT NULL +); +EOF + +cat > schema/02-permissions.sql << 'EOF' +-- Allow public read-only access +GRANT SELECT ON posts, users TO anon; +GRANT SELECT ON posts, users TO authenticated; + +-- Allow authenticated users to create posts +GRANT INSERT ON posts TO authenticated; +EOF + +# Commit and push git add schema/ -git commit -m "Add new tables" -git push +git commit -m "Add posts and users tables" +git push origin main +``` -# Tekton auto-builds and pushes image +Your schema is now compiled into the Docker image. **Automatic pipeline triggers:** +- Builds Docker image with schema +- Pushes to docker.io/mycompany/my-awesome-api:latest +- ArgoCD detects the change and redeploys +- **Your API endpoints are live** (e.g., GET /posts, POST /posts) -# From gitops repo: review and commit any changes -cd ../gitops/ -git add . -git commit -m "Update configuration" -git push +### Step 4: Evolve Your Schema (Zero-Downtime Migrations) + +Instead of rebuilding images, use migrations: -# ArgoCD auto-syncs +```bash +# Create migration files +mkdir -p db/migration + +cat > db/migration/001_initial.up.sql << 'EOF' +CREATE TABLE posts (id SERIAL PRIMARY KEY, title TEXT, author_id INTEGER); +CREATE TABLE users (id SERIAL PRIMARY KEY, name TEXT, email TEXT UNIQUE); +EOF + +cat > db/migration/001_initial.down.sql << 'EOF' +DROP TABLE IF EXISTS posts, users; +EOF + +# Later: add a new feature without redeploying +cat > db/migration/002_add_published_at.up.sql << 'EOF' +ALTER TABLE posts ADD COLUMN published_at TIMESTAMP; +EOF + +cat > db/migration/002_add_published_at.down.sql << 'EOF' +ALTER TABLE posts DROP COLUMN published_at; +EOF + +# Push your changes +git add db/migration/ +git commit -m "Add published_at to posts" +git push origin main + +# ✨ MAGIC: db-migrate pipeline automatically: +# • Clones repo +# • Runs golang-migrate +# • Reloads PostgREST +# • No downtime, no redeployment, no pod restarts ``` -## Security Considerations +Monitor the migration: + +```bash +# Check webhook triggered +kubectl get eventlistener -w -n production +kubectl logs deployment/el-my-awesome-api-db-migrate-listener -n production -1. **JWT Secrets**: Provide secure `jwtSecret` (32+ characters, random) -2. **Database Credentials**: Not stored in manifests; injected by Operator -3. **Role-Based Access**: Configure `anonRole` and `authenticated` roles in schema -4. **Docker Registry**: Use private registries for proprietary APIs -5. **Ingress TLS**: Ensure HTTPS is configured for production +# Check migration results +kubectl get pipelinerun -n production | grep db-migrate +kubectl describe pipelinerun -n production + +# Your new endpoints work immediately +curl https://api.company.internal/posts # includes published_at +``` ## Troubleshooting ### API Not Responding ```bash -# Check PostgREST logs -kubectl logs -f deployment/my-api -c api +# Check PostgREST pod is running +kubectl get pod -n production -l app=api + +# Check logs for connection errors +kubectl logs deployment/api -n production + +# Verify database is running +kubectl get pod -n production -l app=api-db + +# Test DB connection from API pod +kubectl exec -it deployment/api -n production -- \ + sh -c 'curl -s http://localhost:3000/ | head' +``` + +### Migrations Not Triggering +```bash +# Check db-migrate EventListener is running +kubectl get eventlistener -n production + +# Check if webhook is registered in Gitea +kubectl logs deployment/el-my-awesome-api-db-migrate-listener -n production -f --tail=50 + +# Manually trigger by pushing to db/migration/ folder: +echo "-- test" > db/migration/999_test.up.sql +git add db/migration/999_test.up.sql +git commit -m "Trigger migration" +git push origin main + +# Watch the pipeline get created +kubectl get pipelinerun -n production -w +``` + +### Failed Migrations +```bash +# Check migration run status +kubectl get pipelinerun -n production -l tekton.dev/pipeline=db-migrate + +# Get detailed error +kubectl describe pipelinerun -n production + +# Check migration logs +kubectl logs -f -l tekton.dev/pipeline=db-migrate -c step-migrate -n production -# Verify database connection -kubectl exec -it deployment/my-api -- \ - sh -c 'curl -v http://localhost:3000/' +# Common issues: +# - SQL syntax error: Check *.up.sql file format +# - File naming: Must be NNN_description.up.sql (with leading zeros) +# - Database not found: Check dbName matches in helios-app.yaml ``` -### Database Not Initialized +### CI/CD Pipeline Not Triggering ```bash -# Check Operator logs -kubectl logs -f deployment/helios-operator +# Check Tekton EventListener for source code changes +kubectl get eventlistener -n production -# Check database status -kubectl get database -n your-namespace +# Verify webhook URL in Gitea +# Should be: http://el-my-awesome-api-listener.production.svc.cluster.local:8080 -# Check secrets injected -kubectl get secret -n your-namespace | grep db +# Check Tekton controller logs +kubectl logs -f deployment/tekton-triggers-controller -n tekton-pipelines + +# Manual test: push any file to source repo +git add . +git commit -m "Test CI/CD" --allow-empty +git push origin main + +# Check if PipelineRun gets created +kubectl get pipelinerun -n production -w ``` -### CI/CD Not Triggering +## Common Patterns + +### JWT Authentication ```bash -# Check webhook -kubectl get eventlisteners -n default +# Generate a strong JWT secret +openssl rand -base64 32 + +# Pass to template when creating component +JWT Secret: + +# Clients use JWT token to access protected endpoints: +AUTH_TOKEN=$(your-auth-system-generates-jwt) +curl -H "Authorization: Bearer $AUTH_TOKEN" https://api.company.internal/protected-data +``` + +### Role-Based Access Control +```sql +-- In schema/02-permissions.sql +CREATE ROLE anon NOLOGIN; +CREATE ROLE authenticated NOLOGIN; + +-- Public endpoints (unauthenticated) +GRANT SELECT ON public_posts TO anon; + +-- Private endpoints (authenticated users only) +GRANT SELECT ON user_profile TO authenticated; +GRANT INSERT, UPDATE ON user_profile TO authenticated; +``` -# Check Tekton logs -kubectl logs -f tekton-triggers-controller -n tekton-pipelines +### Custom Docker Image +The template uses `postgrest/postgrest:latest` as base, allowing you to: +- Add system packages +- Install extensions +- Configure PostgREST before runtime -# Verify webhook URL in Git repo settings -# Should point to: http://el-{repoName}-listener.default.svc.cluster.local:8080 +Edit `Dockerfile` in source repo: +```dockerfile +FROM postgrest/postgrest:v12.2.0 + +# Add custom packages or configs +RUN apt-get update && apt-get install -y custom-tool + +# Copy your schema +COPY schema/ /schema/ + +# Build pushes to docker.io//:latest ``` ## Files in This Template @@ -256,6 +600,36 @@ kubectl logs -f tekton-triggers-controller -n tekton-pipelines 3. Follow the scaffolding flow to generate their repositories 4. See generated `README.md` in source repo for customization guide +## Quick Reference + +### Common Commands +```bash +# Monitor API deployment +kubectl get pods -n production -l app=api +kubectl logs -f deployment/api -n production + +# Watch database migrations +kubectl get pipelinerun -n production -l tekton.dev/pipeline=db-migrate +kubectl logs -f pipelinerun/ -n production + +# Check webhook integration +kubectl get eventlistener -n production +kubectl logs -f deployment/el-my-awesome-api-listener -n production + +# Direct database access +kubectl port-forward -n production svc/api-db 5432:5432 +psql -h localhost -U postgres -d my-awesome-api-db +``` + +### Debugging Checklist +| Issue | Check This | +|-------|-----------| +| API not responding | `kubectl get pod -n production`, then `kubectl logs` | +| Migrations not running | `kubectl get eventlistener -n production`, check git push was to `db/migration/` | +| Database connection error | Verify Secret exists: `kubectl get secret -n production -db-credentials` | +| Migration fails with SQL error | Review `*.up.sql` file format and `NNN_description` naming | +| Webhook not triggered | Check Gitea webhook URL is: `http://el--listener..svc.cluster.local:8080` | + ## References - [PostgREST Documentation](https://postgrest.org) diff --git a/apps/portal/examples/postgrest-template/content/gitops/helios-app.yaml b/apps/portal/examples/postgrest-template/content/gitops/helios-app.yaml index 0674fbc..9682935 100644 --- a/apps/portal/examples/postgrest-template/content/gitops/helios-app.yaml +++ b/apps/portal/examples/postgrest-template/content/gitops/helios-app.yaml @@ -14,6 +14,7 @@ spec: webhookSecret: ${{ values.name }}-webhook-secret port: ${{ values.port }} replicas: 1 + triggerType: db-migrate components: - name: api @@ -38,7 +39,7 @@ spec: - name: PGRST_MAX_ROWS value: "1000" - name: PGRST_LOG_LEVEL - value: info + value: "info" traits: - type: database properties: diff --git a/apps/portal/examples/postgrest-template/content/gitops/namespace.yaml b/apps/portal/examples/postgrest-template/content/gitops/namespace.yaml new file mode 100644 index 0000000..b20f342 --- /dev/null +++ b/apps/portal/examples/postgrest-template/content/gitops/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ${{ values.namespace }} + labels: + helios.io/managed-by: helios-operator diff --git a/apps/portal/examples/postgrest-template/content/source/postgrestrc.conf b/apps/portal/examples/postgrest-template/content/source/postgrestrc.conf index c7d0042..5d35e29 100644 --- a/apps/portal/examples/postgrest-template/content/source/postgrestrc.conf +++ b/apps/portal/examples/postgrest-template/content/source/postgrestrc.conf @@ -26,7 +26,7 @@ jwt-aud = "${{ values.jwtRole }}" max-rows = 1000 # Logging -log-level = info +log-level = "info" # Connection pool settings (optional) db-pool = 10 diff --git a/apps/portal/examples/postgrest-template/template.yaml b/apps/portal/examples/postgrest-template/template.yaml index 489db41..65cd837 100644 --- a/apps/portal/examples/postgrest-template/template.yaml +++ b/apps/portal/examples/postgrest-template/template.yaml @@ -125,11 +125,11 @@ spec: repoVisibility: public - id: create-webhook - name: Create Webhook + name: Create Webhook for DB Migration Trigger action: gitea:create-webhook input: repoUrl: ${{ parameters.repoUrl }} - webhookUrl: http://el-${{ parameters.repoName }}-listener.${{ parameters.namespace }}.svc.cluster.local:8080 + webhookUrl: http://el-${{ parameters.repoName }}-db-migrate-listener.${{ parameters.namespace }}.svc.cluster.local:8080 webhookSecret: ${{ parameters.repoName }} events: - push @@ -168,7 +168,14 @@ spec: sourcePath: ./gitops repoVisibility: public - # 3. Create webhook secret in cluster + # 3. Apply namespace to cluster + - id: apply-namespace + name: Create Kubernetes Namespace + action: kubernetes:apply + input: + manifestPath: ./gitops/namespace.yaml + + # 4. Create webhook secret in cluster - id: create-webhook-secret name: Create Kubernetes Webhook Secret action: kubernetes:create-secret @@ -181,14 +188,14 @@ spec: # Keep `secretToken` for compatibility with any legacy consumers. secretToken: ${{ parameters.repoName }} - # 4. Deploy HeliosApp to cluster + # 5. Deploy HeliosApp to cluster - id: apply-heliosapp name: Apply HeliosApp to Cluster action: kubernetes:apply input: manifestPath: ./gitops/helios-app.yaml - # 5. Registration + # 6. Registration - id: register name: Register Component action: catalog:register diff --git a/cue/definitions/tekton/pipelines/db-migrate.cue b/cue/definitions/tekton/pipelines/db-migrate.cue index 87403f3..8e34139 100644 --- a/cue/definitions/tekton/pipelines/db-migrate.cue +++ b/cue/definitions/tekton/pipelines/db-migrate.cue @@ -22,9 +22,10 @@ _dbMigrateParams: [ default: "main" }, { - name: "database-url" - description: "Database connection URL (postgres://user:pass@host:port/dbname)" + name: "db-secret-name" + description: "Kubernetes Secret name containing database credentials (expects key PGRST_DB_URI)" type: "string" + default: "api-db-secret" }, { name: "migration-source" @@ -79,7 +80,7 @@ _dbMigrateConfig: { workspace: "source" }] params: [ - {name: "database-url", value: "$(params.database-url)"}, + {name: "db-secret-name", value: "$(params.db-secret-name)"}, {name: "migration-source", value: "$(params.migration-source)"}, ] }, @@ -90,7 +91,7 @@ _dbMigrateConfig: { taskRef: {name: "postgrest-reload"} runAfter: ["run-migrations"] params: [ - {name: "database-url", value: "$(params.database-url)"}, + {name: "db-secret-name", value: "$(params.db-secret-name)"}, ] }, ] diff --git a/cue/definitions/tekton/tasks/db-migrate.cue b/cue/definitions/tekton/tasks/db-migrate.cue index 421539c..5347898 100644 --- a/cue/definitions/tekton/tasks/db-migrate.cue +++ b/cue/definitions/tekton/tasks/db-migrate.cue @@ -4,89 +4,65 @@ import "helios.io/cue/definitions/tekton" // Database Migration Task using golang-migrate // Runs database migrations from the source repository -// Expects DATABASE_URL to be injected from Kubernetes Secret #DBMigrate: tekton.#TektonTask & { -parameter: { -name: "db-migrate" -} - -_config: tekton.#Defaults + parameter: { + name: "db-migrate" + } -output: spec: { -params: [ -{ -name: "migration-source" -description: "Path to migrations directory in the cloned repo (e.g., db/migrations)" -type: "string" -default: "db/migrations" -}, -{ -name: "database-url" -description: "Database connection URL (postgres://user:pass@host:port/dbname)" -type: "string" -}, -] + _config: tekton.#Defaults -workspaces: [{ -name: "source" -description: "Workspace containing cloned repository with migrations" -}] + output: spec: { + params: [ + { + name: "migration-source" + description: "Path to migrations directory in the cloned repo (e.g., db/migration or db/migrations)" + type: "string" + default: "db/migrations" + }, + { + name: "db-secret-name" + description: "Kubernetes Secret name containing database credentials (expects key PGRST_DB_URI)" + type: "string" + default: "api-db-secret" + }, + ] -volumes: [{ -name: "db-credentials" -secret: { -secretName: "database-secret" -} -}] + workspaces: [{ + name: "source" + description: "Workspace containing cloned repository with migrations" + }] -steps: [ -{ -name: "migrate" -image: "migrate/migrate:v4.17.0" -workingDir: "$(workspaces.source.path)" -env: [ -{ -name: "DATABASE_URL" -valueFrom: { -secretKeyRef: { -name: "database-secret" -key: "DATABASE_URL" -} -} -}, -] -volumeMounts: [{ -name: "db-credentials" -mountPath: "/etc/db-credentials" -readOnly: true -}] -script: """ -#!/bin/sh -set -e + steps: [{ + name: "migrate" + image: "migrate/migrate:v4.17.0" + workingDir: "$(workspaces.source.path)" + env: [{ + name: "DATABASE_URL" + valueFrom: { + secretKeyRef: { + name: "$(params.db-secret-name)" + key: "PGRST_DB_URI" + } + } + }] + script: """ + #!/bin/sh + set -e -MIGRATIONS_DIR="$(workspaces.source.path)/$(params.migration-source)" + MIGRATIONS_DIR="$(workspaces.source.path)/$(params.migration-source)" -if [ ! -d "$MIGRATIONS_DIR" ]; then -echo "WARNING: No migrations directory found at $MIGRATIONS_DIR" -echo "Skipping migrations..." -exit 0 -fi + if [ ! -d "$MIGRATIONS_DIR" ]; then + echo "WARNING: No migrations directory found at $MIGRATIONS_DIR" + echo "Skipping migrations..." + exit 0 + fi -echo "Running database migrations from $MIGRATIONS_DIR" + echo "Running database migrations from $MIGRATIONS_DIR" -migrate \\ --path "$MIGRATIONS_DIR" \\ --database "$DATABASE_URL" \\ -up + migrate -path "$MIGRATIONS_DIR" -database "$DATABASE_URL" up -if [ $? -eq 0 ]; then -echo "SUCCESS: Migrations completed successfully" -else -echo "ERROR: Migrations failed" -exit 1 -fi -""" -}, -] -} + echo "SUCCESS: Migrations completed successfully" + """ + }] + } } diff --git a/cue/definitions/tekton/tasks/git-clone.cue b/cue/definitions/tekton/tasks/git-clone.cue index 3e25e78..5fc9195 100644 --- a/cue/definitions/tekton/tasks/git-clone.cue +++ b/cue/definitions/tekton/tasks/git-clone.cue @@ -32,9 +32,22 @@ import "helios.io/cue/definitions/tekton" rm -rf $(workspaces.output.path)/* rm -rf $(workspaces.output.path)/.[!.]* + # Transform localhost URLs for in-cluster access + GIT_URL="$(params.url)" + case "$GIT_URL" in + *localhost:3030*) + echo "Transforming localhost URL to in-cluster address" + GIT_URL=$(echo "$GIT_URL" | sed 's|http://localhost:3030/|http://gitea-http.gitea.svc.cluster.local:3000/|g') + echo "Transformed URL: $GIT_URL" + ;; + *) + echo "Using URL as-is: $GIT_URL" + ;; + esac + # Clone the repository - echo "Cloning $(params.url) to $(workspaces.output.path)" - git clone $(params.url) $(workspaces.output.path) + echo "Cloning $GIT_URL to $(workspaces.output.path)" + git clone "$GIT_URL" $(workspaces.output.path) # Checkout the specified revision cd $(workspaces.output.path) diff --git a/cue/definitions/tekton/tasks/postgrest-reload.cue b/cue/definitions/tekton/tasks/postgrest-reload.cue index 252669e..1b55d1b 100644 --- a/cue/definitions/tekton/tasks/postgrest-reload.cue +++ b/cue/definitions/tekton/tasks/postgrest-reload.cue @@ -6,65 +6,42 @@ import "helios.io/cue/definitions/tekton" // Triggers PostgREST to reload schema cache via NOTIFY command // This ensures the API immediately reflects database changes #PostgRESTReload: tekton.#TektonTask & { -parameter: { -name: "postgrest-reload" -} - -_config: tekton.#Defaults - -output: spec: { -params: [ -{ -name: "database-url" -description: "Database connection URL (postgres://user:pass@host:port/dbname)" -type: "string" -}, -] - -volumes: [{ -name: "db-credentials" -secret: { -secretName: "database-secret" -} -}] - -steps: [ -{ -name: "reload-schema" -image: "postgres:15-alpine" -env: [ -{ -name: "DATABASE_URL" -valueFrom: { -secretKeyRef: { -name: "database-secret" -key: "DATABASE_URL" -} -} -}, -] -volumeMounts: [{ -name: "db-credentials" -mountPath: "/etc/db-credentials" -readOnly: true -}] -script: """ -#!/bin/sh -set -e - -echo "Triggering PostgREST schema reload..." - -psql "$DATABASE_URL" -c "NOTIFY pgrst, 'reload schema';" - -if [ $? -eq 0 ]; then -echo "SUCCESS: Schema reload triggered successfully" -echo "API will reflect new schema within seconds" -else -echo "ERROR: Failed to trigger schema reload" -exit 1 -fi -""" -}, -] -} + parameter: { + name: "postgrest-reload" + } + + _config: tekton.#Defaults + + output: spec: { + params: [{ + name: "db-secret-name" + description: "Kubernetes Secret name containing database credentials (expects key PGRST_DB_URI)" + type: "string" + default: "api-db-secret" + }] + + steps: [{ + name: "reload-schema" + image: "postgres:15-alpine" + env: [{ + name: "DATABASE_URL" + valueFrom: { + secretKeyRef: { + name: "$(params.db-secret-name)" + key: "PGRST_DB_URI" + } + } + }] + script: """ + #!/bin/sh + set -e + + echo "Triggering PostgREST schema reload..." + + psql "$DATABASE_URL" -c "NOTIFY pgrst, 'reload schema';" + + echo "SUCCESS: Schema reload triggered successfully" + """ + }] + } } diff --git a/cue/definitions/tekton/triggers/db-migrate-trigger.cue b/cue/definitions/tekton/triggers/db-migrate-trigger.cue index c2da1ba..1900d2e 100644 --- a/cue/definitions/tekton/triggers/db-migrate-trigger.cue +++ b/cue/definitions/tekton/triggers/db-migrate-trigger.cue @@ -6,7 +6,7 @@ import ( // ===================================================== // DATABASE MIGRATION TRIGGER BUNDLE -// Triggers db-migrate pipeline only on changes to db/** path +// Triggers db-migrate pipeline only on changes to db/migration path // ===================================================== #DatabaseMigrationTriggerBundle: tekton.#TriggerBundle & { @@ -23,7 +23,6 @@ import ( config: params: [ {name: "git-repo-url", value: "$(body.repository.clone_url)"}, {name: "git-revision", value: "$(body.after)"}, - {name: "database-url", value: "$(body.database_url)"}, ] } @@ -40,7 +39,6 @@ import ( params: [ {name: "git-repo-url", description: "Repository URL from webhook"}, {name: "git-revision", description: "Git commit SHA from webhook"}, - {name: "database-url", description: "Database URL (injected from secret or webhook)"}, ] // PipelineRun for db-migrate pipeline @@ -53,7 +51,7 @@ import ( labels: { "helios.io/managed-by": "helios-operator" "app.kubernetes.io/part-of": "helios-platform" - "app.kubernetes.io/instance": _bp.pipelineName + "app.kubernetes.io/instance": "db-migrate" "app.kubernetes.io/name": _bp.appName "janus-idp.io/tekton": _bp.appName "tekton.dev/pipeline": "db-migrate" @@ -64,12 +62,12 @@ import ( name: "db-migrate" } serviceAccountName: _bp.serviceAccount - + params: [ {name: "app-repo-url", value: "$(tt.params.git-repo-url)"}, {name: "app-repo-revision", value: "$(tt.params.git-revision)"}, - {name: "database-url", value: "$(tt.params.database-url)"}, - {name: "migration-source", value: "db/migrations"}, + {name: "db-secret-name", value: "api-db-secret"}, + {name: "migration-source", value: "db/migration"}, {name: "namespace", value: _bp.namespace}, ] @@ -88,7 +86,7 @@ import ( } // 3. EVENT LISTENER - // Listens for push events and filters by db/** path using CEL + // Listens for push events and filters by db/migration path using CEL _listener: tekton.#TektonEventListener & { parameter: { name: "\(bundleParams.appName)-db-migrate-listener" @@ -99,8 +97,8 @@ import ( name: "db-migrate-push" bindings: [{ref: _binding.parameter.name}] template: {ref: _template.parameter.name} - - // CEL interceptor to filter only db/** changes + + // CEL interceptor to filter only db/migration** changes // This ensures migration pipeline only runs when migrations directory is modified interceptors: [ { @@ -115,12 +113,26 @@ import ( ] }, { - // CEL filter to only trigger if db/** path changed - // Handles both single and multiple commits + // CEL interceptor to transform URLs for in-cluster access + // Replaces localhost:3030 with in-cluster Gitea address + ref: {name: "cel", kind: "ClusterInterceptor"} + params: [{ + name: "overlays" + value: [ + { + key: "repository.clone_url" + expression: "body.repository.clone_url.replace('http://localhost:3030/', 'http://gitea-http.gitea.svc.cluster.local:3000/')" + }, + ] + }] + }, + { + // CEL filter to only trigger if migrations path changed + // Includes both added + modified files, matches db/migration or db/migrations ref: {name: "cel", kind: "ClusterInterceptor"} params: [{ name: "filter" - value: "has(body.commits) && body.commits.filter(c, has(c.modified) && c.modified.exists(m, m.startsWith('db/'))).size() > 0" + value: "has(body.commits) && body.commits.exists(c, (has(c.added) && c.added.exists(f, f.startsWith('db/migration/') || f.startsWith('db/migrations/'))) || (has(c.modified) && c.modified.exists(f, f.startsWith('db/migration/') || f.startsWith('db/migrations/'))))" }] }, ] diff --git a/cue/definitions/tekton/triggers/github-push.cue b/cue/definitions/tekton/triggers/github-push.cue index 3e06cdb..12af248 100644 --- a/cue/definitions/tekton/triggers/github-push.cue +++ b/cue/definitions/tekton/triggers/github-push.cue @@ -24,7 +24,7 @@ import ( ] } - // 2. TRIGGER TEMPLATE + // 2. TRIGGER TEMPLATES _template: tekton.#TektonTriggerTemplate & { // Capture bundleParams locally let _bp = bundleParams @@ -104,6 +104,59 @@ import ( } } + _dbMigrateTemplate: tekton.#TektonTriggerTemplate & { + let _bp = bundleParams + + parameter: { + name: "\(_bp.appName)-db-migrate-template" + namespace: _bp.namespace + } + config: { + params: [ + {name: "git-revision", description: "From Webhook"}, + ] + + resourcetemplates: [{ + apiVersion: "tekton.dev/v1beta1" + kind: "PipelineRun" + metadata: { + name: "\(_bp.appName)-migrate-$(uid)" + namespace: _bp.namespace + labels: { + "helios.io/managed-by": "helios-operator" + "app.kubernetes.io/part-of": "helios-platform" + "app.kubernetes.io/instance": "db-migrate" + "app.kubernetes.io/name": _bp.appName + "janus-idp.io/tekton": _bp.appName + "tekton.dev/pipeline": "db-migrate" + } + } + spec: { + pipelineRef: {name: "db-migrate"} + serviceAccountName: _bp.serviceAccount + + params: [ + {name: "app-repo-url", value: _bp.gitRepo}, + {name: "app-repo-revision", value: "$(tt.params.git-revision)"}, + {name: "db-secret-name", value: "api-db-secret"}, + {name: "migration-source", value: "db/migration"}, + {name: "namespace", value: _bp.namespace}, + ] + + workspaces: [{ + name: "source" + volumeClaimTemplate: { + spec: { + accessModes: ["ReadWriteOnce"] + resources: requests: storage: "1Gi" + } + } + }] + } + }] + } + } + // 3. EVENT LISTENER _listener: tekton.#TektonEventListener & { parameter: { @@ -111,23 +164,50 @@ import ( namespace: bundleParams.namespace } config: { - triggers: [{ - name: "gitea-push" - bindings: [{ref: _binding.parameter.name}] - template: {ref: _template.parameter.name} - - // Use the cluster Git webhook interceptor for push event validation. - interceptors: [{ - ref: {name: "github", kind: "ClusterInterceptor"} - params: [ - {name: "secretRef", value: { - secretName: bundleParams.webhookSecret - secretKey: "secret" - }}, - {name: "eventTypes", value: ["push"]}, + triggers: [ + { + name: "gitea-push" + bindings: [{ref: _binding.parameter.name}] + template: {ref: _template.parameter.name} + + // Use the cluster Git webhook interceptor for push event validation. + interceptors: [{ + ref: {name: "github", kind: "ClusterInterceptor"} + params: [ + {name: "secretRef", value: { + secretName: bundleParams.webhookSecret + secretKey: "secret" + }}, + {name: "eventTypes", value: ["push"]}, + ] + }] + }, + { + name: "db-migrate-on-migrations" + bindings: [{ref: _binding.parameter.name}] + template: {ref: _dbMigrateTemplate.parameter.name} + + interceptors: [ + { + ref: {name: "github", kind: "ClusterInterceptor"} + params: [ + {name: "secretRef", value: { + secretName: bundleParams.webhookSecret + secretKey: "secret" + }}, + {name: "eventTypes", value: ["push"]}, + ] + }, + { + ref: {name: "cel", kind: "ClusterInterceptor"} + params: [{ + name: "filter" + value: "has(body.commits) && body.commits.exists(c, (has(c.added) && c.added.exists(f, f.startsWith('db/migration/') || f.startsWith('db/migrations/'))) || (has(c.modified) && c.modified.exists(f, f.startsWith('db/migration/') || f.startsWith('db/migrations/'))))" + }] + }, ] - }] - }] + }, + ] } } @@ -135,6 +215,7 @@ import ( outputs: [ _binding.output, _template.output, + _dbMigrateTemplate.output, _listener.output, ] } \ No newline at end of file diff --git a/cue/engine/tekton_builder.cue b/cue/engine/tekton_builder.cue index 8b8f981..f0026b4 100644 --- a/cue/engine/tekton_builder.cue +++ b/cue/engine/tekton_builder.cue @@ -53,14 +53,27 @@ _tasks: [ } ] -// 2. RENDER PIPELINE -_pipeline: [ +// 2. RENDER PIPELINES +// Render primary pipeline always +_primaryPipeline: [ (pipelines.#RenderPipeline & { pipelineType: tektonInput.pipelineType namespace: tektonInput.namespace }).output ] +// Also render db-migrate pipeline if it's available and needed by triggers +_dbMigratePipeline: [ + if tektonInput.triggerType == "gitea-push" || tektonInput.triggerType == "db-migrate" { + (pipelines.#RenderPipeline & { + pipelineType: "db-migrate" + namespace: tektonInput.namespace + }).output + } +] + +_pipeline: list.Concat([_primaryPipeline, _dbMigratePipeline]) + // 3. RENDER TRIGGERS _triggers: (triggers.#RenderTriggers & { triggerType: tektonInput.triggerType From ea40c073585731026c610a91187d9dc0d0e7d7cb Mon Sep 17 00:00:00 2001 From: Ho Phuoc Nghia Date: Thu, 16 Apr 2026 23:47:39 +0700 Subject: [PATCH 08/12] chore: remove initial schema migration file --- db/migration/001_initial.sql | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 db/migration/001_initial.sql diff --git a/db/migration/001_initial.sql b/db/migration/001_initial.sql deleted file mode 100644 index f7e8490..0000000 --- a/db/migration/001_initial.sql +++ /dev/null @@ -1,5 +0,0 @@ --- Initial schema -CREATE TABLE IF NOT EXISTS users ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL -); From 89ff59fece494ec9ca05ba8719593d8b9f3c718d Mon Sep 17 00:00:00 2001 From: nghiaz160904 Date: Sat, 18 Apr 2026 10:05:12 +0700 Subject: [PATCH 09/12] feat: Add database secret reference and validation for HeliosApp configuration --- apps/operator/api/v1alpha1/heliosapp_types.go | 5 +++ .../controller/heliosapp_controller.go | 36 ++++++++++++++++++- .../internal/controller/tekton/mapper.go | 2 ++ apps/operator/internal/cue/tekton.go | 3 +- cue/definitions/tekton/tasks/db-migrate.cue | 6 ++-- .../tekton/triggers/github-push.cue | 4 +-- cue/engine/tekton_builder.cue | 1 + 7 files changed, 50 insertions(+), 7 deletions(-) diff --git a/apps/operator/api/v1alpha1/heliosapp_types.go b/apps/operator/api/v1alpha1/heliosapp_types.go index 62801f5..54baec6 100644 --- a/apps/operator/api/v1alpha1/heliosapp_types.go +++ b/apps/operator/api/v1alpha1/heliosapp_types.go @@ -126,6 +126,11 @@ type HeliosAppSpec struct { // +optional ContextSubpath string `json:"contextSubpath,omitempty"` + // DatabaseSecretRef is the name of the secret containing database credentials for migrations + // +optional + // +kubebuilder:default="api-db-secret" + DatabaseSecretRef string `json:"databaseSecretRef,omitempty"` + // Components define the workloads of the application Components []Component `json:"components"` } diff --git a/apps/operator/internal/controller/heliosapp_controller.go b/apps/operator/internal/controller/heliosapp_controller.go index 5922879..ec2cd8d 100644 --- a/apps/operator/internal/controller/heliosapp_controller.go +++ b/apps/operator/internal/controller/heliosapp_controller.go @@ -98,6 +98,13 @@ func (r *HeliosAppReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( log.Info("Reconciling HeliosApp", "name", heliosApp.Name, "namespace", heliosApp.Namespace) + // Pre-flight validation: Check if all referenced secrets exist + if err := r.validateSecretReferences(ctx, &heliosApp); err != nil { + log.Error(err, "Pre-flight validation failed: referenced secret does not exist") + r.updateStatus(ctx, &heliosApp, appv1alpha1.PhaseFailed, fmt.Sprintf("Configuration error: %v", err)) + return ctrl.Result{}, err + } + // 2. Map CRD to Application Model appModel, err := mapCRDToModel(&heliosApp) if err != nil { @@ -240,7 +247,8 @@ func (r *HeliosAppReconciler) findObjectsForSecret(ctx context.Context, obj clie for _, app := range heliosAppList.Items { // Check if this app references the changed secret if app.Spec.GitOpsSecretRef == obj.GetName() || - app.Spec.WebhookSecret == obj.GetName() { + app.Spec.WebhookSecret == obj.GetName() || + app.Spec.DatabaseSecretRef == obj.GetName() { requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ Name: app.Name, @@ -252,3 +260,29 @@ func (r *HeliosAppReconciler) findObjectsForSecret(ctx context.Context, obj clie return requests } + +// validateSecretReferences checks if all referenced secrets exist in the cluster. +// This is a pre-flight validation to catch configuration errors early. +func (r *HeliosAppReconciler) validateSecretReferences(ctx context.Context, app *appv1alpha1.HeliosApp) error { + secretsToValidate := map[string]string{ + "webhook secret": app.Spec.WebhookSecret, + "GitOps secret": app.Spec.GitOpsSecretRef, + "database secret": app.Spec.DatabaseSecretRef, + } + + for secretType, secretName := range secretsToValidate { + if secretName == "" { + continue // Skip empty references + } + + var secret corev1.Secret + if err := r.Get(ctx, types.NamespacedName{Name: secretName, Namespace: app.Namespace}, &secret); err != nil { + if errors.IsNotFound(err) { + return fmt.Errorf("%s '%s' not found in namespace '%s'", secretType, secretName, app.Namespace) + } + return fmt.Errorf("failed to validate %s '%s': %w", secretType, secretName, err) + } + } + + return nil +} diff --git a/apps/operator/internal/controller/tekton/mapper.go b/apps/operator/internal/controller/tekton/mapper.go index 0822def..4c0b62c 100644 --- a/apps/operator/internal/controller/tekton/mapper.go +++ b/apps/operator/internal/controller/tekton/mapper.go @@ -36,6 +36,7 @@ func MapCRDToTektonInput(app *appv1alpha1.HeliosApp) cueModel.TektonInput { Port: int(app.Spec.Port), TestCommand: app.Spec.TestCommand, DockerSecret: "docker-credentials", + DatabaseSecretRef: app.Spec.DatabaseSecretRef, ArgoCDNamespace: app.Spec.ArgoCDNamespace, ArgoCDProject: app.Spec.ArgoCDProject, } @@ -44,6 +45,7 @@ func MapCRDToTektonInput(app *appv1alpha1.HeliosApp) cueModel.TektonInput { input.GitOpsBranch = cmp.Or(input.GitOpsBranch, "main") input.GitOpsSecretRef = cmp.Or(input.GitOpsSecretRef, "helios-gitops-bot") input.WebhookSecret = cmp.Or(input.WebhookSecret, "gitea-webhook-secret") + input.DatabaseSecretRef = cmp.Or(input.DatabaseSecretRef, "api-db-secret") input.TriggerType = cmp.Or(input.TriggerType, "gitea-push") if input.PipelineName == "" { input.PipelineName = defaultPipelineName diff --git a/apps/operator/internal/cue/tekton.go b/apps/operator/internal/cue/tekton.go index 2614c23..77484f5 100644 --- a/apps/operator/internal/cue/tekton.go +++ b/apps/operator/internal/cue/tekton.go @@ -60,7 +60,8 @@ type TektonInput struct { TestCommand string `json:"testCommand,omitempty"` // === SECRETS === - DockerSecret string `json:"dockerSecret,omitempty"` + DockerSecret string `json:"dockerSecret,omitempty"` + DatabaseSecretRef string `json:"databaseSecretRef,omitempty"` // === ARGOCD === ArgoCDNamespace string `json:"argoCDNamespace,omitempty"` diff --git a/cue/definitions/tekton/tasks/db-migrate.cue b/cue/definitions/tekton/tasks/db-migrate.cue index 5347898..b5fbb25 100644 --- a/cue/definitions/tekton/tasks/db-migrate.cue +++ b/cue/definitions/tekton/tasks/db-migrate.cue @@ -52,9 +52,9 @@ import "helios.io/cue/definitions/tekton" MIGRATIONS_DIR="$(workspaces.source.path)/$(params.migration-source)" if [ ! -d "$MIGRATIONS_DIR" ]; then - echo "WARNING: No migrations directory found at $MIGRATIONS_DIR" - echo "Skipping migrations..." - exit 0 + echo "ERROR: Migrations directory not found at $MIGRATIONS_DIR" + echo "Expected directory path: $MIGRATIONS_DIR" + exit 1 fi echo "Running database migrations from $MIGRATIONS_DIR" diff --git a/cue/definitions/tekton/triggers/github-push.cue b/cue/definitions/tekton/triggers/github-push.cue index 12af248..e28f752 100644 --- a/cue/definitions/tekton/triggers/github-push.cue +++ b/cue/definitions/tekton/triggers/github-push.cue @@ -138,8 +138,8 @@ import ( params: [ {name: "app-repo-url", value: _bp.gitRepo}, {name: "app-repo-revision", value: "$(tt.params.git-revision)"}, - {name: "db-secret-name", value: "api-db-secret"}, - {name: "migration-source", value: "db/migration"}, + {name: "db-secret-name", value: _bp.databaseSecretRef}, + {name: "migration-source", value: "db/migrations"}, {name: "namespace", value: _bp.namespace}, ] diff --git a/cue/engine/tekton_builder.cue b/cue/engine/tekton_builder.cue index f0026b4..8ef8900 100644 --- a/cue/engine/tekton_builder.cue +++ b/cue/engine/tekton_builder.cue @@ -109,6 +109,7 @@ _triggers: (triggers.#RenderTriggers & { testImage: tekton.#CommonParams.test.image.default serviceAccount: tektonInput.serviceAccount dockerSecret: tektonInput.dockerSecret + databaseSecretRef: tektonInput.databaseSecretRef argoCDNamespace: _argoCDNamespace argoCDAppName: "\(tektonInput.appName)-argocd" From 8a5ef55a5f93c334466362ed96339e265a9a43a6 Mon Sep 17 00:00:00 2001 From: nghiaz160904 Date: Sun, 19 Apr 2026 16:41:35 +0700 Subject: [PATCH 10/12] fix: Update secret validation to exclude database secrets due to auto-creation --- apps/operator/internal/controller/heliosapp_controller.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/operator/internal/controller/heliosapp_controller.go b/apps/operator/internal/controller/heliosapp_controller.go index ec2cd8d..3e19b18 100644 --- a/apps/operator/internal/controller/heliosapp_controller.go +++ b/apps/operator/internal/controller/heliosapp_controller.go @@ -263,11 +263,13 @@ func (r *HeliosAppReconciler) findObjectsForSecret(ctx context.Context, obj clie // validateSecretReferences checks if all referenced secrets exist in the cluster. // This is a pre-flight validation to catch configuration errors early. +// Note: Database secrets are NOT validated here because they are auto-created +// by the operator in Phase 0.5 if database traits are present. func (r *HeliosAppReconciler) validateSecretReferences(ctx context.Context, app *appv1alpha1.HeliosApp) error { secretsToValidate := map[string]string{ - "webhook secret": app.Spec.WebhookSecret, - "GitOps secret": app.Spec.GitOpsSecretRef, - "database secret": app.Spec.DatabaseSecretRef, + "webhook secret": app.Spec.WebhookSecret, + "GitOps secret": app.Spec.GitOpsSecretRef, + // Note: database secret is NOT validated here - it's auto-created in Phase 0.5 } for secretType, secretName := range secretsToValidate { From 37ac09238b1ad96a252fc031dbe3bc0e045a7dd3 Mon Sep 17 00:00:00 2001 From: nghiaz160904 Date: Sun, 19 Apr 2026 16:49:06 +0700 Subject: [PATCH 11/12] feat: Add database secret reference to WebhookIngress configuration --- cue/definitions/tekton/base_trigger.cue | 1 + 1 file changed, 1 insertion(+) diff --git a/cue/definitions/tekton/base_trigger.cue b/cue/definitions/tekton/base_trigger.cue index 115536a..709adaa 100644 --- a/cue/definitions/tekton/base_trigger.cue +++ b/cue/definitions/tekton/base_trigger.cue @@ -159,6 +159,7 @@ package tekton testImage: string | *"node:20" serviceAccount: string | *"default" dockerSecret: string | *"docker-credentials" + databaseSecretRef: string | *"api-db-secret" // Argo CD sync via kubectl patch on Application argoCDNamespace: string | *"argocd" From aa28e114bd846912d8125a4e5b197e7fd8142aec Mon Sep 17 00:00:00 2001 From: nghiaz160904 Date: Sun, 19 Apr 2026 16:56:30 +0700 Subject: [PATCH 12/12] fix: Add databaseSecretRef to Tekton schema for improved secret management --- cue/definitions/tekton/schema.cue | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cue/definitions/tekton/schema.cue b/cue/definitions/tekton/schema.cue index e01c1d1..39eff6e 100644 --- a/cue/definitions/tekton/schema.cue +++ b/cue/definitions/tekton/schema.cue @@ -43,7 +43,8 @@ package tekton testCommand?: string // e.g. "npm test" // === SECRETS === - dockerSecret: string | *"docker-credentials" + dockerSecret: string | *"docker-credentials" + databaseSecretRef: string | *"api-db-secret" // === ARGOCD === argoCDNamespace: string | *"argocd"