From 03086e6709647b6059d527edfb29da222d831571 Mon Sep 17 00:00:00 2001 From: Rudransh Shrivastava Date: Mon, 19 Jan 2026 20:07:58 +0530 Subject: [PATCH 1/7] Add CI/CD to run terraform plan --- .github/workflows/run-ci-cd.yaml | 214 +++++++----------- .../staging/terraform.tfbackend.example | 2 +- .../staging/terraform.tfvars.example | 2 +- 3 files changed, 84 insertions(+), 134 deletions(-) diff --git a/.github/workflows/run-ci-cd.yaml b/.github/workflows/run-ci-cd.yaml index 3a67f3a2ce..04fa339d8a 100644 --- a/.github/workflows/run-ci-cd.yaml +++ b/.github/workflows/run-ci-cd.yaml @@ -569,20 +569,23 @@ jobs: trivy-config: trivy.yaml timeout-minutes: 5 - deploy-staging-nest: - name: Deploy Nest Staging + plan-staging-nest: + name: Plan Nest Staging env: - ANSIBLE_HOST_KEY_CHECKING: false - NEST_HOST_IP_ADDRESS: ${{ secrets.NEST_HOST_IP_ADDRESS }} - NEST_SSH_PRIVATE_KEY_PATH: ${{ vars.NEST_SSH_PRIVATE_KEY_PATH }} - RELEASE_VERSION: ${{ needs.set-release-version.outputs.release_version }} + # NOTE: update to us-east-2 + AWS_REGION: ap-south-1 + AWS_AVAILABILITY_ZONES: '["ap-south-1a", "ap-south-1b", "ap-south-1c"]' + FORCE_COLOR: 1 + TF_INPUT: false + TF_IN_AUTOMATION: true environment: staging - if: | - github.repository == 'OWASP/Nest' && - github.ref == 'refs/heads/main' - needs: - - scan-staging-images - - set-release-version + # NOTE: uncomment + #if: | + #github.repository == 'OWASP/Nest' && + #github.ref == 'refs/heads/main' + #needs: + #- scan-staging-images + #- set-release-version permissions: contents: read runs-on: ubuntu-latest @@ -590,144 +593,91 @@ jobs: - name: Check out repository uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 - - name: Prepare SSH key + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-region: ${{ env.AWS_REGION }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + - name: Install Terraform + uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd + with: + terraform_version: 1.14.0 + + - name: Prepare terraform backend env: - NEST_SSH_PRIVATE_KEY: ${{ secrets.NEST_SSH_PRIVATE_KEY }} - NEST_SSH_PRIVATE_KEY_PATH: ${{ env.NEST_SSH_PRIVATE_KEY_PATH }} + AWS_REGION: ${{ env.AWS_REGION }} + TF_STATE_BUCKET: ${{ secrets.TF_STATE_BUCKET }} + TF_STATE_DYNAMODB_TABLE: ${{ secrets.TF_STATE_DYNAMODB_TABLE }} run: | - SSH_KEY_PATH="${NEST_SSH_PRIVATE_KEY_PATH/#\~/$HOME}" - mkdir -p -m 700 "$(dirname "$SSH_KEY_PATH")" umask 377 - cat > "$SSH_KEY_PATH" < infrastructure/staging/terraform.tfbackend <<-EOF + bucket="$TF_STATE_BUCKET" + dynamodb_table="$TF_STATE_DYNAMODB_TABLE" + region="$AWS_REGION" EOF - - name: Prepare secrets + - name: Prepare terraform variables env: - DJANGO_ALGOLIA_APPLICATION_ID: ${{ secrets.DJANGO_ALGOLIA_APPLICATION_ID }} - DJANGO_ALGOLIA_WRITE_API_KEY: ${{ secrets.DJANGO_ALGOLIA_WRITE_API_KEY }} - DJANGO_ALLOWED_HOSTS: ${{ secrets.DJANGO_ALLOWED_HOSTS }} - DJANGO_AWS_ACCESS_KEY_ID: ${{ secrets.DJANGO_AWS_ACCESS_KEY_ID }} - DJANGO_AWS_SECRET_ACCESS_KEY: ${{ secrets.DJANGO_AWS_SECRET_ACCESS_KEY }} - DJANGO_CONFIGURATION: ${{ secrets.DJANGO_CONFIGURATION }} - DJANGO_DB_HOST: ${{ secrets.DJANGO_DB_HOST }} - DJANGO_DB_NAME: ${{ secrets.DJANGO_DB_NAME }} - DJANGO_DB_PASSWORD: ${{ secrets.DJANGO_DB_PASSWORD }} - DJANGO_DB_PORT: ${{ secrets.DJANGO_DB_PORT }} - DJANGO_DB_USER: ${{ secrets.DJANGO_DB_USER }} - DJANGO_OPEN_AI_SECRET_KEY: ${{ secrets.DJANGO_OPEN_AI_SECRET_KEY }} - DJANGO_REDIS_HOST: ${{ secrets.DJANGO_REDIS_HOST }} - DJANGO_REDIS_PASSWORD: ${{ secrets.DJANGO_REDIS_PASSWORD }} - DJANGO_RELEASE_VERSION: ${{ secrets.DJANGO_RELEASE_VERSION }} - DJANGO_SECRET_KEY: ${{ secrets.DJANGO_SECRET_KEY }} - DJANGO_SENTRY_DSN: ${{ secrets.DJANGO_SENTRY_DSN }} - DJANGO_SETTINGS_MODULE: ${{ secrets.DJANGO_SETTINGS_MODULE }} - DJANGO_SLACK_BOT_TOKEN: ${{ secrets.DJANGO_SLACK_BOT_TOKEN }} - DJANGO_SLACK_SIGNING_SECRET: ${{ secrets.DJANGO_SLACK_SIGNING_SECRET }} - NEXT_SERVER_CSRF_URL: ${{ secrets.NEXT_SERVER_CSRF_URL }} - NEXT_SERVER_GITHUB_CLIENT_ID: ${{ secrets.NEST_GITHUB_CLIENT_ID }} - NEXT_SERVER_GITHUB_CLIENT_SECRET: ${{ secrets.NEST_GITHUB_CLIENT_SECRET }} - NEXT_SERVER_GRAPHQL_URL: ${{ secrets.NEXT_SERVER_GRAPHQL_URL }} - NEXTAUTH_SECRET: ${{ secrets.NEXTAUTH_SECRET }} - NEXTAUTH_URL: ${{ secrets.VITE_API_URL }} - SLACK_BOT_TOKEN_T04T40NHX: ${{ secrets.SLACK_BOT_TOKEN_T04T40NHX }} + AVAILABILITY_ZONES: ${{ env.AWS_AVAILABILITY_ZONES }} + AWS_REGION: ${{ env.AWS_REGION }} + CREATE_RDS_PROXY: false + DOMAIN_NAME: "nest.owasp.dev" + ECS_USE_FARGATE_SPOT: true + ECS_USE_PUBLIC_SUBNETS: false + ENVIRONMENT: "staging" + FRONTEND_USE_FARGATE_SPOT: true + LAMBDA_FUNCTION_NAME: ${{ secrets.ZAPPA_LAMBDA_FUNCTION_NAME }} + PROJECT_NAME: "nest" run: | - # Backend umask 377 - cat > .env.backend < infrastructure/staging/terraform.tfvars <<-EOF + availability_zones=$AVAILABILITY_ZONES + aws_region="$AWS_REGION" + create_rds_proxy=$CREATE_RDS_PROXY + domain_name="$DOMAIN_NAME" + ecs_use_fargate_spot=$ECS_USE_FARGATE_SPOT + ecs_use_public_subnets=$ECS_USE_PUBLIC_SUBNETS + environment="$ENVIRONMENT" + frontend_use_fargate_spot=$FRONTEND_USE_FARGATE_SPOT + lambda_function_name="$LAMBDA_FUNCTION_NAME" + project_name="$PROJECT_NAME" EOF - # Cache - umask 377 - cat > .env.cache < .env.db < .env.frontend < "$SSH_KEY_PATH" <> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + terraform show -no-color tfplan >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY - - name: Run proxy deploy - working-directory: .github/ansible - run: ansible-playbook -i inventory.yaml staging/proxy.yaml -e "github_workspace=$GITHUB_WORKSPACE" - timeout-minutes: 5 + - name: Upload plan artifact + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: tfplan + path: infrastructure/staging/tfplan + retention-days: 3 + timeout-minutes: 15 run-staging-lighthouse-ci: name: Run Lighthouse CI needs: - - deploy-staging-nest-proxy + - deploy-production-nest permissions: contents: read runs-on: ubuntu-latest @@ -759,7 +709,7 @@ jobs: run-staging-zap-baseline-scan: name: Run staging ZAP baseline scan needs: - - deploy-staging-nest-proxy + - deploy-production-nest permissions: contents: read runs-on: ubuntu-latest diff --git a/infrastructure/staging/terraform.tfbackend.example b/infrastructure/staging/terraform.tfbackend.example index dc17c83323..957bf8a97b 100644 --- a/infrastructure/staging/terraform.tfbackend.example +++ b/infrastructure/staging/terraform.tfbackend.example @@ -1,3 +1,3 @@ bucket = "${STATE_BUCKET_NAME}" -dynamodb_table = "owasp-nest-terraform-state-lock" +dynamodb_table = "nest-terraform-state-lock" region = "us-east-2" diff --git a/infrastructure/staging/terraform.tfvars.example b/infrastructure/staging/terraform.tfvars.example index 0d36c1a8dc..bd7c5207f3 100644 --- a/infrastructure/staging/terraform.tfvars.example +++ b/infrastructure/staging/terraform.tfvars.example @@ -7,4 +7,4 @@ ecs_use_public_subnets = false environment = "staging" frontend_use_fargate_spot = true lambda_function_name = null -project_name = "owasp-nest" +project_name = "nest" From 3ef990a048abb7617dedfc812729ad133c4b2e7d Mon Sep 17 00:00:00 2001 From: Rudransh Shrivastava Date: Tue, 20 Jan 2026 20:34:18 +0530 Subject: [PATCH 2/7] Add CI/CD to run terraform apply and zappa deploy --- .github/workflows/run-ci-cd.yaml | 212 ++++++++++++++++-- backend/zappa_callback.py | 9 +- ...mple.json => zappa_settings.template.json} | 5 +- cspell/custom-dict.txt | 1 + infrastructure/README.md | 2 +- infrastructure/modules/parameters/main.tf | 12 +- .../modules/parameters/variables.tf | 10 + infrastructure/staging/main.tf | 2 + infrastructure/staging/outputs.tf | 5 + 9 files changed, 228 insertions(+), 30 deletions(-) rename backend/{zappa_settings.example.json => zappa_settings.template.json} (92%) diff --git a/.github/workflows/run-ci-cd.yaml b/.github/workflows/run-ci-cd.yaml index 04fa339d8a..8bf82b8f08 100644 --- a/.github/workflows/run-ci-cd.yaml +++ b/.github/workflows/run-ci-cd.yaml @@ -447,6 +447,16 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-region: ${{ vars.AWS_REGION }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 + - name: Build backend image uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 with: @@ -463,7 +473,9 @@ jobs: load: true platforms: linux/amd64 push: true - tags: owasp/nest:backend-staging + tags: | + owasp/nest:backend-staging + ${{ secrets.AWS_ECR_BACKEND_REPOSITORY_URL }}:latest - name: Prepare frontend public environment env: @@ -514,7 +526,9 @@ jobs: secrets: | RELEASE_VERSION=${{ needs.set-release-version.outputs.release_version }} SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} - tags: owasp/nest:frontend-staging + tags: | + owasp/nest:frontend-staging + ${{ secrets.AWS_ECR_FRONTEND_REPOSITORY_URL }}:latest - name: Get frontend image size id: frontend-size @@ -532,7 +546,7 @@ jobs: echo "**Backend:** ${{ steps.backend-size.outputs.human_readable }}" echo "**Frontend:** ${{ steps.frontend-size.outputs.human_readable }}" } >> $GITHUB_STEP_SUMMARY - timeout-minutes: 5 + timeout-minutes: 10 scan-staging-images: name: Scan Staging Images @@ -572,20 +586,16 @@ jobs: plan-staging-nest: name: Plan Nest Staging env: - # NOTE: update to us-east-2 - AWS_REGION: ap-south-1 - AWS_AVAILABILITY_ZONES: '["ap-south-1a", "ap-south-1b", "ap-south-1c"]' FORCE_COLOR: 1 TF_INPUT: false TF_IN_AUTOMATION: true environment: staging - # NOTE: uncomment #if: | #github.repository == 'OWASP/Nest' && #github.ref == 'refs/heads/main' - #needs: + needs: #- scan-staging-images - #- set-release-version + - set-release-version permissions: contents: read runs-on: ubuntu-latest @@ -597,7 +607,7 @@ jobs: uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-region: ${{ env.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - name: Install Terraform @@ -607,7 +617,7 @@ jobs: - name: Prepare terraform backend env: - AWS_REGION: ${{ env.AWS_REGION }} + AWS_REGION: ${{ vars.AWS_REGION }} TF_STATE_BUCKET: ${{ secrets.TF_STATE_BUCKET }} TF_STATE_DYNAMODB_TABLE: ${{ secrets.TF_STATE_DYNAMODB_TABLE }} run: | @@ -620,10 +630,10 @@ jobs: - name: Prepare terraform variables env: - AVAILABILITY_ZONES: ${{ env.AWS_AVAILABILITY_ZONES }} - AWS_REGION: ${{ env.AWS_REGION }} + AVAILABILITY_ZONES: ${{ vars.AWS_AVAILABILITY_ZONES }} + AWS_REGION: ${{ vars.AWS_REGION }} CREATE_RDS_PROXY: false - DOMAIN_NAME: "nest.owasp.dev" + DOMAIN_NAME: ${{ vars.DOMAIN_NAME }} ECS_USE_FARGATE_SPOT: true ECS_USE_PUBLIC_SUBNETS: false ENVIRONMENT: "staging" @@ -643,6 +653,11 @@ jobs: frontend_use_fargate_spot=$FRONTEND_USE_FARGATE_SPOT lambda_function_name="$LAMBDA_FUNCTION_NAME" project_name="$PROJECT_NAME" + + secret_recovery_window_in_days = 0 + db_backup_retention_period = 0 + db_deletion_protection = false + db_skip_final_snapshot = true EOF - name: Terraform Init @@ -674,10 +689,175 @@ jobs: retention-days: 3 timeout-minutes: 15 + deploy-staging-nest: + name: Deploy Nest Staging + env: + FORCE_COLOR: 1 + TF_INPUT: false + TF_IN_AUTOMATION: true + environment: staging + #if: | + #github.repository == 'OWASP/Nest' && + #github.ref == 'refs/heads/main' + needs: + - plan-staging-nest + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-region: ${{ vars.AWS_REGION }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + - name: Install Terraform + uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd + with: + terraform_version: 1.14.0 + + - name: Prepare terraform backend + env: + AWS_REGION: ${{ vars.AWS_REGION }} + TF_STATE_BUCKET: ${{ secrets.TF_STATE_BUCKET }} + TF_STATE_DYNAMODB_TABLE: ${{ secrets.TF_STATE_DYNAMODB_TABLE }} + run: | + umask 377 + cat > infrastructure/staging/terraform.tfbackend <<-EOF + bucket="$TF_STATE_BUCKET" + dynamodb_table="$TF_STATE_DYNAMODB_TABLE" + region="$AWS_REGION" + EOF + + - name: Terraform Init + working-directory: infrastructure/staging + run: terraform init -backend-config=terraform.tfbackend + + - name: Download plan artifact + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + with: + name: tfplan + path: infrastructure/staging + + - name: Terraform Apply + working-directory: infrastructure/staging + run: terraform apply -auto-approve tfplan + + - name: Install Poetry + run: pipx install poetry + + - name: Set up Python + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 + with: + cache: 'poetry' + cache-dependency-path: backend/poetry.lock + python-version: '3.13' + + - name: Install backend dependencies + working-directory: backend + run: poetry sync --no-root --without dev --without test --without video + + - name: Prepare Zappa settings + env: + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_PRIVATE_SUBNET_A: ${{ secrets.AWS_PRIVATE_SUBNET_A }} + AWS_PRIVATE_SUBNET_B: ${{ secrets.AWS_PRIVATE_SUBNET_B }} + AWS_PRIVATE_SUBNET_C: ${{ secrets.AWS_PRIVATE_SUBNET_C }} + AWS_REGION: ${{ vars.AWS_REGION }} + ZAPPA_S3_BUCKET: ${{ secrets.ZAPPA_S3_BUCKET }} + ZAPPA_VPC_SECURITY_GROUP_ID: ${{ secrets.ZAPPA_VPC_SECURITY_GROUP_ID }} + run: envsubst < backend/zappa_settings.template.json > backend/zappa_settings.json + + - name: Update Zappa deployment + working-directory: backend + run: poetry run zappa update staging + + - name: Run ECS migrate task + id: migrate-task + env: + CLUSTER_NAME: nest-staging-tasks-cluster + ECS_SECURITY_GROUP_ID: ${{ secrets.AWS_ECS_SECURITY_GROUP_ID }} + PRIVATE_SUBNET_A: ${{ secrets.AWS_PRIVATE_SUBNET_A }} + PRIVATE_SUBNET_B: ${{ secrets.AWS_PRIVATE_SUBNET_B }} + PRIVATE_SUBNET_C: ${{ secrets.AWS_PRIVATE_SUBNET_C }} + TASK_DEFINITION: nest-staging-migrate + run: | + TASK_ARN=$(aws ecs run-task \ + --cluster "$CLUSTER_NAME" \ + --task-definition "$TASK_DEFINITION" \ + --launch-type FARGATE \ + --network-configuration "awsvpcConfiguration={subnets=[$PRIVATE_SUBNET_A,$PRIVATE_SUBNET_B,$PRIVATE_SUBNET_C],securityGroups=[$ECS_SECURITY_GROUP_ID],assignPublicIp=DISABLED}" \ + --query 'tasks[0].taskArn' \ + --output text) + echo "task_arn=$TASK_ARN" >> $GITHUB_OUTPUT + + - name: Wait for migrate task + env: + CLUSTER_NAME: nest-staging-tasks-cluster + run: | + aws ecs wait tasks-stopped \ + --cluster "$CLUSTER_NAME" \ + --tasks "${{ steps.migrate-task.outputs.task_arn }}" + + - name: Check migrate task exit code + env: + CLUSTER_NAME: nest-staging-tasks-cluster + run: | + EXIT_CODE=$(aws ecs describe-tasks \ + --cluster "$CLUSTER_NAME" \ + --tasks "${{ steps.migrate-task.outputs.task_arn }}" \ + --query 'tasks[0].containers[0].exitCode' \ + --output text) + + if [ "$EXIT_CODE" != "0" ]; then + echo "::error::Migrate task failed with exit code $EXIT_CODE" + exit 1 + fi + echo "Migrate task completed successfully" + + - name: Run ECS index-data task + env: + CLUSTER_NAME: nest-staging-tasks-cluster + ECS_SECURITY_GROUP_ID: ${{ secrets.AWS_ECS_SECURITY_GROUP_ID }} + PRIVATE_SUBNET_A: ${{ secrets.AWS_PRIVATE_SUBNET_A }} + PRIVATE_SUBNET_B: ${{ secrets.AWS_PRIVATE_SUBNET_B }} + PRIVATE_SUBNET_C: ${{ secrets.AWS_PRIVATE_SUBNET_C }} + TASK_DEFINITION: nest-staging-index-data + run: | + aws ecs run-task \ + --cluster "$CLUSTER_NAME" \ + --task-definition "$TASK_DEFINITION" \ + --launch-type FARGATE \ + --network-configuration "awsvpcConfiguration={subnets=[$PRIVATE_SUBNET_A,$PRIVATE_SUBNET_B,$PRIVATE_SUBNET_C],securityGroups=[$ECS_SECURITY_GROUP_ID],assignPublicIp=DISABLED}" + echo "Index-data task started (runs async, ~30 min)" + + - name: Force new frontend deployment + env: + CLUSTER_NAME: nest-staging-frontend-cluster + SERVICE_NAME: nest-staging-frontend-service + run: | + aws ecs update-service \ + --cluster "$CLUSTER_NAME" \ + --service "$SERVICE_NAME" \ + --force-new-deployment + + - name: Wait for frontend service stability + env: + CLUSTER_NAME: nest-staging-frontend-cluster + SERVICE_NAME: nest-staging-frontend-service + run: | + aws ecs wait services-stable \ + --cluster "$CLUSTER_NAME" \ + --services "$SERVICE_NAME" + run-staging-lighthouse-ci: name: Run Lighthouse CI needs: - - deploy-production-nest + - deploy-staging-nest permissions: contents: read runs-on: ubuntu-latest @@ -709,7 +889,7 @@ jobs: run-staging-zap-baseline-scan: name: Run staging ZAP baseline scan needs: - - deploy-production-nest + - deploy-staging-nest permissions: contents: read runs-on: ubuntu-latest diff --git a/backend/zappa_callback.py b/backend/zappa_callback.py index aac6296b66..e0edc2d500 100644 --- a/backend/zappa_callback.py +++ b/backend/zappa_callback.py @@ -58,8 +58,13 @@ def update_alias(zappa): print("No published versions found, skipping alias update") return - client.update_alias(FunctionName=zappa.lambda_name, Name="live", FunctionVersion=published[-1]) - print(f"Alias 'live' now points to version {published[-1]}") + try: + client.update_alias( + FunctionName=zappa.lambda_name, Name="live", FunctionVersion=published[-1] + ) + print(f"Alias 'live' now points to version {published[-1]}") + except client.exceptions.ResourceNotFoundException: + print("Alias 'live' not found, skipping alias update") def cleanup_versions(zappa, keep=5): diff --git a/backend/zappa_settings.example.json b/backend/zappa_settings.template.json similarity index 92% rename from backend/zappa_settings.example.json rename to backend/zappa_settings.template.json index 7da7021d81..b2775f4e2a 100644 --- a/backend/zappa_settings.example.json +++ b/backend/zappa_settings.template.json @@ -32,6 +32,7 @@ "/botocore/", "/botocore-", "/data/", + "/generated_videos/", "/pip/", "/pip-", "/setuptools/", @@ -45,7 +46,9 @@ "snap_start": "PublishedVersions", "timeout_seconds": 60, "vpc_config": { - "SecurityGroupIds": ["${AWS_LAMBDA_VPC_SECURITY_GROUP_ID}"], + "SecurityGroupIds": [ + "${ZAPPA_VPC_SECURITY_GROUP_ID}" + ], "SubnetIds": [ "${AWS_PRIVATE_SUBNET_A}", "${AWS_PRIVATE_SUBNET_B}", diff --git a/cspell/custom-dict.txt b/cspell/custom-dict.txt index e2d9cc1bfb..fd69ff3fa0 100644 --- a/cspell/custom-dict.txt +++ b/cspell/custom-dict.txt @@ -71,6 +71,7 @@ dockerhub dsn elevenlabs env +envsubst euo facebookexternalhit gamesec diff --git a/infrastructure/README.md b/infrastructure/README.md index 8af18a25f7..5f5673bdb5 100644 --- a/infrastructure/README.md +++ b/infrastructure/README.md @@ -132,7 +132,7 @@ The Django backend deployment is managed by Zappa. This includes the IAM roles, - Copy the contents from the template file into your new local environment file: ```bash - cat zappa_settings.example.json > zappa_settings.json + cat zappa_settings.template.json > zappa_settings.json ``` 4. **Populate Settings File**: diff --git a/infrastructure/modules/parameters/main.tf b/infrastructure/modules/parameters/main.tf index 9467b7d064..8fa94af792 100644 --- a/infrastructure/modules/parameters/main.tf +++ b/infrastructure/modules/parameters/main.tf @@ -188,11 +188,7 @@ resource "aws_ssm_parameter" "next_server_csrf_url" { name = "/${var.project_name}/${var.environment}/NEXT_SERVER_CSRF_URL" tags = var.common_tags type = "String" - value = "to-be-set-in-aws-console" - - lifecycle { - ignore_changes = [value] - } + value = var.server_csrf_url } resource "aws_ssm_parameter" "next_server_disable_ssr" { @@ -236,11 +232,7 @@ resource "aws_ssm_parameter" "next_server_graphql_url" { name = "/${var.project_name}/${var.environment}/NEXT_SERVER_GRAPHQL_URL" tags = var.common_tags type = "String" - value = "to-be-set-in-aws-console" - - lifecycle { - ignore_changes = [value] - } + value = var.server_graphql_url } resource "aws_ssm_parameter" "nextauth_secret" { diff --git a/infrastructure/modules/parameters/variables.tf b/infrastructure/modules/parameters/variables.tf index 21f0866572..00a7d20a64 100644 --- a/infrastructure/modules/parameters/variables.tf +++ b/infrastructure/modules/parameters/variables.tf @@ -72,6 +72,16 @@ variable "redis_password_arn" { sensitive = true } +variable "server_csrf_url" { + description = "The server-side CSRF URL for Next.js SSR (e.g., https://nest.owasp.dev/csrf/)." + type = string +} + +variable "server_graphql_url" { + description = "The server-side GraphQL URL for Next.js SSR (e.g., https://nest.owasp.dev/graphql/)." + type = string +} + variable "settings_module" { description = "The location of the Django settings module to use (e.g., settings.staging, settings.production)." type = string diff --git a/infrastructure/staging/main.tf b/infrastructure/staging/main.tf index a7caa1573b..10a80475b4 100644 --- a/infrastructure/staging/main.tf +++ b/infrastructure/staging/main.tf @@ -142,6 +142,8 @@ module "parameters" { project_name = var.project_name redis_host = module.cache.redis_primary_endpoint redis_password_arn = module.cache.redis_password_arn + server_csrf_url = var.domain_name != null ? "https://${var.domain_name}/csrf/" : "http://${module.alb.alb_dns_name}/csrf/" + server_graphql_url = var.domain_name != null ? "https://${var.domain_name}/graphql/" : "http://${module.alb.alb_dns_name}/graphql/" } module "security" { diff --git a/infrastructure/staging/outputs.tf b/infrastructure/staging/outputs.tf index 07d711b45e..384b86b87e 100644 --- a/infrastructure/staging/outputs.tf +++ b/infrastructure/staging/outputs.tf @@ -18,6 +18,11 @@ output "backend_ecr_repository_url" { value = module.ecs.ecr_repository_url } +output "ecs_security_group_id" { + description = "The ID of the security group for ECS tasks." + value = module.security.ecs_sg_id +} + output "frontend_ecr_repository_url" { description = "The URL of the frontend ECR repository." value = module.frontend.ecr_repository_url From fa8c292f3a3f763eeede773157867a72bca83d66 Mon Sep 17 00:00:00 2001 From: Rudransh Shrivastava Date: Wed, 21 Jan 2026 11:42:06 +0530 Subject: [PATCH 3/7] Update zappa settings --- backend/zappa_settings.template.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/zappa_settings.template.json b/backend/zappa_settings.template.json index b2775f4e2a..40cc52433d 100644 --- a/backend/zappa_settings.template.json +++ b/backend/zappa_settings.template.json @@ -3,7 +3,7 @@ "apigateway_enabled": false, "app_function": "wsgi.application", "aws_environment_variables": { - "AWS_SYSTEMS_MANAGER_PARAM_STORE_PATH": "/owasp-nest/staging" + "AWS_SYSTEMS_MANAGER_PARAM_STORE_PATH": "/nest/staging" }, "callbacks": { "post": "zappa_callback.post", @@ -18,8 +18,8 @@ "ssm:GetParameter" ], "Resource": [ - "arn:aws:ssm:${AWS_REGION}:${AWS_ACCOUNT_ID}:parameter/owasp-nest/staging", - "arn:aws:ssm:${AWS_REGION}:${AWS_ACCOUNT_ID}:parameter/owasp-nest/staging/*" + "arn:aws:ssm:${AWS_REGION}:${AWS_ACCOUNT_ID}:parameter/nest/staging", + "arn:aws:ssm:${AWS_REGION}:${AWS_ACCOUNT_ID}:parameter/nest/staging/*" ] } ], From caeb22dba3ede7b212cf31c3bd6e61807f415072 Mon Sep 17 00:00:00 2001 From: Rudransh Shrivastava Date: Wed, 21 Jan 2026 12:20:34 +0530 Subject: [PATCH 4/7] update code --- .github/workflows/run-ci-cd.yaml | 19 +++++++------------ backend/Makefile | 4 ++++ backend/zappa_settings.template.json | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/run-ci-cd.yaml b/.github/workflows/run-ci-cd.yaml index 8bf82b8f08..337f2f1660 100644 --- a/.github/workflows/run-ci-cd.yaml +++ b/.github/workflows/run-ci-cd.yaml @@ -590,11 +590,11 @@ jobs: TF_INPUT: false TF_IN_AUTOMATION: true environment: staging - #if: | - #github.repository == 'OWASP/Nest' && - #github.ref == 'refs/heads/main' + if: | + github.repository == 'OWASP/Nest' && + github.ref == 'refs/heads/main' needs: - #- scan-staging-images + - scan-staging-images - set-release-version permissions: contents: read @@ -653,11 +653,6 @@ jobs: frontend_use_fargate_spot=$FRONTEND_USE_FARGATE_SPOT lambda_function_name="$LAMBDA_FUNCTION_NAME" project_name="$PROJECT_NAME" - - secret_recovery_window_in_days = 0 - db_backup_retention_period = 0 - db_deletion_protection = false - db_skip_final_snapshot = true EOF - name: Terraform Init @@ -696,9 +691,9 @@ jobs: TF_INPUT: false TF_IN_AUTOMATION: true environment: staging - #if: | - #github.repository == 'OWASP/Nest' && - #github.ref == 'refs/heads/main' + if: | + github.repository == 'OWASP/Nest' && + github.ref == 'refs/heads/main' needs: - plan-staging-nest permissions: diff --git a/backend/Makefile b/backend/Makefile index 0fd0084621..4d01860155 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -5,7 +5,11 @@ include backend/apps/nest/Makefile include backend/apps/owasp/Makefile include backend/apps/slack/Makefile +ifeq ($(EXEC_MODE),direct) +SHELL := /bin/sh +else SHELL := /bin/bash +endif build-backend-local-image: @DOCKER_BUILDKIT=1 docker build \ diff --git a/backend/zappa_settings.template.json b/backend/zappa_settings.template.json index 40cc52433d..858ee682f8 100644 --- a/backend/zappa_settings.template.json +++ b/backend/zappa_settings.template.json @@ -43,7 +43,7 @@ "runtime": "python3.13", "s3_bucket": "${ZAPPA_S3_BUCKET}", "slim_handler": true, - "snap_start": "PublishedVersions", + "snap_start": "None", "timeout_seconds": 60, "vpc_config": { "SecurityGroupIds": [ From fbf27a926f24995dbdae52c1a78df7ed5de92e2d Mon Sep 17 00:00:00 2001 From: Rudransh Shrivastava Date: Thu, 22 Jan 2026 18:10:47 +0530 Subject: [PATCH 5/7] cleanup old staging setup --- .github/ansible/inventory.yaml | 12 ---- .github/ansible/staging/nest.yaml | 82 --------------------------- .github/ansible/staging/proxy.yaml | 29 ---------- cron/staging | 1 - docker-compose/staging/compose.yaml | 88 ----------------------------- 5 files changed, 212 deletions(-) delete mode 100644 .github/ansible/staging/nest.yaml delete mode 100644 .github/ansible/staging/proxy.yaml delete mode 100644 cron/staging delete mode 100644 docker-compose/staging/compose.yaml diff --git a/.github/ansible/inventory.yaml b/.github/ansible/inventory.yaml index 192f76a56d..bc12d33eb5 100644 --- a/.github/ansible/inventory.yaml +++ b/.github/ansible/inventory.yaml @@ -11,15 +11,3 @@ nest: ansible_python_interpreter: /usr/bin/python3 ansible_ssh_private_key_file: "{{ lookup('env', 'PROXY_SSH_PRIVATE_KEY_PATH') }}" ansible_user: proxy - - staging_nest: - ansible_host: "{{ lookup('env', 'NEST_HOST_IP_ADDRESS') }}" - ansible_python_interpreter: /usr/bin/python3 - ansible_ssh_private_key_file: "{{ lookup('env', 'NEST_SSH_PRIVATE_KEY_PATH') }}" - ansible_user: staging - - staging_nest_proxy: - ansible_host: "{{ lookup('env', 'PROXY_HOST_IP_ADDRESS') }}" - ansible_python_interpreter: /usr/bin/python3 - ansible_ssh_private_key_file: "{{ lookup('env', 'PROXY_SSH_PRIVATE_KEY_PATH') }}" - ansible_user: proxy diff --git a/.github/ansible/staging/nest.yaml b/.github/ansible/staging/nest.yaml deleted file mode 100644 index c8dafb5d55..0000000000 --- a/.github/ansible/staging/nest.yaml +++ /dev/null @@ -1,82 +0,0 @@ - - name: Deploy Nest to Staging - hosts: staging_nest - tasks: - - name: Copy docker-compose.yaml - ansible.builtin.copy: - src: '{{ github_workspace }}/docker-compose/staging/compose.yaml' - dest: ~/docker-compose.yaml - mode: '0644' - - - name: Sync Makefile structure - ansible.builtin.synchronize: - src: '{{ github_workspace }}/' - dest: '~/' - recursive: yes - rsync_opts: - - '--include=*/' - - '--include=Makefile' - - '--include=*/Makefile' - - '--include=*/**/Makefile' - - '--include=*/**/**/Makefile' - - '--exclude=*' - - - name: Update backend makefile for the staging environment - shell: - cmd: sed -i 's/\bnest-backend\b/staging-nest-backend/' ~/backend/Makefile - - - name: Update frontend makefile for the staging environment - shell: - cmd: sed -i 's/\bnest-frontend\b/staging-nest-frontend/' ~/frontend/Makefile - - - name: Ensure data directory exists - ansible.builtin.file: - path: ~/backend/data - state: directory - mode: '0755' - - - name: Copy secrets - copy: - src: '{{ github_workspace }}/{{ item }}' - dest: ~/ - mode: '0400' - loop: - - .env.backend - - .env.cache - - .env.db - - .env.frontend - - - name: Clean up secrets - delegate_to: localhost - file: - path: '{{ github_workspace }}/{{ item }}' - state: absent - loop: - - .env.backend - - .env.cache - - .env.db - - .env.frontend - run_once: true - - - name: Copy crontab - copy: - src: '{{ github_workspace }}/cron/staging' - dest: /tmp/staging_crontab - mode: '0600' - - - name: Install crontab - ansible.builtin.command: - cmd: crontab /tmp/staging_crontab - - - name: Restart services - shell: - cmd: docker compose up -d --pull always - - - name: Prune docker images - shell: - cmd: docker image prune -f - - - name: Index data - async: 1800 # 30 minutes - poll: 0 - shell: | - make index-data > /var/log/nest/staging/index-data.log 2>&1 diff --git a/.github/ansible/staging/proxy.yaml b/.github/ansible/staging/proxy.yaml deleted file mode 100644 index dc75d8c107..0000000000 --- a/.github/ansible/staging/proxy.yaml +++ /dev/null @@ -1,29 +0,0 @@ -- name: Deploy Staging Nest Proxy - hosts: staging_nest_proxy - tasks: - - name: Copy proxy configuration files - copy: - src: '{{ github_workspace }}/proxy/{{ item }}' - dest: ~/ - mode: '0644' - loop: - - blocked_ips.conf - - cloudflare_realip.conf - - headers.conf - - proxy_cache.conf - - redirects.conf - - staging.conf - - - name: Copy docker compose file - copy: - src: '{{ github_workspace }}/docker-compose/proxy/compose.yaml' - dest: ~/docker-compose.yaml - mode: '0644' - - - name: Restart services - shell: - cmd: docker compose up -d --pull always && docker compose restart - - - name: Prune docker images - shell: - cmd: docker image prune -f diff --git a/cron/staging b/cron/staging deleted file mode 100644 index 93765b370d..0000000000 --- a/cron/staging +++ /dev/null @@ -1 +0,0 @@ -# OWASP Nest staging cron job schedule. diff --git a/docker-compose/staging/compose.yaml b/docker-compose/staging/compose.yaml deleted file mode 100644 index 051ce8433d..0000000000 --- a/docker-compose/staging/compose.yaml +++ /dev/null @@ -1,88 +0,0 @@ -services: - staging-nest-backend: - container_name: staging-nest-backend - entrypoint: /home/owasp/entrypoint.sh - image: owasp/nest:backend-staging - env_file: .env.backend - depends_on: - staging-nest-cache: - condition: service_healthy - staging-nest-db: - condition: service_healthy - restart: unless-stopped - networks: - - nest-app-network - - nest-cache-network - - nest-db-network - volumes: - - ./backend/data:/home/owasp/data - - staging-nest-cache: - container_name: staging-nest-cache - image: redis:8.0.5-alpine3.21 - command: > - sh -c ' - redis-server --requirepass $$REDIS_PASSWORD --maxmemory 25mb --maxmemory-policy allkeys-lru - ' - env_file: .env.cache - healthcheck: - interval: 5s - retries: 5 - test: [CMD, redis-cli, -a, $$REDIS_PASSWORD, ping] - timeout: 5s - restart: unless-stopped - volumes: - - ./volumes/cache:/data - networks: - - nest-cache-network - - staging-nest-db: - container_name: staging-nest-db - image: pgvector/pgvector:pg16 - env_file: .env.db - healthcheck: - interval: 5s - retries: 5 - test: [CMD, pg_isready, -U, nest_user_staging, -d, nest_db_staging] - timeout: 5s - restart: unless-stopped - volumes: - - ./volumes/db:/var/lib/postgresql/data - networks: - - nest-db-network - - staging-nest-frontend: - container_name: staging-nest-frontend - env_file: .env.frontend - image: owasp/nest:frontend-staging - restart: unless-stopped - networks: - - nest-app-network - - staging-nest-worker: - container_name: staging-nest-worker - image: owasp/nest:backend-staging - env_file: .env.backend - command: > - sh -c ' - python manage.py rqworker ai --with-scheduler - ' - depends_on: - staging-nest-backend: - condition: service_started - staging-nest-cache: - condition: service_healthy - staging-nest-db: - condition: service_healthy - restart: unless-stopped - networks: - - nest-app-network - - nest-cache-network - - nest-db-network - volumes: - - ./backend/data:/home/owasp/data - -networks: - nest-app-network: - nest-cache-network: - nest-db-network: From d5863c589d94e85fd66578ec29aa918f26ad3a9e Mon Sep 17 00:00:00 2001 From: Rudransh Shrivastava Date: Thu, 22 Jan 2026 19:12:08 +0530 Subject: [PATCH 6/7] update terraform outputs and variables and use terraform output as source of ids --- .github/workflows/run-ci-cd.yaml | 61 +++++++++++++---------- infrastructure/backend/variables.tf | 2 +- infrastructure/modules/ecs/outputs.tf | 11 ++-- infrastructure/modules/storage/outputs.tf | 4 +- infrastructure/staging/main.tf | 6 ++- infrastructure/staging/outputs.tf | 19 ++++++- infrastructure/staging/variables.tf | 4 +- 7 files changed, 70 insertions(+), 37 deletions(-) diff --git a/.github/workflows/run-ci-cd.yaml b/.github/workflows/run-ci-cd.yaml index 337f2f1660..1c8eca065b 100644 --- a/.github/workflows/run-ci-cd.yaml +++ b/.github/workflows/run-ci-cd.yaml @@ -475,7 +475,7 @@ jobs: push: true tags: | owasp/nest:backend-staging - ${{ secrets.AWS_ECR_BACKEND_REPOSITORY_URL }}:latest + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/nest-staging-backend:latest - name: Prepare frontend public environment env: @@ -528,7 +528,7 @@ jobs: SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} tags: | owasp/nest:frontend-staging - ${{ secrets.AWS_ECR_FRONTEND_REPOSITORY_URL }}:latest + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/nest-staging-frontend:latest - name: Get frontend image size id: frontend-size @@ -742,6 +742,21 @@ jobs: working-directory: infrastructure/staging run: terraform apply -auto-approve tfplan + - name: Capture terraform outputs + id: tf-outputs + working-directory: infrastructure/staging + run: | + echo "ecs_security_group_id=$(terraform output -raw ecs_security_group_id)" >> $GITHUB_OUTPUT + echo "frontend_cluster_name=$(terraform output -raw frontend_cluster_name)" >> $GITHUB_OUTPUT + echo "frontend_service_name=$(terraform output -raw frontend_service_name)" >> $GITHUB_OUTPUT + echo "lambda_security_group_id=$(terraform output -raw lambda_security_group_id)" >> $GITHUB_OUTPUT + echo "private_subnet_a=$(terraform output -json private_subnet_ids | jq -r '.[0]')" >> $GITHUB_OUTPUT + echo "private_subnet_b=$(terraform output -json private_subnet_ids | jq -r '.[1]')" >> $GITHUB_OUTPUT + echo "private_subnet_c=$(terraform output -json private_subnet_ids | jq -r '.[2]')" >> $GITHUB_OUTPUT + echo "private_subnet_ids=$(terraform output -json private_subnet_ids | jq -r 'join(",")')" >> $GITHUB_OUTPUT + echo "tasks_cluster_name=$(terraform output -raw tasks_cluster_name)" >> $GITHUB_OUTPUT + echo "zappa_s3_bucket=$(terraform output -raw zappa_s3_bucket_name)" >> $GITHUB_OUTPUT + - name: Install Poetry run: pipx install poetry @@ -759,12 +774,12 @@ jobs: - name: Prepare Zappa settings env: AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - AWS_PRIVATE_SUBNET_A: ${{ secrets.AWS_PRIVATE_SUBNET_A }} - AWS_PRIVATE_SUBNET_B: ${{ secrets.AWS_PRIVATE_SUBNET_B }} - AWS_PRIVATE_SUBNET_C: ${{ secrets.AWS_PRIVATE_SUBNET_C }} + AWS_PRIVATE_SUBNET_A: ${{ steps.tf-outputs.outputs.private_subnet_a }} + AWS_PRIVATE_SUBNET_B: ${{ steps.tf-outputs.outputs.private_subnet_b }} + AWS_PRIVATE_SUBNET_C: ${{ steps.tf-outputs.outputs.private_subnet_c }} AWS_REGION: ${{ vars.AWS_REGION }} - ZAPPA_S3_BUCKET: ${{ secrets.ZAPPA_S3_BUCKET }} - ZAPPA_VPC_SECURITY_GROUP_ID: ${{ secrets.ZAPPA_VPC_SECURITY_GROUP_ID }} + ZAPPA_S3_BUCKET: ${{ steps.tf-outputs.outputs.zappa_s3_bucket }} + ZAPPA_VPC_SECURITY_GROUP_ID: ${{ steps.tf-outputs.outputs.lambda_security_group_id }} run: envsubst < backend/zappa_settings.template.json > backend/zappa_settings.json - name: Update Zappa deployment @@ -774,25 +789,23 @@ jobs: - name: Run ECS migrate task id: migrate-task env: - CLUSTER_NAME: nest-staging-tasks-cluster - ECS_SECURITY_GROUP_ID: ${{ secrets.AWS_ECS_SECURITY_GROUP_ID }} - PRIVATE_SUBNET_A: ${{ secrets.AWS_PRIVATE_SUBNET_A }} - PRIVATE_SUBNET_B: ${{ secrets.AWS_PRIVATE_SUBNET_B }} - PRIVATE_SUBNET_C: ${{ secrets.AWS_PRIVATE_SUBNET_C }} + CLUSTER_NAME: ${{ steps.tf-outputs.outputs.tasks_cluster_name }} + ECS_SECURITY_GROUP_ID: ${{ steps.tf-outputs.outputs.ecs_security_group_id }} + PRIVATE_SUBNET_IDS: ${{ steps.tf-outputs.outputs.private_subnet_ids }} TASK_DEFINITION: nest-staging-migrate run: | TASK_ARN=$(aws ecs run-task \ --cluster "$CLUSTER_NAME" \ --task-definition "$TASK_DEFINITION" \ --launch-type FARGATE \ - --network-configuration "awsvpcConfiguration={subnets=[$PRIVATE_SUBNET_A,$PRIVATE_SUBNET_B,$PRIVATE_SUBNET_C],securityGroups=[$ECS_SECURITY_GROUP_ID],assignPublicIp=DISABLED}" \ + --network-configuration "awsvpcConfiguration={subnets=[$PRIVATE_SUBNET_IDS],securityGroups=[$ECS_SECURITY_GROUP_ID],assignPublicIp=DISABLED}" \ --query 'tasks[0].taskArn' \ --output text) echo "task_arn=$TASK_ARN" >> $GITHUB_OUTPUT - name: Wait for migrate task env: - CLUSTER_NAME: nest-staging-tasks-cluster + CLUSTER_NAME: ${{ steps.tf-outputs.outputs.tasks_cluster_name }} run: | aws ecs wait tasks-stopped \ --cluster "$CLUSTER_NAME" \ @@ -800,7 +813,7 @@ jobs: - name: Check migrate task exit code env: - CLUSTER_NAME: nest-staging-tasks-cluster + CLUSTER_NAME: ${{ steps.tf-outputs.outputs.tasks_cluster_name }} run: | EXIT_CODE=$(aws ecs describe-tasks \ --cluster "$CLUSTER_NAME" \ @@ -816,24 +829,22 @@ jobs: - name: Run ECS index-data task env: - CLUSTER_NAME: nest-staging-tasks-cluster - ECS_SECURITY_GROUP_ID: ${{ secrets.AWS_ECS_SECURITY_GROUP_ID }} - PRIVATE_SUBNET_A: ${{ secrets.AWS_PRIVATE_SUBNET_A }} - PRIVATE_SUBNET_B: ${{ secrets.AWS_PRIVATE_SUBNET_B }} - PRIVATE_SUBNET_C: ${{ secrets.AWS_PRIVATE_SUBNET_C }} + CLUSTER_NAME: ${{ steps.tf-outputs.outputs.tasks_cluster_name }} + ECS_SECURITY_GROUP_ID: ${{ steps.tf-outputs.outputs.ecs_security_group_id }} + PRIVATE_SUBNET_IDS: ${{ steps.tf-outputs.outputs.private_subnet_ids }} TASK_DEFINITION: nest-staging-index-data run: | aws ecs run-task \ --cluster "$CLUSTER_NAME" \ --task-definition "$TASK_DEFINITION" \ --launch-type FARGATE \ - --network-configuration "awsvpcConfiguration={subnets=[$PRIVATE_SUBNET_A,$PRIVATE_SUBNET_B,$PRIVATE_SUBNET_C],securityGroups=[$ECS_SECURITY_GROUP_ID],assignPublicIp=DISABLED}" + --network-configuration "awsvpcConfiguration={subnets=[$PRIVATE_SUBNET_IDS],securityGroups=[$ECS_SECURITY_GROUP_ID],assignPublicIp=DISABLED}" echo "Index-data task started (runs async, ~30 min)" - name: Force new frontend deployment env: - CLUSTER_NAME: nest-staging-frontend-cluster - SERVICE_NAME: nest-staging-frontend-service + CLUSTER_NAME: ${{ steps.tf-outputs.outputs.frontend_cluster_name }} + SERVICE_NAME: ${{ steps.tf-outputs.outputs.frontend_service_name }} run: | aws ecs update-service \ --cluster "$CLUSTER_NAME" \ @@ -842,8 +853,8 @@ jobs: - name: Wait for frontend service stability env: - CLUSTER_NAME: nest-staging-frontend-cluster - SERVICE_NAME: nest-staging-frontend-service + CLUSTER_NAME: ${{ steps.tf-outputs.outputs.frontend_cluster_name }} + SERVICE_NAME: ${{ steps.tf-outputs.outputs.frontend_service_name }} run: | aws ecs wait services-stable \ --cluster "$CLUSTER_NAME" \ diff --git a/infrastructure/backend/variables.tf b/infrastructure/backend/variables.tf index 72789673c3..a0365669a4 100644 --- a/infrastructure/backend/variables.tf +++ b/infrastructure/backend/variables.tf @@ -25,5 +25,5 @@ variable "noncurrent_version_expiration_days" { variable "project_name" { description = "The name of the project." type = string - default = "owasp-nest" + default = "nest" } diff --git a/infrastructure/modules/ecs/outputs.tf b/infrastructure/modules/ecs/outputs.tf index 849a9b5e52..9231b1e852 100644 --- a/infrastructure/modules/ecs/outputs.tf +++ b/infrastructure/modules/ecs/outputs.tf @@ -1,9 +1,14 @@ +output "ecr_repository_url" { + description = "The URL of the ECR repository." + value = aws_ecr_repository.main.repository_url +} + output "ecs_cluster_arn" { description = "The ARN of the ECS cluster." value = aws_ecs_cluster.main.arn } -output "ecr_repository_url" { - description = "The URL of the ECR repository." - value = aws_ecr_repository.main.repository_url +output "ecs_cluster_name" { + description = "The name of the ECS tasks cluster." + value = aws_ecs_cluster.main.name } diff --git a/infrastructure/modules/storage/outputs.tf b/infrastructure/modules/storage/outputs.tf index a771d3ccfb..94887ccc20 100644 --- a/infrastructure/modules/storage/outputs.tf +++ b/infrastructure/modules/storage/outputs.tf @@ -13,9 +13,9 @@ output "fixtures_s3_bucket_name" { value = module.fixtures_bucket.bucket.id } -output "zappa_s3_bucket" { +output "zappa_s3_bucket_name" { description = "The name of the S3 bucket for Zappa deployments." - value = module.zappa_bucket.bucket + value = module.zappa_bucket.bucket.id } output "zappa_s3_bucket_arn" { diff --git a/infrastructure/staging/main.tf b/infrastructure/staging/main.tf index 10a80475b4..d2468d7681 100644 --- a/infrastructure/staging/main.tf +++ b/infrastructure/staging/main.tf @@ -15,6 +15,8 @@ locals { ManagedBy = "Terraform" Project = var.project_name } + fixtures_bucket_name = coalesce(var.fixtures_bucket_name, "${var.project_name}-fixtures") + zappa_bucket_name = coalesce(var.zappa_bucket_name, "${var.project_name}-zappa-deployments") } module "alb" { @@ -165,7 +167,7 @@ module "storage" { common_tags = local.common_tags environment = var.environment - fixtures_bucket_name = var.fixtures_bucket_name + fixtures_bucket_name = local.fixtures_bucket_name project_name = var.project_name - zappa_bucket_name = var.zappa_bucket_name + zappa_bucket_name = local.zappa_bucket_name } diff --git a/infrastructure/staging/outputs.tf b/infrastructure/staging/outputs.tf index 384b86b87e..6eecba8dea 100644 --- a/infrastructure/staging/outputs.tf +++ b/infrastructure/staging/outputs.tf @@ -23,11 +23,21 @@ output "ecs_security_group_id" { value = module.security.ecs_sg_id } +output "frontend_cluster_name" { + description = "The name of the ECS frontend cluster." + value = module.frontend.ecs_cluster_name +} + output "frontend_ecr_repository_url" { description = "The URL of the frontend ECR repository." value = module.frontend.ecr_repository_url } +output "frontend_service_name" { + description = "The name of the ECS frontend service." + value = module.frontend.ecs_service_name +} + output "frontend_url" { description = "The URL to access the frontend." value = var.domain_name != null ? "https://${var.domain_name}" : "http://${module.alb.alb_dns_name}" @@ -43,7 +53,12 @@ output "private_subnet_ids" { value = module.networking.private_subnet_ids } -output "zappa_s3_bucket" { +output "tasks_cluster_name" { + description = "The name of the ECS tasks cluster." + value = module.ecs.ecs_cluster_name +} + +output "zappa_s3_bucket_name" { description = "The name of the S3 bucket for Zappa deployments." - value = module.storage.zappa_s3_bucket.bucket + value = module.storage.zappa_s3_bucket_name } diff --git a/infrastructure/staging/variables.tf b/infrastructure/staging/variables.tf index 35fe3cd7a4..fa1b2c0da0 100644 --- a/infrastructure/staging/variables.tf +++ b/infrastructure/staging/variables.tf @@ -160,7 +160,7 @@ variable "environment" { variable "fixtures_bucket_name" { description = "The name of the S3 bucket for fixtures." type = string - default = "owasp-nest-fixtures" + default = null } variable "frontend_desired_count" { @@ -285,5 +285,5 @@ variable "vpc_cidr" { variable "zappa_bucket_name" { description = "The name of the S3 bucket for Zappa deployments." type = string - default = "owasp-nest-zappa-deployments" + default = null } From ede4614426b3057211bc031abd7fc92eb5fab63b Mon Sep 17 00:00:00 2001 From: Rudransh Shrivastava Date: Fri, 23 Jan 2026 01:05:11 +0530 Subject: [PATCH 7/7] append _NAME to environment secrets --- .github/workflows/run-ci-cd.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/run-ci-cd.yaml b/.github/workflows/run-ci-cd.yaml index 1c8eca065b..3501b3c4ed 100644 --- a/.github/workflows/run-ci-cd.yaml +++ b/.github/workflows/run-ci-cd.yaml @@ -618,13 +618,13 @@ jobs: - name: Prepare terraform backend env: AWS_REGION: ${{ vars.AWS_REGION }} - TF_STATE_BUCKET: ${{ secrets.TF_STATE_BUCKET }} - TF_STATE_DYNAMODB_TABLE: ${{ secrets.TF_STATE_DYNAMODB_TABLE }} + TF_STATE_BUCKET_NAME: ${{ secrets.TF_STATE_BUCKET_NAME }} + TF_STATE_DYNAMODB_TABLE_NAME: ${{ secrets.TF_STATE_DYNAMODB_TABLE_NAME }} run: | umask 377 cat > infrastructure/staging/terraform.tfbackend <<-EOF - bucket="$TF_STATE_BUCKET" - dynamodb_table="$TF_STATE_DYNAMODB_TABLE" + bucket="$TF_STATE_BUCKET_NAME" + dynamodb_table="$TF_STATE_DYNAMODB_TABLE_NAME" region="$AWS_REGION" EOF @@ -718,13 +718,13 @@ jobs: - name: Prepare terraform backend env: AWS_REGION: ${{ vars.AWS_REGION }} - TF_STATE_BUCKET: ${{ secrets.TF_STATE_BUCKET }} - TF_STATE_DYNAMODB_TABLE: ${{ secrets.TF_STATE_DYNAMODB_TABLE }} + TF_STATE_BUCKET_NAME: ${{ secrets.TF_STATE_BUCKET_NAME }} + TF_STATE_DYNAMODB_TABLE_NAME: ${{ secrets.TF_STATE_DYNAMODB_TABLE_NAME }} run: | umask 377 cat > infrastructure/staging/terraform.tfbackend <<-EOF - bucket="$TF_STATE_BUCKET" - dynamodb_table="$TF_STATE_DYNAMODB_TABLE" + bucket="$TF_STATE_BUCKET_NAME" + dynamodb_table="$TF_STATE_DYNAMODB_TABLE_NAME" region="$AWS_REGION" EOF