Fix long-running ECS/Fargate Tasks (#2620)

* Use AWS SSM Parameter Store to handle environment variables

* Use focused policy for read access

* Update documentation

* Add flag for create_rds_proxy

* set default value of create_rds_proxy to false

* Populate Zappa/Lambda environment variables from ssm/parameter store

* Update documentation

* Update example

* add default configurations

* add security group db from lambda

* fix load-data task by adding a --fixture-path flag

* fix ecs tasks by introducing ecs-* make targets

* change ecs run steps

* remove ecs-* and clean code

* add --no-cache

* use call_command

* add test for --fixture-path

* Update code

* Update backend/wsgi.py

---------

Co-authored-by: Arkadii Yakovets <[email protected]>
Co-authored-by: Arkadii Yakovets <[email protected]>

Author: rudransh-shrivastava

backend/Makefile

@@ -24,10 +24,18 @@ create-superuser:
 	@CMD="python manage.py createsuperuser" $(MAKE) exec-backend-command-it
 
 exec-backend-command:
+ifeq ($(EXEC_MODE),direct)
+	@$(CMD)
+else
 	@docker exec -i nest-backend $(CMD)
+endif
 
 exec-backend-command-it:
+ifeq ($(EXEC_MODE),direct)
+	@$(CMD)
+else
 	@docker exec -it nest-backend $(CMD) 2>/dev/null
+endif
 
 exec-db-command-it:
 	@docker exec -it nest-db $(CMD)

backend/apps/common/management/commands/load_data.py

@@ -10,8 +10,23 @@
 class Command(BaseCommand):
     help = "Load OWASP Nest data."
 
+    def add_arguments(self, parser) -> None:
+        """Add command-line arguments to the parser.
+
+        Args:
+            parser (argparse.ArgumentParser): The argument parser instance.
+
+        """
+        parser.add_argument(
+            "--fixture-path",
+            default="data/nest.json.gz",
+            required=False,
+            type=str,
+            help="Path to the fixture file",
+        )
+
     def handle(self, *_args, **_options) -> None:
         """Load data into the OWASP Nest application."""
         with index.disable_indexing(), transaction.atomic():
             # Run loaddata
-            call_command("loaddata", "data/nest.json.gz", "-v", "3")
+            call_command("loaddata", _options["fixture_path"], "-v", "3")

backend/docker/Dockerfile

@@ -38,16 +38,20 @@ RUN --mount=type=cache,target=${POETRY_CACHE_DIR},uid=${OWASP_UID},gid=${OWASP_G
 COPY apps apps
 COPY docker/entrypoint.sh entrypoint.sh
 COPY manage.py wsgi.py ./
+COPY Makefile Makefile
 COPY settings settings
 COPY static static
 COPY templates templates
 
+# Required to run make targets
+RUN ln -s . backend
+
 FROM python:3.13.7-alpine
 
 ARG OWASP_GID
 ARG OWASP_UID
 
-RUN apk update && \
+RUN apk update && apk --no-cache add make && \
     addgroup -S -g ${OWASP_GID} owasp && \
     adduser -S -h /home/owasp -u ${OWASP_UID} -G owasp owasp
 

backend/tests/apps/common/management/commands/load_data_test.py

@@ -1,7 +1,7 @@
 import contextlib
 from unittest.mock import MagicMock, patch
 
-from apps.common.management.commands.load_data import Command
+from django.core.management import call_command
 
 
 class TestLoadDataCommand:
@@ -26,8 +26,7 @@ def test_handle(
         mock_unregister.return_value = None
         mock_register.return_value = None
 
-        command = Command()
-        command.handle()
+        call_command("load_data")
 
         mock_unregister.assert_called_once()
         mock_register.assert_called_once()
@@ -36,6 +35,38 @@ def test_handle(
 
         mock_atomic.assert_called_once()
 
+    @patch("apps.core.utils.index.DisableIndexing.unregister_indexes")
+    @patch("apps.core.utils.index.DisableIndexing.register_indexes")
+    @patch("apps.common.management.commands.load_data.call_command")
+    @patch("apps.common.management.commands.load_data.transaction.atomic")
+    def test_handle_with_custom_fixture_path(
+        self,
+        mock_atomic,
+        mock_call_command,
+        mock_register,
+        mock_unregister,
+    ):
+        mock_model = MagicMock()
+        mock_app_config = MagicMock()
+        mock_app_config.get_models.return_value = [mock_model]
+
+        mock_atomic.return_value.__enter__ = MagicMock()
+        mock_atomic.return_value.__exit__ = MagicMock()
+
+        mock_unregister.return_value = None
+        mock_register.return_value = None
+
+        call_command("load_data", fixture_path="custom/path/to/fixture.json.gz")
+
+        mock_unregister.assert_called_once()
+        mock_register.assert_called_once()
+
+        mock_call_command.assert_called_once_with(
+            "loaddata", "custom/path/to/fixture.json.gz", "-v", "3"
+        )
+
+        mock_atomic.assert_called_once()
+
     @patch("apps.core.utils.index.DisableIndexing.unregister_indexes")
     @patch("apps.core.utils.index.DisableIndexing.register_indexes")
     @patch("apps.common.management.commands.load_data.call_command")
@@ -50,13 +81,12 @@ def test_handle_with_exception_during_call_command(
         """Test that indexing is re-enabled even if call_command fails."""
         mock_call_command.side_effect = Exception("Call command failed")
 
-        command = Command()
         with patch("contextlib.suppress") as mock_suppress:
             mock_suppress.return_value.__enter__ = MagicMock()
             mock_suppress.return_value.__exit__ = MagicMock()
 
             with contextlib.suppress(Exception):
-                command.handle()
+                call_command("load_data")
 
         mock_unregister.assert_called_once()
         mock_register.assert_called_once()

backend/wsgi.py

@@ -1,16 +1,15 @@
 """WSGI config for OWASP Nest project."""
 
 import os
+from pathlib import Path
 
+import boto3
 
-def _populate_environ_from_ssm():
-    ssm_param_path = os.getenv("AWS_SYSTEMS_MANAGER_PARAM_STORE_PATH")
-    if not ssm_param_path:
-        return
-
-    from pathlib import Path
 
-    import boto3
+def populate_environ_from_ssm():
+    """Populate environment variables from AWS Systems Manager Parameter Store."""
+    if not (ssm_param_path := os.getenv("AWS_SYSTEMS_MANAGER_PARAM_STORE_PATH")):
+        return
 
     client = boto3.client("ssm")
     paginator = client.get_paginator("get_parameters_by_path")
@@ -21,7 +20,7 @@ def _populate_environ_from_ssm():
             os.environ[Path(param["Name"]).name] = param["Value"]
 
 
-_populate_environ_from_ssm()
+populate_environ_from_ssm()
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.local")
 os.environ.setdefault("DJANGO_CONFIGURATION", "Local")

infrastructure/README.md

@@ -158,12 +158,12 @@ Migrate and load data into the new database.
 3. **Run ECS Tasks**:
    - Head over to Elastic Container Service in the AWS Console.
    - Click on `owasp-nest-staging-migrate` in `Task Definitions` section.
+   - Select the task definition revision.
    - Click Deploy > Run Task.
    - Use the following configuration:
-      - Task details
-         - Task definition revision: LATEST
       - Networking:
          - VPC: owasp-nest-staging-vpc
+         - Subnets: subnets will be auto-selected due to VPC selection.
          - Security group name: select all with `owasp-nest-staging-` prefix.
             (*Note*: temporary step, will be further improved)
    - Click "Create"

infrastructure/modules/ecs/main.tf

@@ -120,7 +120,7 @@ module "sync_data_task" {
   source = "./modules/task"
 
   aws_region                   = var.aws_region
-  command                      = ["python", "manage.py", "sync-data"]
+  command                      = ["/bin/sh", "-c", "EXEC_MODE=direct make sync-data"]
   common_tags                  = var.common_tags
   container_parameters_arns    = var.container_parameters_arns
   cpu                          = var.sync_data_task_cpu
@@ -140,8 +140,16 @@ module "sync_data_task" {
 module "owasp_update_project_health_metrics_task" {
   source = "./modules/task"
 
-  aws_region                   = var.aws_region
-  command                      = ["/bin/sh", "-c", "python manage.py owasp-update-project-health-requirements && python manage.py owasp-update-project-health-metrics"]
+  aws_region = var.aws_region
+  command = [
+    "/bin/sh",
+    "-c",
+    <<-EOT
+    set -e
+    EXEC_MODE=direct make owasp-update-project-health-requirements
+    EXEC_MODE=direct make owasp-update-project-health-metrics
+    EOT
+  ]
   common_tags                  = var.common_tags
   container_parameters_arns    = var.container_parameters_arns
   cpu                          = var.update_project_health_metrics_task_cpu
@@ -162,7 +170,7 @@ module "owasp_update_project_health_scores_task" {
   source = "./modules/task"
 
   aws_region                   = var.aws_region
-  command                      = ["python", "manage.py", "owasp-update-project-health-scores"]
+  command                      = ["/bin/sh", "-c", "EXEC_MODE=direct make owasp-update-project-health-scores"]
   common_tags                  = var.common_tags
   container_parameters_arns    = var.container_parameters_arns
   cpu                          = var.update_project_health_scores_task_cpu
@@ -183,7 +191,7 @@ module "migrate_task" {
   source = "./modules/task"
 
   aws_region                   = var.aws_region
-  command                      = ["python", "manage.py", "migrate"]
+  command                      = ["/bin/sh", "-c", "EXEC_MODE=direct make migrate"]
   common_tags                  = var.common_tags
   container_parameters_arns    = var.container_parameters_arns
   cpu                          = var.migrate_task_cpu
@@ -210,7 +218,7 @@ module "load_data_task" {
     pip install --target=/tmp/awscli-packages awscli
     export PYTHONPATH="/tmp/awscli-packages:$PYTHONPATH"
     python /tmp/awscli-packages/bin/aws s3 cp s3://${var.fixtures_s3_bucket}/nest.json.gz /tmp/nest.json.gz
-    python manage.py loaddata /tmp/nest.json.gz -v 3
+    python manage.py load_data --fixture-path /tmp/nest.json.gz
     EOT
   ]
   common_tags                  = var.common_tags
@@ -231,17 +239,8 @@ module "load_data_task" {
 module "index_data_task" {
   source = "./modules/task"
 
-  aws_region = var.aws_region
-  command = [
-    "/bin/sh",
-    "-c",
-    <<-EOT
-    set -e
-    python manage.py algolia_reindex
-    python manage.py algolia_update_replicas
-    python manage.py algolia_update_synonyms
-    EOT
-  ]
+  aws_region                   = var.aws_region
+  command                      = ["/bin/sh", "-c", "EXEC_MODE=direct make index-data"]
   common_tags                  = var.common_tags
   container_parameters_arns    = var.container_parameters_arns
   cpu                          = var.index_data_task_cpu

infrastructure/modules/ecs/variables.tf

@@ -68,7 +68,7 @@ variable "migrate_task_cpu" {
 variable "migrate_task_memory" {
   description = "The memory for the migrate task"
   type        = string
-  default     = "2048"
+  default     = "1024"
 }
 
 variable "private_subnet_ids" {
@@ -90,7 +90,7 @@ variable "sync_data_task_cpu" {
 variable "sync_data_task_memory" {
   description = "The memory for the sync-data task"
   type        = string
-  default     = "2048"
+  default     = "1024"
 }
 
 variable "update_project_health_metrics_task_cpu" {
@@ -102,7 +102,7 @@ variable "update_project_health_metrics_task_cpu" {
 variable "update_project_health_metrics_task_memory" {
   description = "The memory for the update-project-health-metrics task"
   type        = string
-  default     = "2048"
+  default     = "1024"
 }
 
 variable "update_project_health_scores_task_cpu" {
@@ -114,5 +114,5 @@ variable "update_project_health_scores_task_cpu" {
 variable "update_project_health_scores_task_memory" {
   description = "The memory for the update-project-health-scores task"
   type        = string
-  default     = "2048"
+  default     = "1024"
 }