diff --git a/.github/workflows/run-ci-cd.yaml b/.github/workflows/run-ci-cd.yaml index 04a8a11435..afd289a004 100644 --- a/.github/workflows/run-ci-cd.yaml +++ b/.github/workflows/run-ci-cd.yaml @@ -226,6 +226,17 @@ jobs: --health-retries=5 ports: - 5432:5432 + cache: + image: redis:8.0.5-alpine3.21 + env: + REDIS_PASSWORD: nest-cache-e2e-password + options: >- + --health-cmd="redis-cli -a $$REDIS_PASSWORD ping" + --health-interval=5s + --health-timeout=5s + --health-retries=5 + ports: + - 6379:6379 steps: - name: Check out repository uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 @@ -320,6 +331,17 @@ jobs: --health-retries=5 ports: - 5432:5432 + cache: + image: redis:8.0.5-alpine3.21 + env: + REDIS_PASSWORD: nest-fuzz-cache-password + options: >- + --health-cmd="redis-cli -a $$REDIS_PASSWORD ping" + --health-interval=5s + --health-timeout=5s + --health-retries=5 + ports: + - 6379:6379 steps: - name: Check out repository uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 diff --git a/.github/workflows/setup-e2e-environment/action.yaml b/.github/workflows/setup-e2e-environment/action.yaml deleted file mode 100644 index ed5f795504..0000000000 --- a/.github/workflows/setup-e2e-environment/action.yaml +++ /dev/null @@ -1,65 +0,0 @@ -name: Set up E2E environment - -description: Sets up the environment for end-to-end testing. - -runs: - using: composite - steps: - - name: Wait for database to be ready - run: | - timeout 5m bash -c ' - until docker exec ${{ job.services.db.id }} pg_isready -U nest_user_e2e -d nest_db_e2e; do - echo "Waiting for database..." - sleep 5 - done - ' - shell: bash - - - name: Install PostgreSQL client - run: sudo apt-get install -y postgresql-client - shell: bash - - - name: Build backend e2e image - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 - with: - cache-from: | - type=gha - cache-to: | - type=gha,compression=zstd - context: backend - file: backend/docker/Dockerfile - load: true - platforms: linux/amd64 - tags: owasp/nest:test-backend-e2e-latest - - - name: Start Backend in the background - run: | - docker run -d --rm --name e2e-nest-backend \ - --env-file backend/.env.e2e.example \ - --network host \ - -e DJANGO_DB_HOST=localhost \ - -p 9000:9000 \ - owasp/nest:test-backend-e2e-latest \ - sh -c ' - python manage.py migrate && - gunicorn wsgi:application --bind 0.0.0.0:9000 - ' - shell: bash - - - name: Waiting for the backend to be ready - run: | - timeout 5m bash -c ' - until wget --spider http://localhost:9000/a; do - echo "Waiting for backend..." - sleep 5 - done - ' - echo "Backend is up!" - shell: bash - - - name: Load Postgres data - env: - PGPASSWORD: nest_user_e2e_password - run: | - gunzip -c backend/data/nest.sql.gz | psql -h localhost -U nest_user_e2e -d nest_db_e2e - shell: bash diff --git a/backend/.env.e2e.example b/backend/.env.e2e.example index 7043b2cd60..08d07d195d 100644 --- a/backend/.env.e2e.example +++ b/backend/.env.e2e.example @@ -13,8 +13,8 @@ DJANGO_DB_PASSWORD=nest_user_e2e_password DJANGO_DB_PORT=5432 DJANGO_OPEN_AI_SECRET_KEY=None DJANGO_PUBLIC_IP_ADDRESS="127.0.0.1" -DJANGO_REDIS_HOST=None -DJANGO_REDIS_PASSWORD=None +DJANGO_REDIS_HOST=cache +DJANGO_REDIS_PASSWORD=nest-cache-e2e-password DJANGO_RELEASE_VERSION=None DJANGO_SECRET_KEY=None DJANGO_SENTRY_DSN=None diff --git a/backend/.env.fuzz.example b/backend/.env.fuzz.example index 6cca22ab06..7b8021b0ab 100644 --- a/backend/.env.fuzz.example +++ b/backend/.env.fuzz.example @@ -13,8 +13,8 @@ DJANGO_DB_PASSWORD=nest_user_fuzz_password DJANGO_DB_PORT=5432 DJANGO_OPEN_AI_SECRET_KEY=None DJANGO_PUBLIC_IP_ADDRESS="127.0.0.1" -DJANGO_REDIS_HOST=None -DJANGO_REDIS_PASSWORD=None +DJANGO_REDIS_HOST=cache +DJANGO_REDIS_PASSWORD=nest-fuzz-cache-password DJANGO_RELEASE_VERSION=None DJANGO_SECRET_KEY=None DJANGO_SENTRY_DSN=None diff --git a/backend/Makefile b/backend/Makefile index 33ff53a177..0fd907f7ca 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -132,11 +132,11 @@ restore-backup: run-backend-e2e: @DOCKER_BUILDKIT=1 \ - docker compose --project-name nest-e2e -f docker-compose/e2e.yaml up --build --remove-orphans --abort-on-container-exit backend db + docker compose --project-name nest-e2e -f docker-compose/e2e/compose.yaml up --build --remove-orphans --abort-on-container-exit backend db cache run-backend-fuzz: @COMPOSE_BAKE=true DOCKER_BUILDKIT=1 \ - docker compose --project-name nest-fuzz -f docker-compose/fuzz.yaml up --build --remove-orphans --abort-on-container-exit backend db + docker compose --project-name nest-fuzz -f docker-compose/fuzz/compose.yaml up --build --remove-orphans --abort-on-container-exit backend db cache save-backup: @echo "Saving Nest backup" @@ -168,9 +168,9 @@ test-fuzz: @docker container rm -f fuzz-nest-db >/dev/null 2>&1 || true @docker volume rm -f nest-fuzz_fuzz-db-data >/dev/null 2>&1 || true @COMPOSE_BAKE=true DOCKER_BUILDKIT=1 \ - docker compose --project-name nest-fuzz -f docker-compose/fuzz.yaml up --build --remove-orphans --abort-on-container-exit db backend data-loader + docker compose --project-name nest-fuzz -f docker-compose/fuzz/compose.yaml up --build --remove-orphans --abort-on-container-exit db cache backend data-loader @COMPOSE_BAKE=true DOCKER_BUILDKIT=1 \ - docker compose --project-name nest-fuzz -f docker-compose/fuzz.yaml up --build --remove-orphans --abort-on-container-exit db backend graphql + docker compose --project-name nest-fuzz -f docker-compose/fuzz/compose.yaml up --build --remove-orphans --abort-on-container-exit db cache backend graphql update-backend-dependencies: @cd backend && poetry update diff --git a/backend/apps/api/rest/v0/__init__.py b/backend/apps/api/rest/v0/__init__.py index 74235cd101..c7a9b758be 100644 --- a/backend/apps/api/rest/v0/__init__.py +++ b/backend/apps/api/rest/v0/__init__.py @@ -68,6 +68,17 @@ ], "throttle": [], } +elif settings.IS_FUZZ_ENVIRONMENT: + api_settings_customization = { + "auth": None, + "servers": [ + { + "description": "Fuzz", + "url": settings.SITE_URL, + } + ], + "throttle": [], + } elif settings.IS_STAGING_ENVIRONMENT: api_settings_customization = { "servers": [ diff --git a/backend/settings/e2e.py b/backend/settings/e2e.py index 4e3895c957..d70e0fc995 100644 --- a/backend/settings/e2e.py +++ b/backend/settings/e2e.py @@ -16,12 +16,6 @@ class E2E(Base): "http://localhost:3000", ) - CACHES = { - "default": { - "BACKEND": "django.core.cache.backends.locmem.LocMemCache", - } - } - CORS_ALLOWED_ORIGINS = ALLOWED_ORIGINS CSRF_TRUSTED_ORIGINS = ALLOWED_ORIGINS diff --git a/backend/settings/fuzz.py b/backend/settings/fuzz.py index 84aae0eb2a..11d3471b59 100644 --- a/backend/settings/fuzz.py +++ b/backend/settings/fuzz.py @@ -11,12 +11,6 @@ class Fuzz(Base): APP_NAME = "OWASP Nest Fuzz Testing" SITE_URL = "http://localhost:9500" - CACHES = { - "default": { - "BACKEND": "django.core.cache.backends.locmem.LocMemCache", - } - } - IS_FUZZ_ENVIRONMENT = True LOGGING = {} PUBLIC_IP_ADDRESS = values.Value() diff --git a/docker-compose/e2e.yaml b/docker-compose/e2e/compose.yaml similarity index 72% rename from docker-compose/e2e.yaml rename to docker-compose/e2e/compose.yaml index ceef646b05..7e8b402cb3 100644 --- a/docker-compose/e2e.yaml +++ b/docker-compose/e2e/compose.yaml @@ -7,12 +7,14 @@ services: gunicorn wsgi:application --bind 0.0.0.0:9000 ' build: - context: ../backend + context: ../../backend dockerfile: docker/Dockerfile depends_on: db: condition: service_healthy - env_file: ../backend/.env.e2e.example + cache: + condition: service_healthy + env_file: ../../backend/.env.e2e.example networks: - e2e-nest-network ports: @@ -40,7 +42,7 @@ services: POSTGRES_USER: ${DJANGO_DB_USER:-nest_user_e2e} POSTGRES_DB: ${DJANGO_DB_NAME:-nest_db_e2e} volumes: - - ../backend/data:/data:ro + - ../../backend/data:/data:ro networks: - e2e-nest-network command: > @@ -68,10 +70,29 @@ services: ports: - 5433:5432 + cache: + command: > + sh -c ' + redis-server --requirepass $$REDIS_PASSWORD --maxmemory 100mb --maxmemory-policy allkeys-lru + ' + container_name: e2e-nest-cache + image: redis:8.0.5-alpine3.21 + environment: + REDIS_PASSWORD: ${DJANGO_REDIS_PASSWORD:-nest-cache-e2e-password} + healthcheck: + interval: 5s + retries: 5 + test: [CMD, redis-cli, -a, $$REDIS_PASSWORD, ping] + timeout: 5s + networks: + - e2e-nest-network + volumes: + - e2e-cache-data:/data + e2e-tests: container_name: e2e-nest-tests build: - context: ../frontend + context: ../../frontend dockerfile: docker/Dockerfile.e2e.test command: > sh -c ' @@ -80,7 +101,7 @@ services: depends_on: backend: condition: service_healthy - env_file: ../frontend/.env.e2e.example + env_file: ../../frontend/.env.e2e.example networks: - e2e-nest-network @@ -88,3 +109,4 @@ networks: e2e-nest-network: volumes: e2e-db-data: + e2e-cache-data: diff --git a/docker-compose/fuzz.yaml b/docker-compose/fuzz/compose.yaml similarity index 72% rename from docker-compose/fuzz.yaml rename to docker-compose/fuzz/compose.yaml index 490e201d5e..0db02c1060 100644 --- a/docker-compose/fuzz.yaml +++ b/docker-compose/fuzz/compose.yaml @@ -7,12 +7,14 @@ services: gunicorn wsgi:application --bind 0.0.0.0:9500 ' build: - context: ../backend + context: ../../backend dockerfile: docker/Dockerfile depends_on: db: condition: service_healthy - env_file: ../backend/.env.fuzz.example + cache: + condition: service_healthy + env_file: ../../backend/.env.fuzz.example networks: - fuzz-nest-network ports: @@ -40,7 +42,7 @@ services: POSTGRES_USER: ${DJANGO_DB_USER:-nest_user_fuzz} POSTGRES_DB: ${DJANGO_DB_NAME:-nest_db_fuzz} volumes: - - ../backend/data:/data:ro + - ../../backend/data:/data:ro networks: - fuzz-nest-network command: > @@ -67,10 +69,30 @@ services: - fuzz-db-data:/var/lib/postgresql/data ports: - 5434:5432 + + cache: + command: > + sh -c ' + redis-server --requirepass $$REDIS_PASSWORD --maxmemory 100mb --maxmemory-policy allkeys-lru + ' + container_name: fuzz-nest-cache + image: redis:8.0.5-alpine3.21 + environment: + REDIS_PASSWORD: ${DJANGO_REDIS_PASSWORD:-nest-fuzz-cache-password} + healthcheck: + interval: 5s + retries: 5 + test: [CMD, redis-cli, -a, $$REDIS_PASSWORD, ping] + timeout: 5s + networks: + - fuzz-nest-network + volumes: + - fuzz-cache-data:/data + graphql: container_name: fuzz-nest-graphql build: - context: ../backend/docker + context: ../../backend/docker dockerfile: Dockerfile.fuzz environment: BASE_URL: http://backend:9500 @@ -78,7 +100,7 @@ services: backend: condition: service_healthy volumes: - - ../backend/fuzzing_results:/home/owasp/fuzzing_results + - ../../backend/fuzzing_results:/home/owasp/fuzzing_results networks: - fuzz-nest-network @@ -87,3 +109,4 @@ networks: volumes: fuzz-db-data: + fuzz-cache-data: diff --git a/frontend/Makefile b/frontend/Makefile index 5d876e4eb4..d4acd8176a 100644 --- a/frontend/Makefile +++ b/frontend/Makefile @@ -60,9 +60,9 @@ test-frontend-e2e: @docker container rm -f e2e-nest-db >/dev/null 2>&1 || true @docker volume rm -f nest-e2e_e2e-db-data >/dev/null 2>&1 || true @DOCKER_BUILDKIT=1 \ - docker compose --project-name nest-e2e -f docker-compose/e2e.yaml up --build --remove-orphans --abort-on-container-exit db backend data-loader + docker compose --project-name nest-e2e -f docker-compose/e2e/compose.yaml up --build --remove-orphans --abort-on-container-exit db cache backend data-loader @DOCKER_BUILDKIT=1 NEXT_PUBLIC_ENVIRONMENT=local \ - docker compose --project-name nest-e2e -f docker-compose/e2e.yaml up --build --remove-orphans --abort-on-container-exit db backend e2e-tests + docker compose --project-name nest-e2e -f docker-compose/e2e/compose.yaml up --build --remove-orphans --abort-on-container-exit db cache backend e2e-tests test-frontend-unit: