diff --git a/.github/ansible/production/nest.yaml b/.github/ansible/production/nest.yaml index 9e5ff7d6b2..5e1d877dd6 100644 --- a/.github/ansible/production/nest.yaml +++ b/.github/ansible/production/nest.yaml @@ -87,6 +87,12 @@ dest: ~/ mode: '0400' + - name: Copy .env.cache + copy: + src: '{{ github_workspace }}/.env.cache' + dest: ~/ + mode: '0400' + - name: Copy .env.db copy: src: '{{ github_workspace }}/.env.db' diff --git a/.github/ansible/staging/nest.yaml b/.github/ansible/staging/nest.yaml index 7c9b65801c..3939832108 100644 --- a/.github/ansible/staging/nest.yaml +++ b/.github/ansible/staging/nest.yaml @@ -99,6 +99,12 @@ dest: ~/ mode: '0400' + - name: Copy .env.cache + copy: + src: '{{ github_workspace }}/.env.cache' + dest: ~/ + mode: '0400' + - name: Copy .env.db copy: src: '{{ github_workspace }}/.env.db' diff --git a/.github/workflows/run-ci-cd.yaml b/.github/workflows/run-ci-cd.yaml index d5b3b97a5d..8619eff885 100644 --- a/.github/workflows/run-ci-cd.yaml +++ b/.github/workflows/run-ci-cd.yaml @@ -337,6 +337,8 @@ jobs: echo "DJANGO_DB_PORT=${{ secrets.DJANGO_DB_PORT }}" >> .env.backend echo "DJANGO_DB_USER=${{ secrets.DJANGO_DB_USER }}" >> .env.backend echo "DJANGO_OPEN_AI_SECRET_KEY=${{ secrets.DJANGO_OPEN_AI_SECRET_KEY }}" >> .env.backend + echo "DJANGO_REDIS_HOST=${{ secrets.DJANGO_REDIS_HOST }}" >> .env.backend + echo "DJANGO_REDIS_PASSWORD=${{ secrets.DJANGO_REDIS_PASSWORD }}" >> .env.backend echo "DJANGO_RELEASE_VERSION=$(date '+%y.%-m.%-d')-${GITHUB_SHA:0:7}" >> .env.backend echo "DJANGO_SECRET_KEY=${{ secrets.DJANGO_SECRET_KEY }}" >> .env.backend echo "DJANGO_SENTRY_DSN=${{ secrets.DJANGO_SENTRY_DSN }}" >> .env.backend @@ -345,6 +347,10 @@ jobs: echo "DJANGO_SLACK_SIGNING_SECRET=${{ secrets.DJANGO_SLACK_SIGNING_SECRET }}" >> .env.backend echo "GITHUB_TOKEN=${{ secrets.DJANGO_GITHUB_TOKEN }}" >> .env.backend + # Cache + touch .env.cache + echo "REDIS_PASSWORD=${{ secrets.DJANGO_REDIS_PASSWORD }}" >> .env.cache + # Database touch .env.db echo "POSTGRES_DB=${{ secrets.DJANGO_DB_NAME }}" >> .env.db @@ -511,6 +517,8 @@ jobs: echo "DJANGO_DB_PORT=${{ secrets.DJANGO_DB_PORT }}" >> .env.backend echo "DJANGO_DB_USER=${{ secrets.DJANGO_DB_USER }}" >> .env.backend echo "DJANGO_OPEN_AI_SECRET_KEY=${{ secrets.DJANGO_OPEN_AI_SECRET_KEY }}" >> .env.backend + echo "DJANGO_REDIS_HOST=${{ secrets.DJANGO_REDIS_HOST }}" >> .env.backend + echo "DJANGO_REDIS_PASSWORD=${{ secrets.DJANGO_REDIS_PASSWORD }}" >> .env.backend echo "DJANGO_RELEASE_VERSION=${{ github.event.release.tag_name }}" >> .env.backend echo "DJANGO_SECRET_KEY=${{ secrets.DJANGO_SECRET_KEY }}" >> .env.backend echo "DJANGO_SENTRY_DSN=${{ secrets.DJANGO_SENTRY_DSN }}" >> .env.backend @@ -519,6 +527,10 @@ jobs: echo "DJANGO_SLACK_SIGNING_SECRET=${{ secrets.DJANGO_SLACK_SIGNING_SECRET }}" >> .env.backend echo "GITHUB_TOKEN=${{ secrets.DJANGO_GITHUB_TOKEN }}" >> .env.backend + # Cache + touch .env.cache + echo "REDIS_PASSWORD=${{ secrets.DJANGO_REDIS_PASSWORD }}" >> .env.cache + # Database touch .env.db echo "POSTGRES_DB=${{ secrets.DJANGO_DB_NAME }}" >> .env.db diff --git a/backend/.env.example b/backend/.env.example index e64165ed0c..2916e696d1 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -12,6 +12,8 @@ DJANGO_DB_PORT=None DJANGO_DB_USER=None DJANGO_OPEN_AI_SECRET_KEY=None DJANGO_PUBLIC_IP_ADDRESS="127.0.0.1" +DJANGO_REDIS_HOST=None +DJANGO_REDIS_PASSWORD=None DJANGO_RELEASE_VERSION=None DJANGO_SECRET_KEY=None DJANGO_SENTRY_DSN=None diff --git a/backend/docker/Dockerfile.local b/backend/docker/Dockerfile.local index d925e53728..90e39aff03 100644 --- a/backend/docker/Dockerfile.local +++ b/backend/docker/Dockerfile.local @@ -20,7 +20,7 @@ FROM python:3.13.3-alpine SHELL ["/bin/sh", "-o", "pipefail", "-c"] RUN apk update && \ - apk add postgresql-client && \ + apk add postgresql-client redis && \ addgroup -S owasp && \ adduser -S -h /home/owasp -G owasp owasp && \ python -m pip install --no-cache-dir poetry diff --git a/backend/poetry.lock b/backend/poetry.lock index 7abda5daf4..e0b8197db5 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -786,6 +786,25 @@ files = [ [package.dependencies] Django = ">=4.2" +[[package]] +name = "django-redis" +version = "5.4.0" +description = "Full featured redis cache backend for Django." +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "django-redis-5.4.0.tar.gz", hash = "sha256:6a02abaa34b0fea8bf9b707d2c363ab6adc7409950b2db93602e6cb292818c42"}, + {file = "django_redis-5.4.0-py3-none-any.whl", hash = "sha256:ebc88df7da810732e2af9987f7f426c96204bf89319df4c6da6ca9a2942edd5b"}, +] + +[package.dependencies] +Django = ">=3.2" +redis = ">=3,<4.0.0 || >4.0.0,<4.0.1 || >4.0.1" + +[package.extras] +hiredis = ["redis[hiredis] (>=3,!=4.0.0,!=4.0.1)"] + [[package]] name = "django-storages" version = "1.14.6" @@ -2492,6 +2511,22 @@ files = [ {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, ] +[[package]] +name = "redis" +version = "5.2.1" +description = "Python client for Redis database and key-value store" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "redis-5.2.1-py3-none-any.whl", hash = "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4"}, + {file = "redis-5.2.1.tar.gz", hash = "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f"}, +] + +[package.extras] +hiredis = ["hiredis (>=3.0.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==23.2.1)", "requests (>=2.31.0)"] + [[package]] name = "regex" version = "2024.11.6" @@ -3116,4 +3151,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = "^3.13" -content-hash = "83bfbf685725f1bf168770d9a4f146f6fc2d0c2010d3c200670be35745ade04a" +content-hash = "8dc9e85a835bc82849fe37e2adf4fa037d1be36fcbbc2208cc105386aa4b249e" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index acab6f4122..4dba2c7538 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -27,6 +27,7 @@ django = "^5.1" django-configurations = "^2.5.1" django-cors-headers = "^4.7.0" django-filter = "^25.1" +django-redis = "^5.4.0" django-storages = { extras = ["s3"], version = "^1.14.4" } djangorestframework = "^3.15.2" geopy = "^2.4.1" diff --git a/backend/settings/base.py b/backend/settings/base.py index b7800cc978..160252686b 100644 --- a/backend/settings/base.py +++ b/backend/settings/base.py @@ -122,11 +122,19 @@ class Base(Configuration): "INDEX_PREFIX": ENVIRONMENT.lower(), } + REDIS_HOST = values.SecretValue(environ_name="REDIS_HOST") + REDIS_PASSWORD = values.SecretValue(environ_name="REDIS_PASSWORD") CACHES = { "default": { - "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": f"redis://:{REDIS_PASSWORD}@{REDIS_HOST}:6379", + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + }, + "TIMEOUT": 300, } } + # Database # https://docs.djangoproject.com/en/5.1/ref/settings/#databases DATABASES = { diff --git a/backend/tests/apps/core/api/algolia_test.py b/backend/tests/apps/core/api/algolia_test.py index 98cb6345bb..aa7b7f8879 100644 --- a/backend/tests/apps/core/api/algolia_test.py +++ b/backend/tests/apps/core/api/algolia_test.py @@ -3,7 +3,6 @@ import pytest import requests -from django.core.cache import cache from apps.core.api.algolia import algolia_search @@ -18,15 +17,15 @@ CLIENT_IP_ADDRESS = "127.0.0.1" -@pytest.fixture -def _clear_cache(): - """Clear the cache before and after each test.""" - cache.clear() - yield - cache.clear() +@pytest.fixture(autouse=True) +def mock_redis_cache(): + """Mock Redis cache used in algolia.py.""" + with patch("apps.core.api.algolia.cache") as mock_cache: + mock_cache.get.return_value = None + mock_cache.set.return_value = True + yield mock_cache -@pytest.mark.usefixtures("_clear_cache") class TestAlgoliaSearch: @pytest.mark.parametrize( ("index_name", "query", "page", "hits_per_page", "facet_filters", "expected_result"), @@ -92,7 +91,6 @@ def test_algolia_search_invalid_method(self): @pytest.mark.parametrize( ("index_name", "query", "page", "hits_per_page", "facet_filters", "error_message"), [ - # Index name tests ( 5, "owasp", @@ -101,14 +99,38 @@ def test_algolia_search_invalid_method(self): ["idx_is_active:true"], "indexName is required and must be a string.", ), - # Query tests - ("chapters", 5, 2, 20, ["idx_is_active:true"], "query must be a string."), - # Page tests - ("committees", "review", "0", 5, [], "page value must be an integer."), - # hitsPerPage tests - ("committees", "review", 1, "1001", [], "hitsPerPage must be an integer."), - # Facet filters tests - ("issues", "bug", 1, 10, "idx_is_active:true", "facetFilters must be a list."), + ( + "chapters", + 5, + 2, + 20, + ["idx_is_active:true"], + "query must be a string.", + ), + ( + "committees", + "review", + "0", + 5, + [], + "page value must be an integer.", + ), + ( + "committees", + "review", + 1, + "1001", + [], + "hitsPerPage must be an integer.", + ), + ( + "issues", + "bug", + 1, + 10, + "idx_is_active:true", + "facetFilters must be a list.", + ), ], ) def test_algolia_search_invalid_request( diff --git a/cspell/custom-dict.txt b/cspell/custom-dict.txt index dc2591bfad..f21d02365a 100644 --- a/cspell/custom-dict.txt +++ b/cspell/custom-dict.txt @@ -87,6 +87,7 @@ pygoat pymdownx pyyaml repositorycontributor +requirepass rsc saft sakanashi diff --git a/docker/docker-compose-local.yaml b/docker/docker-compose-local.yaml index 7849a95650..467ea32257 100644 --- a/docker/docker-compose-local.yaml +++ b/docker/docker-compose-local.yaml @@ -10,6 +10,8 @@ services: context: ../backend dockerfile: docker/Dockerfile.local depends_on: + cache: + condition: service_healthy db: condition: service_healthy env_file: ../backend/.env @@ -19,6 +21,8 @@ services: DJANGO_DB_PASSWORD: ${DJANGO_DB_PASSWORD:-nest_user_dev_password} DJANGO_DB_PORT: ${DJANGO_DB_PORT:-5432} DJANGO_DB_USER: ${DJANGO_DB_USER:-nest_user_dev} + DJANGO_REDIS_HOST: ${DJANGO_REDIS_HOST:-nest-cache} + DJANGO_REDIS_PASSWORD: ${DJANGO_REDIS_HOST:-nest-cache-password} networks: - nest-network ports: @@ -27,6 +31,25 @@ services: - ../backend:/home/owasp - backend-venv:/home/owasp/.venv + cache: + command: > + sh -c ' + redis-server --requirepass $$REDIS_PASSWORD --maxmemory 25mb --maxmemory-policy allkeys-lru + ' + container_name: nest-cache + image: redis:7.2.7-alpine3.21 + environment: + REDIS_PASSWORD: ${DJANGO_REDIS_PASSWORD:-nest-cache-password} + healthcheck: + interval: 5s + retries: 5 + test: [CMD, redis-cli, -a, $REDIS_PASSWORD, ping] + timeout: 5s + networks: + - nest-network + volumes: + - cache-data:/data + db: container_name: nest-db image: postgres:16.4 @@ -91,6 +114,7 @@ networks: volumes: backend-venv: + cache-data: db-data: docs-venv: frontend-next: diff --git a/docker/docker-compose-production.yaml b/docker/docker-compose-production.yaml index da57a77c03..5c97c37f97 100644 --- a/docker/docker-compose-production.yaml +++ b/docker/docker-compose-production.yaml @@ -4,37 +4,60 @@ services: image: arkid15r/owasp-nest-backend:production env_file: .env.backend depends_on: + production-nest-cache: + condition: service_healthy production-nest-db: condition: service_healthy restart: unless-stopped networks: - nest-app-network + - nest-cache-network - nest-db-network volumes: - ./data:/home/owasp/data - production-nest-frontend: - container_name: production-nest-frontend - image: arkid15r/owasp-nest-frontend:production + production-nest-cache: + container_name: production-nest-cache + image: redis:7.2.7-alpine3.21 + command: > + sh -c ' + redis-server --requirepass $$REDIS_PASSWORD --maxmemory 100mb --maxmemory-policy allkeys-lru + ' + env_file: .env.cache + healthcheck: + interval: 5s + retries: 5 + test: [CMD, redis-cli, -a, $REDIS_PASSWORD, ping] + timeout: 5s restart: unless-stopped + volumes: + - ./volumes/cache:/data networks: - - nest-app-network + - nest-cache-network production-nest-db: container_name: production-nest-db image: postgres:16.4 env_file: .env.db healthcheck: - test: [CMD, pg_isready, -U, nest_user_production, -d, nest_db_production] interval: 5s - timeout: 5s retries: 5 + test: [CMD, pg_isready, -U, nest_user_production, -d, nest_db_production] + timeout: 5s restart: unless-stopped volumes: - ./volumes/db:/var/lib/postgresql/data networks: - nest-db-network + production-nest-frontend: + container_name: production-nest-frontend + image: arkid15r/owasp-nest-frontend:production + restart: unless-stopped + networks: + - nest-app-network + networks: nest-app-network: + nest-cache-network: nest-db-network: diff --git a/docker/docker-compose-staging.yaml b/docker/docker-compose-staging.yaml index 9756fb1fa9..eaa20422bb 100644 --- a/docker/docker-compose-staging.yaml +++ b/docker/docker-compose-staging.yaml @@ -4,37 +4,60 @@ services: image: arkid15r/owasp-nest-backend:staging env_file: .env.backend depends_on: + staging-nest-cache: + condition: service_healthy staging-nest-db: condition: service_healthy restart: unless-stopped networks: - nest-app-network + - nest-cache-network - nest-db-network volumes: - ./data:/home/owasp/data - staging-nest-frontend: - container_name: staging-nest-frontend - image: arkid15r/owasp-nest-frontend:staging + staging-nest-cache: + container_name: staging-nest-cache + image: redis:7.2.7-alpine3.21 + command: > + sh -c ' + redis-server --requirepass $$REDIS_PASSWORD --maxmemory 25mb --maxmemory-policy allkeys-lru + ' + env_file: .env.cache + healthcheck: + interval: 5s + retries: 5 + test: [CMD, redis-cli, -a, $REDIS_PASSWORD, ping] + timeout: 5s restart: unless-stopped + volumes: + - ./volumes/cache:/data networks: - - nest-app-network + - nest-cache-network staging-nest-db: container_name: staging-nest-db image: postgres:16.4 env_file: .env.db healthcheck: - test: [CMD, pg_isready, -U, nest_user_staging, -d, nest_db_staging] interval: 5s - timeout: 5s retries: 5 + test: [CMD, pg_isready, -U, nest_user_staging, -d, nest_db_staging] + timeout: 5s restart: unless-stopped volumes: - ./volumes/db:/var/lib/postgresql/data networks: - nest-db-network + staging-nest-frontend: + container_name: staging-nest-frontend + image: arkid15r/owasp-nest-frontend:staging + restart: unless-stopped + networks: + - nest-app-network + networks: nest-app-network: + nest-cache-network: nest-db-network: