Migrate Fuzz tests tool to Schemathesis and add REST Fuzz tests

.github/workflows/run-ci-cd.yaml

@@ -299,98 +299,24 @@ jobs:
             echo "release_version=$(date '+%y.%-m.%-d')-${GITHUB_SHA::7}" >> $GITHUB_OUTPUT
           fi
 
-  run-fuzz-tests:
-    name: Run fuzz tests
+  run-graphql-fuzz-tests:
+    name: Run GraphQL fuzz tests
     needs:
       - scan-code
       - scan-ci-dependencies
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    services:
-      db:
-        image: pgvector/pgvector:pg16
-        env:
-          POSTGRES_DB: nest_db_fuzz
-          POSTGRES_PASSWORD: nest_user_fuzz_password
-          POSTGRES_USER: nest_user_fuzz
-        options: >-
-          --health-cmd="pg_isready -U nest_user_fuzz -d nest_db_fuzz -h localhost -p 5432"
-          --health-interval=5s
-          --health-timeout=5s
-          --health-retries=5
-        ports:
-          - 5432:5432
-    steps:
-      - name: Check out repository
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
-
-      - name: Set up Docker buildx
-        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f
-
-      - name: Setup Backend environment
-        uses: ./.github/workflows/setup-backend-environment
-        with:
-          db_username: nest_user_fuzz
-          db_name: nest_db_fuzz
-
-      - name: Run backend with fuzz environment variables
-        run: |
-          docker run -d --rm --name fuzz-nest-backend \
-            --env-file backend/.env.fuzz.example \
-            --network host \
-            -e DJANGO_DB_HOST=localhost \
-            -p 9500:9500 \
-            owasp/nest:test-backend-latest \
-            sh -c '
-              python manage.py migrate &&
-              gunicorn wsgi:application --bind 0.0.0.0:9500
-            '
+    uses: ./.github/workflows/run-fuzz-tests.yaml
+    with:
+      test-file: test_graphql.py
 
-      - name: Waiting for the backend to be ready
-        run: |
-          timeout 5m bash -c '
-            until wget --spider http://localhost:9500/a; do
-              echo "Waiting for backend..."
-              sleep 5
-            done
-          '
-          echo "Backend is up!"
-
-      - name: Load Postgres data
-        env:
-          PGPASSWORD: nest_user_fuzz_password
-        run: |
-          pg_restore -h localhost -U nest_user_fuzz -d nest_db_fuzz < backend/data/nest.dump
-
-      - name: Build Fuzz-testing image
-        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83
-        with:
-          cache-from: |
-            type=gha
-            type=registry,ref=owasp/nest:test-fuzz-backend-cache
-          cache-to: |
-            type=gha,compression=zstd
-          context: backend/docker
-          file: backend/docker/Dockerfile.fuzz
-          load: true
-          platforms: linux/amd64
-          tags: owasp/nest:test-fuzz-backend-latest
-
-      - name: Run backend fuzz tests
-        run: |
-          mkdir -p ${{ github.workspace }}/fuzzing_results &&
-          chmod -R 777 ${{ github.workspace }}/fuzzing_results &&
-          docker run -e BASE_URL=http://localhost:9500 --network host \
-          -v ${{ github.workspace }}/fuzzing_results:/home/owasp/fuzzing_results \
-          owasp/nest:test-fuzz-backend-latest
-
-      - name: Upload fuzzing results
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
-        if: always()
-        with:
-          name: fuzzing-results
-          path: fuzzing_results/
-          retention-days: 30
+  run-rest-fuzz-tests:
+    name: Run REST fuzz tests
+    needs:
+      - scan-code
+      - scan-ci-dependencies
+    uses: ./.github/workflows/run-fuzz-tests.yaml
+    with:
+      test-file: test_rest.py
+      rest-url: http://localhost:9500/api/v0
 
   build-staging-images:
     name: Build Staging Images

.github/workflows/run-fuzz-tests.yaml

@@ -0,0 +1,95 @@
+name: Run fuzz tests
+
+on:
+  workflow_call:
+    inputs:
+      test-file:
+        description: 'The test file to run fuzz tests on'
+        required: true
+        type: string
+      rest-url:
+        description: 'The REST API URL to test against'
+        required: false
+        type: string
+        default: ''
+
+jobs:
+  run-fuzz-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    services:
+      db:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_DB: nest_db_fuzz
+          POSTGRES_PASSWORD: nest_user_fuzz_password
+          POSTGRES_USER: nest_user_fuzz
+        options: >-
+          --health-cmd="pg_isready -U nest_user_fuzz -d nest_db_fuzz -h localhost -p 5432"
+          --health-interval=5s
+          --health-timeout=5s
+          --health-retries=5
+        ports:
+          - 5432:5432
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
+
+      - name: Set up Docker buildx
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f
+
+      - name: Setup Backend environment
+        uses: ./.github/workflows/setup-backend-environment
+        with:
+          db_username: nest_user_fuzz
+          db_name: nest_db_fuzz
+
+      - name: Run backend with fuzz environment variables
+        run: |
+          docker run -d --rm --name fuzz-nest-backend \
+            --env-file backend/.env.fuzz.example \
+            --network host \
+            -e DJANGO_DB_HOST=localhost \
+            -p 9500:9500 \
+            owasp/nest:test-backend-latest \
+            sh -c '
+              python manage.py migrate &&
+              gunicorn wsgi:application --bind 0.0.0.0:9500
+            '
+
+      - name: Waiting for the backend to be ready
+        run: |
+          timeout 5m bash -c '
+            until wget --spider http://localhost:9500/a; do
+              echo "Waiting for backend..."
+              sleep 5
+            done
+          '
+          echo "Backend is up!"
+
+      - name: Load Postgres data
+        env:
+          PGPASSWORD: nest_user_fuzz_password
+        run: |
+          pg_restore -h localhost -U nest_user_fuzz -d nest_db_fuzz < backend/data/nest.dump
+
+      - name: Build Fuzz-testing image
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83
+        with:
+          cache-from: |
+            type=gha
+            type=registry,ref=owasp/nest:test-fuzz-backend-cache
+          cache-to: |
+            type=gha,compression=zstd
+          context: backend/docker/fuzz
+          file: backend/docker/fuzz/Dockerfile
+          load: true
+          platforms: linux/amd64
+          tags: owasp/nest:test-fuzz-backend-latest
+
+      - name: Run fuzz tests
+        env:
+          TEST_FILE: ${{ inputs.test-file }}
+          REST_URL: ${{ inputs.rest-url }}
+        run: |
+          docker run -e BASE_URL=http://localhost:9500 -e TEST_FILE="$TEST_FILE" -e REST_URL="$REST_URL" --network host owasp/nest:test-fuzz-backend-latest

backend/Makefile

@@ -169,8 +169,12 @@ test-fuzz:
 	@docker volume rm -f nest-fuzz_fuzz-db-data >/dev/null 2>&1 || true
 	@COMPOSE_BAKE=true DOCKER_BUILDKIT=1 \
 	docker compose --project-name nest-fuzz -f docker-compose/fuzz.yaml up --build --remove-orphans --abort-on-container-exit db backend data-loader
+	echo "Running GraphQL fuzz tests..."
 	@COMPOSE_BAKE=true DOCKER_BUILDKIT=1 \
 	docker compose --project-name nest-fuzz -f docker-compose/fuzz.yaml up --build --remove-orphans --abort-on-container-exit db backend graphql
+	echo "Running REST API fuzz tests..."
+	@COMPOSE_BAKE=true DOCKER_BUILDKIT=1 \
+	docker compose --project-name nest-fuzz -f docker-compose/fuzz.yaml up --build --remove-orphans --abort-on-container-exit db backend rest
 
 update-backend-dependencies:
 	@cd backend && poetry update

backend/apps/api/rest/v0/__init__.py

@@ -68,6 +68,18 @@
         ],
         "throttle": [],
     }
+
+elif settings.IS_FUZZ_ENVIRONMENT:
+    api_settings_customization = {
+        "auth": None,
+        "servers": [
+            {
+                "description": "Fuzz",
+                "url": settings.SITE_URL,
+            }
+        ],
+        "throttle": [],
+    }
 elif settings.IS_STAGING_ENVIRONMENT:
     api_settings_customization = {
         "servers": [

backend/apps/owasp/models/project_health_metrics.py

@@ -4,7 +4,7 @@
 
 from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import models
-from django.db.models.functions import ExtractMonth, TruncDate
+from django.db.models.functions import Coalesce, ExtractMonth, TruncDate
 from django.utils import timezone
 
 from apps.common.models import BulkSaveModel, TimestampedModel
@@ -170,13 +170,11 @@ def get_latest_health_metrics() -> models.QuerySet["ProjectHealthMetrics"]:
             QuerySet[ProjectHealthMetrics]: QuerySet of project health metrics.
 
         """
-        return ProjectHealthMetrics.objects.filter(
-            nest_created_at=models.Subquery(
-                ProjectHealthMetrics.objects.filter(project=models.OuterRef("project"))
-                .order_by("-nest_created_at")
-                .values("nest_created_at")[:1]
-            ),
-            project__is_active=True,
+        return (
+            ProjectHealthMetrics.objects.filter(project__is_active=True)
+            .select_related("project")
+            .order_by("project_id", "-nest_created_at")
+            .distinct("project_id")
         )
 
     @staticmethod
@@ -187,27 +185,29 @@ def get_stats() -> ProjectHealthStatsNode:
             ProjectHealthStatsNode: The overall health stats of all projects.
 
         """
-        metrics = ProjectHealthMetrics.get_latest_health_metrics()
-
-        projects_count_healthy = metrics.filter(
-            score__gte=HEALTH_SCORE_THRESHOLD_HEALTHY,
-        ).count()
-        projects_count_need_attention = metrics.filter(
-            score__lt=HEALTH_SCORE_THRESHOLD_HEALTHY,
-            score__gte=HEALTH_SCORE_THRESHOLD_NEED_ATTENTION,
-        ).count()
-        projects_count_unhealthy = metrics.filter(
-            score__lt=HEALTH_SCORE_THRESHOLD_NEED_ATTENTION
-        ).count()
-
-        projects_count_total = metrics.count() or 1  # Avoid division by zero
-
-        aggregation = metrics.aggregate(
-            average_score=models.Avg("score"),
-            total_contributors=models.Sum("contributors_count"),
-            total_forks=models.Sum("forks_count"),
-            total_stars=models.Sum("stars_count"),
+        stats = ProjectHealthMetrics.objects.filter(
+            id__in=ProjectHealthMetrics.get_latest_health_metrics().values_list("id", flat=True)
+        ).aggregate(
+            projects_count_healthy=models.Count(
+                "id", filter=models.Q(score__gte=HEALTH_SCORE_THRESHOLD_HEALTHY)
+            ),
+            projects_count_need_attention=models.Count(
+                "id",
+                filter=models.Q(
+                    score__lt=HEALTH_SCORE_THRESHOLD_HEALTHY,
+                    score__gte=HEALTH_SCORE_THRESHOLD_NEED_ATTENTION,
+                ),
+            ),
+            projects_count_unhealthy=models.Count(
+                "id", filter=models.Q(score__lt=HEALTH_SCORE_THRESHOLD_NEED_ATTENTION)
+            ),
+            projects_count_total=models.Count("id"),
+            average_score=Coalesce(models.Avg("score"), 0.0),
+            total_contributors=Coalesce(models.Sum("contributors_count"), 0),
+            total_forks=Coalesce(models.Sum("forks_count"), 0),
+            total_stars=Coalesce(models.Sum("stars_count"), 0),
         )
+        total = stats["projects_count_total"] or 1  # Avoid division by zero
         monthly_overall_metrics = (
             ProjectHealthMetrics.objects.annotate(month=ExtractMonth("nest_created_at"))
             .filter(
@@ -220,22 +220,28 @@ def get_stats() -> ProjectHealthStatsNode:
                 score=models.Avg("score"),
             )
         )
+        months = []
+        scores = []
+        for entry in monthly_overall_metrics:
+            months.append(entry["month"])
+            scores.append(entry["score"])
         return ProjectHealthStatsNode(
-            average_score=aggregation.get("average_score", 0.0),
+            average_score=stats["average_score"],
             # We use all metrics instead of latest metrics to get the monthly trend
-            monthly_overall_scores=list(monthly_overall_metrics.values_list("score", flat=True)),
-            monthly_overall_scores_months=list(
-                monthly_overall_metrics.values_list("month", flat=True)
-            ),
-            projects_count_healthy=projects_count_healthy,
-            projects_count_need_attention=projects_count_need_attention,
-            projects_count_unhealthy=projects_count_unhealthy,
-            projects_percentage_healthy=(projects_count_healthy / projects_count_total) * 100,
+            monthly_overall_scores=scores,
+            monthly_overall_scores_months=months,
+            projects_count_healthy=stats["projects_count_healthy"],
+            projects_count_need_attention=stats["projects_count_need_attention"],
+            projects_count_unhealthy=stats["projects_count_unhealthy"],
+            projects_percentage_healthy=(
+                stats["projects_count_healthy"] / stats["projects_count_total"]
+            )
+            * 100,
             projects_percentage_need_attention=(
-                (projects_count_need_attention / projects_count_total) * 100
+                (stats["projects_count_need_attention"] / stats["projects_count_total"]) * 100
             ),
-            projects_percentage_unhealthy=(projects_count_unhealthy / projects_count_total) * 100,
-            total_contributors=(aggregation.get("total_contributors", 0)),
-            total_forks=(aggregation.get("total_forks", 0)),
-            total_stars=(aggregation.get("total_stars", 0)),
+            projects_percentage_unhealthy=(stats["projects_count_unhealthy"] / total) * 100,
+            total_contributors=(stats.get("total_contributors", 0)),
+            total_forks=(stats.get("total_forks", 0)),
+            total_stars=(stats.get("total_stars", 0)),
         )