test: add automated testing runner and matrix

automated_testing/.env.example

@@ -0,0 +1,41 @@
+# =============================================================================
+# Gleec QA Automation — Environment Configuration
+# =============================================================================
+# Copy this file to .env and fill in values for your environment.
+#
+# PLATFORM NOTES:
+#   Linux:   Ollama runs natively. Use http://host.docker.internal:11434
+#   Windows: Run Ollama natively on Windows (best GPU perf).
+#            Run this runner + Docker inside WSL2.
+#            Ollama URL from WSL2/Docker: http://host.docker.internal:11434
+#            (Docker Desktop shares ports between Windows and WSL2 automatically)
+# =============================================================================
+
+# --- Skyvern + Ollama integration ---
+ENV=local
+ENABLE_OLLAMA=true
+LLM_KEY=OLLAMA
+OLLAMA_SERVER_URL=http://host.docker.internal:11434
+OLLAMA_MODEL=qwen2.5-vl:32b
+OLLAMA_SUPPORTS_VISION=true
+
+# --- Database (managed by Docker Compose) ---
+DATABASE_STRING=postgresql+psycopg://skyvern:skyvern@postgres:5432/skyvern
+
+# --- Browser ---
+BROWSER_TYPE=chromium-headful
+VIDEO_PATH=/app/videos
+BROWSER_ACTION_TIMEOUT_MS=10000
+MAX_STEPS_PER_RUN=50
+
+# --- Skyvern server ---
+LOG_LEVEL=INFO
+PORT=8000
+
+# --- Runner configuration (read by runner.py, not by Docker) ---
+# APP_BASE_URL=https://app.gleecwallet.com
+# OLLAMA_URL=http://localhost:11434
+# SKYVERN_URL=http://localhost:8000
+# VRAM_MIN_GB=15
+# DEFAULT_RETRIES=3
+# CRITICAL_RETRIES=5

automated_testing/ci-pipeline.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# =============================================================================
+# Gleec QA Automation — CI Pipeline
+# =============================================================================
+# Exit codes:
+#   0 = all tests passed
+#   1 = test failures or errors
+#   2 = pre-flight / infrastructure failure
+#   3 = all passed but some flaky
+# =============================================================================
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+MATRIX="${MATRIX:-test_matrix.yaml}"
+ARTIFACTS_DIR="${CI_ARTIFACTS_DIR:-results}"
+
+echo "=== Gleec QA CI Pipeline ==="
+
+# ---------------------------------------------------------------------------
+# Infrastructure
+# ---------------------------------------------------------------------------
+echo "[infra] Verifying Ollama..."
+if ! curl -sf http://localhost:11434/api/tags > /dev/null 2>&1; then
+    echo "[infra] Starting Ollama..."
+    ollama serve &
+    sleep 5
+fi
+
+echo "[infra] Starting Docker stack..."
+docker compose up -d
+sleep 10
+
+# ---------------------------------------------------------------------------
+# Smoke gate (fast, blocks deployment on failure)
+# ---------------------------------------------------------------------------
+echo "[smoke] Running smoke gate..."
+python -m runner.runner --matrix "$MATRIX" --tag smoke --single
+SMOKE_EXIT=$?
+
+if [ $SMOKE_EXIT -eq 2 ]; then
+    echo "[smoke] INFRASTRUCTURE FAILURE — aborting pipeline"
+    exit 2
+fi
+
+if [ $SMOKE_EXIT -eq 1 ]; then
+    echo "[smoke] SMOKE GATE FAILED — blocking deployment"
+    # Copy whatever reports exist
+    cp results/run_*/report.html "$ARTIFACTS_DIR/" 2>/dev/null || true
+    exit 1
+fi
+
+echo "[smoke] Smoke gate passed (exit=$SMOKE_EXIT)"
+
+# ---------------------------------------------------------------------------
+# Full suite (with retries and majority vote)
+# ---------------------------------------------------------------------------
+echo "[full] Running full automated suite..."
+python -m runner.runner --matrix "$MATRIX"
+FULL_EXIT=$?
+
+# ---------------------------------------------------------------------------
+# Collect artifacts
+# ---------------------------------------------------------------------------
+echo "[artifacts] Collecting reports..."
+mkdir -p "$ARTIFACTS_DIR"
+
+LATEST_RUN=$(ls -td results/run_* 2>/dev/null | head -1)
+if [ -n "$LATEST_RUN" ]; then
+    cp "$LATEST_RUN/report.html" "$ARTIFACTS_DIR/" 2>/dev/null || true
+    cp "$LATEST_RUN/results.json" "$ARTIFACTS_DIR/" 2>/dev/null || true
+fi
+
+echo "[done] Pipeline complete (exit=$FULL_EXIT)"
+exit $FULL_EXIT

automated_testing/docker-compose.yml

@@ -0,0 +1,33 @@
+services:
+  postgres:
+    image: postgres:15
+    environment:
+      POSTGRES_USER: skyvern
+      POSTGRES_PASSWORD: skyvern
+      POSTGRES_DB: skyvern
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U skyvern"]
+      interval: 5s
+      retries: 5
+
+  skyvern:
+    image: ghcr.io/skyvern-ai/skyvern:latest
+    depends_on:
+      postgres:
+        condition: service_healthy
+    ports:
+      - "8000:8000"
+    env_file:
+      - .env
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - ./results/videos:/app/videos
+      - ./results/screenshots:/app/artifacts
+
+volumes:
+  pgdata: