diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..2e5e944 --- /dev/null +++ b/.env.example @@ -0,0 +1,61 @@ +# Archon Docker Compose Environment Configuration +# This file contains environment variables for all services in the Archon stack + +# ============================================================================= +# Service Ports and Networking +# ============================================================================= +HOST=0.0.0.0 +LOG_LEVEL=INFO + +# Frontend Service +VITE_PORT=5173 +NODE_ENV=development + +# Server Service +ARCHON_SERVER_PORT=8181 +ARCHON_SERVER_HOST=archon-server + +# Agents Service +ARCHON_AGENTS_PORT=8052 + +# MCP Service +ARCHON_MCP_PORT=8051 + +# ============================================================================= +# Database Configuration +# ============================================================================= +# Supabase Configuration (replace with your actual values) +SUPABASE_URL={your_supabase_url} +SUPABASE_ANON_KEY={your_supabase_anon_key} +SUPABASE_SERVICE_KEY={your_supabase_service_key} + +# Alternative database URL format +DATABASE_URL=postgresql://user:password@host:port/database + +# ============================================================================= +# Security and Authentication +# ============================================================================= +# Add your authentication keys here +# JWT_SECRET=your_jwt_secret_here +# API_KEY=your_api_key_here + +# ============================================================================= +# Feature Flags +# ============================================================================= +PROJECTS_ENABLED=true + +# ============================================================================= +# Development/Testing Overrides +# ============================================================================= +# Uncomment these for local development/testing +# SUPABASE_URL=http://localhost:54321 +# SUPABASE_ANON_KEY=test_key +# SUPABASE_SERVICE_KEY=test_service_key + +# ============================================================================= +# Production Configuration +# ============================================================================= +# For production deployments, set these values: +# NODE_ENV=production +# LOG_LEVEL=WARN +# Add your production database URLs and keys above \ No newline at end of file diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 002f815..3f5eba4 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -25,7 +25,7 @@ on: env: REGISTRY: ghcr.io UPSTREAM_REPO: coleam00/Archon - + jobs: build-images: runs-on: ubuntu-latest @@ -107,6 +107,7 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + no-cache: true # Force rebuild to see actual dependency installation cache-from: type=gha,scope=build-${{ matrix.component.name }} cache-to: type=gha,mode=max,scope=build-${{ matrix.component.name }} platforms: linux/amd64 # AMD64 only for faster builds diff --git a/.github/workflows/check-updates.yml b/.github/workflows/check-updates.yml index aa49484..138e9c3 100644 --- a/.github/workflows/check-updates.yml +++ b/.github/workflows/check-updates.yml @@ -17,9 +17,9 @@ on: default: false type: boolean -env: +env: UPSTREAM_REPO: coleam00/Archon - + jobs: check-for-updates: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7160869 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +logs/ + +# Environment variables +.env + +# Claude +.claude/ +CLAUDE.md diff --git a/README.md b/README.md index f6c53df..dc51285 100644 --- a/README.md +++ b/README.md @@ -85,11 +85,18 @@ services: image: ghcr.io/yourusername/archon-mcp:latest ports: - "8051:8051" + environment: + - ARCHON_MCP_PORT=8051 + - ARCHON_SERVER_PORT=8181 + - ARCHON_AGENTS_PORT=8052 + - VITE_PORT=5173 archon-agents: image: ghcr.io/yourusername/archon-agents:latest ports: - "8052:8052" + environment: + - ARCHON_AGENTS_PORT=8052 archon-frontend: image: ghcr.io/yourusername/archon-frontend:latest @@ -104,10 +111,10 @@ services: docker run -p 8181:8181 ghcr.io/yourusername/archon-server:latest # Run MCP service -docker run -p 8051:8051 ghcr.io/yourusername/archon-mcp:latest +docker run -p 8051:8051 -e ARCHON_MCP_PORT=8051 -e ARCHON_SERVER_PORT=8181 -e ARCHON_AGENTS_PORT=8052 -e VITE_PORT=5173 ghcr.io/yourusername/archon-mcp:latest # Run agents service -docker run -p 8052:8052 ghcr.io/yourusername/archon-agents:latest +docker run -p 8052:8052 -e ARCHON_AGENTS_PORT=8052 ghcr.io/yourusername/archon-agents:latest # Run frontend - FIXED: Use consistent port mapping docker run -p 5173:5173 ghcr.io/yourusername/archon-frontend:latest diff --git a/docker-compose.registry.yml b/docker-compose.registry.yml index d419f36..1cb4ef0 100644 --- a/docker-compose.registry.yml +++ b/docker-compose.registry.yml @@ -4,13 +4,13 @@ services: frontend: image: ghcr.io/dapperdivers/archon-frontend:latest ports: - - "5173:5173" + - "${VITE_PORT}:${VITE_PORT}" environment: - - NODE_ENV=development - - HOST=0.0.0.0 - - PORT=5173 + - NODE_ENV=${NODE_ENV} + - HOST=${HOST} + - PORT=${VITE_PORT} healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:5173/health.html || exit 1"] + test: ["CMD-SHELL", "curl -f http://localhost:${VITE_PORT}/health.html || exit 1"] interval: 30s timeout: 10s retries: 3 @@ -19,14 +19,14 @@ services: agents: image: ghcr.io/dapperdivers/archon-agents:latest ports: - - "8052:8052" + - "${ARCHON_AGENTS_PORT}:${ARCHON_AGENTS_PORT}" environment: - - HOST=0.0.0.0 - - PORT=8052 - - ARCHON_AGENTS_PORT=8052 - - LOG_LEVEL=INFO + - HOST=${HOST} + - PORT=${ARCHON_AGENTS_PORT} + - ARCHON_AGENTS_PORT=${ARCHON_AGENTS_PORT} + - LOG_LEVEL=${LOG_LEVEL} healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:8052/health || exit 1"] + test: ["CMD-SHELL", "curl -f http://localhost:${ARCHON_AGENTS_PORT}/health || exit 1"] interval: 30s timeout: 10s retries: 3 @@ -35,15 +35,15 @@ services: server: image: ghcr.io/dapperdivers/archon-server:latest ports: - - "8181:8181" + - "${ARCHON_SERVER_PORT}:${ARCHON_SERVER_PORT}" environment: - - HOST=0.0.0.0 - - PORT=8181 - - ARCHON_SERVER_PORT=8181 - - LOG_LEVEL=INFO - - SUPABASE_URL=https://localhost:54321 - - SUPABASE_ANON_KEY=test_key - - SUPABASE_SERVICE_KEY=test_service_key + - HOST=${HOST} + - PORT=${ARCHON_SERVER_PORT} + - ARCHON_SERVER_PORT=${ARCHON_SERVER_PORT} + - LOG_LEVEL=${LOG_LEVEL} + - SUPABASE_URL=${SUPABASE_URL} + - SUPABASE_ANON_KEY=${SUPABASE_ANON_KEY} + - SUPABASE_SERVICE_KEY=${SUPABASE_SERVICE_KEY} healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8181/health || exit 1"] interval: 30s @@ -54,12 +54,12 @@ services: mcp: image: ghcr.io/dapperdivers/archon-mcp:latest ports: - - "8051:8051" + - "${ARCHON_MCP_PORT}:${ARCHON_MCP_PORT}" environment: - - HOST=0.0.0.0 - - PORT=8051 - - ARCHON_MCP_PORT=8051 - - LOG_LEVEL=INFO + - HOST=${HOST} + - PORT=${ARCHON_MCP_PORT} + - ARCHON_MCP_PORT=${ARCHON_MCP_PORT} + - LOG_LEVEL=${LOG_LEVEL} # Note: MCP service may not have a /health endpoint healthcheck: test: ["CMD-SHELL", "netstat -tuln | grep :8051 || exit 1"] diff --git a/dockerfiles/agents/Dockerfile b/dockerfiles/agents/Dockerfile index 7c07ea8..b32332e 100644 --- a/dockerfiles/agents/Dockerfile +++ b/dockerfiles/agents/Dockerfile @@ -1,5 +1,5 @@ # Fast single-stage agents build with security hardening -FROM python:3.11-slim +FROM python:3.12-slim # Set proper environment defaults for Kubernetes ENV HOST=0.0.0.0 @@ -20,8 +20,8 @@ RUN apt-get update && apt-get install -y \ && apt-get clean # Copy and install requirements from local source -COPY requirements.agents.txt . -RUN pip install --no-cache-dir -r requirements.agents.txt +COPY pyproject.toml uv.lock ./ +RUN pip install --upgrade pip && pip install --no-cache-dir -e . --group agents --group server # Copy application code COPY . . diff --git a/dockerfiles/mcp/Dockerfile b/dockerfiles/mcp/Dockerfile index 5751b9d..12947b7 100644 --- a/dockerfiles/mcp/Dockerfile +++ b/dockerfiles/mcp/Dockerfile @@ -1,5 +1,5 @@ # MCP Service - Lightweight HTTP-based microservice -FROM python:3.11-slim +FROM python:3.12-slim WORKDIR /app @@ -11,14 +11,14 @@ RUN apt-get update && apt-get install -y \ && apt-get clean # Install dependencies -COPY requirements.mcp.txt . -RUN pip install --no-cache-dir -r requirements.mcp.txt +COPY pyproject.toml uv.lock ./ +RUN pip install --upgrade pip && pip install --no-cache-dir -e . --group mcp --group server # Create minimal directory structure -RUN mkdir -p src/mcp/modules src/server/services src/server/config +RUN mkdir -p src/mcp_server/modules src/server/services src/server/config # Copy only MCP-specific files (lightweight protocol wrapper) -COPY src/mcp/ src/mcp/ +COPY src/mcp_server/ src/mcp_server/ COPY src/__init__.py src/ # Copy only the minimal server files MCP needs for HTTP communication @@ -54,4 +54,4 @@ EXPOSE ${ARCHON_MCP_PORT} # Health check disabled - use TCP checks in Kubernetes instead # Run the MCP server using module syntax (avoids naming conflicts) -CMD ["python", "-m", "src.mcp.mcp_server"] \ No newline at end of file +CMD ["python", "-m", "src.mcp_server.mcp_server"] \ No newline at end of file diff --git a/dockerfiles/server/Dockerfile b/dockerfiles/server/Dockerfile index fe0fd26..c6de0a8 100644 --- a/dockerfiles/server/Dockerfile +++ b/dockerfiles/server/Dockerfile @@ -1,5 +1,26 @@ -# Fast single-stage server build with security hardening -FROM python:3.11-slim +# Server Service - Web crawling and document processing microservice with security hardening + +# Build stage +FROM python:3.12 AS builder + +WORKDIR /build + +# Install build dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Copy and install Python dependencies +COPY pyproject.toml uv.lock ./ +RUN pip install --upgrade pip && pip install --user --no-cache-dir -e . --group server + +# Runtime stage +FROM python:3.12-slim + +# Re-declare ARG for runtime stage +ARG BUILD_VERSION="2025-09-01T13:00:00Z-remove-manual-playwright" + +WORKDIR /app # Set proper environment defaults for Kubernetes ENV HOST=0.0.0.0 @@ -8,13 +29,6 @@ ENV ARCHON_SERVER_PORT=8181 ENV LOG_LEVEL=INFO ENV PYTHONUNBUFFERED=1 ENV PYTHONDONTWRITEBYTECODE=1 -ENV PYTHONPATH=/app:/app/src - -WORKDIR /app - -# Copy and install requirements from local source -COPY requirements.server.txt . -RUN pip install --no-cache-dir -r requirements.server.txt # Install runtime dependencies for Playwright (minimal set) RUN apt-get update && apt-get install -y \ @@ -42,22 +56,39 @@ RUN apt-get update && apt-get install -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +# Copy Python packages from builder +COPY --from=builder /root/.local /root/.local +# Ensure playwright browsers are available +ENV PATH=/root/.local/bin:$PATH +ENV PLAYWRIGHT_BROWSERS_PATH=/opt/playwright-browsers +ENV PYTHONPATH="/root/.local/lib/python3.12/site-packages" +# Install playwright browsers (playwright itself comes from crawl4ai dependency) +RUN echo "=== BUILD VERSION: ${BUILD_VERSION} ===" && \ + mkdir -p /opt/playwright-browsers && \ + python -m playwright install chromium && \ + chmod -R 755 /opt/playwright-browsers -# Copy application code -COPY . . +# Copy server code and tests (selective copying like upstream) +COPY src/server/ src/server/ +COPY src/__init__.py src/ +COPY tests/ tests/ -# Create non-root user first +# Create non-root user for runtime security RUN addgroup --gid 1001 archon && \ adduser --disabled-password --gecos '' --uid 1001 --gid 1001 archon && \ - chown -R archon:archon /app + # Create home directory structure + mkdir -p /home/archon/.cache && \ + chown -R archon:archon /app /home/archon && \ + # Allow non-root user access to Python packages in /root/.local + chmod -R 755 /root/.local && \ + chmod 755 /root -# Switch to non-root user and install Playwright browsers for this user +# Switch to non-root user for runtime USER 1001 ENV HOME=/home/archon - -# Install Playwright browsers as the archon user so they're accessible at runtime -RUN python -m playwright install chromium +ENV PLAYWRIGHT_BROWSERS_PATH=/opt/playwright-browsers +ENV PYTHONPATH="/app:/root/.local/lib/python3.12/site-packages:$PYTHONPATH" EXPOSE ${ARCHON_SERVER_PORT} @@ -65,5 +96,5 @@ EXPOSE ${ARCHON_SERVER_PORT} HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \ CMD python -c "import urllib.request, os; urllib.request.urlopen(f'http://localhost:{os.environ[\"ARCHON_SERVER_PORT\"]}/health').read()" || exit 1 -# Run the server with configurable port (matching original Archon structure) -CMD sh -c "python -m uvicorn src.server.main:socket_app --host 0.0.0.0 --port ${ARCHON_SERVER_PORT} --workers 1" \ No newline at end of file +# Run the server with configurable port (using JSON array format for better signal handling) +CMD ["sh", "-c", "python -m uvicorn src.server.main:socket_app --host 0.0.0.0 --port ${ARCHON_SERVER_PORT} --workers 1"] \ No newline at end of file diff --git a/tests/docker-compose.test.yml b/tests/docker-compose.test.yml index 477d73b..ec5a477 100644 --- a/tests/docker-compose.test.yml +++ b/tests/docker-compose.test.yml @@ -1,10 +1,8 @@ -version: '3.8' - services: agents: build: context: /tmp/archon-source/python - dockerfile: /home/derek/projects/archon-images/dockerfiles/agents/Dockerfile + dockerfile: /Volumes/Storage/Projects/archon-images/dockerfiles/agents/Dockerfile image: archon-agents:test environment: - ARCHON_AGENTS_PORT=8052 @@ -24,7 +22,7 @@ services: archon-server: build: context: /tmp/archon-source/python - dockerfile: /home/derek/projects/archon-images/dockerfiles/server/Dockerfile + dockerfile: /Volumes/Storage/Projects/archon-images/dockerfiles/server/Dockerfile image: archon-server:test environment: - ARCHON_SERVER_PORT=8181 @@ -44,7 +42,7 @@ services: mcp: build: context: /tmp/archon-source/python - dockerfile: /home/derek/projects/archon-images/dockerfiles/mcp/Dockerfile + dockerfile: /Volumes/Storage/Projects/archon-images/dockerfiles/mcp/Dockerfile image: archon-mcp:test environment: - ARCHON_MCP_PORT=8051 @@ -56,7 +54,7 @@ services: frontend: build: context: /tmp/archon-source/archon-ui-main - dockerfile: /home/derek/projects/archon-images/dockerfiles/frontend/Dockerfile + dockerfile: /Volumes/Storage/Projects/archon-images/dockerfiles/frontend/Dockerfile image: archon-frontend:test environment: - VITE_PORT=5173 diff --git a/tests/test-containers.sh b/tests/test-containers.sh index 7df1520..093da0b 100755 --- a/tests/test-containers.sh +++ b/tests/test-containers.sh @@ -32,7 +32,8 @@ print_warning() { # Cleanup function cleanup() { print_status "Cleaning up containers and networks..." - docker compose -f docker-compose.test.yml down --remove-orphans 2>/dev/null || true + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + docker compose -f "$SCRIPT_DIR/docker-compose.test.yml" down --remove-orphans 2>/dev/null || true docker system prune -f 2>/dev/null || true } @@ -54,7 +55,8 @@ fi # Build all images print_status "Building Docker images..." -docker compose -f docker-compose.test.yml build --no-cache +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +docker compose -f "$SCRIPT_DIR/docker-compose.test.yml" build --no-cache if [ $? -eq 0 ]; then print_success "All images built successfully" @@ -65,7 +67,7 @@ fi # Start containers print_status "Starting containers..." -docker compose -f docker-compose.test.yml up -d +docker compose -f "$SCRIPT_DIR/docker-compose.test.yml" up -d # Wait for containers to start print_status "Waiting for containers to start (30 seconds)..." @@ -73,7 +75,7 @@ sleep 30 # Check container status print_status "Checking container status..." -docker compose -f docker-compose.test.yml ps +docker compose -f "$SCRIPT_DIR/docker-compose.test.yml" ps # Test each service test_service() { @@ -84,7 +86,7 @@ test_service() { print_status "Testing $service on port $port..." # Check if container is running - if ! docker compose -f docker-compose.test.yml ps $service | grep -q "Up"; then + if ! docker compose -f "$SCRIPT_DIR/docker-compose.test.yml" ps $service | grep -q "Up"; then print_error "$service container is not running" return 1 fi @@ -102,7 +104,7 @@ test_service() { # Show recent logs print_status "Recent logs for $service:" - docker compose -f docker-compose.test.yml logs --tail=10 $service + docker compose -f "$SCRIPT_DIR/docker-compose.test.yml" logs --tail=10 $service } # Test all services @@ -113,13 +115,13 @@ test_service "mcp" "8051" "" # Show overall health print_status "Health check status:" -docker compose -f docker-compose.test.yml ps +docker compose -f "$SCRIPT_DIR/docker-compose.test.yml" ps # Check for any error logs print_status "Checking for error logs..." error_count=0 for service in frontend agents server mcp; do - if docker compose -f docker-compose.test.yml logs $service | grep -i "error\|exception\|failed" | grep -v "test"; then + if docker compose -f "$SCRIPT_DIR/docker-compose.test.yml" logs $service | grep -i "error\|exception\|failed" | grep -v "test"; then print_error "Found errors in $service logs" ((error_count++)) fi @@ -133,7 +135,7 @@ fi print_status "Test complete. Containers will be cleaned up automatically." print_status "If you want to keep containers running, press Ctrl+C now and run:" -print_status " docker compose -f docker-compose.test.yml down" +print_status " docker compose -f $SCRIPT_DIR/docker-compose.test.yml down" # Keep running for a bit to allow manual inspection sleep 10 \ No newline at end of file