chore: remove etcd in favor of redis

.env.ci

@@ -23,7 +23,8 @@ ATTESTATION_HOST = "attestation"
 ATTESTATION_PORT = 8080
 
 # nilAuth Trusted URLs
-NILAUTH_TRUSTED_ROOT_ISSUERS = "http://nilauth:30921"
+NILAUTH_TRUSTED_ROOT_ISSUERS = "http://nilauth-credit-server:3000" # "http://nilauth:30921"
+CREDIT_API_TOKEN = "n i l l i o n"
 
 # Postgres Docker Compose Config
 POSTGRES_HOST = "postgres"
@@ -37,9 +38,9 @@ POSTGRES_PORT = 5432
 # Redis Docker Compose Config
 REDIS_URL = "redis://redis:6379"
 
-# Etcd Docker Compose Config
-ETCD_HOST = "etcd"
-ETCD_PORT = 2379
+# Model Discovery Redis Docker Compose Config
+DISCOVERY_HOST = "redis"
+DISCOVERY_PORT = 6379
 
 # Grafana Docker Compose Config
 GF_SECURITY_ADMIN_USER = "admin"

.github/workflows/cicd.yml

@@ -11,10 +11,14 @@ on:
 permissions:
   id-token: write # Required for OIDC
   contents: read  # Required for checkout
+  packages: read  # Required for GHCR access
 
 jobs:
   test:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test-type: [pyright, unit, integration]
 
     steps:
       - uses: actions/checkout@v4
@@ -39,9 +43,11 @@ jobs:
           uv sync
 
       - name: Run Ruff format check
+        if: matrix.test-type == 'pyright'
         run: uv run ruff format --check
 
       - name: Run Ruff linting
+        if: matrix.test-type == 'pyright'
         run: uv run ruff check --exclude packages/verifier/
 
       - name: Create .env for tests
@@ -51,13 +57,16 @@ jobs:
           sed -i 's/HF_TOKEN=.*/HF_TOKEN=dummy_token/' .env
           sed -i 's/BRAVE_SEARCH_API=.*/BRAVE_SEARCH_API=dummy_api/' .env
 
-      - name: pyright
+      - name: Run pyright
+        if: matrix.test-type == 'pyright'
         run: uv run pyright
 
       - name: Run unit tests
+        if: matrix.test-type == 'unit'
         run: uv run pytest -v tests/unit
 
       - name: Run integration tests
+        if: matrix.test-type == 'integration'
         run: uv run pytest -v tests/integration
 
   start-runner:
@@ -73,7 +82,7 @@ jobs:
         with:
           aws-access-key-id: ${{ secrets.GH_AWS_ACCESS_KEY }}
           aws-secret-access-key: ${{ secrets.GH_AWS_SECRET_KEY }}
-          aws-region: "eu-west-1"
+          aws-region: "us-east-1"
       - name: Start EC2 runner
         id: start-ec2-runner
         uses: NillionNetwork/[email protected]
@@ -82,12 +91,12 @@ jobs:
           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
           runners-per-machine: 3
           number-of-machines: 1
-          ec2-image-id: ami-0174a246556e8750b
-          ec2-instance-type: g4dn.xlarge
-          subnet-id: subnet-0ec4c353621eabae2
-          security-group-id: sg-03ee5c56e1f467aa0
-          key-name: production-github-runner-key
-          iam-role-name: github-runners-production-github-runner-ec2
+          ec2-image-id: ami-0e70d84403fc045d7
+          ec2-instance-type: g6.xlarge
+          subnet-id: subnet-0bb357f46d1bc355c
+          security-group-id: sg-022a5cdcf57e9618b
+          key-name: us-east-1-github-runner-key
+          iam-role-name: github-runners-us-east-1-github-runner-ec2
           aws-resource-tags: >
             [
               {"Key": "Name", "Value": "github-runner-${{ github.run_id }}-${{ github.run_number }}"},
@@ -96,7 +105,7 @@ jobs:
               {"Key": "Deployment", "Value": "github-runners"},
               {"Key": "Type", "Value": "GithubRunner"},
               {"Key": "User", "Value": "ec2-user"},
-              {"Key": "Environment", "Value": "production"}
+              {"Key": "Environment", "Value": "us-east-1"}
             ]
 
   build-images:
@@ -105,7 +114,7 @@ jobs:
     runs-on: ${{ needs.start-runner.outputs.label }}
     strategy:
       matrix:
-        component: [vllm, attestation, api]
+        component: [vllm, api]
         include:
           - component: api
             build_args: "--target nilai --platform linux/amd64"
@@ -148,6 +157,13 @@ jobs:
          sed -i 's/NILDB_BUILDER_PRIVATE_KEY=.*/NILDB_BUILDER_PRIVATE_KEY=${{ secrets.NILDB_BUILDER_PRIVATE_KEY }}/' .env
          sed -i 's/NILDB_COLLECTION=.*/NILDB_COLLECTION=${{ secrets.NILDB_COLLECTION }}/' .env
 
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GH_PAT }}
+
       - name: Compose docker-compose.yml
         run: python3 ./scripts/docker-composer.py --dev -f docker/compose/docker-compose.llama-1b-gpu.ci.yml -o development-compose.yml
 
@@ -279,7 +295,7 @@ jobs:
     if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'release'
     strategy:
       matrix:
-        component: [vllm, attestation, api]
+        component: [vllm, api]
     steps:
       - name: Configure AWS credentials for ECR
         uses: aws-actions/configure-aws-credentials@v4
@@ -327,7 +343,7 @@ jobs:
         with:
           aws-access-key-id: ${{ secrets.GH_AWS_ACCESS_KEY }}
           aws-secret-access-key: ${{ secrets.GH_AWS_SECRET_KEY }}
-          aws-region: "eu-west-1"
+          aws-region: "us-east-1"
 
       - name: Stop EC2 runner
         uses: NillionNetwork/[email protected]

README.md

@@ -163,60 +163,6 @@ docker compose -f production-compose.yml up -d
 docker compose -f production-compose.yml logs -f
 ```
 
-### 3. Manual Component Deployment
-
-#### Components
-
-- **API Frontend**: Handles user requests and routes model interactions
-- **Databases**:
-  - **SQLite**: User registry and access management
-  - **etcd3**: Distributed key-value store for model lifecycle management
-
-#### Setup Steps
-
-1. **Start etcd3 Instance**
-   ```shell
-   docker run -d --name etcd-server \
-     -p 2379:2379 -p 2380:2380 \
-     -e ALLOW_NONE_AUTHENTICATION=yes \
-     bitnami/etcd:latest
-
-   docker run -d --name redis \
-     -p 6379:6379 \
-     redis:latest
-    ```
-
-2. **Start PostgreSQL**
-    ```shell
-    docker run -d --name postgres \
-      -e POSTGRES_USER=${POSTGRES_USER} \
-      -e POSTGRES_PASSWORD=${POSTGRES_PASSWORD} \
-      -e POSTGRES_DB=${POSTGRES_DB} \
-      -p 5432:5432 \
-      --network frontend_net \
-      --volume postgres_data:/var/lib/postgresql/data \
-      postgres:16
-    ```
-
-2. **Run API Server**
-   ```shell
-   # Development Environment
-    fastapi dev nilai-api/src/nilai_api/__main__.py --port 8080
-
-   # Production Environment
-   uv run fastapi run nilai-api/src/nilai_api/__main__.py --port 8080
-   ```
-
-3. **Run Model Instances**
-   ```shell
-   # Example: Llama 3.2 1B Model
-   # Development Environment
-   uv run fastapi dev nilai-models/src/nilai_models/models/llama_1b_cpu/__init__.py
-
-   # Production Environment
-   uv run fastapi run nilai-models/src/nilai_models/models/llama_1b_cpu/__init__.py
-   ```
-
 ## Developer Workflow
 
 ### Code Quality and Formatting
@@ -229,7 +175,7 @@ uv run pre-commit install
 
 ## Model Lifecycle Management
 
-- Models register themselves in the etcd database
+- Models register themselves in the Redis Discovery database
 - Registration includes address information with an auto-expiring lifetime
 - If a model disconnects, it is automatically removed from the available models
 

docker-compose.dev.yml

@@ -64,8 +64,6 @@ services:
     image: 'grafana/grafana:11.5.1'
     restart: unless-stopped
     user: "$UID:$GID"
-    depends_on:
-      - prometheus
     environment:
       - GF_USERS_ALLOW_SIGN_UP=false
     healthcheck:
@@ -99,6 +97,7 @@ services:
 
   nilauth-credit-server:
     image: ghcr.io/nillionnetwork/nilauth-credit:sha-cb9e36a
+    platform: linux/amd64 # for macOS to force running on Rosetta 2
     environment:
       DATABASE_URL: postgresql://nilauth:nilauth_dev_password@nilauth-postgres:5432/nilauth_credit
       HOST: 0.0.0.0
@@ -108,8 +107,10 @@ services:
     depends_on:
       nilauth-postgres:
         condition: service_healthy
+    volumes:
+      - ./scripts/credit-init.sql:/app/migrations/20251015000006_seed_test_data.sql
     healthcheck:
-      test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3000/health"]
+      test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:3000/health"]
       interval: 30s
       retries: 3
       start_period: 15s

docker-compose.testnet.yml

@@ -8,7 +8,7 @@ services:
     container_name: testnet-nilai-nuc-api
     image: nillion/nilai-api:latest
     depends_on:
-      etcd:
+      redis:
         condition: service_healthy
     restart: unless-stopped
     healthcheck:

docker-compose.yml

@@ -1,17 +1,4 @@
 services:
-  etcd:
-    container_name: etcd
-    image: 'bitnamilegacy/etcd'
-    environment:
-      - ALLOW_NONE_AUTHENTICATION=yes
-      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
-    restart: unless-stopped
-    healthcheck:
-      test: ["CMD", "etcdctl", "endpoint", "health"]
-      interval: 10s
-      timeout: 5s
-      retries: 3
-      start_period: 5s
   redis:
     container_name: redis
     image: 'redis:latest'
@@ -26,7 +13,7 @@ services:
     container_name: nilai-api
     image: nillion/nilai-api:latest
     depends_on:
-      etcd:
+      redis:
         condition: service_healthy
     restart: unless-stopped
     healthcheck:
@@ -39,7 +26,7 @@ services:
     container_name: nilai-nuc-api
     image: nillion/nilai-api:latest
     depends_on:
-      etcd:
+      redis:
         condition: service_healthy
       api:
         condition: service_healthy

docker/compose/docker-compose.deepseek-14b-gpu.yml

@@ -14,11 +14,6 @@ services:
     env_file:
       - .env
     restart: unless-stopped
-    depends_on:
-      etcd:
-        condition: service_healthy
-      llama_8b_gpu:
-        condition: service_healthy
     command: >
       --model deepseek-ai/DeepSeek-R1-Distill-Qwen-14B
       --gpu-memory-utilization 0.39
@@ -29,8 +24,8 @@ services:
     environment:
       - SVC_HOST=deepseek_14b_gpu
       - SVC_PORT=8000
-      - ETCD_HOST=etcd
-      - ETCD_PORT=2379
+      - DISCOVERY_HOST=redis
+      - DISCOVERY_PORT=6379
       - TOOL_SUPPORT=false
     volumes:
       - hugging_face_models:/root/.cache/huggingface  # cache models

docker/compose/docker-compose.gemma-27b-gpu.yml

@@ -16,7 +16,7 @@ services:
       - .env
     restart: unless-stopped
     depends_on:
-      etcd:
+      redis:
         condition: service_healthy
     command: >
       --model google/gemma-3-27b-it
@@ -29,8 +29,8 @@ services:
     environment:
       - SVC_HOST=gemma_27b_gpu
       - SVC_PORT=8000
-      - ETCD_HOST=etcd
-      - ETCD_PORT=2379
+      - DISCOVERY_HOST=redis
+      - DISCOVERY_PORT=6379
       - TOOL_SUPPORT=false
       - MULTIMODAL_SUPPORT=true
       - MODEL_NUM_RETRIES=60