feat: CI improvements

.github/workflows/job_test_api_local.yaml

@@ -6,14 +6,12 @@ permissions:
 jobs:
   test:
     name: API Test Local
-    timeout-minutes: 25
+    timeout-minutes: 90
     runs-on: blacksmith-8vcpu-ubuntu-2404
     steps:
       - uses: actions/checkout@v4
       - name: Set up Docker Buildx
         uses: useblacksmith/setup-docker-builder@v1
-      - name: Delete huge unnecessary tools folder
-        run: rm -rf /opt/hostedtoolcache
       - name: Run containers
         run: docker compose -f ./deployment/docker-compose.yaml up mysql redis clickhouse planetscale agent s3 apiv2 api -d --wait
         env:

.github/workflows/job_test_go_api_local.yaml

@@ -7,22 +7,30 @@ jobs:
   tests:
     name: Test
     runs-on: blacksmith-16vcpu-ubuntu-2404
-    timeout-minutes: 25
     steps:
       - uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: useblacksmith/setup-docker-builder@v1
       - name: Setup Go
         uses: ./.github/actions/setup-go
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
       - name: Install Go test tools
-        run: |
-          go install github.com/mfridman/tparse@v0.14.0
+        run: go install github.com/mfridman/tparse@v0.14.0
         working-directory: go
-      - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
-      - name: Test
-        run: make test-full
+      - name: Run unit and light integration tests
+        run: make test-integration
+        working-directory: go
+        env:
+          DOCKER_BUILDKIT: 1
+          COMPOSE_DOCKER_CLI_BUILD: 1
+      - name: Run stress tests
+        # Run only if we're being merged (to main) or we're a new tag
+        if: github.event_name == 'merge_group' || startsWith(github.ref, 'refs/tags/')
+        run: make test-stress
         working-directory: go
         env:
           INTEGRATION_TEST: true
           SIMULATION_TEST: false
+          DOCKER_BUILDKIT: 1
+          COMPOSE_DOCKER_CLI_BUILD: 1

deployment/config/prometheus.yml

@@ -1,9 +1,20 @@
 global:
   scrape_interval: 15s
+  # Enable legacy metric name validation if needed for compatibility
+  metric_name_validation_scheme: legacy
 
 scrape_configs:
   - job_name: "prometheus"
-
+    # Explicitly specify scrape protocol fallback for compatibility
+    scrape_protocols: 
+      - "OpenMetricsText1.0.0"
+      - "OpenMetricsText0.0.1" 
+      - "PrometheusText0.0.4"
+
+    # Enable HTTP/2 for scraping if supported
+    http_config:
+      enable_http2: true
+
     http_sd_configs:
       - url: http://apiv2:2112/sd
         refresh_interval: "60s"

go/Makefile

@@ -1,4 +1,4 @@
-.PHONY: install fmt test-unit test-full build generate lint pull up down
+.PHONY: install fmt test-unit test-integration test-integration-long test-stress test build generate pull up clean
 
 # Detect OS and set GOMAXPROCS accordingly
 UNAME_S := $(shell uname -s)
@@ -16,15 +16,15 @@ PARALLEL_PROCS := $(shell if [ $(GOMAXPROCS_VAL) -gt 1 ]; then expr $(GOMAXPROCS
 install:
 	go mod tidy
 
-fmt: lint
+fmt:
 	go fmt ./...
 	golangci-lint run
 
 pull:
 	@docker compose -f ../deployment/docker-compose.yaml pull
 
-up:
-	@docker compose -f ../deployment/docker-compose.yaml up --force-recreate -d mysql planetscale redis clickhouse s3 otel
+up: pull
+	@docker compose -f ../deployment/docker-compose.yaml up -d planetscale mysql redis clickhouse s3 otel
 	@echo "Starting ClickHouse migrations (will retry if ClickHouse isn't ready)..."
 	@for i in {1..10}; do \
 		echo "Migration attempt $$i..."; \
@@ -37,23 +37,36 @@ up:
 		fi; \
 	done
 
-down:
+clean:
 	@docker compose -f ../deployment/docker-compose.yaml down --volumes
 
-test-full: export INTEGRATION_TEST=true
-test-full: export SIMULATION_TEST=false
-test-full: up
-	@echo "Running full tests..."
-	@echo "Using $(PARALLEL_PROCS) parallel test processes"
-	@go test -json -failfast -timeout=15m -parallel=$(PARALLEL_PROCS) ./... | tparse -all -progress -smallscreen
-
-test-unit: up
-	@echo "Using $(PARALLEL_PROCS) parallel test processes"
-	@go test -json -failfast -timeout=15m -parallel=$(PARALLEL_PROCS) -race ./... | tparse -all -progress -smallscreen
-
 build:
 	go build -o unkey ./main.go
 
 generate:
 	buf generate
 	go generate ./...
+
+test: test-unit
+
+test-unit: up
+	@echo "Running tests w/$(PARALLEL_PROCS) parallel test processes"
+	@go test -json -failfast -timeout=15m -parallel=$(PARALLEL_PROCS) -race ./... | tparse -all -progress -smallscreen
+
+test-stress: export INTEGRATION_TEST=true
+test-stress: export SIMULATION_TEST=false
+test-stress: up
+	@echo "Running stress tests w/$(PARALLEL_PROCS) parallel test processes"
+	@go test -tags=stress,integration,integration_long -json -failfast -timeout=15m -parallel=$(PARALLEL_PROCS) ./... | tparse -all -progress -smallscreen
+
+test-integration-long: export INTEGRATION_TEST=true
+test-integration-long: export SIMULATION_TEST=false
+test-integration-long: up
+	@echo "Running long-ish integration tests w/$(PARALLEL_PROCS) parallel test processes"
+	@go test -tags=integration,integration_long -json -failfast -timeout=15m -parallel=$(PARALLEL_PROCS) ./... | tparse -all -progress -smallscreen
+
+test-integration: export INTEGRATION_TEST=true
+test-integration: export SIMULATION_TEST=false
+test-integration: up
+	@echo "Running integration tests w/$(PARALLEL_PROCS) parallel test processes"
+	@go test -tags=integration -json -failfast -timeout=15m -parallel=$(PARALLEL_PROCS) ./... | tparse -all -progress -smallscreen

go/apps/api/integration/multi_node_ratelimiting/generate_tests/main.go

@@ -1,17 +1,15 @@
-package main
-
 //go:generate go run main.go
+package main
 
 import (
 	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 	"text/template"
-	"time"
 )
 
-// TestCase defines a single rate limit test configuration
+// TestCase defines a single ratelimit test configuration
 type TestCase struct {
 	NodeCount   int
 	Limit       int64
@@ -20,6 +18,54 @@ type TestCase struct {
 	WindowCount int
 }
 
+var extremeEdgeCases = []TestCase{
+	{NodeCount: 9, Limit: 100, Duration: 30000, LoadFactor: 2.0, WindowCount: 5},
+	{NodeCount: 9, Limit: 100, Duration: 300000, LoadFactor: 1.0, WindowCount: 3},
+}
+
+var realisticCombinations = []TestCase{
+	{NodeCount: 1, Limit: 100, Duration: 10000, LoadFactor: 0.9, WindowCount: 10},
+	{NodeCount: 3, Limit: 100, Duration: 10000, LoadFactor: 0.9, WindowCount: 10},
+	{NodeCount: 5, Limit: 100, Duration: 10000, LoadFactor: 0.9, WindowCount: 10},
+
+	{NodeCount: 1, Limit: 100, Duration: 10000, LoadFactor: 1.0, WindowCount: 10},
+	{NodeCount: 3, Limit: 100, Duration: 10000, LoadFactor: 1.0, WindowCount: 10},
+	{NodeCount: 5, Limit: 100, Duration: 10000, LoadFactor: 1.0, WindowCount: 10},
+
+	{NodeCount: 1, Limit: 100, Duration: 10000, LoadFactor: 2.0, WindowCount: 10},
+	{NodeCount: 3, Limit: 100, Duration: 10000, LoadFactor: 2.0, WindowCount: 10},
+	{NodeCount: 5, Limit: 100, Duration: 10000, LoadFactor: 2.0, WindowCount: 10},
+
+	{NodeCount: 1, Limit: 100, Duration: 300000, LoadFactor: 0.9, WindowCount: 10},
+	{NodeCount: 3, Limit: 100, Duration: 300000, LoadFactor: 0.9, WindowCount: 10},
+	{NodeCount: 5, Limit: 100, Duration: 300000, LoadFactor: 0.9, WindowCount: 10},
+
+	{NodeCount: 3, Limit: 100, Duration: 300000, LoadFactor: 1.5, WindowCount: 10},
+	{NodeCount: 5, Limit: 100, Duration: 300000, LoadFactor: 1.5, WindowCount: 10},
+
+	{NodeCount: 1, Limit: 10, Duration: 60000, LoadFactor: 0.9, WindowCount: 10},
+	{NodeCount: 3, Limit: 10, Duration: 60000, LoadFactor: 0.9, WindowCount: 10},
+	{NodeCount: 5, Limit: 10, Duration: 60000, LoadFactor: 0.9, WindowCount: 10},
+	{NodeCount: 3, Limit: 10, Duration: 60000, LoadFactor: 1.5, WindowCount: 10},
+	{NodeCount: 5, Limit: 10, Duration: 60000, LoadFactor: 1.5, WindowCount: 10},
+}
+
+// BuildTag returns the go:build constraint line (or empty) appropriate for tc.
+// It is used by the generator template to decide which tests are compiled by default.
+func (tc TestCase) BuildTag() string {
+	switch {
+	// Single nodes always run
+	case tc.NodeCount == 1:
+		return "//go:build integration"
+	// Anything less than 5 is an integration test
+	case tc.NodeCount <= 5:
+		return "//go:build integration_long"
+	// Assume its a stress test to ensure fast tests stay fast
+	default:
+		return "//go:build stress"
+	}
+}
+
 func (tc TestCase) PackageName() string {
 	return fmt.Sprintf("ratelimit_nodes%02d_limit%04d_duration%09d_load%s_windows%03d",
 		tc.NodeCount,
@@ -28,7 +74,6 @@ func (tc TestCase) PackageName() string {
 		strings.ReplaceAll(fmt.Sprintf("%05.2f", tc.LoadFactor), ".", "_"),
 		tc.WindowCount,
 	)
-
 }
 
 func (tc TestCase) TestName() string {
@@ -39,19 +84,11 @@ func (tc TestCase) TestName() string {
 		strings.ReplaceAll(fmt.Sprintf("%05.2f", tc.LoadFactor), ".", "_"),
 		tc.WindowCount,
 	)
-
 }
 
-// Define test matrix parameters
-var (
-	nodeCounts   = []int{1, 3, 9}
-	limits       = []int64{5, 100}
-	durations    = []time.Duration{time.Second, time.Minute, time.Hour}
-	loadFactors  = []float64{0.9, 2.0, 10.0}
-	windowCounts = []int{10}
-)
-
-const testTemplate = `// Code generated by go generate; DO NOT EDIT.
+const testTemplate = `// Code generated by go generate; DO NOT EDIT.{{ if .BuildTag }}
+{{ .BuildTag }}
+{{ end }}
 package {{ .PackageName }}
 
 import (
@@ -88,7 +125,7 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
-	err = os.MkdirAll(baseDir, 0755)
+	err = os.MkdirAll(baseDir, 0o755)
 	if err != nil {
 		panic(err)
 	}
@@ -99,30 +136,15 @@ func main() {
 		panic(err)
 	}
 
-	// Generate test cases
-	var testCases []TestCase
-	for _, nodeCount := range nodeCounts {
-		for _, limit := range limits {
-			for _, duration := range durations {
-				for _, loadFactor := range loadFactors {
-					for _, windowCount := range windowCounts {
-						testCases = append(testCases, TestCase{
-							NodeCount:   nodeCount,
-							Limit:       limit,
-							Duration:    duration.Milliseconds(),
-							LoadFactor:  loadFactor,
-							WindowCount: windowCount,
-						})
-					}
-				}
-			}
-		}
-	}
+	// Combine all test cases
+	testCases := make([]TestCase, 0, len(realisticCombinations)+len(extremeEdgeCases))
+	testCases = append(testCases, realisticCombinations...)
+	testCases = append(testCases, extremeEdgeCases...)
 
 	// Generate test files
 	for _, tc := range testCases {
 		packageDir := filepath.Join(baseDir, tc.PackageName())
-		err := os.MkdirAll(packageDir, 0755)
+		err := os.MkdirAll(packageDir, 0o755)
 		if err != nil {
 			panic(err)
 		}
@@ -144,5 +166,6 @@ func main() {
 		}
 	}
 
-	fmt.Printf("Generated %d test cases\n", len(testCases))
+	fmt.Printf("Generated %d test cases (%d realistic + %d extreme)\n",
+		len(testCases), len(realisticCombinations), len(extremeEdgeCases))
 }