clickhouse support for log_store

.gitignore

@@ -187,3 +187,71 @@ tests/cmd/seedvks/seedvks
 
 # routing harness ledgers (local run journals, never committed)
 tests/e2e/api/routing/ledger-*
+
+
+# Build artifacts
+*/build/
+*.so
+*.dll
+*.dylib
+
+# Go build cache
+*.exe
+*.exe~
+*.test
+*.out
+
+# Dependency directories
+vendor/
+
+reports/*
+!reports/.keep
+
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/versions
+
+# testing
+/coverage
+
+# build output
+ui/.next/
+ui/out/
+
+# production
+ui/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.pnpm-debug.log*
+
+# env files (can opt-in for committing if needed)
+.env*
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+
+# auto-generated TanStack Router route tree
+ui/app/routeTree.gen.ts
+
+
+bifrost-migration-cli
+
+tests/e2e/clis/reports

Makefile

@@ -823,18 +823,19 @@ test-framework: install-gotestsum ## Run framework tests
 	@$(EXPOSE_ENV); \
 	$(ECHO) "$(GREEN)Running framework tests...$(NC)"; \
 	mkdir -p $(TEST_REPORTS_DIR); \
+	rm -f $(TEST_REPORTS_DIR)/.framework-failed; \
 	cd framework && find . -name "*.go" -path "*/tests/*" -o -name "*_test.go" | head -1 > /dev/null && \
 		for dir in $$(find . -name "*_test.go" -exec dirname {} \; | sort -u); do \
 			pkg_name=$$(echo $$dir | sed 's|^\./||' | sed 's|/|-|g'); \
 			$(ECHO) "Testing $$dir..."; \
-			cd $$dir && gotestsum \
+			( cd $$dir && gotestsum \
 				--format=$(GOTESTSUM_FORMAT) \
-				--junitfile=../../$(TEST_REPORTS_DIR)/framework-$$pkg_name.xml \
-				-- -v ./... && cd - > /dev/null; \
+				--junitfile=$(CURDIR)/$(TEST_REPORTS_DIR)/framework-$$pkg_name.xml \
+				-- -v ./... ) || touch $(CURDIR)/$(TEST_REPORTS_DIR)/.framework-failed; \
 			if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
 				if which junit-viewer > /dev/null 2>&1; then \
 					$(ECHO) "$(YELLOW)Generating HTML report for $$pkg_name...$(NC)"; \
-					junit-viewer --results=../$(TEST_REPORTS_DIR)/framework-$$pkg_name.xml --save=../$(TEST_REPORTS_DIR)/framework-$$pkg_name.html 2>/dev/null || true; \
+					junit-viewer --results=$(CURDIR)/$(TEST_REPORTS_DIR)/framework-$$pkg_name.xml --save=$(CURDIR)/$(TEST_REPORTS_DIR)/framework-$$pkg_name.html 2>/dev/null || true; \
 				fi; \
 			fi; \
 		done || $(ECHO) "No framework tests found"
@@ -848,6 +849,10 @@ test-framework: install-gotestsum ## Run framework tests
 		SUMMARY_LABEL="Framework" \
 		SUMMARY_STRIP="framework-" \
 		SUMMARY_FILES="$(TEST_REPORTS_DIR)/framework-*.xml"
+	@if [ -f $(TEST_REPORTS_DIR)/.framework-failed ]; then \
+		rm -f $(TEST_REPORTS_DIR)/.framework-failed; \
+		exit 1; \
+	fi
 
 # Internal: render a table of test reports + a final pass/fail scenario.
 # Usage: $(MAKE) print-test-summary SUMMARY_LABEL="Framework" SUMMARY_STRIP="framework-" SUMMARY_FILES="<glob or files>"

core/schemas/async.go

@@ -1,10 +1,19 @@
 package schemas
 
-import "time"
+import (
+	"database/sql/driver"
+	"time"
+)
 
 // AsyncJobStatus represents the status of an async job
 type AsyncJobStatus string
 
+// Value implements driver.Valuer so database drivers that append typed
+// column values (e.g. clickhouse-go batch inserts) can serialize the type.
+func (s AsyncJobStatus) Value() (driver.Value, error) {
+	return string(s), nil
+}
+
 const (
 	AsyncJobStatusPending    AsyncJobStatus = "pending"
 	AsyncJobStatusProcessing AsyncJobStatus = "processing"

core/schemas/bifrost.go

@@ -2,6 +2,7 @@
 package schemas
 
 import (
+	"database/sql/driver"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -117,6 +118,12 @@ var StandardProviders = []ModelProvider{
 // RequestType represents the type of request being made to a provider.
 type RequestType string
 
+// Value implements driver.Valuer so database drivers that append typed
+// column values (e.g. clickhouse-go batch inserts) can serialize the type.
+func (r RequestType) Value() (driver.Value, error) {
+	return string(r), nil
+}
+
 const (
 	ListModelsRequest            RequestType = "list_models"
 	TextCompletionRequest        RequestType = "text_completion"

examples/configs/withclickhouselogstore/config.json

@@ -0,0 +1,34 @@
+{
+    "$schema": "https://www.getbifrost.ai/schema",
+    "config_store": {
+        "enabled": true,
+        "type": "sqlite",
+        "config": {
+            "path": "../../examples/configs/withclickhouselogstore/config.db"
+        }
+    },
+    "logs_store": {
+        "enabled": true,
+        "type": "clickhouse",
+        "retention_days": 30,
+        "config": {
+            "host": "localhost",
+            "port": "9001",
+            "database": "bifrost",
+            "username": "bifrost",
+            "password": "bifrost_password"
+        }
+    },
+    "providers": {
+        "openai": {
+            "keys": [
+                {
+                    "name": "openai-key-1",
+                    "value": "sk-proj-abc",
+                    "weight": 1,
+                    "models": ["*"]
+                }
+            ]
+        }
+    }
+}

examples/configs/withclickhouselogstorehttp/config.json

@@ -0,0 +1,35 @@
+{
+    "$schema": "https://www.getbifrost.ai/schema",
+    "config_store": {
+        "enabled": true,
+        "type": "sqlite",
+        "config": {
+            "path": "../../examples/configs/withclickhouselogstorehttp/config.db"
+        }
+    },
+    "logs_store": {
+        "enabled": true,
+        "type": "clickhouse",
+        "retention_days": 30,
+        "config": {
+            "host": "localhost",
+            "port": "8123",
+            "database": "bifrost",
+            "username": "bifrost",
+            "password": "bifrost_password",
+            "protocol": "http"
+        }
+    },
+    "providers": {
+        "openai": {
+            "keys": [
+                {
+                    "name": "openai-key-1",
+                    "value": "sk-proj-abc",
+                    "weight": 1,
+                    "models": ["*"]
+                }
+            ]
+        }
+    }
+}

framework/docker-compose.yml

@@ -8,6 +8,11 @@
 #             For production, use cloud service with PINECONE_API_KEY and PINECONE_INDEX_HOST
 #             See: https://docs.pinecone.io/guides/operations/local-development
 #
+# Supported Log Stores:
+# - Postgres: shared with configstore (port 5432)
+# - ClickHouse: native protocol on host port 9001 (container 9000; host 9000 is
+#               taken by Weaviate), HTTP on 8123. Used by logstore clickhouse tests.
+#
 services:
   postgres:
     image: postgres:16-alpine
@@ -30,6 +35,31 @@ services:
     networks:
       - bifrost_network
 
+  clickhouse:
+    image: clickhouse/clickhouse-server:24.8-alpine
+    container_name: bifrost-clickhouse-fw
+    environment:
+      CLICKHOUSE_DB: bifrost
+      CLICKHOUSE_USER: bifrost
+      CLICKHOUSE_PASSWORD: bifrost_password
+    ports:
+      - "9001:9000"
+      - "8123:8123"
+    volumes:
+      - clickhouse_data:/var/lib/clickhouse
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:8123/ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    restart: unless-stopped
+    networks:
+      - bifrost_network
+
   redis:
     image: redis/redis-stack:latest
     container_name: bifrost-redis
@@ -112,6 +142,8 @@ networks:
 volumes:
   postgres_data:
     driver: local
+  clickhouse_data:
+    driver: local
   weaviate_data:
     driver: local
   redis_data:

framework/go.mod

@@ -18,6 +18,7 @@ require (
 	golang.org/x/crypto v0.52.0
 	golang.org/x/sync v0.20.0
 	google.golang.org/api v0.282.0
+	gorm.io/driver/clickhouse v0.7.0
 	gorm.io/driver/postgres v1.6.0
 	gorm.io/driver/sqlite v1.6.0
 	gorm.io/gorm v1.31.1
@@ -34,6 +35,8 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
+	github.com/ClickHouse/ch-go v0.61.5 // indirect
+	github.com/ClickHouse/clickhouse-go/v2 v2.30.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
@@ -49,6 +52,8 @@ require (
 	github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-faster/city v1.0.1 // indirect
+	github.com/go-faster/errors v0.7.1 // indirect
 	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
@@ -69,13 +74,18 @@ require (
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.16 // indirect
 	github.com/googleapis/gax-go/v2 v2.22.0 // indirect
+	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/oapi-codegen/runtime v1.1.1 // indirect
+	github.com/paulmach/orb v0.11.1 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
+	github.com/segmentio/asm v1.2.0 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect