From 599ad8a503abb9e7e574c794d6ab620c453d94bd Mon Sep 17 00:00:00 2001 From: akshaydeo Date: Sun, 28 Jun 2026 14:25:10 +0530 Subject: [PATCH] clickhouse support for log_store --- .gitignore | 68 ++ Makefile | 13 +- core/schemas/async.go | 11 +- core/schemas/bifrost.go | 7 + .../withclickhouselogstore/config.json | 34 + .../withclickhouselogstorehttp/config.json | 35 + examples/plugins/hello-world/.gitignore | 15 - framework/docker-compose.yml | 32 + framework/go.mod | 10 + framework/go.sum | 78 +++ framework/logstore/clickhouse.go | 190 ++++++ framework/logstore/clickhousemigrate.go | 289 +++++++++ framework/logstore/clickhousestore.go | 331 ++++++++++ framework/logstore/clickhousestore_test.go | 599 ++++++++++++++++++ framework/logstore/config.go | 6 + framework/logstore/dialectsql.go | 27 + framework/logstore/rdb.go | 393 ++++++------ framework/logstore/store.go | 11 +- scripts/bifrost-migration-cli/.gitignore | 1 - tests/cmd/e2eseed/go.mod | 11 + tests/cmd/e2eseed/go.sum | 11 + tests/cmd/seed/go.mod | 11 + tests/cmd/seed/go.sum | 11 + tests/e2e/clis/.gitignore | 2 - tests/e2e/clis/reports/.keep | 0 tests/semanticcache/.gitignore | 2 - transports/config.schema.json | 57 +- ui/.gitignore | 43 -- 28 files changed, 2023 insertions(+), 275 deletions(-) create mode 100644 examples/configs/withclickhouselogstore/config.json create mode 100644 examples/configs/withclickhouselogstorehttp/config.json delete mode 100644 examples/plugins/hello-world/.gitignore create mode 100644 framework/logstore/clickhouse.go create mode 100644 framework/logstore/clickhousemigrate.go create mode 100644 framework/logstore/clickhousestore.go create mode 100644 framework/logstore/clickhousestore_test.go create mode 100644 framework/logstore/dialectsql.go delete mode 100644 scripts/bifrost-migration-cli/.gitignore delete mode 100644 tests/e2e/clis/.gitignore delete mode 100644 tests/e2e/clis/reports/.keep delete mode 100644 tests/semanticcache/.gitignore delete mode 100644 ui/.gitignore diff --git a/.gitignore b/.gitignore index 63b8ebf7a0b..b14674855f4 100644 --- a/.gitignore +++ b/.gitignore @@ -187,3 +187,71 @@ tests/cmd/seedvks/seedvks # routing harness ledgers (local run journals, never committed) tests/e2e/api/routing/ledger-* + + +# Build artifacts +*/build/ +*.so +*.dll +*.dylib + +# Go build cache +*.exe +*.exe~ +*.test +*.out + +# Dependency directories +vendor/ + +reports/* +!reports/.keep + +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/node_modules +/.pnp +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/versions + +# testing +/coverage + +# build output +ui/.next/ +ui/out/ + +# production +ui/build + +# misc +.DS_Store +*.pem + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.pnpm-debug.log* + +# env files (can opt-in for committing if needed) +.env* + +# vercel +.vercel + +# typescript +*.tsbuildinfo + +# auto-generated TanStack Router route tree +ui/app/routeTree.gen.ts + + +bifrost-migration-cli + +tests/e2e/clis/reports \ No newline at end of file diff --git a/Makefile b/Makefile index 6cc602891ac..2720763bb37 100644 --- a/Makefile +++ b/Makefile @@ -823,18 +823,19 @@ test-framework: install-gotestsum ## Run framework tests @$(EXPOSE_ENV); \ $(ECHO) "$(GREEN)Running framework tests...$(NC)"; \ mkdir -p $(TEST_REPORTS_DIR); \ + rm -f $(TEST_REPORTS_DIR)/.framework-failed; \ cd framework && find . -name "*.go" -path "*/tests/*" -o -name "*_test.go" | head -1 > /dev/null && \ for dir in $$(find . -name "*_test.go" -exec dirname {} \; | sort -u); do \ pkg_name=$$(echo $$dir | sed 's|^\./||' | sed 's|/|-|g'); \ $(ECHO) "Testing $$dir..."; \ - cd $$dir && gotestsum \ + ( cd $$dir && gotestsum \ --format=$(GOTESTSUM_FORMAT) \ - --junitfile=../../$(TEST_REPORTS_DIR)/framework-$$pkg_name.xml \ - -- -v ./... && cd - > /dev/null; \ + --junitfile=$(CURDIR)/$(TEST_REPORTS_DIR)/framework-$$pkg_name.xml \ + -- -v ./... ) || touch $(CURDIR)/$(TEST_REPORTS_DIR)/.framework-failed; \ if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \ if which junit-viewer > /dev/null 2>&1; then \ $(ECHO) "$(YELLOW)Generating HTML report for $$pkg_name...$(NC)"; \ - junit-viewer --results=../$(TEST_REPORTS_DIR)/framework-$$pkg_name.xml --save=../$(TEST_REPORTS_DIR)/framework-$$pkg_name.html 2>/dev/null || true; \ + junit-viewer --results=$(CURDIR)/$(TEST_REPORTS_DIR)/framework-$$pkg_name.xml --save=$(CURDIR)/$(TEST_REPORTS_DIR)/framework-$$pkg_name.html 2>/dev/null || true; \ fi; \ fi; \ done || $(ECHO) "No framework tests found" @@ -848,6 +849,10 @@ test-framework: install-gotestsum ## Run framework tests SUMMARY_LABEL="Framework" \ SUMMARY_STRIP="framework-" \ SUMMARY_FILES="$(TEST_REPORTS_DIR)/framework-*.xml" + @if [ -f $(TEST_REPORTS_DIR)/.framework-failed ]; then \ + rm -f $(TEST_REPORTS_DIR)/.framework-failed; \ + exit 1; \ + fi # Internal: render a table of test reports + a final pass/fail scenario. # Usage: $(MAKE) print-test-summary SUMMARY_LABEL="Framework" SUMMARY_STRIP="framework-" SUMMARY_FILES="" diff --git a/core/schemas/async.go b/core/schemas/async.go index 6166b7f98f9..0d368cae2d0 100644 --- a/core/schemas/async.go +++ b/core/schemas/async.go @@ -1,10 +1,19 @@ package schemas -import "time" +import ( + "database/sql/driver" + "time" +) // AsyncJobStatus represents the status of an async job type AsyncJobStatus string +// Value implements driver.Valuer so database drivers that append typed +// column values (e.g. clickhouse-go batch inserts) can serialize the type. +func (s AsyncJobStatus) Value() (driver.Value, error) { + return string(s), nil +} + const ( AsyncJobStatusPending AsyncJobStatus = "pending" AsyncJobStatusProcessing AsyncJobStatus = "processing" diff --git a/core/schemas/bifrost.go b/core/schemas/bifrost.go index 9562a57d002..b4f8dc683cc 100644 --- a/core/schemas/bifrost.go +++ b/core/schemas/bifrost.go @@ -2,6 +2,7 @@ package schemas import ( + "database/sql/driver" "encoding/json" "errors" "fmt" @@ -117,6 +118,12 @@ var StandardProviders = []ModelProvider{ // RequestType represents the type of request being made to a provider. type RequestType string +// Value implements driver.Valuer so database drivers that append typed +// column values (e.g. clickhouse-go batch inserts) can serialize the type. +func (r RequestType) Value() (driver.Value, error) { + return string(r), nil +} + const ( ListModelsRequest RequestType = "list_models" TextCompletionRequest RequestType = "text_completion" diff --git a/examples/configs/withclickhouselogstore/config.json b/examples/configs/withclickhouselogstore/config.json new file mode 100644 index 00000000000..c9ae21385a1 --- /dev/null +++ b/examples/configs/withclickhouselogstore/config.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://www.getbifrost.ai/schema", + "config_store": { + "enabled": true, + "type": "sqlite", + "config": { + "path": "../../examples/configs/withclickhouselogstore/config.db" + } + }, + "logs_store": { + "enabled": true, + "type": "clickhouse", + "retention_days": 30, + "config": { + "host": "localhost", + "port": "9001", + "database": "bifrost", + "username": "bifrost", + "password": "bifrost_password" + } + }, + "providers": { + "openai": { + "keys": [ + { + "name": "openai-key-1", + "value": "sk-proj-abc", + "weight": 1, + "models": ["*"] + } + ] + } + } +} diff --git a/examples/configs/withclickhouselogstorehttp/config.json b/examples/configs/withclickhouselogstorehttp/config.json new file mode 100644 index 00000000000..07b6cc8db57 --- /dev/null +++ b/examples/configs/withclickhouselogstorehttp/config.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://www.getbifrost.ai/schema", + "config_store": { + "enabled": true, + "type": "sqlite", + "config": { + "path": "../../examples/configs/withclickhouselogstorehttp/config.db" + } + }, + "logs_store": { + "enabled": true, + "type": "clickhouse", + "retention_days": 30, + "config": { + "host": "localhost", + "port": "8123", + "database": "bifrost", + "username": "bifrost", + "password": "bifrost_password", + "protocol": "http" + } + }, + "providers": { + "openai": { + "keys": [ + { + "name": "openai-key-1", + "value": "sk-proj-abc", + "weight": 1, + "models": ["*"] + } + ] + } + } +} diff --git a/examples/plugins/hello-world/.gitignore b/examples/plugins/hello-world/.gitignore deleted file mode 100644 index 76de1335085..00000000000 --- a/examples/plugins/hello-world/.gitignore +++ /dev/null @@ -1,15 +0,0 @@ -# Build artifacts -build/ -*.so -*.dll -*.dylib - -# Go build cache -*.exe -*.exe~ -*.test -*.out - -# Dependency directories -vendor/ - diff --git a/framework/docker-compose.yml b/framework/docker-compose.yml index 9845e1b70fb..4585fbb97a1 100644 --- a/framework/docker-compose.yml +++ b/framework/docker-compose.yml @@ -8,6 +8,11 @@ # For production, use cloud service with PINECONE_API_KEY and PINECONE_INDEX_HOST # See: https://docs.pinecone.io/guides/operations/local-development # +# Supported Log Stores: +# - Postgres: shared with configstore (port 5432) +# - ClickHouse: native protocol on host port 9001 (container 9000; host 9000 is +# taken by Weaviate), HTTP on 8123. Used by logstore clickhouse tests. +# services: postgres: image: postgres:16-alpine @@ -30,6 +35,31 @@ services: networks: - bifrost_network + clickhouse: + image: clickhouse/clickhouse-server:24.8-alpine + container_name: bifrost-clickhouse-fw + environment: + CLICKHOUSE_DB: bifrost + CLICKHOUSE_USER: bifrost + CLICKHOUSE_PASSWORD: bifrost_password + ports: + - "9001:9000" + - "8123:8123" + volumes: + - clickhouse_data:/var/lib/clickhouse + ulimits: + nofile: + soft: 262144 + hard: 262144 + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:8123/ping"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + networks: + - bifrost_network + redis: image: redis/redis-stack:latest container_name: bifrost-redis @@ -112,6 +142,8 @@ networks: volumes: postgres_data: driver: local + clickhouse_data: + driver: local weaviate_data: driver: local redis_data: diff --git a/framework/go.mod b/framework/go.mod index e1793679965..da05679f34c 100644 --- a/framework/go.mod +++ b/framework/go.mod @@ -18,6 +18,7 @@ require ( golang.org/x/crypto v0.52.0 golang.org/x/sync v0.20.0 google.golang.org/api v0.282.0 + gorm.io/driver/clickhouse v0.7.0 gorm.io/driver/postgres v1.6.0 gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.31.1 @@ -34,6 +35,8 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect + github.com/ClickHouse/ch-go v0.61.5 // indirect + github.com/ClickHouse/clickhouse-go/v2 v2.30.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect @@ -49,6 +52,8 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -69,13 +74,18 @@ require ( github.com/google/s2a-go v0.1.9 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.16 // indirect github.com/googleapis/gax-go/v2 v2.22.0 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/oapi-codegen/runtime v1.1.1 // indirect + github.com/paulmach/orb v0.11.1 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/segmentio/asm v1.2.0 // indirect + github.com/shopspring/decimal v1.4.0 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect diff --git a/framework/go.sum b/framework/go.sum index 40c7bdd503b..f2df915956d 100644 --- a/framework/go.sum +++ b/framework/go.sum @@ -32,6 +32,10 @@ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= +github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= +github.com/ClickHouse/clickhouse-go/v2 v2.30.0 h1:AG4D/hW39qa58+JHQIFOSnxyL46H6h2lrmGGk17dhFo= +github.com/ClickHouse/clickhouse-go/v2 v2.30.0/go.mod h1:i9ZQAojcayW3RsdCb3YR+n+wC2h65eJsZCscZ1Z1wyo= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= @@ -125,6 +129,10 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= +github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -183,10 +191,15 @@ github.com/go-openapi/validate v0.25.1/go.mod h1:RMVyVFYte0gbSTaZ0N4KmTn6u/kClvA github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= @@ -201,6 +214,8 @@ github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= github.com/hajimehoshi/go-mp3 v0.3.4 h1:NUP7pBYH8OguP4diaTZ9wJbUbk3tC0KlfzsEpWmYj68= github.com/hajimehoshi/go-mp3 v0.3.4/go.mod h1:fRtZraRFcWb0pu7ok0LqyFhCUrPeMsGRSVop0eemFmo= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E= github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -218,12 +233,18 @@ github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/ github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -247,6 +268,11 @@ github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmt github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= +github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pinecone-io/go-pinecone/v5 v5.3.0 h1:0YQlEtmXGWK/I8ztkOVM6PuBYgFJZhjSdb0ddU+bHPE= github.com/pinecone-io/go-pinecone/v5 v5.3.0/go.mod h1:6Fg85fcyvMUQFf9KW7zniN81kelSYvsjF+KPLdc1MGA= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= @@ -269,6 +295,10 @@ github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= github.com/savsgio/gotils v0.0.0-20250408102913-196191ec6287 h1:qIQ0tWF9vxGtkJa24bR+2i53WBCz1nW/Pc47oVYauC4= github.com/savsgio/gotils v0.0.0-20250408102913-196191ec6287/go.mod h1:sM7Mt7uEoCeFSCBM+qBrqvEo+/9vdmj19wzp3yzUhmg= +github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= +github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= @@ -281,6 +311,7 @@ github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/ github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -293,6 +324,7 @@ github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= @@ -309,10 +341,17 @@ github.com/weaviate/weaviate-go-client/v5 v5.7.1 h1:vEMxh486QqRqWaq58UEe/TiTbGbo github.com/weaviate/weaviate-go-client/v5 v5.7.1/go.mod h1:T/JDErjN074GrnYIa0AgK1TGUGP/6A/8vqXNPlv4c6E= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.mongodb.org/mongo-driver v1.17.7 h1:a9w+U3Vt67eYzcfq3k/OAv284/uUUkL0uP75VE5rCOU= go.mongodb.org/mongo-driver v1.17.7/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= @@ -341,24 +380,58 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/arch v0.23.0 h1:lKF64A2jF6Zd8L0knGltUnegD62JMFBiCPBmQpToHhg= golang.org/x/arch v0.23.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.282.0 h1:WmJiSVqUnKqJCpJOx7YADbXaC+9DDsnGSfllFSj7R2I= @@ -371,14 +444,19 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20260523011958-0a33c5d7ca68 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20260523011958-0a33c5d7ca68/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/clickhouse v0.7.0 h1:BCrqvgONayvZRgtuA6hdya+eAW5P2QVagV3OlEp1vtA= +gorm.io/driver/clickhouse v0.7.0/go.mod h1:TmNo0wcVTsD4BBObiRnCahUgHJHjBIwuRejHwYt3JRs= gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= diff --git a/framework/logstore/clickhouse.go b/framework/logstore/clickhouse.go new file mode 100644 index 00000000000..9981ce33f25 --- /dev/null +++ b/framework/logstore/clickhouse.go @@ -0,0 +1,190 @@ +package logstore + +import ( + "context" + "fmt" + "net" + "net/url" + "strings" + "time" + + "github.com/maximhq/bifrost/core/schemas" + clickhousedriver "gorm.io/driver/clickhouse" + "gorm.io/gorm" +) + +// ClickHouseConfig represents the configuration for a ClickHouse log store. +// +// ClickHouse is an append-only columnar OLAP store. The backend uses +// ReplacingMergeTree tables with a connection-level `final = 1` setting so +// reads transparently see the latest version of each row (see clickhousestore.go +// for the mutation strategy). +type ClickHouseConfig struct { + Host *schemas.SecretVar `json:"host"` + Port *schemas.SecretVar `json:"port"` + Database *schemas.SecretVar `json:"database"` + Username *schemas.SecretVar `json:"username"` + Password *schemas.SecretVar `json:"password"` + // Protocol selects the ClickHouse wire protocol: "native" (default, port + // 9000/9440) or "http" (port 8123/8443). clickhouse-go derives the protocol + // from the DSN scheme, so this maps to clickhouse:// vs http(s)://. + Protocol string `json:"protocol,omitempty"` + // Secure enables TLS (native: secure=true; http: switches to https). + Secure bool `json:"secure,omitempty"` + // DialTimeout is the connection dial timeout in milliseconds (JSON config + // duration fields are integer milliseconds). 0 means the 10s default. + DialTimeout int `json:"dial_timeout,omitempty"` + // Cluster, when set, makes DDL run as `ON CLUSTER ` against + // ReplicatedReplacingMergeTree engines. Empty means single-node. + Cluster string `json:"cluster,omitempty"` +} + +const ( + defaultClickHouseNativePort = "9000" + defaultClickHouseNativeTLSPort = "9440" + defaultClickHouseHTTPPort = "8123" + defaultClickHouseHTTPSPort = "8443" + defaultClickHouseDatabase = "default" + defaultClickHouseDialTimeout = 10 * time.Second + clickHouseProtocolNative = "native" + clickHouseProtocolHTTP = "http" +) + +func secretValue(v *schemas.SecretVar) string { + if v == nil { + return "" + } + return v.GetValue() +} + +// buildClickHouseDSN assembles a clickhouse-go v2 DSN. The wire protocol is +// selected via the URL scheme (clickhouse:// = native, http(s):// = HTTP), and +// unknown query params (here, `final`) are passed through as ClickHouse +// settings, so every pooled connection applies FINAL automatically. +func buildClickHouseDSN(config *ClickHouseConfig) (string, error) { + host := secretValue(config.Host) + if host == "" { + return "", fmt.Errorf("clickhouse: host is required") + } + + // Resolve protocol -> URL scheme + default port. clickhouse-go requires + // scheme "https" (not "http" + secure) for HTTP-over-TLS. + var scheme, defaultPort string + switch strings.ToLower(strings.TrimSpace(config.Protocol)) { + case "", clickHouseProtocolNative: + scheme = "clickhouse" + if config.Secure { + defaultPort = defaultClickHouseNativeTLSPort + } else { + defaultPort = defaultClickHouseNativePort + } + case clickHouseProtocolHTTP: + if config.Secure { + scheme = "https" + defaultPort = defaultClickHouseHTTPSPort + } else { + scheme = "http" + defaultPort = defaultClickHouseHTTPPort + } + default: + return "", fmt.Errorf("clickhouse: unsupported protocol %q (use %q or %q)", config.Protocol, clickHouseProtocolNative, clickHouseProtocolHTTP) + } + + port := secretValue(config.Port) + if port == "" { + port = defaultPort + } + + database := secretValue(config.Database) + if database == "" { + database = defaultClickHouseDatabase + } + + dialTimeout := defaultClickHouseDialTimeout + if config.DialTimeout > 0 { + dialTimeout = time.Duration(config.DialTimeout) * time.Millisecond + } + + u := url.URL{ + Scheme: scheme, + Host: net.JoinHostPort(host, port), + Path: "/" + database, + } + if user := secretValue(config.Username); user != "" { + if pass := secretValue(config.Password); pass != "" { + u.User = url.UserPassword(user, pass) + } else { + u.User = url.User(user) + } + } + + q := url.Values{} + // Apply FINAL to every query so ReplacingMergeTree dedup is transparent to + // the reused analytics read path (see clickhousestore.go). + q.Set("final", "1") + // The GORM ClickHouse driver rewrites DELETE/UPDATE into ALTER TABLE + // mutations, which are asynchronous by default - a read right after a + // delete would still see the rows. mutations_sync=1 makes the connection + // wait until the mutation is applied on the current replica. + q.Set("mutations_sync", "1") + q.Set("dial_timeout", dialTimeout.String()) + // clickhouse-go: native TLS is requested via secure=true; the https scheme + // also requires secure=true; plain http must NOT set it. + if config.Secure { + q.Set("secure", "true") + } + u.RawQuery = q.Encode() + + return u.String(), nil +} + +// newClickHouseLogStore creates a new ClickHouse log store. retentionDays drives +// the table TTL; values < 1 leave TTL unset (the LogsCleaner still prunes via +// DeleteLogsBatch). +func newClickHouseLogStore(ctx context.Context, config *ClickHouseConfig, retentionDays int, logger schemas.Logger) (LogStore, error) { + dsn, err := buildClickHouseDSN(config) + if err != nil { + return nil, err + } + + logger.Info("logstore: opening clickhouse connection (if this step hangs, the database host/port is likely unreachable)") + db, err := gorm.Open(clickhousedriver.Open(dsn), &gorm.Config{ + Logger: newGormLogger(logger), + }) + if err != nil { + logger.Error("logstore: failed to open clickhouse connection: %v", err) + return nil, err + } + + // Release the pool on any startup failure past this point; ownership + // transfers to the returned store only on success. + constructed := false + defer func() { + if constructed { + return + } + if sqlDB, dbErr := db.DB(); dbErr == nil { + if closeErr := sqlDB.Close(); closeErr != nil { + logger.Error("logstore: failed to close clickhouse pool after startup failure: %v", closeErr) + } + } + }() + + if err := db.WithContext(ctx).Exec("SELECT 1").Error; err != nil { + logger.Error("logstore: clickhouse ping failed: %v", err) + return nil, fmt.Errorf("clickhouse ping failed: %w", err) + } + + logger.Info("logstore: running clickhouse schema migrations") + if err := triggerClickHouseMigrations(ctx, db, config.Cluster, retentionDays, logger); err != nil { + logger.Error("logstore: clickhouse schema migrations failed: %v", err) + return nil, err + } + logger.Info("logstore: clickhouse schema migrations complete") + + constructed = true + return &ClickHouseLogStore{ + RDBLogStore: &RDBLogStore{db: db, logger: logger}, + cluster: config.Cluster, + }, nil +} diff --git a/framework/logstore/clickhousemigrate.go b/framework/logstore/clickhousemigrate.go new file mode 100644 index 00000000000..f51641c3e35 --- /dev/null +++ b/framework/logstore/clickhousemigrate.go @@ -0,0 +1,289 @@ +package logstore + +import ( + "context" + "fmt" + "reflect" + "strings" + "time" + + "github.com/maximhq/bifrost/core/schemas" + "gorm.io/gorm" + "gorm.io/gorm/schema" +) + +// clickhouseColumnType maps a GORM-parsed field to a ClickHouse column type. +// Pointer fields become Nullable(...). This keeps the ClickHouse DDL in lockstep +// with the shared Log/MCPToolLog/AsyncJob structs so the reused read path (which +// references DB column names) never drifts from the physical schema. +func clickhouseColumnType(f *schema.Field) string { + ft := f.FieldType + nullable := false + for ft.Kind() == reflect.Ptr { + nullable = true + ft = ft.Elem() + } + + base := "String" + switch ft.Kind() { + case reflect.String: + base = "String" + case reflect.Bool: + base = "Bool" + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + base = "Int64" + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + base = "UInt64" + case reflect.Float32, reflect.Float64: + base = "Float64" + case reflect.Struct: + if ft == reflect.TypeFor[time.Time]() { + base = "DateTime64(3)" + } + } + + if nullable { + return "Nullable(" + base + ")" + } + return base +} + +// chColumnOverrides maps column names to full ClickHouse column definitions +// that replace the default type derived from the Go struct. Used for columns +// that need DEFAULT expressions computed by ClickHouse at INSERT time. +var chColumnOverrides = map[string]string{ + // inc_number: monotonically increasing per-row insert-order number. + // generateSnowflakeID() produces a unique UInt64 for every row in a batch + // (12-bit counter = 4096/ms), cast to Int64 for Go compatibility. + // The DEFAULT fires on fresh inserts (where the column is omitted) and is + // preserved on re-inserts (updates) because the existing value is carried + // through the read-modify-write cycle. + "inc_number": "`inc_number` Int64 DEFAULT CAST(generateSnowflakeID() AS Int64)", +} + +// clickhouseColumnDefs parses the GORM schema for model and returns column +// definitions ("`name` Type") for every persisted field, in struct order. +func clickhouseColumnDefs(db *gorm.DB, model any) ([]string, error) { + st, err := schema.Parse(model, &chSchemaCache, db.NamingStrategy) + if err != nil { + return nil, fmt.Errorf("failed to parse schema for clickhouse DDL: %w", err) + } + var cols []string + for _, f := range st.Fields { + if f.DBName == "" || f.IgnoreMigration { + continue + } + if override, ok := chColumnOverrides[f.DBName]; ok { + cols = append(cols, override) + continue + } + cols = append(cols, fmt.Sprintf("`%s` %s", f.DBName, clickhouseColumnType(f))) + } + return cols, nil +} + +// chEscapeIdentifier escapes a value for embedding inside a backtick-quoted +// ClickHouse identifier (backticks are escaped by doubling). Used for the +// config-supplied cluster name, which reaches DDL via fmt.Sprintf. +func chEscapeIdentifier(s string) string { + return strings.ReplaceAll(s, "`", "``") +} + +// chTableOpts describes the engine-level options for a ClickHouse table. +type chTableOpts struct { + table string + partitionBy string // empty = no PARTITION BY + orderBy string // e.g. "(timestamp, id)" + ttl string // empty = no TTL + skipIndexes []string // full "INDEX ..." clauses +} + +// clickhouseCreateTable derives the column list from the GORM model, appends the +// ReplacingMergeTree version column (`ver`, defaulted to now64() so every INSERT +// is auto-versioned), and runs an idempotent CREATE TABLE IF NOT EXISTS. +func clickhouseCreateTable(ctx context.Context, db *gorm.DB, model any, opts chTableOpts, cluster string) error { + cols, err := clickhouseColumnDefs(db, model) + if err != nil { + return err + } + // Version column for ReplacingMergeTree dedup. Not part of the Go struct, so + // INSERTs omit it and ClickHouse fills now64(); a later re-insert (cost + // backfill, has_object flip, idempotent retry) gets a higher ver and wins. + // Nanosecond precision: with now64()'s default millisecond resolution, a + // create + immediate update landing in the same millisecond would tie on + // `ver` and leave the winner to merge order instead of latest-write-wins. + cols = append(cols, "`ver` DateTime64(9) DEFAULT now64(9)") + cols = append(cols, opts.skipIndexes...) + + engine := "ReplacingMergeTree(ver)" + onCluster := "" + if cluster != "" { + onCluster = fmt.Sprintf(" ON CLUSTER `%s`", chEscapeIdentifier(cluster)) + engine = fmt.Sprintf("ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/%s', '{replica}', ver)", opts.table) + } + + var b strings.Builder + fmt.Fprintf(&b, "CREATE TABLE IF NOT EXISTS `%s`%s (\n %s\n) ENGINE = %s\n", opts.table, onCluster, strings.Join(cols, ",\n "), engine) + if opts.partitionBy != "" { + fmt.Fprintf(&b, "PARTITION BY %s\n", opts.partitionBy) + } + fmt.Fprintf(&b, "ORDER BY %s\n", opts.orderBy) + if opts.ttl != "" { + fmt.Fprintf(&b, "TTL %s\n", opts.ttl) + } + b.WriteString("SETTINGS index_granularity = 8192") + + return db.WithContext(ctx).Exec(b.String()).Error +} + +// clickhouseExistingColumns returns the set of column names already present on a +// ClickHouse table. +func clickhouseExistingColumns(ctx context.Context, db *gorm.DB, table string) (map[string]struct{}, error) { + var names []string + if err := db.WithContext(ctx). + Raw("SELECT name FROM system.columns WHERE database = currentDatabase() AND table = ?", table). + Scan(&names).Error; err != nil { + return nil, err + } + set := make(map[string]struct{}, len(names)) + for _, n := range names { + set[n] = struct{}{} + } + return set, nil +} + +// clickhouseReconcileColumns adds any model columns missing from the live table +// via ALTER TABLE ... ADD COLUMN IF NOT EXISTS. This is the forward-evolution +// path: the shared Log/MCPToolLog structs gain fields over time, and CREATE +// TABLE IF NOT EXISTS only ever runs once. We do this ourselves (rather than the +// gorm driver's AutoMigrate) because AutoMigrate maps the structs' Postgres/ +// SQLite tags (type:varchar(255) -> FixedString(255), etc.) incorrectly for +// ClickHouse; clickhouseColumnType maps Go kinds to clean String/Nullable types. +func clickhouseReconcileColumns(ctx context.Context, db *gorm.DB, model any, table, cluster string, logger schemas.Logger) error { + existing, err := clickhouseExistingColumns(ctx, db, table) + if err != nil { + return fmt.Errorf("clickhouse: read columns for %s: %w", table, err) + } + st, err := schema.Parse(model, &chSchemaCache, db.NamingStrategy) + if err != nil { + return fmt.Errorf("clickhouse: parse schema for %s: %w", table, err) + } + onCluster := "" + if cluster != "" { + onCluster = fmt.Sprintf(" ON CLUSTER `%s`", chEscapeIdentifier(cluster)) + } + for _, f := range st.Fields { + if f.DBName == "" || f.IgnoreMigration { + continue + } + if _, ok := existing[f.DBName]; ok { + continue + } + columnDef := fmt.Sprintf("`%s` %s", f.DBName, clickhouseColumnType(f)) + if override, ok := chColumnOverrides[f.DBName]; ok { + columnDef = override + } + stmt := fmt.Sprintf("ALTER TABLE `%s`%s ADD COLUMN IF NOT EXISTS %s", table, onCluster, columnDef) + logger.Info("[logstore] clickhouse: adding column %s.%s", table, f.DBName) + if err := db.WithContext(ctx).Exec(stmt).Error; err != nil { + return fmt.Errorf("clickhouse: add column %s.%s: %w", table, f.DBName, err) + } + } + return nil +} + +// chLogsTTL derives the logs/mcp_tool_logs TTL clause from the configured +// retention. Values < 1 leave TTL unset (the LogsCleaner still prunes via +// DeleteLogsBatch). +func chLogsTTL(retentionDays int) string { + if retentionDays < 1 { + return "" + } + return fmt.Sprintf("toDateTime(created_at) + INTERVAL %d DAY", retentionDays) +} + +// clickhouseMigrationStep is one per-table migration: create the table if +// missing, then reconcile any columns added to its model since. +type clickhouseMigrationStep func(ctx context.Context, db *gorm.DB, cluster string, retentionDays int, logger schemas.Logger) error + +// migrationClickHouseLogsTable creates the logs table and reconciles it with +// the Log struct. +func migrationClickHouseLogsTable(ctx context.Context, db *gorm.DB, cluster string, retentionDays int, logger schemas.Logger) error { + logger.Info("[logstore] clickhouse: creating table logs") + if err := clickhouseCreateTable(ctx, db, &Log{}, chTableOpts{ + table: "logs", + partitionBy: "toYYYYMM(timestamp)", + orderBy: "(timestamp, id)", + ttl: chLogsTTL(retentionDays), + skipIndexes: []string{ + "INDEX idx_logs_provider provider TYPE bloom_filter GRANULARITY 1", + "INDEX idx_logs_model model TYPE bloom_filter GRANULARITY 1", + "INDEX idx_logs_status status TYPE bloom_filter GRANULARITY 1", + "INDEX idx_logs_team_id team_id TYPE bloom_filter GRANULARITY 1", + "INDEX idx_logs_virtual_key_id virtual_key_id TYPE bloom_filter GRANULARITY 1", + "INDEX idx_logs_user_id user_id TYPE bloom_filter GRANULARITY 1", + "INDEX idx_logs_selected_key_id selected_key_id TYPE bloom_filter GRANULARITY 1", + }, + }, cluster); err != nil { + return fmt.Errorf("clickhouse: create logs table: %w", err) + } + return clickhouseReconcileColumns(ctx, db, &Log{}, "logs", cluster, logger) +} + +// migrationClickHouseMCPToolLogsTable creates the mcp_tool_logs table and +// reconciles it with the MCPToolLog struct. +func migrationClickHouseMCPToolLogsTable(ctx context.Context, db *gorm.DB, cluster string, retentionDays int, logger schemas.Logger) error { + logger.Info("[logstore] clickhouse: creating table mcp_tool_logs") + if err := clickhouseCreateTable(ctx, db, &MCPToolLog{}, chTableOpts{ + table: "mcp_tool_logs", + partitionBy: "toYYYYMM(timestamp)", + orderBy: "(timestamp, id)", + ttl: chLogsTTL(retentionDays), + skipIndexes: []string{ + "INDEX idx_mcp_logs_status status TYPE bloom_filter GRANULARITY 1", + "INDEX idx_mcp_logs_virtual_key_id virtual_key_id TYPE bloom_filter GRANULARITY 1", + "INDEX idx_mcp_logs_tool_name tool_name TYPE bloom_filter GRANULARITY 1", + }, + }, cluster); err != nil { + return fmt.Errorf("clickhouse: create mcp_tool_logs table: %w", err) + } + return clickhouseReconcileColumns(ctx, db, &MCPToolLog{}, "mcp_tool_logs", cluster, logger) +} + +// migrationClickHouseAsyncJobsTable creates the async_jobs table and reconciles +// it with the AsyncJob struct. async_jobs is a small queue: a hard 7-day TTL on +// created_at is a safety backstop (independent of the logs retention setting); +// the AsyncJobCleaner's DeleteExpired/DeleteStale handle normal (sub-hour) +// expiry. +func migrationClickHouseAsyncJobsTable(ctx context.Context, db *gorm.DB, cluster string, _ int, logger schemas.Logger) error { + logger.Info("[logstore] clickhouse: creating table async_jobs") + if err := clickhouseCreateTable(ctx, db, &AsyncJob{}, chTableOpts{ + table: "async_jobs", + orderBy: "id", + ttl: "toDateTime(created_at) + INTERVAL 7 DAY", + }, cluster); err != nil { + return fmt.Errorf("clickhouse: create async_jobs table: %w", err) + } + return clickhouseReconcileColumns(ctx, db, &AsyncJob{}, "async_jobs", cluster, logger) +} + +// clickhouseMigrationSteps lists the per-table migrations in execution order, +// mirroring logstoreMigrationSteps for the SQL stores. +var clickhouseMigrationSteps = []clickhouseMigrationStep{ + migrationClickHouseLogsTable, + migrationClickHouseMCPToolLogsTable, + migrationClickHouseAsyncJobsTable, +} + +// triggerClickHouseMigrations runs all registered ClickHouse table migrations +// in order. Analogous to triggerMigrations for Postgres/SQLite, but with no +// migration ledger or advisory lock: CREATE TABLE IF NOT EXISTS and ADD COLUMN +// IF NOT EXISTS are inherently idempotent and concurrency-safe across pods. +func triggerClickHouseMigrations(ctx context.Context, db *gorm.DB, cluster string, retentionDays int, logger schemas.Logger) error { + for _, step := range clickhouseMigrationSteps { + if err := step(ctx, db, cluster, retentionDays, logger); err != nil { + return err + } + } + return nil +} diff --git a/framework/logstore/clickhousestore.go b/framework/logstore/clickhousestore.go new file mode 100644 index 00000000000..212ee152fd1 --- /dev/null +++ b/framework/logstore/clickhousestore.go @@ -0,0 +1,331 @@ +package logstore + +import ( + "context" + "errors" + "fmt" + "hash/fnv" + "reflect" + "sort" + "sync" + + "gorm.io/gorm" + "gorm.io/gorm/schema" +) + +// ClickHouseLogStore is a LogStore backed by ClickHouse. It embeds *RDBLogStore +// to reuse the (dialect-aware) analytics/read path and overrides only the +// methods ClickHouse cannot satisfy through plain GORM: +// +// - inserts that relied on ON CONFLICT DO NOTHING (ClickHouse has no upsert; +// idempotency comes from ReplacingMergeTree dedup + the connection-level +// `final = 1` setting, so a plain INSERT is correct), +// - row updates (ClickHouse has no cheap UPDATE; we read-modify-write and +// re-insert, letting the `ver` DEFAULT now64() column make the newest +// insert win on merge - see clickhousemigrate.go). +// +// Deletes are left to the embedded methods: the GORM ClickHouse driver emits +// lightweight `DELETE ... WHERE`, and TTL is the primary retention mechanism. +type ClickHouseLogStore struct { + *RDBLogStore + // cluster is the optional ON CLUSTER name (empty = single-node). Retained + // for future cluster-aware DDL. + cluster string + // rmwLocks serializes read-modify-write cycles per row key within this + // process. Because updates re-insert the whole row, two concurrent updaters + // of the same id (e.g. object offload setting has_object while the + // completion writer sets status/cost) would otherwise both read the same + // base row and the higher `ver` would silently drop the other's patch. + // Cross-pod races are not covered, but a given request id is only mutated + // by the pod that processed it. + rmwLocks [chRMWShards]sync.Mutex +} + +// chRMWShards is the number of RMW lock shards; keys are hashed onto them. +const chRMWShards = 128 + +func chRMWShard(table, id string) int { + h := fnv.New32a() + h.Write([]byte(table)) + h.Write([]byte{0}) + h.Write([]byte(id)) + return int(h.Sum32() % chRMWShards) +} + +// lockRMW locks the shard for a single row key and returns the unlock func. +func (s *ClickHouseLogStore) lockRMW(table, id string) func() { + mu := &s.rmwLocks[chRMWShard(table, id)] + mu.Lock() + return mu.Unlock +} + +// lockRMWBatch locks the distinct shards covering a set of row keys in +// ascending shard order (so concurrent batch lockers cannot deadlock) and +// returns the unlock func. +func (s *ClickHouseLogStore) lockRMWBatch(table string, ids []string) func() { + seen := make(map[int]struct{}, len(ids)) + for _, id := range ids { + seen[chRMWShard(table, id)] = struct{}{} + } + shards := make([]int, 0, len(seen)) + for sh := range seen { + shards = append(shards, sh) + } + sort.Ints(shards) + for _, sh := range shards { + s.rmwLocks[sh].Lock() + } + return func() { + for i := len(shards) - 1; i >= 0; i-- { + s.rmwLocks[shards[i]].Unlock() + } + } +} + +// chSchemaCache is a shared GORM schema parse cache reused across RMW calls. +var chSchemaCache sync.Map + +func chParseSchema(db *gorm.DB, model interface{}) (*schema.Schema, error) { + return schema.Parse(model, &chSchemaCache, db.NamingStrategy) +} + +// chImmutableColumns are the ReplacingMergeTree dedup key columns (the tables' +// ORDER BY is `(timestamp, id)` / `id` - see clickhousemigrate.go). Updates must +// never rewrite them: a reinserted row with a different key value would be a +// new logical row instead of replacing the old one, so the helpers below skip +// them the same way a SQL UPDATE never rewrites its WHERE key. +var chImmutableColumns = map[string]struct{}{ + "id": {}, + "timestamp": {}, + "inc_number": {}, // DB-assigned monotonic insert-order number; must survive re-inserts +} + +// chApplyUpdateMap applies a column->value map onto a struct pointer using the +// GORM schema field setters (which handle pointer / typed conversions). +// Dedup key columns are skipped. +func chApplyUpdateMap(ctx context.Context, st *schema.Schema, dest reflect.Value, updates map[string]interface{}) error { + for col, val := range updates { + if _, immutable := chImmutableColumns[col]; immutable { + continue + } + f, ok := st.FieldsByDBName[col] + if !ok { + continue + } + if err := f.Set(ctx, dest, val); err != nil { + return fmt.Errorf("clickhouse: set column %s: %w", col, err) + } + } + return nil +} + +// chApplyStructUpdate overlays the non-zero fields of src onto dest, mirroring +// GORM's Updates(struct) semantics (zero-valued fields are not written). +// Dedup key columns are skipped. +func chApplyStructUpdate(ctx context.Context, st *schema.Schema, dest, src reflect.Value) error { + for _, f := range st.Fields { + if f.DBName == "" { + continue + } + if _, immutable := chImmutableColumns[f.DBName]; immutable { + continue + } + val, isZero := f.ValueOf(ctx, src) + if isZero { + continue + } + if err := f.Set(ctx, dest, val); err != nil { + return fmt.Errorf("clickhouse: set column %s: %w", f.DBName, err) + } + } + return nil +} + +// chReinsert re-inserts a (possibly patched) row with hooks skipped so the +// BeforeCreate serialization does not clobber already-serialized base columns. +// The omitted `ver` column defaults to now64(), so this insert supersedes the +// prior version on the next ReplacingMergeTree merge (and immediately under +// `final = 1` reads). +func (s *ClickHouseLogStore) chReinsert(ctx context.Context, v interface{}) error { + return s.db.WithContext(ctx).Session(&gorm.Session{SkipHooks: true}).Create(v).Error +} + +// --- Inserts (no ON CONFLICT; RMT dedup handles idempotency) --- + +// CreateIfNotExists inserts a log entry. Duplicate ids collapse on merge (and +// are hidden by `final = 1` reads), so a plain INSERT is idempotent. +func (s *ClickHouseLogStore) CreateIfNotExists(ctx context.Context, entry *Log) error { + if entry == nil { + return fmt.Errorf("log entry is nil") + } + // Omit inc_number so ClickHouse's DEFAULT generateSnowflakeID() fires. + return s.db.WithContext(ctx).Omit("inc_number").Create(entry).Error +} + +// BatchCreateIfNotExists inserts multiple log entries. See CreateIfNotExists. +func (s *ClickHouseLogStore) BatchCreateIfNotExists(ctx context.Context, entries []*Log) error { + if len(entries) == 0 { + return nil + } + // Omit inc_number so ClickHouse's DEFAULT generateSnowflakeID() fires. + return s.db.WithContext(ctx).Omit("inc_number").Create(&entries).Error +} + +// BatchCreateMCPToolLogsIfNotExists inserts multiple MCP tool log entries. See +// CreateIfNotExists. +func (s *ClickHouseLogStore) BatchCreateMCPToolLogsIfNotExists(ctx context.Context, entries []*MCPToolLog) error { + if len(entries) == 0 { + return nil + } + // Omit inc_number so ClickHouse's DEFAULT generateSnowflakeID() fires. + return s.db.WithContext(ctx).Omit("inc_number").Create(&entries).Error +} + +// --- Updates (read-modify-write + re-insert) --- + +// Update applies an update (a column->value map, or a *Log/Log whose non-zero +// fields are written) to the log row by re-inserting a patched copy. +func (s *ClickHouseLogStore) Update(ctx context.Context, id string, entry any) error { + st, err := chParseSchema(s.db, &Log{}) + if err != nil { + return err + } + defer s.lockRMW("logs", id)() + var existing Log + if err := s.db.WithContext(ctx).Where("id = ?", id).First(&existing).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return ErrNotFound + } + return err + } + dest := reflect.ValueOf(&existing).Elem() + switch v := entry.(type) { + case map[string]interface{}: + if err := chApplyUpdateMap(ctx, st, dest, v); err != nil { + return err + } + case *Log: + if v == nil { + return fmt.Errorf("clickhouse: nil *Log update") + } + if err := v.SerializeFields(); err != nil { + return err + } + if err := chApplyStructUpdate(ctx, st, dest, reflect.ValueOf(v).Elem()); err != nil { + return err + } + case Log: + if err := v.SerializeFields(); err != nil { + return err + } + if err := chApplyStructUpdate(ctx, st, dest, reflect.ValueOf(&v).Elem()); err != nil { + return err + } + default: + return fmt.Errorf("clickhouse: unsupported Update entry type %T", entry) + } + return s.chReinsert(ctx, &existing) +} + +// BulkUpdateCost backfills costs by reading each chunk of rows, patching cost, +// and re-inserting. Reading the full row is required because the re-insert must +// reproduce every column (the ReplacingMergeTree dedup key includes timestamp). +func (s *ClickHouseLogStore) BulkUpdateCost(ctx context.Context, updates map[string]float64) error { + if len(updates) == 0 { + return nil + } + ids := make([]string, 0, len(updates)) + for id := range updates { + ids = append(ids, id) + } + for start := 0; start < len(ids); start += bulkUpdateCostChunkSize { + end := start + bulkUpdateCostChunkSize + if end > len(ids) { + end = len(ids) + } + chunk := ids[start:end] + if err := func() error { + defer s.lockRMWBatch("logs", chunk)() + var rows []*Log + if err := s.db.WithContext(ctx).Where("id IN ?", chunk).Find(&rows).Error; err != nil { + return err + } + if len(rows) == 0 { + return nil + } + for _, r := range rows { + cost := updates[r.ID] + r.Cost = &cost + } + return s.chReinsert(ctx, &rows) + }(); err != nil { + return err + } + } + return nil +} + +// UpdateMCPToolLog applies an update to an MCP tool log row via read-modify-write. +func (s *ClickHouseLogStore) UpdateMCPToolLog(ctx context.Context, id string, entry any) error { + st, err := chParseSchema(s.db, &MCPToolLog{}) + if err != nil { + return err + } + defer s.lockRMW("mcp_tool_logs", id)() + var existing MCPToolLog + if err := s.db.WithContext(ctx).Where("id = ?", id).First(&existing).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return ErrNotFound + } + return err + } + dest := reflect.ValueOf(&existing).Elem() + switch v := entry.(type) { + case map[string]interface{}: + if err := chApplyUpdateMap(ctx, st, dest, v); err != nil { + return err + } + case *MCPToolLog: + if v == nil { + return fmt.Errorf("clickhouse: nil *MCPToolLog update") + } + if err := v.SerializeFields(); err != nil { + return err + } + if err := chApplyStructUpdate(ctx, st, dest, reflect.ValueOf(v).Elem()); err != nil { + return err + } + case MCPToolLog: + if err := v.SerializeFields(); err != nil { + return err + } + if err := chApplyStructUpdate(ctx, st, dest, reflect.ValueOf(&v).Elem()); err != nil { + return err + } + default: + return fmt.Errorf("clickhouse: unsupported UpdateMCPToolLog entry type %T", entry) + } + return s.chReinsert(ctx, &existing) +} + +// UpdateAsyncJob applies a column->value map to an async job row via +// read-modify-write. +func (s *ClickHouseLogStore) UpdateAsyncJob(ctx context.Context, id string, updates map[string]interface{}) error { + st, err := chParseSchema(s.db, &AsyncJob{}) + if err != nil { + return err + } + defer s.lockRMW("async_jobs", id)() + var existing AsyncJob + if err := s.db.WithContext(ctx).Where("id = ?", id).First(&existing).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return ErrNotFound + } + return err + } + dest := reflect.ValueOf(&existing).Elem() + if err := chApplyUpdateMap(ctx, st, dest, updates); err != nil { + return err + } + return s.chReinsert(ctx, &existing) +} diff --git a/framework/logstore/clickhousestore_test.go b/framework/logstore/clickhousestore_test.go new file mode 100644 index 00000000000..ad31d5b203a --- /dev/null +++ b/framework/logstore/clickhousestore_test.go @@ -0,0 +1,599 @@ +package logstore + +import ( + "context" + "fmt" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/maximhq/bifrost/core/schemas" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + gormschema "gorm.io/gorm/schema" +) + +// ClickHouse test connection matches the clickhouse service in +// framework/docker-compose.yml (native protocol on host port 9001; host 9000 +// is taken by Weaviate). +const ( + clickhouseTestHost = "localhost" + clickhouseTestPort = "9001" + clickhouseTestDatabase = "bifrost" + clickhouseTestUser = "bifrost" + clickhouseTestPassword = "bifrost_password" +) + +func clickhouseTestConfig() *ClickHouseConfig { + return &ClickHouseConfig{ + Host: schemas.NewSecretVar(clickhouseTestHost), + Port: schemas.NewSecretVar(clickhouseTestPort), + Database: schemas.NewSecretVar(clickhouseTestDatabase), + Username: schemas.NewSecretVar(clickhouseTestUser), + Password: schemas.NewSecretVar(clickhouseTestPassword), + } +} + +// trySetupClickHouseStore connects to the docker-compose ClickHouse, runs +// migrations, and truncates the log tables for a clean slate. Skips the test +// when ClickHouse is unavailable. +func trySetupClickHouseStore(t *testing.T) *ClickHouseLogStore { + t.Helper() + ctx := context.Background() + store, err := newClickHouseLogStore(ctx, clickhouseTestConfig(), 0, testLogger{}) + if err != nil { + t.Skipf("ClickHouse not available, skipping test: %v", err) + } + ch := store.(*ClickHouseLogStore) + for _, table := range []string{"logs", "mcp_tool_logs", "async_jobs"} { + require.NoError(t, ch.db.Exec("TRUNCATE TABLE "+table).Error) + } + t.Cleanup(func() { _ = ch.Close(context.Background()) }) + return ch +} + +func chTestLog(id string, ts time.Time) *Log { + return &Log{ + ID: id, + Timestamp: ts, + Object: "chat.completion", + Provider: "openai", + Model: "gpt-4o", + Status: "processing", + CreatedAt: ts, + } +} + +// chCountRows counts logical rows visible for an id; with the connection-level +// final=1 setting, ReplacingMergeTree duplicates must collapse to one. +func chCountRows(t *testing.T, db *gorm.DB, table, id string) int64 { + t.Helper() + var count int64 + require.NoError(t, db.Raw(fmt.Sprintf("SELECT count() FROM `%s` WHERE id = ?", table), id).Scan(&count).Error) + return count +} + +// --- Pure unit tests (no server required) --- + +func TestBuildClickHouseDSN(t *testing.T) { + t.Run("NativeDefaults", func(t *testing.T) { + dsn, err := buildClickHouseDSN(&ClickHouseConfig{Host: schemas.NewSecretVar("ch.local")}) + require.NoError(t, err) + assert.True(t, strings.HasPrefix(dsn, "clickhouse://ch.local:9000/default?"), dsn) + assert.Contains(t, dsn, "final=1") + assert.Contains(t, dsn, "dial_timeout=10s") + assert.NotContains(t, dsn, "secure=") + }) + + t.Run("NativeSecureUsesTLSPort", func(t *testing.T) { + dsn, err := buildClickHouseDSN(&ClickHouseConfig{Host: schemas.NewSecretVar("ch.local"), Secure: true}) + require.NoError(t, err) + assert.Contains(t, dsn, "ch.local:9440") + assert.Contains(t, dsn, "secure=true") + }) + + t.Run("HTTPProtocol", func(t *testing.T) { + dsn, err := buildClickHouseDSN(&ClickHouseConfig{Host: schemas.NewSecretVar("ch.local"), Protocol: "http"}) + require.NoError(t, err) + assert.True(t, strings.HasPrefix(dsn, "http://ch.local:8123/default?"), dsn) + }) + + t.Run("HTTPSecureUsesHTTPSScheme", func(t *testing.T) { + dsn, err := buildClickHouseDSN(&ClickHouseConfig{Host: schemas.NewSecretVar("ch.local"), Protocol: "http", Secure: true}) + require.NoError(t, err) + assert.True(t, strings.HasPrefix(dsn, "https://ch.local:8443/default?"), dsn) + assert.Contains(t, dsn, "secure=true") + }) + + t.Run("CredentialsPortAndDatabase", func(t *testing.T) { + dsn, err := buildClickHouseDSN(clickhouseTestConfig()) + require.NoError(t, err) + assert.Contains(t, dsn, "bifrost:bifrost_password@localhost:9001/bifrost") + }) + + t.Run("DialTimeoutMilliseconds", func(t *testing.T) { + dsn, err := buildClickHouseDSN(&ClickHouseConfig{Host: schemas.NewSecretVar("ch.local"), DialTimeout: 2500}) + require.NoError(t, err) + assert.Contains(t, dsn, "dial_timeout=2.5s") + }) + + t.Run("MissingHost", func(t *testing.T) { + _, err := buildClickHouseDSN(&ClickHouseConfig{}) + require.Error(t, err) + }) + + t.Run("UnsupportedProtocol", func(t *testing.T) { + _, err := buildClickHouseDSN(&ClickHouseConfig{Host: schemas.NewSecretVar("ch.local"), Protocol: "grpc"}) + require.Error(t, err) + }) +} + +func TestChEscapeIdentifier(t *testing.T) { + assert.Equal(t, "prod_cluster", chEscapeIdentifier("prod_cluster")) + assert.Equal(t, "a``b", chEscapeIdentifier("a`b")) + assert.Equal(t, "````", chEscapeIdentifier("``")) +} + +// chUnitSchemaDB returns a gorm.DB usable for schema parsing without a live +// connection (chParseSchema only needs the naming strategy). +func chUnitSchemaDB() *gorm.DB { + return &gorm.DB{Config: &gorm.Config{NamingStrategy: gormschema.NamingStrategy{}}} +} + +func TestChApplyUpdateMapSkipsDedupKeys(t *testing.T) { + ctx := context.Background() + st, err := chParseSchema(chUnitSchemaDB(), &Log{}) + require.NoError(t, err) + + ts := time.Now().UTC().Truncate(time.Millisecond) + row := *chTestLog("log-1", ts) + dest := reflect.ValueOf(&row).Elem() + + err = chApplyUpdateMap(ctx, st, dest, map[string]interface{}{ + "status": "success", + "id": "hijacked", + "timestamp": ts.Add(time.Hour), + "cost": 0.42, + }) + require.NoError(t, err) + + assert.Equal(t, "success", row.Status) + require.NotNil(t, row.Cost) + assert.Equal(t, 0.42, *row.Cost) + // Dedup key columns must survive untouched. + assert.Equal(t, "log-1", row.ID) + assert.Equal(t, ts, row.Timestamp) +} + +func TestChApplyStructUpdateSkipsDedupKeys(t *testing.T) { + ctx := context.Background() + st, err := chParseSchema(chUnitSchemaDB(), &Log{}) + require.NoError(t, err) + + ts := time.Now().UTC().Truncate(time.Millisecond) + row := *chTestLog("log-1", ts) + dest := reflect.ValueOf(&row).Elem() + + update := Log{ID: "hijacked", Timestamp: ts.Add(time.Hour), Status: "error", Model: "gpt-4o-mini"} + require.NoError(t, chApplyStructUpdate(ctx, st, dest, reflect.ValueOf(&update).Elem())) + + assert.Equal(t, "error", row.Status) + assert.Equal(t, "gpt-4o-mini", row.Model) + assert.Equal(t, "log-1", row.ID) + assert.Equal(t, ts, row.Timestamp) + // Zero-valued fields in the update struct must not clobber existing values. + assert.Equal(t, "openai", row.Provider) +} + +// --- Integration tests (require docker-compose clickhouse) --- + +func TestClickHouseCreateAndFind(t *testing.T) { + store := trySetupClickHouseStore(t) + ctx := context.Background() + ts := time.Now().UTC().Truncate(time.Millisecond) + + require.NoError(t, store.CreateIfNotExists(ctx, chTestLog("ch-create-1", ts))) + + found, err := store.FindByID(ctx, "ch-create-1") + require.NoError(t, err) + assert.Equal(t, "openai", found.Provider) + assert.Equal(t, "gpt-4o", found.Model) + assert.Equal(t, "processing", found.Status) + + present, err := store.IsLogEntryPresent(ctx, "ch-create-1") + require.NoError(t, err) + assert.True(t, present) + + _, err = store.FindByID(ctx, "does-not-exist") + assert.ErrorIs(t, err, ErrNotFound) + + hasLogs, err := store.HasLogs(ctx) + require.NoError(t, err) + assert.True(t, hasLogs) + + require.NoError(t, store.Ping(ctx)) +} + +func TestClickHouseIdempotentCreate(t *testing.T) { + store := trySetupClickHouseStore(t) + ctx := context.Background() + ts := time.Now().UTC().Truncate(time.Millisecond) + + entry := chTestLog("ch-idem-1", ts) + require.NoError(t, store.CreateIfNotExists(ctx, entry)) + require.NoError(t, store.CreateIfNotExists(ctx, chTestLog("ch-idem-1", ts))) + + // final=1 must collapse the duplicate inserts into a single logical row. + assert.Equal(t, int64(1), chCountRows(t, store.db, "logs", "ch-idem-1")) +} + +func TestClickHouseBatchCreate(t *testing.T) { + store := trySetupClickHouseStore(t) + ctx := context.Background() + ts := time.Now().UTC().Truncate(time.Millisecond) + + entries := []*Log{ + chTestLog("ch-batch-1", ts), + chTestLog("ch-batch-2", ts.Add(time.Millisecond)), + chTestLog("ch-batch-3", ts.Add(2*time.Millisecond)), + } + require.NoError(t, store.BatchCreateIfNotExists(ctx, entries)) + require.NoError(t, store.BatchCreateIfNotExists(ctx, nil)) // no-op + + for _, id := range []string{"ch-batch-1", "ch-batch-2", "ch-batch-3"} { + _, err := store.FindByID(ctx, id) + require.NoError(t, err) + } +} + +func TestClickHouseUpdateWithMap(t *testing.T) { + store := trySetupClickHouseStore(t) + ctx := context.Background() + ts := time.Now().UTC().Truncate(time.Millisecond) + + require.NoError(t, store.CreateIfNotExists(ctx, chTestLog("ch-upd-map", ts))) + require.NoError(t, store.Update(ctx, "ch-upd-map", map[string]interface{}{ + "status": "success", + "cost": 1.25, + })) + + found, err := store.FindByID(ctx, "ch-upd-map") + require.NoError(t, err) + assert.Equal(t, "success", found.Status) + require.NotNil(t, found.Cost) + assert.Equal(t, 1.25, *found.Cost) + // Untouched columns must survive the re-insert. + assert.Equal(t, "openai", found.Provider) + assert.Equal(t, "gpt-4o", found.Model) + assert.Equal(t, int64(1), chCountRows(t, store.db, "logs", "ch-upd-map")) + + assert.ErrorIs(t, store.Update(ctx, "missing-id", map[string]interface{}{"status": "success"}), ErrNotFound) +} + +func TestClickHouseUpdateWithStruct(t *testing.T) { + store := trySetupClickHouseStore(t) + ctx := context.Background() + ts := time.Now().UTC().Truncate(time.Millisecond) + + require.NoError(t, store.CreateIfNotExists(ctx, chTestLog("ch-upd-struct", ts))) + + latency := 123.5 + require.NoError(t, store.Update(ctx, "ch-upd-struct", &Log{Status: "success", Latency: &latency})) + + found, err := store.FindByID(ctx, "ch-upd-struct") + require.NoError(t, err) + assert.Equal(t, "success", found.Status) + require.NotNil(t, found.Latency) + assert.Equal(t, 123.5, *found.Latency) + assert.Equal(t, "gpt-4o", found.Model) +} + +func TestClickHouseUpdateCannotRewriteDedupKey(t *testing.T) { + store := trySetupClickHouseStore(t) + ctx := context.Background() + ts := time.Now().UTC().Truncate(time.Millisecond) + + require.NoError(t, store.CreateIfNotExists(ctx, chTestLog("ch-upd-key", ts))) + + // An update that tries to move the dedup key must not fork a second + // logical row (the table ORDER BY is (timestamp, id)). + require.NoError(t, store.Update(ctx, "ch-upd-key", map[string]interface{}{ + "timestamp": ts.Add(time.Hour), + "id": "ch-upd-key-forged", + "status": "success", + })) + + assert.Equal(t, int64(1), chCountRows(t, store.db, "logs", "ch-upd-key")) + assert.Equal(t, int64(0), chCountRows(t, store.db, "logs", "ch-upd-key-forged")) + + found, err := store.FindByID(ctx, "ch-upd-key") + require.NoError(t, err) + assert.Equal(t, "success", found.Status) + assert.Equal(t, ts.UnixMilli(), found.Timestamp.UnixMilli()) +} + +func TestClickHouseConcurrentUpdatesPreserveBothPatches(t *testing.T) { + store := trySetupClickHouseStore(t) + ctx := context.Background() + + // The object-offload path (has_object) racing the completion path + // (status/cost) is the exact lost-update scenario the per-id RMW locks + // exist for; without them one patch silently vanishes. + for i := 0; i < 10; i++ { + id := fmt.Sprintf("ch-race-%d", i) + ts := time.Now().UTC().Truncate(time.Millisecond) + require.NoError(t, store.CreateIfNotExists(ctx, chTestLog(id, ts))) + + var wg sync.WaitGroup + errs := make([]error, 2) + wg.Add(2) + go func() { + defer wg.Done() + errs[0] = store.Update(ctx, id, map[string]interface{}{"status": "success", "cost": 0.5}) + }() + go func() { + defer wg.Done() + errs[1] = store.Update(ctx, id, map[string]interface{}{"has_object": true}) + }() + wg.Wait() + require.NoError(t, errs[0]) + require.NoError(t, errs[1]) + + found, err := store.FindByID(ctx, id) + require.NoError(t, err) + assert.Equal(t, "success", found.Status, "status patch lost for %s", id) + assert.True(t, found.HasObject, "has_object patch lost for %s", id) + } +} + +func TestClickHouseBulkUpdateCost(t *testing.T) { + store := trySetupClickHouseStore(t) + ctx := context.Background() + ts := time.Now().UTC().Truncate(time.Millisecond) + + updates := map[string]float64{} + for i := 0; i < 5; i++ { + id := fmt.Sprintf("ch-cost-%d", i) + require.NoError(t, store.CreateIfNotExists(ctx, chTestLog(id, ts.Add(time.Duration(i)*time.Millisecond)))) + updates[id] = float64(i) * 0.1 + } + // Unknown ids must be ignored, not error. + updates["ch-cost-missing"] = 9.9 + + require.NoError(t, store.BulkUpdateCost(ctx, updates)) + require.NoError(t, store.BulkUpdateCost(ctx, nil)) // no-op + + for i := 0; i < 5; i++ { + id := fmt.Sprintf("ch-cost-%d", i) + found, err := store.FindByID(ctx, id) + require.NoError(t, err) + require.NotNil(t, found.Cost, "cost missing for %s", id) + assert.InDelta(t, float64(i)*0.1, *found.Cost, 1e-9) + assert.Equal(t, int64(1), chCountRows(t, store.db, "logs", id)) + } +} + +func TestClickHouseSearchAndStats(t *testing.T) { + store := trySetupClickHouseStore(t) + ctx := context.Background() + ts := time.Now().UTC().Truncate(time.Millisecond) + + for i := 0; i < 3; i++ { + entry := chTestLog(fmt.Sprintf("ch-search-%d", i), ts.Add(time.Duration(i)*time.Second)) + entry.Status = "success" + if i == 2 { + entry.Provider = "anthropic" + entry.Model = "claude-sonnet-4-5" + } + require.NoError(t, store.CreateIfNotExists(ctx, entry)) + } + + result, err := store.SearchLogs(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + assert.Len(t, result.Logs, 3) + + filtered, err := store.SearchLogs(ctx, SearchFilters{Providers: []string{"anthropic"}}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + assert.Len(t, filtered.Logs, 1) + + stats, err := store.GetStats(ctx, SearchFilters{}) + require.NoError(t, err) + assert.Equal(t, int64(3), stats.TotalRequests) + + models, err := store.GetDistinctModels(ctx, 10, "") + require.NoError(t, err) + assert.ElementsMatch(t, []string{"gpt-4o", "claude-sonnet-4-5"}, models) +} + +func TestClickHouseDeleteLogs(t *testing.T) { + store := trySetupClickHouseStore(t) + ctx := context.Background() + ts := time.Now().UTC().Truncate(time.Millisecond) + + for _, id := range []string{"ch-del-1", "ch-del-2", "ch-del-3"} { + require.NoError(t, store.CreateIfNotExists(ctx, chTestLog(id, ts))) + } + + require.NoError(t, store.DeleteLog(ctx, "ch-del-1")) + require.NoError(t, store.DeleteLogs(ctx, []string{"ch-del-2", "ch-del-3"})) + + for _, id := range []string{"ch-del-1", "ch-del-2", "ch-del-3"} { + _, err := store.FindByID(ctx, id) + assert.ErrorIs(t, err, ErrNotFound, "log %s should be deleted", id) + } +} + +func TestClickHouseDeleteLogsBatch(t *testing.T) { + store := trySetupClickHouseStore(t) + ctx := context.Background() + + old := time.Now().UTC().Add(-48 * time.Hour).Truncate(time.Millisecond) + fresh := time.Now().UTC().Truncate(time.Millisecond) + require.NoError(t, store.CreateIfNotExists(ctx, chTestLog("ch-old", old))) + require.NoError(t, store.CreateIfNotExists(ctx, chTestLog("ch-fresh", fresh))) + + _, err := store.DeleteLogsBatch(ctx, time.Now().UTC().Add(-24*time.Hour), 100) + require.NoError(t, err) + + _, err = store.FindByID(ctx, "ch-old") + assert.ErrorIs(t, err, ErrNotFound) + _, err = store.FindByID(ctx, "ch-fresh") + assert.NoError(t, err) +} + +func chTestMCPToolLog(id string, ts time.Time) *MCPToolLog { + return &MCPToolLog{ + ID: id, + Timestamp: ts, + ToolName: "search_web", + Status: "processing", + CreatedAt: ts, + } +} + +func TestClickHouseMCPToolLogs(t *testing.T) { + store := trySetupClickHouseStore(t) + ctx := context.Background() + ts := time.Now().UTC().Truncate(time.Millisecond) + + entries := []*MCPToolLog{ + chTestMCPToolLog("ch-mcp-1", ts), + chTestMCPToolLog("ch-mcp-2", ts.Add(time.Millisecond)), + } + require.NoError(t, store.BatchCreateMCPToolLogsIfNotExists(ctx, entries)) + require.NoError(t, store.BatchCreateMCPToolLogsIfNotExists(ctx, nil)) // no-op + + found, err := store.FindMCPToolLog(ctx, "ch-mcp-1") + require.NoError(t, err) + assert.Equal(t, "search_web", found.ToolName) + + // Map update. + latency := 42.0 + require.NoError(t, store.UpdateMCPToolLog(ctx, "ch-mcp-1", map[string]interface{}{ + "status": "success", + "latency": latency, + })) + found, err = store.FindMCPToolLog(ctx, "ch-mcp-1") + require.NoError(t, err) + assert.Equal(t, "success", found.Status) + require.NotNil(t, found.Latency) + assert.Equal(t, 42.0, *found.Latency) + assert.Equal(t, int64(1), chCountRows(t, store.db, "mcp_tool_logs", "ch-mcp-1")) + + // Struct update preserves untouched fields and the dedup key. + require.NoError(t, store.UpdateMCPToolLog(ctx, "ch-mcp-2", &MCPToolLog{Status: "error", Timestamp: ts.Add(time.Hour)})) + found, err = store.FindMCPToolLog(ctx, "ch-mcp-2") + require.NoError(t, err) + assert.Equal(t, "error", found.Status) + assert.Equal(t, "search_web", found.ToolName) + assert.Equal(t, ts.Add(time.Millisecond).UnixMilli(), found.Timestamp.UnixMilli()) + assert.Equal(t, int64(1), chCountRows(t, store.db, "mcp_tool_logs", "ch-mcp-2")) + + assert.ErrorIs(t, store.UpdateMCPToolLog(ctx, "missing-id", map[string]interface{}{"status": "success"}), ErrNotFound) + + hasLogs, err := store.HasMCPToolLogs(ctx) + require.NoError(t, err) + assert.True(t, hasLogs) + + result, err := store.SearchMCPToolLogs(ctx, MCPToolLogSearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + assert.Len(t, result.Logs, 2) +} + +func TestClickHouseAsyncJobs(t *testing.T) { + store := trySetupClickHouseStore(t) + ctx := context.Background() + now := time.Now().UTC().Truncate(time.Millisecond) + + job := &AsyncJob{ + ID: "ch-job-1", + Status: schemas.AsyncJobStatusProcessing, + RequestType: schemas.ChatCompletionRequest, + CreatedAt: now, + } + require.NoError(t, store.CreateAsyncJob(ctx, job)) + + found, err := store.FindAsyncJobByID(ctx, "ch-job-1") + require.NoError(t, err) + assert.Equal(t, schemas.AsyncJobStatusProcessing, found.Status) + + completedAt := now.Add(time.Second) + require.NoError(t, store.UpdateAsyncJob(ctx, "ch-job-1", map[string]interface{}{ + "status": string(schemas.AsyncJobStatusCompleted), + "response": `{"ok":true}`, + "completed_at": completedAt, + })) + found, err = store.FindAsyncJobByID(ctx, "ch-job-1") + require.NoError(t, err) + assert.Equal(t, schemas.AsyncJobStatusCompleted, found.Status) + assert.Equal(t, `{"ok":true}`, found.Response) + assert.Equal(t, int64(1), chCountRows(t, store.db, "async_jobs", "ch-job-1")) + + // Expired job cleanup. + expiredAt := now.Add(-time.Hour) + expired := &AsyncJob{ + ID: "ch-job-expired", + Status: schemas.AsyncJobStatusCompleted, + RequestType: schemas.ChatCompletionRequest, + ExpiresAt: &expiredAt, + CreatedAt: now.Add(-2 * time.Hour), + } + require.NoError(t, store.CreateAsyncJob(ctx, expired)) + _, err = store.DeleteExpiredAsyncJobs(ctx) + require.NoError(t, err) + _, err = store.FindAsyncJobByID(ctx, "ch-job-expired") + assert.Error(t, err, "expired job should be deleted") + + // Stale processing job cleanup. + stale := &AsyncJob{ + ID: "ch-job-stale", + Status: schemas.AsyncJobStatusProcessing, + RequestType: schemas.ChatCompletionRequest, + CreatedAt: now.Add(-48 * time.Hour), + } + require.NoError(t, store.CreateAsyncJob(ctx, stale)) + _, err = store.DeleteStaleAsyncJobs(ctx, now.Add(-24*time.Hour)) + require.NoError(t, err) + _, err = store.FindAsyncJobByID(ctx, "ch-job-stale") + assert.Error(t, err, "stale processing job should be deleted") + + // The completed job must survive both cleanups. + _, err = store.FindAsyncJobByID(ctx, "ch-job-1") + assert.NoError(t, err) +} + +func TestClickHouseHistograms(t *testing.T) { + store := trySetupClickHouseStore(t) + ctx := context.Background() + ts := time.Now().UTC().Truncate(time.Millisecond) + + for i := 0; i < 4; i++ { + entry := chTestLog(fmt.Sprintf("ch-hist-%d", i), ts.Add(time.Duration(i)*time.Second)) + entry.Status = "success" + cost := 0.25 + entry.Cost = &cost + entry.TotalTokens = 100 + entry.PromptTokens = 60 + entry.CompletionTokens = 40 + require.NoError(t, store.CreateIfNotExists(ctx, entry)) + } + + hist, err := store.GetHistogram(ctx, SearchFilters{}, 60) + require.NoError(t, err) + require.NotNil(t, hist) + + costHist, err := store.GetCostHistogram(ctx, SearchFilters{}, 60) + require.NoError(t, err) + require.NotNil(t, costHist) + + tokenHist, err := store.GetTokenHistogram(ctx, SearchFilters{}, 60) + require.NoError(t, err) + require.NotNil(t, tokenHist) + + modelRankings, err := store.GetModelRankings(ctx, SearchFilters{}) + require.NoError(t, err) + require.NotNil(t, modelRankings) +} diff --git a/framework/logstore/config.go b/framework/logstore/config.go index b784aa59772..458c046ff00 100644 --- a/framework/logstore/config.go +++ b/framework/logstore/config.go @@ -110,6 +110,12 @@ func (c *Config) UnmarshalJSON(data []byte) error { return fmt.Errorf("failed to unmarshal postgres config: %w", err) } c.Config = &postgresConfig + case LogStoreTypeClickHouse: + var clickhouseConfig ClickHouseConfig + if err := json.Unmarshal(temp.Config, &clickhouseConfig); err != nil { + return fmt.Errorf("failed to unmarshal clickhouse config: %w", err) + } + c.Config = &clickhouseConfig default: return fmt.Errorf("unknown log store type: %s", temp.Type) } diff --git a/framework/logstore/dialectsql.go b/framework/logstore/dialectsql.go new file mode 100644 index 00000000000..e8b610cfe20 --- /dev/null +++ b/framework/logstore/dialectsql.go @@ -0,0 +1,27 @@ +package logstore + +import "fmt" + +// unixBucketExpr returns a SQL expression that truncates the `timestamp` column +// to a bucket boundary and yields an integer unix-seconds value, per dialect. +// The returned string is a complete SQL fragment (the sqlite branch contains a +// literal strftime '%s' specifier, which is safe to pass as an argument to a +// later fmt.Sprintf since only the format string is scanned for verbs). +// +// Keeping the per-dialect bucket math in one place lets the ~17 histogram +// queries in rdb.go share a single, dialect-correct expression instead of +// branching inline (which previously routed ClickHouse into the Postgres +// EXTRACT(EPOCH ...) path - invalid ClickHouse SQL). +func unixBucketExpr(dialect string, bucketSizeSeconds int64) string { + switch dialect { + case "sqlite": + return fmt.Sprintf("(CAST(strftime('%%s', timestamp) AS INTEGER) / %d) * %d", bucketSizeSeconds, bucketSizeSeconds) + case "mysql": + return fmt.Sprintf("(FLOOR(UNIX_TIMESTAMP(timestamp) / %d) * %d)", bucketSizeSeconds, bucketSizeSeconds) + case "clickhouse": + return fmt.Sprintf("toInt64(intDiv(toUnixTimestamp(timestamp), %d) * %d)", bucketSizeSeconds, bucketSizeSeconds) + default: + // PostgreSQL (and others) + return fmt.Sprintf("CAST(FLOOR(EXTRACT(EPOCH FROM timestamp) / %d) * %d AS BIGINT)", bucketSizeSeconds, bucketSizeSeconds) + } +} diff --git a/framework/logstore/rdb.go b/framework/logstore/rdb.go index eba5806d4a4..56cdc68dd8b 100644 --- a/framework/logstore/rdb.go +++ b/framework/logstore/rdb.go @@ -290,13 +290,19 @@ func (s *RDBLogStore) applyFilters(baseQuery *gorm.DB, filters SearchFilters) *g } } if len(valid) > 0 { - if s.db.Dialector.Name() == "postgres" { + switch s.db.Dialector.Name() { + case "postgres": // Match the same loose-JSON guard used by aggregateCacheHits so the regex extract is safe. baseQuery = baseQuery.Where( "cache_debug IS NOT NULL AND cache_debug <> '' AND cache_debug ~ '^\\s*\\{.*\\}\\s*$' AND substring(cache_debug from '\"hit_type\"[[:space:]]*:[[:space:]]*\"([^\"]+)\"') IN ?", valid, ) - } else { + case "clickhouse": + baseQuery = baseQuery.Where( + "cache_debug IS NOT NULL AND cache_debug != '' AND isValidJSON(cache_debug) AND JSONExtractString(cache_debug, 'hit_type') IN ?", + valid, + ) + default: baseQuery = baseQuery.Where( "cache_debug IS NOT NULL AND cache_debug != '' AND json_valid(cache_debug) AND json_extract(cache_debug, '$.hit_type') IN ?", valid, @@ -318,9 +324,12 @@ func (s *RDBLogStore) applyFilters(baseQuery *gorm.DB, filters SearchFilters) *g dialect := s.db.Dialector.Name() // Guard must match the partial-index predicate so the planner uses the GIN index. // SQLite does not support IS JSON OBJECT, so fall back to the equivalent json_type check. - if dialect == "postgres" { + switch dialect { + case "postgres": baseQuery = baseQuery.Where("metadata IS NOT NULL AND metadata IS JSON OBJECT") - } else { + case "clickhouse": + baseQuery = baseQuery.Where("metadata IS NOT NULL AND isValidJSON(metadata)") + default: baseQuery = baseQuery.Where("metadata IS NOT NULL AND json_valid(metadata) AND json_type(metadata) = 'object'") } for key, value := range filters.MetadataFilters { @@ -334,6 +343,10 @@ func (s *RDBLogStore) applyFilters(baseQuery *gorm.DB, filters SearchFilters) *g // strings — always match as a string to avoid type mismatch with jsonb. jsonFragment := fmt.Sprintf(`{%q: %q}`, key, value) baseQuery = baseQuery.Where("metadata::jsonb @> ?::jsonb", jsonFragment) + case "clickhouse": + // Metadata values are stored as JSON strings (see postgres note); + // match them as strings via JSONExtractString. + baseQuery = baseQuery.Where("JSONExtractString(metadata, ?) = ?", key, value) default: // SQLite: quote the member name so dots/hyphens stay part of the key path := `$."` + key + `"` @@ -894,6 +907,14 @@ func (s *RDBLogStore) listSelectColumns() string { ELSE bifrost_safe_jsonb(responses_input_history) END AS responses_input_history` outputMessageExpr = `CASE WHEN object_type = 'realtime.turn' THEN output_message ELSE NULL END AS output_message` + case "clickhouse": + // ClickHouse: return the full history columns as-is. The last-message + // truncation optimization the SQLite/Postgres list path applies is + // deferred (correctness over payload size); hybrid offloading and + // content_summary already bound list payloads in practice. + inputHistoryExpr = `input_history AS input_history` + responsesInputExpr = `responses_input_history AS responses_input_history` + outputMessageExpr = `CASE WHEN object_type = 'realtime.turn' THEN output_message ELSE NULL END AS output_message` default: // sqlite inputHistoryExpr = `CASE WHEN object_type = 'realtime.turn' THEN input_history @@ -1053,7 +1074,8 @@ func (s *RDBLogStore) aggregateCacheHits(ctx context.Context, base *gorm.DB, fil SemanticHits sql.NullInt64 `gorm:"column:semantic_hits"` } q := s.applyFilters(base, filters) - if s.db.Dialector.Name() == "postgres" { + switch s.db.Dialector.Name() { + case "postgres": q = q.Where("cache_debug IS NOT NULL AND cache_debug <> '' AND cache_debug ~ '^\\s*\\{.*\\}\\s*$'") if err := q.Select( `SUM(CASE WHEN substring(cache_debug from '"hit_type"[[:space:]]*:[[:space:]]*"([^"]+)"') = 'direct' THEN 1 ELSE 0 END) AS direct_hits, ` + @@ -1061,7 +1083,15 @@ func (s *RDBLogStore) aggregateCacheHits(ctx context.Context, base *gorm.DB, fil ).Scan(&result).Error; err != nil { return nil, nil, fmt.Errorf("failed to aggregate cache-hit stats: %w", err) } - } else { + case "clickhouse": + q = q.Where("cache_debug IS NOT NULL AND cache_debug != '' AND isValidJSON(cache_debug)") + if err := q.Select( + `SUM(CASE WHEN JSONExtractString(cache_debug, 'hit_type') = 'direct' THEN 1 ELSE 0 END) AS direct_hits, ` + + `SUM(CASE WHEN JSONExtractString(cache_debug, 'hit_type') = 'semantic' THEN 1 ELSE 0 END) AS semantic_hits`, + ).Scan(&result).Error; err != nil { + return nil, nil, fmt.Errorf("failed to aggregate cache-hit stats: %w", err) + } + default: q = q.Where("cache_debug IS NOT NULL AND cache_debug != '' AND json_valid(cache_debug)") if err := q.Select( `SUM(CASE WHEN json_extract(cache_debug, '$.hit_type') = 'direct' THEN 1 ELSE 0 END) AS direct_hits, ` + @@ -1105,36 +1135,13 @@ func (s *RDBLogStore) GetHistogram(ctx context.Context, filters SearchFilters, b } // Build select clause with database-specific unix timestamp calculation - var selectClause string - switch dialect { - case "sqlite": - // SQLite: use strftime to get unix timestamp, then bucket - selectClause = fmt.Sprintf(` - (CAST(strftime('%%s', timestamp) AS INTEGER) / %d) * %d as bucket_timestamp, - COUNT(*) as total, - SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) as success, - SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as error_count, - SUM(CASE WHEN status = 'cancelled' THEN 1 ELSE 0 END) as cancelled_count - `, bucketSizeSeconds, bucketSizeSeconds) - case "mysql": - // MySQL: use UNIX_TIMESTAMP - selectClause = fmt.Sprintf(` - (FLOOR(UNIX_TIMESTAMP(timestamp) / %d) * %d) as bucket_timestamp, - COUNT(*) as total, - SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) as success, - SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as error_count, - SUM(CASE WHEN status = 'cancelled' THEN 1 ELSE 0 END) as cancelled_count - `, bucketSizeSeconds, bucketSizeSeconds) - default: - // PostgreSQL (and others): use EXTRACT(EPOCH FROM timestamp) - selectClause = fmt.Sprintf(` - CAST(FLOOR(EXTRACT(EPOCH FROM timestamp) / %d) * %d AS BIGINT) as bucket_timestamp, + selectClause := fmt.Sprintf(` + %s as bucket_timestamp, COUNT(*) as total, SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) as success, SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as error_count, - SUM(CASE WHEN status = 'cancelled' THEN 1 ELSE 0 END) as cancelled_count - `, bucketSizeSeconds, bucketSizeSeconds) - } + SUM(CASE WHEN status = 'cancelled' THEN 1 ELSE 0 END) as cancelled_count + `, unixBucketExpr(dialect, bucketSizeSeconds)) if err := baseQuery. Select(selectClause). @@ -1237,33 +1244,13 @@ func (s *RDBLogStore) GetTokenHistogram(ctx context.Context, filters SearchFilte CachedReadTokens int64 `gorm:"column:cached_read_tokens"` } - var selectClause string - switch dialect { - case "sqlite": - selectClause = fmt.Sprintf(` - (CAST(strftime('%%s', timestamp) AS INTEGER) / %d) * %d as bucket_timestamp, - COALESCE(SUM(prompt_tokens), 0) as prompt_tokens, - COALESCE(SUM(completion_tokens), 0) as completion_tokens, - COALESCE(SUM(total_tokens), 0) as total_tokens, - COALESCE(SUM(cached_read_tokens), 0) as cached_read_tokens - `, bucketSizeSeconds, bucketSizeSeconds) - case "mysql": - selectClause = fmt.Sprintf(` - (FLOOR(UNIX_TIMESTAMP(timestamp) / %d) * %d) as bucket_timestamp, - COALESCE(SUM(prompt_tokens), 0) as prompt_tokens, - COALESCE(SUM(completion_tokens), 0) as completion_tokens, - COALESCE(SUM(total_tokens), 0) as total_tokens, - COALESCE(SUM(cached_read_tokens), 0) as cached_read_tokens - `, bucketSizeSeconds, bucketSizeSeconds) - default: - selectClause = fmt.Sprintf(` - CAST(FLOOR(EXTRACT(EPOCH FROM timestamp) / %d) * %d AS BIGINT) as bucket_timestamp, + selectClause := fmt.Sprintf(` + %s as bucket_timestamp, COALESCE(SUM(prompt_tokens), 0) as prompt_tokens, COALESCE(SUM(completion_tokens), 0) as completion_tokens, COALESCE(SUM(total_tokens), 0) as total_tokens, COALESCE(SUM(cached_read_tokens), 0) as cached_read_tokens - `, bucketSizeSeconds, bucketSizeSeconds) - } + `, unixBucketExpr(dialect, bucketSizeSeconds)) if err := baseQuery. Select(selectClause). @@ -1363,27 +1350,11 @@ func (s *RDBLogStore) GetCostHistogram(ctx context.Context, filters SearchFilter TotalCost float64 `gorm:"column:total_cost"` } - var selectClause string - switch dialect { - case "sqlite": - selectClause = fmt.Sprintf(` - (CAST(strftime('%%s', timestamp) AS INTEGER) / %d) * %d as bucket_timestamp, - model, - COALESCE(SUM(cost), 0) as total_cost - `, bucketSizeSeconds, bucketSizeSeconds) - case "mysql": - selectClause = fmt.Sprintf(` - (FLOOR(UNIX_TIMESTAMP(timestamp) / %d) * %d) as bucket_timestamp, - model, - COALESCE(SUM(cost), 0) as total_cost - `, bucketSizeSeconds, bucketSizeSeconds) - default: - selectClause = fmt.Sprintf(` - CAST(FLOOR(EXTRACT(EPOCH FROM timestamp) / %d) * %d AS BIGINT) as bucket_timestamp, + selectClause := fmt.Sprintf(` + %s as bucket_timestamp, model, COALESCE(SUM(cost), 0) as total_cost - `, bucketSizeSeconds, bucketSizeSeconds) - } + `, unixBucketExpr(dialect, bucketSizeSeconds)) if err := baseQuery. Select(selectClause). @@ -1486,36 +1457,14 @@ func (s *RDBLogStore) GetModelHistogram(ctx context.Context, filters SearchFilte Cancelled int64 `gorm:"column:cancelled_count"` } - var selectClause string - switch dialect { - case "sqlite": - selectClause = fmt.Sprintf(` - (CAST(strftime('%%s', timestamp) AS INTEGER) / %d) * %d as bucket_timestamp, - model, - COUNT(*) as total, - SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) as success, - SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as error_count, - SUM(CASE WHEN status = 'cancelled' THEN 1 ELSE 0 END) as cancelled_count - `, bucketSizeSeconds, bucketSizeSeconds) - case "mysql": - selectClause = fmt.Sprintf(` - (FLOOR(UNIX_TIMESTAMP(timestamp) / %d) * %d) as bucket_timestamp, - model, - COUNT(*) as total, - SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) as success, - SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as error_count, - SUM(CASE WHEN status = 'cancelled' THEN 1 ELSE 0 END) as cancelled_count - `, bucketSizeSeconds, bucketSizeSeconds) - default: - selectClause = fmt.Sprintf(` - CAST(FLOOR(EXTRACT(EPOCH FROM timestamp) / %d) * %d AS BIGINT) as bucket_timestamp, + selectClause := fmt.Sprintf(` + %s as bucket_timestamp, model, COUNT(*) as total, SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) as success, SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as error_count, - SUM(CASE WHEN status = 'cancelled' THEN 1 ELSE 0 END) as cancelled_count - `, bucketSizeSeconds, bucketSizeSeconds) - } + SUM(CASE WHEN status = 'cancelled' THEN 1 ELSE 0 END) as cancelled_count + `, unixBucketExpr(dialect, bucketSizeSeconds)) if err := baseQuery. Select(selectClause). @@ -1643,11 +1592,60 @@ func (s *RDBLogStore) GetLatencyHistogram(ctx context.Context, filters SearchFil return s.getLatencyHistogramSQLite(ctx, baseQuery, filters, bucketSizeSeconds) case "mysql": return s.getLatencyHistogramMySQL(ctx, baseQuery, filters, bucketSizeSeconds) + case "clickhouse": + return s.getLatencyHistogramClickHouse(ctx, baseQuery, filters, bucketSizeSeconds) default: return s.getLatencyHistogramPercentileCont(ctx, baseQuery, filters, bucketSizeSeconds) } } +// getLatencyHistogramClickHouse computes latency percentiles with ClickHouse's +// quantile() aggregate (ClickHouse does not support percentile_cont ... WITHIN +// GROUP). Shape mirrors getLatencyHistogramPercentileCont. +func (s *RDBLogStore) getLatencyHistogramClickHouse(ctx context.Context, baseQuery *gorm.DB, filters SearchFilters, bucketSizeSeconds int64) (*LatencyHistogramResult, error) { + var results []struct { + BucketTimestamp int64 `gorm:"column:bucket_timestamp"` + AvgLatency sql.NullFloat64 `gorm:"column:avg_latency"` + P90Latency sql.NullFloat64 `gorm:"column:p90_latency"` + P95Latency sql.NullFloat64 `gorm:"column:p95_latency"` + P99Latency sql.NullFloat64 `gorm:"column:p99_latency"` + TotalRequests int64 `gorm:"column:total_requests"` + } + + selectClause := fmt.Sprintf(` + %s as bucket_timestamp, + AVG(latency) as avg_latency, + quantile(0.90)(latency) as p90_latency, + quantile(0.95)(latency) as p95_latency, + quantile(0.99)(latency) as p99_latency, + COUNT(*) as total_requests + `, unixBucketExpr("clickhouse", bucketSizeSeconds)) + + if err := baseQuery. + Select(selectClause). + Group("bucket_timestamp"). + Order("bucket_timestamp ASC"). + Find(&results).Error; err != nil { + return nil, fmt.Errorf("failed to get latency histogram: %w", err) + } + + computedBuckets := make(map[int64]LatencyHistogramBucket, len(results)) + var orderedKeys []int64 + for _, r := range results { + orderedKeys = append(orderedKeys, r.BucketTimestamp) + computedBuckets[r.BucketTimestamp] = LatencyHistogramBucket{ + Timestamp: time.Unix(r.BucketTimestamp, 0).UTC(), + AvgLatency: r.AvgLatency.Float64, + P90Latency: r.P90Latency.Float64, + P95Latency: r.P95Latency.Float64, + P99Latency: r.P99Latency.Float64, + TotalRequests: r.TotalRequests, + } + } + + return s.buildLatencyHistogramResult(computedBuckets, orderedKeys, filters, bucketSizeSeconds) +} + // getLatencyHistogramPercentileCont uses database-level percentile_cont for PostgreSQL. // Returns 1 aggregated row per bucket instead of loading all individual latency values. func (s *RDBLogStore) getLatencyHistogramPercentileCont(ctx context.Context, baseQuery *gorm.DB, filters SearchFilters, bucketSizeSeconds int64) (*LatencyHistogramResult, error) { @@ -2307,27 +2305,11 @@ func (s *RDBLogStore) GetProviderCostHistogram(ctx context.Context, filters Sear TotalCost float64 `gorm:"column:total_cost"` } - var selectClause string - switch dialect { - case "sqlite": - selectClause = fmt.Sprintf(` - (CAST(strftime('%%s', timestamp) AS INTEGER) / %d) * %d as bucket_timestamp, - provider, - COALESCE(SUM(cost), 0) as total_cost - `, bucketSizeSeconds, bucketSizeSeconds) - case "mysql": - selectClause = fmt.Sprintf(` - (FLOOR(UNIX_TIMESTAMP(timestamp) / %d) * %d) as bucket_timestamp, - provider, - COALESCE(SUM(cost), 0) as total_cost - `, bucketSizeSeconds, bucketSizeSeconds) - default: - selectClause = fmt.Sprintf(` - CAST(FLOOR(EXTRACT(EPOCH FROM timestamp) / %d) * %d AS BIGINT) as bucket_timestamp, + selectClause := fmt.Sprintf(` + %s as bucket_timestamp, provider, COALESCE(SUM(cost), 0) as total_cost - `, bucketSizeSeconds, bucketSizeSeconds) - } + `, unixBucketExpr(dialect, bucketSizeSeconds)) if err := baseQuery. Select(selectClause). @@ -2419,33 +2401,13 @@ func (s *RDBLogStore) GetProviderTokenHistogram(ctx context.Context, filters Sea TotalTokens int64 `gorm:"column:total_tokens"` } - var selectClause string - switch dialect { - case "sqlite": - selectClause = fmt.Sprintf(` - (CAST(strftime('%%s', timestamp) AS INTEGER) / %d) * %d as bucket_timestamp, - provider, - COALESCE(SUM(prompt_tokens), 0) as prompt_tokens, - COALESCE(SUM(completion_tokens), 0) as completion_tokens, - COALESCE(SUM(total_tokens), 0) as total_tokens - `, bucketSizeSeconds, bucketSizeSeconds) - case "mysql": - selectClause = fmt.Sprintf(` - (FLOOR(UNIX_TIMESTAMP(timestamp) / %d) * %d) as bucket_timestamp, - provider, - COALESCE(SUM(prompt_tokens), 0) as prompt_tokens, - COALESCE(SUM(completion_tokens), 0) as completion_tokens, - COALESCE(SUM(total_tokens), 0) as total_tokens - `, bucketSizeSeconds, bucketSizeSeconds) - default: - selectClause = fmt.Sprintf(` - CAST(FLOOR(EXTRACT(EPOCH FROM timestamp) / %d) * %d AS BIGINT) as bucket_timestamp, + selectClause := fmt.Sprintf(` + %s as bucket_timestamp, provider, COALESCE(SUM(prompt_tokens), 0) as prompt_tokens, COALESCE(SUM(completion_tokens), 0) as completion_tokens, COALESCE(SUM(total_tokens), 0) as total_tokens - `, bucketSizeSeconds, bucketSizeSeconds) - } + `, unixBucketExpr(dialect, bucketSizeSeconds)) if err := baseQuery. Select(selectClause). @@ -2544,11 +2506,81 @@ func (s *RDBLogStore) GetProviderLatencyHistogram(ctx context.Context, filters S return s.getProviderLatencyHistogramSQLite(ctx, baseQuery, filters, bucketSizeSeconds) case "mysql": return s.getProviderLatencyHistogramMySQL(ctx, baseQuery, filters, bucketSizeSeconds) + case "clickhouse": + return s.getProviderLatencyHistogramClickHouse(ctx, baseQuery, filters, bucketSizeSeconds) default: return s.getProviderLatencyHistogramPercentileCont(ctx, baseQuery, filters, bucketSizeSeconds) } } +// getProviderLatencyHistogramClickHouse computes per-provider latency +// percentiles with ClickHouse's quantile() aggregate. Shape mirrors +// getProviderLatencyHistogramPercentileCont. +func (s *RDBLogStore) getProviderLatencyHistogramClickHouse(ctx context.Context, baseQuery *gorm.DB, filters SearchFilters, bucketSizeSeconds int64) (*ProviderLatencyHistogramResult, error) { + var results []struct { + BucketTimestamp int64 `gorm:"column:bucket_timestamp"` + Provider string `gorm:"column:provider"` + AvgLatency sql.NullFloat64 `gorm:"column:avg_latency"` + P90Latency sql.NullFloat64 `gorm:"column:p90_latency"` + P95Latency sql.NullFloat64 `gorm:"column:p95_latency"` + P99Latency sql.NullFloat64 `gorm:"column:p99_latency"` + TotalRequests int64 `gorm:"column:total_requests"` + } + + selectClause := fmt.Sprintf(` + %s as bucket_timestamp, + provider, + AVG(latency) as avg_latency, + quantile(0.90)(latency) as p90_latency, + quantile(0.95)(latency) as p95_latency, + quantile(0.99)(latency) as p99_latency, + COUNT(*) as total_requests + `, unixBucketExpr("clickhouse", bucketSizeSeconds)) + + if err := baseQuery. + Select(selectClause). + Group("bucket_timestamp, provider"). + Order("bucket_timestamp ASC, provider ASC"). + Find(&results).Error; err != nil { + return nil, fmt.Errorf("failed to get provider latency histogram: %w", err) + } + + providersSet := make(map[string]bool) + computedBuckets := make(map[int64]*ProviderLatencyHistogramBucket) + var orderedBuckets []int64 + seenBuckets := make(map[int64]bool) + + for _, r := range results { + providersSet[r.Provider] = true + if !seenBuckets[r.BucketTimestamp] { + seenBuckets[r.BucketTimestamp] = true + orderedBuckets = append(orderedBuckets, r.BucketTimestamp) + } + stats := ProviderLatencyStats{ + AvgLatency: r.AvgLatency.Float64, + P90Latency: r.P90Latency.Float64, + P95Latency: r.P95Latency.Float64, + P99Latency: r.P99Latency.Float64, + TotalRequests: r.TotalRequests, + } + if bucket, exists := computedBuckets[r.BucketTimestamp]; exists { + bucket.ByProvider[r.Provider] = stats + } else { + computedBuckets[r.BucketTimestamp] = &ProviderLatencyHistogramBucket{ + Timestamp: time.Unix(r.BucketTimestamp, 0).UTC(), + ByProvider: map[string]ProviderLatencyStats{r.Provider: stats}, + } + } + } + + providers := make([]string, 0, len(providersSet)) + for provider := range providersSet { + providers = append(providers, provider) + } + + return s.buildProviderLatencyHistogramResult(computedBuckets, orderedBuckets, providers, filters, bucketSizeSeconds) +} + // getProviderLatencyHistogramPercentileCont uses database-level percentile_cont for PostgreSQL. // Returns 1 aggregated row per (bucket, provider) instead of loading all individual latency values. func (s *RDBLogStore) getProviderLatencyHistogramPercentileCont(ctx context.Context, baseQuery *gorm.DB, filters SearchFilters, bucketSizeSeconds int64) (*ProviderLatencyHistogramResult, error) { @@ -2835,13 +2867,7 @@ func (s *RDBLogStore) GetDimensionCostHistogram(ctx context.Context, filters Sea baseQuery = baseQuery.Where("status IN ?", terminalLogStatuses) baseQuery = baseQuery.Where("cost IS NOT NULL AND cost > 0") - var bucketExpr string - switch dialect { - case "sqlite": - bucketExpr = fmt.Sprintf("CAST((CAST(strftime('%%s', timestamp) AS INTEGER) / %d) * %d AS INTEGER)", bucketSizeSeconds, bucketSizeSeconds) - default: - bucketExpr = fmt.Sprintf("CAST(FLOOR(EXTRACT(EPOCH FROM timestamp) / %d) * %d AS BIGINT)", bucketSizeSeconds, bucketSizeSeconds) - } + bucketExpr := unixBucketExpr(dialect, bucketSizeSeconds) var results []struct { BucketTimestamp int64 `gorm:"column:bucket_timestamp"` @@ -2943,13 +2969,7 @@ func (s *RDBLogStore) GetDimensionTokenHistogram(ctx context.Context, filters Se baseQuery = s.applyFilters(baseQuery, filters) baseQuery = baseQuery.Where("status IN ?", terminalLogStatuses) - var bucketExpr string - switch dialect { - case "sqlite": - bucketExpr = fmt.Sprintf("CAST((CAST(strftime('%%s', timestamp) AS INTEGER) / %d) * %d AS INTEGER)", bucketSizeSeconds, bucketSizeSeconds) - default: - bucketExpr = fmt.Sprintf("CAST(FLOOR(EXTRACT(EPOCH FROM timestamp) / %d) * %d AS BIGINT)", bucketSizeSeconds, bucketSizeSeconds) - } + bucketExpr := unixBucketExpr(dialect, bucketSizeSeconds) var results []struct { BucketTimestamp int64 `gorm:"column:bucket_timestamp"` @@ -3060,13 +3080,7 @@ func (s *RDBLogStore) GetDimensionLatencyHistogram(ctx context.Context, filters baseQuery = baseQuery.Where("status IN ?", terminalLogStatuses) baseQuery = baseQuery.Where("latency IS NOT NULL") - var bucketExpr string - switch dialect { - case "sqlite": - bucketExpr = fmt.Sprintf("CAST((CAST(strftime('%%s', timestamp) AS INTEGER) / %d) * %d AS INTEGER)", bucketSizeSeconds, bucketSizeSeconds) - default: - bucketExpr = fmt.Sprintf("CAST(FLOOR(EXTRACT(EPOCH FROM timestamp) / %d) * %d AS BIGINT)", bucketSizeSeconds, bucketSizeSeconds) - } + bucketExpr := unixBucketExpr(dialect, bucketSizeSeconds) var results []struct { BucketTimestamp int64 `gorm:"column:bucket_timestamp"` @@ -3382,9 +3396,12 @@ func (s *RDBLogStore) GetDistinctMetadataKeys(ctx context.Context, limit int, qu var metadataStrings []string // Guard must match the partial-index predicate so the planner uses the GIN index. var metadataGuard string - if s.db.Dialector.Name() == "postgres" { + switch s.db.Dialector.Name() { + case "postgres": metadataGuard = "metadata IS NOT NULL AND metadata IS JSON OBJECT AND metadata != '{}' AND timestamp >= ?" - } else { + case "clickhouse": + metadataGuard = "metadata IS NOT NULL AND isValidJSON(metadata) AND metadata != '{}' AND timestamp >= ?" + default: metadataGuard = "metadata IS NOT NULL AND json_valid(metadata) AND json_type(metadata) = 'object' AND metadata != '{}' AND timestamp >= ?" } err := s.ScopedDB(ctx).Model(&Log{}). @@ -3934,30 +3951,12 @@ func (s *RDBLogStore) GetMCPHistogram(ctx context.Context, filters MCPToolLogSea Error int64 `gorm:"column:error"` } - var selectClause string - switch dialect { - case "sqlite": - selectClause = fmt.Sprintf(` - (CAST(strftime('%%s', timestamp) AS INTEGER) / %d) * %d as bucket_timestamp, - COUNT(*) as count, - SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) as success, - SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as error - `, bucketSizeSeconds, bucketSizeSeconds) - case "mysql": - selectClause = fmt.Sprintf(` - (FLOOR(UNIX_TIMESTAMP(timestamp) / %d) * %d) as bucket_timestamp, - COUNT(*) as count, - SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) as success, - SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as error - `, bucketSizeSeconds, bucketSizeSeconds) - default: - selectClause = fmt.Sprintf(` - CAST(FLOOR(EXTRACT(EPOCH FROM timestamp) / %d) * %d AS BIGINT) as bucket_timestamp, + selectClause := fmt.Sprintf(` + %s as bucket_timestamp, COUNT(*) as count, SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) as success, SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as error - `, bucketSizeSeconds, bucketSizeSeconds) - } + `, unixBucketExpr(dialect, bucketSizeSeconds)) if err := baseQuery. Select(selectClause). @@ -4029,24 +4028,10 @@ func (s *RDBLogStore) GetMCPCostHistogram(ctx context.Context, filters MCPToolLo TotalCost float64 `gorm:"column:total_cost"` } - var selectClause string - switch dialect { - case "sqlite": - selectClause = fmt.Sprintf(` - (CAST(strftime('%%s', timestamp) AS INTEGER) / %d) * %d as bucket_timestamp, - COALESCE(SUM(cost), 0) as total_cost - `, bucketSizeSeconds, bucketSizeSeconds) - case "mysql": - selectClause = fmt.Sprintf(` - (FLOOR(UNIX_TIMESTAMP(timestamp) / %d) * %d) as bucket_timestamp, - COALESCE(SUM(cost), 0) as total_cost - `, bucketSizeSeconds, bucketSizeSeconds) - default: - selectClause = fmt.Sprintf(` - CAST(FLOOR(EXTRACT(EPOCH FROM timestamp) / %d) * %d AS BIGINT) as bucket_timestamp, + selectClause := fmt.Sprintf(` + %s as bucket_timestamp, COALESCE(SUM(cost), 0) as total_cost - `, bucketSizeSeconds, bucketSizeSeconds) - } + `, unixBucketExpr(dialect, bucketSizeSeconds)) if err := baseQuery. Select(selectClause). diff --git a/framework/logstore/store.go b/framework/logstore/store.go index 56e8be8b3d4..3c59d580304 100644 --- a/framework/logstore/store.go +++ b/framework/logstore/store.go @@ -14,8 +14,9 @@ type LogStoreType string // LogStoreTypeSQLite is the type of log store for SQLite. const ( - LogStoreTypeSQLite LogStoreType = "sqlite" - LogStoreTypePostgres LogStoreType = "postgres" + LogStoreTypeSQLite LogStoreType = "sqlite" + LogStoreTypePostgres LogStoreType = "postgres" + LogStoreTypeClickHouse LogStoreType = "clickhouse" ) // LogStore is the interface for the log store. @@ -122,6 +123,12 @@ func NewLogStore(ctx context.Context, config *Config, logger schemas.Logger) (Lo } else { return nil, fmt.Errorf("invalid postgres config: %T", config.Config) } + case LogStoreTypeClickHouse: + if clickhouseConfig, ok := config.Config.(*ClickHouseConfig); ok { + inner, err = newClickHouseLogStore(ctx, clickhouseConfig, config.RetentionDays, logger) + } else { + return nil, fmt.Errorf("invalid clickhouse config: %T", config.Config) + } default: return nil, fmt.Errorf("unsupported log store type: %s", config.Type) } diff --git a/scripts/bifrost-migration-cli/.gitignore b/scripts/bifrost-migration-cli/.gitignore deleted file mode 100644 index 9f10d22fcc2..00000000000 --- a/scripts/bifrost-migration-cli/.gitignore +++ /dev/null @@ -1 +0,0 @@ -bifrost-migration-cli diff --git a/tests/cmd/e2eseed/go.mod b/tests/cmd/e2eseed/go.mod index 0e18c5395eb..42fb6b5114d 100644 --- a/tests/cmd/e2eseed/go.mod +++ b/tests/cmd/e2eseed/go.mod @@ -23,6 +23,8 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect + github.com/ClickHouse/ch-go v0.61.5 // indirect + github.com/ClickHouse/clickhouse-go/v2 v2.30.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect @@ -57,6 +59,8 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -65,6 +69,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.16 // indirect github.com/googleapis/gax-go/v2 v2.22.0 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect github.com/invopop/jsonschema v0.13.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect @@ -82,9 +87,14 @@ require ( github.com/mattn/go-sqlite3 v1.14.32 // indirect github.com/maximhq/bifrost/core v1.6.2 // indirect github.com/maximhq/bifrost/framework v1.3.16 // indirect + github.com/paulmach/orb v0.11.1 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/rs/zerolog v1.34.0 // indirect + github.com/segmentio/asm v1.2.0 // indirect + github.com/shopspring/decimal v1.4.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect @@ -121,6 +131,7 @@ require ( google.golang.org/grpc v1.81.1 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + gorm.io/driver/clickhouse v0.7.0 // indirect gorm.io/driver/postgres v1.6.0 // indirect gorm.io/driver/sqlite v1.6.0 // indirect gorm.io/gorm v1.31.1 // indirect diff --git a/tests/cmd/e2eseed/go.sum b/tests/cmd/e2eseed/go.sum index 4030f84974b..1c0a3064924 100644 --- a/tests/cmd/e2eseed/go.sum +++ b/tests/cmd/e2eseed/go.sum @@ -32,6 +32,8 @@ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= +github.com/ClickHouse/clickhouse-go/v2 v2.30.0 h1:AG4D/hW39qa58+JHQIFOSnxyL46H6h2lrmGGk17dhFo= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= @@ -115,6 +117,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -141,6 +145,7 @@ github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= github.com/hajimehoshi/go-mp3 v0.3.4 h1:NUP7pBYH8OguP4diaTZ9wJbUbk3tC0KlfzsEpWmYj68= github.com/hajimehoshi/go-mp3 v0.3.4/go.mod h1:fRtZraRFcWb0pu7ok0LqyFhCUrPeMsGRSVop0eemFmo= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E= github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -180,8 +185,11 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= @@ -195,6 +203,8 @@ github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= github.com/savsgio/gotils v0.0.0-20250408102913-196191ec6287 h1:qIQ0tWF9vxGtkJa24bR+2i53WBCz1nW/Pc47oVYauC4= github.com/savsgio/gotils v0.0.0-20250408102913-196191ec6287/go.mod h1:sM7Mt7uEoCeFSCBM+qBrqvEo+/9vdmj19wzp3yzUhmg= +github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= @@ -294,6 +304,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/clickhouse v0.7.0 h1:BCrqvgONayvZRgtuA6hdya+eAW5P2QVagV3OlEp1vtA= gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= diff --git a/tests/cmd/seed/go.mod b/tests/cmd/seed/go.mod index b2bfb75731c..e2ba4983a1a 100644 --- a/tests/cmd/seed/go.mod +++ b/tests/cmd/seed/go.mod @@ -28,6 +28,8 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect + github.com/ClickHouse/ch-go v0.61.5 // indirect + github.com/ClickHouse/clickhouse-go/v2 v2.30.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect @@ -62,6 +64,8 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -70,6 +74,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.16 // indirect github.com/googleapis/gax-go/v2 v2.22.0 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect github.com/invopop/jsonschema v0.13.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect @@ -85,9 +90,14 @@ require ( github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-sqlite3 v1.14.32 // indirect + github.com/paulmach/orb v0.11.1 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/rs/zerolog v1.34.0 // indirect + github.com/segmentio/asm v1.2.0 // indirect + github.com/shopspring/decimal v1.4.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect @@ -124,4 +134,5 @@ require ( google.golang.org/grpc v1.81.1 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + gorm.io/driver/clickhouse v0.7.0 // indirect ) diff --git a/tests/cmd/seed/go.sum b/tests/cmd/seed/go.sum index 4030f84974b..1c0a3064924 100644 --- a/tests/cmd/seed/go.sum +++ b/tests/cmd/seed/go.sum @@ -32,6 +32,8 @@ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= +github.com/ClickHouse/clickhouse-go/v2 v2.30.0 h1:AG4D/hW39qa58+JHQIFOSnxyL46H6h2lrmGGk17dhFo= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= @@ -115,6 +117,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -141,6 +145,7 @@ github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= github.com/hajimehoshi/go-mp3 v0.3.4 h1:NUP7pBYH8OguP4diaTZ9wJbUbk3tC0KlfzsEpWmYj68= github.com/hajimehoshi/go-mp3 v0.3.4/go.mod h1:fRtZraRFcWb0pu7ok0LqyFhCUrPeMsGRSVop0eemFmo= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E= github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -180,8 +185,11 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= @@ -195,6 +203,8 @@ github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= github.com/savsgio/gotils v0.0.0-20250408102913-196191ec6287 h1:qIQ0tWF9vxGtkJa24bR+2i53WBCz1nW/Pc47oVYauC4= github.com/savsgio/gotils v0.0.0-20250408102913-196191ec6287/go.mod h1:sM7Mt7uEoCeFSCBM+qBrqvEo+/9vdmj19wzp3yzUhmg= +github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= @@ -294,6 +304,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/clickhouse v0.7.0 h1:BCrqvgONayvZRgtuA6hdya+eAW5P2QVagV3OlEp1vtA= gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= diff --git a/tests/e2e/clis/.gitignore b/tests/e2e/clis/.gitignore deleted file mode 100644 index 573b3a8b4ab..00000000000 --- a/tests/e2e/clis/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -reports/* -!reports/.keep diff --git a/tests/e2e/clis/reports/.keep b/tests/e2e/clis/reports/.keep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/semanticcache/.gitignore b/tests/semanticcache/.gitignore deleted file mode 100644 index 6c7f6431d4b..00000000000 --- a/tests/semanticcache/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -reports/ -*.log diff --git a/transports/config.schema.json b/transports/config.schema.json index fb30e5f48e0..50c8f8a766e 100644 --- a/transports/config.schema.json +++ b/transports/config.schema.json @@ -1260,7 +1260,7 @@ }, "type": { "type": "string", - "enum": ["sqlite", "postgres"], + "enum": ["sqlite", "postgres", "clickhouse"], "description": "Logs store type" }, "config": { @@ -1388,6 +1388,61 @@ ], "additionalProperties": false } + }, + { + "if": { + "properties": { + "../type": { + "const": "clickhouse" + } + } + }, + "then": { + "properties": { + "host": { + "type": "string", + "description": "ClickHouse host" + }, + "port": { + "type": "string", + "description": "ClickHouse port. Defaults by protocol: native 9000 (9440 TLS), http 8123 (8443 TLS)" + }, + "database": { + "type": "string", + "description": "ClickHouse database name (default: default)" + }, + "username": { + "type": "string", + "description": "ClickHouse username" + }, + "password": { + "type": "string", + "description": "ClickHouse password" + }, + "protocol": { + "type": "string", + "enum": ["native", "http"], + "description": "ClickHouse wire protocol (default: native)" + }, + "secure": { + "type": "boolean", + "description": "Enable TLS (native: secure=true; http: switches to https)", + "default": false + }, + "dial_timeout": { + "type": "integer", + "description": "Connection dial timeout in milliseconds (default: 10000)", + "minimum": 1, + "default": 10000 + }, + "cluster": { + "type": "string", + "description": "Optional cluster name; when set, DDL runs ON CLUSTER with replicated table engines" + } + }, + "required": ["host"], + "additionalProperties": false + } } ] }, diff --git a/ui/.gitignore b/ui/.gitignore deleted file mode 100644 index e1bfa611088..00000000000 --- a/ui/.gitignore +++ /dev/null @@ -1,43 +0,0 @@ -# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. - -# dependencies -/node_modules -/.pnp -.pnp.* -.yarn/* -!.yarn/patches -!.yarn/plugins -!.yarn/releases -!.yarn/versions - -# testing -/coverage - -# build output -/.next/ -/out/ - -# production -/build - -# misc -.DS_Store -*.pem - -# debug -npm-debug.log* -yarn-debug.log* -yarn-error.log* -.pnpm-debug.log* - -# env files (can opt-in for committing if needed) -.env* - -# vercel -.vercel - -# typescript -*.tsbuildinfo - -# auto-generated TanStack Router route tree -/app/routeTree.gen.ts