diff --git a/README.md b/README.md index 25f28eb..1ad1c91 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ Heimdall supports a growing set of pluggable command types: | `snowflake` | [Query execution in Snowflake](https://github.com/patterninc/heimdall/blob/main/plugins/snowflake/README.md) | Async | | `spark` | [SparkSQL query execution on EMR on EKS](https://github.com/patterninc/heimdall/blob/main/plugins/spark/README.md) | Async | | `trino` | [Query execution in Trino](https://github.com/patterninc/heimdall/blob/main/plugins/trino/README.md) | Async | +| `clickhouse`| [Query execution in Clickhouse](https://github.com/patterninc/heimdall/blob/main/plugins/clickhouse/README.md) | Sync | --- diff --git a/go.mod b/go.mod index 034ca49..6fa4cb9 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/patterninc/heimdall go 1.24.6 require ( + github.com/ClickHouse/clickhouse-go/v2 v2.40.3 + github.com/antlr4-go/antlr/v4 v4.13.1 github.com/aws/aws-sdk-go-v2 v1.38.0 github.com/aws/aws-sdk-go-v2/config v1.30.3 github.com/aws/aws-sdk-go-v2/credentials v1.18.3 @@ -20,6 +22,7 @@ require ( github.com/kubeflow/spark-operator/v2 v2.3.0 github.com/lib/pq v1.10.9 github.com/linkedin/goavro v2.1.0+incompatible + github.com/shopspring/decimal v1.4.0 github.com/snowflakedb/gosnowflake v1.15.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.33.4 @@ -35,8 +38,8 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 // indirect github.com/BurntSushi/toml v1.5.0 // indirect + github.com/ClickHouse/ch-go v0.68.0 // indirect github.com/andybalholm/brotli v1.2.0 // indirect - github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/apache/arrow-go/v18 v18.4.0 // indirect github.com/apache/thrift v0.22.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 // indirect @@ -61,6 +64,8 @@ require ( github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.9 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect @@ -86,6 +91,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mtibben/percent v0.2.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/paulmach/orb v0.11.1 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect @@ -93,22 +99,24 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/segmentio/asm v1.2.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect - go.opentelemetry.io/otel v1.37.0 // indirect - go.opentelemetry.io/otel/trace v1.37.0 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect - golang.org/x/crypto v0.41.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.42.0 // indirect golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 // indirect golang.org/x/mod v0.27.0 // indirect - golang.org/x/net v0.43.0 // indirect + golang.org/x/net v0.44.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.35.0 // indirect - golang.org/x/term v0.34.0 // indirect - golang.org/x/text v0.28.0 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/sys v0.36.0 // indirect + golang.org/x/term v0.35.0 // indirect + golang.org/x/text v0.29.0 // indirect golang.org/x/time v0.9.0 // indirect golang.org/x/tools v0.36.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect diff --git a/go.sum b/go.sum index 0ce2383..181e3c0 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,10 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJ github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/ClickHouse/ch-go v0.68.0 h1:zd2VD8l2aVYnXFRyhTyKCrxvhSz1AaY4wBUXu/f0GiU= +github.com/ClickHouse/ch-go v0.68.0/go.mod h1:C89Fsm7oyck9hr6rRo5gqqiVtaIY6AjdD0WFMyNRQ5s= +github.com/ClickHouse/clickhouse-go/v2 v2.40.3 h1:46jB4kKwVDUOnECpStKMVXxvR0Cg9zeV9vdbPjtn6po= +github.com/ClickHouse/clickhouse-go/v2 v2.40.3/go.mod h1:qO0HwvjCnTB4BPL/k6EE3l4d9f/uF+aoimAhJX70eKA= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= @@ -92,6 +96,10 @@ github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= +github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -112,12 +120,16 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -140,10 +152,12 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -169,6 +183,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -178,6 +193,9 @@ github.com/onsi/ginkgo/v2 v2.23.3 h1:edHxnszytJ4lD9D5Jjc4tiDkPBZ3siDeJJkUZJJVkp0 github.com/onsi/ginkgo/v2 v2.23.3/go.mod h1:zXTP6xIp3U8aVuXN8ENK9IXRaTjFnpVB9mGmaSRvxnM= github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= +github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= +github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= @@ -197,6 +215,10 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= +github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/snowflakedb/gosnowflake v1.15.0 h1:1V4dG1EmJ9O81Hv8y1LAE9koZebmx4tnRAPKWvDf8xA= @@ -207,38 +229,46 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= -go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= -go.yaml.in/yaml/v3 v3.0.3 h1:bXOww4E/J3f66rav3pX3m8w6jDE4knZjGOw8b5Y6iNE= -go.yaml.in/yaml/v3 v3.0.3/go.mod h1:tBHosrYAkRZjRAOREWbDnBXUf08JOwYq++0QNwQiWzI= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= -golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= +golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4= golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -249,28 +279,36 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= +golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= -golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= +golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -291,9 +329,12 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/pkg/object/command/clickhouse/clickhouse.go b/internal/pkg/object/command/clickhouse/clickhouse.go new file mode 100644 index 0000000..819f83a --- /dev/null +++ b/internal/pkg/object/command/clickhouse/clickhouse.go @@ -0,0 +1,183 @@ +package clickhouse + +import ( + "context" + "fmt" + + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" + "github.com/hladush/go-telemetry/pkg/telemetry" + hdctx "github.com/patterninc/heimdall/pkg/context" + "github.com/patterninc/heimdall/pkg/object/cluster" + "github.com/patterninc/heimdall/pkg/object/job" + "github.com/patterninc/heimdall/pkg/object/job/status" + "github.com/patterninc/heimdall/pkg/plugin" + "github.com/patterninc/heimdall/pkg/result" + "github.com/patterninc/heimdall/pkg/result/column" +) + +type commandContext struct { + Username string `yaml:"username,omitempty" json:"username,omitempty"` + Password string `yaml:"password,omitempty" json:"password,omitempty"` +} + +type clusterContext struct { + Endpoints []string `yaml:"endpoints" json:"endpoints"` + Database string `yaml:"database,omitempty" json:"database,omitempty"` +} + +type jobContext struct { + Query string `yaml:"query" json:"query"` + Params map[string]string `yaml:"params,omitempty" json:"params,omitempty"` + ReturnResult bool `yaml:"return_result,omitempty" json:"return_result,omitempty"` + conn driver.Conn +} + +const ( + serviceName = "clickhouse" +) + +var ( + dummyRowsInstance = dummyRows() + handleMethod = telemetry.NewMethod("handle", serviceName) + createExcMethod = telemetry.NewMethod("createExc", serviceName) + collectResultsMethod = telemetry.NewMethod("collectResults", serviceName) +) + +// New creates a new clickhouse plugin handler +func New(ctx *hdctx.Context) (plugin.Handler, error) { + t := &commandContext{} + + if ctx != nil { + if err := ctx.Unmarshal(t); err != nil { + return nil, err + } + } + + return t.handler, nil +} + +func (cmd *commandContext) handler(r *plugin.Runtime, j *job.Job, c *cluster.Cluster) error { + ctx := context.Background() + + jobContext, err := cmd.createJobContext(j, c) + if err != nil { + handleMethod.LogAndCountError(err, "create_job_context") + return err + } + + rows, err := jobContext.execute(ctx) + if err != nil { + handleMethod.LogAndCountError(err, "execute") + return err + } + res, err := collectResults(rows) + if err != nil { + handleMethod.LogAndCountError(err, "collect_results") + return err + } + j.Result = res + j.Status = status.Succeeded + + return nil +} + +func (cmd *commandContext) createJobContext(j *job.Job, c *cluster.Cluster) (*jobContext, error) { + // get cluster context + clusterCtx := &clusterContext{} + if c.Context != nil { + if err := c.Context.Unmarshal(clusterCtx); err != nil { + createExcMethod.CountError("unmarshal_cluster_context") + return nil, fmt.Errorf("failed to unmarshal cluster context: %v", err) + } + } + + // get job context + jobCtx := &jobContext{} + if j.Context != nil { + if err := j.Context.Unmarshal(jobCtx); err != nil { + createExcMethod.CountError("unmarshal_job_context") + return nil, fmt.Errorf("failed to unmarshal job context: %v", err) + } + } + + conn, err := clickhouse.Open(&clickhouse.Options{ + Addr: clusterCtx.Endpoints, + Auth: clickhouse.Auth{ + Database: clusterCtx.Database, + Username: cmd.Username, + Password: cmd.Password, + }, + }) + if err != nil { + createExcMethod.CountError("open_connection") + return nil, fmt.Errorf("failed to open ClickHouse connection: %v", err) + } + jobCtx.conn = conn + return jobCtx, nil +} + +func (j *jobContext) execute(ctx context.Context) (driver.Rows, error) { + var args []any + for k, v := range j.Params { + args = append(args, clickhouse.Named(k, v)) + } + if j.ReturnResult { + return j.conn.Query(ctx, j.Query, args...) + } + return dummyRowsInstance, j.conn.Exec(ctx, j.Query, args...) + +} + +func collectResults(rows driver.Rows) (*result.Result, error) { + defer rows.Close() + + cols := rows.Columns() + colTypes := rows.ColumnTypes() + + out := &result.Result{ + Columns: make([]*column.Column, len(cols)), + Data: make([][]any, 0, 128), + } + for i, c := range cols { + base, _ := unwrapCHType(colTypes[i].DatabaseTypeName()) + columnTypeName := colTypes[i].DatabaseTypeName() + if val, ok := chTypeToResultTypeName[base]; ok { + columnTypeName = val + } + out.Columns[i] = &column.Column{ + Name: c, + Type: column.Type(columnTypeName), + } + } + + // For each column we keep: scan target and a reader that returns a normalized interface{} + + for rows.Next() { + scanTargets := make([]any, len(cols)) + readers := make([]func() any, len(cols)) + + for i, ct := range colTypes { + base, nullable := unwrapCHType(ct.DatabaseTypeName()) + + if handler, ok := chTypeHandlers[base]; ok { + scanTargets[i], readers[i] = handler(nullable) + } else { + // Fallback (covers unknown + legacy decimal detection) + scanTargets[i], readers[i] = handleDefault(nullable) + } + } + + if err := rows.Scan(scanTargets...); err != nil { + collectResultsMethod.CountError("row_scan") + return nil, fmt.Errorf("row scan error: %w", err) + } + + row := make([]any, len(cols)) + for i := range readers { + row[i] = readers[i]() + } + out.Data = append(out.Data, row) + } + return out, nil +} diff --git a/internal/pkg/object/command/clickhouse/column_types.go b/internal/pkg/object/command/clickhouse/column_types.go new file mode 100644 index 0000000..b7348be --- /dev/null +++ b/internal/pkg/object/command/clickhouse/column_types.go @@ -0,0 +1,133 @@ +package clickhouse + +import ( + "strings" + "time" + + "github.com/shopspring/decimal" +) + +var chTypeToResultTypeName = map[string]string{ + "UInt8": "int", + "UInt16": "int", + "UInt32": "int", + "UInt64": "long", + "Int8": "int", + "Int16": "int", + "Int32": "int", + "Int64": "long", + "Float32": "float", + "Float64": "double", + "String": "string", + "FixedString": "string", + "Date": "int32", + "Date32": "int32", + "DateTime": "long", + "DateTime64": "long", + "Bool": "boolean", +} + +// Map of base type -> handler +var chTypeHandlers = map[string]chScanHandler{ + "UInt8": handleUInt8, + "UInt16": handleUInt16, + "UInt32": handleUInt32, + "UInt64": handleUInt64, + "Int8": handleInt8, + "Int16": handleInt16, + "Int32": handleInt32, + "Int64": handleInt64, + "Float32": handleFloat32, + "Float64": handleFloat64, + "String": handleString, + "FixedString": handleString, + "Date": handleTime, + "Date32": handleTime, + "DateTime": handleTime, + "DateTime64": handleTime, + "Decimal": handleDecimal, + "Bool": handleBool, +} + +// Type handler signature +type chScanHandler func(nullable bool) (scanTarget any, reader func() any) + +// unified nullable helper +func makeScanTarget[T any](nullable bool) (any, func() any) { + if nullable { + var p *T + return &p, func() any { + if p == nil { + return nil + } + return *p + } + } + var v T + return &v, func() any { return v } +} + +// Individual handlers (all via generic helper) +func handleUInt8(nullable bool) (any, func() any) { return makeScanTarget[uint8](nullable) } +func handleUInt16(nullable bool) (any, func() any) { return makeScanTarget[uint16](nullable) } +func handleUInt32(nullable bool) (any, func() any) { return makeScanTarget[uint32](nullable) } +func handleUInt64(nullable bool) (any, func() any) { return makeScanTarget[uint64](nullable) } +func handleInt8(nullable bool) (any, func() any) { return makeScanTarget[int8](nullable) } +func handleInt16(nullable bool) (any, func() any) { return makeScanTarget[int16](nullable) } +func handleInt32(nullable bool) (any, func() any) { return makeScanTarget[int32](nullable) } +func handleInt64(nullable bool) (any, func() any) { return makeScanTarget[int64](nullable) } +func handleFloat32(nullable bool) (any, func() any) { return makeScanTarget[float32](nullable) } +func handleFloat64(nullable bool) (any, func() any) { return makeScanTarget[float64](nullable) } +func handleString(nullable bool) (any, func() any) { return makeScanTarget[string](nullable) } +func handleTime(nullable bool) (any, func() any) { return makeScanTarget[time.Time](nullable) } +func handleBool(nullable bool) (any, func() any) { return makeScanTarget[bool](nullable) } +func handleDecimal(nullable bool) (any, func() any) { + if nullable { + var p decimal.NullDecimal + return &p, func() any { + if p.Valid { + val, _ := p.Decimal.Float64() + return val + } + return nil + } + } + var v decimal.Decimal + return &v, func() any { + val, _ := v.Float64() + return val + } +} + +func handleDefault(nullable bool) (any, func() any) { + // Treat Decimal, UUID, IPv4, IPv6 as string; fallback also string + return makeScanTarget[string](nullable) +} + +func unwrapCHType(t string) (base string, nullable bool) { + // Unwrap Nullable and LowCardinality; keep Array out for brevity + s := t + for { + if strings.HasPrefix(s, "Nullable(") && strings.HasSuffix(s, ")") { + nullable = true + s = s[len("Nullable(") : len(s)-1] + continue + } + if strings.HasPrefix(s, "LowCardinality(") && strings.HasSuffix(s, ")") { + s = s[len("LowCardinality(") : len(s)-1] + continue + } + break + } + // Decimal(N,S) normalize to "Decimal" + if isDecimal(s) { + return "Decimal", nullable + } + return s, nullable +} + +func isDecimal(s string) bool { + return strings.HasPrefix(s, "Decimal(") || strings.HasPrefix(s, "Decimal32(") || + strings.HasPrefix(s, "Decimal64(") || strings.HasPrefix(s, "Decimal128(") || + strings.HasPrefix(s, "Decimal256(") +} diff --git a/internal/pkg/object/command/clickhouse/dummy_rows.go b/internal/pkg/object/command/clickhouse/dummy_rows.go new file mode 100644 index 0000000..d8cd894 --- /dev/null +++ b/internal/pkg/object/command/clickhouse/dummy_rows.go @@ -0,0 +1,47 @@ +package clickhouse + +import ( + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" +) + +func dummyRows() driver.Rows { + return &dummyRowsStruct{} +} + +type dummyRowsStruct struct { + // driver.Rows +} + +// ColumnTypes implements driver.Rows. +func (d *dummyRowsStruct) ColumnTypes() []driver.ColumnType { + return nil +} + +// Scan implements driver.Rows. +func (d *dummyRowsStruct) Scan(dest ...any) error { + return nil +} + +// ScanStruct implements driver.Rows. +func (d *dummyRowsStruct) ScanStruct(dest any) error { + return nil +} + +// Totals implements driver.Rows. +func (d *dummyRowsStruct) Totals(dest ...any) error { + return nil +} + +func (d *dummyRowsStruct) Next() bool { + return false +} + +func (d *dummyRowsStruct) Columns() []string { + return nil +} +func (d *dummyRowsStruct) Close() error { + return nil +} +func (d *dummyRowsStruct) Err() error { + return nil +} diff --git a/plugins/clickhouse/README.md b/plugins/clickhouse/README.md new file mode 100644 index 0000000..34e2439 --- /dev/null +++ b/plugins/clickhouse/README.md @@ -0,0 +1,138 @@ +# 🗃️ ClickHouse Plugin + +The **ClickHouse Plugin** enables Heimdall to execute SQL queries on configured ClickHouse clusters. It connects directly to ClickHouse instances and executes queries, with support for parameterized queries and optional result collection. + +--- + +## 🧩 Plugin Overview + +* **Plugin Name:** `clickhouse` +* **Use Case:** Running SQL queries against ClickHouse databases with optional result retrieval + +--- + +## ⚙️ Defining a ClickHouse Command + +A ClickHouse command defines authentication credentials for connecting to ClickHouse clusters. The credentials are shared across all jobs using this command. + +```yaml + - name: clickhouse-analytics + status: active + plugin: clickhouse + version: 24.8 + description: Execute ClickHouse queries + context: + username: analytics_user + password: secure_password + tags: + - type:clickhouse + cluster_tags: + - type:clickhouse +``` + +🔸 The command stores authentication credentials (`username` and `password`) that will be used for all ClickHouse connections. These credentials are applied to all jobs targeting ClickHouse clusters. + +--- + +## 🖥️ Cluster Configuration + +Each ClickHouse cluster must define connection `endpoints` and optionally specify a default `database`. + +```yaml + - name: clickhouse-prod + status: active + version: 24.8 + description: Production ClickHouse cluster + context: + endpoints: + - "clickhouse-node1.company.com:9000" + - "clickhouse-node2.company.com:9000" + - "clickhouse-node3.company.com:9000" + database: analytics + tags: + - type:clickhouse + - env:production +``` + +🔹 The `endpoints` array defines ClickHouse server addresses with ports. The optional `database` parameter sets the default database for connections to this cluster. + +--- + +## 🚀 Submitting a ClickHouse Job + +A ClickHouse job provides the SQL query to execute, optional parameters, and result handling preferences. + +```json +{ + "name": "user-analytics-query", + "version": "1.0.0", + "command_criteria": ["type:clickhouse"], + "cluster_criteria": ["env:production"], + "context": { + "query": "SELECT user_id, COUNT(*) AS events FROM user_events WHERE date >= {date:Date} AND user_type = {user_type:String} GROUP BY user_id", + "params": { + "date": "2024-01-01", + "user_type": "premium" + }, + "return_result": true + } +} +``` + +🔹 The job executes the SQL query with the provided parameters and returns results if `return_result` is enabled. Parameters are safely bound to prevent SQL injection. + +--- + +## 📦 Job Context & Runtime + +The ClickHouse plugin handles: + +* **Connection Management**: Establishes secure connections to ClickHouse clusters using provided credentials +* **Query Execution**: Executes SQL queries with parameter binding for security +* **Type Handling**: Properly handles ClickHouse data types including nullable and low cardinality variants +* **Result Collection**: Optionally collects and formats query results for API responses + +### Supported ClickHouse Data Types + +The plugin supports comprehensive ClickHouse type mapping: + +| ClickHouse Type | Go Type | Nullable Support | +|----------------|---------|------------------| +| `UInt8`, `UInt16`, `UInt32`, `Int8`, `Int16`, `Int32`, | `int`, | ✅ | +| `UInt64` `Int64` | `int64` | ✅ | +| `Float32`, `Float64` | `float32`, `float64` | ✅ | +| `String`, `FixedString` | `string` | ✅ | +| `Date`, `Date32`, `DateTime`, | `time.Time` | ✅ | +| `Decimal(P,S)`, | `string` | ✅ | + +🔸 The plugin automatically handles: +- **Nullable types**: `Nullable(String)` → `*string` +- **Low cardinality**: `LowCardinality(String)` → `string` +- **Complex wrappers**: `Nullable(LowCardinality(String))` → `*string` +- **Decimal variants**: `Decimal64(18,4)` → `string` + +--- + +## 📊 Returning Job Results + +When `return_result` is enabled, query results are available via: + +``` +GET /api/v1/job//result +``` + +Results are returned in structured format: + +```json +{ + "columns": [ + {"name": "user_id", "type": "long"}, + {"name": "events", "type": "long"} + ], + "data": [ + [12345, 156], + [67890, 203], + [11111, 89] + ] +} +``` \ No newline at end of file diff --git a/plugins/clickhouse/clickhouse.go b/plugins/clickhouse/clickhouse.go new file mode 100644 index 0000000..f15422e --- /dev/null +++ b/plugins/clickhouse/clickhouse.go @@ -0,0 +1,12 @@ +package main + +import ( + "github.com/patterninc/heimdall/internal/pkg/object/command/clickhouse" + "github.com/patterninc/heimdall/pkg/context" + "github.com/patterninc/heimdall/pkg/plugin" +) + +// New creates a new clickhouse plugin handler +func New(commandContext *context.Context) (plugin.Handler, error) { + return clickhouse.New(commandContext) +}