From 6854d813eb88f88a40d3c04f5765ee3273fcc72d Mon Sep 17 00:00:00 2001 From: Aris Tzoumas Date: Mon, 9 Sep 2024 14:53:05 +0300 Subject: [PATCH 1/4] feat: support dialect-aware query conditions --- go.mod | 1 + go.sum | 4 + sqlconnect/db.go | 73 +++-- sqlconnect/internal/base/dbopts.go | 7 + sqlconnect/internal/base/dialect.go | 4 +- sqlconnect/internal/base/goqu_dialect.go | 183 ++++++++++++ sqlconnect/internal/bigquery/db.go | 2 +- sqlconnect/internal/bigquery/dialect.go | 4 +- sqlconnect/internal/bigquery/goqu_dialect.go | 29 ++ .../internal/bigquery/goqu_dialect_test.go | 48 ++++ .../bigquery/testdata/goqu-test-seed.sql | 12 + sqlconnect/internal/databricks/db.go | 2 +- sqlconnect/internal/databricks/dialect.go | 4 +- .../internal/databricks/goqu_dialect.go | 28 ++ .../internal/databricks/goqu_dialect_test.go | 48 ++++ .../databricks/testdata/goqu-test-seed.sql | 12 + .../db_integration_test_scenario.go | 266 +++++++++++++++++- sqlconnect/internal/mysql/db.go | 2 +- sqlconnect/internal/mysql/dialect.go | 4 +- sqlconnect/internal/mysql/goqu_dialect.go | 92 ++++++ .../internal/mysql/goqu_dialect_test.go | 48 ++++ .../mysql/testdata/goqu-test-seed.sql | 12 + sqlconnect/internal/postgres/db.go | 1 + sqlconnect/internal/postgres/goqu_dialect.go | 29 ++ .../internal/postgres/goqu_dialect_test.go | 48 ++++ .../postgres/testdata/goqu-test-seed.sql | 12 + sqlconnect/internal/redshift/db.go | 2 +- sqlconnect/internal/redshift/dialect.go | 1 + sqlconnect/internal/redshift/goqu_dialect.go | 27 ++ .../internal/redshift/goqu_dialect_test.go | 48 ++++ .../redshift/testdata/goqu-test-seed.sql | 12 + sqlconnect/internal/snowflake/db.go | 2 +- sqlconnect/internal/snowflake/dialect.go | 4 +- sqlconnect/internal/snowflake/goqu_dialect.go | 28 ++ .../internal/snowflake/goqu_dialect_test.go | 48 ++++ .../snowflake/testdata/goqu-test-seed.sql | 12 + sqlconnect/internal/trino/db.go | 2 +- sqlconnect/internal/trino/dialect.go | 4 +- sqlconnect/internal/trino/goqu_dialect.go | 43 +++ .../internal/trino/goqu_dialect_test.go | 48 ++++ .../trino/testdata/goqu-test-seed.sql | 12 + sqlconnect/op/operator.go | 45 +++ sqlconnect/querydef.go | 2 +- sqlconnect/querydef_test.go | 4 +- 44 files changed, 1288 insertions(+), 31 deletions(-) create mode 100644 sqlconnect/internal/base/goqu_dialect.go create mode 100644 sqlconnect/internal/bigquery/goqu_dialect.go create mode 100644 sqlconnect/internal/bigquery/goqu_dialect_test.go create mode 100644 sqlconnect/internal/bigquery/testdata/goqu-test-seed.sql create mode 100644 sqlconnect/internal/databricks/goqu_dialect.go create mode 100644 sqlconnect/internal/databricks/goqu_dialect_test.go create mode 100644 sqlconnect/internal/databricks/testdata/goqu-test-seed.sql create mode 100644 sqlconnect/internal/mysql/goqu_dialect.go create mode 100644 sqlconnect/internal/mysql/goqu_dialect_test.go create mode 100644 sqlconnect/internal/mysql/testdata/goqu-test-seed.sql create mode 100644 sqlconnect/internal/postgres/goqu_dialect.go create mode 100644 sqlconnect/internal/postgres/goqu_dialect_test.go create mode 100644 sqlconnect/internal/postgres/testdata/goqu-test-seed.sql create mode 100644 sqlconnect/internal/redshift/goqu_dialect.go create mode 100644 sqlconnect/internal/redshift/goqu_dialect_test.go create mode 100644 sqlconnect/internal/redshift/testdata/goqu-test-seed.sql create mode 100644 sqlconnect/internal/snowflake/goqu_dialect.go create mode 100644 sqlconnect/internal/snowflake/goqu_dialect_test.go create mode 100644 sqlconnect/internal/snowflake/testdata/goqu-test-seed.sql create mode 100644 sqlconnect/internal/trino/goqu_dialect.go create mode 100644 sqlconnect/internal/trino/goqu_dialect_test.go create mode 100644 sqlconnect/internal/trino/testdata/goqu-test-seed.sql create mode 100644 sqlconnect/op/operator.go diff --git a/go.mod b/go.mod index 0425366..1ac1685 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/google/uuid v1.6.0 github.com/lib/pq v1.10.9 github.com/ory/dockertest/v3 v3.11.0 + github.com/rudderlabs/goqu/v10 v10.3.0 github.com/rudderlabs/rudder-go-kit v0.40.0 github.com/rudderlabs/sql-tunnels v0.1.7 github.com/samber/lo v1.47.0 diff --git a/go.sum b/go.sum index 60a1b00..1840c06 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,8 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg6 github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= +github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= @@ -299,6 +301,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= +github.com/rudderlabs/goqu/v10 v10.3.0 h1:FaZioS8fRYJVYoLO5lieuyiVvEHhmQ6jP5sKwPIcKSs= +github.com/rudderlabs/goqu/v10 v10.3.0/go.mod h1:LH2vI5gGHBxEQuESqFyk5ZA2anGINc8o25hbidDWOYw= github.com/rudderlabs/rudder-go-kit v0.40.0 h1:Vk/NZm2DUuOiMmTSKUWYQVbXkl4If9KdGQOjNpXCPC4= github.com/rudderlabs/rudder-go-kit v0.40.0/go.mod h1:GtOYIFfVvNcXabgGytoGdsjdpKTH6PipFIom0bY94WQ= github.com/rudderlabs/sql-tunnels v0.1.7 h1:wDCRl6zY4M5gfWazf7XkSTGQS3yjBzUiUgEMBIfHNDA= diff --git a/sqlconnect/db.go b/sqlconnect/db.go index 7917c99..40ba07e 100644 --- a/sqlconnect/db.go +++ b/sqlconnect/db.go @@ -5,7 +5,10 @@ import ( "database/sql" "database/sql/driver" "errors" + "fmt" "time" + + "github.com/rudderlabs/goqu/v10" ) var ( @@ -98,23 +101,61 @@ type JsonRowMapper interface { JSONRowMapper() RowMapper[map[string]any] } -type Dialect interface { - // QuoteTable quotes a table name - QuoteTable(table RelationRef) string +type ( + Dialect interface { + // QuoteTable quotes a table name + QuoteTable(table RelationRef) string - // QuoteIdentifier quotes an identifier, e.g. a column name - QuoteIdentifier(name string) string + // QuoteIdentifier quotes an identifier, e.g. a column name + QuoteIdentifier(name string) string - // FormatTableName formats a table name, typically by lower or upper casing it, depending on the database - // - // Deprecated: to be removed in future versions, since its behaviour is not consistent across databases, e.g. using lowercase for BigQuery while it shouldn't. - // If you want to have a consistent behaviour across databases, use [NormaliseIdentifier] and [ParseRelationRef] instead. - FormatTableName(name string) string + // FormatTableName formats a table name, typically by lower or upper casing it, depending on the database + // + // Deprecated: to be removed in future versions, since its behaviour is not consistent across databases, e.g. using lowercase for BigQuery while it shouldn't. + // If you want to have a consistent behaviour across databases, use [NormaliseIdentifier] and [ParseRelationRef] instead. + FormatTableName(name string) string - // NormaliseIdentifier normalises the identifier's parts that are unquoted, typically by lower or upper casing them, depending on the database - NormaliseIdentifier(identifier string) string + // NormaliseIdentifier normalises the identifier's parts that are unquoted, typically by lower or upper casing them, depending on the database + NormaliseIdentifier(identifier string) string - // ParseRelationRef parses a string into a RelationRef after normalising the identifier and stripping out surrounding quotes. - // The result is a RelationRef with case-sensitive fields, i.e. it can be safely quoted (see [QuoteTable] and, for instance, used for matching against the database's information schema. - ParseRelationRef(identifier string) (RelationRef, error) -} + // ParseRelationRef parses a string into a RelationRef after normalising the identifier and stripping out surrounding quotes. + // The result is a RelationRef with case-sensitive fields, i.e. it can be safely quoted (see [QuoteTable] and, for instance, used for matching against the database's information schema. + ParseRelationRef(identifier string) (RelationRef, error) + + // QueryCondition returns a dialect-specific query condition sql string for the provided identifier, operator and value(s). + // + // E.g. QueryCondition("age", "gt", 18) returns "age > 18" + // + // Each operator has a different number of arguments, e.g. [eq] requires one argument, [in] requires at least one argument, etc. + // See [op] package for the list of supported operators + QueryCondition(identifier, operator string, args ...any) (sql string, err error) + + // GoquExpressionToSQL converts an Expression to a SQL string + GoquExpressionToSQL(expression GoquExpression) (sql string, err error) + + // Expressions returns the dialect-specific expressions + Expressions() Expressions + } + + // GoquExpression represents a goqu expression + GoquExpression = goqu.Expression + + // Expressions provides dialect-specific expressions + Expressions interface { + // TimestampAdd returns an expression that adds the interval to the timestamp value. + // The value can either be a string literal (column, timestamp, function etc.) or a [time.Time] value. + TimestampAdd(timeValue any, interval int, unit string) (Expression, error) + + // DateAdd returns an expression that adds the interval to the date value. + // The value can either be a string literal (column, timestamp, function etc.) or a [time.Time] value. + // Values are cast to [DATE]. + DateAdd(dateValue any, interval int, unit string) (Expression, error) + } + + // Expression represents a dialect-specific expression. + // One can get the expression's SQL string by calling [String()] on it. + Expression interface { + GoquExpression() GoquExpression + fmt.Stringer + } +) diff --git a/sqlconnect/internal/base/dbopts.go b/sqlconnect/internal/base/dbopts.go index 3614b36..3158aa9 100644 --- a/sqlconnect/internal/base/dbopts.go +++ b/sqlconnect/internal/base/dbopts.go @@ -44,6 +44,13 @@ func WithDialect(dialect sqlconnect.Dialect) Option { } } +// WithGoquDialect sets the goqu dialect for the client +func WithGoquDialect(gqd *GoquDialect) Option { + return func(db *DB) { + db.Dialect = &dialect{gqd} + } +} + // WithSQLCommandsOverride allows for overriding some of the sql commands that the client uses func WithSQLCommandsOverride(override func(defaultCommands SQLCommands) SQLCommands) Option { return func(db *DB) { diff --git a/sqlconnect/internal/base/dialect.go b/sqlconnect/internal/base/dialect.go index cd7aa5e..6d49980 100644 --- a/sqlconnect/internal/base/dialect.go +++ b/sqlconnect/internal/base/dialect.go @@ -7,7 +7,9 @@ import ( "github.com/rudderlabs/sqlconnect-go/sqlconnect" ) -type dialect struct{} +type dialect struct { + *GoquDialect +} // QuoteTable quotes a table name func (d dialect) QuoteTable(table sqlconnect.RelationRef) string { diff --git a/sqlconnect/internal/base/goqu_dialect.go b/sqlconnect/internal/base/goqu_dialect.go new file mode 100644 index 0000000..98fc669 --- /dev/null +++ b/sqlconnect/internal/base/goqu_dialect.go @@ -0,0 +1,183 @@ +package base + +import ( + "fmt" + + "github.com/samber/lo" + + "github.com/rudderlabs/goqu/v10" + "github.com/rudderlabs/goqu/v10/exp" + "github.com/rudderlabs/goqu/v10/sqlgen" + "github.com/rudderlabs/sqlconnect-go/sqlconnect" +) + +func NewGoquDialect(dialect string, o *sqlgen.SQLDialectOptions, expressions *Expressions) *GoquDialect { + return &GoquDialect{ + esg: sqlgen.NewExpressionSQLGenerator(dialect, o), + expressions: expressions, + } +} + +type GoquDialect struct { + esg sqlgen.ExpressionSQLGenerator + expressions *Expressions +} + +type Expressions struct { + TimestampAdd func(time any, interval int, unit string) goqu.Expression + DateAdd func(date any, interval int, unit string) goqu.Expression +} + +func (gq *GoquDialect) QueryCondition(identifier, operator string, args ...any) (sql string, err error) { + args = lo.Map(args, func(a any, _ int) any { + if s, ok := a.(sqlconnect.Expression); ok { + return s.GoquExpression() + } + return a + }) + var expr goqu.Expression + switch operator { + case "eq": + if len(args) != 1 { + return "", fmt.Errorf("eq operator requires exactly one argument, got %d", len(args)) + } + expr = goqu.C(identifier).Eq(args[0]) + case "neq": + if len(args) != 1 { + return "", fmt.Errorf("neq operator requires exactly one argument, got %d", len(args)) + } + expr = goqu.C(identifier).Neq(args[0]) + case "in": + if len(args) == 0 { + return "", fmt.Errorf("in operator requires at least one argument") + } + expr = goqu.C(identifier).In(args...) + case "notin": + if len(args) == 0 { + return "", fmt.Errorf("notin operator requires at least one argument") + } + expr = goqu.C(identifier).NotIn(args...) + case "like": + if len(args) != 1 { + return "", fmt.Errorf("like operator requires exactly one argument, got %d", len(args)) + } + expr = goqu.C(identifier).Like(args[0]) + case "notlike": + if len(args) != 1 { + return "", fmt.Errorf("notlike operator requires exactly one argument, got %d", len(args)) + } + expr = goqu.C(identifier).NotLike(args[0]) + case "isset": + if len(args) != 0 { + return "", fmt.Errorf("isset operator requires no arguments, got %d", len(args)) + } + expr = goqu.C(identifier).IsNotNull() + case "notset": + if len(args) != 0 { + return "", fmt.Errorf("isnotset operator requires no arguments, got %d", len(args)) + } + expr = goqu.C(identifier).IsNull() + case "gt": + if len(args) != 1 { + return "", fmt.Errorf("gt operator requires exactly one argument, got %d", len(args)) + } + expr = goqu.C(identifier).Gt(args[0]) + case "gte": + if len(args) != 1 { + return "", fmt.Errorf("gte operator requires exactly one argument, got %d", len(args)) + } + expr = goqu.C(identifier).Gte(args[0]) + case "lt": + if len(args) != 1 { + return "", fmt.Errorf("lt operator requires exactly one argument, got %d", len(args)) + } + expr = goqu.C(identifier).Lt(args[0]) + case "lte": + if len(args) != 1 { + return "", fmt.Errorf("lte operator requires exactly one argument, got %d", len(args)) + } + expr = goqu.C(identifier).Lte(args[0]) + case "between": + if len(args) != 2 { + return "", fmt.Errorf("between operator requires exactly two arguments, got %d", len(args)) + } + expr = goqu.C(identifier).Between(exp.NewRangeVal(args[0], args[1])) + case "notbetween": + if len(args) != 2 { + return "", fmt.Errorf("notbetween operator requires exactly two arguments, got %d", len(args)) + } + expr = goqu.C(identifier).NotBetween(exp.NewRangeVal(args[0], args[1])) + default: + return "", fmt.Errorf("unsupported operator: %s", operator) + } + + return gq.GoquExpressionToSQL(expr) +} + +func (gq *GoquDialect) GoquExpressionToSQL(expression sqlconnect.GoquExpression) (sql string, err error) { + sql, _, err = sqlgen.GenerateExpressionSQL(gq.esg, false, expression) + return +} + +func (gq *GoquDialect) Expressions() sqlconnect.Expressions { + return gq +} + +func (gq *GoquDialect) TimestampAdd(timeValue any, interval int, unit string) (sqlconnect.Expression, error) { + switch unit { + case "second", "minute", "hour", "day", "month", "year": + case "week": + unit = "day" + interval *= 7 + default: + return nil, fmt.Errorf("unsupported unit: %s", unit) + } + + var v any + switch timeValue := timeValue.(type) { + case string: + v = goqu.L(timeValue) + default: + v = timeValue + } + + goquExpression := gq.expressions.TimestampAdd(v, interval, unit) + sql, _, err := sqlgen.GenerateExpressionSQL(gq.esg, false, goquExpression) + return &expression{Expression: goquExpression, sql: sql}, err +} + +func (gq *GoquDialect) DateAdd(timeValue any, interval int, unit string) (sqlconnect.Expression, error) { + switch unit { + case "day", "month", "year": + case "week": + unit = "day" + interval *= 7 + default: + return nil, fmt.Errorf("unsupported unit: %s", unit) + } + + var v any + switch timeValue := timeValue.(type) { + case string: + v = goqu.L(timeValue) + default: + v = timeValue + } + + goquExpression := gq.expressions.DateAdd(v, interval, unit) + sql, _, err := sqlgen.GenerateExpressionSQL(gq.esg, false, goquExpression) + return &expression{Expression: goquExpression, sql: sql}, err +} + +type expression struct { + goqu.Expression + sql string +} + +func (e *expression) GoquExpression() goqu.Expression { + return e.Expression +} + +func (e *expression) String() string { + return e.sql +} diff --git a/sqlconnect/internal/bigquery/db.go b/sqlconnect/internal/bigquery/db.go index 839c733..f08c3a2 100644 --- a/sqlconnect/internal/bigquery/db.go +++ b/sqlconnect/internal/bigquery/db.go @@ -37,7 +37,7 @@ func NewDB(configJSON json.RawMessage) (*DB, error) { DB: base.NewDB( db, sshtunnel.NoTunnelCloser, - base.WithDialect(dialect{}), + base.WithDialect(dialect{base.NewGoquDialect(DatabaseType, GoquDialectOptions(), GoquExpressions())}), base.WithColumnTypeMapper(getColumnTypeMapper(config)), base.WithJsonRowMapper(getJonRowMapper(config)), base.WithSQLCommandsOverride(func(cmds base.SQLCommands) base.SQLCommands { diff --git a/sqlconnect/internal/bigquery/dialect.go b/sqlconnect/internal/bigquery/dialect.go index 84f1fe8..db83c63 100644 --- a/sqlconnect/internal/bigquery/dialect.go +++ b/sqlconnect/internal/bigquery/dialect.go @@ -8,7 +8,9 @@ import ( "github.com/rudderlabs/sqlconnect-go/sqlconnect/internal/base" ) -type dialect struct{} +type dialect struct { + *base.GoquDialect +} var ( escape = regexp.MustCompile("('|\"|`)") diff --git a/sqlconnect/internal/bigquery/goqu_dialect.go b/sqlconnect/internal/bigquery/goqu_dialect.go new file mode 100644 index 0000000..caab37d --- /dev/null +++ b/sqlconnect/internal/bigquery/goqu_dialect.go @@ -0,0 +1,29 @@ +package bigquery + +import ( + "fmt" + "strings" + + "github.com/rudderlabs/goqu/v10" + "github.com/rudderlabs/goqu/v10/sqlgen" + "github.com/rudderlabs/sqlconnect-go/sqlconnect/internal/base" +) + +func GoquDialectOptions() *sqlgen.SQLDialectOptions { + o := sqlgen.DefaultDialectOptions() + o.QuoteIdentifiers = false + o.QuoteRune = '`' + return o +} + +func GoquExpressions() *base.Expressions { + return &base.Expressions{ + TimestampAdd: func(timeValue any, interval int, unit string) goqu.Expression { + return goqu.L(fmt.Sprintf("TIMESTAMP_ADD(?, INTERVAL %d %s)", interval, strings.ToUpper(unit)), timeValue) + }, + + DateAdd: func(dateValue any, interval int, unit string) goqu.Expression { + return goqu.L(fmt.Sprintf("DATE_ADD(DATE(?), INTERVAL %d %s)", interval, strings.ToUpper(unit)), dateValue) + }, + } +} diff --git a/sqlconnect/internal/bigquery/goqu_dialect_test.go b/sqlconnect/internal/bigquery/goqu_dialect_test.go new file mode 100644 index 0000000..90d1260 --- /dev/null +++ b/sqlconnect/internal/bigquery/goqu_dialect_test.go @@ -0,0 +1,48 @@ +package bigquery + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/rudderlabs/goqu/v10" + "github.com/rudderlabs/goqu/v10/sqlgen" +) + +func TestExpressions(t *testing.T) { + expressions := GoquExpressions() + + t.Run("TimestampAdd", func(t *testing.T) { + t.Run("literal", func(t *testing.T) { + require.Equal(t, "TIMESTAMP_ADD(CURRENT_DATE, INTERVAL 1 HOUR)", toSQL(t, expressions.TimestampAdd(goqu.L("CURRENT_DATE"), 1, "hour"))) + require.Equal(t, "TIMESTAMP_ADD('2020-01-01', INTERVAL 1 DAY)", toSQL(t, expressions.TimestampAdd(goqu.L("'2020-01-01'"), 1, "day"))) + }) + + t.Run("time", func(t *testing.T) { + now, err := time.Parse(time.RFC3339, "2020-01-01T00:00:00Z") + require.NoError(t, err) + require.Equal(t, "TIMESTAMP_ADD('2020-01-01T00:00:00Z', INTERVAL 1 DAY)", toSQL(t, expressions.TimestampAdd(now, 1, "day"))) + }) + }) + + t.Run("DateAdd", func(t *testing.T) { + t.Run("literal", func(t *testing.T) { + require.Equal(t, "DATE_ADD(DATE(CURRENT_DATE), INTERVAL 1 DAY)", toSQL(t, expressions.DateAdd(goqu.L("CURRENT_DATE"), 1, "day"))) + require.Equal(t, "DATE_ADD(DATE('2020-01-01'), INTERVAL 1 DAY)", toSQL(t, expressions.DateAdd(goqu.L("'2020-01-01'"), 1, "day"))) + }) + + t.Run("time", func(t *testing.T) { + now, err := time.Parse(time.RFC3339, "2020-01-01T00:00:00Z") + require.NoError(t, err) + require.Equal(t, "DATE_ADD(DATE('2020-01-01T00:00:00Z'), INTERVAL 1 DAY)", toSQL(t, expressions.DateAdd(now, 1, "day"))) + }) + }) +} + +func toSQL(t *testing.T, expression interface{}) string { + esg := sqlgen.NewExpressionSQLGenerator(DatabaseType, GoquDialectOptions()) + sql, _, err := sqlgen.GenerateExpressionSQL(esg, false, expression) + require.NoError(t, err) + return sql +} diff --git a/sqlconnect/internal/bigquery/testdata/goqu-test-seed.sql b/sqlconnect/internal/bigquery/testdata/goqu-test-seed.sql new file mode 100644 index 0000000..1c12b2d --- /dev/null +++ b/sqlconnect/internal/bigquery/testdata/goqu-test-seed.sql @@ -0,0 +1,12 @@ +CREATE TABLE `{{.schema}}`.`goqu_test` ( + _string STRING(10), + _int INT, + _float BIGNUMERIC(2,1), + _boolean BOOLEAN, + _timestamp TIMESTAMP +); + +INSERT INTO `{{.schema}}`.`goqu_test` + (_string, _int, _float, _boolean, _timestamp) +VALUES + ('string', 1, 1.1, TRUE, '2021-01-01T00:00:00Z'); \ No newline at end of file diff --git a/sqlconnect/internal/databricks/db.go b/sqlconnect/internal/databricks/db.go index 31ae96f..cd781f6 100644 --- a/sqlconnect/internal/databricks/db.go +++ b/sqlconnect/internal/databricks/db.go @@ -69,7 +69,7 @@ func NewDB(configJson json.RawMessage) (*DB, error) { DB: base.NewDB( db, tunnelCloser, - base.WithDialect(dialect{}), + base.WithDialect(dialect{base.NewGoquDialect(DatabaseType, GoquDialectOptions(), GoquExpressions())}), base.WithColumnTypeMapper(getColumnTypeMapper(config)), base.WithJsonRowMapper(getJonRowMapper(config)), base.WithSQLCommandsOverride(func(cmds base.SQLCommands) base.SQLCommands { diff --git a/sqlconnect/internal/databricks/dialect.go b/sqlconnect/internal/databricks/dialect.go index 359cc2a..d54c9f8 100644 --- a/sqlconnect/internal/databricks/dialect.go +++ b/sqlconnect/internal/databricks/dialect.go @@ -7,7 +7,9 @@ import ( "github.com/rudderlabs/sqlconnect-go/sqlconnect/internal/base" ) -type dialect struct{} +type dialect struct { + *base.GoquDialect +} // QuoteTable quotes a table name func (d dialect) QuoteTable(table sqlconnect.RelationRef) string { diff --git a/sqlconnect/internal/databricks/goqu_dialect.go b/sqlconnect/internal/databricks/goqu_dialect.go new file mode 100644 index 0000000..b1f4070 --- /dev/null +++ b/sqlconnect/internal/databricks/goqu_dialect.go @@ -0,0 +1,28 @@ +package databricks + +import ( + "fmt" + "strings" + + "github.com/rudderlabs/goqu/v10" + "github.com/rudderlabs/goqu/v10/sqlgen" + "github.com/rudderlabs/sqlconnect-go/sqlconnect/internal/base" +) + +func GoquDialectOptions() *sqlgen.SQLDialectOptions { + o := sqlgen.DefaultDialectOptions() + o.QuoteIdentifiers = false + o.QuoteRune = '`' + return o +} + +func GoquExpressions() *base.Expressions { + return &base.Expressions{ + TimestampAdd: func(timeValue any, interval int, unit string) goqu.Expression { + return goqu.L(fmt.Sprintf("DATEADD(%s, %d, ?)", strings.ToUpper(unit), interval), timeValue) + }, + DateAdd: func(dateValue any, interval int, unit string) goqu.Expression { + return goqu.L(fmt.Sprintf("DATEADD(%s, %d, DATE(?))", strings.ToUpper(unit), interval), dateValue) + }, + } +} diff --git a/sqlconnect/internal/databricks/goqu_dialect_test.go b/sqlconnect/internal/databricks/goqu_dialect_test.go new file mode 100644 index 0000000..9c9e9f8 --- /dev/null +++ b/sqlconnect/internal/databricks/goqu_dialect_test.go @@ -0,0 +1,48 @@ +package databricks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/rudderlabs/goqu/v10" + "github.com/rudderlabs/goqu/v10/sqlgen" +) + +func TestExpressions(t *testing.T) { + expressions := GoquExpressions() + + t.Run("TimestampAdd", func(t *testing.T) { + t.Run("literal", func(t *testing.T) { + require.Equal(t, "DATEADD(HOUR, 1, CURRENT_DATE)", toSQL(t, expressions.TimestampAdd(goqu.L("CURRENT_DATE"), 1, "hour"))) + require.Equal(t, "DATEADD(DAY, 1, '2020-01-01')", toSQL(t, expressions.TimestampAdd(goqu.L("'2020-01-01'"), 1, "day"))) + }) + + t.Run("time", func(t *testing.T) { + now, err := time.Parse(time.RFC3339, "2020-01-01T00:00:00Z") + require.NoError(t, err) + require.Equal(t, "DATEADD(DAY, 1, '2020-01-01T00:00:00Z')", toSQL(t, expressions.TimestampAdd(now, 1, "day"))) + }) + }) + + t.Run("DateAdd", func(t *testing.T) { + t.Run("literal", func(t *testing.T) { + require.Equal(t, "DATEADD(DAY, 1, DATE(CURRENT_DATE))", toSQL(t, expressions.DateAdd(goqu.L("CURRENT_DATE"), 1, "day"))) + require.Equal(t, "DATEADD(DAY, 1, DATE('2020-01-01'))", toSQL(t, expressions.DateAdd(goqu.L("'2020-01-01'"), 1, "day"))) + }) + + t.Run("time", func(t *testing.T) { + now, err := time.Parse(time.RFC3339, "2020-01-01T00:00:00Z") + require.NoError(t, err) + require.Equal(t, "DATEADD(DAY, 1, DATE('2020-01-01T00:00:00Z'))", toSQL(t, expressions.DateAdd(now, 1, "day"))) + }) + }) +} + +func toSQL(t *testing.T, expression interface{}) string { + esg := sqlgen.NewExpressionSQLGenerator(DatabaseType, GoquDialectOptions()) + sql, _, err := sqlgen.GenerateExpressionSQL(esg, false, expression) + require.NoError(t, err) + return sql +} diff --git a/sqlconnect/internal/databricks/testdata/goqu-test-seed.sql b/sqlconnect/internal/databricks/testdata/goqu-test-seed.sql new file mode 100644 index 0000000..dc76e12 --- /dev/null +++ b/sqlconnect/internal/databricks/testdata/goqu-test-seed.sql @@ -0,0 +1,12 @@ +CREATE TABLE `{{.schema}}`.`goqu_test` ( + _string STRING, + _int INT, + _float NUMERIC(2,1), + _boolean BOOLEAN, + _timestamp TIMESTAMP +); + +INSERT INTO `{{.schema}}`.`goqu_test` + (_string, _int, _float, _boolean, _timestamp) +VALUES + ('string', 1, 1.1, TRUE, '2021-01-01T00:00:00Z'); \ No newline at end of file diff --git a/sqlconnect/internal/integration_test/db_integration_test_scenario.go b/sqlconnect/internal/integration_test/db_integration_test_scenario.go index e15429d..afdc3b1 100644 --- a/sqlconnect/internal/integration_test/db_integration_test_scenario.go +++ b/sqlconnect/internal/integration_test/db_integration_test_scenario.go @@ -19,6 +19,7 @@ import ( "github.com/rudderlabs/rudder-go-kit/testhelper/rand" "github.com/rudderlabs/sqlconnect-go/sqlconnect" + "github.com/rudderlabs/sqlconnect-go/sqlconnect/op" sqlconnectutil "github.com/rudderlabs/sqlconnect-go/sqlconnect/util" ) @@ -143,6 +144,268 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe }) }) + t.Run("goqu dialect", func(t *testing.T) { + table := sqlconnect.NewRelationRef(formatfn("goqu_test"), sqlconnect.WithSchema(schema.Name)) + ExecuteStatements(t, db, schema.Name, "testdata/goqu-test-seed.sql") + + const ( + stringCol = "_string" + stringVal = "string" + + intCol = "_int" + intVal = 1 + + floatCol = "_float" + floatVal = 1.1 + + boolCol = "_boolean" + boolVal = true + + timeCol = "_timestamp" + timeVal = "2021-01-01T00:00:00Z" + ) + timestampVal, err := time.Parse(time.RFC3339, timeVal) + require.NoError(t, err, "it should be able to parse the timestamp value") + + validateCondition := func(t *testing.T, condition string, count int) { + rows, err := db.GetRowCountForQuery(ctx, fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE %s", table, condition)) + require.NoErrorf(t, err, "it should be able to get row count for query with condition %q", condition) + require.Equal(t, count, rows, "it should return %d rows for query with condition %q", count, condition) + } + + getQueryCondition := func(t *testing.T, col, op string, val ...any) string { + sql, err := db.QueryCondition(col, op, val...) + require.NoError(t, err, "it should be able to generate a query condition") + return sql + } + + getTimestampAddExpression := func(t *testing.T, timeValue any, interval int, unit string) any { + expr, err := db.Expressions().TimestampAdd(timeValue, interval, unit) + require.NoError(t, err, "it should be able to generate a time add expression") + return expr + } + + getDateAddExpression := func(t *testing.T, dateValue any, interval int, unit string) any { + expr, err := db.Expressions().DateAdd(dateValue, interval, unit) + require.NoError(t, err, "it should be able to generate a date add expression") + return expr + } + + t.Run("isset", func(t *testing.T) { + op := string(op.IsSet) + rowCount := 1 + validateCondition(t, getQueryCondition(t, stringCol, op), rowCount) + validateCondition(t, getQueryCondition(t, intCol, op), rowCount) + validateCondition(t, getQueryCondition(t, floatCol, op), rowCount) + validateCondition(t, getQueryCondition(t, boolCol, op), rowCount) + validateCondition(t, getQueryCondition(t, timeCol, op), rowCount) + + t.Run("with invalid arguments", func(t *testing.T) { + _, err := db.QueryCondition(stringCol, op, "invalid") + require.Error(t, err, "it should return an error for invalid arguments") + }) + }) + + t.Run("notset", func(t *testing.T) { + op := string(op.NotSet) + rowCount := 0 + validateCondition(t, getQueryCondition(t, stringCol, op), rowCount) + validateCondition(t, getQueryCondition(t, intCol, op), rowCount) + validateCondition(t, getQueryCondition(t, floatCol, op), rowCount) + validateCondition(t, getQueryCondition(t, boolCol, op), rowCount) + validateCondition(t, getQueryCondition(t, timeCol, op), rowCount) + + t.Run("with invalid arguments", func(t *testing.T) { + _, err := db.QueryCondition(stringCol, op, "invalid") + require.Error(t, err, "it should return an error for invalid arguments") + }) + }) + + t.Run("eq", func(t *testing.T) { + op := string(op.Eq) + rowCount := 1 + validateCondition(t, getQueryCondition(t, stringCol, op, stringVal), rowCount) + validateCondition(t, getQueryCondition(t, intCol, op, intVal), rowCount) + validateCondition(t, getQueryCondition(t, floatCol, op, floatVal), rowCount) + validateCondition(t, getQueryCondition(t, boolCol, op, boolVal), rowCount) + validateCondition(t, getQueryCondition(t, timeCol, op, timestampVal), rowCount) + + t.Run("with invalid arguments", func(t *testing.T) { + _, err := db.QueryCondition(stringCol, op, "one", "two") + require.Error(t, err, "it should return an error for invalid arguments") + }) + }) + + t.Run("neq", func(t *testing.T) { + op := string(op.Neq) + rowCount := 0 + validateCondition(t, getQueryCondition(t, stringCol, op, stringVal), rowCount) + validateCondition(t, getQueryCondition(t, intCol, op, intVal), rowCount) + validateCondition(t, getQueryCondition(t, floatCol, op, floatVal), rowCount) + validateCondition(t, getQueryCondition(t, boolCol, op, boolVal), rowCount) + validateCondition(t, getQueryCondition(t, timeCol, op, timestampVal), rowCount) + + t.Run("with invalid arguments", func(t *testing.T) { + _, err := db.QueryCondition(stringCol, op, "one", "two") + require.Error(t, err, "it should return an error for invalid arguments") + }) + }) + + t.Run("in", func(t *testing.T) { + op := string(op.In) + rowCount := 1 + validateCondition(t, getQueryCondition(t, stringCol, op, stringVal), rowCount) + validateCondition(t, getQueryCondition(t, intCol, op, intVal), rowCount) + validateCondition(t, getQueryCondition(t, floatCol, op, floatVal), rowCount) + validateCondition(t, getQueryCondition(t, boolCol, op, boolVal), rowCount) + // in for timestamps is not supported for databricks + // validateCondition(t, getQueryCondition(t, timeCol, op, timestampVal), rowCount) + + t.Run("with invalid arguments", func(t *testing.T) { + _, err := db.QueryCondition(stringCol, op) + require.Error(t, err, "it should return an error for invalid arguments") + }) + }) + + t.Run("notin", func(t *testing.T) { + op := string(op.NotIn) + rowCount := 0 + validateCondition(t, getQueryCondition(t, stringCol, op, stringVal), rowCount) + validateCondition(t, getQueryCondition(t, intCol, op, intVal), rowCount) + validateCondition(t, getQueryCondition(t, floatCol, op, floatVal), rowCount) + validateCondition(t, getQueryCondition(t, boolCol, op, boolVal), rowCount) + // in for timestamps is not supported for databricks + // validateCondition(t, getQueryCondition(t, timeCol, op, timestampVal), rowCount) + + t.Run("with invalid arguments", func(t *testing.T) { + _, err := db.QueryCondition(stringCol, op) + require.Error(t, err, "it should return an error for invalid arguments") + }) + }) + + t.Run("like", func(t *testing.T) { + op := string(op.Like) + rowCount := 1 + validateCondition(t, getQueryCondition(t, stringCol, op, stringVal), rowCount) + + t.Run("with invalid arguments", func(t *testing.T) { + _, err := db.QueryCondition(stringCol, op) + require.Error(t, err, "it should return an error for invalid arguments") + }) + }) + + t.Run("notlike", func(t *testing.T) { + op := string(op.NotLike) + rowCount := 0 + validateCondition(t, getQueryCondition(t, stringCol, op, stringVal), rowCount) + + t.Run("with invalid arguments", func(t *testing.T) { + _, err := db.QueryCondition(stringCol, op) + require.Error(t, err, "it should return an error for invalid arguments") + }) + }) + + t.Run("gt", func(t *testing.T) { + op := string(op.Gt) + rowCount := 1 + validateCondition(t, getQueryCondition(t, intCol, op, intVal-1), rowCount) + validateCondition(t, getQueryCondition(t, floatCol, op, floatVal-1.0), rowCount) + validateCondition(t, getQueryCondition(t, timeCol, op, timestampVal.Add(-1*time.Hour)), rowCount) + + t.Run("with invalid arguments", func(t *testing.T) { + _, err := db.QueryCondition(stringCol, op) + require.Error(t, err, "it should return an error for invalid arguments") + }) + }) + + t.Run("gte", func(t *testing.T) { + op := string(op.Gte) + rowCount := 1 + validateCondition(t, getQueryCondition(t, intCol, op, intVal), rowCount) + validateCondition(t, getQueryCondition(t, floatCol, op, floatVal), rowCount) + validateCondition(t, getQueryCondition(t, timeCol, op, timestampVal), rowCount) + + t.Run("with invalid arguments", func(t *testing.T) { + _, err := db.QueryCondition(stringCol, op) + require.Error(t, err, "it should return an error for invalid arguments") + }) + }) + + t.Run("lt", func(t *testing.T) { + op := string(op.Lt) + rowCount := 1 + validateCondition(t, getQueryCondition(t, intCol, op, intVal+1), rowCount) + validateCondition(t, getQueryCondition(t, floatCol, op, floatVal+1.0), rowCount) + validateCondition(t, getQueryCondition(t, timeCol, op, timestampVal.Add(time.Hour)), rowCount) + + t.Run("with invalid arguments", func(t *testing.T) { + _, err := db.QueryCondition(stringCol, op) + require.Error(t, err, "it should return an error for invalid arguments") + }) + }) + + t.Run("lte", func(t *testing.T) { + op := string(op.Lte) + rowCount := 1 + validateCondition(t, getQueryCondition(t, intCol, op, intVal), rowCount) + validateCondition(t, getQueryCondition(t, floatCol, op, floatVal), rowCount) + validateCondition(t, getQueryCondition(t, timeCol, op, timestampVal), rowCount) + + t.Run("with invalid arguments", func(t *testing.T) { + _, err := db.QueryCondition(stringCol, op) + require.Error(t, err, "it should return an error for invalid arguments") + }) + }) + + t.Run("between", func(t *testing.T) { + op := string(op.Between) + rowCount := 1 + validateCondition(t, getQueryCondition(t, intCol, op, intVal-1, intVal+1), rowCount) + validateCondition(t, getQueryCondition(t, floatCol, op, floatVal-1.0, floatVal+1.0), rowCount) + validateCondition(t, getQueryCondition(t, timeCol, op, timestampVal.Add(-1*time.Hour), timestampVal.Add(time.Hour)), rowCount) + + t.Run("with invalid arguments", func(t *testing.T) { + _, err := db.QueryCondition(stringCol, op) + require.Error(t, err, "it should return an error for invalid arguments") + }) + }) + + t.Run("notbetween", func(t *testing.T) { + op := string(op.NotBetween) + rowCount := 0 + validateCondition(t, getQueryCondition(t, intCol, op, intVal-1, intVal+1), rowCount) + validateCondition(t, getQueryCondition(t, floatCol, op, floatVal-1.0, floatVal+1.0), rowCount) + validateCondition(t, getQueryCondition(t, timeCol, op, timestampVal.Add(-1*time.Hour), timestampVal.Add(time.Hour)), rowCount) + + t.Run("with invalid arguments", func(t *testing.T) { + _, err := db.QueryCondition(stringCol, op) + require.Error(t, err, "it should return an error for invalid arguments") + }) + }) + + t.Run("invalid operator", func(t *testing.T) { + _, err = db.QueryCondition("column", "someop") + require.Error(t, err, "it should return an error for an invalid operator") + require.ErrorContains(t, err, "unsupported operator: someop", "it should return an error for an invalid operator") + }) + + t.Run("time add", func(t *testing.T) { + op := string(op.Lt) + rowCount := 1 + + validateCondition(t, getQueryCondition(t, timeCol, op, getTimestampAddExpression(t, timestampVal, 1, "hour")), rowCount) + validateCondition(t, getQueryCondition(t, timeCol, op, getTimestampAddExpression(t, "CURRENT_TIMESTAMP", -1, "day")), rowCount) + }) + + t.Run("date add", func(t *testing.T) { + op := string(op.Lt) + rowCount := 1 + + validateCondition(t, getQueryCondition(t, "DATE("+timeCol+")", op, getDateAddExpression(t, timestampVal, 1, "day")), rowCount) + validateCondition(t, getQueryCondition(t, "DATE("+timeCol+")", op, getDateAddExpression(t, "CURRENT_TIMESTAMP", -1, "day")), rowCount) + }) + }) + t.Run("dialect", func(t *testing.T) { t.Run("with unquoted table", func(t *testing.T) { identifier := db.QuoteIdentifier(schema.Name) + "." + "UnQuoted_TablE" @@ -291,12 +554,13 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe tableWithCatalog := table tableWithCatalog.Catalog = currentCatalog columns, err := db.ListColumns(ctx, tableWithCatalog) + require.NoErrorf(t, err, "it should be able to list columns for %s", tableWithCatalog) columns = lo.Map(columns, func(col sqlconnect.ColumnRef, _ int) sqlconnect.ColumnRef { require.NotEmptyf(t, col.RawType, "it should return the raw type for column %q", col.Name) col.RawType = "" return col }) - require.NoError(t, err, "it should be able to list columns") + require.Len(t, columns, 2, "it should return the correct number of columns") require.ElementsMatch(t, columns, []sqlconnect.ColumnRef{ {Name: formatfn("c1"), Type: "int"}, diff --git a/sqlconnect/internal/mysql/db.go b/sqlconnect/internal/mysql/db.go index e4e4750..4200135 100644 --- a/sqlconnect/internal/mysql/db.go +++ b/sqlconnect/internal/mysql/db.go @@ -63,7 +63,7 @@ func NewDB(configJSON json.RawMessage) (*DB, error) { } return cmds }), - base.WithDialect(dialect{}), + base.WithDialect(dialect{base.NewGoquDialect(DatabaseType, GoquDialectOptions(), GoquExpressions())}), ), }, nil } diff --git a/sqlconnect/internal/mysql/dialect.go b/sqlconnect/internal/mysql/dialect.go index be9a0ed..3d8b153 100644 --- a/sqlconnect/internal/mysql/dialect.go +++ b/sqlconnect/internal/mysql/dialect.go @@ -7,7 +7,9 @@ import ( "github.com/rudderlabs/sqlconnect-go/sqlconnect/internal/base" ) -type dialect struct{} +type dialect struct { + *base.GoquDialect +} // QuoteTable quotes a table name func (d dialect) QuoteTable(table sqlconnect.RelationRef) string { diff --git a/sqlconnect/internal/mysql/goqu_dialect.go b/sqlconnect/internal/mysql/goqu_dialect.go new file mode 100644 index 0000000..d7882bc --- /dev/null +++ b/sqlconnect/internal/mysql/goqu_dialect.go @@ -0,0 +1,92 @@ +package mysql + +import ( + "fmt" + "strings" + + "github.com/rudderlabs/goqu/v10" + "github.com/rudderlabs/goqu/v10/exp" + "github.com/rudderlabs/goqu/v10/sqlgen" + "github.com/rudderlabs/sqlconnect-go/sqlconnect/internal/base" +) + +func GoquDialectOptions() *sqlgen.SQLDialectOptions { + o := sqlgen.DefaultDialectOptions() + o.QuoteIdentifiers = false + o.SupportsReturn = false + o.SupportsOrderByOnUpdate = true + o.SupportsLimitOnUpdate = true + o.SupportsLimitOnDelete = true + o.SupportsOrderByOnDelete = true + o.SupportsConflictUpdateWhere = false + o.SupportsInsertIgnoreSyntax = true + o.SupportsConflictTarget = false + o.SupportsWithCTE = false + o.SupportsWithCTERecursive = false + o.SupportsDistinctOn = false + o.SupportsWindowFunction = false + o.SupportsDeleteTableHint = true + + o.UseFromClauseForMultipleUpdateTables = false + + o.PlaceHolderFragment = []byte("?") + o.IncludePlaceholderNum = false + o.QuoteRune = '`' + o.DefaultValuesFragment = []byte("") + o.True = []byte("1") + o.False = []byte("0") + o.TimeFormat = "2006-01-02 15:04:05" + o.BooleanOperatorLookup = map[exp.BooleanOperation][]byte{ + exp.EqOp: []byte("="), + exp.NeqOp: []byte("!="), + exp.GtOp: []byte(">"), + exp.GteOp: []byte(">="), + exp.LtOp: []byte("<"), + exp.LteOp: []byte("<="), + exp.InOp: []byte("IN"), + exp.NotInOp: []byte("NOT IN"), + exp.IsOp: []byte("IS"), + exp.IsNotOp: []byte("IS NOT"), + exp.LikeOp: []byte("LIKE BINARY"), + exp.NotLikeOp: []byte("NOT LIKE BINARY"), + exp.ILikeOp: []byte("LIKE"), + exp.NotILikeOp: []byte("NOT LIKE"), + exp.RegexpLikeOp: []byte("REGEXP BINARY"), + exp.RegexpNotLikeOp: []byte("NOT REGEXP BINARY"), + exp.RegexpILikeOp: []byte("REGEXP"), + exp.RegexpNotILikeOp: []byte("NOT REGEXP"), + } + o.BitwiseOperatorLookup = map[exp.BitwiseOperation][]byte{ + exp.BitwiseInversionOp: []byte("~"), + exp.BitwiseOrOp: []byte("|"), + exp.BitwiseAndOp: []byte("&"), + exp.BitwiseXorOp: []byte("^"), + exp.BitwiseLeftShiftOp: []byte("<<"), + exp.BitwiseRightShiftOp: []byte(">>"), + } + o.EscapedRunes = map[rune][]byte{ + '\'': []byte("\\'"), + '"': []byte("\\\""), + '\\': []byte("\\\\"), + '\n': []byte("\\n"), + '\r': []byte("\\r"), + 0: []byte("\\x00"), + 0x1a: []byte("\\x1a"), + } + o.InsertIgnoreClause = []byte("INSERT IGNORE INTO") + o.ConflictFragment = []byte("") + o.ConflictDoUpdateFragment = []byte(" ON DUPLICATE KEY UPDATE ") + o.ConflictDoNothingFragment = []byte("") + return o +} + +func GoquExpressions() *base.Expressions { + return &base.Expressions{ + TimestampAdd: func(timeValue any, interval int, unit string) goqu.Expression { + return goqu.L(fmt.Sprintf("DATE_ADD(?, INTERVAL %d %s)", interval, strings.ToUpper(unit)), timeValue) + }, + DateAdd: func(dateValue any, interval int, unit string) goqu.Expression { + return goqu.L(fmt.Sprintf("DATE_ADD(DATE(?), INTERVAL %d %s)", interval, strings.ToUpper(unit)), dateValue) + }, + } +} diff --git a/sqlconnect/internal/mysql/goqu_dialect_test.go b/sqlconnect/internal/mysql/goqu_dialect_test.go new file mode 100644 index 0000000..2c10edb --- /dev/null +++ b/sqlconnect/internal/mysql/goqu_dialect_test.go @@ -0,0 +1,48 @@ +package mysql + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/rudderlabs/goqu/v10" + "github.com/rudderlabs/goqu/v10/sqlgen" +) + +func TestExpressions(t *testing.T) { + expressions := GoquExpressions() + + t.Run("TimestampAdd", func(t *testing.T) { + t.Run("literal", func(t *testing.T) { + require.Equal(t, "DATE_ADD(CURRENT_DATE, INTERVAL 1 HOUR)", toSQL(t, expressions.TimestampAdd(goqu.L("CURRENT_DATE"), 1, "hour"))) + require.Equal(t, "DATE_ADD('2020-01-01', INTERVAL 1 DAY)", toSQL(t, expressions.TimestampAdd(goqu.L("'2020-01-01'"), 1, "day"))) + }) + + t.Run("time", func(t *testing.T) { + now, err := time.Parse(time.RFC3339, "2020-01-01T00:00:00Z") + require.NoError(t, err) + require.Equal(t, "DATE_ADD('2020-01-01 00:00:00', INTERVAL 1 DAY)", toSQL(t, expressions.TimestampAdd(now, 1, "day"))) + }) + }) + + t.Run("DateAdd", func(t *testing.T) { + t.Run("literal", func(t *testing.T) { + require.Equal(t, "DATE_ADD(DATE(CURRENT_DATE), INTERVAL 1 DAY)", toSQL(t, expressions.DateAdd(goqu.L("CURRENT_DATE"), 1, "day"))) + require.Equal(t, "DATE_ADD(DATE('2020-01-01'), INTERVAL 1 DAY)", toSQL(t, expressions.DateAdd(goqu.L("'2020-01-01'"), 1, "day"))) + }) + + t.Run("time", func(t *testing.T) { + now, err := time.Parse(time.RFC3339, "2020-01-01T00:00:00Z") + require.NoError(t, err) + require.Equal(t, "DATE_ADD(DATE('2020-01-01 00:00:00'), INTERVAL 1 DAY)", toSQL(t, expressions.DateAdd(now, 1, "day"))) + }) + }) +} + +func toSQL(t *testing.T, expression interface{}) string { + esg := sqlgen.NewExpressionSQLGenerator(DatabaseType, GoquDialectOptions()) + sql, _, err := sqlgen.GenerateExpressionSQL(esg, false, expression) + require.NoError(t, err) + return sql +} diff --git a/sqlconnect/internal/mysql/testdata/goqu-test-seed.sql b/sqlconnect/internal/mysql/testdata/goqu-test-seed.sql new file mode 100644 index 0000000..5192802 --- /dev/null +++ b/sqlconnect/internal/mysql/testdata/goqu-test-seed.sql @@ -0,0 +1,12 @@ +CREATE TABLE `{{.schema}}`.`goqu_test` ( + _string VARCHAR(10), + _int INT, + _float NUMERIC(2,1), + _boolean BOOLEAN, + _timestamp TIMESTAMP +); + +INSERT INTO `{{.schema}}`.`goqu_test` + (_string, _int, _float, _boolean, _timestamp) +VALUES + ('string', 1, 1.1, TRUE, '2021-01-01 00:00:00'); \ No newline at end of file diff --git a/sqlconnect/internal/postgres/db.go b/sqlconnect/internal/postgres/db.go index 2671b9c..b8f6847 100644 --- a/sqlconnect/internal/postgres/db.go +++ b/sqlconnect/internal/postgres/db.go @@ -44,6 +44,7 @@ func NewDB(credentialsJSON json.RawMessage) (*DB, error) { DB: base.NewDB( db, tunnelCloser, + base.WithGoquDialect(base.NewGoquDialect(DatabaseType, GoquDialectOptions(), GoquExpressions())), base.WithColumnTypeMappings(getColumnTypeMappings(config)), base.WithJsonRowMapper(getJonRowMapper(config)), ), diff --git a/sqlconnect/internal/postgres/goqu_dialect.go b/sqlconnect/internal/postgres/goqu_dialect.go new file mode 100644 index 0000000..d3376d2 --- /dev/null +++ b/sqlconnect/internal/postgres/goqu_dialect.go @@ -0,0 +1,29 @@ +package postgres + +import ( + "fmt" + + "github.com/rudderlabs/goqu/v10" + "github.com/rudderlabs/goqu/v10/sqlgen" + "github.com/rudderlabs/sqlconnect-go/sqlconnect/internal/base" +) + +func GoquDialectOptions() *sqlgen.SQLDialectOptions { + o := sqlgen.DefaultDialectOptions() + o.QuoteIdentifiers = false + o.QuoteRune = '"' + o.PlaceHolderFragment = []byte("$") + o.IncludePlaceholderNum = true + return o +} + +func GoquExpressions() *base.Expressions { + return &base.Expressions{ + TimestampAdd: func(timeValue any, interval int, unit string) goqu.Expression { + return goqu.L(fmt.Sprintf("(?::TIMESTAMP + INTERVAL '%d %s')", interval, unit), timeValue) + }, + DateAdd: func(dateValue any, interval int, unit string) goqu.Expression { + return goqu.L(fmt.Sprintf("(DATE(?::TIMESTAMP) + INTERVAL '%d %s')", interval, unit), dateValue) + }, + } +} diff --git a/sqlconnect/internal/postgres/goqu_dialect_test.go b/sqlconnect/internal/postgres/goqu_dialect_test.go new file mode 100644 index 0000000..47c2af8 --- /dev/null +++ b/sqlconnect/internal/postgres/goqu_dialect_test.go @@ -0,0 +1,48 @@ +package postgres + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/rudderlabs/goqu/v10" + "github.com/rudderlabs/goqu/v10/sqlgen" +) + +func TestExpressions(t *testing.T) { + expressions := GoquExpressions() + + t.Run("TimestampAdd", func(t *testing.T) { + t.Run("literal", func(t *testing.T) { + require.Equal(t, "(CURRENT_DATE::TIMESTAMP + INTERVAL '1 hour')", toSQL(t, expressions.TimestampAdd(goqu.L("CURRENT_DATE"), 1, "hour"))) + require.Equal(t, "('2020-01-01'::TIMESTAMP + INTERVAL '1 day')", toSQL(t, expressions.TimestampAdd(goqu.L("'2020-01-01'"), 1, "day"))) + }) + + t.Run("time", func(t *testing.T) { + now, err := time.Parse(time.RFC3339, "2020-01-01T00:00:00Z") + require.NoError(t, err) + require.Equal(t, "('2020-01-01T00:00:00Z'::TIMESTAMP + INTERVAL '1 day')", toSQL(t, expressions.TimestampAdd(now, 1, "day"))) + }) + }) + + t.Run("DateAdd", func(t *testing.T) { + t.Run("literal", func(t *testing.T) { + require.Equal(t, "(DATE(CURRENT_DATE::TIMESTAMP) + INTERVAL '1 day')", toSQL(t, expressions.DateAdd(goqu.L("CURRENT_DATE"), 1, "day"))) + require.Equal(t, "(DATE('2020-01-01'::TIMESTAMP) + INTERVAL '1 day')", toSQL(t, expressions.DateAdd(goqu.L("'2020-01-01'"), 1, "day"))) + }) + + t.Run("time", func(t *testing.T) { + now, err := time.Parse(time.RFC3339, "2020-01-01T00:00:00Z") + require.NoError(t, err) + require.Equal(t, "(DATE('2020-01-01T00:00:00Z'::TIMESTAMP) + INTERVAL '1 day')", toSQL(t, expressions.DateAdd(now, 1, "day"))) + }) + }) +} + +func toSQL(t *testing.T, expression interface{}) string { + esg := sqlgen.NewExpressionSQLGenerator(DatabaseType, GoquDialectOptions()) + sql, _, err := sqlgen.GenerateExpressionSQL(esg, false, expression) + require.NoError(t, err) + return sql +} diff --git a/sqlconnect/internal/postgres/testdata/goqu-test-seed.sql b/sqlconnect/internal/postgres/testdata/goqu-test-seed.sql new file mode 100644 index 0000000..d01b884 --- /dev/null +++ b/sqlconnect/internal/postgres/testdata/goqu-test-seed.sql @@ -0,0 +1,12 @@ +CREATE TABLE "{{.schema}}"."goqu_test" ( + _string VARCHAR(10), + _int INT, + _float NUMERIC(2,1), + _boolean BOOLEAN, + _timestamp TIMESTAMP +); + +INSERT INTO "{{.schema}}"."goqu_test" + (_string, _int, _float, _boolean, _timestamp) +VALUES + ('string', 1, 1.1, TRUE, '2021-01-01T00:00:00Z'); \ No newline at end of file diff --git a/sqlconnect/internal/redshift/db.go b/sqlconnect/internal/redshift/db.go index 999a15b..5bbb11c 100644 --- a/sqlconnect/internal/redshift/db.go +++ b/sqlconnect/internal/redshift/db.go @@ -52,7 +52,7 @@ func NewDB(credentialsJSON json.RawMessage) (*DB, error) { DB: base.NewDB( db, tunnelCloser, - base.WithDialect(dialect{caseSensitive: caseSensitive == "on"}), + base.WithDialect(dialect{GoquDialect: base.NewGoquDialect(DatabaseType, GoquDialectOptions(), GoquExpressions()), caseSensitive: caseSensitive == "on"}), base.WithColumnTypeMappings(getColumnTypeMappings(useLegacyMappings)), base.WithJsonRowMapper(getJonRowMapper(useLegacyMappings)), base.WithSQLCommandsOverride(func(cmds base.SQLCommands) base.SQLCommands { diff --git a/sqlconnect/internal/redshift/dialect.go b/sqlconnect/internal/redshift/dialect.go index 118e01d..37045ed 100644 --- a/sqlconnect/internal/redshift/dialect.go +++ b/sqlconnect/internal/redshift/dialect.go @@ -9,6 +9,7 @@ import ( ) type dialect struct { + *base.GoquDialect caseSensitive bool } diff --git a/sqlconnect/internal/redshift/goqu_dialect.go b/sqlconnect/internal/redshift/goqu_dialect.go new file mode 100644 index 0000000..eb0d97f --- /dev/null +++ b/sqlconnect/internal/redshift/goqu_dialect.go @@ -0,0 +1,27 @@ +package redshift + +import ( + "fmt" + + "github.com/rudderlabs/goqu/v10" + "github.com/rudderlabs/goqu/v10/sqlgen" + "github.com/rudderlabs/sqlconnect-go/sqlconnect/internal/base" +) + +func GoquDialectOptions() *sqlgen.SQLDialectOptions { + o := sqlgen.DefaultDialectOptions() + o.QuoteIdentifiers = false + o.QuoteRune = '"' + return o +} + +func GoquExpressions() *base.Expressions { + return &base.Expressions{ + TimestampAdd: func(timeValue any, interval int, unit string) goqu.Expression { + return goqu.L(fmt.Sprintf("DATEADD(%s, %d, CAST(? AS TIMESTAMP))", unit, interval), timeValue) + }, + DateAdd: func(dateValue any, interval int, unit string) goqu.Expression { + return goqu.L(fmt.Sprintf("DATEADD(%s, %d, DATE(CAST(? AS TIMESTAMP)))", unit, interval), dateValue) + }, + } +} diff --git a/sqlconnect/internal/redshift/goqu_dialect_test.go b/sqlconnect/internal/redshift/goqu_dialect_test.go new file mode 100644 index 0000000..1dad96c --- /dev/null +++ b/sqlconnect/internal/redshift/goqu_dialect_test.go @@ -0,0 +1,48 @@ +package redshift + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/rudderlabs/goqu/v10" + "github.com/rudderlabs/goqu/v10/sqlgen" +) + +func TestExpressions(t *testing.T) { + expressions := GoquExpressions() + + t.Run("TimestampAdd", func(t *testing.T) { + t.Run("literal", func(t *testing.T) { + require.Equal(t, "DATEADD(hour, 1, CAST(CURRENT_DATE AS TIMESTAMP))", toSQL(t, expressions.TimestampAdd(goqu.L("CURRENT_DATE"), 1, "hour"))) + require.Equal(t, "DATEADD(day, 1, CAST('2020-01-01' AS TIMESTAMP))", toSQL(t, expressions.TimestampAdd(goqu.L("'2020-01-01'"), 1, "day"))) + }) + + t.Run("time", func(t *testing.T) { + now, err := time.Parse(time.RFC3339, "2020-01-01T00:00:00Z") + require.NoError(t, err) + require.Equal(t, "DATEADD(day, 1, CAST('2020-01-01T00:00:00Z' AS TIMESTAMP))", toSQL(t, expressions.TimestampAdd(now, 1, "day"))) + }) + }) + + t.Run("DateAdd", func(t *testing.T) { + t.Run("literal", func(t *testing.T) { + require.Equal(t, "DATEADD(day, 1, DATE(CAST(CURRENT_DATE AS TIMESTAMP)))", toSQL(t, expressions.DateAdd(goqu.L("CURRENT_DATE"), 1, "day"))) + require.Equal(t, "DATEADD(day, 1, DATE(CAST('2020-01-01' AS TIMESTAMP)))", toSQL(t, expressions.DateAdd(goqu.L("'2020-01-01'"), 1, "day"))) + }) + + t.Run("time", func(t *testing.T) { + now, err := time.Parse(time.RFC3339, "2020-01-01T00:00:00Z") + require.NoError(t, err) + require.Equal(t, "DATEADD(day, 1, DATE(CAST('2020-01-01T00:00:00Z' AS TIMESTAMP)))", toSQL(t, expressions.DateAdd(now, 1, "day"))) + }) + }) +} + +func toSQL(t *testing.T, expression interface{}) string { + esg := sqlgen.NewExpressionSQLGenerator(DatabaseType, GoquDialectOptions()) + sql, _, err := sqlgen.GenerateExpressionSQL(esg, false, expression) + require.NoError(t, err) + return sql +} diff --git a/sqlconnect/internal/redshift/testdata/goqu-test-seed.sql b/sqlconnect/internal/redshift/testdata/goqu-test-seed.sql new file mode 100644 index 0000000..d01b884 --- /dev/null +++ b/sqlconnect/internal/redshift/testdata/goqu-test-seed.sql @@ -0,0 +1,12 @@ +CREATE TABLE "{{.schema}}"."goqu_test" ( + _string VARCHAR(10), + _int INT, + _float NUMERIC(2,1), + _boolean BOOLEAN, + _timestamp TIMESTAMP +); + +INSERT INTO "{{.schema}}"."goqu_test" + (_string, _int, _float, _boolean, _timestamp) +VALUES + ('string', 1, 1.1, TRUE, '2021-01-01T00:00:00Z'); \ No newline at end of file diff --git a/sqlconnect/internal/snowflake/db.go b/sqlconnect/internal/snowflake/db.go index f73d821..df6e28c 100644 --- a/sqlconnect/internal/snowflake/db.go +++ b/sqlconnect/internal/snowflake/db.go @@ -38,7 +38,7 @@ func NewDB(configJSON json.RawMessage) (*DB, error) { DB: base.NewDB( db, sshtunnel.NoTunnelCloser, - base.WithDialect(dialect{}), + base.WithDialect(dialect{base.NewGoquDialect(DatabaseType, GoquDialectOptions(), GoquExpressions())}), base.WithColumnTypeMapper(getColumnTypeMapper(config)), base.WithJsonRowMapper(getJonRowMapper(config)), base.WithSQLCommandsOverride(func(cmds base.SQLCommands) base.SQLCommands { diff --git a/sqlconnect/internal/snowflake/dialect.go b/sqlconnect/internal/snowflake/dialect.go index 60f5bf3..4e8c4f0 100644 --- a/sqlconnect/internal/snowflake/dialect.go +++ b/sqlconnect/internal/snowflake/dialect.go @@ -7,7 +7,9 @@ import ( "github.com/rudderlabs/sqlconnect-go/sqlconnect/internal/base" ) -type dialect struct{} +type dialect struct { + *base.GoquDialect +} // QuoteTable quotes a table name func (d dialect) QuoteTable(table sqlconnect.RelationRef) string { diff --git a/sqlconnect/internal/snowflake/goqu_dialect.go b/sqlconnect/internal/snowflake/goqu_dialect.go new file mode 100644 index 0000000..17c2718 --- /dev/null +++ b/sqlconnect/internal/snowflake/goqu_dialect.go @@ -0,0 +1,28 @@ +package snowflake + +import ( + "fmt" + + "github.com/rudderlabs/goqu/v10" + "github.com/rudderlabs/goqu/v10/sqlgen" + "github.com/rudderlabs/sqlconnect-go/sqlconnect/internal/base" +) + +func GoquDialectOptions() *sqlgen.SQLDialectOptions { + o := sqlgen.DefaultDialectOptions() + o.QuoteIdentifiers = false + o.QuoteRune = '"' + o.UseEqForBooleanDataTypes = true + return o +} + +func GoquExpressions() *base.Expressions { + return &base.Expressions{ + TimestampAdd: func(timeValue any, interval int, unit string) goqu.Expression { + return goqu.L(fmt.Sprintf("DATEADD(%s, %d, ?)", unit, interval), timeValue) + }, + DateAdd: func(timeValue any, interval int, unit string) goqu.Expression { + return goqu.L(fmt.Sprintf("DATEADD(%s, %d, DATE(?))", unit, interval), timeValue) + }, + } +} diff --git a/sqlconnect/internal/snowflake/goqu_dialect_test.go b/sqlconnect/internal/snowflake/goqu_dialect_test.go new file mode 100644 index 0000000..6759bc3 --- /dev/null +++ b/sqlconnect/internal/snowflake/goqu_dialect_test.go @@ -0,0 +1,48 @@ +package snowflake + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/rudderlabs/goqu/v10" + "github.com/rudderlabs/goqu/v10/sqlgen" +) + +func TestExpressions(t *testing.T) { + expressions := GoquExpressions() + + t.Run("TimestampAdd", func(t *testing.T) { + t.Run("literal", func(t *testing.T) { + require.Equal(t, "DATEADD(hour, 1, CURRENT_DATE)", toSQL(t, expressions.TimestampAdd(goqu.L("CURRENT_DATE"), 1, "hour"))) + require.Equal(t, "DATEADD(day, 1, '2020-01-01')", toSQL(t, expressions.TimestampAdd(goqu.L("'2020-01-01'"), 1, "day"))) + }) + + t.Run("time", func(t *testing.T) { + now, err := time.Parse(time.RFC3339, "2020-01-01T00:00:00Z") + require.NoError(t, err) + require.Equal(t, "DATEADD(day, 1, '2020-01-01T00:00:00Z')", toSQL(t, expressions.TimestampAdd(now, 1, "day"))) + }) + }) + + t.Run("DateAdd", func(t *testing.T) { + t.Run("literal", func(t *testing.T) { + require.Equal(t, "DATEADD(day, 1, DATE(CURRENT_DATE))", toSQL(t, expressions.DateAdd(goqu.L("CURRENT_DATE"), 1, "day"))) + require.Equal(t, "DATEADD(day, 1, DATE('2020-01-01'))", toSQL(t, expressions.DateAdd(goqu.L("'2020-01-01'"), 1, "day"))) + }) + + t.Run("time", func(t *testing.T) { + now, err := time.Parse(time.RFC3339, "2020-01-01T00:00:00Z") + require.NoError(t, err) + require.Equal(t, "DATEADD(day, 1, DATE('2020-01-01T00:00:00Z'))", toSQL(t, expressions.DateAdd(now, 1, "day"))) + }) + }) +} + +func toSQL(t *testing.T, expression interface{}) string { + esg := sqlgen.NewExpressionSQLGenerator(DatabaseType, GoquDialectOptions()) + sql, _, err := sqlgen.GenerateExpressionSQL(esg, false, expression) + require.NoError(t, err) + return sql +} diff --git a/sqlconnect/internal/snowflake/testdata/goqu-test-seed.sql b/sqlconnect/internal/snowflake/testdata/goqu-test-seed.sql new file mode 100644 index 0000000..f835100 --- /dev/null +++ b/sqlconnect/internal/snowflake/testdata/goqu-test-seed.sql @@ -0,0 +1,12 @@ +CREATE TABLE "{{.schema}}"."GOQU_TEST" ( + _string VARCHAR(10), + _int INT, + _float NUMERIC(2,1), + _boolean BOOLEAN, + _timestamp TIMESTAMP +); + +INSERT INTO "{{.schema}}"."GOQU_TEST" + (_string, _int, _float, _boolean, _timestamp) +VALUES + ('string', 1, 1.1, TRUE, '2021-01-01T00:00:00Z'); \ No newline at end of file diff --git a/sqlconnect/internal/trino/db.go b/sqlconnect/internal/trino/db.go index c3cdd95..2c1d274 100644 --- a/sqlconnect/internal/trino/db.go +++ b/sqlconnect/internal/trino/db.go @@ -45,7 +45,7 @@ func NewDB(configJSON json.RawMessage) (*DB, error) { DB: base.NewDB( db, tunnelCloser, - base.WithDialect(dialect{}), + base.WithDialect(dialect{base.NewGoquDialect(DatabaseType, GoquDialectOptions(), GoquExpressions())}), base.WithColumnTypeMapper(columnTypeMapper), base.WithJsonRowMapper(jsonRowMapper), base.WithSQLCommandsOverride(func(cmds base.SQLCommands) base.SQLCommands { diff --git a/sqlconnect/internal/trino/dialect.go b/sqlconnect/internal/trino/dialect.go index 0d870c9..e4e6dec 100644 --- a/sqlconnect/internal/trino/dialect.go +++ b/sqlconnect/internal/trino/dialect.go @@ -8,7 +8,9 @@ import ( "github.com/rudderlabs/sqlconnect-go/sqlconnect/internal/base" ) -type dialect struct{} +type dialect struct { + *base.GoquDialect +} // QuoteTable quotes a table name func (d dialect) QuoteTable(table sqlconnect.RelationRef) string { diff --git a/sqlconnect/internal/trino/goqu_dialect.go b/sqlconnect/internal/trino/goqu_dialect.go new file mode 100644 index 0000000..c7946d0 --- /dev/null +++ b/sqlconnect/internal/trino/goqu_dialect.go @@ -0,0 +1,43 @@ +package trino + +import ( + "fmt" + "strings" + + "github.com/rudderlabs/goqu/v10" + "github.com/rudderlabs/goqu/v10/exp" + "github.com/rudderlabs/goqu/v10/sqlgen" + "github.com/rudderlabs/sqlconnect-go/sqlconnect/internal/base" +) + +func GoquDialectOptions() *sqlgen.SQLDialectOptions { + o := sqlgen.DefaultDialectOptions() + o.QuoteIdentifiers = false + o.UseEqForBooleanDataTypes = true + o.QuoteRune = '"' + o.TimeFunctionLiteral = "from_iso8601_timestamp(?)" + return o +} + +func GoquExpressions() *base.Expressions { + return &base.Expressions{ + TimestampAdd: func(timeValue any, interval int, unit string) goqu.Expression { + switch tt := timeValue.(type) { + case exp.LiteralExpression: + if strings.HasPrefix(tt.Literal(), "'") && strings.HasSuffix(tt.Literal(), "'") { + timeValue = goqu.L(fmt.Sprintf("TIMESTAMP %s", tt.Literal()), tt.Args()...) + } + } + return goqu.L(fmt.Sprintf("DATE_ADD('%s', %d, ?)", unit, interval), timeValue) + }, + DateAdd: func(dateValue any, interval int, unit string) goqu.Expression { + switch dd := dateValue.(type) { + case exp.LiteralExpression: + if strings.HasPrefix(dd.Literal(), "'") && strings.HasSuffix(dd.Literal(), "'") { + dateValue = goqu.L(fmt.Sprintf("TIMESTAMP %s", dd.Literal()), dd.Args()...) + } + } + return goqu.L(fmt.Sprintf("DATE_ADD('%s', %d, DATE(?))", unit, interval), dateValue) + }, + } +} diff --git a/sqlconnect/internal/trino/goqu_dialect_test.go b/sqlconnect/internal/trino/goqu_dialect_test.go new file mode 100644 index 0000000..527f231 --- /dev/null +++ b/sqlconnect/internal/trino/goqu_dialect_test.go @@ -0,0 +1,48 @@ +package trino + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/rudderlabs/goqu/v10" + "github.com/rudderlabs/goqu/v10/sqlgen" +) + +func TestExpressions(t *testing.T) { + expressions := GoquExpressions() + + t.Run("TimestampAdd", func(t *testing.T) { + t.Run("literal", func(t *testing.T) { + require.Equal(t, "DATE_ADD('hour', 1, CURRENT_DATE)", toSQL(t, expressions.TimestampAdd(goqu.L("CURRENT_DATE"), 1, "hour"))) + require.Equal(t, "DATE_ADD('day', 1, TIMESTAMP '2020-01-01')", toSQL(t, expressions.TimestampAdd(goqu.L("'2020-01-01'"), 1, "day"))) + }) + + t.Run("time", func(t *testing.T) { + now, err := time.Parse(time.RFC3339, "2020-01-01T00:00:00Z") + require.NoError(t, err) + require.Equal(t, "DATE_ADD('day', 1, from_iso8601_timestamp('2020-01-01T00:00:00Z'))", toSQL(t, expressions.TimestampAdd(now, 1, "day"))) + }) + }) + + t.Run("DateAdd", func(t *testing.T) { + t.Run("literal", func(t *testing.T) { + require.Equal(t, "DATE_ADD('day', 1, DATE(CURRENT_DATE))", toSQL(t, expressions.DateAdd(goqu.L("CURRENT_DATE"), 1, "day"))) + require.Equal(t, "DATE_ADD('day', 1, DATE(TIMESTAMP '2020-01-01'))", toSQL(t, expressions.DateAdd(goqu.L("'2020-01-01'"), 1, "day"))) + }) + + t.Run("time", func(t *testing.T) { + now, err := time.Parse(time.RFC3339, "2020-01-01T00:00:00Z") + require.NoError(t, err) + require.Equal(t, "DATE_ADD('day', 1, DATE(from_iso8601_timestamp('2020-01-01T00:00:00Z')))", toSQL(t, expressions.DateAdd(now, 1, "day"))) + }) + }) +} + +func toSQL(t *testing.T, expression interface{}) string { + esg := sqlgen.NewExpressionSQLGenerator(DatabaseType, GoquDialectOptions()) + sql, _, err := sqlgen.GenerateExpressionSQL(esg, false, expression) + require.NoError(t, err) + return sql +} diff --git a/sqlconnect/internal/trino/testdata/goqu-test-seed.sql b/sqlconnect/internal/trino/testdata/goqu-test-seed.sql new file mode 100644 index 0000000..0bcb956 --- /dev/null +++ b/sqlconnect/internal/trino/testdata/goqu-test-seed.sql @@ -0,0 +1,12 @@ +CREATE TABLE "{{.schema}}"."goqu_test" ( + _string VARCHAR(10), + _int INT, + _float DECIMAL(2,1), + _boolean BOOLEAN, + _timestamp TIMESTAMP +); + +INSERT INTO "{{.schema}}"."goqu_test" + (_string, _int, _float, _boolean, _timestamp) +VALUES + ('string', 1, 1.1, TRUE, from_iso8601_timestamp('2021-01-01T00:00:00Z')); \ No newline at end of file diff --git a/sqlconnect/op/operator.go b/sqlconnect/op/operator.go new file mode 100644 index 0000000..ae93f95 --- /dev/null +++ b/sqlconnect/op/operator.go @@ -0,0 +1,45 @@ +package op + +type Operator string + +const ( + // = + Eq Operator = "eq" + // != + Neq Operator = "neq" + // IN (...) + In Operator = "in" + // NOT IN (...) + NotIn Operator = "notin" + + // > + Gt Operator = "gt" + // >= + Gte Operator = "gte" + // < + Lt Operator = "lt" + // <= + Lte Operator = "lte" + + // LIKE + Like Operator = "like" + // NOT LIKE + NotLike Operator = "notlike" + // left <= v <= right + Between Operator = "between" + // v < left OR v > right + NotBetween Operator = "notbetween" + + // IS NOT NULL + IsSet Operator = "isset" + // IS NULL + NotSet Operator = "notset" +) + +func IsValid(op Operator) bool { + switch op { + case Eq, Neq, In, NotIn, Gt, Gte, Lt, Lte, Like, NotLike, Between, NotBetween, IsSet, NotSet: + return true + } + return false +} diff --git a/sqlconnect/querydef.go b/sqlconnect/querydef.go index fc22541..ff80fd6 100644 --- a/sqlconnect/querydef.go +++ b/sqlconnect/querydef.go @@ -30,7 +30,7 @@ type QueryOrder struct { func (query *QueryDef) ToSQL(d Dialect) string { var cols string - if query.Columns == nil || len(query.Columns) == 0 { + if len(query.Columns) == 0 { cols = "*" } else { for i, column := range query.Columns { diff --git a/sqlconnect/querydef_test.go b/sqlconnect/querydef_test.go index 024c699..9e4e391 100644 --- a/sqlconnect/querydef_test.go +++ b/sqlconnect/querydef_test.go @@ -48,7 +48,9 @@ func TestQueryDef(t *testing.T) { }) } -type testDialect struct{} +type testDialect struct { + *base.GoquDialect +} func (d testDialect) FormatTableName(name string) string { return name From d8671aef806ec4dae61bc2a358be9341e339ff39 Mon Sep 17 00:00:00 2001 From: Aris Tzoumas Date: Tue, 10 Sep 2024 15:24:20 +0300 Subject: [PATCH 2/4] fixup! feat: support dialect-aware query conditions --- sqlconnect/internal/base/goqu_dialect.go | 2 +- sqlconnect/internal/postgres/goqu_dialect.go | 2 +- sqlconnect/internal/postgres/goqu_dialect_test.go | 6 +++--- sqlconnect/internal/redshift/goqu_dialect.go | 2 +- sqlconnect/internal/redshift/goqu_dialect_test.go | 6 +++--- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sqlconnect/internal/base/goqu_dialect.go b/sqlconnect/internal/base/goqu_dialect.go index 98fc669..3efca20 100644 --- a/sqlconnect/internal/base/goqu_dialect.go +++ b/sqlconnect/internal/base/goqu_dialect.go @@ -30,7 +30,7 @@ type Expressions struct { func (gq *GoquDialect) QueryCondition(identifier, operator string, args ...any) (sql string, err error) { args = lo.Map(args, func(a any, _ int) any { - if s, ok := a.(sqlconnect.Expression); ok { + if s, ok := a.(sqlconnect.Expression); ok { // unwrap sqlconnect.Expression return s.GoquExpression() } return a diff --git a/sqlconnect/internal/postgres/goqu_dialect.go b/sqlconnect/internal/postgres/goqu_dialect.go index d3376d2..8202cb2 100644 --- a/sqlconnect/internal/postgres/goqu_dialect.go +++ b/sqlconnect/internal/postgres/goqu_dialect.go @@ -23,7 +23,7 @@ func GoquExpressions() *base.Expressions { return goqu.L(fmt.Sprintf("(?::TIMESTAMP + INTERVAL '%d %s')", interval, unit), timeValue) }, DateAdd: func(dateValue any, interval int, unit string) goqu.Expression { - return goqu.L(fmt.Sprintf("(DATE(?::TIMESTAMP) + INTERVAL '%d %s')", interval, unit), dateValue) + return goqu.L(fmt.Sprintf("(DATE(?) + INTERVAL '%d %s')", interval, unit), dateValue) }, } } diff --git a/sqlconnect/internal/postgres/goqu_dialect_test.go b/sqlconnect/internal/postgres/goqu_dialect_test.go index 47c2af8..633d7e1 100644 --- a/sqlconnect/internal/postgres/goqu_dialect_test.go +++ b/sqlconnect/internal/postgres/goqu_dialect_test.go @@ -28,14 +28,14 @@ func TestExpressions(t *testing.T) { t.Run("DateAdd", func(t *testing.T) { t.Run("literal", func(t *testing.T) { - require.Equal(t, "(DATE(CURRENT_DATE::TIMESTAMP) + INTERVAL '1 day')", toSQL(t, expressions.DateAdd(goqu.L("CURRENT_DATE"), 1, "day"))) - require.Equal(t, "(DATE('2020-01-01'::TIMESTAMP) + INTERVAL '1 day')", toSQL(t, expressions.DateAdd(goqu.L("'2020-01-01'"), 1, "day"))) + require.Equal(t, "(DATE(CURRENT_DATE) + INTERVAL '1 day')", toSQL(t, expressions.DateAdd(goqu.L("CURRENT_DATE"), 1, "day"))) + require.Equal(t, "(DATE('2020-01-01') + INTERVAL '1 day')", toSQL(t, expressions.DateAdd(goqu.L("'2020-01-01'"), 1, "day"))) }) t.Run("time", func(t *testing.T) { now, err := time.Parse(time.RFC3339, "2020-01-01T00:00:00Z") require.NoError(t, err) - require.Equal(t, "(DATE('2020-01-01T00:00:00Z'::TIMESTAMP) + INTERVAL '1 day')", toSQL(t, expressions.DateAdd(now, 1, "day"))) + require.Equal(t, "(DATE('2020-01-01T00:00:00Z') + INTERVAL '1 day')", toSQL(t, expressions.DateAdd(now, 1, "day"))) }) }) } diff --git a/sqlconnect/internal/redshift/goqu_dialect.go b/sqlconnect/internal/redshift/goqu_dialect.go index eb0d97f..aab9574 100644 --- a/sqlconnect/internal/redshift/goqu_dialect.go +++ b/sqlconnect/internal/redshift/goqu_dialect.go @@ -21,7 +21,7 @@ func GoquExpressions() *base.Expressions { return goqu.L(fmt.Sprintf("DATEADD(%s, %d, CAST(? AS TIMESTAMP))", unit, interval), timeValue) }, DateAdd: func(dateValue any, interval int, unit string) goqu.Expression { - return goqu.L(fmt.Sprintf("DATEADD(%s, %d, DATE(CAST(? AS TIMESTAMP)))", unit, interval), dateValue) + return goqu.L(fmt.Sprintf("DATEADD(%s, %d, DATE(?))", unit, interval), dateValue) }, } } diff --git a/sqlconnect/internal/redshift/goqu_dialect_test.go b/sqlconnect/internal/redshift/goqu_dialect_test.go index 1dad96c..1255c3e 100644 --- a/sqlconnect/internal/redshift/goqu_dialect_test.go +++ b/sqlconnect/internal/redshift/goqu_dialect_test.go @@ -28,14 +28,14 @@ func TestExpressions(t *testing.T) { t.Run("DateAdd", func(t *testing.T) { t.Run("literal", func(t *testing.T) { - require.Equal(t, "DATEADD(day, 1, DATE(CAST(CURRENT_DATE AS TIMESTAMP)))", toSQL(t, expressions.DateAdd(goqu.L("CURRENT_DATE"), 1, "day"))) - require.Equal(t, "DATEADD(day, 1, DATE(CAST('2020-01-01' AS TIMESTAMP)))", toSQL(t, expressions.DateAdd(goqu.L("'2020-01-01'"), 1, "day"))) + require.Equal(t, "DATEADD(day, 1, DATE(CURRENT_DATE))", toSQL(t, expressions.DateAdd(goqu.L("CURRENT_DATE"), 1, "day"))) + require.Equal(t, "DATEADD(day, 1, DATE('2020-01-01'))", toSQL(t, expressions.DateAdd(goqu.L("'2020-01-01'"), 1, "day"))) }) t.Run("time", func(t *testing.T) { now, err := time.Parse(time.RFC3339, "2020-01-01T00:00:00Z") require.NoError(t, err) - require.Equal(t, "DATEADD(day, 1, DATE(CAST('2020-01-01T00:00:00Z' AS TIMESTAMP)))", toSQL(t, expressions.DateAdd(now, 1, "day"))) + require.Equal(t, "DATEADD(day, 1, DATE('2020-01-01T00:00:00Z'))", toSQL(t, expressions.DateAdd(now, 1, "day"))) }) }) } From 2308aefcb2015dcc11b2a2d4cf6f8df753bb29bf Mon Sep 17 00:00:00 2001 From: Aris Tzoumas Date: Thu, 12 Sep 2024 09:52:49 +0300 Subject: [PATCH 3/4] chore: add nbfinterval operation --- sqlconnect/internal/base/goqu_dialect.go | 20 +++++++++++++++++++ .../db_integration_test_scenario.go | 20 +++++++++++++++++++ sqlconnect/op/operator.go | 4 +++- 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/sqlconnect/internal/base/goqu_dialect.go b/sqlconnect/internal/base/goqu_dialect.go index 3efca20..165d1cc 100644 --- a/sqlconnect/internal/base/goqu_dialect.go +++ b/sqlconnect/internal/base/goqu_dialect.go @@ -107,6 +107,26 @@ func (gq *GoquDialect) QueryCondition(identifier, operator string, args ...any) return "", fmt.Errorf("notbetween operator requires exactly two arguments, got %d", len(args)) } expr = goqu.C(identifier).NotBetween(exp.NewRangeVal(args[0], args[1])) + case "nbfinterval": + if len(args) != 2 { + return "", fmt.Errorf("nbfinterval operator requires exactly two arguments, got %d", len(args)) + } + var ( + interval int + unit string + ok bool + ) + if interval, ok = args[0].(int); !ok { + return "", fmt.Errorf("nbfinterval operator requires first argument to be an integer") + } + if unit, ok = args[1].(string); !ok { + return "", fmt.Errorf("nbfinterval operator requires second argument to be a string") + } + dateAddExpr, err := gq.DateAdd("CURRENT_DATE", -interval, unit) + if err != nil { + return "", err + } + expr = goqu.C(identifier).Gte(dateAddExpr.GoquExpression()) default: return "", fmt.Errorf("unsupported operator: %s", operator) } diff --git a/sqlconnect/internal/integration_test/db_integration_test_scenario.go b/sqlconnect/internal/integration_test/db_integration_test_scenario.go index afdc3b1..9ec41d0 100644 --- a/sqlconnect/internal/integration_test/db_integration_test_scenario.go +++ b/sqlconnect/internal/integration_test/db_integration_test_scenario.go @@ -389,6 +389,26 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe require.ErrorContains(t, err, "unsupported operator: someop", "it should return an error for an invalid operator") }) + t.Run("nbfinterval operator", func(t *testing.T) { + op := string(op.NbfInterval) + rowCount := 0 + validateCondition(t, getQueryCondition(t, "DATE("+timeCol+")", op, 1, "day"), rowCount) + + t.Run("with invalid arguments", func(t *testing.T) { + _, err := db.QueryCondition("col", op) + require.Error(t, err, "it should return an error for no arguments") + + _, err = db.QueryCondition("col", op, "1", "day") + require.Error(t, err, "it should return an error for invalid 1st argument") + + _, err = db.QueryCondition("col", op, 1, 2) + require.Error(t, err, "it should return an error for invalid 2nd argument") + + _, err = db.QueryCondition("col", op, 1, "day", 3) + require.Error(t, err, "it should return an error for invalid number of arguments") + }) + }) + t.Run("time add", func(t *testing.T) { op := string(op.Lt) rowCount := 1 diff --git a/sqlconnect/op/operator.go b/sqlconnect/op/operator.go index ae93f95..6211a6f 100644 --- a/sqlconnect/op/operator.go +++ b/sqlconnect/op/operator.go @@ -29,6 +29,8 @@ const ( Between Operator = "between" // v < left OR v > right NotBetween Operator = "notbetween" + // left >= now() - INTERVAL right + NbfInterval Operator = "nbfinterval" // IS NOT NULL IsSet Operator = "isset" @@ -38,7 +40,7 @@ const ( func IsValid(op Operator) bool { switch op { - case Eq, Neq, In, NotIn, Gt, Gte, Lt, Lte, Like, NotLike, Between, NotBetween, IsSet, NotSet: + case Eq, Neq, In, NotIn, Gt, Gte, Lt, Lte, Like, NotLike, Between, NotBetween, NbfInterval, IsSet, NotSet: return true } return false From da28b2b1f5ac97502870d6b10a5b1d71803a6e36 Mon Sep 17 00:00:00 2001 From: Aris Tzoumas Date: Thu, 12 Sep 2024 12:42:57 +0300 Subject: [PATCH 4/4] chore: operator (short) names --- sqlconnect/internal/base/goqu_dialect.go | 64 ++++++++++--------- .../db_integration_test_scenario.go | 44 ++++++------- sqlconnect/op/operator.go | 19 +++--- 3 files changed, 65 insertions(+), 62 deletions(-) diff --git a/sqlconnect/internal/base/goqu_dialect.go b/sqlconnect/internal/base/goqu_dialect.go index 165d1cc..d329b9a 100644 --- a/sqlconnect/internal/base/goqu_dialect.go +++ b/sqlconnect/internal/base/goqu_dialect.go @@ -2,6 +2,7 @@ package base import ( "fmt" + "strings" "github.com/samber/lo" @@ -9,6 +10,7 @@ import ( "github.com/rudderlabs/goqu/v10/exp" "github.com/rudderlabs/goqu/v10/sqlgen" "github.com/rudderlabs/sqlconnect-go/sqlconnect" + "github.com/rudderlabs/sqlconnect-go/sqlconnect/op" ) func NewGoquDialect(dialect string, o *sqlgen.SQLDialectOptions, expressions *Expressions) *GoquDialect { @@ -36,80 +38,80 @@ func (gq *GoquDialect) QueryCondition(identifier, operator string, args ...any) return a }) var expr goqu.Expression - switch operator { - case "eq": + switch op.Operator(strings.ToLower(operator)) { + case op.Eq: if len(args) != 1 { - return "", fmt.Errorf("eq operator requires exactly one argument, got %d", len(args)) + return "", fmt.Errorf("%s operator requires exactly one argument, got %d", operator, len(args)) } expr = goqu.C(identifier).Eq(args[0]) - case "neq": + case op.Neq: if len(args) != 1 { - return "", fmt.Errorf("neq operator requires exactly one argument, got %d", len(args)) + return "", fmt.Errorf("%s operator requires exactly one argument, got %d", operator, len(args)) } expr = goqu.C(identifier).Neq(args[0]) - case "in": + case op.In: if len(args) == 0 { - return "", fmt.Errorf("in operator requires at least one argument") + return "", fmt.Errorf("%s operator requires at least one argument", operator) } expr = goqu.C(identifier).In(args...) - case "notin": + case op.Nin: if len(args) == 0 { - return "", fmt.Errorf("notin operator requires at least one argument") + return "", fmt.Errorf("%s operator requires at least one argument", operator) } expr = goqu.C(identifier).NotIn(args...) - case "like": + case op.Like: if len(args) != 1 { - return "", fmt.Errorf("like operator requires exactly one argument, got %d", len(args)) + return "", fmt.Errorf("%s operator requires exactly one argument, got %d", operator, len(args)) } expr = goqu.C(identifier).Like(args[0]) - case "notlike": + case op.NLike: if len(args) != 1 { - return "", fmt.Errorf("notlike operator requires exactly one argument, got %d", len(args)) + return "", fmt.Errorf("%s operator requires exactly one argument, got %d", operator, len(args)) } expr = goqu.C(identifier).NotLike(args[0]) - case "isset": + case op.Nnull: if len(args) != 0 { - return "", fmt.Errorf("isset operator requires no arguments, got %d", len(args)) + return "", fmt.Errorf("%s operator requires no arguments, got %d", operator, len(args)) } expr = goqu.C(identifier).IsNotNull() - case "notset": + case op.Null: if len(args) != 0 { - return "", fmt.Errorf("isnotset operator requires no arguments, got %d", len(args)) + return "", fmt.Errorf("%s operator requires no arguments, got %d", operator, len(args)) } expr = goqu.C(identifier).IsNull() - case "gt": + case op.Gt: if len(args) != 1 { - return "", fmt.Errorf("gt operator requires exactly one argument, got %d", len(args)) + return "", fmt.Errorf("%s operator requires exactly one argument, got %d", operator, len(args)) } expr = goqu.C(identifier).Gt(args[0]) - case "gte": + case op.Gte: if len(args) != 1 { - return "", fmt.Errorf("gte operator requires exactly one argument, got %d", len(args)) + return "", fmt.Errorf("%s operator requires exactly one argument, got %d", operator, len(args)) } expr = goqu.C(identifier).Gte(args[0]) - case "lt": + case op.Lt: if len(args) != 1 { - return "", fmt.Errorf("lt operator requires exactly one argument, got %d", len(args)) + return "", fmt.Errorf("%s operator requires exactly one argument, got %d", operator, len(args)) } expr = goqu.C(identifier).Lt(args[0]) - case "lte": + case op.Lte: if len(args) != 1 { - return "", fmt.Errorf("lte operator requires exactly one argument, got %d", len(args)) + return "", fmt.Errorf("%s operator requires exactly one argument, got %d", operator, len(args)) } expr = goqu.C(identifier).Lte(args[0]) - case "between": + case op.Btw: if len(args) != 2 { - return "", fmt.Errorf("between operator requires exactly two arguments, got %d", len(args)) + return "", fmt.Errorf("%s operator requires exactly two arguments, got %d", operator, len(args)) } expr = goqu.C(identifier).Between(exp.NewRangeVal(args[0], args[1])) - case "notbetween": + case op.Nbtw: if len(args) != 2 { - return "", fmt.Errorf("notbetween operator requires exactly two arguments, got %d", len(args)) + return "", fmt.Errorf("%s operator requires exactly two arguments, got %d", operator, len(args)) } expr = goqu.C(identifier).NotBetween(exp.NewRangeVal(args[0], args[1])) - case "nbfinterval": + case op.Inlast: if len(args) != 2 { - return "", fmt.Errorf("nbfinterval operator requires exactly two arguments, got %d", len(args)) + return "", fmt.Errorf("%s operator requires exactly two arguments, got %d", operator, len(args)) } var ( interval int diff --git a/sqlconnect/internal/integration_test/db_integration_test_scenario.go b/sqlconnect/internal/integration_test/db_integration_test_scenario.go index 9ec41d0..080dfd6 100644 --- a/sqlconnect/internal/integration_test/db_integration_test_scenario.go +++ b/sqlconnect/internal/integration_test/db_integration_test_scenario.go @@ -191,8 +191,8 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe return expr } - t.Run("isset", func(t *testing.T) { - op := string(op.IsSet) + t.Run(string(op.Nnull), func(t *testing.T) { + op := string(op.Nnull) rowCount := 1 validateCondition(t, getQueryCondition(t, stringCol, op), rowCount) validateCondition(t, getQueryCondition(t, intCol, op), rowCount) @@ -206,8 +206,8 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe }) }) - t.Run("notset", func(t *testing.T) { - op := string(op.NotSet) + t.Run(string(op.Null), func(t *testing.T) { + op := string(op.Null) rowCount := 0 validateCondition(t, getQueryCondition(t, stringCol, op), rowCount) validateCondition(t, getQueryCondition(t, intCol, op), rowCount) @@ -221,7 +221,7 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe }) }) - t.Run("eq", func(t *testing.T) { + t.Run(string(op.Eq), func(t *testing.T) { op := string(op.Eq) rowCount := 1 validateCondition(t, getQueryCondition(t, stringCol, op, stringVal), rowCount) @@ -236,7 +236,7 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe }) }) - t.Run("neq", func(t *testing.T) { + t.Run(string(op.Neq), func(t *testing.T) { op := string(op.Neq) rowCount := 0 validateCondition(t, getQueryCondition(t, stringCol, op, stringVal), rowCount) @@ -251,7 +251,7 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe }) }) - t.Run("in", func(t *testing.T) { + t.Run(string(op.In), func(t *testing.T) { op := string(op.In) rowCount := 1 validateCondition(t, getQueryCondition(t, stringCol, op, stringVal), rowCount) @@ -267,8 +267,8 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe }) }) - t.Run("notin", func(t *testing.T) { - op := string(op.NotIn) + t.Run(string(op.Nin), func(t *testing.T) { + op := string(op.Nin) rowCount := 0 validateCondition(t, getQueryCondition(t, stringCol, op, stringVal), rowCount) validateCondition(t, getQueryCondition(t, intCol, op, intVal), rowCount) @@ -283,7 +283,7 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe }) }) - t.Run("like", func(t *testing.T) { + t.Run(string(op.Like), func(t *testing.T) { op := string(op.Like) rowCount := 1 validateCondition(t, getQueryCondition(t, stringCol, op, stringVal), rowCount) @@ -294,8 +294,8 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe }) }) - t.Run("notlike", func(t *testing.T) { - op := string(op.NotLike) + t.Run(string(op.NLike), func(t *testing.T) { + op := string(op.NLike) rowCount := 0 validateCondition(t, getQueryCondition(t, stringCol, op, stringVal), rowCount) @@ -305,7 +305,7 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe }) }) - t.Run("gt", func(t *testing.T) { + t.Run(string(op.Gt), func(t *testing.T) { op := string(op.Gt) rowCount := 1 validateCondition(t, getQueryCondition(t, intCol, op, intVal-1), rowCount) @@ -318,7 +318,7 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe }) }) - t.Run("gte", func(t *testing.T) { + t.Run(string(op.Gte), func(t *testing.T) { op := string(op.Gte) rowCount := 1 validateCondition(t, getQueryCondition(t, intCol, op, intVal), rowCount) @@ -331,7 +331,7 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe }) }) - t.Run("lt", func(t *testing.T) { + t.Run(string(op.Lt), func(t *testing.T) { op := string(op.Lt) rowCount := 1 validateCondition(t, getQueryCondition(t, intCol, op, intVal+1), rowCount) @@ -344,7 +344,7 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe }) }) - t.Run("lte", func(t *testing.T) { + t.Run(string(op.Lte), func(t *testing.T) { op := string(op.Lte) rowCount := 1 validateCondition(t, getQueryCondition(t, intCol, op, intVal), rowCount) @@ -357,8 +357,8 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe }) }) - t.Run("between", func(t *testing.T) { - op := string(op.Between) + t.Run(string(op.Btw), func(t *testing.T) { + op := string(op.Btw) rowCount := 1 validateCondition(t, getQueryCondition(t, intCol, op, intVal-1, intVal+1), rowCount) validateCondition(t, getQueryCondition(t, floatCol, op, floatVal-1.0, floatVal+1.0), rowCount) @@ -370,8 +370,8 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe }) }) - t.Run("notbetween", func(t *testing.T) { - op := string(op.NotBetween) + t.Run(string(op.Nbtw), func(t *testing.T) { + op := string(op.Nbtw) rowCount := 0 validateCondition(t, getQueryCondition(t, intCol, op, intVal-1, intVal+1), rowCount) validateCondition(t, getQueryCondition(t, floatCol, op, floatVal-1.0, floatVal+1.0), rowCount) @@ -389,8 +389,8 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe require.ErrorContains(t, err, "unsupported operator: someop", "it should return an error for an invalid operator") }) - t.Run("nbfinterval operator", func(t *testing.T) { - op := string(op.NbfInterval) + t.Run(string(op.Inlast)+" operator", func(t *testing.T) { + op := string(op.Inlast) rowCount := 0 validateCondition(t, getQueryCondition(t, "DATE("+timeCol+")", op, 1, "day"), rowCount) diff --git a/sqlconnect/op/operator.go b/sqlconnect/op/operator.go index 6211a6f..73fa717 100644 --- a/sqlconnect/op/operator.go +++ b/sqlconnect/op/operator.go @@ -10,7 +10,7 @@ const ( // IN (...) In Operator = "in" // NOT IN (...) - NotIn Operator = "notin" + Nin Operator = "nin" // > Gt Operator = "gt" @@ -24,23 +24,24 @@ const ( // LIKE Like Operator = "like" // NOT LIKE - NotLike Operator = "notlike" + NLike Operator = "nlike" // left <= v <= right - Between Operator = "between" + Btw Operator = "btw" // v < left OR v > right - NotBetween Operator = "notbetween" + Nbtw Operator = "nbtw" // left >= now() - INTERVAL right - NbfInterval Operator = "nbfinterval" + Inlast Operator = "inlast" - // IS NOT NULL - IsSet Operator = "isset" // IS NULL - NotSet Operator = "notset" + Null Operator = "null" + + // IS NOT NULL + Nnull Operator = "nnull" ) func IsValid(op Operator) bool { switch op { - case Eq, Neq, In, NotIn, Gt, Gte, Lt, Lte, Like, NotLike, Between, NotBetween, NbfInterval, IsSet, NotSet: + case Eq, Neq, In, Nin, Gt, Gte, Lt, Lte, Like, NLike, Btw, Nbtw, Inlast, Nnull, Null: return true } return false