From d8b108f86842a1ae3182092b24ecf009dacec4e7 Mon Sep 17 00:00:00 2001 From: JP Moresmau Date: Mon, 2 Jan 2023 13:39:42 +0100 Subject: [PATCH] Handle date time with nanosecond precision, with specific functions to create Trino data types explicitly. --- README.md | 13 +- trino/integration_test.go | 4 +- trino/serial.go | 71 ++++++++- trino/serial_test.go | 49 +++++- trino/trino.go | 52 +++++-- trino/trino_test.go | 320 ++++++++++++++++++++++++++++++++++++-- 6 files changed, 471 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index b774ad52..debf49ec 100644 --- a/README.md +++ b/README.md @@ -198,12 +198,17 @@ When passing arguments to queries, the driver supports the following Go data typ * `string` * slices * `trino.Numeric` - a string representation of a number +* `time.Time` - passed to Trino as a timestamp with a time zone +* the result of `trino.Date(year, month, day)` - passed to Trino as a date +* the result of `trino.Time(hour, minute, second, nanosecond)` - passed to Trino as a time without a time zone +* the result of `trino.TimeTz(hour, minute, second, nanosecond, location)` - passed to Trino as a time with a time zone +* the result of `trino.Timestamp(year, month, day, hour, minute, second, nanosecond)` - passed to Trino as a timestamp without a time zone It's not yet possible to pass: * `nil` * `float32` or `float64` * `byte` -* `time.Time` or `time.Duration` +* `time.Duration` * `json.RawMessage` * maps @@ -215,7 +220,11 @@ SELECT * FROM table WHERE col_double = cast(? AS DOUBLE) OR col_timestamp = CAST ### Response rows When reading response rows, the driver supports most Trino data types, except: -* time and timestamps with precision - all time types are returned as `time.Time` +* time and timestamps with precision - all time types are returned as `time.Time`. + All precisions up to nanoseconds (`TIMESTAMP(9)` or `TIME(9)`) are supported (since + this is the maximum precision Golang's `time.Time` supports). If a query returns columns + defined with a greater precision, use `CAST` to reduce the returned precision, or convert the + value to a string that then can be parsed manually. * `DECIMAL` - returned as string * `IPADDRESS` - returned as string * `INTERVAL YEAR TO MONTH` and `INTERVAL DAY TO SECOND` - returned as string diff --git a/trino/integration_test.go b/trino/integration_test.go index 7adfd984..13cb8d12 100644 --- a/trino/integration_test.go +++ b/trino/integration_test.go @@ -422,7 +422,7 @@ func TestIntegrationArgsConversion(t *testing.T) { AND col_big = ? AND col_real = cast(? as real) AND col_double = cast(? as double) - AND col_ts = cast(? as timestamp) + AND col_ts = ? AND col_varchar = ? AND col_array = ?`, int16(1), @@ -431,7 +431,7 @@ func TestIntegrationArgsConversion(t *testing.T) { int64(1), Numeric("1"), Numeric("1"), - "2017-07-10 01:02:03.004 UTC", + time.Date(2017, 7, 10, 1, 2, 3, 4*1000000, time.UTC), "string", []string{"A", "B"}).Scan(&value) if err != nil { diff --git a/trino/serial.go b/trino/serial.go index b00df442..0bf78374 100644 --- a/trino/serial.go +++ b/trino/serial.go @@ -35,6 +35,65 @@ func (e UnsupportedArgError) Error() string { // If another string format is used it will error to serialise type Numeric string +// trinoDate represents a Date type in Trino. +type trinoDate struct { + year int + month time.Month + day int +} + +// Date creates a representation of a Trino Date type. +func Date(year int, month time.Month, day int) trinoDate { + return trinoDate{year, month, day} +} + +// trinoTime represents a Time type in Trino. +type trinoTime struct { + hour int + minute int + second int + nanosecond int +} + +// Time creates a representation of a Trino Time type. To represent time with precision higher than nanoseconds, pass the value as a string and use a cast in the query. +func Time(hour int, + minute int, + second int, + nanosecond int) trinoTime { + return trinoTime{hour, minute, second, nanosecond} +} + +// trinoTimeTz represents a Time(9) With Timezone type in Trino. +type trinoTimeTz time.Time + +// TimeTz creates a representation of a Trino Time(9) With Timezone type. +func TimeTz(hour int, + minute int, + second int, + nanosecond int, + location *time.Location) trinoTimeTz { + // When reading a time, a nil location indicates UTC. + // However, passing nil to time.Date() panics. + if location == nil { + location = time.UTC + } + return trinoTimeTz(time.Date(0, 0, 0, hour, minute, second, nanosecond, location)) +} + +// Timestamp indicates we want a TimeStamp type WITHOUT a time zone in Trino from a Golang time. +type trinoTimestamp time.Time + +// Timestamp creates a representation of a Trino Timestamp(9) type. +func Timestamp(year int, + month time.Month, + day int, + hour int, + minute int, + second int, + nanosecond int) trinoTimestamp { + return trinoTimestamp(time.Date(year, month, day, hour, minute, second, nanosecond, time.UTC)) +} + // Serial converts any supported value to its equivalent string for as a Trino parameter // See https://trino.io/docs/current/language/types.html func Serial(v interface{}) (string, error) { @@ -92,9 +151,17 @@ func Serial(v interface{}) (string, error) { case []byte: return "", UnsupportedArgError{"[]byte"} - // time.Time and time.Duration not supported as time and date take several different formats in Trino + case trinoDate: + return fmt.Sprintf("DATE '%04d-%02d-%02d'", x.year, x.month, x.day), nil + case trinoTime: + return fmt.Sprintf("TIME '%02d:%02d:%02d.%09d'", x.hour, x.minute, x.second, x.nanosecond), nil + case trinoTimeTz: + return "TIME " + time.Time(x).Format("'15:04:05.999999999 Z07:00'"), nil + case trinoTimestamp: + return "TIMESTAMP " + time.Time(x).Format("'2006-01-02 15:04:05.999999999'"), nil case time.Time: - return "", UnsupportedArgError{"time.Time"} + return "TIMESTAMP " + time.Time(x).Format("'2006-01-02 15:04:05.999999999 Z07:00'"), nil + case time.Duration: return "", UnsupportedArgError{"time.Duration"} diff --git a/trino/serial_test.go b/trino/serial_test.go index 1d19d378..62ef0fdb 100644 --- a/trino/serial_test.go +++ b/trino/serial_test.go @@ -14,9 +14,16 @@ package trino -import "testing" +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) func TestSerial(t *testing.T) { + paris, err := time.LoadLocation("Europe/Paris") + require.NoError(t, err) scenarios := []struct { name string value interface{} @@ -113,6 +120,46 @@ func TestSerial(t *testing.T) { value: false, expectedSerial: "false", }, + { + name: "date", + value: Date(2017, 7, 10), + expectedSerial: "DATE '2017-07-10'", + }, + { + name: "time without timezone", + value: Time(11, 34, 25, 123456), + expectedSerial: "TIME '11:34:25.000123456'", + }, + { + name: "time with timezone", + value: TimeTz(11, 34, 25, 123456, time.FixedZone("test zone", +2*3600)), + expectedSerial: "TIME '11:34:25.000123456 +02:00'", + }, + { + name: "time with timezone", + value: TimeTz(11, 34, 25, 123456, nil), + expectedSerial: "TIME '11:34:25.000123456 Z'", + }, + { + name: "timestamp without timezone", + value: Timestamp(2017, 7, 10, 11, 34, 25, 123456), + expectedSerial: "TIMESTAMP '2017-07-10 11:34:25.000123456'", + }, + { + name: "timestamp with time zone in Fixed Zone", + value: time.Date(2017, 7, 10, 11, 34, 25, 123456, time.FixedZone("test zone", +2*3600)), + expectedSerial: "TIMESTAMP '2017-07-10 11:34:25.000123456 +02:00'", + }, + { + name: "timestamp with time zone in Named Zone", + value: time.Date(2017, 7, 10, 11, 34, 25, 123456, paris), + expectedSerial: "TIMESTAMP '2017-07-10 11:34:25.000123456 +02:00'", + }, + { + name: "timestamp with time zone in UTC", + value: time.Date(2017, 7, 10, 11, 34, 25, 123456, time.UTC), + expectedSerial: "TIMESTAMP '2017-07-10 11:34:25.000123456 Z'", + }, { name: "nil", value: nil, diff --git a/trino/trino.go b/trino/trino.go index d871183c..ce0a966b 100644 --- a/trino/trino.go +++ b/trino/trino.go @@ -624,20 +624,24 @@ func (st *driverStmt) ExecContext(ctx context.Context, args []driver.NamedValue) } func (st *driverStmt) CheckNamedValue(arg *driver.NamedValue) error { - _, ok := arg.Value.(Numeric) - if ok { - return nil - } - if reflect.TypeOf(arg.Value).Kind() == reflect.Slice { + switch arg.Value.(type) { + case Numeric, trinoDate, trinoTime, trinoTimeTz, trinoTimestamp: return nil - } + default: + { + if reflect.TypeOf(arg.Value).Kind() == reflect.Slice { + return nil + } - if arg.Name == trinoProgressCallbackParam { - return nil - } - if arg.Name == trinoProgressCallbackPeriodParam { - return nil + if arg.Name == trinoProgressCallbackParam { + return nil + } + if arg.Name == trinoProgressCallbackPeriodParam { + return nil + } + } } + return driver.ErrSkip } @@ -733,10 +737,11 @@ func (st *driverStmt) QueryContext(ctx context.Context, args []driver.NamedValue func (st *driverStmt) exec(ctx context.Context, args []driver.NamedValue) (*stmtResponse, error) { query := st.query - var hs http.Header + hs := make(http.Header) + // Ensure the server returns timestamps preserving their precision, without truncating them to timestamp(3). + hs.Add("X-Trino-Client-Capabilities", "PARAMETRIC_DATETIME") if len(args) > 0 { - hs = make(http.Header) var ss []string for _, arg := range args { if arg.Name == trinoProgressCallbackParam { @@ -1279,7 +1284,15 @@ func newTypeConverter(typeName string, signature typeSignature) (*typeConverter, } result.scale = newOptionalInt64(signature.Arguments[1].long) } + case "time", "time with time zone", "timestamp", "timestamp with time zone": + if len(signature.Arguments) > 0 { + if signature.Arguments[0].Kind != KIND_LONG { + return nil, ErrInvalidResponseType + } + result.precision = newOptionalInt64(signature.Arguments[0].long) + } } + return result, nil } @@ -1863,16 +1876,21 @@ func (s *NullSlice3Float64) Scan(value interface{}) error { return nil } +// Layout for time and timestamp WITHOUT time zone. +// Trino can support up to 12 digits sub second precision, but Go only 9. +// (Requires X-Trino-Client-Capabilities: PARAMETRIC_DATETIME) var timeLayouts = []string{ "2006-01-02", - "15:04:05.000", - "2006-01-02 15:04:05.000", + "15:04:05.999999999", + "2006-01-02 15:04:05.999999999", } // Layout for time and timestamp WITH time zone. +// Trino can support up to 12 digits sub second precision, but Go only 9. +// (Requires X-Trino-Client-Capabilities: PARAMETRIC_DATETIME) var timeLayoutsTZ = []string{ - "15:04:05.000 -07:00", - "2006-01-02 15:04:05.000 -07:00", + "15:04:05.999999999 -07:00", + "2006-01-02 15:04:05.999999999 -07:00", } func scanNullTime(v interface{}) (NullTime, error) { diff --git a/trino/trino_test.go b/trino/trino_test.go index 2f8026fc..cf38b81b 100644 --- a/trino/trino_test.go +++ b/trino/trino_test.go @@ -593,8 +593,8 @@ func TestQueryColumns(t *testing.T) { }, { "TIME", - false, - 0, + true, + 3, 0, false, 0, @@ -602,8 +602,8 @@ func TestQueryColumns(t *testing.T) { }, { "TIME", - false, - 0, + true, + 6, 0, false, 0, @@ -611,8 +611,8 @@ func TestQueryColumns(t *testing.T) { }, { "TIME WITH TIME ZONE", - false, - 0, + true, + 3, 0, false, 0, @@ -620,8 +620,8 @@ func TestQueryColumns(t *testing.T) { }, { "TIMESTAMP", - false, - 0, + true, + 3, 0, false, 0, @@ -629,8 +629,8 @@ func TestQueryColumns(t *testing.T) { }, { "TIMESTAMP", - false, - 0, + true, + 6, 0, false, 0, @@ -638,8 +638,8 @@ func TestQueryColumns(t *testing.T) { }, { "TIMESTAMP WITH TIME ZONE", - false, - 0, + true, + 3, 0, false, 0, @@ -647,8 +647,8 @@ func TestQueryColumns(t *testing.T) { }, { "TIMESTAMP WITH TIME ZONE", - false, - 0, + true, + 6, 0, false, 0, @@ -765,6 +765,212 @@ func TestQueryColumns(t *testing.T) { assert.Equal(t, actualTypes, expectedTypes) } +func TestMaxTrinoPrecisionDateTime(t *testing.T) { + c := &Config{ + ServerURI: *integrationServerFlag, + SessionProperties: map[string]string{"query_priority": "1"}, + } + + dsn, err := c.FormatDSN() + require.NoError(t, err) + + db, err := sql.Open("trino", dsn) + require.NoError(t, err) + + t.Cleanup(func() { + assert.NoError(t, db.Close()) + }) + + rows, err := db.Query(`SELECT + TIME '15:55:23.383345123456' AS timep, + TIME '15:55:23.383345123456 +08:00' AS timeptz, + TIMESTAMP '2020-06-10 15:55:23.383345123456' AS tsp, + TIMESTAMP '2020-06-10 15:55:23.383345123456 UTC' AS tsptz`) + require.NoError(t, err, "Failed executing query") + assert.NotNil(t, rows) + + columns, err := rows.Columns() + require.NoError(t, err, "Failed reading result columns") + + assert.Equal(t, 4, len(columns), "Expected 4 result column") + expectedNames := []string{ + "timep", + "timeptz", + "tsp", + "tsptz", + } + assert.Equal(t, expectedNames, columns) + + columnTypes, err := rows.ColumnTypes() + require.NoError(t, err, "Failed reading result column types") + + assert.Equal(t, 4, len(columnTypes), "Expected 4 result column type") + + type columnType struct { + typeName string + hasScale bool + precision int64 + scale int64 + hasLength bool + length int64 + scanType reflect.Type + } + expectedTypes := []columnType{ + { + "TIME", + true, + 12, + 0, + false, + 0, + reflect.TypeOf(sql.NullTime{}), + }, + { + "TIME WITH TIME ZONE", + true, + 12, + 0, + false, + 0, + reflect.TypeOf(sql.NullTime{}), + }, + { + "TIMESTAMP", + true, + 12, + 0, + false, + 0, + reflect.TypeOf(sql.NullTime{}), + }, + { + "TIMESTAMP WITH TIME ZONE", + true, + 12, + 0, + false, + 0, + reflect.TypeOf(sql.NullTime{}), + }, + } + actualTypes := make([]columnType, 4) + for i, column := range columnTypes { + actualTypes[i].typeName = column.DatabaseTypeName() + actualTypes[i].precision, actualTypes[i].scale, actualTypes[i].hasScale = column.DecimalSize() + actualTypes[i].length, actualTypes[i].hasLength = column.Length() + actualTypes[i].scanType = column.ScanType() + } + + assert.Equal(t, actualTypes, expectedTypes) + + assert.False(t, rows.Next()) + assert.Equal(t, "parsing time \"15:55:23.383345123456\" as \"2006-01-02 15:04:05.999999999\": cannot parse \"5:23.383345123456\" as \"2006\"", rows.Err().Error()) + +} + +func TestMaxGoPrecisionDateTime(t *testing.T) { + c := &Config{ + ServerURI: *integrationServerFlag, + SessionProperties: map[string]string{"query_priority": "1"}, + } + + dsn, err := c.FormatDSN() + require.NoError(t, err) + + db, err := sql.Open("trino", dsn) + require.NoError(t, err) + + t.Cleanup(func() { + assert.NoError(t, db.Close()) + }) + + rows, err := db.Query(`SELECT + cast(current_time as time(9)) AS timep, + cast(current_time as time(9) with time zone) AS timeptz, + cast(current_time as timestamp(9)) AS tsp, + cast(current_time as timestamp(9) with time zone) AS tsptz`) + require.NoError(t, err, "Failed executing query") + assert.NotNil(t, rows) + + columns, err := rows.Columns() + require.NoError(t, err, "Failed reading result columns") + + assert.Equal(t, 4, len(columns), "Expected 4 result column") + expectedNames := []string{ + "timep", + "timeptz", + "tsp", + "tsptz", + } + assert.Equal(t, expectedNames, columns) + + columnTypes, err := rows.ColumnTypes() + require.NoError(t, err, "Failed reading result column types") + + assert.Equal(t, 4, len(columnTypes), "Expected 4 result column type") + + type columnType struct { + typeName string + hasScale bool + precision int64 + scale int64 + hasLength bool + length int64 + scanType reflect.Type + } + expectedTypes := []columnType{ + { + "TIME", + true, + 9, + 0, + false, + 0, + reflect.TypeOf(sql.NullTime{}), + }, + { + "TIME WITH TIME ZONE", + true, + 9, + 0, + false, + 0, + reflect.TypeOf(sql.NullTime{}), + }, + { + "TIMESTAMP", + true, + 9, + 0, + false, + 0, + reflect.TypeOf(sql.NullTime{}), + }, + { + "TIMESTAMP WITH TIME ZONE", + true, + 9, + 0, + false, + 0, + reflect.TypeOf(sql.NullTime{}), + }, + } + actualTypes := make([]columnType, 4) + for i, column := range columnTypes { + actualTypes[i].typeName = column.DatabaseTypeName() + actualTypes[i].precision, actualTypes[i].scale, actualTypes[i].hasScale = column.DecimalSize() + actualTypes[i].length, actualTypes[i].hasLength = column.Length() + actualTypes[i].scanType = column.ScanType() + } + + assert.Equal(t, actualTypes, expectedTypes) + + assert.True(t, rows.Next()) + require.NoError(t, rows.Err()) + +} + func TestQueryCancellation(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) @@ -960,6 +1166,8 @@ func TestUnsupportedTransaction(t *testing.T) { func TestTypeConversion(t *testing.T) { utc, err := time.LoadLocation("UTC") require.NoError(t, err) + paris, err := time.LoadLocation("Europe/Paris") + require.NoError(t, err) testcases := []struct { DataType string @@ -1034,6 +1242,48 @@ func TestTypeConversion(t *testing.T) { ResponseUnmarshalledSample: "01:02:03.000-05:00", ExpectedGoValue: time.Date(0, 1, 1, 1, 2, 3, 0, time.FixedZone("", -5*3600)), }, + { + DataType: "time", + RawType: "time", + ResponseUnmarshalledSample: "01:02:03.123456789", + ExpectedGoValue: time.Date(0, 1, 1, 1, 2, 3, 123456789, time.Local), + }, + { + DataType: "time with time zone", + RawType: "time with time zone", + ResponseUnmarshalledSample: "01:02:03.123456789 UTC", + ExpectedGoValue: time.Date(0, 1, 1, 1, 2, 3, 123456789, utc), + }, + { + DataType: "time with time zone", + RawType: "time with time zone", + ResponseUnmarshalledSample: "01:02:03.123456789 +03:00", + ExpectedGoValue: time.Date(0, 1, 1, 1, 2, 3, 123456789, time.FixedZone("", 3*3600)), + }, + { + DataType: "time with time zone", + RawType: "time with time zone", + ResponseUnmarshalledSample: "01:02:03.123456789+03:00", + ExpectedGoValue: time.Date(0, 1, 1, 1, 2, 3, 123456789, time.FixedZone("", 3*3600)), + }, + { + DataType: "time with time zone", + RawType: "time with time zone", + ResponseUnmarshalledSample: "01:02:03.123456789 -05:00", + ExpectedGoValue: time.Date(0, 1, 1, 1, 2, 3, 123456789, time.FixedZone("", -5*3600)), + }, + { + DataType: "time with time zone", + RawType: "time with time zone", + ResponseUnmarshalledSample: "01:02:03.123456789-05:00", + ExpectedGoValue: time.Date(0, 1, 1, 1, 2, 3, 123456789, time.FixedZone("", -5*3600)), + }, + { + DataType: "time with time zone", + RawType: "time with time zone", + ResponseUnmarshalledSample: "01:02:03.123456789 Europe/Paris", + ExpectedGoValue: time.Date(0, 1, 1, 1, 2, 3, 123456789, paris), + }, { DataType: "timestamp", RawType: "timestamp", @@ -1070,6 +1320,48 @@ func TestTypeConversion(t *testing.T) { ResponseUnmarshalledSample: "2017-07-10 01:02:03.000-04:00", ExpectedGoValue: time.Date(2017, 7, 10, 1, 2, 3, 0, time.FixedZone("", -4*3600)), }, + { + DataType: "timestamp", + RawType: "timestamp", + ResponseUnmarshalledSample: "2017-07-10 01:02:03.123456789", + ExpectedGoValue: time.Date(2017, 7, 10, 1, 2, 3, 123456789, time.Local), + }, + { + DataType: "timestamp with time zone", + RawType: "timestamp with time zone", + ResponseUnmarshalledSample: "2017-07-10 01:02:03.123456789 UTC", + ExpectedGoValue: time.Date(2017, 7, 10, 1, 2, 3, 123456789, utc), + }, + { + DataType: "timestamp with time zone", + RawType: "timestamp with time zone", + ResponseUnmarshalledSample: "2017-07-10 01:02:03.123456789 +03:00", + ExpectedGoValue: time.Date(2017, 7, 10, 1, 2, 3, 123456789, time.FixedZone("", 3*3600)), + }, + { + DataType: "timestamp with time zone", + RawType: "timestamp with time zone", + ResponseUnmarshalledSample: "2017-07-10 01:02:03.123456789+03:00", + ExpectedGoValue: time.Date(2017, 7, 10, 1, 2, 3, 123456789, time.FixedZone("", 3*3600)), + }, + { + DataType: "timestamp with time zone", + RawType: "timestamp with time zone", + ResponseUnmarshalledSample: "2017-07-10 01:02:03.123456789 -04:00", + ExpectedGoValue: time.Date(2017, 7, 10, 1, 2, 3, 123456789, time.FixedZone("", -4*3600)), + }, + { + DataType: "timestamp with time zone", + RawType: "timestamp with time zone", + ResponseUnmarshalledSample: "2017-07-10 01:02:03.123456789-04:00", + ExpectedGoValue: time.Date(2017, 7, 10, 1, 2, 3, 123456789, time.FixedZone("", -4*3600)), + }, + { + DataType: "timestamp with time zone", + RawType: "timestamp with time zone", + ResponseUnmarshalledSample: "2017-07-10 01:02:03.123456789 Europe/Paris", + ExpectedGoValue: time.Date(2017, 7, 10, 1, 2, 3, 123456789, paris), + }, { DataType: "map(varchar,varchar)", RawType: "map",