Merge pull request #9 from goccy/group_by

Supports HAVING / ORDER BY / GROUP BY (ROLLUP) / QUALIFY / JOIN and some window functions

README.md

@@ -147,6 +147,11 @@ A list of ZetaSQL specifications and features supported by go-zetasqlite.
 - [x] WINDOW
 - [x] WITH
 - [x] UNION
+- [X] HAVING
+- [x] ORDER BY
+- [X] GROUP BY - ROLLUP
+- [X] INNER/LEFT JOIN
+- [x] QUALIFY
 
 ### Aggregate functions
 
@@ -195,7 +200,7 @@ A list of ZetaSQL specifications and features supported by go-zetasqlite.
 ### Numbering functions
 
 - [x] RANK
-- [ ] DENSE_RANK
+- [x] DENSE_RANK
 - [ ] PERCENT_RANK
 - [ ] CUME_DIST
 - [ ] NTILE
@@ -276,7 +281,7 @@ A list of ZetaSQL specifications and features supported by go-zetasqlite.
 - [x] LAST_VALUE
 - [ ] NTH_VALUE
 - [ ] LEAD
-- [ ] LAG
+- [x] LAG
 - [ ] PERCENTILE_CONT
 - [ ] PERCENTILE_DISC
 

cmd/zetasqlite-cli/go.mod

@@ -0,0 +1,14 @@
+module github.com/goccy/go-zetasqlite/cmd/zetasqlite-cli
+
+go 1.18
+
+require (
+	github.com/chzyer/readline v1.5.1
+	github.com/goccy/go-zetasqlite v0.3.1
+)
+
+require (
+	github.com/goccy/go-zetasql v0.2.8 // indirect
+	github.com/mattn/go-sqlite3 v1.14.14 // indirect
+	golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 // indirect
+)

cmd/zetasqlite-cli/go.sum

@@ -0,0 +1,15 @@
+github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM=
+github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ=
+github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI=
+github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk=
+github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04=
+github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8=
+github.com/goccy/go-zetasql v0.2.8 h1:Kh8zQgBxN7Kg+cNJ3Z724G0oyaGoUYngeATSp6IOF/A=
+github.com/goccy/go-zetasql v0.2.8/go.mod h1:6W14CJVKh7crrSPyj6NPk4c49L2NWnxvyDLsRkOm4BI=
+github.com/goccy/go-zetasqlite v0.3.1 h1:rOWZZrQp59iu0FLeYkrOjdXBQbF/XPKNq3wGdbsTNYE=
+github.com/goccy/go-zetasqlite v0.3.1/go.mod h1:PT3M02F7jVGdLmrczfr3ZaLnjxH9bowlsP6qy77OG1g=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/mattn/go-sqlite3 v1.14.14 h1:qZgc/Rwetq+MtyE18WhzjokPD93dNqLGNT3QJuLvBGw=
+github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 h1:y/woIyUBFbpQGKS0u1aHF/40WUDnek3fPOyD08H5Vng=
+golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

cmd/zetasqlite-cli/main.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"io"
+	"log"
+	"reflect"
+	"strings"
+
+	"github.com/chzyer/readline"
+	_ "github.com/goccy/go-zetasqlite"
+)
+
+func main() {
+	if err := run(context.Background()); err != nil {
+		log.Fatalf("%+v", err)
+	}
+}
+
+func run(ctx context.Context) error {
+	db, err := sql.Open("zetasqlite_sqlite3", ":memory:")
+	if err != nil {
+		return err
+	}
+	rl, err := readline.NewEx(&readline.Config{
+		Prompt:      ">> ",
+		HistoryFile: "./zetasqlite.history",
+	})
+	if err != nil {
+		return err
+	}
+	defer rl.Close()
+	for {
+		line, err := rl.Readline()
+		if err == io.EOF || err == readline.ErrInterrupt {
+			break
+		}
+		line = strings.TrimSpace(line)
+		rows, err := db.QueryContext(ctx, line)
+		if err != nil {
+			fmt.Printf("ERROR: %v\n", err)
+			continue
+		}
+		columns, err := rows.Columns()
+		if err != nil {
+			fmt.Printf("ERROR: %v\n", err)
+			continue
+		}
+		columnNum := len(columns)
+		args := make([]interface{}, columnNum)
+		for i := 0; i < columnNum; i++ {
+			var v interface{}
+			args[i] = &v
+		}
+		header := strings.Join(columns, "|")
+		fmt.Printf("%s\n", header)
+		fmt.Printf("%s\n", strings.Repeat("-", len(header)))
+		for rows.Next() {
+			if err := rows.Scan(args...); err != nil {
+				fmt.Printf("ERROR: %v", err)
+				break
+			}
+			values := make([]string, 0, len(args))
+			for _, arg := range args {
+				v := reflect.ValueOf(arg).Elem().Interface()
+				values = append(values, fmt.Sprint(v))
+			}
+			fmt.Printf("%s\n", strings.Join(values, "|"))
+		}
+	}
+	return nil
+}

driver.go

@@ -118,7 +118,7 @@ func (c *ZetaSQLiteConn) ExecContext(ctx context.Context, query string, args []d
 	if err != nil {
 		return nil, fmt.Errorf("failed to analyze query: %w", err)
 	}
-	newNamedValues, err := internal.ConvertNamedValues(args)
+	newNamedValues, err := internal.EncodeNamedValues(args, out.Params())
 	if err != nil {
 		return nil, err
 	}
@@ -134,7 +134,7 @@ func (c *ZetaSQLiteConn) QueryContext(ctx context.Context, query string, args []
 	if err != nil {
 		return nil, fmt.Errorf("failed to analyze query: %w", err)
 	}
-	newNamedValues, err := internal.ConvertNamedValues(args)
+	newNamedValues, err := internal.EncodeNamedValues(args, out.Params())
 	if err != nil {
 		return nil, err
 	}

go.mod

@@ -3,7 +3,7 @@ module github.com/goccy/go-zetasqlite
 go 1.17
 
 require (
-	github.com/goccy/go-zetasql v0.2.7
+	github.com/goccy/go-zetasql v0.2.8
 	github.com/mattn/go-sqlite3 v1.14.14
 )
 

go.sum

@@ -1,5 +1,5 @@
-github.com/goccy/go-zetasql v0.2.7 h1:zpWBg0hb9LqDRpllrPvs8Ei6LsVVLNWLlVV1RgnQvx4=
-github.com/goccy/go-zetasql v0.2.7/go.mod h1:6W14CJVKh7crrSPyj6NPk4c49L2NWnxvyDLsRkOm4BI=
+github.com/goccy/go-zetasql v0.2.8 h1:Kh8zQgBxN7Kg+cNJ3Z724G0oyaGoUYngeATSp6IOF/A=
+github.com/goccy/go-zetasql v0.2.8/go.mod h1:6W14CJVKh7crrSPyj6NPk4c49L2NWnxvyDLsRkOm4BI=
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/mattn/go-sqlite3 v1.14.14 h1:qZgc/Rwetq+MtyE18WhzjokPD93dNqLGNT3QJuLvBGw=

internal/analyzer.go

@@ -22,7 +22,7 @@ type AnalyzerOutput struct {
 	node           ast.Node
 	query          string
 	formattedQuery string
-	argsNum        int
+	params         []*ast.ParameterNode
 	isQuery        bool
 	tableSpec      *TableSpec
 	outputColumns  []*ColumnSpec
@@ -31,6 +31,10 @@ type AnalyzerOutput struct {
 	QueryContext   func(context.Context, *sql.Conn, ...interface{}) (driver.Rows, error)
 }
 
+func (o *AnalyzerOutput) Params() []*ast.ParameterNode {
+	return o.params
+}
+
 func NewAnalyzer(catalog *Catalog) *Analyzer {
 	return &Analyzer{
 		catalog: catalog,
@@ -67,6 +71,9 @@ func newAnalyzerOptions() *zetasql.AnalyzerOptions {
 		zetasql.FeatureV13ExtendedDateTimeSignatures,
 		zetasql.FeatureV12CivilTime,
 		zetasql.FeatureIntervalType,
+		zetasql.FeatureGroupByRollup,
+		zetasql.FeatureV13NullsFirstLastInOrderBy,
+		zetasql.FeatureV13Qualify,
 	})
 	langOpt.SetSupportedStatementKinds([]ast.Kind{
 		ast.QueryStmt,
@@ -136,7 +143,7 @@ func (a *Analyzer) analyzeCreateTableStmt(query string, node *ast.CreateTableStm
 	return &AnalyzerOutput{
 		node:      node,
 		query:     query,
-		argsNum:   a.getParamNumFromNode(node),
+		params:    a.getParamsFromNode(node),
 		tableSpec: spec,
 		Prepare: func(ctx context.Context, conn *sql.Conn) (driver.Stmt, error) {
 			if spec.CreateMode == ast.CreateOrReplaceMode {
@@ -203,18 +210,18 @@ func (a *Analyzer) analyzeDMLStmt(ctx context.Context, query string, node ast.No
 	if formattedQuery == "" {
 		return nil, fmt.Errorf("failed to format query %s", query)
 	}
-	argsNum := a.getParamNumFromNode(node)
+	params := a.getParamsFromNode(node)
 	return &AnalyzerOutput{
 		node:           node,
 		query:          query,
 		formattedQuery: formattedQuery,
-		argsNum:        argsNum,
+		params:         params,
 		Prepare: func(ctx context.Context, conn *sql.Conn) (driver.Stmt, error) {
 			s, err := conn.PrepareContext(ctx, formattedQuery)
 			if err != nil {
 				return nil, fmt.Errorf("failed to prepare %s: %w", query, err)
 			}
-			return newDMLStmt(s, argsNum, formattedQuery), nil
+			return newDMLStmt(s, params, formattedQuery), nil
 		},
 		ExecContext: func(ctx context.Context, conn *sql.Conn, args ...interface{}) (driver.Result, error) {
 			if _, err := conn.ExecContext(ctx, formattedQuery, args...); err != nil {
@@ -240,19 +247,19 @@ func (a *Analyzer) analyzeQueryStmt(ctx context.Context, query string, node *ast
 	if formattedQuery == "" {
 		return nil, fmt.Errorf("failed to format query %s", query)
 	}
-	argsNum := a.getParamNumFromNode(node)
+	params := a.getParamsFromNode(node)
 	return &AnalyzerOutput{
 		node:           node,
 		query:          query,
 		formattedQuery: formattedQuery,
-		argsNum:        argsNum,
+		params:         params,
 		isQuery:        true,
 		Prepare: func(ctx context.Context, conn *sql.Conn) (driver.Stmt, error) {
 			s, err := conn.PrepareContext(ctx, formattedQuery)
 			if err != nil {
 				return nil, fmt.Errorf("failed to prepare %s: %w", query, err)
 			}
-			return newQueryStmt(s, argsNum, formattedQuery, outputColumns), nil
+			return newQueryStmt(s, params, formattedQuery, outputColumns), nil
 		},
 		QueryContext: func(ctx context.Context, conn *sql.Conn, args ...interface{}) (driver.Rows, error) {
 			rows, err := conn.QueryContext(ctx, formattedQuery, args...)
@@ -266,7 +273,7 @@ func (a *Analyzer) analyzeQueryStmt(ctx context.Context, query string, node *ast
 
 func (a *Analyzer) getFullNamePath(query string) (*fullNamePath, error) {
 	fullpath := &fullNamePath{}
-	parsedAST, err := zetasql.ParseStatement(query)
+	parsedAST, err := zetasql.ParseStatement(query, a.opt.ParserOptions())
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse statement: %w", err)
 	}
@@ -311,14 +318,14 @@ func (a *Analyzer) getFullNamePath(query string) (*fullNamePath, error) {
 	return fullpath, nil
 }
 
-func (a *Analyzer) getParamNumFromNode(node ast.Node) int {
-	var numInput int
+func (a *Analyzer) getParamsFromNode(node ast.Node) []*ast.ParameterNode {
+	var params []*ast.ParameterNode
 	ast.Walk(node, func(n ast.Node) error {
-		_, ok := n.(*ast.ParameterNode)
+		param, ok := n.(*ast.ParameterNode)
 		if ok {
-			numInput++
+			params = append(params, param)
 		}
 		return nil
 	})
-	return numInput
+	return params
 }

internal/catalog.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"reflect"
+	"strings"
 	"sync"
 	"time"
 
@@ -252,6 +253,14 @@ func (c *Catalog) addTableSpecRecursive(cat *types.SimpleCatalog, spec *TableSpe
 		if !c.existsCatalog(cat, subCatalogName) {
 			cat.AddCatalog(subCatalog)
 		}
+		fullTableName := strings.Join(spec.NamePath, ".")
+		if !c.existsTable(cat, fullTableName) {
+			table, err := c.createSimpleTable(fullTableName, spec)
+			if err != nil {
+				return err
+			}
+			cat.AddTable(table)
+		}
 		newNamePath := spec.NamePath[1:]
 		// add sub catalog to root catalog
 		if err := c.addTableSpecRecursive(cat, c.copyTableSpec(spec, newNamePath)); err != nil {
@@ -271,18 +280,26 @@ func (c *Catalog) addTableSpecRecursive(cat *types.SimpleCatalog, spec *TableSpe
 	if c.existsTable(cat, tableName) {
 		return nil
 	}
+	table, err := c.createSimpleTable(tableName, spec)
+	if err != nil {
+		return err
+	}
+	cat.AddTable(table)
+	return nil
+}
+
+func (c *Catalog) createSimpleTable(tableName string, spec *TableSpec) (*types.SimpleTable, error) {
 	columns := []types.Column{}
 	for _, column := range spec.Columns {
 		typ, err := column.Type.ToZetaSQLType()
 		if err != nil {
-			return err
+			return nil, err
 		}
 		columns = append(columns, types.NewSimpleColumn(
 			tableName, column.Name, typ,
 		))
 	}
-	cat.AddTable(types.NewSimpleTable(tableName, columns))
-	return nil
+	return types.NewSimpleTable(tableName, columns), nil
 }
 
 func (c *Catalog) addFunctionSpecRecursive(cat *types.SimpleCatalog, spec *FunctionSpec) error {