Skip to content

Commit

Permalink
Merge #25477
Browse files Browse the repository at this point in the history 
25477: opt: Add additional decorrelation rules + tests r=andy-kimball a=andy-kimball

This PR adds additional decorrelation rules, as well as a suite of benchmark/ORM queries that require decorrelation to work.

Co-authored-by: Andrew Kimball <[email protected]>
  • Loading branch information
@andy-kimball
craig[bot] and andy-kimball committed May 16, 2018
2 parents cd753ac + bd7ea79 commit 024c2fa
Show file tree
Hide file tree
Showing 35 changed files with 8,540 additions and 1,126 deletions.
8 changes: 5 additions & 3 deletions pkg/sql/opt/memo/expr_view_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (
"github.com/cockroachdb/cockroach/pkg/settings/cluster"
"github.com/cockroachdb/cockroach/pkg/sql/opt/memo"
"github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder"
"github.com/cockroachdb/cockroach/pkg/sql/opt/testutils"
"github.com/cockroachdb/cockroach/pkg/sql/opt/testutils/testcat"
"github.com/cockroachdb/cockroach/pkg/sql/opt/xform"
"github.com/cockroachdb/cockroach/pkg/sql/parser"
"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
Expand All @@ -31,8 +31,10 @@ import (
func BenchmarkExprView(b *testing.B) {
semaCtx := tree.MakeSemaContext(false /* privileged */)
evalCtx := tree.MakeTestingEvalContext(cluster.MakeTestingClusterSettings())
catalog := testutils.NewTestCatalog()
testutils.ExecuteTestDDL(b, "CREATE TABLE a (x INT PRIMARY KEY, y INT)", catalog)
catalog := testcat.New()
if _, err := catalog.ExecuteDDL("CREATE TABLE a (x INT PRIMARY KEY, y INT)"); err != nil {
b.Fatal(err)
}

cases := []string{
"SELECT x, y FROM a WHERE x + y * 2 - (x * y) > 0",
Expand Down
13 changes: 9 additions & 4 deletions pkg/sql/opt/memo/logical_props_builder_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (

"github.com/cockroachdb/cockroach/pkg/sql/opt/memo"
"github.com/cockroachdb/cockroach/pkg/sql/opt/testutils"
"github.com/cockroachdb/cockroach/pkg/sql/opt/testutils/testcat"
)

func TestLogicalPropsBuilder(t *testing.T) {
Expand Down Expand Up @@ -64,9 +65,13 @@ func TestHasCorrelatedSubquery(t *testing.T) {
}
}

func createLogPropsCatalog(t *testing.T) *testutils.TestCatalog {
cat := testutils.NewTestCatalog()
testutils.ExecuteTestDDL(t, "CREATE TABLE a (x INT PRIMARY KEY, y INT)", cat)
testutils.ExecuteTestDDL(t, "CREATE TABLE b (x INT PRIMARY KEY, z INT NOT NULL)", cat)
func createLogPropsCatalog(t *testing.T) *testcat.Catalog {
cat := testcat.New()
if _, err := cat.ExecuteDDL("CREATE TABLE a (x INT PRIMARY KEY, y INT)"); err != nil {
t.Fatal(err)
}
if _, err := cat.ExecuteDDL("CREATE TABLE b (x INT PRIMARY KEY, z INT NOT NULL)"); err != nil {
t.Fatal(err)
}
return cat
}
3 changes: 2 additions & 1 deletion pkg/sql/opt/memo/memo_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (

"github.com/cockroachdb/cockroach/pkg/sql/opt/memo"
"github.com/cockroachdb/cockroach/pkg/sql/opt/testutils"
"github.com/cockroachdb/cockroach/pkg/sql/opt/testutils/testcat"
"github.com/cockroachdb/cockroach/pkg/testutils/datadriven"
)

Expand All @@ -35,7 +36,7 @@ func TestMemo(t *testing.T) {
// See OptTester.Handle for supported commands.
func runDataDrivenTest(t *testing.T, path string, fmtFlags memo.ExprFmtFlags) {
datadriven.Walk(t, path, func(t *testing.T, path string) {
catalog := testutils.NewTestCatalog()
catalog := testcat.New()
datadriven.RunTest(t, path, func(d *datadriven.TestData) string {
tester := testutils.NewOptTester(catalog, d.Input)
tester.Flags.ExprFormat = fmtFlags
Expand Down
23 changes: 12 additions & 11 deletions pkg/sql/opt/memo/private_defs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,22 +20,23 @@ import (
"github.com/cockroachdb/cockroach/pkg/sql/opt"
"github.com/cockroachdb/cockroach/pkg/sql/opt/memo"
"github.com/cockroachdb/cockroach/pkg/sql/opt/props"
"github.com/cockroachdb/cockroach/pkg/sql/opt/testutils"
"github.com/cockroachdb/cockroach/pkg/sql/opt/testutils/testcat"
)

func TestCanProvideOrdering(t *testing.T) {
cat := testutils.NewTestCatalog()
testutils.ExecuteTestDDL(
t,
"CREATE TABLE a ("+
"k INT PRIMARY KEY, "+
"i INT, "+
"s STRING, "+
"f FLOAT, "+
"INDEX (i, k), "+
cat := testcat.New()
_, err := cat.ExecuteDDL(
"CREATE TABLE a (" +
"k INT PRIMARY KEY, " +
"i INT, " +
"s STRING, " +
"f FLOAT, " +
"INDEX (i, k), " +
"INDEX (s DESC) STORING(f))",
cat,
)
if err != nil {
t.Fatal(err)
}

md := opt.NewMetadata()
a := md.AddTable(cat.Table("a"))
Expand Down
35 changes: 14 additions & 21 deletions pkg/sql/opt/memo/testdata/logprops/join
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,37 +50,30 @@ project
├── columns: column7:7(int)
├── stats: [rows=1000]
├── inner-join-apply
│ ├── columns: xysd.x:1(int!null) xysd.y:2(int) column6:6(int)
│ ├── columns: xysd.x:1(int!null) xysd.y:2(int) column1:5(int)
│ ├── stats: [rows=1000]
│ ├── keys: (1)
│ ├── scan xysd
│ │ ├── columns: xysd.x:1(int!null) xysd.y:2(int)
│ │ ├── stats: [rows=1000]
│ │ └── keys: (1)
│ ├── project
│ │ ├── columns: column6:6(int)
│ ├── max1-row
│ │ ├── columns: column1:5(int)
│ │ ├── outer: (1,2)
│ │ ├── cardinality: [1 - 1]
│ │ ├── stats: [rows=1]
│ │ ├── max1-row
│ │ │ ├── columns: column1:5(int)
│ │ │ ├── outer: (1,2)
│ │ │ ├── cardinality: [1 - 1]
│ │ │ ├── stats: [rows=1]
│ │ │ └── values
│ │ │ ├── columns: column1:5(int)
│ │ │ ├── outer: (1,2)
│ │ │ ├── cardinality: [2 - 2]
│ │ │ ├── stats: [rows=2]
│ │ │ ├── tuple [type=tuple{int}, outer=(1)]
│ │ │ │ └── variable: xysd.x [type=int, outer=(1)]
│ │ │ └── tuple [type=tuple{int}, outer=(2)]
│ │ │ └── variable: xysd.y [type=int, outer=(2)]
│ │ └── projections [outer=(5)]
│ │ └── variable: column1 [type=int, outer=(5)]
│ │ └── values
│ │ ├── columns: column1:5(int)
│ │ ├── outer: (1,2)
│ │ ├── cardinality: [2 - 2]
│ │ ├── stats: [rows=2]
│ │ ├── tuple [type=tuple{int}, outer=(1)]
│ │ │ └── variable: xysd.x [type=int, outer=(1)]
│ │ └── tuple [type=tuple{int}, outer=(2)]
│ │ └── variable: xysd.y [type=int, outer=(2)]
│ └── true [type=bool]
└── projections [outer=(6)]
└── variable: column6 [type=int, outer=(6)]
└── projections [outer=(5)]
└── variable: column1 [type=int, outer=(5)]

# Left-join.
build
Expand Down
39 changes: 39 additions & 0 deletions pkg/sql/opt/memo/testdata/typing
View file
Original file line number Diff line number Diff line change
Expand Up @@ -493,3 +493,42 @@ group-by
│ └── variable: column20 [type=jsonb]
└── jsonb-agg [type=jsonb]
└── variable: column22 [type=jsonb]

# AnyNotNull internal aggregate function.
opt
SELECT * FROM (SELECT x, x::string, y FROM a) WHERE (SELECT MAX(x) FROM b WHERE y=z::int) > 'foo'
----
project
├── columns: x:1(int!null) column3:3(string) y:2(int)
└── select
├── columns: a.x:1(int!null) a.y:2(int) column3:3(string) column6:6(string!null)
├── group-by
│ ├── columns: a.x:1(int!null) a.y:2(int) column3:3(string) column6:6(string)
│ ├── grouping columns: a.x:1(int!null)
│ ├── left-join
│ │ ├── columns: a.x:1(int!null) a.y:2(int) column3:3(string) b.x:4(string) b.z:5(decimal)
│ │ ├── project
│ │ │ ├── columns: column3:3(string) a.x:1(int!null) a.y:2(int)
│ │ │ ├── scan a
│ │ │ │ └── columns: a.x:1(int!null) a.y:2(int)
│ │ │ └── projections
│ │ │ └── cast: string [type=string]
│ │ │ └── variable: a.x [type=int]
│ │ ├── scan b
│ │ │ └── columns: b.x:4(string!null) b.z:5(decimal!null)
│ │ └── filters [type=bool]
│ │ └── eq [type=bool]
│ │ ├── variable: a.y [type=int]
│ │ └── cast: int [type=int]
│ │ └── variable: b.z [type=decimal]
│ └── aggregations
│ ├── max [type=string]
│ │ └── variable: b.x [type=string]
│ ├── any-not-null [type=int]
│ │ └── variable: a.y [type=int]
│ └── any-not-null [type=string]
│ └── variable: column3 [type=string]
└── filters [type=bool]
└── gt [type=bool]
├── variable: column6 [type=string]
└── const: 'foo' [type=string]
16 changes: 9 additions & 7 deletions pkg/sql/opt/memo/typing.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,13 +126,15 @@ func init() {
opt.ArrayOp: typeAsPrivate,

// Override default typeAsAggregate behavior for aggregate functions with
// a large number of possible overloads.
opt.ArrayAggOp: typeArrayAgg,
opt.MaxOp: typeAsFirstArg,
opt.MinOp: typeAsFirstArg,

// ExistsAggOp is not part of SQL, and so is not part of the Builtins
// table that typeAsAggregate relies upon.
// a large number of possible overloads or where ReturnType depends on
// argument types.
opt.ArrayAggOp: typeArrayAgg,
opt.MaxOp: typeAsFirstArg,
opt.MinOp: typeAsFirstArg,
opt.AnyNotNullOp: typeAsFirstArg,

// These aggregate functions are not part of SQL, and so are not part of
// the Builtins table that typeAsAggregate relies upon.
opt.ExistsAggOp: typeAsBool,
}

Expand Down
66 changes: 34 additions & 32 deletions pkg/sql/opt/metadata_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import (
"testing"

"github.com/cockroachdb/cockroach/pkg/sql/opt"
"github.com/cockroachdb/cockroach/pkg/sql/opt/testutils"
"github.com/cockroachdb/cockroach/pkg/sql/opt/testutils/testcat"
"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
"github.com/cockroachdb/cockroach/pkg/sql/sem/types"
"github.com/cockroachdb/cockroach/pkg/util"
Expand Down Expand Up @@ -73,10 +73,10 @@ func TestMetadataTables(t *testing.T) {
md := opt.NewMetadata()

// Add a table reference to the metadata.
a := &testutils.TestTable{}
a := &testcat.Table{}
a.Name = tree.MakeUnqualifiedTableName(tree.Name("a"))
x := &testutils.TestColumn{Name: "x"}
y := &testutils.TestColumn{Name: "y"}
x := &testcat.Column{Name: "x"}
y := &testcat.Column{Name: "y"}
a.Columns = append(a.Columns, x, y)

tabID := md.AddTable(a)
Expand All @@ -100,8 +100,8 @@ func TestMetadataTables(t *testing.T) {
}

// Add a table reference without a name to the metadata.
b := &testutils.TestTable{}
b.Columns = append(b.Columns, &testutils.TestColumn{Name: "x"})
b := &testcat.Table{}
b.Columns = append(b.Columns, &testcat.Column{Name: "x"})

tabID = md.AddTable(b)
if tabID != 3 {
Expand Down Expand Up @@ -140,20 +140,22 @@ func TestMetadataWeakKeys(t *testing.T) {
// 4. Non-unique index (should always be superset of primary key).
// 5. Unique index that has subset of cols of another unique index, but
// which is defined afterwards (triggers removal of previous weak key).
cat := testutils.NewTestCatalog()
testutils.ExecuteTestDDL(t,
"CREATE TABLE a ("+
"k INT, "+
"i INT, "+
"d DECIMAL, "+
"f FLOAT, "+
"s STRING, "+
"PRIMARY KEY (k, i), "+
"UNIQUE INDEX (f) STORING (s, i),"+
"UNIQUE INDEX (d DESC, i, s),"+
"UNIQUE INDEX (d, i DESC) STORING (f),"+
"INDEX (s DESC, i))",
cat)
cat := testcat.New()
_, err := cat.ExecuteDDL(
"CREATE TABLE a (" +
"k INT, " +
"i INT, " +
"d DECIMAL, " +
"f FLOAT, " +
"s STRING, " +
"PRIMARY KEY (k, i), " +
"UNIQUE INDEX (f) STORING (s, i)," +
"UNIQUE INDEX (d DESC, i, s)," +
"UNIQUE INDEX (d, i DESC) STORING (f)," +
"INDEX (s DESC, i))")
if err != nil {
t.Fatal(err)
}
a := md.AddTable(cat.Table("a"))

wk := md.TableWeakKeys(a)
Expand Down Expand Up @@ -208,18 +210,18 @@ func TestMetadataWeakKeys(t *testing.T) {

// TestIndexColumns tests that we can extract a set of columns from an index ordinal.
func TestIndexColumns(t *testing.T) {
cat := testutils.NewTestCatalog()
testutils.ExecuteTestDDL(
t,
"CREATE TABLE a ("+
"k INT PRIMARY KEY, "+
"i INT, "+
"s STRING, "+
"f FLOAT, "+
"INDEX (i, k), "+
"INDEX (s DESC) STORING(f))",
cat,
)
cat := testcat.New()
_, err := cat.ExecuteDDL(
"CREATE TABLE a (" +
"k INT PRIMARY KEY, " +
"i INT, " +
"s STRING, " +
"f FLOAT, " +
"INDEX (i, k), " +
"INDEX (s DESC) STORING(f))")
if err != nil {
t.Fatal(err)
}

md := opt.NewMetadata()
a := md.AddTable(cat.Table("a"))
Expand Down
Loading

0 comments on commit 024c2fa

Please sign in to comment.