Added generated system tables to pg_catalog tables

core/context.go

@@ -58,8 +58,8 @@ func getContextValues(ctx *sql.Context) (*contextValues, error) {
 	return cv, nil
 }
 
-// getRootFromContext returns the working session's root from the context, along with the session.
-func getRootFromContext(ctx *sql.Context) (*dsess.DoltSession, *RootValue, error) {
+// GetRootFromContext returns the working session's root from the context, along with the session.
+func GetRootFromContext(ctx *sql.Context) (*dsess.DoltSession, *RootValue, error) {
 	return getRootFromContextForDatabase(ctx, "")
 }
 
@@ -120,7 +120,7 @@ func SetPgCatalogCache(ctx *sql.Context, pgCatalogCache any) error {
 
 // GetDoltTableFromContext returns the Dolt table from the context. Returns nil if no table was found.
 func GetDoltTableFromContext(ctx *sql.Context, tableName doltdb.TableName) (*doltdb.Table, error) {
-	_, root, err := getRootFromContext(ctx)
+	_, root, err := GetRootFromContext(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -220,7 +220,7 @@ func GetFunctionsCollectionFromContext(ctx *sql.Context) (*functions.Collection,
 	if err != nil {
 		return nil, err
 	}
-	_, root, err := getRootFromContext(ctx)
+	_, root, err := GetRootFromContext(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -293,7 +293,7 @@ func GetTypesCollectionFromContext(ctx *sql.Context) (*typecollection.TypeCollec
 		return nil, err
 	}
 	if cv.types == nil {
-		_, root, err := getRootFromContext(ctx)
+		_, root, err := GetRootFromContext(ctx)
 		if err != nil {
 			return nil, err
 		}

core/schema.go

@@ -22,7 +22,7 @@ import (
 // GetCurrentSchema returns the current schema used by the context. Defaults to "public" if the context does not specify
 // a schema.
 func GetCurrentSchema(ctx *sql.Context) (string, error) {
-	_, root, err := getRootFromContext(ctx)
+	_, root, err := GetRootFromContext(ctx)
 	if err != nil {
 		return "", nil
 	}

go.mod

@@ -6,13 +6,13 @@ require (
 	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/cockroachdb/apd/v2 v2.0.3-0.20200518165714-d020e156310a
 	github.com/cockroachdb/errors v1.7.5
-	github.com/dolthub/dolt/go v0.40.5-0.20250610173122-93301be37322
+	github.com/dolthub/dolt/go v0.40.5-0.20250619014715-a87c3da8efaf
 	github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20241119094239-f4e529af734d
 	github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2
 	github.com/dolthub/go-icu-regex v0.0.0-20250327004329-6799764f2dad
-	github.com/dolthub/go-mysql-server v0.20.1-0.20250609235012-75f15bbda63a
+	github.com/dolthub/go-mysql-server v0.20.1-0.20250618175653-7d22a78f4c71
 	github.com/dolthub/sqllogictest/go v0.0.0-20240618184124-ca47f9354216
-	github.com/dolthub/vitess v0.0.0-20250609213846-75541d7ef20a
+	github.com/dolthub/vitess v0.0.0-20250611225316-90a5898bfe26
 	github.com/fatih/color v1.13.0
 	github.com/goccy/go-json v0.10.2
 	github.com/gogo/protobuf v1.3.2

go.sum

@@ -256,8 +256,8 @@ github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5Xh
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dolthub/aws-sdk-go-ini-parser v0.0.0-20250305001723-2821c37f6c12 h1:IdqX7J8vi/Kn3T3Ee0VzqnLqwFmgA2hr8WZETPcQjfM=
 github.com/dolthub/aws-sdk-go-ini-parser v0.0.0-20250305001723-2821c37f6c12/go.mod h1:rN7X8BHwkjPcfMQQ2QTAq/xM3leUSGLfb+1Js7Y6TVo=
-github.com/dolthub/dolt/go v0.40.5-0.20250610173122-93301be37322 h1:a1RFFm9S7evoi52dDLYalO4dJ0ODZp0fpDzeGaRnMmU=
-github.com/dolthub/dolt/go v0.40.5-0.20250610173122-93301be37322/go.mod h1:bCnd7kgVzluInixqje/NRPwykkjStWr8bLM+O8q6YB4=
+github.com/dolthub/dolt/go v0.40.5-0.20250619014715-a87c3da8efaf h1:enouuGG0sJAlCN11sthy0ngFYN26vVvxDkgILI8VRRw=
+github.com/dolthub/dolt/go v0.40.5-0.20250619014715-a87c3da8efaf/go.mod h1:iqXXu2G7HkrrV8G5v5a0I60HaxLeLSfjZLdP64ZepYM=
 github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20241119094239-f4e529af734d h1:gO9+wrmNHXukPNCO1tpfCcXIdMlW/qppbUStfLvqz/U=
 github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20241119094239-f4e529af734d/go.mod h1:L5RDYZbC9BBWmoU2+TjTekeqqhFXX5EqH9ln00O0stY=
 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 h1:u3PMzfF8RkKd3lB9pZ2bfn0qEG+1Gms9599cr0REMww=
@@ -266,8 +266,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
 github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
 github.com/dolthub/go-icu-regex v0.0.0-20250327004329-6799764f2dad h1:66ZPawHszNu37VPQckdhX1BPPVzREsGgNxQeefnlm3g=
 github.com/dolthub/go-icu-regex v0.0.0-20250327004329-6799764f2dad/go.mod h1:ylU4XjUpsMcvl/BKeRRMXSH7e7WBrPXdSLvnRJYrxEA=
-github.com/dolthub/go-mysql-server v0.20.1-0.20250609235012-75f15bbda63a h1:Q7obvACozcTOz11+L55dh1muPtNweIJdzNY5m0PLfDU=
-github.com/dolthub/go-mysql-server v0.20.1-0.20250609235012-75f15bbda63a/go.mod h1:yVcBcDGOyYHmciKmQ20JZ+Mr12lOjSnzpmuBMZKPQGo=
+github.com/dolthub/go-mysql-server v0.20.1-0.20250618175653-7d22a78f4c71 h1:KMg3/8YX0Jcs/AtghLroZ+RYp8FRo0Vvww9i2PmpUfw=
+github.com/dolthub/go-mysql-server v0.20.1-0.20250618175653-7d22a78f4c71/go.mod h1:zuYoQ3keJHAvWUWMLzbP9anvR32b3sy1Fm8wB8ukNxQ=
 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI=
 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q=
 github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE=
@@ -276,8 +276,8 @@ github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 h1:bMGS25NWAGTE
 github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71/go.mod h1:2/2zjLQ/JOOSbbSboojeg+cAwcRV0fDLzIiWch/lhqI=
 github.com/dolthub/sqllogictest/go v0.0.0-20240618184124-ca47f9354216 h1:JWkKRE4EHUcEVQCMRBej8DYxjYjRz/9MdF/NNQh0o70=
 github.com/dolthub/sqllogictest/go v0.0.0-20240618184124-ca47f9354216/go.mod h1:e/FIZVvT2IR53HBCAo41NjqgtEnjMJGKca3Y/dAmZaA=
-github.com/dolthub/vitess v0.0.0-20250609213846-75541d7ef20a h1:DWQt6KSgrkZYuxzvGflImldau0a3IfINhEGQnFst/pw=
-github.com/dolthub/vitess v0.0.0-20250609213846-75541d7ef20a/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70=
+github.com/dolthub/vitess v0.0.0-20250611225316-90a5898bfe26 h1:9Npf0JYVCrwe9edTfYD/pjIncCePNDiu4j50xLcV334=
+github.com/dolthub/vitess v0.0.0-20250611225316-90a5898bfe26/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=

server/ast/expr.go

@@ -361,36 +361,43 @@ func nodeExpr(ctx *Context, node tree.Expr) (vitess.Expr, error) {
 				Expression: pgexprs.NewBinaryOperator(framework.Operator_BinaryNotEqual),
 				Children:   vitess.Exprs{left, right},
 			}, nil
-		case tree.In:
+		case tree.In, tree.NotIn:
+			var innerExpression vitess.InjectedExpr
 			switch right := right.(type) {
 			case vitess.ValTuple:
-				return vitess.InjectedExpr{
+				innerExpression = vitess.InjectedExpr{
 					Expression: pgexprs.NewInTuple(),
 					Children:   vitess.Exprs{left, right},
-				}, nil
+				}
 			case *vitess.Subquery:
-				return vitess.InjectedExpr{
+				innerExpression = vitess.InjectedExpr{
 					Expression: pgexprs.NewInSubquery(),
 					Children:   vitess.Exprs{left, right},
-				}, nil
+				}
 			case vitess.InjectedExpr:
 				if _, ok := right.Expression.(*pgexprs.RecordExpr); ok {
-					return vitess.InjectedExpr{
+					innerExpression = vitess.InjectedExpr{
 						Expression: pgexprs.NewInTuple(),
 						Children:   vitess.Exprs{left, vitess.ValTuple(right.Children)},
-					}, nil
+					}
 				}
 			}
-			return nil, errors.Errorf("right side of IN expression is not a tuple or subquery, got %T", right)
-		case tree.NotIn:
-			innerExpr := vitess.InjectedExpr{
-				Expression: pgexprs.NewInTuple(),
-				Children:   vitess.Exprs{left, right},
+
+			if innerExpression.Expression == nil {
+				return nil, errors.Errorf("right side of IN expression is not a tuple or subquery, got %T", right)
+			}
+
+			switch node.Operator {
+			case tree.In:
+				return innerExpression, nil
+			case tree.NotIn:
+				return vitess.InjectedExpr{
+					Expression: pgexprs.NewNot(),
+					Children:   vitess.Exprs{innerExpression},
+				}, nil
+			default:
+				return nil, errors.Errorf("unknown comparison operator used")
 			}
-			return vitess.InjectedExpr{
-				Expression: pgexprs.NewNot(),
-				Children:   vitess.Exprs{innerExpr},
-			}, nil
 		case tree.Like:
 			operator = vitess.LikeStr
 		case tree.NotLike:

server/expression/in_subquery.go

@@ -15,12 +15,12 @@
 package expression
 
 import (
-	"context"
 	"fmt"
 
 	"github.com/cockroachdb/errors"
 	"github.com/dolthub/go-mysql-server/sql"
 	"github.com/dolthub/go-mysql-server/sql/expression"
+	"github.com/dolthub/go-mysql-server/sql/hash"
 	"github.com/dolthub/go-mysql-server/sql/plan"
 	"github.com/dolthub/go-mysql-server/sql/types"
 	vitess "github.com/dolthub/vitess/go/vt/sqlparser"
@@ -46,7 +46,7 @@ var _ sql.Expression = (*InSubquery)(nil)
 var _ expression.BinaryExpression = (*InSubquery)(nil)
 
 // nilKey is the hash of a row with a single nil value.
-var nilKey, _ = sql.HashOf(context.TODO(), sql.NewRow(nil))
+var nilKey, _ = hash.HashOf(nil, nil, sql.NewRow(nil))
 
 // NewInSubquery returns a new *InSubquery.
 func NewInSubquery() *InSubquery {
@@ -97,7 +97,7 @@ func (in *InSubquery) Eval(ctx *sql.Context, row sql.Row) (any, error) {
 
 	// TODO: it might be possible for the left value to hash to a different value than the right even though they pass
 	//  an equality check. We need to perform a type conversion here to catch this case.
-	key, err := sql.HashOf(ctx, sql.NewRow(left))
+	key, err := hash.HashOf(ctx, nil, sql.NewRow(left))
 	if err != nil {
 		return nil, err
 	}

server/tables/database.go

@@ -35,7 +35,7 @@ func (d Database) GetTableInsensitive(ctx *sql.Context, tblName string) (sql.Tab
 	// exact search.
 	if tableMap, ok := handlers[d.db.Schema()]; ok {
 		if handler, ok := tableMap[tblName]; ok {
-			return NewVirtualTable(handler), true, nil
+			return NewVirtualTable(handler, d.db), true, nil
 		}
 	}
 	return nil, false, nil

server/tables/pgcatalog/init.go

@@ -14,11 +14,17 @@
 
 package pgcatalog
 
+import "os"
+
 // PgCatalogName is a constant to the pg_catalog name.
 const PgCatalogName = "pg_catalog"
 
+// includeSystemTables is a flag to determine whether to include system tables in the pg_catalog tables.
+var includeSystemTables = true
+
 // Init initializes everything necessary for the pg_catalog tables.
 func Init() {
+	InitIncludeSystemTables()
 	InitPgAggregate()
 	InitPgAm()
 	InitPgAmop()
@@ -159,3 +165,11 @@ func Init() {
 	InitPgUserMappings()
 	InitPgViews()
 }
+
+func InitIncludeSystemTables() {
+	if _, ok := os.LookupEnv("REGRESSION_TESTING"); ok {
+		// In CI regression tests, we exclude system tables to make them faster.
+		// None of them rely on the presence of system tables.
+		includeSystemTables = false
+	}
+}

server/tables/pgcatalog/pg_attribute.go

@@ -17,8 +17,12 @@ package pgcatalog
 import (
 	"io"
 
+	"github.com/cockroachdb/errors"
+	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
+	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve"
 	"github.com/dolthub/go-mysql-server/sql"
 
+	"github.com/dolthub/doltgresql/core"
 	"github.com/dolthub/doltgresql/core/id"
 	"github.com/dolthub/doltgresql/server/functions"
 	"github.com/dolthub/doltgresql/server/tables"
@@ -69,6 +73,38 @@ func (p PgAttributeHandler) RowIter(ctx *sql.Context) (sql.RowIter, error) {
 		if err != nil {
 			return nil, err
 		}
+
+		if includeSystemTables {
+			_, root, err := core.GetRootFromContext(ctx)
+			if err != nil {
+				return nil, err
+			}
+
+			systemTables, err := resolve.GetGeneratedSystemTables(ctx, root)
+			if err != nil {
+				return nil, err
+			}
+
+			db := ctx.GetCurrentDatabase()
+			for _, tblName := range systemTables {
+				tbl, err := core.GetSqlTableFromContext(ctx, db, tblName)
+				if err != nil {
+					// Some of the system tables exist conditionally when accessed, so just skip them in this case
+					if errors.Is(doltdb.ErrTableNotFound, err) {
+						continue
+					}
+					return nil, err
+				}
+
+				schema := tbl.Schema()
+				for i, col := range schema {
+					cols = append(cols, col)
+					colIdxs = append(colIdxs, i)
+					tableOIDs = append(tableOIDs, id.NewTable(tblName.Schema, tblName.Name).AsId())
+				}
+			}
+		}
+
 		pgCatalogCache.attributeCols = cols
 		pgCatalogCache.attributeColIdxs = colIdxs
 		pgCatalogCache.attributeTableOIDs = tableOIDs

server/tables/pgcatalog/pg_catalog_cache.go

@@ -16,6 +16,7 @@ package pgcatalog
 
 import (
 	"github.com/cockroachdb/errors"
+	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
 
 	"github.com/dolthub/go-mysql-server/sql"
 
@@ -75,7 +76,7 @@ type pgCatalogCache struct {
 
 	// pg_tables
 	tables       []sql.Table
-	tableSchemas []string
+	systemTables []doltdb.TableName
 }
 
 // newPgCatalogCache creates a new pgCatalogCache, with the query/process ID set to |pid|. The PID is important,

server/tables/pgcatalog/pg_class.go

@@ -18,8 +18,10 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve"
 	"github.com/dolthub/go-mysql-server/sql"
 
+	"github.com/dolthub/doltgresql/core"
 	"github.com/dolthub/doltgresql/core/id"
 	"github.com/dolthub/doltgresql/server/functions"
 	"github.com/dolthub/doltgresql/server/tables"
@@ -103,6 +105,28 @@ func (p PgClassHandler) RowIter(ctx *sql.Context) (sql.RowIter, error) {
 		if err != nil {
 			return nil, err
 		}
+
+		if includeSystemTables {
+			_, root, err := core.GetRootFromContext(ctx)
+			if err != nil {
+				return nil, err
+			}
+
+			systemTables, err := resolve.GetGeneratedSystemTables(ctx, root)
+			if err != nil {
+				return nil, err
+			}
+
+			for _, tblName := range systemTables {
+				classes = append(classes, pgClass{
+					oid:       id.NewTable(tblName.Schema, tblName.Name).AsId(),
+					name:      tblName.Name,
+					schemaOid: id.NewNamespace(tblName.Schema).AsId(),
+					kind:      "r",
+				})
+			}
+		}
+
 		pgCatalogCache.pgClasses = classes
 	}