Skip to content

Added storage capabilities for GMS Extended Types #7457

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Hydrocharged merged 2 commits into from
Feb 7, 2024
Merged

Added storage capabilities for GMS Extended Types #7457

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
36efa2c
Added storage capabilities for GMS Extended Types
Hydrocharged Feb 7, 2024
0e4fe05
[ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/upda…
Hydrocharged Feb 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
186 changes: 96 additions & 90 deletions go/gen/fb/serial/encoding.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions go/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ require (
github.com/dolthub/fslock v0.0.3
github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488
github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81
github.com/dolthub/vitess v0.0.0-20240206204925-6acf16fa777c
github.com/dolthub/vitess v0.0.0-20240207121055-c057d2347007
github.com/dustin/go-humanize v1.0.1
github.com/fatih/color v1.13.0
github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
Expand Down Expand Up @@ -57,7 +57,7 @@ require (
github.com/cespare/xxhash v1.1.0
github.com/creasty/defaults v1.6.0
github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2
github.com/dolthub/go-mysql-server v0.17.1-0.20240207001922-0aaa10ab846e
github.com/dolthub/go-mysql-server v0.17.1-0.20240207124505-c0f397a6aaca
github.com/dolthub/swiss v0.1.0
github.com/goccy/go-json v0.10.2
github.com/google/go-github/v57 v57.0.0
Expand Down
10 changes: 4 additions & 6 deletions go/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,8 +183,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y=
github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168=
github.com/dolthub/go-mysql-server v0.17.1-0.20240207001922-0aaa10ab846e h1:adkEQion8xhowSQSk9NenvrpGt0U6sG6Yr1OaOA7Xjs=
github.com/dolthub/go-mysql-server v0.17.1-0.20240207001922-0aaa10ab846e/go.mod h1:Kjfapc/1feJ3/WgTgiY0LoR5x/3ax5PJ26brPnvYUO8=
github.com/dolthub/go-mysql-server v0.17.1-0.20240207124505-c0f397a6aaca h1:tI3X4fIUTOT0N8n+GYkPNa384WlJoOBcztK5c5mBzjU=
github.com/dolthub/go-mysql-server v0.17.1-0.20240207124505-c0f397a6aaca/go.mod h1:ANK0a6tyjrZ2cOzDJT3nFsDp80xksI4UfeijFlvnjwE=
github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514=
github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto=
github.com/dolthub/jsonpath v0.0.2-0.20240201003050-392940944c15 h1:sfTETOpsrNJPDn2KydiCtDgVu6Xopq8k3JP8PjFT22s=
Expand All @@ -195,10 +195,8 @@ github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9X
github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY=
github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc=
github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ=
github.com/dolthub/vitess v0.0.0-20240205203605-9e6c6d650813 h1:tGwsoLAMFQ+7FDEyIWOIJ1Vc/nptbFi0Fh7SQahB8ro=
github.com/dolthub/vitess v0.0.0-20240205203605-9e6c6d650813/go.mod h1:IwjNXSQPymrja5pVqmfnYdcy7Uv7eNJNBPK/MEh9OOw=
github.com/dolthub/vitess v0.0.0-20240206204925-6acf16fa777c h1:Zt23BHsxvPHGfpHV9k/FcsHqWZjfybyQQux2OLpRni8=
github.com/dolthub/vitess v0.0.0-20240206204925-6acf16fa777c/go.mod h1:IwjNXSQPymrja5pVqmfnYdcy7Uv7eNJNBPK/MEh9OOw=
github.com/dolthub/vitess v0.0.0-20240207121055-c057d2347007 h1:MvFoe0FnHhxQLyp4Ldw0HRj1yu83YErbtbr7XxhaIFk=
github.com/dolthub/vitess v0.0.0-20240207121055-c057d2347007/go.mod h1:IwjNXSQPymrja5pVqmfnYdcy7Uv7eNJNBPK/MEh9OOw=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
Expand Down
3 changes: 3 additions & 0 deletions go/libraries/doltcore/migrate/tuples.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,9 @@ func translateNomsField(ctx context.Context, ns tree.NodeStore, value types.Valu
case types.BlobKind:
return translateBlobField(ctx, ns, value.(types.Blob), idx, b)

case types.ExtendedKind:
return fmt.Errorf("extended types are invalid during migration")

default:
return fmt.Errorf("encountered unexpected NomsKind %s",
types.KindToString[nk])
Expand Down
11 changes: 10 additions & 1 deletion go/libraries/doltcore/schema/encoding/serialization.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -638,6 +638,15 @@ func sqlTypeString(t typeinfo.TypeInfo) string {
return typ.String()
}

// Extended types are string serializable, so we'll just prepend a tag
if extendedType, ok := typ.(sqltypes.ExtendedType); ok {
serializedType, err := sqltypes.SerializeTypeToString(extendedType)
if err != nil {
panic(err)
}
return planbuilder.ExtendedTypeTag + serializedType
}

return typ.String()
}

Expand All @@ -650,7 +659,7 @@ func typeinfoFromSqlType(s string) (typeinfo.TypeInfo, error) {
}

func encodingFromTypeinfo(t typeinfo.TypeInfo) serial.Encoding {
return schema.EncodingFromSqlType(t.ToSqlType().Type())
return schema.EncodingFromSqlType(t.ToSqlType())
}

func constraintsFromSerialColumn(col *serial.Column) (cc []schema.ColConstraint) {
Expand Down
32 changes: 24 additions & 8 deletions go/libraries/doltcore/schema/schema_impl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"strings"

"github.com/dolthub/go-mysql-server/sql"
gmstypes "github.com/dolthub/go-mysql-server/sql/types"
"github.com/dolthub/vitess/go/vt/proto/query"

"github.com/dolthub/dolt/go/gen/fb/serial"
Expand Down Expand Up @@ -438,6 +439,7 @@ func (si *schemaImpl) getKeyColumnsDescriptor(convertAddressColumns bool) val.Tu
}

var tt []val.Type
var handlers []val.TupleTypeHandler
useCollations := false // We only use collations if a string exists
var collations []sql.CollationID
_ = si.GetPKCols().Iter(func(tag uint64, col Column) (stop bool, err error) {
Expand All @@ -452,12 +454,12 @@ func (si *schemaImpl) getKeyColumnsDescriptor(convertAddressColumns bool) val.Tu

if convertAddressColumns && !contentHashedField && queryType == query.Type_BLOB {
t = val.Type{
Enc: val.Encoding(EncodingFromSqlType(query.Type_VARBINARY)),
Enc: val.Encoding(EncodingFromQueryType(query.Type_VARBINARY)),
Nullable: columnMissingNotNullConstraint(col),
}
} else if convertAddressColumns && !contentHashedField && queryType == query.Type_TEXT {
t = val.Type{
Enc: val.Encoding(EncodingFromSqlType(query.Type_VARCHAR)),
Enc: val.Encoding(EncodingFromQueryType(query.Type_VARCHAR)),
Nullable: columnMissingNotNullConstraint(col),
}
} else if convertAddressColumns && !contentHashedField && queryType == query.Type_GEOMETRY {
Expand All @@ -467,7 +469,7 @@ func (si *schemaImpl) getKeyColumnsDescriptor(convertAddressColumns bool) val.Tu
}
} else {
t = val.Type{
Enc: val.Encoding(EncodingFromSqlType(queryType)),
Enc: val.Encoding(EncodingFromSqlType(sqlType)),
Nullable: columnMissingNotNullConstraint(col),
}
}
Expand All @@ -478,6 +480,12 @@ func (si *schemaImpl) getKeyColumnsDescriptor(convertAddressColumns bool) val.Tu
} else {
collations = append(collations, sql.Collation_Unspecified)
}

if extendedType, ok := sqlType.(gmstypes.ExtendedType); ok {
handlers = append(handlers, extendedType)
} else {
handlers = append(handlers, nil)
}
return
})

Expand All @@ -486,18 +494,20 @@ func (si *schemaImpl) getKeyColumnsDescriptor(convertAddressColumns bool) val.Tu
panic(fmt.Errorf("cannot create tuple descriptor from %d collations and %d types", len(collations), len(tt)))
}
cmp := CollationTupleComparator{Collations: collations}
return val.NewTupleDescriptorWithComparator(cmp, tt...)
return val.NewTupleDescriptorWithArgs(val.TupleDescriptorArgs{Comparator: cmp, Handlers: handlers}, tt...)
} else {
return val.NewTupleDescriptor(tt...)
return val.NewTupleDescriptorWithArgs(val.TupleDescriptorArgs{Handlers: handlers}, tt...)
}
}

// GetValueDescriptor implements the Schema interface.
func (si *schemaImpl) GetValueDescriptor() val.TupleDesc {
var tt []val.Type
var handlers []val.TupleTypeHandler
var collations []sql.CollationID
if IsKeyless(si) {
tt = []val.Type{val.KeylessCardType}
handlers = []val.TupleTypeHandler{nil}
collations = []sql.CollationID{sql.Collation_Unspecified}
}

Expand All @@ -510,7 +520,7 @@ func (si *schemaImpl) GetValueDescriptor() val.TupleDesc {
sqlType := col.TypeInfo.ToSqlType()
queryType := sqlType.Type()
tt = append(tt, val.Type{
Enc: val.Encoding(EncodingFromSqlType(queryType)),
Enc: val.Encoding(EncodingFromSqlType(sqlType)),
Nullable: col.IsNullable(),
})
if queryType == query.Type_CHAR || queryType == query.Type_VARCHAR {
Expand All @@ -519,6 +529,12 @@ func (si *schemaImpl) GetValueDescriptor() val.TupleDesc {
} else {
collations = append(collations, sql.Collation_Unspecified)
}

if extendedType, ok := sqlType.(gmstypes.ExtendedType); ok {
handlers = append(handlers, extendedType)
} else {
handlers = append(handlers, nil)
}
return
})

Expand All @@ -527,9 +543,9 @@ func (si *schemaImpl) GetValueDescriptor() val.TupleDesc {
panic(fmt.Errorf("cannot create tuple descriptor from %d collations and %d types", len(collations), len(tt)))
}
cmp := CollationTupleComparator{Collations: collations}
return val.NewTupleDescriptorWithComparator(cmp, tt...)
return val.NewTupleDescriptorWithArgs(val.TupleDescriptorArgs{Comparator: cmp, Handlers: handlers}, tt...)
} else {
return val.NewTupleDescriptor(tt...)
return val.NewTupleDescriptorWithArgs(val.TupleDescriptorArgs{Handlers: handlers}, tt...)
}
}

Expand Down
Loading