Skip to content

Root Objects #8965

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Hydrocharged merged 2 commits into from
Mar 28, 2025
Merged

Root Objects #8965

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4808f7c
Root Objects
Hydrocharged Mar 25, 2025
db61ca6
[ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/upda…
Hydrocharged Mar 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 1 addition & 7 deletions go/go.work.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -447,7 +447,6 @@ github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pS
github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-github/v45 v45.2.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28=
github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
Expand Down Expand Up @@ -526,6 +525,7 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
Expand Down Expand Up @@ -692,8 +692,6 @@ go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4=
golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
Expand All @@ -707,17 +705,13 @@ golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4
golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2 h1:IRJeR9r1pYWsHKTRe/IInb7lYvbBVIqOgsX/u0mbOWY=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
Expand Down
174 changes: 109 additions & 65 deletions go/libraries/doltcore/diff/table_deltas.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/utils/set"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
Expand Down Expand Up @@ -54,6 +55,8 @@ type TableDelta struct {
ToName doltdb.TableName
FromTable *doltdb.Table
ToTable *doltdb.Table
FromRootObject doltdb.RootObject
ToRootObject doltdb.RootObject
FromNodeStore tree.NodeStore
ToNodeStore tree.NodeStore
FromVRW types.ValueReadWriter
Expand Down Expand Up @@ -145,9 +148,23 @@ func GetTableDeltas(ctx context.Context, fromRoot, toRoot doltdb.RootValue) (del
if err != nil {
return nil, err
}
err = fromRoot.IterRootObjects(ctx, func(name doltdb.TableName, tbl doltdb.RootObject) (stop bool, err error) {
fromDeltas = append(fromDeltas, TableDelta{
FromName: name,
FromRootObject: tbl,
FromSch: schema.EmptySchema,
FromVRW: fromVRW,
FromNodeStore: fromNS,
ToVRW: toVRW,
ToNodeStore: toNS,
})
return
})
if err != nil {
return nil, err
}

toDeltas := make([]TableDelta, 0)

err = toRoot.IterTables(ctx, func(name doltdb.TableName, tbl *doltdb.Table, sch schema.Schema) (stop bool, err error) {
c, err := toRoot.GetForeignKeyCollection(ctx)
if err != nil {
Expand Down Expand Up @@ -176,6 +193,21 @@ func GetTableDeltas(ctx context.Context, fromRoot, toRoot doltdb.RootValue) (del
if err != nil {
return nil, err
}
err = toRoot.IterRootObjects(ctx, func(name doltdb.TableName, tbl doltdb.RootObject) (stop bool, err error) {
toDeltas = append(toDeltas, TableDelta{
ToName: name,
ToRootObject: tbl,
ToSch: schema.EmptySchema,
FromVRW: fromVRW,
FromNodeStore: fromNS,
ToVRW: toVRW,
ToNodeStore: toNS,
})
return
})
if err != nil {
return nil, err
}

deltas = matchTableDeltas(fromDeltas, toDeltas)
deltas, err = filterUnmodifiedTableDeltas(deltas)
Expand Down Expand Up @@ -235,7 +267,7 @@ func getFkParentSchs(ctx context.Context, root doltdb.RootValue, fks ...doltdb.F
func filterUnmodifiedTableDeltas(deltas []TableDelta) ([]TableDelta, error) {
var filtered []TableDelta
for _, d := range deltas {
if d.ToTable == nil || d.FromTable == nil {
if d.IsAdd() || d.IsDrop() {
// Table was added or dropped
filtered = append(filtered, d)
continue
Expand Down Expand Up @@ -276,6 +308,8 @@ func matchTableDeltas(fromDeltas, toDeltas []TableDelta) (deltas []TableDelta) {
ToName: t.ToName,
FromTable: f.FromTable,
ToTable: t.ToTable,
FromRootObject: f.FromRootObject,
ToRootObject: t.ToRootObject,
FromSch: f.FromSch,
ToSch: t.ToSch,
FromFks: f.FromFks,
Expand Down Expand Up @@ -339,12 +373,12 @@ func schemasOverlap(from, to schema.Schema) bool {

// IsAdd returns true if the table was added between the fromRoot and toRoot.
func (td TableDelta) IsAdd() bool {
return td.FromTable == nil && td.ToTable != nil
return (td.FromTable == nil && td.ToTable != nil) || (td.FromRootObject == nil && td.ToRootObject != nil)
}

// IsDrop returns true if the table was dropped between the fromRoot and toRoot.
func (td TableDelta) IsDrop() bool {
return td.FromTable != nil && td.ToTable == nil
return (td.FromTable != nil && td.ToTable == nil) || (td.FromRootObject != nil && td.ToRootObject == nil)
}

// IsRename return true if the table was renamed between the fromRoot and toRoot.
Expand All @@ -362,22 +396,39 @@ func (td TableDelta) HasHashChanged() (bool, error) {
return true, nil
}

toHash, err := td.ToTable.HashOf()
if err != nil {
return false, err
}
var toHash hash.Hash
var fromHash hash.Hash
var err error

fromHash, err := td.FromTable.HashOf()
if err != nil {
return false, err
if td.FromTable != nil || td.ToTable != nil {
toHash, err = td.ToTable.HashOf()
if err != nil {
return false, err
}
fromHash, err = td.FromTable.HashOf()
if err != nil {
return false, err
}
} else {
ctx := context.Background()
toHash, err = td.ToRootObject.HashOf(ctx)
if err != nil {
return false, err
}
fromHash, err = td.FromRootObject.HashOf(ctx)
if err != nil {
return false, err
}
}

return !toHash.Equal(fromHash), nil
}

// HasSchemaChanged returns true if the table schema has changed between the
// fromRoot and toRoot.
func (td TableDelta) HasSchemaChanged(ctx context.Context) (bool, error) {
if td.FromRootObject != nil || td.ToRootObject != nil {
return false, nil
}
// Database collation change is a schema change
if td.FromTable == nil && td.ToTable == nil {
return true, nil
Expand Down Expand Up @@ -419,71 +470,56 @@ func (td TableDelta) HasSchemaChanged(ctx context.Context) (bool, error) {
}

func (td TableDelta) HasChangesIgnoringColumnTags(ctx context.Context) (bool, error) {

if td.FromTable == nil && td.ToTable == nil {
if td.FromTable == nil && td.ToTable == nil && td.FromRootObject == nil && td.ToRootObject == nil {
return true, nil
}

if td.IsAdd() || td.IsDrop() {
return true, nil
}

if td.IsRename() {
return true, nil
}

if td.HasFKChanges() {
return true, nil
}

fromRowDataHash, err := td.FromTable.GetRowDataHash(ctx)
if err != nil {
return false, err
}

toRowDataHash, err := td.ToTable.GetRowDataHash(ctx)
if err != nil {
return false, err
}

// Any change to the table data counts as a change
if !fromRowDataHash.Equal(toRowDataHash) {
return true, nil
}

fromTableHash, err := td.FromTable.HashOf()
if err != nil {
return false, err
}

toTableHash, err := td.FromTable.HashOf()
if err != nil {
return false, err
}

// If the data hashes have changed, the table has obviously changed.
if !fromTableHash.Equal(toTableHash) {
return true, nil
}

fromSchemaHash, err := td.FromTable.GetSchemaHash(ctx)
if err != nil {
return false, err
}

toSchemaHash, err := td.ToTable.GetSchemaHash(ctx)
if err != nil {
return false, err
}
// There are specific checks that we make for regular tables
if td.FromTable != nil || td.ToTable != nil {
// Any change to the table data counts as a change
fromRowDataHash, err := td.FromTable.GetRowDataHash(ctx)
if err != nil {
return false, err
}
toRowDataHash, err := td.ToTable.GetRowDataHash(ctx)
if err != nil {
return false, err
}
if !fromRowDataHash.Equal(toRowDataHash) {
return true, nil
}

// If neither data nor schema hashes have changed, the table is obviously the same.
if fromSchemaHash.Equal(toSchemaHash) {
return false, nil
// If neither data nor schema hashes have changed, the table is the same
fromSchemaHash, err := td.FromTable.GetSchemaHash(ctx)
if err != nil {
return false, err
}
toSchemaHash, err := td.ToTable.GetSchemaHash(ctx)
if err != nil {
return false, err
}
if fromSchemaHash.Equal(toSchemaHash) {
return false, nil
}
} else {
// If we're not dealing with regular tables, then we must be dealing with root objects.
// In that case, it's a straight-forward check to see if the hashes changed.
if ok, err := td.HasHashChanged(); err != nil || ok {
return ok, err
}
}

// The schema hash has changed but the data has remained the same. We must inspect the schema to determine
// whether the change is observable or if only column tags have changed.

fromSchema, toSchema, err := td.GetSchemas(ctx)
if err != nil {
return false, err
Expand All @@ -495,19 +531,21 @@ func (td TableDelta) HasChangesIgnoringColumnTags(ctx context.Context) (bool, er

func (td TableDelta) HasDataChanged(ctx context.Context) (bool, error) {
// Database collation change is not a data change
if td.FromTable == nil && td.ToTable == nil {
if td.FromTable == nil && td.ToTable == nil && td.FromRootObject == nil && td.ToRootObject == nil {
return false, nil
}

if td.FromRootObject != nil || td.ToRootObject != nil {
return td.HasHashChanged()
}

if td.IsAdd() {
isEmpty, err := isTableDataEmpty(ctx, td.ToTable)
if err != nil {
return false, err
}

return !isEmpty, nil
}

if td.IsDrop() {
isEmpty, err := isTableDataEmpty(ctx, td.FromTable)
if err != nil {
Expand All @@ -520,12 +558,10 @@ func (td TableDelta) HasDataChanged(ctx context.Context) (bool, error) {
if err != nil {
return false, err
}

toRowDataHash, err := td.ToTable.GetRowDataHash(ctx)
if err != nil {
return false, err
}

return !fromRowDataHash.Equal(toRowDataHash), nil
}

Expand Down Expand Up @@ -596,6 +632,9 @@ func (td TableDelta) GetSchemas(ctx context.Context) (from, to schema.Schema, er

// Format returns the format of the tables in this delta.
func (td TableDelta) Format() *types.NomsBinFormat {
if td.FromRootObject != nil || td.ToRootObject != nil {
return types.Format_DOLT
}
if td.FromTable != nil {
return td.FromTable.Format()
}
Expand Down Expand Up @@ -693,6 +732,11 @@ func (td TableDelta) GetSummary(ctx context.Context) (*TableDeltaSummary, error)

// GetRowData returns the table's row data at the fromRoot and toRoot, or an empty map if the table did not exist.
func (td TableDelta) GetRowData(ctx context.Context) (from, to durable.Index, err error) {
if td.FromRootObject != nil || td.ToRootObject != nil {
// TODO: determine if this should error
return nil, nil, nil
}

if td.FromTable == nil && td.ToTable == nil {
return nil, nil, fmt.Errorf("both from and to tables are missing from table delta")
}
Expand Down
30 changes: 30 additions & 0 deletions go/libraries/doltcore/doltdb/root_object.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package doltdb

import (
"context"

"github.com/dolthub/dolt/go/store/hash"
)

// RootObject is an object that is located on the root, rather than inside another object (table, schema, etc.). This is
// primarily used by Doltgres to store root-level objects (sequences, functions, etc.).
type RootObject interface {
// HashOf returns the hash of the underlying struct.
HashOf(ctx context.Context) (hash.Hash, error)
// Name returns the name of the custom table.
Name() TableName
}
Loading
Loading