Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
149 commits
Select commit Hold shift + click to select a range
796564b
[stats] event loop
max-hoffman Dec 30, 2024
5034635
more progress
max-hoffman Dec 31, 2024
8de66e4
basic scheduler test working
max-hoffman Jan 2, 2025
d6882e4
analyze
max-hoffman Jan 3, 2025
4c5bd3f
add/drop hooks
max-hoffman Jan 3, 2025
c2876de
gc
max-hoffman Jan 6, 2025
4d8d8f0
delete an alter
max-hoffman Jan 6, 2025
bb6ab3c
drop index and table
max-hoffman Jan 6, 2025
542bc40
fix other tests
max-hoffman Jan 6, 2025
14cf9fd
branch management
max-hoffman Jan 8, 2025
d949b3d
starter for kv
max-hoffman Jan 9, 2025
397aaa9
gc and refactor maintanance
max-hoffman Jan 14, 2025
16ff4ff
fix bucket doubling
max-hoffman Jan 15, 2025
1d04f74
delete log
max-hoffman Jan 15, 2025
2be37c1
better bucket counting
max-hoffman Jan 21, 2025
ee16cf1
test for disk round trip
max-hoffman Jan 21, 2025
d18b524
more prolly stats gc tests
max-hoffman Jan 21, 2025
ee2286b
rotate backing stats db
max-hoffman Jan 22, 2025
4a91332
progress towards swapping old for new, deleting old code
max-hoffman Jan 22, 2025
128efd5
fix gc bucket overflow
max-hoffman Jan 23, 2025
347d3f5
test for gc overflow
max-hoffman Jan 23, 2025
9bdb958
org and closers
max-hoffman Jan 23, 2025
d503c4e
save progress update
max-hoffman Jan 24, 2025
76a45ff
finally get first two bats running
max-hoffman Jan 25, 2025
373aa9a
startup bound hash issue
max-hoffman Jan 27, 2025
c56dd06
rewrite GC to be synchronous, fix more bugs
max-hoffman Jan 28, 2025
14eae29
fix session freshness
max-hoffman Jan 28, 2025
6ab5193
fix branch gc
max-hoffman Jan 29, 2025
474a85f
cache writes and gc are serialized
max-hoffman Jan 30, 2025
d8e6c09
fix gc/branch update dropped hashes
max-hoffman Jan 30, 2025
31d3780
fix gc race, doubling race, jobs race
max-hoffman Jan 31, 2025
c2c4f05
fix more races
max-hoffman Feb 3, 2025
e23cf1f
docs
max-hoffman Feb 4, 2025
4da767b
convert bats to script tests
max-hoffman Feb 4, 2025
8d3c07f
more tests, purge/stop
max-hoffman Feb 5, 2025
6578011
validate
max-hoffman Feb 6, 2025
aafeec7
docs
max-hoffman Feb 6, 2025
6dd1fb4
some PR cleanup
max-hoffman Feb 6, 2025
4635cfa
more cleanup
max-hoffman Feb 6, 2025
ec8ed11
stash for pull
max-hoffman Feb 6, 2025
c95fcda
merge
max-hoffman Feb 6, 2025
2e424eb
fix bucket hash conflicts
max-hoffman Feb 7, 2025
e849f27
Fix more collection bugs.
max-hoffman Feb 10, 2025
8224b09
bump, timer proc
max-hoffman Feb 10, 2025
43e739d
more test fixes
max-hoffman Feb 11, 2025
b209532
cache bats changes
max-hoffman Feb 11, 2025
043c174
merge main
max-hoffman Feb 11, 2025
7291054
Another deadlock
max-hoffman Feb 11, 2025
40cdce0
delete comment
max-hoffman Feb 11, 2025
d040cfa
fmt
max-hoffman Feb 11, 2025
6df2999
no read replica stats
max-hoffman Feb 11, 2025
fe72f62
fix plan tests
max-hoffman Feb 11, 2025
1be0f49
branch qualified analyze fix
max-hoffman Feb 11, 2025
7a35c3d
[no-release-notes] go: statspro/jobqueue: Create a SerialQueue, which…
reltuk Feb 12, 2025
468dafc
go: statspro/jobqueue: A bit of cleanup, fix a flakey test.
reltuk Feb 12, 2025
9c595c9
Merge branch 'aaron/jobqueue' into max/stats-event-loop
max-hoffman Feb 13, 2025
efa0412
rewrite with GDQ
max-hoffman Feb 14, 2025
e3811ee
prog
max-hoffman Feb 15, 2025
f46a1c4
tests run
max-hoffman Feb 17, 2025
ad9ed8e
fix info and storage
max-hoffman Feb 17, 2025
4910909
outline for gc impl
max-hoffman Feb 18, 2025
c31dd08
fix tests and races
max-hoffman Feb 18, 2025
6d2ea07
bump
max-hoffman Feb 18, 2025
f171a50
better error and panic management
max-hoffman Feb 18, 2025
8d8398a
better start/stop/wait
max-hoffman Feb 19, 2025
d637699
Add rate limiting
max-hoffman Feb 19, 2025
3c5f1a9
gc ticker
max-hoffman Feb 19, 2025
aa23dde
docs
max-hoffman Feb 19, 2025
4f926d3
doc
max-hoffman Feb 19, 2025
9e260cb
test prog
max-hoffman Feb 20, 2025
8a64097
fix more tests
max-hoffman Feb 20, 2025
9b3a8cb
finish up listener tests
max-hoffman Feb 21, 2025
ff64cfb
add comments
max-hoffman Feb 21, 2025
99c4e91
gc concurrency
max-hoffman Feb 21, 2025
f070e05
enginetests and statspro tests passing
max-hoffman Feb 25, 2025
791dff0
simplify listeners
max-hoffman Feb 25, 2025
c394276
bats progress
max-hoffman Feb 27, 2025
f53de92
small edits
max-hoffman Feb 28, 2025
7e58b09
tests progress
max-hoffman Mar 4, 2025
8e48425
merge main
max-hoffman Mar 4, 2025
e666a48
bats are running
max-hoffman Mar 4, 2025
40930aa
fmt
max-hoffman Mar 4, 2025
28947eb
build
max-hoffman Mar 4, 2025
1823dcc
edits
max-hoffman Mar 5, 2025
9535ffb
fix interface
max-hoffman Mar 5, 2025
a20401f
merge main
max-hoffman Mar 5, 2025
2c3c271
fix build
max-hoffman Mar 5, 2025
2d02535
stats alternate index types
max-hoffman Mar 5, 2025
258f5e9
fix mem test
max-hoffman Mar 5, 2025
5775bfe
tidy
max-hoffman Mar 5, 2025
002ece8
build
max-hoffman Mar 5, 2025
f656924
fix more tests
max-hoffman Mar 5, 2025
39ee000
fmt
max-hoffman Mar 5, 2025
380c51b
more fmt
max-hoffman Mar 5, 2025
657c032
merge main
max-hoffman Mar 5, 2025
2a32486
merge main
max-hoffman Mar 6, 2025
a28f36f
copyright
max-hoffman Mar 6, 2025
e3295a6
license
max-hoffman Mar 6, 2025
4e12e77
Merge branch 'main' into max/stats-event-loop
max-hoffman Mar 6, 2025
7e1d1b7
fix races
max-hoffman Mar 6, 2025
10f8bc3
syntax error
max-hoffman Mar 6, 2025
10e67a7
fix windows path
max-hoffman Mar 6, 2025
6b36433
nil mcv panic
max-hoffman Mar 6, 2025
58879d6
fix test races
max-hoffman Mar 6, 2025
e9d58f2
bump def job interval to 30ms
max-hoffman Mar 6, 2025
4d76ab2
deterministic tests
max-hoffman Mar 6, 2025
0e4c2e5
more tests
max-hoffman Mar 6, 2025
11bbba7
TEMP COMMIT: valctx plus some other stuff...
reltuk Feb 26, 2025
f07fef3
Merge branch 'aaron/nbs-session-lifecycle-sanity-checks' into max/sta…
max-hoffman Mar 6, 2025
bb69370
shorter concurrency tests
max-hoffman Mar 6, 2025
687f497
[ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/upda…
max-hoffman Mar 6, 2025
e73931a
nondeterministic test
max-hoffman Mar 6, 2025
7864cf1
Merge branch 'max/stats-event-loop' of github.com:dolthub/dolt into m…
max-hoffman Mar 6, 2025
b13e8cf
try to make queue tests less racy
max-hoffman Mar 7, 2025
6c8a0db
missed one start
max-hoffman Mar 7, 2025
c36381b
stats granular session locks
max-hoffman Mar 7, 2025
d1631d4
simplify a little
max-hoffman Mar 7, 2025
9e15822
merge main
max-hoffman Mar 7, 2025
7b68907
try to avoid serialq test deadlock
max-hoffman Mar 11, 2025
7579620
merge main
max-hoffman Mar 11, 2025
15d51a6
try to fix flakes
max-hoffman Mar 11, 2025
66e8c71
more races
max-hoffman Mar 11, 2025
a35b9c7
bump
max-hoffman Mar 11, 2025
9923a6e
merge and worker batch
max-hoffman Mar 11, 2025
cd6fb69
merge main
max-hoffman Mar 11, 2025
73db71f
another race
max-hoffman Mar 12, 2025
0772a00
cleanup
max-hoffman Mar 12, 2025
3f238d0
more cleanup
max-hoffman Mar 12, 2025
ac16aca
revert ctx validation
max-hoffman Mar 12, 2025
8591963
most zach comments
max-hoffman Mar 12, 2025
6ba5996
more comments
max-hoffman Mar 13, 2025
06d3419
more race
max-hoffman Mar 17, 2025
ca41655
Merge branch 'main' into max/stats-event-loop
max-hoffman Mar 17, 2025
22d7801
bump
max-hoffman Mar 17, 2025
0fb4128
merge main
max-hoffman Mar 17, 2025
91f0cf6
more race
max-hoffman Mar 17, 2025
a8fd924
bump
max-hoffman Mar 18, 2025
de47ba3
merge main
max-hoffman Mar 18, 2025
2a75378
schemas
max-hoffman Mar 18, 2025
4d0bab5
skip windows racees
max-hoffman Mar 19, 2025
05bcf2c
standardize server config init, use background threads management
max-hoffman Mar 19, 2025
c701726
[ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/upda…
max-hoffman Mar 19, 2025
cb57493
merge main
max-hoffman Mar 20, 2025
f71fbd5
Merge branch 'max/stats-event-loop' of github.com:dolthub/dolt into m…
max-hoffman Mar 20, 2025
a4d4f72
default stats noop
max-hoffman Mar 20, 2025
252efd0
threads management improvements
max-hoffman Mar 20, 2025
8fd7a05
undo change
max-hoffman Mar 20, 2025
bc7f15a
move stats initialization back to engine
max-hoffman Mar 20, 2025
73fcedc
[ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/upda…
max-hoffman Mar 20, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 0 additions & 33 deletions go/Godeps/LICENSES

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 8 additions & 2 deletions go/cmd/dolt/commands/archive.go
Original file line number Diff line number Diff line change
Expand Up @@ -310,8 +310,14 @@ func relateCommitToParentChunks(ctx context.Context, commit hash.Hash, groupings

from, to, err := delta.GetRowData(ctx)

f := durable.ProllyMapFromIndex(from)
t := durable.ProllyMapFromIndex(to)
f, err := durable.ProllyMapFromIndex(from)
if err != nil {
return err
}
t, err := durable.ProllyMapFromIndex(to)
if err != nil {
return err
}

if f.Node().Level() != t.Node().Level() {
continue
Expand Down
39 changes: 31 additions & 8 deletions go/cmd/dolt/commands/engine/sqlengine.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ package engine

import (
"context"
"fmt"
"os"
"strconv"
"strings"
Expand Down Expand Up @@ -45,7 +44,6 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/kvexec"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/mysql_file_handler"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/statsnoms"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/statspro"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/writer"
"github.com/dolthub/dolt/go/libraries/utils/config"
Expand Down Expand Up @@ -85,6 +83,7 @@ type SqlEngineConfig struct {
AutoGCController *dsqle.AutoGCController
BinlogReplicaController binlogreplication.BinlogReplicaController
EventSchedulerStatus eventscheduler.SchedulerStatus
StatsController sql.StatsProvider
}

// NewSqlEngine returns a SqlEngine
Expand Down Expand Up @@ -201,9 +200,6 @@ func NewSqlEngine(
"authentication_dolt_jwt": NewAuthenticateDoltJWTPlugin(config.JwksConfig),
})

statsPro := statspro.NewProvider(pro, statsnoms.NewNomsStatsFactory(mrEnv.RemoteDialProvider()))
engine.Analyzer.Catalog.StatsProvider = statsPro

if config.AutoGCController != nil {
err = config.AutoGCController.RunBackgroundThread(bThreads, sqlEngine.NewDefaultContext)
if err != nil {
Expand All @@ -216,8 +212,15 @@ func NewSqlEngine(
dprocedures.UseSessionAwareSafepointController = true
}

_, enabled, _ := sql.SystemVariables.GetGlobal(dsess.DoltStatsEnabled)
if enabled.(int8) == 1 {
config.StatsController = statspro.NewStatsController(logrus.StandardLogger(), mrEnv.GetEnv(mrEnv.GetFirstDatabase()))
} else {
config.StatsController = statspro.StatsNoop{}
}

engine.Analyzer.ExecBuilder = rowexec.NewOverrideBuilder(kvexec.Builder{})
sessFactory := doltSessionFactory(pro, statsPro, mrEnv.Config(), bcController, gcSafepointController, config.Autocommit)
sessFactory := doltSessionFactory(pro, config.StatsController, mrEnv.Config(), bcController, gcSafepointController, config.Autocommit)
sqlEngine.provider = pro
sqlEngine.contextFactory = sqlContextFactory
sqlEngine.dsessFactory = sessFactory
Expand All @@ -236,8 +239,28 @@ func NewSqlEngine(

// configuring stats depends on sessionBuilder
// sessionBuilder needs ref to statsProv
if err = statsPro.Configure(ctx, sqlEngine.NewDefaultContext, bThreads, dbs); err != nil {
fmt.Fprintln(cli.CliErr, err)
if sc, ok := config.StatsController.(*statspro.StatsController); ok {
_, memOnly, _ := sql.SystemVariables.GetGlobal(dsess.DoltStatsMemoryOnly)
sc.SetMemOnly(memOnly.(int8) == 1)

pro.InitDatabaseHooks = append(pro.InitDatabaseHooks, statspro.NewInitDatabaseHook(sc))
pro.DropDatabaseHooks = append(pro.DropDatabaseHooks, statspro.NewDropDatabaseHook(sc))

var sqlDbs []sql.Database
for _, db := range dbs {
sqlDbs = append(sqlDbs, db)
}

err = sc.Init(ctx, pro, sqlEngine.NewDefaultContext, bThreads, sqlDbs)
if err != nil {
return nil, err
}

if _, paused, _ := sql.SystemVariables.GetGlobal(dsess.DoltStatsPaused); paused.(int8) == 0 {
if err = sc.Restart(); err != nil {
return nil, err
}
}
}

// Load MySQL Db information
Expand Down
14 changes: 3 additions & 11 deletions go/cmd/dolt/commands/sqlserver/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -303,17 +303,9 @@ func ConfigureServices(
var sqlEngine *engine.SqlEngine
InitSqlEngine := &svcs.AnonService{
InitF: func(ctx context.Context) (err error) {
if statsOn, err := mrEnv.Config().GetString(env.SqlServerGlobalsPrefix + "." + dsess.DoltStatsAutoRefreshEnabled); err != nil {
// Auto-stats is off by default for every command except
// sql-server. Unless the config specifies a specific
// behavior, enable server stats collection.
sql.SystemVariables.SetGlobal(dsess.DoltStatsAutoRefreshEnabled, 1)
} else if statsOn != "0" {
// do not bootstrap if auto-stats enabled
} else if _, err := mrEnv.Config().GetString(env.SqlServerGlobalsPrefix + "." + dsess.DoltStatsBootstrapEnabled); err != nil {
// If we've disabled stats collection and config does not
// specify bootstrap behavior, enable bootstrapping.
sql.SystemVariables.SetGlobal(dsess.DoltStatsBootstrapEnabled, 1)
if _, err := mrEnv.Config().GetString(env.SqlServerGlobalsPrefix + "." + dsess.DoltStatsPaused); err != nil {
// unless otherwise specified, run stats writer alongside server
sql.SystemVariables.SetGlobal(dsess.DoltStatsPaused, 0)
}
sqlEngine, err = engine.NewSqlEngine(
ctx,
Expand Down
14 changes: 8 additions & 6 deletions go/cmd/dolt/commands/sqlserver/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package sqlserver

import (
"fmt"
"net/http"
"os"
"path/filepath"
Expand Down Expand Up @@ -184,11 +185,6 @@ func TestServerBadArgs(t *testing.T) {

func TestServerGoodParams(t *testing.T) {
ctx := context.Background()
env, err := sqle.CreateEnvWithSeedData()
require.NoError(t, err)
defer func() {
assert.NoError(t, env.DoltDB(ctx).Close())
}()

tests := []servercfg.ServerConfig{
DefaultCommandLineServerConfig(),
Expand All @@ -210,6 +206,11 @@ func TestServerGoodParams(t *testing.T) {

for _, test := range tests {
t.Run(servercfg.ConfigInfo(test), func(t *testing.T) {
env, err := sqle.CreateEnvWithSeedData()
require.NoError(t, err)
defer func() {
assert.NoError(t, env.DoltDB(ctx).Close())
}()
sc := svcs.NewController()
go func(config servercfg.ServerConfig, sc *svcs.Controller) {
_, _ = Serve(context.Background(), &Config{
Expand All @@ -219,7 +220,7 @@ func TestServerGoodParams(t *testing.T) {
DoltEnv: env,
})
}(test, sc)
err := sc.WaitForStart()
err = sc.WaitForStart()
require.NoError(t, err)
conn, err := dbr.Open("mysql", servercfg.ConnectionString(test, "dbname"), nil)
require.NoError(t, err)
Expand All @@ -228,6 +229,7 @@ func TestServerGoodParams(t *testing.T) {
sc.Stop()
err = sc.WaitForStop()
assert.NoError(t, err)
fmt.Println("stop server")
})
}
}
Expand Down
3 changes: 1 addition & 2 deletions go/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ require (
github.com/creasty/defaults v1.6.0
github.com/dolthub/aws-sdk-go-ini-parser v0.0.0-20250305001723-2821c37f6c12
github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2
github.com/dolthub/go-mysql-server v0.19.1-0.20250320042421-9a6edfcfab0d
github.com/dolthub/go-mysql-server v0.19.1-0.20250320173422-cce3ea1590af
github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63
github.com/esote/minmaxheap v1.0.0
github.com/goccy/go-json v0.10.2
Expand Down Expand Up @@ -94,7 +94,6 @@ require (
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
golang.org/x/text v0.22.0
gonum.org/v1/plot v0.11.0
gopkg.in/errgo.v2 v2.1.0
gopkg.in/go-jose/go-jose.v2 v2.6.3
gopkg.in/yaml.v3 v3.0.1
)
Expand Down
5 changes: 2 additions & 3 deletions go/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -221,8 +221,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
github.com/dolthub/go-icu-regex v0.0.0-20250319212010-451ea8d003fa h1:NFbzJ4wjWRz32nz2EimbrHpRx1Xt6k+IaR8N+j4x62k=
github.com/dolthub/go-icu-regex v0.0.0-20250319212010-451ea8d003fa/go.mod h1:ylU4XjUpsMcvl/BKeRRMXSH7e7WBrPXdSLvnRJYrxEA=
github.com/dolthub/go-mysql-server v0.19.1-0.20250320042421-9a6edfcfab0d h1:Ra9hv9fvJkSvjihPmtQB4EMGhq9qNp08gUI/mRmF9no=
github.com/dolthub/go-mysql-server v0.19.1-0.20250320042421-9a6edfcfab0d/go.mod h1:9itIc5jYYDRxmchFmegPaLaqdf4XWYX6nua5HhrajgA=
github.com/dolthub/go-mysql-server v0.19.1-0.20250320173422-cce3ea1590af h1:ozgYo2hKV6uQqLxZTS+QElHTaZ8mMiKOln25jZI1gVc=
github.com/dolthub/go-mysql-server v0.19.1-0.20250320173422-cce3ea1590af/go.mod h1:9itIc5jYYDRxmchFmegPaLaqdf4XWYX6nua5HhrajgA=
github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI=
github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q=
github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE=
Expand Down Expand Up @@ -1186,7 +1186,6 @@ gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
Expand Down
9 changes: 9 additions & 0 deletions go/go.work.sum
Original file line number Diff line number Diff line change
Expand Up @@ -386,6 +386,12 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/dolthub/go-mysql-server v0.19.1-0.20250228215144-f8da474ecd9f h1:lwQH9jVmSiPg1DFMYB9rWyyJTMPMoBpGrYRsOGOD/hA=
github.com/dolthub/go-mysql-server v0.19.1-0.20250228215144-f8da474ecd9f/go.mod h1:JTlrabhq5TJqvlL+J3NKlm0EzTHQQugUAH6yAxWi4Ww=
github.com/dolthub/go-mysql-server v0.19.1-0.20250305230031-14a57e076a0a h1:lemFIUt0NCKIeX7vnU2yKF8UIgc0DT8zIoEUn7oy+60=
github.com/dolthub/go-mysql-server v0.19.1-0.20250305230031-14a57e076a0a/go.mod h1:yr+Vv47/YLOKMgiEY+QxHTlbIVpTuiVtkEZ5l+xruY4=
github.com/dolthub/maphash v0.0.0-20221220182448-74e1e1ea1577 h1:SegEguMxToBn045KRHLIUlF2/jR7Y2qD6fF+3tdOfvI=
github.com/dolthub/maphash v0.0.0-20221220182448-74e1e1ea1577/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4=
github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc=
github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ=
github.com/dolthub/vitess v0.0.0-20241104125316-860772ba6683 h1:2/RJeUfNAXS7mbBnEr9C36htiCJHk5XldDPzhxtEsME=
github.com/dolthub/vitess v0.0.0-20241104125316-860772ba6683/go.mod h1:uBvlRluuL+SbEWTCZ68o0xvsdYZER3CEG/35INdzfJM=
github.com/dolthub/vitess v0.0.0-20241231200706-18992bb25fdc/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70=
Expand Down Expand Up @@ -651,6 +657,8 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/sts v1.0.588/go.mod h1:
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/tag v1.0.233/go.mod h1:sX14+NSvMjOhNFaMtP2aDy6Bss8PyFXij21gpY6+DAs=
github.com/tencentyun/cos-go-sdk-v5 v0.7.42/go.mod h1:LUFnaqRmGk6pEHOaRmdn2dCZR2j0cSsM5xowWFPTPao=
github.com/thanhpk/randstr v1.0.4/go.mod h1:M/H2P1eNLZzlDwAzpkkkUvoyNNMbzRGhESZuEQk3r0U=
github.com/thepudds/swisstable v0.0.0-20221011152303-9c77dc657777 h1:5u+6YWU2faS+Sr/x8j9yalMpSDUkatNOZWXV3wMUCGQ=
github.com/thepudds/swisstable v0.0.0-20221011152303-9c77dc657777/go.mod h1:4af3KxEsswy6aTzsTcwa8QZUSh4V+80oHdp1QX9uJHA=
github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e/go.mod h1:/Tnicc6m/lsJE0irFMA0LfIwTBo4QP7A8IfyIv4zZKI=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE=
github.com/tombuildsstuff/giovanni v0.15.1/go.mod h1:0TZugJPEtqzPlMpuJHYfXY6Dq2uLPrXf98D2XQSxNbA=
Expand Down Expand Up @@ -740,6 +748,7 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/cheggaaa/pb.v1 v1.0.25 h1:Ev7yu1/f6+d+b3pi5vPdRPc6nNtP1umSfcWiEfRqv6I=
gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
Expand Down
11 changes: 8 additions & 3 deletions go/libraries/doltcore/diff/diff_stat.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,11 +105,16 @@ func diffProllyTrees(ctx context.Context, ch chan DiffStatProgress, keyless bool

var f, t prolly.Map
if from != nil {
f = durable.ProllyMapFromIndex(from)
f, err = durable.ProllyMapFromIndex(from)
if err != nil {
return err
}
}
if to != nil {
t = durable.ProllyMapFromIndex(to)

t, err = durable.ProllyMapFromIndex(to)
if err != nil {
return err
}
}

_, fVD := f.Descriptors()
Expand Down
12 changes: 6 additions & 6 deletions go/libraries/doltcore/doltdb/doltdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -2110,17 +2110,17 @@ func (ddb *DoltDB) AddStash(ctx context.Context, head *Commit, stash RootValue,
return err
}

func (ddb *DoltDB) SetStatisics(ctx context.Context, branch string, addr hash.Hash) error {
statsDs, err := ddb.db.GetDataset(ctx, ref.NewStatsRef(branch).String())
func (ddb *DoltDB) SetStatistics(ctx context.Context, branch string, addr hash.Hash) error {
statsDs, err := ddb.db.GetDataset(ctx, ref.NewStatsRef().String())
if err != nil {
return err
}
_, err = ddb.db.SetStatsRef(ctx, statsDs, addr)
return err
}

func (ddb *DoltDB) DropStatisics(ctx context.Context, branch string) error {
statsDs, err := ddb.db.GetDataset(ctx, ref.NewStatsRef(branch).String())
func (ddb *DoltDB) DropStatisics(ctx context.Context) error {
statsDs, err := ddb.db.GetDataset(ctx, ref.NewStatsRef().String())

_, err = ddb.db.Delete(ctx, statsDs, "")
if err != nil {
Expand All @@ -2132,8 +2132,8 @@ func (ddb *DoltDB) DropStatisics(ctx context.Context, branch string) error {
var ErrNoStatistics = errors.New("no statistics found")

// GetStatistics returns the value of the singleton ref.StatsRef for this database
func (ddb *DoltDB) GetStatistics(ctx context.Context, branch string) (prolly.Map, error) {
ds, err := ddb.db.GetDataset(ctx, ref.NewStatsRef(branch).String())
func (ddb *DoltDB) GetStatistics(ctx context.Context) (prolly.Map, error) {
ds, err := ddb.db.GetDataset(ctx, ref.NewStatsRef().String())
if err != nil {
return prolly.Map{}, err
}
Expand Down
14 changes: 11 additions & 3 deletions go/libraries/doltcore/doltdb/durable/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -273,8 +273,13 @@ type prollyIndex struct {
}

// ProllyMapFromIndex unwraps the Index and returns the underlying prolly.Map.
func ProllyMapFromIndex(i Index) prolly.Map {
return i.(prollyIndex).index
func ProllyMapFromIndex(i Index) (prolly.Map, error) {
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Kind of a large change. Not necessarily opposed, but is this strictly to accommodate VectorIndexes? And if so, is there a less intrusive change that accomplishes it?

Seems like a panic is as good as an error for catching this kind of mismatch in logic where it needs to be updated.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I mean i can undo it and handle it separately for stats if you'd prefer. I'd be surprised if this doesn't also affect merges and schema changes

switch i := i.(type) {
case prollyIndex:
return i.index, nil
default:
return prolly.Map{}, fmt.Errorf("expected prollyIndex, found: %T", i)
}
}

// xxx: don't use this, temporary fix waiting for bigger
Expand Down Expand Up @@ -369,7 +374,10 @@ func (i prollyIndex) AddColumnToRows(ctx context.Context, newCol string, newSche
}

// If not, then we have to iterate over this table's rows and update all the offsets for the new column
rowMap := ProllyMapFromIndex(i)
rowMap, err := ProllyMapFromIndex(i)
if err != nil {
return nil, err
}
mutator := rowMap.Mutate()

iter, err := mutator.IterAll(ctx)
Expand Down
5 changes: 4 additions & 1 deletion go/libraries/doltcore/merge/fulltext_rebuild.go
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,10 @@ func createRowIterForTable(ctx *sql.Context, t *doltdb.Table, sch schema.Schema)
if err != nil {
return nil, err
}
rows := durable.ProllyMapFromIndex(rowData)
rows, err := durable.ProllyMapFromIndex(rowData)
if err != nil {
return nil, err
}
rowCount, err := rows.Count()
if err != nil {
return nil, err
Expand Down
5 changes: 4 additions & 1 deletion go/libraries/doltcore/merge/fulltext_table.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,10 @@ func (table *fulltextTable) ApplyToTable(ctx *sql.Context) (*doltdb.Table, error
if err != nil {
return nil, err
}
m := durable.ProllyMapFromIndex(idx)
m, err := durable.ProllyMapFromIndex(idx)
if err != nil {
return nil, err
}
keyDesc, valDesc := m.Descriptors()
keyMap, valMap := ordinalMappingsFromSchema(table.SqlSch, table.Sch)
mut := m.Mutate()
Expand Down
4 changes: 2 additions & 2 deletions go/libraries/doltcore/merge/keyless_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -403,7 +403,7 @@ func assertNomsConflicts(t *testing.T, ctx context.Context, tbl *doltdb.Table, e
func mustGetRowValueFromTable(t *testing.T, ctx context.Context, tbl *doltdb.Table, key val.Tuple) val.Tuple {
idx, err := tbl.GetRowData(ctx)
require.NoError(t, err)
m := durable.ProllyMapFromIndex(idx)
m, _ := durable.ProllyMapFromIndex(idx)

var value val.Tuple
err = m.Get(ctx, key, func(_, v val.Tuple) error {
Expand Down Expand Up @@ -438,7 +438,7 @@ func assertKeylessRows(t *testing.T, ctx context.Context, tbl *doltdb.Table, exp
func assertKeylessProllyRows(t *testing.T, ctx context.Context, tbl *doltdb.Table, expected []keylessEntry) {
idx, err := tbl.GetRowData(ctx)
require.NoError(t, err)
m := durable.ProllyMapFromIndex(idx)
m, _ := durable.ProllyMapFromIndex(idx)

expectedSet := mustHash128Set(expected...)

Expand Down
Loading
Loading