From d5b2092702c21a957679c237837d34734479bcd0 Mon Sep 17 00:00:00 2001 From: macneale4 <46170177+macneale4@users.noreply.github.com> Date: Tue, 25 Mar 2025 03:19:27 +0000 Subject: [PATCH 01/16] [ga-bump-dep] Bump dependency in Dolt by macneale4 --- go/go.mod | 4 ++-- go/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go/go.mod b/go/go.mod index d853d3f1e67..ea9de6ce73f 100644 --- a/go/go.mod +++ b/go/go.mod @@ -14,7 +14,7 @@ require ( github.com/dolthub/fslock v0.0.3 github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 - github.com/dolthub/vitess v0.0.0-20250320231804-0e77d549294c + github.com/dolthub/vitess v0.0.0-20250325024605-8131be3ca6d3 github.com/dustin/go-humanize v1.0.1 github.com/fatih/color v1.13.0 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 @@ -60,7 +60,7 @@ require ( github.com/creasty/defaults v1.6.0 github.com/dolthub/aws-sdk-go-ini-parser v0.0.0-20250305001723-2821c37f6c12 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.19.1-0.20250321183729-47285a54773c + github.com/dolthub/go-mysql-server v0.19.1-0.20250325031739-45c4c4ce0449 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 github.com/esote/minmaxheap v1.0.0 github.com/goccy/go-json v0.10.2 diff --git a/go/go.sum b/go/go.sum index e84c69f9eb9..ea48af52a5a 100644 --- a/go/go.sum +++ b/go/go.sum @@ -221,8 +221,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= github.com/dolthub/go-icu-regex v0.0.0-20250319212010-451ea8d003fa h1:NFbzJ4wjWRz32nz2EimbrHpRx1Xt6k+IaR8N+j4x62k= github.com/dolthub/go-icu-regex v0.0.0-20250319212010-451ea8d003fa/go.mod h1:ylU4XjUpsMcvl/BKeRRMXSH7e7WBrPXdSLvnRJYrxEA= -github.com/dolthub/go-mysql-server v0.19.1-0.20250321183729-47285a54773c h1:bCPelFSgt6aOPUqvrKV3OY8S9tjcBxQTgFNGARsrcSU= -github.com/dolthub/go-mysql-server v0.19.1-0.20250321183729-47285a54773c/go.mod h1:WyimbJzsrNiYsQ7nfnXYLIoFAfNX0I7rHH1WaDt7faM= +github.com/dolthub/go-mysql-server v0.19.1-0.20250325031739-45c4c4ce0449 h1:pW0XNR2cdUTGmLXyvFux5hZQRFJlHK12fWeC5vdmWNo= +github.com/dolthub/go-mysql-server v0.19.1-0.20250325031739-45c4c4ce0449/go.mod h1:vKBT9NZQDwG0Db6CT4D92Mw6kaJpWbIDy1hBX7XWUCA= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q= github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE= @@ -231,8 +231,8 @@ github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 h1:bMGS25NWAGTE github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71/go.mod h1:2/2zjLQ/JOOSbbSboojeg+cAwcRV0fDLzIiWch/lhqI= github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9XGFa6q5Ap4Z/OhNkAMBaK5YeuEzwJt+NZdhiE= github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY= -github.com/dolthub/vitess v0.0.0-20250320231804-0e77d549294c h1:Dv2DfEGb8WRBi8I5KF5Sy39TuZi/FI692mpobKWcv4g= -github.com/dolthub/vitess v0.0.0-20250320231804-0e77d549294c/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70= +github.com/dolthub/vitess v0.0.0-20250325024605-8131be3ca6d3 h1:euU+adNAYw46Zcp1HnoaSDWhqjfaL8s/1SPU+i16gYM= +github.com/dolthub/vitess v0.0.0-20250325024605-8131be3ca6d3/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= From 115aec82d77281854a19c0322badac759f639560 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Wed, 19 Mar 2025 15:40:36 -0700 Subject: [PATCH 02/16] Add support for back_log option to server config and flags. No tests. No SQL support --- .../dolt/commands/sqlserver/command_line_config.go | 12 ++++++++++++ go/cmd/dolt/commands/sqlserver/server.go | 1 + go/cmd/dolt/commands/sqlserver/sqlserver.go | 4 ++++ go/libraries/doltcore/servercfg/serverconfig.go | 5 +++++ go/libraries/doltcore/servercfg/yaml_config.go | 12 ++++++++++++ 5 files changed, 34 insertions(+) diff --git a/go/cmd/dolt/commands/sqlserver/command_line_config.go b/go/cmd/dolt/commands/sqlserver/command_line_config.go index b90b1e1adb1..6c7de3887c7 100755 --- a/go/cmd/dolt/commands/sqlserver/command_line_config.go +++ b/go/cmd/dolt/commands/sqlserver/command_line_config.go @@ -42,6 +42,7 @@ type commandLineServerConfig struct { autoCommit bool doltTransactionCommit bool maxConnections uint64 + maxWaitConnections uint32 tlsKey string tlsCert string requireSecureTransport bool @@ -72,6 +73,7 @@ func DefaultCommandLineServerConfig() *commandLineServerConfig { logFormat: servercfg.DefaultLogFormat, autoCommit: servercfg.DefaultAutoCommit, maxConnections: servercfg.DefaultMaxConnections, + maxWaitConnections: servercfg.DefaultMaxWaitConnections, dataDir: servercfg.DefaultDataDir, cfgDir: filepath.Join(servercfg.DefaultDataDir, servercfg.DefaultCfgDir), privilegeFilePath: filepath.Join(servercfg.DefaultDataDir, servercfg.DefaultCfgDir, servercfg.DefaultPrivilegeFilePath), @@ -169,6 +171,10 @@ func NewCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResult config.withMaxConnections(uint64(maxConnections)) } + if maxWaitConnections, ok := apr.GetInt(maxWaitConnectionsFlag); ok { + config.maxWaitConnections = uint32(maxWaitConnections) + } + config.autoCommit = !apr.Contains(noAutoCommitFlag) if apr.Contains(noAutoCommitFlag) { config.valuesSet[servercfg.AutoCommitKey] = struct{}{} @@ -258,6 +264,12 @@ func (cfg *commandLineServerConfig) MaxConnections() uint64 { return cfg.maxConnections } +// MaxWaitConnections returns the maximum number of simultaneous connections that the server will allow to block waiting +// for a connection before new connections result in immediate rejection. +func (cfg *commandLineServerConfig) MaxWaitConnections() uint32 { + return cfg.maxWaitConnections +} + // TLSKey returns a path to the servers PEM-encoded private TLS key. "" if there is none. func (cfg *commandLineServerConfig) TLSKey() string { return cfg.tlsKey diff --git a/go/cmd/dolt/commands/sqlserver/server.go b/go/cmd/dolt/commands/sqlserver/server.go index 0f8087a9561..366c4af12c3 100644 --- a/go/cmd/dolt/commands/sqlserver/server.go +++ b/go/cmd/dolt/commands/sqlserver/server.go @@ -1092,6 +1092,7 @@ func getConfigFromServerConfig(serverConfig servercfg.ServerConfig, plf server.P serverConf.ConnReadTimeout = readTimeout serverConf.ConnWriteTimeout = writeTimeout serverConf.MaxConnections = serverConfig.MaxConnections() + serverConf.MaxWaitConnections = serverConfig.MaxWaitConnections() serverConf.TLSConfig = tlsConfig serverConf.RequireSecureTransport = serverConfig.RequireSecureTransport() serverConf.MaxLoggedQueryLen = serverConfig.MaxLoggedQueryLen() diff --git a/go/cmd/dolt/commands/sqlserver/sqlserver.go b/go/cmd/dolt/commands/sqlserver/sqlserver.go index 48f2df51538..402e698e667 100644 --- a/go/cmd/dolt/commands/sqlserver/sqlserver.go +++ b/go/cmd/dolt/commands/sqlserver/sqlserver.go @@ -48,6 +48,7 @@ const ( configFileFlag = "config" queryParallelismFlag = "query-parallelism" maxConnectionsFlag = "max-connections" + maxWaitConnectionsFlag = "back-log" allowCleartextPasswordsFlag = "allow-cleartext-passwords" socketFlag = "socket" remotesapiPortFlag = "remotesapi-port" @@ -107,6 +108,8 @@ SUPPORTED CONFIG FILE FIELDS: {{.EmphasisLeft}}listener.max_connections{{.EmphasisRight}}: The number of simultaneous connections that the server will accept +{{.EmphasisLeft}}listener.back_log{{.EmphasisRight}}: The number of simultaneous connections that the server will allow to block waiting for a connection before new connections result in immediate rejection. Default 50. + {{.EmphasisLeft}}listener.read_timeout_millis{{.EmphasisRight}}: The number of milliseconds that the server will wait for a read operation {{.EmphasisLeft}}listener.write_timeout_millis{{.EmphasisRight}}: The number of milliseconds that the server will wait for a write operation @@ -179,6 +182,7 @@ func (cmd SqlServerCmd) ArgParserWithName(name string) *argparser.ArgParser { ap.SupportsFlag(noAutoCommitFlag, "", "Set @@autocommit = off for the server.") ap.SupportsInt(queryParallelismFlag, "", "num-go-routines", "Deprecated, no effect in current versions of Dolt") ap.SupportsInt(maxConnectionsFlag, "", "max-connections", fmt.Sprintf("Set the number of connections handled by the server. Defaults to `%d`.", serverConfig.MaxConnections())) + ap.SupportsInt(maxWaitConnectionsFlag, "", "back-log", fmt.Sprintf("Set the number of connections that can block waiting for a connection before new connections are rejected. Defaults to `%d`.", serverConfig.MaxWaitConnections())) ap.SupportsString(commands.PrivsFilePathFlag, "", "privilege file", "Path to a file to load and store users and grants. Defaults to `$doltcfg-dir/privileges.db`. Will be created as needed.") ap.SupportsString(commands.BranchCtrlPathFlag, "", "branch control file", "Path to a file to load and store branch control permissions. Defaults to `$doltcfg-dir/branch_control.db`. Will be created as needed.") ap.SupportsString(allowCleartextPasswordsFlag, "", "allow-cleartext-passwords", "Allows use of cleartext passwords. Defaults to false.") diff --git a/go/libraries/doltcore/servercfg/serverconfig.go b/go/libraries/doltcore/servercfg/serverconfig.go index 2b48640387e..51c83d97ef4 100644 --- a/go/libraries/doltcore/servercfg/serverconfig.go +++ b/go/libraries/doltcore/servercfg/serverconfig.go @@ -59,6 +59,7 @@ const ( DefaultAutoGCBehaviorEnable = false DefaultDoltTransactionCommit = false DefaultMaxConnections = 1000 + DefaultMaxWaitConnections = 50 DefaultDataDir = "." DefaultCfgDir = ".doltcfg" DefaultPrivilegeFilePath = "privileges.db" @@ -159,6 +160,9 @@ type ServerConfig interface { CfgDir() string // MaxConnections returns the maximum number of simultaneous connections the server will allow. The default is 1 MaxConnections() uint64 + // MaxWaitConnections returns the maximum number of simultaneous connections that the server will allow to block waiting + // for a connection before new connections result in immediate rejection + MaxWaitConnections() uint32 // TLSKey returns a path to the servers PEM-encoded private TLS key. "" if there is none. TLSKey() string // TLSCert returns a path to the servers PEM-encoded TLS certificate chain. "" if there is none. @@ -240,6 +244,7 @@ func defaultServerConfigYAML() *YAMLConfig { HostStr: ptr(DefaultHost), PortNumber: ptr(DefaultPort), MaxConnections: ptr(uint64(DefaultMaxConnections)), + BackLog: ptr(uint32(DefaultMaxWaitConnections)), ReadTimeoutMillis: ptr(uint64(DefaultTimeout)), WriteTimeoutMillis: ptr(uint64(DefaultTimeout)), AllowCleartextPasswords: ptr(DefaultAllowCleartextPasswords), diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index c5bfe483033..21afacf29a7 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -80,6 +80,7 @@ type ListenerYAMLConfig struct { HostStr *string `yaml:"host,omitempty"` PortNumber *int `yaml:"port,omitempty"` MaxConnections *uint64 `yaml:"max_connections,omitempty"` + BackLog *uint32 `yaml:"back_log,omitempty"` ReadTimeoutMillis *uint64 `yaml:"read_timeout_millis,omitempty"` WriteTimeoutMillis *uint64 `yaml:"write_timeout_millis,omitempty"` // TLSKey is a file system path to an unencrypted private TLS key in PEM format. @@ -487,6 +488,9 @@ func (cfg YAMLConfig) withDefaultsFilledIn() YAMLConfig { if withDefaults.ListenerConfig.MaxConnections == nil { withDefaults.ListenerConfig.MaxConnections = defaults.ListenerConfig.MaxConnections } + if withDefaults.ListenerConfig.BackLog == nil { + withDefaults.ListenerConfig.BackLog = defaults.ListenerConfig.BackLog + } if withDefaults.ListenerConfig.ReadTimeoutMillis == nil { withDefaults.ListenerConfig.ReadTimeoutMillis = defaults.ListenerConfig.ReadTimeoutMillis } @@ -659,6 +663,14 @@ func (cfg YAMLConfig) MaxConnections() uint64 { return *cfg.ListenerConfig.MaxConnections } +func (cfg YAMLConfig) MaxWaitConnections() uint32 { + if cfg.ListenerConfig.BackLog == nil { + return DefaultMaxWaitConnections + } + + return *cfg.ListenerConfig.BackLog +} + // DisableClientMultiStatements returns true if the server should run in a mode // where the CLIENT_MULTI_STATEMENTS option are ignored and every incoming // ComQuery packet is assumed to be a standalone query. From 309ba1d3d44926aa5cd20b97b826d497af938cf0 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Wed, 19 Mar 2025 16:26:56 -0700 Subject: [PATCH 03/16] Test the --back-log value gets written to config.yaml --- go/cmd/dolt/commands/sqlserver/command_line_config.go | 8 +++++++- go/libraries/doltcore/servercfg/serverconfig.go | 1 + go/libraries/doltcore/servercfg/yaml_config.go | 4 ++++ .../bats/sql-server-config-file-generation.bats | 2 ++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/go/cmd/dolt/commands/sqlserver/command_line_config.go b/go/cmd/dolt/commands/sqlserver/command_line_config.go index 6c7de3887c7..05eff0cf7f6 100755 --- a/go/cmd/dolt/commands/sqlserver/command_line_config.go +++ b/go/cmd/dolt/commands/sqlserver/command_line_config.go @@ -172,7 +172,7 @@ func NewCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResult } if maxWaitConnections, ok := apr.GetInt(maxWaitConnectionsFlag); ok { - config.maxWaitConnections = uint32(maxWaitConnections) + config.withMaxWaitConnections(uint32(maxWaitConnections)) } config.autoCommit = !apr.Contains(noAutoCommitFlag) @@ -437,6 +437,12 @@ func (cfg *commandLineServerConfig) withMaxConnections(maxConnections uint64) *c return cfg } +func (cfg *commandLineServerConfig) withMaxWaitConnections(maxWaitConnections uint32) *commandLineServerConfig { + cfg.maxWaitConnections = maxWaitConnections + cfg.valuesSet[servercfg.MaxWaitConnectionsKey] = struct{}{} + return cfg +} + // withDataDir updates the path to a directory to use as the data dir. func (cfg *commandLineServerConfig) withDataDir(dataDir string) *commandLineServerConfig { cfg.dataDir = dataDir diff --git a/go/libraries/doltcore/servercfg/serverconfig.go b/go/libraries/doltcore/servercfg/serverconfig.go index 51c83d97ef4..d12fc16a7c0 100644 --- a/go/libraries/doltcore/servercfg/serverconfig.go +++ b/go/libraries/doltcore/servercfg/serverconfig.go @@ -311,6 +311,7 @@ const ( DataDirKey = "data_dir" CfgDirKey = "cfg_dir" MaxConnectionsKey = "max_connections" + MaxWaitConnectionsKey = "back_log" TLSKeyKey = "tls_key" TLSCertKey = "tls_cert" RequireSecureTransportKey = "require_secure_transport" diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index 21afacf29a7..664d68e04a4 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -198,6 +198,7 @@ func ServerConfigAsYAMLConfig(cfg ServerConfig) *YAMLConfig { HostStr: ptr(cfg.Host()), PortNumber: ptr(cfg.Port()), MaxConnections: ptr(cfg.MaxConnections()), + BackLog: ptr(cfg.MaxWaitConnections()), ReadTimeoutMillis: ptr(cfg.ReadTimeout()), WriteTimeoutMillis: ptr(cfg.WriteTimeout()), TLSKey: nillableStrPtr(cfg.TLSKey()), @@ -268,6 +269,7 @@ func ServerConfigSetValuesAsYAMLConfig(cfg ServerConfig) *YAMLConfig { HostStr: zeroIf(ptr(cfg.Host()), !cfg.ValueSet(HostKey)), PortNumber: zeroIf(ptr(cfg.Port()), !cfg.ValueSet(PortKey)), MaxConnections: zeroIf(ptr(cfg.MaxConnections()), !cfg.ValueSet(MaxConnectionsKey)), + BackLog: zeroIf(ptr(cfg.MaxWaitConnections()), !cfg.ValueSet(MaxWaitConnectionsKey)), ReadTimeoutMillis: zeroIf(ptr(cfg.ReadTimeout()), !cfg.ValueSet(ReadTimeoutKey)), WriteTimeoutMillis: zeroIf(ptr(cfg.WriteTimeout()), !cfg.ValueSet(WriteTimeoutKey)), TLSKey: zeroIf(ptr(cfg.TLSKey()), !cfg.ValueSet(TLSKeyKey)), @@ -958,6 +960,8 @@ func (cfg YAMLConfig) ValueSet(value string) bool { return cfg.ListenerConfig.WriteTimeoutMillis != nil case MaxConnectionsKey: return cfg.ListenerConfig.MaxConnections != nil + case MaxWaitConnectionsKey: + return cfg.ListenerConfig.BackLog != nil case EventSchedulerKey: return cfg.BehaviorConfig.EventSchedulerStatus != nil } diff --git a/integration-tests/bats/sql-server-config-file-generation.bats b/integration-tests/bats/sql-server-config-file-generation.bats index 924a16541ed..ee3fa60235c 100644 --- a/integration-tests/bats/sql-server-config-file-generation.bats +++ b/integration-tests/bats/sql-server-config-file-generation.bats @@ -98,6 +98,7 @@ EOF --max-connections 77 \ --timeout 7777777 \ --allow-cleartext-passwords true \ + --back-log 767 \ --host 0.0.0.0 run cat "$CONFIG_FILE_NAME" @@ -108,6 +109,7 @@ EOF [[ "$output" =~ "read_timeout_millis: 7777777" ]] || false [[ "$output" =~ "write_timeout_millis: 7777777" ]] || false [[ "$output" =~ "allow_cleartext_passwords: true" ]] || false + [[ "$output" =~ "back_log: 767" ]] || false } @test "sql-server-config-file-generation: generated config file uses default values as placeholders for unset fields" { From 66180b5623edc29761ed2e06f0ab0d276e82fb4e Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Thu, 20 Mar 2025 13:13:11 -0700 Subject: [PATCH 04/16] Reliable first test for max-conns and back-log --- integration-tests/bats/sql-server.bats | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/integration-tests/bats/sql-server.bats b/integration-tests/bats/sql-server.bats index db48a231994..8f0e856fc3b 100644 --- a/integration-tests/bats/sql-server.bats +++ b/integration-tests/bats/sql-server.bats @@ -2115,3 +2115,29 @@ EOF [[ "$output" =~ "br3 | true" ]] || false [[ "$output" =~ "main | false" ]] || false } + +@test "sql-server: test --max-connections and --back-log flags" { + cd repo1 + # 3 connections allowed, everything else immediate failure. + start_sql_server_with_args --max-connections 3 --back-log 0 + + # Start 3 long running connections. + set +e # errors expected. + pids=() + for i in {1..3}; do + dolt sql & + pids+=($!) + done + + # Give the first 3 connections time to start/connect + sleep 5 + + run dolt sql -q "select(1);" + [ $status -ne 0 ] + [[ "$output" =~ "bad connection" ]] || false + + # Kill all the running shells + for pid in "${pids[@]}"; do + kill -9 "$pid" + done +} From 11dcccd1cc4e3b9068d931bed4bc037160154274 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Thu, 20 Mar 2025 13:53:44 -0700 Subject: [PATCH 05/16] Add more tests for max cons and back log --- integration-tests/bats/sql-server.bats | 93 ++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/integration-tests/bats/sql-server.bats b/integration-tests/bats/sql-server.bats index 8f0e856fc3b..98dda65fa3e 100644 --- a/integration-tests/bats/sql-server.bats +++ b/integration-tests/bats/sql-server.bats @@ -2141,3 +2141,96 @@ EOF kill -9 "$pid" done } + +@test "sql-server: test --max-connections 3 flag" { + cd repo1 + + dolt sql -q "CREATE TABLE test_table ( + id INT AUTO_INCREMENT PRIMARY KEY, + str VARCHAR(20) + );" + dolt commit -A -m "create test_table" + + # 3 connections allowed, default back-log (50). + start_sql_server_with_args --max-connections 3 + + set +e # errors expected. + + # Start 3 long running connections. + pids=() + for i in {1..3}; do + dolt sql & + pids+=($!) + done + + sleep 1 # give all connections a chance to start + + # These jobs will wait until there is a connection available. verify + # by ensuring the inserts complete. + dolt sql -q "insert into test_table (str) values ('test4223');" & + second_to_last_pid=$! + dolt sql -q "insert into test_table (str) values ('test9119');" & + last_pid=$! + + sleep 1 + + # Now have two jobs waiting for a connection. + + # Kill all the running shells + for pid in "${pids[@]}"; do + kill -9 "$pid" + done + + wait "$second_to_last_pid" + wait "$last_pid" + + run dolt sql -q "select * from test_table;" + [ $status -eq 0 ] + [[ "$output" =~ "test4223" ]] || false + [[ "$output" =~ "test9119" ]] || false +} + +@test "sql-server: test --max-connections 3 and --back-log 1 flags" { + cd repo1 + + dolt sql -q "CREATE TABLE test_table ( + id INT AUTO_INCREMENT PRIMARY KEY, + str VARCHAR(20) + );" + dolt commit -A -m "create test_table" + + # 3 connections allowed, 1 connection in back-log. + start_sql_server_with_args --max-connections 3 --back-log 1 + + set +e # errors expected. + # Start 3 long running connections. + pids=() + for i in {1..3}; do + dolt sql & + pids+=($!) + done + + # This job will wait until there is a connection available. verify + # by ensuring the insert completes. + dolt sql -q "insert into test_table (str) values ('test4223');" & + last_pid=$! + + sleep 1 # give all connections a chance to start + + # This operation should fail immediately. + run dolt sql -q "insert into test_table (str) values ('testxxxx');" + [ "$status" -ne 0 ] + [[ "$stderr" =~ "bad connection" ]] || false + + # Kill all the running shells + for pid in "${pids[@]}"; do + kill -9 "$pid" + done + + wait "$last_pid" + + run dolt sql -q "select * from test_table;" + [ $status -eq 0 ] + [[ "$output" =~ "test4223" ]] || false + [[ ! "$output" =~ "testxxxx" ]] || false +} From 27ff3beaac71a98a3c794b25a2bc14f26b570685 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Fri, 21 Mar 2025 11:21:10 -0700 Subject: [PATCH 06/16] Add max connections timeout --- .../commands/sqlserver/command_line_config.go | 22 ++++++++ go/cmd/dolt/commands/sqlserver/server.go | 1 + go/cmd/dolt/commands/sqlserver/sqlserver.go | 4 ++ .../doltcore/servercfg/serverconfig.go | 52 +++++++++++-------- .../doltcore/servercfg/yaml_config.go | 29 ++++++++--- .../sql-server-config-file-generation.bats | 7 +++ 6 files changed, 86 insertions(+), 29 deletions(-) diff --git a/go/cmd/dolt/commands/sqlserver/command_line_config.go b/go/cmd/dolt/commands/sqlserver/command_line_config.go index 05eff0cf7f6..6716c20b37a 100755 --- a/go/cmd/dolt/commands/sqlserver/command_line_config.go +++ b/go/cmd/dolt/commands/sqlserver/command_line_config.go @@ -19,6 +19,7 @@ import ( "path/filepath" "strconv" "strings" + "time" "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" "github.com/dolthub/dolt/go/libraries/utils/filesys" @@ -43,6 +44,7 @@ type commandLineServerConfig struct { doltTransactionCommit bool maxConnections uint64 maxWaitConnections uint32 + maxWaitConnsTimeout time.Duration tlsKey string tlsCert string requireSecureTransport bool @@ -74,6 +76,7 @@ func DefaultCommandLineServerConfig() *commandLineServerConfig { autoCommit: servercfg.DefaultAutoCommit, maxConnections: servercfg.DefaultMaxConnections, maxWaitConnections: servercfg.DefaultMaxWaitConnections, + maxWaitConnsTimeout: servercfg.DefaultMaxWaitConnectionsTimeout, dataDir: servercfg.DefaultDataDir, cfgDir: filepath.Join(servercfg.DefaultDataDir, servercfg.DefaultCfgDir), privilegeFilePath: filepath.Join(servercfg.DefaultDataDir, servercfg.DefaultCfgDir, servercfg.DefaultPrivilegeFilePath), @@ -175,6 +178,14 @@ func NewCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResult config.withMaxWaitConnections(uint32(maxWaitConnections)) } + if maxWaitConnsTimeoutStr, ok := apr.GetValue(maxWaitConsTimeoutFlag); ok { + maxWaitConnsTimeout, err := time.ParseDuration(maxWaitConnsTimeoutStr) + if err != nil { + return nil, fmt.Errorf("invalid value for --max-wait-connections-timeout '%s'", maxWaitConnsTimeoutStr) + } + config.maxWaitConnsTimeout = maxWaitConnsTimeout + } + config.autoCommit = !apr.Contains(noAutoCommitFlag) if apr.Contains(noAutoCommitFlag) { config.valuesSet[servercfg.AutoCommitKey] = struct{}{} @@ -270,6 +281,10 @@ func (cfg *commandLineServerConfig) MaxWaitConnections() uint32 { return cfg.maxWaitConnections } +func (cfg *commandLineServerConfig) MaxWaitConnectionsTimeout() time.Duration { + return cfg.maxWaitConnsTimeout +} + // TLSKey returns a path to the servers PEM-encoded private TLS key. "" if there is none. func (cfg *commandLineServerConfig) TLSKey() string { return cfg.tlsKey @@ -437,12 +452,19 @@ func (cfg *commandLineServerConfig) withMaxConnections(maxConnections uint64) *c return cfg } +// NM4 - I think we can drop this. Or take a str? func (cfg *commandLineServerConfig) withMaxWaitConnections(maxWaitConnections uint32) *commandLineServerConfig { cfg.maxWaitConnections = maxWaitConnections cfg.valuesSet[servercfg.MaxWaitConnectionsKey] = struct{}{} return cfg } +func (cfg *commandLineServerConfig) withMaxWaitConnectionsTimeout(maxWaitConnsTimeout time.Duration) *commandLineServerConfig { + cfg.maxWaitConnsTimeout = maxWaitConnsTimeout + cfg.valuesSet[servercfg.MaxWaitConnectionsTimeoutKey] = struct{}{} + return cfg +} + // withDataDir updates the path to a directory to use as the data dir. func (cfg *commandLineServerConfig) withDataDir(dataDir string) *commandLineServerConfig { cfg.dataDir = dataDir diff --git a/go/cmd/dolt/commands/sqlserver/server.go b/go/cmd/dolt/commands/sqlserver/server.go index 366c4af12c3..05443d84dd3 100644 --- a/go/cmd/dolt/commands/sqlserver/server.go +++ b/go/cmd/dolt/commands/sqlserver/server.go @@ -1093,6 +1093,7 @@ func getConfigFromServerConfig(serverConfig servercfg.ServerConfig, plf server.P serverConf.ConnWriteTimeout = writeTimeout serverConf.MaxConnections = serverConfig.MaxConnections() serverConf.MaxWaitConnections = serverConfig.MaxWaitConnections() + serverConf.MaxWaitConnectionsTimeout = serverConfig.MaxWaitConnectionsTimeout() serverConf.TLSConfig = tlsConfig serverConf.RequireSecureTransport = serverConfig.RequireSecureTransport() serverConf.MaxLoggedQueryLen = serverConfig.MaxLoggedQueryLen() diff --git a/go/cmd/dolt/commands/sqlserver/sqlserver.go b/go/cmd/dolt/commands/sqlserver/sqlserver.go index 402e698e667..d1ddbcb6075 100644 --- a/go/cmd/dolt/commands/sqlserver/sqlserver.go +++ b/go/cmd/dolt/commands/sqlserver/sqlserver.go @@ -49,6 +49,7 @@ const ( queryParallelismFlag = "query-parallelism" maxConnectionsFlag = "max-connections" maxWaitConnectionsFlag = "back-log" + maxWaitConsTimeoutFlag = "max-connections-timeout" allowCleartextPasswordsFlag = "allow-cleartext-passwords" socketFlag = "socket" remotesapiPortFlag = "remotesapi-port" @@ -110,6 +111,8 @@ SUPPORTED CONFIG FILE FIELDS: {{.EmphasisLeft}}listener.back_log{{.EmphasisRight}}: The number of simultaneous connections that the server will allow to block waiting for a connection before new connections result in immediate rejection. Default 50. +{{.EmphasisLeft}}listener.max_wait_connections_timeout{{.EmphasisRight}}: The maximum amount of time that a connection will block waiting for a connection before being rejected. + {{.EmphasisLeft}}listener.read_timeout_millis{{.EmphasisRight}}: The number of milliseconds that the server will wait for a read operation {{.EmphasisLeft}}listener.write_timeout_millis{{.EmphasisRight}}: The number of milliseconds that the server will wait for a write operation @@ -183,6 +186,7 @@ func (cmd SqlServerCmd) ArgParserWithName(name string) *argparser.ArgParser { ap.SupportsInt(queryParallelismFlag, "", "num-go-routines", "Deprecated, no effect in current versions of Dolt") ap.SupportsInt(maxConnectionsFlag, "", "max-connections", fmt.Sprintf("Set the number of connections handled by the server. Defaults to `%d`.", serverConfig.MaxConnections())) ap.SupportsInt(maxWaitConnectionsFlag, "", "back-log", fmt.Sprintf("Set the number of connections that can block waiting for a connection before new connections are rejected. Defaults to `%d`.", serverConfig.MaxWaitConnections())) + ap.SupportsString(maxWaitConsTimeoutFlag, "", "max-connections-timeout", fmt.Sprintf("Set the maximum duration that a connection will block waiting for a connection before being rejected. Defaults to `%v`.", serverConfig.MaxWaitConnectionsTimeout())) ap.SupportsString(commands.PrivsFilePathFlag, "", "privilege file", "Path to a file to load and store users and grants. Defaults to `$doltcfg-dir/privileges.db`. Will be created as needed.") ap.SupportsString(commands.BranchCtrlPathFlag, "", "branch control file", "Path to a file to load and store branch control permissions. Defaults to `$doltcfg-dir/branch_control.db`. Will be created as needed.") ap.SupportsString(allowCleartextPasswordsFlag, "", "allow-cleartext-passwords", "Allows use of cleartext passwords. Defaults to false.") diff --git a/go/libraries/doltcore/servercfg/serverconfig.go b/go/libraries/doltcore/servercfg/serverconfig.go index d12fc16a7c0..02a67231ad8 100644 --- a/go/libraries/doltcore/servercfg/serverconfig.go +++ b/go/libraries/doltcore/servercfg/serverconfig.go @@ -22,6 +22,7 @@ import ( "path/filepath" "runtime" "strings" + "time" ) var DefaultUnixSocketFilePath = DefaultMySQLUnixSocketFilePath @@ -47,29 +48,30 @@ const ( ) const ( - DefaultHost = "localhost" - DefaultPort = 3306 - DefaultUser = "root" - DefaultPass = "" - DefaultTimeout = 8 * 60 * 60 * 1000 // 8 hours, same as MySQL - DefaultReadOnly = false - DefaultLogLevel = LogLevel_Info - DefaultLogFormat = LogFormat_Text - DefaultAutoCommit = true - DefaultAutoGCBehaviorEnable = false - DefaultDoltTransactionCommit = false - DefaultMaxConnections = 1000 - DefaultMaxWaitConnections = 50 - DefaultDataDir = "." - DefaultCfgDir = ".doltcfg" - DefaultPrivilegeFilePath = "privileges.db" - DefaultBranchControlFilePath = "branch_control.db" - DefaultMetricsHost = "" - DefaultMetricsPort = -1 - DefaultAllowCleartextPasswords = false - DefaultMySQLUnixSocketFilePath = "/tmp/mysql.sock" - DefaultMaxLoggedQueryLen = 0 - DefaultEncodeLoggedQuery = false + DefaultHost = "localhost" + DefaultPort = 3306 + DefaultUser = "root" + DefaultPass = "" + DefaultTimeout = 8 * 60 * 60 * 1000 // 8 hours, same as MySQL + DefaultReadOnly = false + DefaultLogLevel = LogLevel_Info + DefaultLogFormat = LogFormat_Text + DefaultAutoCommit = true + DefaultAutoGCBehaviorEnable = false + DefaultDoltTransactionCommit = false + DefaultMaxConnections = 1000 + DefaultMaxWaitConnections = 50 + DefaultMaxWaitConnectionsTimeout = 60 * time.Second + DefaultDataDir = "." + DefaultCfgDir = ".doltcfg" + DefaultPrivilegeFilePath = "privileges.db" + DefaultBranchControlFilePath = "branch_control.db" + DefaultMetricsHost = "" + DefaultMetricsPort = -1 + DefaultAllowCleartextPasswords = false + DefaultMySQLUnixSocketFilePath = "/tmp/mysql.sock" + DefaultMaxLoggedQueryLen = 0 + DefaultEncodeLoggedQuery = false ) func ptr[T any](t T) *T { @@ -163,6 +165,8 @@ type ServerConfig interface { // MaxWaitConnections returns the maximum number of simultaneous connections that the server will allow to block waiting // for a connection before new connections result in immediate rejection MaxWaitConnections() uint32 + // MaxWaitConnectionsTimeout returns the maximum amount of time that a connection will block waiting for a connection + MaxWaitConnectionsTimeout() time.Duration // TLSKey returns a path to the servers PEM-encoded private TLS key. "" if there is none. TLSKey() string // TLSCert returns a path to the servers PEM-encoded TLS certificate chain. "" if there is none. @@ -245,6 +249,7 @@ func defaultServerConfigYAML() *YAMLConfig { PortNumber: ptr(DefaultPort), MaxConnections: ptr(uint64(DefaultMaxConnections)), BackLog: ptr(uint32(DefaultMaxWaitConnections)), + MaxConnectionsTimeoutMs: ptr(uint64(DefaultMaxWaitConnectionsTimeout.Milliseconds())), ReadTimeoutMillis: ptr(uint64(DefaultTimeout)), WriteTimeoutMillis: ptr(uint64(DefaultTimeout)), AllowCleartextPasswords: ptr(DefaultAllowCleartextPasswords), @@ -312,6 +317,7 @@ const ( CfgDirKey = "cfg_dir" MaxConnectionsKey = "max_connections" MaxWaitConnectionsKey = "back_log" + MaxWaitConnectionsTimeoutKey = "max_connections_timeout" TLSKeyKey = "tls_key" TLSCertKey = "tls_cert" RequireSecureTransportKey = "require_secure_transport" diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index 664d68e04a4..2c537e59e4c 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -18,6 +18,7 @@ import ( "fmt" "path/filepath" "strings" + "time" "unicode" "unicode/utf8" @@ -77,12 +78,13 @@ type UserYAMLConfig struct { // ListenerYAMLConfig contains information on the network connection that the server will open type ListenerYAMLConfig struct { - HostStr *string `yaml:"host,omitempty"` - PortNumber *int `yaml:"port,omitempty"` - MaxConnections *uint64 `yaml:"max_connections,omitempty"` - BackLog *uint32 `yaml:"back_log,omitempty"` - ReadTimeoutMillis *uint64 `yaml:"read_timeout_millis,omitempty"` - WriteTimeoutMillis *uint64 `yaml:"write_timeout_millis,omitempty"` + HostStr *string `yaml:"host,omitempty"` + PortNumber *int `yaml:"port,omitempty"` + MaxConnections *uint64 `yaml:"max_connections,omitempty"` + BackLog *uint32 `yaml:"back_log,omitempty"` + MaxConnectionsTimeoutMs *uint64 `yaml:"max_connections_timeout_millis,omitempty"` + ReadTimeoutMillis *uint64 `yaml:"read_timeout_millis,omitempty"` + WriteTimeoutMillis *uint64 `yaml:"write_timeout_millis,omitempty"` // TLSKey is a file system path to an unencrypted private TLS key in PEM format. TLSKey *string `yaml:"tls_key,omitempty"` // TLSCert is a file system path to a TLS certificate chain in PEM format. @@ -199,6 +201,7 @@ func ServerConfigAsYAMLConfig(cfg ServerConfig) *YAMLConfig { PortNumber: ptr(cfg.Port()), MaxConnections: ptr(cfg.MaxConnections()), BackLog: ptr(cfg.MaxWaitConnections()), + MaxConnectionsTimeoutMs: ptr(uint64(cfg.MaxWaitConnectionsTimeout().Milliseconds())), ReadTimeoutMillis: ptr(cfg.ReadTimeout()), WriteTimeoutMillis: ptr(cfg.WriteTimeout()), TLSKey: nillableStrPtr(cfg.TLSKey()), @@ -270,6 +273,7 @@ func ServerConfigSetValuesAsYAMLConfig(cfg ServerConfig) *YAMLConfig { PortNumber: zeroIf(ptr(cfg.Port()), !cfg.ValueSet(PortKey)), MaxConnections: zeroIf(ptr(cfg.MaxConnections()), !cfg.ValueSet(MaxConnectionsKey)), BackLog: zeroIf(ptr(cfg.MaxWaitConnections()), !cfg.ValueSet(MaxWaitConnectionsKey)), + MaxConnectionsTimeoutMs: zeroIf(ptr(uint64(cfg.MaxWaitConnectionsTimeout().Milliseconds())), !cfg.ValueSet(MaxWaitConnectionsTimeoutKey)), ReadTimeoutMillis: zeroIf(ptr(cfg.ReadTimeout()), !cfg.ValueSet(ReadTimeoutKey)), WriteTimeoutMillis: zeroIf(ptr(cfg.WriteTimeout()), !cfg.ValueSet(WriteTimeoutKey)), TLSKey: zeroIf(ptr(cfg.TLSKey()), !cfg.ValueSet(TLSKeyKey)), @@ -493,6 +497,9 @@ func (cfg YAMLConfig) withDefaultsFilledIn() YAMLConfig { if withDefaults.ListenerConfig.BackLog == nil { withDefaults.ListenerConfig.BackLog = defaults.ListenerConfig.BackLog } + if withDefaults.ListenerConfig.MaxConnectionsTimeoutMs == nil { + withDefaults.ListenerConfig.MaxConnectionsTimeoutMs = defaults.ListenerConfig.MaxConnectionsTimeoutMs + } if withDefaults.ListenerConfig.ReadTimeoutMillis == nil { withDefaults.ListenerConfig.ReadTimeoutMillis = defaults.ListenerConfig.ReadTimeoutMillis } @@ -673,6 +680,14 @@ func (cfg YAMLConfig) MaxWaitConnections() uint32 { return *cfg.ListenerConfig.BackLog } +func (cfg YAMLConfig) MaxWaitConnectionsTimeout() time.Duration { + if cfg.ListenerConfig.MaxConnectionsTimeoutMs == nil { + return DefaultMaxWaitConnectionsTimeout + } + + return time.Duration(*cfg.ListenerConfig.MaxConnectionsTimeoutMs) * time.Millisecond +} + // DisableClientMultiStatements returns true if the server should run in a mode // where the CLIENT_MULTI_STATEMENTS option are ignored and every incoming // ComQuery packet is assumed to be a standalone query. @@ -962,6 +977,8 @@ func (cfg YAMLConfig) ValueSet(value string) bool { return cfg.ListenerConfig.MaxConnections != nil case MaxWaitConnectionsKey: return cfg.ListenerConfig.BackLog != nil + case MaxWaitConnectionsTimeoutKey: + return cfg.ListenerConfig.MaxConnectionsTimeoutMs != nil case EventSchedulerKey: return cfg.BehaviorConfig.EventSchedulerStatus != nil } diff --git a/integration-tests/bats/sql-server-config-file-generation.bats b/integration-tests/bats/sql-server-config-file-generation.bats index ee3fa60235c..f5b908fed59 100644 --- a/integration-tests/bats/sql-server-config-file-generation.bats +++ b/integration-tests/bats/sql-server-config-file-generation.bats @@ -99,6 +99,7 @@ EOF --timeout 7777777 \ --allow-cleartext-passwords true \ --back-log 767 \ + --max-connections-timeout 13s \ --host 0.0.0.0 run cat "$CONFIG_FILE_NAME" @@ -110,6 +111,12 @@ EOF [[ "$output" =~ "write_timeout_millis: 7777777" ]] || false [[ "$output" =~ "allow_cleartext_passwords: true" ]] || false [[ "$output" =~ "back_log: 767" ]] || false + + echo "--------------------------" + echo "$output" + echo "--------------------------" + + [[ "$output" =~ "max_connections_timeout: 676" ]] || false } @test "sql-server-config-file-generation: generated config file uses default values as placeholders for unset fields" { From ae18118f31273789ff9339235c4d59f24a673c18 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Fri, 21 Mar 2025 13:19:12 -0700 Subject: [PATCH 07/16] Get max timeout for waiting connections all the way through --- go/cmd/dolt/commands/sqlserver/command_line_config.go | 4 ++-- .../bats/sql-server-config-file-generation.bats | 9 ++------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/go/cmd/dolt/commands/sqlserver/command_line_config.go b/go/cmd/dolt/commands/sqlserver/command_line_config.go index 6716c20b37a..458c451ea7b 100755 --- a/go/cmd/dolt/commands/sqlserver/command_line_config.go +++ b/go/cmd/dolt/commands/sqlserver/command_line_config.go @@ -181,9 +181,9 @@ func NewCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResult if maxWaitConnsTimeoutStr, ok := apr.GetValue(maxWaitConsTimeoutFlag); ok { maxWaitConnsTimeout, err := time.ParseDuration(maxWaitConnsTimeoutStr) if err != nil { - return nil, fmt.Errorf("invalid value for --max-wait-connections-timeout '%s'", maxWaitConnsTimeoutStr) + return nil, fmt.Errorf("invalid duration value for --max-wait-connections-timeout '%s'", maxWaitConnsTimeoutStr) } - config.maxWaitConnsTimeout = maxWaitConnsTimeout + config.withMaxWaitConnectionsTimeout(maxWaitConnsTimeout) } config.autoCommit = !apr.Contains(noAutoCommitFlag) diff --git a/integration-tests/bats/sql-server-config-file-generation.bats b/integration-tests/bats/sql-server-config-file-generation.bats index f5b908fed59..f4e1837acd3 100644 --- a/integration-tests/bats/sql-server-config-file-generation.bats +++ b/integration-tests/bats/sql-server-config-file-generation.bats @@ -99,7 +99,7 @@ EOF --timeout 7777777 \ --allow-cleartext-passwords true \ --back-log 767 \ - --max-connections-timeout 13s \ + --max-connections-timeout 6m13s \ --host 0.0.0.0 run cat "$CONFIG_FILE_NAME" @@ -111,12 +111,7 @@ EOF [[ "$output" =~ "write_timeout_millis: 7777777" ]] || false [[ "$output" =~ "allow_cleartext_passwords: true" ]] || false [[ "$output" =~ "back_log: 767" ]] || false - - echo "--------------------------" - echo "$output" - echo "--------------------------" - - [[ "$output" =~ "max_connections_timeout: 676" ]] || false + [[ "$output" =~ "max_connections_timeout_millis: 373000" ]] || false } @test "sql-server-config-file-generation: generated config file uses default values as placeholders for unset fields" { From aacde670b0983b72f83e54deb8ba3088ebfd3b48 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Mon, 24 Mar 2025 10:20:58 -0700 Subject: [PATCH 08/16] Test the --max-connections-timeout has an effect --- integration-tests/bats/sql-server.bats | 30 ++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/integration-tests/bats/sql-server.bats b/integration-tests/bats/sql-server.bats index 98dda65fa3e..35a248502ea 100644 --- a/integration-tests/bats/sql-server.bats +++ b/integration-tests/bats/sql-server.bats @@ -2234,3 +2234,33 @@ EOF [[ "$output" =~ "test4223" ]] || false [[ ! "$output" =~ "testxxxx" ]] || false } + +# bats test_tags=no_lambda +@test "sql-server: test --max-connections-timeout 15s and --max-connections 3 flags" { + cd repo1 + + # Default is 60s, but I don't want to extend the test time for this. + start_sql_server_with_args --max-connections-timeout=15s --max-connections=3 + + pids=() + for i in {1..3}; do + dolt sql & + pids+=($!) + done + + # Attempt to connect with a fourth connection using mysql client because dolt sql has + # 9 retries which we don't have the patience to wait for. + start_time=$(date +%s) + run mysql -h 127.0.0.1 -P 3306 -u root -e "SELECT 1;" + end_time=$(date +%s) + + elapsed_time=$((end_time - start_time)) + [[ $elapsed_time -lt 17 ]] || false + [[ $elapsed_time -gt 13 ]] || false + [ $status -ne 0 ] + [[ "$output" =~ "Lost connection to MySQL server at 'reading initial communication packet'" ]] || false + + for pid in "${pids[@]}"; do + kill -9 "$pid" + done +} From 44313be0d86d4060eb1cffa6453f867b66e79613 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Mon, 24 Mar 2025 10:53:09 -0700 Subject: [PATCH 09/16] Cleanup --- go/cmd/dolt/commands/sqlserver/command_line_config.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/cmd/dolt/commands/sqlserver/command_line_config.go b/go/cmd/dolt/commands/sqlserver/command_line_config.go index 458c451ea7b..9ad8cf86f27 100755 --- a/go/cmd/dolt/commands/sqlserver/command_line_config.go +++ b/go/cmd/dolt/commands/sqlserver/command_line_config.go @@ -452,7 +452,6 @@ func (cfg *commandLineServerConfig) withMaxConnections(maxConnections uint64) *c return cfg } -// NM4 - I think we can drop this. Or take a str? func (cfg *commandLineServerConfig) withMaxWaitConnections(maxWaitConnections uint32) *commandLineServerConfig { cfg.maxWaitConnections = maxWaitConnections cfg.valuesSet[servercfg.MaxWaitConnectionsKey] = struct{}{} From cd407bcbc8db774195035e4bd10fe203a2986698 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Mon, 24 Mar 2025 15:25:28 -0700 Subject: [PATCH 10/16] Update missed go test --- go/cmd/dolt/commands/sqlserver/server_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/cmd/dolt/commands/sqlserver/server_test.go b/go/cmd/dolt/commands/sqlserver/server_test.go index 9a1d7ce9216..1d532a92d15 100644 --- a/go/cmd/dolt/commands/sqlserver/server_test.go +++ b/go/cmd/dolt/commands/sqlserver/server_test.go @@ -595,6 +595,8 @@ listener: # host: localhost # port: 3306 # max_connections: 1000 + # back_log: 50 + # max_connections_timeout_millis: 60000 read_timeout_millis: 11000 write_timeout_millis: 11000 # tls_key: key.pem From ff8320fce5099121331b95c19cf7cefd355bae69 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Mon, 24 Mar 2025 15:39:33 -0700 Subject: [PATCH 11/16] More test goodness --- .../doltcore/servercfg/testdata/minver_validation.txt | 4 +++- go/libraries/doltcore/servercfg/yaml_config_test.go | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go/libraries/doltcore/servercfg/testdata/minver_validation.txt b/go/libraries/doltcore/servercfg/testdata/minver_validation.txt index cbb390420a9..aac26ffbf93 100644 --- a/go/libraries/doltcore/servercfg/testdata/minver_validation.txt +++ b/go/libraries/doltcore/servercfg/testdata/minver_validation.txt @@ -21,6 +21,8 @@ ListenerConfig servercfg.ListenerYAMLConfig 0.0.0 listener,omitempty -HostStr *string 0.0.0 host,omitempty -PortNumber *int 0.0.0 port,omitempty -MaxConnections *uint64 0.0.0 max_connections,omitempty +-BackLog *uint32 0.0.0 back_log,omitempty +-MaxConnectionsTimeoutMs *uint64 0.0.0 max_connections_timeout_millis,omitempty -ReadTimeoutMillis *uint64 0.0.0 read_timeout_millis,omitempty -WriteTimeoutMillis *uint64 0.0.0 write_timeout_millis,omitempty -TLSKey *string 0.0.0 tls_key,omitempty @@ -64,4 +66,4 @@ ClusterCfg *servercfg.ClusterYAMLConfig 0.0.0 cluster,omitempty --TLSCert_ string 0.0.0 tls_cert --TLSCA_ string 0.0.0 tls_ca --URLMatches []string 0.0.0 server_name_urls ---DNSMatches []string 0.0.0 server_name_dns \ No newline at end of file +--DNSMatches []string 0.0.0 server_name_dns diff --git a/go/libraries/doltcore/servercfg/yaml_config_test.go b/go/libraries/doltcore/servercfg/yaml_config_test.go index 8214e8f230d..bd218de95ec 100644 --- a/go/libraries/doltcore/servercfg/yaml_config_test.go +++ b/go/libraries/doltcore/servercfg/yaml_config_test.go @@ -41,6 +41,8 @@ listener: host: localhost port: 3306 max_connections: 1000 + back_log: 50 + max_connections_timeout_millis: 60000 read_timeout_millis: 28800000 write_timeout_millis: 28800000 From 24c4e7ca204653e313c32912965607654cae7a9d Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Mon, 24 Mar 2025 17:47:22 -0700 Subject: [PATCH 12/16] Make the last timeout test more reliable --- integration-tests/bats/sql-server.bats | 33 +++++++++++++++++++------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/integration-tests/bats/sql-server.bats b/integration-tests/bats/sql-server.bats index 35a248502ea..61aec0fdd20 100644 --- a/integration-tests/bats/sql-server.bats +++ b/integration-tests/bats/sql-server.bats @@ -2237,30 +2237,45 @@ EOF # bats test_tags=no_lambda @test "sql-server: test --max-connections-timeout 15s and --max-connections 3 flags" { + skiponwindows "mysql client required" + cd repo1 + export PORT=$( definePORT ) + # Default is 60s, but I don't want to extend the test time for this. - start_sql_server_with_args --max-connections-timeout=15s --max-connections=3 + start_sql_server_with_args_no_port --max-connections-timeout=10s --max-connections=3 --back-log=10 --port=$PORT + # For this test we use the mysql client, which doesn't retry connections, but also seems to exit early + # if we don't give it a query. So we use a sleep to hang the three connections. These will be killed, so + # they won't run the full 30s. pids=() for i in {1..3}; do - dolt sql & + mysql -h 127.0.0.1 -P $PORT -u root -D repo1 -e "select sleep(30)" & pids+=($!) done - # Attempt to connect with a fourth connection using mysql client because dolt sql has - # 9 retries which we don't have the patience to wait for. + sleep 3 + + # Attempt to connect with a fourth connection - should fail after 10s. `run` caused a lot of variance in the test + # time, so we avoid using it here. start_time=$(date +%s) - run mysql -h 127.0.0.1 -P 3306 -u root -e "SELECT 1;" + set +e + mysql -h 127.0.0.1 -P $PORT -u root > $BATS_TMPDIR/mysql.out + status=$? end_time=$(date +%s) + [ $status -ne 0 ] + # There is a high amount of variance in the time it takes to fail, so we just check that it is within a range + # with a pretty high upper bound. elapsed_time=$((end_time - start_time)) - [[ $elapsed_time -lt 17 ]] || false - [[ $elapsed_time -gt 13 ]] || false - [ $status -ne 0 ] + [[ $elapsed_time -lt 15 ]] || false + [[ $elapsed_time -gt 9 ]] || false + + run cat $BATS_TMPDIR/mysql.out [[ "$output" =~ "Lost connection to MySQL server at 'reading initial communication packet'" ]] || false for pid in "${pids[@]}"; do - kill -9 "$pid" + kill "$pid" 2>/dev/null done } From 9b35e9b879ad9834057898b22d4bf67a1464f194 Mon Sep 17 00:00:00 2001 From: Aaron Son Date: Tue, 25 Mar 2025 15:02:16 -0700 Subject: [PATCH 13/16] go: binlogreplication: binlog_primary_test.go: Fix up for timestamp rounding instead of truncation. --- .../doltcore/sqle/binlogreplication/binlog_primary_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/libraries/doltcore/sqle/binlogreplication/binlog_primary_test.go b/go/libraries/doltcore/sqle/binlogreplication/binlog_primary_test.go index 799c7bff41f..313decd34f0 100644 --- a/go/libraries/doltcore/sqle/binlogreplication/binlog_primary_test.go +++ b/go/libraries/doltcore/sqle/binlogreplication/binlog_primary_test.go @@ -171,9 +171,9 @@ func TestBinlogPrimary(t *testing.T) { {"1", "42", nil, nil, "123", "123", "123", "123", "200", "200", "200", "200", "200", float32(1.0101), float64(2.02030405060708), "1981", "1981-02-16", "-123:45:30.123456", - "1981-02-16 06:01:02", "1981-02-16 06:01:02.2", "1981-02-16 06:01:02.23", "1981-02-16 06:01:02.234", - "1981-02-16 06:01:02.2345", "1981-02-16 06:01:02.23456", "1981-02-16 06:01:02.234567", - "2024-04-08 10:30:42", "2024-04-08 10:30:42.8", "2024-04-08 10:30:42.87", "2024-04-08 10:30:42.876", + "1981-02-16 06:01:02", "1981-02-16 06:01:02.2", "1981-02-16 06:01:02.23", "1981-02-16 06:01:02.235", + "1981-02-16 06:01:02.2346", "1981-02-16 06:01:02.23457", "1981-02-16 06:01:02.234567", + "2024-04-08 10:30:43", "2024-04-08 10:30:42.9", "2024-04-08 10:30:42.88", "2024-04-08 10:30:42.877", "2024-04-08 10:30:42.8765", "2024-04-08 10:30:42.87654", "2024-04-08 10:30:42.876543", "\x01\x03", "green", "pants,tie,belt", From 9f47fa929730dff5d8e1e1f86a2b18ffa48d184c Mon Sep 17 00:00:00 2001 From: Aaron Son Date: Tue, 25 Mar 2025 15:24:26 -0700 Subject: [PATCH 14/16] go/store/datas/commit_meta.go: Round the CommitMeta Timestamp when we format it for presentation. Better matches the behavior of these timestamps on the wire. --- go/store/datas/commit_meta.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/go/store/datas/commit_meta.go b/go/store/datas/commit_meta.go index d0839dd8f08..7928929e335 100644 --- a/go/store/datas/commit_meta.go +++ b/go/store/datas/commit_meta.go @@ -205,8 +205,10 @@ func (cm *CommitMeta) Time() time.Time { // FormatTS takes the internal timestamp and turns it into a human readable string in the time.RubyDate format // which looks like: "Mon Jan 02 15:04:05 -0700 2006" +// +// We round this to the nearest second, which is what MySQL timestamp does by default. func (cm *CommitMeta) FormatTS() string { - return cm.Time().In(CommitLoc).Format(time.RubyDate) + return cm.Time().In(CommitLoc).Round(time.Second).Format(time.RubyDate) } // String returns the human readable string representation of the commit data From 9e0e0a2f3f2d0861214d4eab5287b4e2225b9e54 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Tue, 25 Mar 2025 15:33:42 -0700 Subject: [PATCH 15/16] PR Feedback --- go/libraries/doltcore/servercfg/yaml_config.go | 4 ++-- integration-tests/bats/sql-server.bats | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index 2c537e59e4c..3a8bf1d36f1 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -81,8 +81,8 @@ type ListenerYAMLConfig struct { HostStr *string `yaml:"host,omitempty"` PortNumber *int `yaml:"port,omitempty"` MaxConnections *uint64 `yaml:"max_connections,omitempty"` - BackLog *uint32 `yaml:"back_log,omitempty"` - MaxConnectionsTimeoutMs *uint64 `yaml:"max_connections_timeout_millis,omitempty"` + BackLog *uint32 `yaml:"back_log,omitempty" minver:"1.50.10"` + MaxConnectionsTimeoutMs *uint64 `yaml:"max_connections_timeout_millis,omitempty" minver:"1.50.10"` ReadTimeoutMillis *uint64 `yaml:"read_timeout_millis,omitempty"` WriteTimeoutMillis *uint64 `yaml:"write_timeout_millis,omitempty"` // TLSKey is a file system path to an unencrypted private TLS key in PEM format. diff --git a/integration-tests/bats/sql-server.bats b/integration-tests/bats/sql-server.bats index 61aec0fdd20..59a8da0c267 100644 --- a/integration-tests/bats/sql-server.bats +++ b/integration-tests/bats/sql-server.bats @@ -2236,7 +2236,7 @@ EOF } # bats test_tags=no_lambda -@test "sql-server: test --max-connections-timeout 15s and --max-connections 3 flags" { +@test "sql-server: test --max-connections-timeout 10s and --max-connections 3 flags" { skiponwindows "mysql client required" cd repo1 @@ -2244,7 +2244,7 @@ EOF export PORT=$( definePORT ) # Default is 60s, but I don't want to extend the test time for this. - start_sql_server_with_args_no_port --max-connections-timeout=10s --max-connections=3 --back-log=10 --port=$PORT + start_sql_server_with_args_no_port --max-connections-timeout=10s --max-connections=3 --port=$PORT # For this test we use the mysql client, which doesn't retry connections, but also seems to exit early # if we don't give it a query. So we use a sleep to hang the three connections. These will be killed, so From 8a73e44231debe2b99af505abbef9078e81078f9 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Tue, 25 Mar 2025 15:42:37 -0700 Subject: [PATCH 16/16] More test fixes --- .../doltcore/servercfg/testdata/minver_validation.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/libraries/doltcore/servercfg/testdata/minver_validation.txt b/go/libraries/doltcore/servercfg/testdata/minver_validation.txt index aac26ffbf93..8bd6035586e 100644 --- a/go/libraries/doltcore/servercfg/testdata/minver_validation.txt +++ b/go/libraries/doltcore/servercfg/testdata/minver_validation.txt @@ -21,8 +21,8 @@ ListenerConfig servercfg.ListenerYAMLConfig 0.0.0 listener,omitempty -HostStr *string 0.0.0 host,omitempty -PortNumber *int 0.0.0 port,omitempty -MaxConnections *uint64 0.0.0 max_connections,omitempty --BackLog *uint32 0.0.0 back_log,omitempty --MaxConnectionsTimeoutMs *uint64 0.0.0 max_connections_timeout_millis,omitempty +-BackLog *uint32 1.50.10 back_log,omitempty +-MaxConnectionsTimeoutMs *uint64 1.50.10 max_connections_timeout_millis,omitempty -ReadTimeoutMillis *uint64 0.0.0 read_timeout_millis,omitempty -WriteTimeoutMillis *uint64 0.0.0 write_timeout_millis,omitempty -TLSKey *string 0.0.0 tls_key,omitempty