diff --git a/go/cmd/dolt/commands/sqlserver/sqlserver.go b/go/cmd/dolt/commands/sqlserver/sqlserver.go index 1e21163af66..0315b042ac1 100644 --- a/go/cmd/dolt/commands/sqlserver/sqlserver.go +++ b/go/cmd/dolt/commands/sqlserver/sqlserver.go @@ -83,6 +83,17 @@ var sqlServerDocs = cli.CommandDocumentationContent{ " other command line arguments are ignored.\n\nThis is an example yaml configuration file showing all supported" + " items and their default values:\n\n" + indentLines(servercfg.ServerConfigAsYAMLConfig(DefaultCommandLineServerConfig()).String()) + "\n\n" + ` +ENVIRONMENT VARIABLE INTERPOLATION: + +SQL server yaml configs support environment variable interpolation: + + ${VAR} Expands to the value of VAR (error if VAR is unset or empty) + $$ Escapes to a literal '$' + +Notes: + - Interpolation happens before YAML parsing. + - Quote values for string fields when needed (e.g. values containing ':'), but do not quote placeholders intended for numeric/bool fields. + SUPPORTED CONFIG FILE FIELDS: {{.EmphasisLeft}}data_dir{{.EmphasisRight}}: A directory where the server will load dolt databases to serve, and create new ones. Defaults to the current directory. diff --git a/go/libraries/doltcore/servercfg/env_interpolate.go b/go/libraries/doltcore/servercfg/env_interpolate.go new file mode 100644 index 00000000000..18a9a7b9b45 --- /dev/null +++ b/go/libraries/doltcore/servercfg/env_interpolate.go @@ -0,0 +1,191 @@ +// Copyright 2025 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package servercfg + +import ( + "bytes" + "fmt" + "os" + "regexp" +) + +type envPlaceholder struct { + varName string + closingBrace int +} + +// interpolateEnv expands environment variable placeholders in |data|. +// +// Supported syntax: +// - ${VAR} : expands to VAR's value; error if VAR is unset or empty +// - $$ : escapes to a literal '$' +// +// Notes: +// - Expansion is applied to the input text only (env var values are inserted literally). +func interpolateEnv(data []byte) ([]byte, error) { + out := make([]byte, 0, len(data)) + for i := 0; i < len(data); { + relDollar := bytes.IndexByte(data[i:], '$') + if relDollar == -1 { + out = append(out, data[i:]...) + break + } + + dollarIdx := i + relDollar + out = append(out, data[i:dollarIdx]...) + + // Escape sequence: $$ -> $ + if dollarIdx+1 < len(data) && data[dollarIdx+1] == '$' { + out = append(out, '$') + i = dollarIdx + 2 + continue + } + + // Placeholder: ${...} + if dollarIdx+1 < len(data) && data[dollarIdx+1] == '{' { + ph, err := parseEnvPlaceholder(data, dollarIdx) + if err != nil { + return nil, err + } + + out, err = appendEnvExpansion(out, ph) + if err != nil { + return nil, err + } + + // Continue after the closing brace. + i = ph.closingBrace + 1 + continue + } + + // Leave stray '$' untouched + out = append(out, '$') + i = dollarIdx + 1 + } + + return out, nil +} + +func parseEnvPlaceholder(data []byte, dollarIdx int) (envPlaceholder, error) { + // data[dollarIdx] == '$' and data[dollarIdx+1] == '{' expected. + start := dollarIdx + 2 // after ${ + + rest := data[start:] + + relBrace := bytes.IndexByte(rest, '}') + relNL := bytes.IndexByte(rest, '\n') + relCR := bytes.IndexByte(rest, '\r') + + relLineEnd := minPositive(relNL, relCR) + if relLineEnd != -1 && (relBrace == -1 || relLineEnd < relBrace) { + // Placeholders must be on a single line. This prevents bad/missing braces from + // consuming the rest of the file and producing giant error messages. + return envPlaceholder{}, envErrorAt(data, dollarIdx, "unterminated environment placeholder") + } + + if relBrace == -1 { + return envPlaceholder{}, envErrorAt(data, dollarIdx, "unterminated environment placeholder") + } + + closingBrace := start + relBrace + + expr := data[start:closingBrace] + varName, err := parseEnvExpr(expr) + if err != nil { + return envPlaceholder{}, envErrorAt(data, dollarIdx, err.Error()) + } + + return envPlaceholder{ + varName: varName, + closingBrace: closingBrace, + }, nil +} + +func minPositive(a, b int) int { + if a == -1 { + return b + } + if b == -1 { + return a + } + if a < b { + return a + } + return b +} + +func appendEnvExpansion(out []byte, ph envPlaceholder) ([]byte, error) { + val, ok := os.LookupEnv(ph.varName) + if !ok || val == "" { + return nil, fmt.Errorf("environment variable %q is not set or empty", ph.varName) + } + return append(out, []byte(val)...), nil +} + +func parseEnvExpr(expr []byte) (varName string, err error) { + // Default expressions are intentionally unsupported to avoid silently masking misconfiguration. + if bytes.Contains(expr, []byte(":-")) { + return "", fmt.Errorf("environment variable default expressions are not supported (found %q)", string(expr)) + } + + if len(expr) == 0 { + return "", fmt.Errorf("invalid environment placeholder: empty variable name") + } + if !isValidEnvVarName(expr) { + return "", fmt.Errorf("invalid environment variable name %q", string(expr)) + } + + return string(expr), nil +} + +var envVarRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +func isValidEnvVarName(b []byte) bool { + return envVarRe.Match(b) +} + +func envErrorAt(data []byte, idx int, msg string) error { + line, col := lineAndColAt(data, idx) + return fmt.Errorf("%s (line %d, column %d)", msg, line, col) +} + +func lineAndColAt(data []byte, idx int) (line int, col int) { + if idx < 0 { + idx = 0 + } + if idx > len(data) { + idx = len(data) + } + + line = 1 + col = 1 + for i := 0; i < idx; i++ { + switch data[i] { + case '\n': + line++ + col = 1 + case '\r': + line++ + col = 1 + // Handle CRLF as a single newline. + if i+1 < idx && data[i+1] == '\n' { + i++ + } + default: + col++ + } + } + return line, col +} diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index d71fcb6b78a..9f5aa5f2493 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -194,6 +194,11 @@ func YamlConfigFromFile(fs filesys.Filesys, path string) (ServerConfig, error) { return nil, fmt.Errorf("Failed to read file '%s'. Error: %s", path, err.Error()) } + data, err = interpolateEnv(data) + if err != nil { + return nil, fmt.Errorf("Failed to interpolate environment variables in yaml file '%s'. Error: %s", path, err.Error()) + } + cfg, err := NewYamlConfig(data) if err != nil { return nil, fmt.Errorf("Failed to parse yaml file '%s'. Error: %s", path, err.Error()) diff --git a/go/libraries/doltcore/servercfg/yaml_config_test.go b/go/libraries/doltcore/servercfg/yaml_config_test.go index 42678dc997f..87ff3b90276 100644 --- a/go/libraries/doltcore/servercfg/yaml_config_test.go +++ b/go/libraries/doltcore/servercfg/yaml_config_test.go @@ -20,6 +20,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" + + "github.com/dolthub/dolt/go/libraries/utils/filesys" ) var trueValue = true @@ -206,6 +208,110 @@ cluster: require.Equal(t, "http://doltdb-1.doltdb:50051/{database}", config.ClusterConfig().StandbyRemotes()[0].RemoteURLTemplate()) } +func TestYamlConfigFromFileEnvInterpolation_String(t *testing.T) { + t.Setenv("DOLT_TEST_SQLSERVER_HOST", "127.0.0.1") + + fs := filesys.EmptyInMemFS("/") + err := fs.WriteFile("config.yaml", []byte(` +listener: + host: ${DOLT_TEST_SQLSERVER_HOST} +`), 0o644) + require.NoError(t, err) + + cfg, err := YamlConfigFromFile(fs, "config.yaml") + require.NoError(t, err) + require.Equal(t, "127.0.0.1", cfg.Host()) +} + +func TestYamlConfigFromFileEnvInterpolation_Int(t *testing.T) { + t.Setenv("DOLT_TEST_SQLSERVER_PORT", "15200") + + fs := filesys.EmptyInMemFS("/") + err := fs.WriteFile("config.yaml", []byte(` +listener: + port: ${DOLT_TEST_SQLSERVER_PORT} +`), 0o644) + require.NoError(t, err) + + cfg, err := YamlConfigFromFile(fs, "config.yaml") + require.NoError(t, err) + require.Equal(t, 15200, cfg.Port()) +} + +func TestYamlConfigFromFileEnvInterpolation_MissingVarErrors(t *testing.T) { + // Empty env vars result in an interpolation error. + t.Setenv("DOLT_TEST_SQLSERVER_MISSING", "") + + fs := filesys.EmptyInMemFS("/") + err := fs.WriteFile("config.yaml", []byte(` +listener: + port: ${DOLT_TEST_SQLSERVER_MISSING} +`), 0o644) + require.NoError(t, err) + + _, err = YamlConfigFromFile(fs, "config.yaml") + require.Error(t, err) + require.Contains(t, err.Error(), "DOLT_TEST_SQLSERVER_MISSING") +} + +func TestYamlConfigFromFileEnvInterpolation_UnsetVarErrors(t *testing.T) { + fs := filesys.EmptyInMemFS("/") + err := fs.WriteFile("config.yaml", []byte(` +listener: + port: ${DOLT_TEST_SQLSERVER_UNSET} +`), 0o644) + require.NoError(t, err) + + _, err = YamlConfigFromFile(fs, "config.yaml") + require.Error(t, err) + require.Contains(t, err.Error(), "DOLT_TEST_SQLSERVER_UNSET") +} + +func TestYamlConfigFromFileEnvInterpolation_DefaultSyntaxErrors(t *testing.T) { + fs := filesys.EmptyInMemFS("/") + err := fs.WriteFile("config.yaml", []byte(` +listener: + port: ${DOLT_TEST_SQLSERVER_PORT:-15200} +`), 0o644) + require.NoError(t, err) + + _, err = YamlConfigFromFile(fs, "config.yaml") + require.Error(t, err) + require.Contains(t, err.Error(), "default expressions") +} + +func TestYamlConfigFromFileEnvInterpolation_EscapeDollar(t *testing.T) { + fs := filesys.EmptyInMemFS("/") + err := fs.WriteFile("config.yaml", []byte(` +golden_mysql_conn: "$$dollar" +`), 0o644) + require.NoError(t, err) + + cfg, err := YamlConfigFromFile(fs, "config.yaml") + require.NoError(t, err) + yc, ok := cfg.(*YAMLConfig) + require.True(t, ok) + require.Equal(t, "$dollar", yc.GoldenMysqlConnectionString()) +} + +func TestYamlConfigFromFileEnvInterpolation_UnterminatedStopsAtNewline(t *testing.T) { + fs := filesys.EmptyInMemFS("/") + err := fs.WriteFile("config.yaml", []byte(` +listener: + port: ${DOLT_TEST_SQLSERVER_PORT +behavior: + read_only: true +junk: "}" +`), 0o644) + require.NoError(t, err) + + _, err = YamlConfigFromFile(fs, "config.yaml") + require.Error(t, err) + require.Contains(t, err.Error(), "unterminated environment placeholder") + require.Contains(t, err.Error(), "line 3") + require.Contains(t, err.Error(), "column") +} + func TestValidateClusterConfig(t *testing.T) { cases := []struct { Name string diff --git a/integration-tests/bats/sql-server-config-env-interpolation.bats b/integration-tests/bats/sql-server-config-env-interpolation.bats new file mode 100644 index 00000000000..d0209709ade --- /dev/null +++ b/integration-tests/bats/sql-server-config-env-interpolation.bats @@ -0,0 +1,157 @@ +#!/usr/bin/env bats +load $BATS_TEST_DIRNAME/helper/common.bash +load $BATS_TEST_DIRNAME/helper/query-server-common.bash + +make_repo() { + mkdir "$1" + cd "$1" + dolt init + cd .. +} + +# wait_for_http_ok +wait_for_http_ok() { + url="$1" + timeout_ms="$2" + end_time=$((SECONDS+($timeout_ms/1000))) + while [ $SECONDS -lt $end_time ]; do + if curl -fsS "$url" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + return 1 +} + +setup() { + skiponwindows "tests are flaky on Windows" + if [ "$SQL_ENGINE" = "remote-engine" ]; then + skip "This test tests remote connections directly, SQL_ENGINE is not needed." + fi + setup_no_dolt_init + make_repo repo1 +} + +teardown() { + stop_sql_server 1 && sleep 0.5 + rm -rf "$BATS_TMPDIR/sql-server-config-env-interpolation-test$$" + teardown_common +} + +# bats test_tags=no_lambda +@test "sql-server-config-env-interpolation: required env var expands listener.port" { + cd repo1 + + SQL_PORT=$(definePORT) + + # Use a single-quoted heredoc so bash does not expand ${...} + cat > config.yml <<'EOF' +listener: + host: "0.0.0.0" + port: ${DOLT_TEST_SQLSERVER_PORT} +EOF + + if [ "$IS_WINDOWS" == true ]; then + DOLT_TEST_SQLSERVER_PORT=$SQL_PORT PORT=$SQL_PORT dolt sql-server --config ./config.yml & + else + DOLT_TEST_SQLSERVER_PORT=$SQL_PORT PORT=$SQL_PORT dolt sql-server --config ./config.yml --socket "dolt.$SQL_PORT.sock" & + fi + SERVER_PID=$! + + run wait_for_connection "$SQL_PORT" 8500 + [ $status -eq 0 ] +} + +# bats test_tags=no_lambda +@test "sql-server-config-env-interpolation: default expression syntax fails clearly" { + cd repo1 + + SQL_PORT=$(definePORT) + + # Use a single-quoted heredoc so bash does not expand ${...} + cat > config.yml <<'EOF' +listener: + host: "0.0.0.0" + port: ${DOLT_TEST_SQLSERVER_PORT:-15200} +EOF + + run dolt sql-server --config ./config.yml + [ $status -ne 0 ] + log_output_has "Failed to interpolate environment variables in yaml file" + log_output_has "default expressions" +} + +# bats test_tags=no_lambda +@test "sql-server-config-env-interpolation: missing env var fails with clear error" { + cd repo1 + + unset DOLT_TEST_SQLSERVER_MISSING + + # Use a single-quoted heredoc so bash does not expand ${...} + cat > config.yml <<'EOF' +listener: + host: "0.0.0.0" + port: ${DOLT_TEST_SQLSERVER_MISSING} +EOF + + run dolt sql-server --config ./config.yml + [ $status -ne 0 ] + log_output_has "Failed to interpolate environment variables in yaml file" + log_output_has "DOLT_TEST_SQLSERVER_MISSING" +} + +# bats test_tags=no_lambda +@test "sql-server-config-env-interpolation: dollar escaping and env composition works (metrics labels)" { + cd repo1 + + SQL_PORT=$(definePORT) + METRICS_PORT=$(definePORT) + export DOLT_TEST_LABEL="foo" + + # We want the YAML to contain $$${DOLT_TEST_LABEL} literally: + # - $$ becomes a literal '$' + # - ${DOLT_TEST_LABEL} expands to "foo" + # => final label value should be "$foo" + cat > config.yml < "$BATS_TMPDIR/metrics_env_interp_$$.out" 2>/dev/null || true + if grep -F "$expectedLabel" "$BATS_TMPDIR/metrics_env_interp_$$.out" >/dev/null 2>&1; then + found=1 + break + fi + sleep 1 + done + + [ "$found" -eq 1 ] +} +