From 0af2edb3c41cbd7253322992f98294362e0a3023 Mon Sep 17 00:00:00 2001 From: elianddb Date: Thu, 24 Jul 2025 21:00:11 +0000 Subject: [PATCH 01/17] impl client-side binary-as-hex --- go/cmd/dolt/commands/sql.go | 34 ++++++++++++++ .../dolt/commands/sqlserver/queryist_utils.go | 24 +++++++--- go/libraries/doltcore/sqle/sqlutil/sql_row.go | 1 + .../untyped/tabular/fixedwidth_tablewriter.go | 38 +++++++++++++++ integration-tests/bats/sql-server.bats | 28 +++++++++++ integration-tests/bats/sql.bats | 46 +++++++++++++++++++ 6 files changed, 164 insertions(+), 7 deletions(-) diff --git a/go/cmd/dolt/commands/sql.go b/go/cmd/dolt/commands/sql.go index 43d8bdc7e07..12618bebbf1 100644 --- a/go/cmd/dolt/commands/sql.go +++ b/go/cmd/dolt/commands/sql.go @@ -43,6 +43,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/env" dsqle "github.com/dolthub/dolt/go/libraries/doltcore/sqle" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" + "github.com/dolthub/dolt/go/libraries/doltcore/table/untyped/tabular" "github.com/dolthub/dolt/go/libraries/utils/argparser" "github.com/dolthub/dolt/go/libraries/utils/iohelp" "github.com/dolthub/dolt/go/libraries/utils/osutil" @@ -95,6 +96,11 @@ const ( ProfileFlag = "profile" timeFlag = "time" outputFlag = "output" + binaryAsHexFlag = "binary-as-hex" + skipBinaryAsHexFlag = "skip-binary-as-hex" + // TODO: Consider simplifying to use MySQL's skip pattern with single flag definition + // MySQL handles both --binary-as-hex and --skip-binary-as-hex with one option definition + // and uses disabled_my_option to distinguish between enable/disable welcomeMsg = `# Welcome to the DoltSQL shell. # Statements must be terminated with ';'. @@ -106,6 +112,22 @@ func init() { dsqle.AddDoltSystemVariables() } +// applyBinaryAsHexContext sets binary-as-hex context if enabled based on flags and default +func applyBinaryAsHexContext(sqlCtx *sql.Context, apr *argparser.ArgParseResults, defaultValue bool) *sql.Context { + // Priority: --skip-binary-as-hex > --binary-as-hex > default + enabled := defaultValue + if apr.Contains(skipBinaryAsHexFlag) { + enabled = false + } else if apr.Contains(binaryAsHexFlag) { + enabled = true + } + + if enabled { + return tabular.WithBinaryAsHex(sqlCtx, true) + } + return sqlCtx +} + type SqlCmd struct { VersionStr string } @@ -144,6 +166,9 @@ func (cmd SqlCmd) ArgParser() *argparser.ArgParser { ap.SupportsFlag(BatchFlag, "b", "Use to enable more efficient batch processing for large SQL import scripts. This mode is no longer supported and this flag is a no-op. To speed up your SQL imports, use either LOAD DATA, or structure your SQL import script to insert many rows per statement.") ap.SupportsFlag(continueFlag, "c", "Continue running queries on an error. Used for batch mode only.") ap.SupportsString(fileInputFlag, "f", "input file", "Execute statements from the file given.") + ap.SupportsFlag(binaryAsHexFlag, "", "Print binary data as hex. Enabled by default for interactive terminals.") + // --skip-binary-as-hex is supported but not shown in help, matching MySQL's behavior + ap.SupportsFlag(skipBinaryAsHexFlag, "", "") return ap } @@ -219,11 +244,17 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE } if query, queryOK := apr.GetValue(QueryFlag); queryOK { + // Apply binary-as-hex logic for query mode (default: false for non-interactive) + sqlCtx = applyBinaryAsHexContext(sqlCtx, apr, false) + if apr.Contains(saveFlag) { return execSaveQuery(sqlCtx, dEnv, queryist, apr, query, format, usage) } return queryMode(sqlCtx, queryist, apr, query, format, usage) } else if savedQueryName, exOk := apr.GetValue(executeFlag); exOk { + // Apply binary-as-hex logic for execute saved query mode (default: false for non-interactive) + sqlCtx = applyBinaryAsHexContext(sqlCtx, apr, false) + return executeSavedQuery(sqlCtx, queryist, dEnv, savedQueryName, format, usage) } else if apr.Contains(listSavedFlag) { return listSavedQueries(sqlCtx, queryist, dEnv, format, usage) @@ -261,6 +292,9 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE defer fileReadProg.close() } + // Determine binary-as-hex behavior based on flags and TTY detection (like MySQL) + sqlCtx = applyBinaryAsHexContext(sqlCtx, apr, isTty) + if isTty { err := execShell(sqlCtx, queryist, format, cliCtx) if err != nil { diff --git a/go/cmd/dolt/commands/sqlserver/queryist_utils.go b/go/cmd/dolt/commands/sqlserver/queryist_utils.go index 5af74de4260..95512f01f03 100644 --- a/go/cmd/dolt/commands/sqlserver/queryist_utils.go +++ b/go/cmd/dolt/commands/sqlserver/queryist_utils.go @@ -24,6 +24,7 @@ import ( "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/types" + "github.com/dolthub/vitess/go/sqltypes" "github.com/dolthub/vitess/go/vt/sqlparser" "github.com/go-sql-driver/mysql" "github.com/gocraft/dbr/v2" @@ -36,6 +37,7 @@ import ( "github.com/dolthub/dolt/go/libraries/utils/filesys" ) + // BuildConnectionStringQueryist returns a Queryist that connects to the server specified by the given server config. Presence in this // module isn't ideal, but it's the only way to get the server config into the queryist. func BuildConnectionStringQueryist(ctx context.Context, cwdFS filesys.Filesys, creds *cli.UserPassword, apr *argparser.ArgParseResults, host string, port int, useTLS bool, dbRev string) (cli.LateBindQueryist, error) { @@ -147,18 +149,26 @@ type MysqlRowWrapper struct { var _ sql.RowIter = (*MysqlRowWrapper)(nil) func NewMysqlRowWrapper(sqlRows *sql2.Rows) (*MysqlRowWrapper, error) { - colNames, err := sqlRows.Columns() + colTypes, err := sqlRows.ColumnTypes() if err != nil { return nil, err } - schema := make(sql.Schema, len(colNames)) - vRow := make([]*string, len(colNames)) - iRow := make([]interface{}, len(colNames)) + schema := make(sql.Schema, len(colTypes)) + vRow := make([]*string, len(colTypes)) + iRow := make([]interface{}, len(colTypes)) rows := make([]sql.Row, 0) - for i, colName := range colNames { + for i, colType := range colTypes { + // Preserve binary type information for client-side binary-as-hex formatting + sqlType := types.LongText + dbTypeName := strings.ToUpper(colType.DatabaseTypeName()) + if dbTypeName == "VARBINARY" || dbTypeName == "BINARY" { + // Normalize both BINARY and VARBINARY to VarBinary for simpler formatting logic + sqlType = types.MustCreateBinary(sqltypes.VarBinary, 255) + } + schema[i] = &sql.Column{ - Name: colName, - Type: types.LongText, + Name: colType.Name(), + Type: sqlType, Nullable: true, } iRow[i] = &vRow[i] diff --git a/go/libraries/doltcore/sqle/sqlutil/sql_row.go b/go/libraries/doltcore/sqle/sqlutil/sql_row.go index f8b3a99400d..7137939b6d5 100644 --- a/go/libraries/doltcore/sqle/sqlutil/sql_row.go +++ b/go/libraries/doltcore/sqle/sqlutil/sql_row.go @@ -28,6 +28,7 @@ import ( "github.com/dolthub/dolt/go/store/types" ) + // DoltRowToSqlRow constructs a go-mysql-server sql.Row from a Dolt row.Row. func DoltRowToSqlRow(doltRow row.Row, sch schema.Schema) (sql.Row, error) { if doltRow == nil { diff --git a/go/libraries/doltcore/table/untyped/tabular/fixedwidth_tablewriter.go b/go/libraries/doltcore/table/untyped/tabular/fixedwidth_tablewriter.go index 6b72cb03114..423a069386f 100644 --- a/go/libraries/doltcore/table/untyped/tabular/fixedwidth_tablewriter.go +++ b/go/libraries/doltcore/table/untyped/tabular/fixedwidth_tablewriter.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/vitess/go/sqltypes" "github.com/fatih/color" "github.com/dolthub/dolt/go/libraries/doltcore/diff" @@ -32,6 +33,20 @@ import ( const writeBufSize = 256 * 1024 +// Context key for binary-as-hex flag +type ctxKey int + +const binaryAsHexKey ctxKey = iota + +// WithBinaryAsHex returns a new context with binary-as-hex flag set +func WithBinaryAsHex(ctx *sql.Context, enabled bool) *sql.Context { + if ctx == nil { + return ctx + } + newCtx := context.WithValue(ctx.Context, binaryAsHexKey, enabled) + return ctx.WithContext(newCtx) +} + // FixedWidthTableWriter is a TableWriter that applies a fixed width transform to its fields. All fields are // expected to be strings. type FixedWidthTableWriter struct { @@ -219,6 +234,29 @@ func (w *FixedWidthTableWriter) stringValue(ctx *sql.Context, idx int, i interfa return "NULL", nil } + // Client-side binary-as-hex formatting (like MySQL client does) + if ctx != nil { + if val := ctx.Value(binaryAsHexKey); val != nil { + if enabled, ok := val.(bool); ok && enabled { + // Check if this is a binary type that should be formatted as hex + // Only check for VarBinary since MysqlRowWrapper normalizes both BINARY and VARBINARY to VarBinary + // TODO: Add support for other binary types per MySQL 8.4 docs: + // https://dev.mysql.com/doc/refman/8.4/en/mysql-command-options.html#option_mysql_binary-as-hex + if stringType, ok := w.schema[idx].Type.(sql.StringType); ok { + if stringType.Type() == sqltypes.VarBinary { + if binaryBytes, ok := i.([]byte); ok { + return fmt.Sprintf("0x%X", binaryBytes), nil + } + // Handle binary data that comes as string (from server connections) + if binaryString, ok := i.(string); ok { + return fmt.Sprintf("0x%X", []byte(binaryString)), nil + } + } + } + } + } + } + return sqlutil.SqlColToStr(ctx, w.schema[idx].Type, i) } diff --git a/integration-tests/bats/sql-server.bats b/integration-tests/bats/sql-server.bats index 7f733daf1d4..341e74298b4 100644 --- a/integration-tests/bats/sql-server.bats +++ b/integration-tests/bats/sql-server.bats @@ -2197,3 +2197,31 @@ EOF run grep -F "Dropping persisted '__dolt_local_user__@localhost' because this account name is reserved for Dolt" server_log.txt [ $status -eq 0 ] } + + +@test "sql-server: client interactive binary-as-hex behavior works with server connections" { + skiponwindows "Missing dependencies" + which expect > /dev/null || skip "expect not available" + + cd repo1 + + # Setup: Create table with binary data and commit it + dolt sql -q "DROP TABLE IF EXISTS binary_test; CREATE TABLE binary_test (pk INT PRIMARY KEY, vb VARBINARY(20)); INSERT INTO binary_test VALUES (1, 0x0A000000), (2, 'abc');" + dolt add . + dolt commit -m "Add binary test data" + + start_sql_server repo1 + + # Test --binary-as-hex flag in interactive mode (should show hex) + run expect "$BATS_TEST_DIRNAME/binary-as-hex-server-interactive.expect" $PORT + echo "EXPECT OUTPUT: $output" + echo "EXPECT STATUS: $status" + [ $status -eq 0 ] + + # Test --skip-binary-as-hex flag in interactive mode (should show raw) + run expect "$BATS_TEST_DIRNAME/binary-as-hex-skip-flag-interactive.expect" $PORT + [ $status -eq 0 ] + + stop_sql_server 1 +} + diff --git a/integration-tests/bats/sql.bats b/integration-tests/bats/sql.bats index f6c60bf8819..10e1694ae46 100755 --- a/integration-tests/bats/sql.bats +++ b/integration-tests/bats/sql.bats @@ -2913,3 +2913,49 @@ SQL dolt sql < $BATS_TEST_DIRNAME/helper/with_utf16be_bom.sql dolt table rm t1 } + +@test "sql: client binary-as-hex behavior works with local database" { + # Setup: Create table with binary data + dolt sql -q "CREATE TABLE binary_test (pk INT PRIMARY KEY, vb VARBINARY(20)); INSERT INTO binary_test VALUES (1, 0x0A000000), (2, 'abc');" + + # Client non-interactive mode (using -q flag) should show raw bytes by default + run dolt sql -q "SELECT vb FROM binary_test WHERE pk = 1" + [ $status -eq 0 ] + # Should show raw bytes (invisible chars), not hex format + ! [[ "$output" =~ "0x0A000000" ]] || false + + # Printable data should show as text + run dolt sql -q "SELECT vb FROM binary_test WHERE pk = 2" + [ $status -eq 0 ] + [[ "$output" =~ "abc" ]] || false + + # HEX() function should work consistently + run dolt sql -q "SELECT HEX(vb) FROM binary_test WHERE pk = 1" + [ $status -eq 0 ] + [[ "$output" =~ "0A000000" ]] || false + + # Test --binary-as-hex flag forces hex display in non-interactive mode + run dolt sql --binary-as-hex -q "SELECT vb FROM binary_test WHERE pk = 1" + [ $status -eq 0 ] + [[ "$output" =~ "0x0A000000" ]] || false + + # Test --binary-as-hex with printable data + run dolt sql --binary-as-hex -q "SELECT vb FROM binary_test WHERE pk = 2" + [ $status -eq 0 ] + [[ "$output" =~ "0x616263" ]] || false + + # Test --skip-binary-as-hex flag forces raw display + run dolt sql --skip-binary-as-hex -q "SELECT vb FROM binary_test WHERE pk = 2" + [ $status -eq 0 ] + [[ "$output" =~ "abc" ]] || false + ! [[ "$output" =~ "0x616263" ]] || false + + # Test flag priority: --skip-binary-as-hex overrides --binary-as-hex + run dolt sql --binary-as-hex --skip-binary-as-hex -q "SELECT vb FROM binary_test WHERE pk = 2" + [ $status -eq 0 ] + [[ "$output" =~ "abc" ]] || false + ! [[ "$output" =~ "0x616263" ]] || false + + # Cleanup + dolt sql -q "DROP TABLE binary_test" +} From e782684753dbbe06bdc84dbd20c2d0ec50d4a522 Mon Sep 17 00:00:00 2001 From: elianddb Date: Thu, 24 Jul 2025 21:42:10 +0000 Subject: [PATCH 02/17] Add expect files for binary-as-hex tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- go/cmd/dolt/commands/sql.go | 6 ++--- .../dolt/commands/sqlserver/queryist_utils.go | 3 +-- go/libraries/doltcore/sqle/sqlutil/sql_row.go | 1 - .../binary-as-hex-server-interactive.expect | 26 +++++++++++++++++++ ...binary-as-hex-skip-flag-interactive.expect | 26 +++++++++++++++++++ 5 files changed, 56 insertions(+), 6 deletions(-) create mode 100755 integration-tests/bats/binary-as-hex-server-interactive.expect create mode 100755 integration-tests/bats/binary-as-hex-skip-flag-interactive.expect diff --git a/go/cmd/dolt/commands/sql.go b/go/cmd/dolt/commands/sql.go index 12618bebbf1..b0de24288cc 100644 --- a/go/cmd/dolt/commands/sql.go +++ b/go/cmd/dolt/commands/sql.go @@ -121,7 +121,7 @@ func applyBinaryAsHexContext(sqlCtx *sql.Context, apr *argparser.ArgParseResults } else if apr.Contains(binaryAsHexFlag) { enabled = true } - + if enabled { return tabular.WithBinaryAsHex(sqlCtx, true) } @@ -246,7 +246,7 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE if query, queryOK := apr.GetValue(QueryFlag); queryOK { // Apply binary-as-hex logic for query mode (default: false for non-interactive) sqlCtx = applyBinaryAsHexContext(sqlCtx, apr, false) - + if apr.Contains(saveFlag) { return execSaveQuery(sqlCtx, dEnv, queryist, apr, query, format, usage) } @@ -254,7 +254,7 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE } else if savedQueryName, exOk := apr.GetValue(executeFlag); exOk { // Apply binary-as-hex logic for execute saved query mode (default: false for non-interactive) sqlCtx = applyBinaryAsHexContext(sqlCtx, apr, false) - + return executeSavedQuery(sqlCtx, queryist, dEnv, savedQueryName, format, usage) } else if apr.Contains(listSavedFlag) { return listSavedQueries(sqlCtx, queryist, dEnv, format, usage) diff --git a/go/cmd/dolt/commands/sqlserver/queryist_utils.go b/go/cmd/dolt/commands/sqlserver/queryist_utils.go index 95512f01f03..c38344813e8 100644 --- a/go/cmd/dolt/commands/sqlserver/queryist_utils.go +++ b/go/cmd/dolt/commands/sqlserver/queryist_utils.go @@ -37,7 +37,6 @@ import ( "github.com/dolthub/dolt/go/libraries/utils/filesys" ) - // BuildConnectionStringQueryist returns a Queryist that connects to the server specified by the given server config. Presence in this // module isn't ideal, but it's the only way to get the server config into the queryist. func BuildConnectionStringQueryist(ctx context.Context, cwdFS filesys.Filesys, creds *cli.UserPassword, apr *argparser.ArgParseResults, host string, port int, useTLS bool, dbRev string) (cli.LateBindQueryist, error) { @@ -165,7 +164,7 @@ func NewMysqlRowWrapper(sqlRows *sql2.Rows) (*MysqlRowWrapper, error) { // Normalize both BINARY and VARBINARY to VarBinary for simpler formatting logic sqlType = types.MustCreateBinary(sqltypes.VarBinary, 255) } - + schema[i] = &sql.Column{ Name: colType.Name(), Type: sqlType, diff --git a/go/libraries/doltcore/sqle/sqlutil/sql_row.go b/go/libraries/doltcore/sqle/sqlutil/sql_row.go index 7137939b6d5..f8b3a99400d 100644 --- a/go/libraries/doltcore/sqle/sqlutil/sql_row.go +++ b/go/libraries/doltcore/sqle/sqlutil/sql_row.go @@ -28,7 +28,6 @@ import ( "github.com/dolthub/dolt/go/store/types" ) - // DoltRowToSqlRow constructs a go-mysql-server sql.Row from a Dolt row.Row. func DoltRowToSqlRow(doltRow row.Row, sch schema.Schema) (sql.Row, error) { if doltRow == nil { diff --git a/integration-tests/bats/binary-as-hex-server-interactive.expect b/integration-tests/bats/binary-as-hex-server-interactive.expect new file mode 100755 index 00000000000..62ad47a8376 --- /dev/null +++ b/integration-tests/bats/binary-as-hex-server-interactive.expect @@ -0,0 +1,26 @@ +#!/usr/bin/expect + +set port [lindex $argv 0] +set timeout 10 + +spawn dolt --host 127.0.0.1 --port $port --no-tls sql --binary-as-hex +expect ">" + +send "USE repo1;\r" +expect ">" + +send "SELECT * FROM binary_test;\r" +expect { + "0x0A000000" { + # Found hex output - this is correct + expect ">" + send "exit\r" + expect eof + exit 0 + } + timeout { + send "exit\r" + expect eof + exit 1 + } +} \ No newline at end of file diff --git a/integration-tests/bats/binary-as-hex-skip-flag-interactive.expect b/integration-tests/bats/binary-as-hex-skip-flag-interactive.expect new file mode 100755 index 00000000000..dac896897c3 --- /dev/null +++ b/integration-tests/bats/binary-as-hex-skip-flag-interactive.expect @@ -0,0 +1,26 @@ +#!/usr/bin/expect + +set port [lindex $argv 0] +set timeout 10 + +spawn dolt --host 127.0.0.1 --port $port --no-tls sql --skip-binary-as-hex +expect ">" + +send "USE repo1;\r" +expect ">" + +send "SELECT * FROM binary_test;\r" +expect { + -re "\\| 1 \\|.*\\|" { + # Found raw binary output (not hex) - this is correct for skip flag + expect ">" + send "exit\r" + expect eof + exit 0 + } + timeout { + send "exit\r" + expect eof + exit 1 + } +} \ No newline at end of file From 77b0e3a7b4c9fdaa7879effdfa6c52aac0a548d3 Mon Sep 17 00:00:00 2001 From: elianddb Date: Fri, 25 Jul 2025 17:04:49 +0000 Subject: [PATCH 03/17] Complete binary-as-hex implementation for local and server connections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Move binary-as-hex decision to main Exec() function with TTY detection * Implement row-level binary data transformation for all output formats * Add server connection binary type detection via DatabaseTypeName() * Handle both local ([]byte) and server (string) binary data formats * Support BINARY and VARBINARY types following MySQL 8.4+ behavior * Remove redundant SQL context usage and function parameters * All binary-as-hex tests now pass for local and server connections 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- go/cmd/dolt/commands/debug.go | 2 +- go/cmd/dolt/commands/sql.go | 133 ++++++++++++------ .../dolt/commands/sqlserver/queryist_utils.go | 14 +- .../untyped/tabular/fixedwidth_tablewriter.go | 37 ----- 4 files changed, 98 insertions(+), 88 deletions(-) diff --git a/go/cmd/dolt/commands/debug.go b/go/cmd/dolt/commands/debug.go index 059a41f1cde..490e492e5eb 100644 --- a/go/cmd/dolt/commands/debug.go +++ b/go/cmd/dolt/commands/debug.go @@ -436,5 +436,5 @@ func execDebugMode(ctx *sql.Context, qryist cli.Queryist, queryFile *os.File, co }() input := bufio.NewReader(transform.NewReader(queryFile, textunicode.BOMOverride(transform.Nop))) - return execBatchMode(ctx, qryist, input, continueOnErr, format) + return execBatchMode(ctx, qryist, input, continueOnErr, format, false) } diff --git a/go/cmd/dolt/commands/sql.go b/go/cmd/dolt/commands/sql.go index b0de24288cc..c9888b4c6d1 100644 --- a/go/cmd/dolt/commands/sql.go +++ b/go/cmd/dolt/commands/sql.go @@ -43,11 +43,11 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/env" dsqle "github.com/dolthub/dolt/go/libraries/doltcore/sqle" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" - "github.com/dolthub/dolt/go/libraries/doltcore/table/untyped/tabular" "github.com/dolthub/dolt/go/libraries/utils/argparser" "github.com/dolthub/dolt/go/libraries/utils/iohelp" "github.com/dolthub/dolt/go/libraries/utils/osutil" eventsapi "github.com/dolthub/eventsapi_schema/dolt/services/eventsapi/v1alpha1" + "github.com/dolthub/vitess/go/sqltypes" ) var sqlDocs = cli.CommandDocumentationContent{ @@ -112,22 +112,52 @@ func init() { dsqle.AddDoltSystemVariables() } -// applyBinaryAsHexContext sets binary-as-hex context if enabled based on flags and default -func applyBinaryAsHexContext(sqlCtx *sql.Context, apr *argparser.ArgParseResults, defaultValue bool) *sql.Context { - // Priority: --skip-binary-as-hex > --binary-as-hex > default - enabled := defaultValue - if apr.Contains(skipBinaryAsHexFlag) { - enabled = false - } else if apr.Contains(binaryAsHexFlag) { - enabled = true +// binaryAsHexRowIter wraps a row iterator and converts binary data to hex format +type binaryAsHexRowIter struct { + inner sql.RowIter + schema sql.Schema +} + +func newBinaryAsHexRowIter(inner sql.RowIter, schema sql.Schema) sql.RowIter { + return &binaryAsHexRowIter{ + inner: inner, + schema: schema, } +} - if enabled { - return tabular.WithBinaryAsHex(sqlCtx, true) +func (iter *binaryAsHexRowIter) Next(ctx *sql.Context) (sql.Row, error) { + row, err := iter.inner.Next(ctx) + if err != nil { + return nil, err + } + + // Transform binary data to hex format + for i, val := range row { + if val != nil && i < len(iter.schema) { + if stringType, ok := iter.schema[i].Type.(sql.StringType); ok { + sqlType := stringType.Type() + switch sqlType { + case sqltypes.Binary, sqltypes.VarBinary: + // Handle byte slice for VARBINARY/BINARY columns (local connections) + if binaryBytes, ok := val.([]byte); ok { + row[i] = fmt.Sprintf("0x%X", binaryBytes) + } else if binaryString, ok := val.(string); ok { + // Handle string data that contains binary (server connections) + row[i] = fmt.Sprintf("0x%X", []byte(binaryString)) + } + } + } + } } - return sqlCtx + + return row, nil +} + +func (iter *binaryAsHexRowIter) Close(ctx *sql.Context) error { + return iter.inner.Close(ctx) } + type SqlCmd struct { VersionStr string } @@ -235,6 +265,25 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage) } + // Determine if we're running in a TTY (used by all modes) + isTty := false + fi, err := os.Stdin.Stat() + if err != nil { + if !osutil.IsWindows { + return HandleVErrAndExitCode(errhand.BuildDError("Couldn't stat STDIN. This is a bug.").Build(), usage) + } + } else { + isTty = fi.Mode()&os.ModeCharDevice != 0 + } + + // Determine binary-as-hex behavior once (default based on TTY, flags can override) + binaryAsHex := isTty + if apr.Contains(skipBinaryAsHexFlag) { + binaryAsHex = false + } else if apr.Contains(binaryAsHexFlag) { + binaryAsHex = true + } + queryist, sqlCtx, closeFunc, err := cliCtx.QueryEngine(ctx) if err != nil { return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage) @@ -244,32 +293,16 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE } if query, queryOK := apr.GetValue(QueryFlag); queryOK { - // Apply binary-as-hex logic for query mode (default: false for non-interactive) - sqlCtx = applyBinaryAsHexContext(sqlCtx, apr, false) - if apr.Contains(saveFlag) { - return execSaveQuery(sqlCtx, dEnv, queryist, apr, query, format, usage) + return execSaveQuery(sqlCtx, dEnv, queryist, apr, query, format, usage, binaryAsHex) } - return queryMode(sqlCtx, queryist, apr, query, format, usage) + return queryMode(sqlCtx, queryist, apr, query, format, usage, binaryAsHex) } else if savedQueryName, exOk := apr.GetValue(executeFlag); exOk { - // Apply binary-as-hex logic for execute saved query mode (default: false for non-interactive) - sqlCtx = applyBinaryAsHexContext(sqlCtx, apr, false) - - return executeSavedQuery(sqlCtx, queryist, dEnv, savedQueryName, format, usage) + return executeSavedQuery(sqlCtx, queryist, dEnv, savedQueryName, format, usage, binaryAsHex) } else if apr.Contains(listSavedFlag) { return listSavedQueries(sqlCtx, queryist, dEnv, format, usage) } else { // Run in either batch mode for piped input, or shell mode for interactive - isTty := false - - fi, err := os.Stdin.Stat() - if err != nil { - if !osutil.IsWindows { - return sqlHandleVErrAndExitCode(queryist, errhand.BuildDError("Couldn't stat STDIN. This is a bug.").Build(), usage) - } - } else { - isTty = fi.Mode()&os.ModeCharDevice != 0 - } _, continueOnError := apr.GetValue(continueFlag) @@ -292,17 +325,14 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE defer fileReadProg.close() } - // Determine binary-as-hex behavior based on flags and TTY detection (like MySQL) - sqlCtx = applyBinaryAsHexContext(sqlCtx, apr, isTty) - if isTty { - err := execShell(sqlCtx, queryist, format, cliCtx) + err := execShell(sqlCtx, queryist, format, cliCtx, binaryAsHex) if err != nil { return sqlHandleVErrAndExitCode(queryist, errhand.VerboseErrorFromError(err), usage) } } else { input = transform.NewReader(input, textunicode.BOMOverride(transform.Nop)) - err := execBatchMode(sqlCtx, queryist, input, continueOnError, format) + err := execBatchMode(sqlCtx, queryist, input, continueOnError, format, binaryAsHex) if err != nil { return sqlHandleVErrAndExitCode(queryist, errhand.VerboseErrorFromError(err), usage) } @@ -392,10 +422,10 @@ func listSavedQueries(ctx *sql.Context, qryist cli.Queryist, dEnv *env.DoltEnv, } query := "SELECT * FROM " + doltdb.DoltQueryCatalogTableName - return sqlHandleVErrAndExitCode(qryist, execSingleQuery(ctx, qryist, query, format), usage) + return sqlHandleVErrAndExitCode(qryist, execSingleQuery(ctx, qryist, query, format, false), usage) } -func executeSavedQuery(ctx *sql.Context, qryist cli.Queryist, dEnv *env.DoltEnv, savedQueryName string, format engine.PrintResultFormat, usage cli.UsagePrinter) int { +func executeSavedQuery(ctx *sql.Context, qryist cli.Queryist, dEnv *env.DoltEnv, savedQueryName string, format engine.PrintResultFormat, usage cli.UsagePrinter, binaryAsHex bool) int { if !dEnv.Valid() { return sqlHandleVErrAndExitCode(qryist, errhand.BuildDError("error: --%s must be used in a dolt database directory.", executeFlag).Build(), usage) } @@ -412,7 +442,7 @@ func executeSavedQuery(ctx *sql.Context, qryist cli.Queryist, dEnv *env.DoltEnv, } cli.PrintErrf("Executing saved query '%s':\n%s\n", savedQueryName, sq.Query) - return sqlHandleVErrAndExitCode(qryist, execSingleQuery(ctx, qryist, sq.Query, format), usage) + return sqlHandleVErrAndExitCode(qryist, execSingleQuery(ctx, qryist, sq.Query, format, binaryAsHex), usage) } func queryMode( @@ -422,12 +452,12 @@ func queryMode( query string, format engine.PrintResultFormat, usage cli.UsagePrinter, + binaryAsHex bool, ) int { - _, continueOnError := apr.GetValue(continueFlag) input := strings.NewReader(query) - err := execBatchMode(ctx, qryist, input, continueOnError, format) + err := execBatchMode(ctx, qryist, input, continueOnError, format, binaryAsHex) if err != nil { return sqlHandleVErrAndExitCode(qryist, errhand.VerboseErrorFromError(err), usage) } @@ -435,14 +465,14 @@ func queryMode( return 0 } -func execSaveQuery(ctx *sql.Context, dEnv *env.DoltEnv, qryist cli.Queryist, apr *argparser.ArgParseResults, query string, format engine.PrintResultFormat, usage cli.UsagePrinter) int { +func execSaveQuery(ctx *sql.Context, dEnv *env.DoltEnv, qryist cli.Queryist, apr *argparser.ArgParseResults, query string, format engine.PrintResultFormat, usage cli.UsagePrinter, binaryAsHex bool) int { if !dEnv.Valid() { return sqlHandleVErrAndExitCode(qryist, errhand.BuildDError("error: --%s must be used in a dolt database directory.", saveFlag).Build(), usage) } saveName := apr.GetValueOrDefault(saveFlag, "") - verr := execSingleQuery(ctx, qryist, query, format) + verr := execSingleQuery(ctx, qryist, query, format, binaryAsHex) if verr != nil { return sqlHandleVErrAndExitCode(qryist, verr, usage) } @@ -473,6 +503,7 @@ func execSingleQuery( qryist cli.Queryist, query string, format engine.PrintResultFormat, + binaryAsHex bool, ) errhand.VerboseError { sqlSch, rowIter, _, err := processQuery(sqlCtx, query, qryist) @@ -481,6 +512,10 @@ func execSingleQuery( } if rowIter != nil { + // Apply binary-as-hex formatting if enabled + if binaryAsHex { + rowIter = newBinaryAsHexRowIter(rowIter, sqlSch) + } err = engine.PrettyPrintResults(sqlCtx, format, sqlSch, rowIter, false, false, false) if err != nil { return errhand.VerboseErrorFromError(err) @@ -649,7 +684,7 @@ func saveQuery(ctx *sql.Context, root doltdb.RootValue, query string, name strin } // execBatchMode runs all the queries in the input reader -func execBatchMode(ctx *sql.Context, qryist cli.Queryist, input io.Reader, continueOnErr bool, format engine.PrintResultFormat) error { +func execBatchMode(ctx *sql.Context, qryist cli.Queryist, input io.Reader, continueOnErr bool, format engine.PrintResultFormat, binaryAsHex bool) error { scanner := NewStreamScanner(input) var query string for scanner.Scan() { @@ -697,6 +732,10 @@ func execBatchMode(ctx *sql.Context, qryist cli.Queryist, input io.Reader, conti fileReadProg.printNewLineIfNeeded() } } + // Apply binary-as-hex formatting if enabled + if binaryAsHex { + rowIter = newBinaryAsHexRowIter(rowIter, sqlSch) + } err = engine.PrettyPrintResults(ctx, format, sqlSch, rowIter, false, false, false) if err != nil { err = buildBatchSqlErr(scanner.state.statementStartLine, query, err) @@ -723,7 +762,7 @@ func buildBatchSqlErr(stmtStartLine int, query string, err error) error { // execShell starts a SQL shell. Returns when the user exits the shell. The Root of the sqlEngine may // be updated by any queries which were processed. -func execShell(sqlCtx *sql.Context, qryist cli.Queryist, format engine.PrintResultFormat, cliCtx cli.CliContext) error { +func execShell(sqlCtx *sql.Context, qryist cli.Queryist, format engine.PrintResultFormat, cliCtx cli.CliContext, binaryAsHex bool) error { _ = iohelp.WriteLine(cli.CliOut, welcomeMsg) historyFile := filepath.Join(".sqlhistory") // history file written to working dir @@ -878,6 +917,10 @@ func execShell(sqlCtx *sql.Context, qryist cli.Queryist, format engine.PrintResu verr := formatQueryError("", err) shell.Println(verr.Verbose()) } else if rowIter != nil { + // Apply binary-as-hex formatting to row data if enabled + if binaryAsHex { + rowIter = newBinaryAsHexRowIter(rowIter, sqlSch) + } switch closureFormat { case engine.FormatTabular, engine.FormatVertical: err = engine.PrettyPrintResultsExtended(sqlCtx, closureFormat, sqlSch, rowIter, pagerEnabled, toggleWarnings, true) diff --git a/go/cmd/dolt/commands/sqlserver/queryist_utils.go b/go/cmd/dolt/commands/sqlserver/queryist_utils.go index c38344813e8..0129a7145be 100644 --- a/go/cmd/dolt/commands/sqlserver/queryist_utils.go +++ b/go/cmd/dolt/commands/sqlserver/queryist_utils.go @@ -37,6 +37,7 @@ import ( "github.com/dolthub/dolt/go/libraries/utils/filesys" ) + // BuildConnectionStringQueryist returns a Queryist that connects to the server specified by the given server config. Presence in this // module isn't ideal, but it's the only way to get the server config into the queryist. func BuildConnectionStringQueryist(ctx context.Context, cwdFS filesys.Filesys, creds *cli.UserPassword, apr *argparser.ArgParseResults, host string, port int, useTLS bool, dbRev string) (cli.LateBindQueryist, error) { @@ -157,12 +158,15 @@ func NewMysqlRowWrapper(sqlRows *sql2.Rows) (*MysqlRowWrapper, error) { iRow := make([]interface{}, len(colTypes)) rows := make([]sql.Row, 0) for i, colType := range colTypes { - // Preserve binary type information for client-side binary-as-hex formatting - sqlType := types.LongText - dbTypeName := strings.ToUpper(colType.DatabaseTypeName()) - if dbTypeName == "VARBINARY" || dbTypeName == "BINARY" { - // Normalize both BINARY and VARBINARY to VarBinary for simpler formatting logic + // Check if this is a binary type by examining the database type name + typeName := strings.ToLower(colType.DatabaseTypeName()) + var sqlType sql.Type + switch typeName { + case "binary", "varbinary", "tinyblob", "blob", "mediumblob", "longblob": sqlType = types.MustCreateBinary(sqltypes.VarBinary, 255) + default: + // Default to LongText for all other types (as was done before) + sqlType = types.LongText } schema[i] = &sql.Column{ diff --git a/go/libraries/doltcore/table/untyped/tabular/fixedwidth_tablewriter.go b/go/libraries/doltcore/table/untyped/tabular/fixedwidth_tablewriter.go index 423a069386f..de9d9863f29 100644 --- a/go/libraries/doltcore/table/untyped/tabular/fixedwidth_tablewriter.go +++ b/go/libraries/doltcore/table/untyped/tabular/fixedwidth_tablewriter.go @@ -22,7 +22,6 @@ import ( "strings" "github.com/dolthub/go-mysql-server/sql" - "github.com/dolthub/vitess/go/sqltypes" "github.com/fatih/color" "github.com/dolthub/dolt/go/libraries/doltcore/diff" @@ -33,19 +32,6 @@ import ( const writeBufSize = 256 * 1024 -// Context key for binary-as-hex flag -type ctxKey int - -const binaryAsHexKey ctxKey = iota - -// WithBinaryAsHex returns a new context with binary-as-hex flag set -func WithBinaryAsHex(ctx *sql.Context, enabled bool) *sql.Context { - if ctx == nil { - return ctx - } - newCtx := context.WithValue(ctx.Context, binaryAsHexKey, enabled) - return ctx.WithContext(newCtx) -} // FixedWidthTableWriter is a TableWriter that applies a fixed width transform to its fields. All fields are // expected to be strings. @@ -234,29 +220,6 @@ func (w *FixedWidthTableWriter) stringValue(ctx *sql.Context, idx int, i interfa return "NULL", nil } - // Client-side binary-as-hex formatting (like MySQL client does) - if ctx != nil { - if val := ctx.Value(binaryAsHexKey); val != nil { - if enabled, ok := val.(bool); ok && enabled { - // Check if this is a binary type that should be formatted as hex - // Only check for VarBinary since MysqlRowWrapper normalizes both BINARY and VARBINARY to VarBinary - // TODO: Add support for other binary types per MySQL 8.4 docs: - // https://dev.mysql.com/doc/refman/8.4/en/mysql-command-options.html#option_mysql_binary-as-hex - if stringType, ok := w.schema[idx].Type.(sql.StringType); ok { - if stringType.Type() == sqltypes.VarBinary { - if binaryBytes, ok := i.([]byte); ok { - return fmt.Sprintf("0x%X", binaryBytes), nil - } - // Handle binary data that comes as string (from server connections) - if binaryString, ok := i.(string); ok { - return fmt.Sprintf("0x%X", []byte(binaryString)), nil - } - } - } - } - } - } - return sqlutil.SqlColToStr(ctx, w.schema[idx].Type, i) } From a3e6bbaf88e2e95f2d2c6f65884101701f00f71a Mon Sep 17 00:00:00 2001 From: elianddb Date: Fri, 25 Jul 2025 20:06:31 +0000 Subject: [PATCH 04/17] mv impl to print layer --- go/cmd/dolt/commands/blame.go | 2 +- .../commands/cvcmds/verify_constraints.go | 2 +- go/cmd/dolt/commands/engine/sql_print.go | 65 ++++++++++++- go/cmd/dolt/commands/schcmds/tags.go | 2 +- go/cmd/dolt/commands/sql.go | 97 ++++--------------- .../dolt/commands/sqlserver/queryist_utils.go | 17 +--- go/libraries/doltcore/sqle/sqlutil/sql_row.go | 22 +++++ .../untyped/tabular/fixedwidth_tablewriter.go | 1 - ...-as-hex-flag-precedence-interactive.expect | 46 +++++++++ ...y-as-hex-server-default-interactive.expect | 41 ++++++++ .../binary-as-hex-server-interactive.expect | 23 ++++- ...binary-as-hex-skip-flag-interactive.expect | 30 +++++- integration-tests/bats/sql-server.bats | 53 +++++++++- 13 files changed, 286 insertions(+), 115 deletions(-) create mode 100644 integration-tests/bats/binary-as-hex-flag-precedence-interactive.expect create mode 100644 integration-tests/bats/binary-as-hex-server-default-interactive.expect diff --git a/go/cmd/dolt/commands/blame.go b/go/cmd/dolt/commands/blame.go index 0ec7b9e5b9e..c22045b0ed0 100644 --- a/go/cmd/dolt/commands/blame.go +++ b/go/cmd/dolt/commands/blame.go @@ -130,7 +130,7 @@ func (cmd BlameCmd) Exec(ctx context.Context, commandStr string, args []string, return 1 } - err = engine.PrettyPrintResults(sqlCtx, engine.FormatTabular, schema, ri, false, false, true) + err = engine.PrettyPrintResults(sqlCtx, engine.FormatTabular, schema, ri, false, false, true, false) if err != nil { iohelp.WriteLine(cli.CliOut, err.Error()) return 1 diff --git a/go/cmd/dolt/commands/cvcmds/verify_constraints.go b/go/cmd/dolt/commands/cvcmds/verify_constraints.go index 3445cb69105..e473978781f 100644 --- a/go/cmd/dolt/commands/cvcmds/verify_constraints.go +++ b/go/cmd/dolt/commands/cvcmds/verify_constraints.go @@ -181,7 +181,7 @@ func printViolationsForTable(ctx *sql.Context, dbName, tblName string, tbl *dolt limitItr := &sqlLimitIter{itr: sqlItr, limit: 50} - err = engine.PrettyPrintResults(ctx, engine.FormatTabular, sqlSch, limitItr, false, false, false) + err = engine.PrettyPrintResults(ctx, engine.FormatTabular, sqlSch, limitItr, false, false, false, false) if err != nil { return errhand.BuildDError("Error outputting rows").AddCause(err).Build() } diff --git a/go/cmd/dolt/commands/engine/sql_print.go b/go/cmd/dolt/commands/engine/sql_print.go index ba715a62b7b..1b593982158 100644 --- a/go/cmd/dolt/commands/engine/sql_print.go +++ b/go/cmd/dolt/commands/engine/sql_print.go @@ -23,6 +23,7 @@ import ( "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/types" + "github.com/dolthub/vitess/go/sqltypes" "github.com/fatih/color" "github.com/dolthub/dolt/go/cmd/dolt/cli" @@ -56,16 +57,16 @@ const ( ) // PrettyPrintResults prints the result of a query in the format provided -func PrettyPrintResults(ctx *sql.Context, resultFormat PrintResultFormat, sqlSch sql.Schema, rowIter sql.RowIter, pageResults, showWarnings, printOkResult bool) (rerr error) { - return prettyPrintResultsWithSummary(ctx, resultFormat, sqlSch, rowIter, PrintNoSummary, pageResults, showWarnings, printOkResult) +func PrettyPrintResults(ctx *sql.Context, resultFormat PrintResultFormat, sqlSch sql.Schema, rowIter sql.RowIter, pageResults, showWarnings, printOkResult, binaryAsHex bool) (rerr error) { + return prettyPrintResultsWithSummary(ctx, resultFormat, sqlSch, rowIter, PrintNoSummary, pageResults, showWarnings, printOkResult, binaryAsHex) } // PrettyPrintResultsExtended prints the result of a query in the format provided, including row count and timing info -func PrettyPrintResultsExtended(ctx *sql.Context, resultFormat PrintResultFormat, sqlSch sql.Schema, rowIter sql.RowIter, pageResults, showWarnings, printOkResult bool) (rerr error) { - return prettyPrintResultsWithSummary(ctx, resultFormat, sqlSch, rowIter, PrintRowCountAndTiming, pageResults, showWarnings, printOkResult) +func PrettyPrintResultsExtended(ctx *sql.Context, resultFormat PrintResultFormat, sqlSch sql.Schema, rowIter sql.RowIter, pageResults, showWarnings, printOkResult, binaryAsHex bool) (rerr error) { + return prettyPrintResultsWithSummary(ctx, resultFormat, sqlSch, rowIter, PrintRowCountAndTiming, pageResults, showWarnings, printOkResult, binaryAsHex) } -func prettyPrintResultsWithSummary(ctx *sql.Context, resultFormat PrintResultFormat, sqlSch sql.Schema, rowIter sql.RowIter, summary PrintSummaryBehavior, pageResults, showWarnings, printOkResult bool) (rerr error) { +func prettyPrintResultsWithSummary(ctx *sql.Context, resultFormat PrintResultFormat, sqlSch sql.Schema, rowIter sql.RowIter, summary PrintSummaryBehavior, pageResults, showWarnings, printOkResult, binaryAsHex bool) (rerr error) { defer func() { closeErr := rowIter.Close(ctx) if rerr == nil && closeErr != nil { @@ -126,6 +127,11 @@ func prettyPrintResultsWithSummary(ctx *sql.Context, resultFormat PrintResultFor } } + // Wrap iterator with binary-to-hex transformation if needed + if binaryAsHex { + rowIter = newBinaryHexIterator(rowIter, sqlSch) + } + numRows, err = writeResultSet(ctx, rowIter, wr) } @@ -197,6 +203,55 @@ func printResultSetSummary(numRows int, numWarnings uint16, warningsList string, return nil } +// binaryHexIterator wraps a row iterator and transforms binary data to hex format +type binaryHexIterator struct { + inner sql.RowIter + schema sql.Schema +} + +// newBinaryHexIterator creates a new iterator that transforms binary data to hex format. +// It wraps the provided iterator and transforms BINARY and VARBINARY column values +// to hex string representation (e.g., "0x41424344"). +func newBinaryHexIterator(inner sql.RowIter, schema sql.Schema) sql.RowIter { + return &binaryHexIterator{ + inner: inner, + schema: schema, + } +} + +// Next returns the next row from the wrapped iterator with binary data transformed to hex format. +// Binary and VarBinary column values are converted to uppercase hex strings prefixed with "0x". +func (iter *binaryHexIterator) Next(ctx *sql.Context) (sql.Row, error) { + row, err := iter.inner.Next(ctx) + if err != nil { + return nil, err + } + + // Transform binary column values to hex string format in place + // Currently supports BINARY and VARBINARY types only. + // TODO: Add support for BLOB types (TINYBLOB, BLOB, MEDIUMBLOB, LONGBLOB) and BIT type + // as confirmed by testing MySQL 8.4+ binary-as-hex behavior + for i, val := range row { + if val != nil && i < len(iter.schema) { + switch iter.schema[i].Type.Type() { + case sqltypes.Binary, sqltypes.VarBinary: + if bytes, ok := val.([]byte); ok { + row[i] = fmt.Sprintf("0x%X", bytes) + } else { + row[i] = fmt.Sprintf("0x%X", []byte(fmt.Sprint(val))) + } + } + } + } + + return row, nil +} + +// Close closes the wrapped iterator and releases any resources. +func (iter *binaryHexIterator) Close(ctx *sql.Context) error { + return iter.inner.Close(ctx) +} + // writeResultSet drains the iterator given, printing rows from it to the writer given. Returns the number of rows. func writeResultSet(ctx *sql.Context, rowIter sql.RowIter, wr table.SqlRowWriter) (int, error) { i := 0 diff --git a/go/cmd/dolt/commands/schcmds/tags.go b/go/cmd/dolt/commands/schcmds/tags.go index a790d1ae3c9..f5fcf23bd33 100644 --- a/go/cmd/dolt/commands/schcmds/tags.go +++ b/go/cmd/dolt/commands/schcmds/tags.go @@ -140,7 +140,7 @@ func (cmd TagsCmd) Exec(ctx context.Context, commandStr string, args []string, d } sqlCtx := sql.NewContext(ctx) - err = engine.PrettyPrintResults(sqlCtx, outputFmt, headerSchema, sql.RowsToRowIter(rows...), false, false, false) + err = engine.PrettyPrintResults(sqlCtx, outputFmt, headerSchema, sql.RowsToRowIter(rows...), false, false, false, false) return commands.HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage) } diff --git a/go/cmd/dolt/commands/sql.go b/go/cmd/dolt/commands/sql.go index c9888b4c6d1..f1df2c5a99a 100644 --- a/go/cmd/dolt/commands/sql.go +++ b/go/cmd/dolt/commands/sql.go @@ -112,52 +112,6 @@ func init() { dsqle.AddDoltSystemVariables() } -// binaryAsHexRowIter wraps a row iterator and converts binary data to hex format -type binaryAsHexRowIter struct { - inner sql.RowIter - schema sql.Schema -} - -func newBinaryAsHexRowIter(inner sql.RowIter, schema sql.Schema) sql.RowIter { - return &binaryAsHexRowIter{ - inner: inner, - schema: schema, - } -} - -func (iter *binaryAsHexRowIter) Next(ctx *sql.Context) (sql.Row, error) { - row, err := iter.inner.Next(ctx) - if err != nil { - return nil, err - } - - // Transform binary data to hex format - for i, val := range row { - if val != nil && i < len(iter.schema) { - if stringType, ok := iter.schema[i].Type.(sql.StringType); ok { - sqlType := stringType.Type() - switch sqlType { - case sqltypes.Binary, sqltypes.VarBinary: - // Handle byte slice for VARBINARY/BINARY columns (local connections) - if binaryBytes, ok := val.([]byte); ok { - row[i] = fmt.Sprintf("0x%X", binaryBytes) - } else if binaryString, ok := val.(string); ok { - // Handle string data that contains binary (server connections) - row[i] = fmt.Sprintf("0x%X", []byte(binaryString)) - } - } - } - } - } - - return row, nil -} - -func (iter *binaryAsHexRowIter) Close(ctx *sql.Context) error { - return iter.inner.Close(ctx) -} - - type SqlCmd struct { VersionStr string } @@ -197,8 +151,8 @@ func (cmd SqlCmd) ArgParser() *argparser.ArgParser { ap.SupportsFlag(continueFlag, "c", "Continue running queries on an error. Used for batch mode only.") ap.SupportsString(fileInputFlag, "f", "input file", "Execute statements from the file given.") ap.SupportsFlag(binaryAsHexFlag, "", "Print binary data as hex. Enabled by default for interactive terminals.") - // --skip-binary-as-hex is supported but not shown in help, matching MySQL's behavior - ap.SupportsFlag(skipBinaryAsHexFlag, "", "") + // TODO: MySQL uses a skip- pattern for negating flags and doesn't show them in help + ap.SupportsFlag(skipBinaryAsHexFlag, "", "Disable binary data as hex output.") return ap } @@ -265,19 +219,8 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage) } - // Determine if we're running in a TTY (used by all modes) - isTty := false - fi, err := os.Stdin.Stat() - if err != nil { - if !osutil.IsWindows { - return HandleVErrAndExitCode(errhand.BuildDError("Couldn't stat STDIN. This is a bug.").Build(), usage) - } - } else { - isTty = fi.Mode()&os.ModeCharDevice != 0 - } - - // Determine binary-as-hex behavior once (default based on TTY, flags can override) - binaryAsHex := isTty + // Determine binary-as-hex behavior from flags (default false for non-interactive modes) + binaryAsHex := false if apr.Contains(skipBinaryAsHexFlag) { binaryAsHex = false } else if apr.Contains(binaryAsHexFlag) { @@ -303,6 +246,15 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE return listSavedQueries(sqlCtx, queryist, dEnv, format, usage) } else { // Run in either batch mode for piped input, or shell mode for interactive + isTty := false + fi, err := os.Stdin.Stat() + if err != nil { + if !osutil.IsWindows { + return sqlHandleVErrAndExitCode(queryist, errhand.BuildDError("Couldn't stat STDIN. This is a bug.").Build(), usage) + } + } else { + isTty = fi.Mode()&os.ModeCharDevice != 0 + } _, continueOnError := apr.GetValue(continueFlag) @@ -326,7 +278,9 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE } if isTty { - err := execShell(sqlCtx, queryist, format, cliCtx, binaryAsHex) + // In shell mode, default to hex format unless explicitly disabled + shellBinaryAsHex := !apr.Contains(skipBinaryAsHexFlag) + err := execShell(sqlCtx, queryist, format, cliCtx, shellBinaryAsHex) if err != nil { return sqlHandleVErrAndExitCode(queryist, errhand.VerboseErrorFromError(err), usage) } @@ -513,10 +467,7 @@ func execSingleQuery( if rowIter != nil { // Apply binary-as-hex formatting if enabled - if binaryAsHex { - rowIter = newBinaryAsHexRowIter(rowIter, sqlSch) - } - err = engine.PrettyPrintResults(sqlCtx, format, sqlSch, rowIter, false, false, false) + err = engine.PrettyPrintResults(sqlCtx, format, sqlSch, rowIter, false, false, false, binaryAsHex) if err != nil { return errhand.VerboseErrorFromError(err) } @@ -732,11 +683,7 @@ func execBatchMode(ctx *sql.Context, qryist cli.Queryist, input io.Reader, conti fileReadProg.printNewLineIfNeeded() } } - // Apply binary-as-hex formatting if enabled - if binaryAsHex { - rowIter = newBinaryAsHexRowIter(rowIter, sqlSch) - } - err = engine.PrettyPrintResults(ctx, format, sqlSch, rowIter, false, false, false) + err = engine.PrettyPrintResults(ctx, format, sqlSch, rowIter, false, false, false, binaryAsHex) if err != nil { err = buildBatchSqlErr(scanner.state.statementStartLine, query, err) if !continueOnErr { @@ -917,15 +864,11 @@ func execShell(sqlCtx *sql.Context, qryist cli.Queryist, format engine.PrintResu verr := formatQueryError("", err) shell.Println(verr.Verbose()) } else if rowIter != nil { - // Apply binary-as-hex formatting to row data if enabled - if binaryAsHex { - rowIter = newBinaryAsHexRowIter(rowIter, sqlSch) - } switch closureFormat { case engine.FormatTabular, engine.FormatVertical: - err = engine.PrettyPrintResultsExtended(sqlCtx, closureFormat, sqlSch, rowIter, pagerEnabled, toggleWarnings, true) + err = engine.PrettyPrintResultsExtended(sqlCtx, closureFormat, sqlSch, rowIter, pagerEnabled, toggleWarnings, true, binaryAsHex) default: - err = engine.PrettyPrintResults(sqlCtx, closureFormat, sqlSch, rowIter, pagerEnabled, toggleWarnings, true) + err = engine.PrettyPrintResults(sqlCtx, closureFormat, sqlSch, rowIter, pagerEnabled, toggleWarnings, true, binaryAsHex) } } else { if _, isUseStmt := sqlStmt.(*sqlparser.Use); isUseStmt { diff --git a/go/cmd/dolt/commands/sqlserver/queryist_utils.go b/go/cmd/dolt/commands/sqlserver/queryist_utils.go index 0129a7145be..ec04c6cde1d 100644 --- a/go/cmd/dolt/commands/sqlserver/queryist_utils.go +++ b/go/cmd/dolt/commands/sqlserver/queryist_utils.go @@ -23,8 +23,6 @@ import ( "strings" "github.com/dolthub/go-mysql-server/sql" - "github.com/dolthub/go-mysql-server/sql/types" - "github.com/dolthub/vitess/go/sqltypes" "github.com/dolthub/vitess/go/vt/sqlparser" "github.com/go-sql-driver/mysql" "github.com/gocraft/dbr/v2" @@ -33,11 +31,11 @@ import ( "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" "github.com/dolthub/dolt/go/libraries/utils/argparser" "github.com/dolthub/dolt/go/libraries/utils/filesys" ) - // BuildConnectionStringQueryist returns a Queryist that connects to the server specified by the given server config. Presence in this // module isn't ideal, but it's the only way to get the server config into the queryist. func BuildConnectionStringQueryist(ctx context.Context, cwdFS filesys.Filesys, creds *cli.UserPassword, apr *argparser.ArgParseResults, host string, port int, useTLS bool, dbRev string) (cli.LateBindQueryist, error) { @@ -158,20 +156,9 @@ func NewMysqlRowWrapper(sqlRows *sql2.Rows) (*MysqlRowWrapper, error) { iRow := make([]interface{}, len(colTypes)) rows := make([]sql.Row, 0) for i, colType := range colTypes { - // Check if this is a binary type by examining the database type name - typeName := strings.ToLower(colType.DatabaseTypeName()) - var sqlType sql.Type - switch typeName { - case "binary", "varbinary", "tinyblob", "blob", "mediumblob", "longblob": - sqlType = types.MustCreateBinary(sqltypes.VarBinary, 255) - default: - // Default to LongText for all other types (as was done before) - sqlType = types.LongText - } - schema[i] = &sql.Column{ Name: colType.Name(), - Type: sqlType, + Type: sqlutil.DatabaseTypeNameToSqlType(colType.DatabaseTypeName()), Nullable: true, } iRow[i] = &vRow[i] diff --git a/go/libraries/doltcore/sqle/sqlutil/sql_row.go b/go/libraries/doltcore/sqle/sqlutil/sql_row.go index f8b3a99400d..4abed78b472 100644 --- a/go/libraries/doltcore/sqle/sqlutil/sql_row.go +++ b/go/libraries/doltcore/sqle/sqlutil/sql_row.go @@ -18,10 +18,14 @@ import ( "context" "errors" "fmt" + "strings" "github.com/dolthub/go-mysql-server/sql" + gmstypes "github.com/dolthub/go-mysql-server/sql/types" + // Necessary for the empty context used by some functions to be initialized with system vars _ "github.com/dolthub/go-mysql-server/sql/variables" + "github.com/dolthub/vitess/go/sqltypes" "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" @@ -248,3 +252,21 @@ func SqlColToStr(ctx *sql.Context, sqlType sql.Type, col interface{}) (string, e return "", nil } + +// DatabaseTypeNameToSqlType converts a MySQL wire protocol database type name +// to a go-mysql-server sql.Type. This uses the same type mapping logic as the existing +// Dolt type system for consistency. +// TODO: Add support for BLOB types (TINYBLOB, BLOB, MEDIUMBLOB, LONGBLOB) and BIT type +// as confirmed by testing MySQL 8.4+ binary-as-hex behavior +func DatabaseTypeNameToSqlType(databaseTypeName string) sql.Type { + typeName := strings.ToLower(databaseTypeName) + switch typeName { + case "binary": + return gmstypes.MustCreateBinary(sqltypes.Binary, 255) + case "varbinary": + return gmstypes.MustCreateBinary(sqltypes.VarBinary, 255) + default: + // Default to LongText for all other types (as was done before) + return gmstypes.LongText + } +} diff --git a/go/libraries/doltcore/table/untyped/tabular/fixedwidth_tablewriter.go b/go/libraries/doltcore/table/untyped/tabular/fixedwidth_tablewriter.go index de9d9863f29..6b72cb03114 100644 --- a/go/libraries/doltcore/table/untyped/tabular/fixedwidth_tablewriter.go +++ b/go/libraries/doltcore/table/untyped/tabular/fixedwidth_tablewriter.go @@ -32,7 +32,6 @@ import ( const writeBufSize = 256 * 1024 - // FixedWidthTableWriter is a TableWriter that applies a fixed width transform to its fields. All fields are // expected to be strings. type FixedWidthTableWriter struct { diff --git a/integration-tests/bats/binary-as-hex-flag-precedence-interactive.expect b/integration-tests/bats/binary-as-hex-flag-precedence-interactive.expect new file mode 100644 index 00000000000..16a0d75ee3d --- /dev/null +++ b/integration-tests/bats/binary-as-hex-flag-precedence-interactive.expect @@ -0,0 +1,46 @@ +#!/usr/bin/expect + +set port [lindex $argv 0] +set timeout 10 + +spawn dolt --host 127.0.0.1 --port $port --no-tls sql --binary-as-hex --skip-binary-as-hex +expect ">" + +send "USE repo1;\r" +expect ">" + +send "SELECT pk, vb FROM binary_test WHERE pk = 2;\r" +expect { + "0x616263" { + send "exit\r" + expect eof + exit 1 + } + "abc" { + expect ">" + send "SELECT pk, vb FROM binary_test WHERE pk = 1;\r" + expect { + "0x0A000000" { + send "exit\r" + expect eof + exit 1 + } + -re "1.*\n" { + expect ">" + send "exit\r" + expect eof + exit 0 + } + timeout { + send "exit\r" + expect eof + exit 1 + } + } + } + timeout { + send "exit\r" + expect eof + exit 1 + } +} \ No newline at end of file diff --git a/integration-tests/bats/binary-as-hex-server-default-interactive.expect b/integration-tests/bats/binary-as-hex-server-default-interactive.expect new file mode 100644 index 00000000000..5bdac7c22a1 --- /dev/null +++ b/integration-tests/bats/binary-as-hex-server-default-interactive.expect @@ -0,0 +1,41 @@ +#!/usr/bin/expect + +set port [lindex $argv 0] +set timeout 10 + +spawn dolt --host 127.0.0.1 --port $port --no-tls sql +expect ">" + +send "USE repo1;\r" +expect ">" + +send "SELECT pk, vb FROM binary_test WHERE pk = 1;\r" +expect { + "0x0A000000" { + expect ">" + send "SELECT pk, vb FROM binary_test WHERE pk = 2;\r" + expect { + "0x616263" { + expect ">" + send "exit\r" + expect eof + exit 0 + } + "abc" { + send "exit\r" + expect eof + exit 1 + } + timeout { + send "exit\r" + expect eof + exit 1 + } + } + } + timeout { + send "exit\r" + expect eof + exit 1 + } +} \ No newline at end of file diff --git a/integration-tests/bats/binary-as-hex-server-interactive.expect b/integration-tests/bats/binary-as-hex-server-interactive.expect index 62ad47a8376..e74498206ce 100755 --- a/integration-tests/bats/binary-as-hex-server-interactive.expect +++ b/integration-tests/bats/binary-as-hex-server-interactive.expect @@ -9,14 +9,29 @@ expect ">" send "USE repo1;\r" expect ">" -send "SELECT * FROM binary_test;\r" +send "SELECT pk, vb FROM binary_test WHERE pk = 2;\r" expect { - "0x0A000000" { - # Found hex output - this is correct + "0x616263" { expect ">" + send "SELECT pk, vb FROM binary_test WHERE pk = 1;\r" + expect { + "0x0A000000" { + expect ">" + send "exit\r" + expect eof + exit 0 + } + timeout { + send "exit\r" + expect eof + exit 1 + } + } + } + "abc" { send "exit\r" expect eof - exit 0 + exit 1 } timeout { send "exit\r" diff --git a/integration-tests/bats/binary-as-hex-skip-flag-interactive.expect b/integration-tests/bats/binary-as-hex-skip-flag-interactive.expect index dac896897c3..423e093f57f 100755 --- a/integration-tests/bats/binary-as-hex-skip-flag-interactive.expect +++ b/integration-tests/bats/binary-as-hex-skip-flag-interactive.expect @@ -9,14 +9,34 @@ expect ">" send "USE repo1;\r" expect ">" -send "SELECT * FROM binary_test;\r" +send "SELECT pk, vb FROM binary_test WHERE pk = 2;\r" expect { - -re "\\| 1 \\|.*\\|" { - # Found raw binary output (not hex) - this is correct for skip flag - expect ">" + "0x616263" { send "exit\r" expect eof - exit 0 + exit 1 + } + "abc" { + expect ">" + send "SELECT pk, vb FROM binary_test WHERE pk = 1;\r" + expect { + "0x0A000000" { + send "exit\r" + expect eof + exit 1 + } + -re "1.*\n" { + expect ">" + send "exit\r" + expect eof + exit 0 + } + timeout { + send "exit\r" + expect eof + exit 1 + } + } } timeout { send "exit\r" diff --git a/integration-tests/bats/sql-server.bats b/integration-tests/bats/sql-server.bats index 341e74298b4..56940f1117c 100644 --- a/integration-tests/bats/sql-server.bats +++ b/integration-tests/bats/sql-server.bats @@ -2199,7 +2199,7 @@ EOF } -@test "sql-server: client interactive binary-as-hex behavior works with server connections" { +@test "sql-server: client binary-as-hex behavior works with server connections" { skiponwindows "Missing dependencies" which expect > /dev/null || skip "expect not available" @@ -2212,16 +2212,59 @@ EOF start_sql_server repo1 - # Test --binary-as-hex flag in interactive mode (should show hex) + # 1. Test default interactive behavior (should show hex by default) + run expect "$BATS_TEST_DIRNAME/binary-as-hex-server-default-interactive.expect" $PORT + [ $status -eq 0 ] + + # 2. Test --binary-as-hex flag in interactive mode (should show hex) run expect "$BATS_TEST_DIRNAME/binary-as-hex-server-interactive.expect" $PORT - echo "EXPECT OUTPUT: $output" - echo "EXPECT STATUS: $status" [ $status -eq 0 ] - # Test --skip-binary-as-hex flag in interactive mode (should show raw) + # 3. Test --skip-binary-as-hex flag in interactive mode (should show raw) run expect "$BATS_TEST_DIRNAME/binary-as-hex-skip-flag-interactive.expect" $PORT [ $status -eq 0 ] + # 4. Test flag precedence: --skip-binary-as-hex overrides --binary-as-hex + run expect "$BATS_TEST_DIRNAME/binary-as-hex-flag-precedence-interactive.expect" $PORT + [ $status -eq 0 ] + + # 5. Test non-interactive server behavior with -q flag (should show raw by default) + run dolt --host 127.0.0.1 --port $PORT --no-tls sql -q "USE repo1; SELECT vb FROM binary_test WHERE pk = 1" + [ $status -eq 0 ] + ! [[ "$output" =~ "0x0A000000" ]] || false + + # 6. Test non-interactive server behavior with --binary-as-hex flag + run dolt --host 127.0.0.1 --port $PORT --no-tls sql --binary-as-hex -q "USE repo1; SELECT vb FROM binary_test WHERE pk = 1" + [ $status -eq 0 ] + [[ "$output" =~ "0x0A000000" ]] || false + + # 7. Test non-interactive server behavior with printable data + run dolt --host 127.0.0.1 --port $PORT --no-tls sql -q "USE repo1; SELECT vb FROM binary_test WHERE pk = 2" + [ $status -eq 0 ] + [[ "$output" =~ "abc" ]] || false + + # 8. Test non-interactive server flag precedence + run dolt --host 127.0.0.1 --port $PORT --no-tls sql --binary-as-hex --skip-binary-as-hex -q "USE repo1; SELECT vb FROM binary_test WHERE pk = 2" + [ $status -eq 0 ] + [[ "$output" =~ "abc" ]] || false + ! [[ "$output" =~ "0x616263" ]] || false + + # 9. Test non-interactive server behavior with -q flag (should show raw by default) + run dolt --host 127.0.0.1 --port $PORT --no-tls sql -q "USE repo1; SELECT vb FROM binary_test WHERE pk = 1" + [ $status -eq 0 ] + ! [[ "$output" =~ "0x0A000000" ]] || false + + # 10. Test non-interactive server behavior with -q and --binary-as-hex flags + run dolt --host 127.0.0.1 --port $PORT --no-tls sql --binary-as-hex -q "USE repo1; SELECT vb FROM binary_test WHERE pk = 1" + [ $status -eq 0 ] + [[ "$output" =~ "0x0A000000" ]] || false + + # 11. Test non-interactive server -q flag precedence + run dolt --host 127.0.0.1 --port $PORT --no-tls sql --binary-as-hex --skip-binary-as-hex -q "USE repo1; SELECT vb FROM binary_test WHERE pk = 2" + [ $status -eq 0 ] + [[ "$output" =~ "abc" ]] || false + ! [[ "$output" =~ "0x616263" ]] || false + stop_sql_server 1 } From 123f24da0372ea54cfac07dd7535d1d623c2342f Mon Sep 17 00:00:00 2001 From: elianddb Date: Mon, 28 Jul 2025 19:47:03 +0000 Subject: [PATCH 05/17] add non-filter tests and interactive test for local db --- ...-as-hex-flag-precedence-interactive.expect | 23 +++++++++++ .../binary-as-hex-local-db-interactive.expect | 38 ++++++++++++++++++ ...ry-as-hex-local-db-skip-interactive.expect | 39 +++++++++++++++++++ ...y-as-hex-server-default-interactive.expect | 22 +++++++++++ .../binary-as-hex-server-interactive.expect | 22 +++++++++++ ...binary-as-hex-skip-flag-interactive.expect | 23 +++++++++++ integration-tests/bats/sql-server.bats | 21 ++++++++++ integration-tests/bats/sql.bats | 14 +++++++ 8 files changed, 202 insertions(+) create mode 100755 integration-tests/bats/binary-as-hex-local-db-interactive.expect create mode 100755 integration-tests/bats/binary-as-hex-local-db-skip-interactive.expect diff --git a/integration-tests/bats/binary-as-hex-flag-precedence-interactive.expect b/integration-tests/bats/binary-as-hex-flag-precedence-interactive.expect index 16a0d75ee3d..e443aaee711 100644 --- a/integration-tests/bats/binary-as-hex-flag-precedence-interactive.expect +++ b/integration-tests/bats/binary-as-hex-flag-precedence-interactive.expect @@ -2,6 +2,7 @@ set port [lindex $argv 0] set timeout 10 +log_user 0 spawn dolt --host 127.0.0.1 --port $port --no-tls sql --binary-as-hex --skip-binary-as-hex expect ">" @@ -9,6 +10,28 @@ expect ">" send "USE repo1;\r" expect ">" +send "SELECT * FROM binary_test;\r" +expect { + "abc" { + expect ">" + } + "0x616263" { + send "exit\r" + expect eof + exit 1 + } + "0x0A000000" { + send "exit\r" + expect eof + exit 1 + } + timeout { + send "exit\r" + expect eof + exit 1 + } +} + send "SELECT pk, vb FROM binary_test WHERE pk = 2;\r" expect { "0x616263" { diff --git a/integration-tests/bats/binary-as-hex-local-db-interactive.expect b/integration-tests/bats/binary-as-hex-local-db-interactive.expect new file mode 100755 index 00000000000..ed89d663bac --- /dev/null +++ b/integration-tests/bats/binary-as-hex-local-db-interactive.expect @@ -0,0 +1,38 @@ +#!/usr/bin/expect + +set timeout 10 +log_user 0 + +# Test with --binary-as-hex flag +spawn dolt sql --binary-as-hex +expect ">" + +send "CREATE TABLE test_binary (pk INT PRIMARY KEY, vb VARBINARY(20));\r" +expect ">" + +send "INSERT INTO test_binary VALUES (1, 0x0A000000), (2, 'abc');\r" +expect ">" + +send "SELECT * FROM test_binary;\r" +expect { + "| 1 | 0x0A000000 |" { + expect { + "| 2 | 0x616263 |" { + expect ">" + send "exit\r" + expect eof + exit 0 + } + timeout { + send "exit\r" + expect eof + exit 1 + } + } + } + timeout { + send "exit\r" + expect eof + exit 1 + } +} \ No newline at end of file diff --git a/integration-tests/bats/binary-as-hex-local-db-skip-interactive.expect b/integration-tests/bats/binary-as-hex-local-db-skip-interactive.expect new file mode 100755 index 00000000000..febd4204a56 --- /dev/null +++ b/integration-tests/bats/binary-as-hex-local-db-skip-interactive.expect @@ -0,0 +1,39 @@ +#!/usr/bin/expect + +set timeout 10 +log_user 0 + +# Test with --skip-binary-as-hex flag +spawn dolt sql --skip-binary-as-hex +expect ">" + +send "CREATE TABLE test_binary (pk INT PRIMARY KEY, vb VARBINARY(20));\r" +expect ">" + +send "INSERT INTO test_binary VALUES (1, 0x0A000000), (2, 'abc');\r" +expect ">" + +send "SELECT * FROM test_binary;\r" +expect { + -re {\\| 2.*\\| abc \\|} { + expect ">" + send "exit\r" + expect eof + exit 0 + } + "0x616263" { + send "exit\r" + expect eof + exit 1 + } + "0x0A000000" { + send "exit\r" + expect eof + exit 1 + } + timeout { + send "exit\r" + expect eof + exit 1 + } +} \ No newline at end of file diff --git a/integration-tests/bats/binary-as-hex-server-default-interactive.expect b/integration-tests/bats/binary-as-hex-server-default-interactive.expect index 5bdac7c22a1..a7aaada8048 100644 --- a/integration-tests/bats/binary-as-hex-server-default-interactive.expect +++ b/integration-tests/bats/binary-as-hex-server-default-interactive.expect @@ -2,6 +2,7 @@ set port [lindex $argv 0] set timeout 10 +log_user 0 spawn dolt --host 127.0.0.1 --port $port --no-tls sql expect ">" @@ -9,6 +10,27 @@ expect ">" send "USE repo1;\r" expect ">" +send "SELECT * FROM binary_test;\r" +expect { + "0x0A000000" { + expect { + "0x616263" { + expect ">" + } + timeout { + send "exit\r" + expect eof + exit 1 + } + } + } + timeout { + send "exit\r" + expect eof + exit 1 + } +} + send "SELECT pk, vb FROM binary_test WHERE pk = 1;\r" expect { "0x0A000000" { diff --git a/integration-tests/bats/binary-as-hex-server-interactive.expect b/integration-tests/bats/binary-as-hex-server-interactive.expect index e74498206ce..c3ca8721c1e 100755 --- a/integration-tests/bats/binary-as-hex-server-interactive.expect +++ b/integration-tests/bats/binary-as-hex-server-interactive.expect @@ -2,6 +2,7 @@ set port [lindex $argv 0] set timeout 10 +log_user 0 spawn dolt --host 127.0.0.1 --port $port --no-tls sql --binary-as-hex expect ">" @@ -9,6 +10,27 @@ expect ">" send "USE repo1;\r" expect ">" +send "SELECT * FROM binary_test;\r" +expect { + "0x0A000000" { + expect { + "0x616263" { + expect ">" + } + timeout { + send "exit\r" + expect eof + exit 1 + } + } + } + timeout { + send "exit\r" + expect eof + exit 1 + } +} + send "SELECT pk, vb FROM binary_test WHERE pk = 2;\r" expect { "0x616263" { diff --git a/integration-tests/bats/binary-as-hex-skip-flag-interactive.expect b/integration-tests/bats/binary-as-hex-skip-flag-interactive.expect index 423e093f57f..627968733ac 100755 --- a/integration-tests/bats/binary-as-hex-skip-flag-interactive.expect +++ b/integration-tests/bats/binary-as-hex-skip-flag-interactive.expect @@ -2,6 +2,7 @@ set port [lindex $argv 0] set timeout 10 +log_user 0 spawn dolt --host 127.0.0.1 --port $port --no-tls sql --skip-binary-as-hex expect ">" @@ -9,6 +10,28 @@ expect ">" send "USE repo1;\r" expect ">" +send "SELECT * FROM binary_test;\r" +expect { + "abc" { + expect ">" + } + "0x616263" { + send "exit\r" + expect eof + exit 1 + } + "0x0A000000" { + send "exit\r" + expect eof + exit 1 + } + timeout { + send "exit\r" + expect eof + exit 1 + } +} + send "SELECT pk, vb FROM binary_test WHERE pk = 2;\r" expect { "0x616263" { diff --git a/integration-tests/bats/sql-server.bats b/integration-tests/bats/sql-server.bats index 56940f1117c..dc5f1060986 100644 --- a/integration-tests/bats/sql-server.bats +++ b/integration-tests/bats/sql-server.bats @@ -2265,6 +2265,27 @@ EOF [[ "$output" =~ "abc" ]] || false ! [[ "$output" =~ "0x616263" ]] || false + # 12. Test non-filtered SELECT with default behavior (should show raw by default) + run dolt --host 127.0.0.1 --port $PORT --no-tls sql -q "USE repo1; SELECT * FROM binary_test" + [ $status -eq 0 ] + ! [[ "$output" =~ "0x0A000000" ]] || false + ! [[ "$output" =~ "0x616263" ]] || false + [[ "$output" =~ "abc" ]] || false + + # 13. Test non-filtered SELECT with --binary-as-hex flag + run dolt --host 127.0.0.1 --port $PORT --no-tls sql --binary-as-hex -q "USE repo1; SELECT * FROM binary_test" + [ $status -eq 0 ] + [[ "$output" =~ "0x0A000000" ]] || false + [[ "$output" =~ "0x616263" ]] || false + + # 14. Test non-filtered SELECT with --skip-binary-as-hex flag + run dolt --host 127.0.0.1 --port $PORT --no-tls sql --skip-binary-as-hex -q "USE repo1; SELECT * FROM binary_test" + [ $status -eq 0 ] + ! [[ "$output" =~ "0x0A000000" ]] || false + ! [[ "$output" =~ "0x616263" ]] || false + [[ "$output" =~ "abc" ]] || false + stop_sql_server 1 } + diff --git a/integration-tests/bats/sql.bats b/integration-tests/bats/sql.bats index 10e1694ae46..5a2aed36827 100755 --- a/integration-tests/bats/sql.bats +++ b/integration-tests/bats/sql.bats @@ -2959,3 +2959,17 @@ SQL # Cleanup dolt sql -q "DROP TABLE binary_test" } + +@test "sql: interactive binary-as-hex behavior with local database non-filtered selects" { + skiponwindows "Missing dependencies" + which expect > /dev/null || skip "expect not available" + + # Test interactive shell with --binary-as-hex flag (includes non-filtered SELECT *) + run expect "$BATS_TEST_DIRNAME/binary-as-hex-local-db-interactive.expect" + [ $status -eq 0 ] + + # Test interactive shell with --skip-binary-as-hex flag (includes non-filtered SELECT *) + run expect "$BATS_TEST_DIRNAME/binary-as-hex-local-db-skip-interactive.expect" + [ $status -eq 0 ] +} + From 130ef212ba6de37af8556be693b2fde6a1b123cd Mon Sep 17 00:00:00 2001 From: elianddb Date: Mon, 28 Jul 2025 20:04:02 +0000 Subject: [PATCH 06/17] rm extra imports --- go/cmd/dolt/commands/sql.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/cmd/dolt/commands/sql.go b/go/cmd/dolt/commands/sql.go index f1df2c5a99a..e0c4b4fdb2e 100644 --- a/go/cmd/dolt/commands/sql.go +++ b/go/cmd/dolt/commands/sql.go @@ -47,7 +47,6 @@ import ( "github.com/dolthub/dolt/go/libraries/utils/iohelp" "github.com/dolthub/dolt/go/libraries/utils/osutil" eventsapi "github.com/dolthub/eventsapi_schema/dolt/services/eventsapi/v1alpha1" - "github.com/dolthub/vitess/go/sqltypes" ) var sqlDocs = cli.CommandDocumentationContent{ From 2393ca393fc1196f0f48035c1b65b1c2c3257cc1 Mon Sep 17 00:00:00 2001 From: elianddb Date: Mon, 28 Jul 2025 22:43:09 +0000 Subject: [PATCH 07/17] bypass column validation on binaryAsHex flag print --- go/cmd/dolt/commands/engine/sql_print.go | 21 +++++++------- go/libraries/doltcore/sqle/sqlutil/sql_row.go | 8 ++++++ integration-tests/bats/sql-server.bats | 28 +++++++++++++++++++ integration-tests/bats/sql.bats | 27 ++++++++++++++++++ 4 files changed, 74 insertions(+), 10 deletions(-) diff --git a/go/cmd/dolt/commands/engine/sql_print.go b/go/cmd/dolt/commands/engine/sql_print.go index 1b593982158..24eaea0686f 100644 --- a/go/cmd/dolt/commands/engine/sql_print.go +++ b/go/cmd/dolt/commands/engine/sql_print.go @@ -203,15 +203,20 @@ func printResultSetSummary(numRows int, numWarnings uint16, warningsList string, return nil } +// binaryAsHexDisplayValue wraps hex-formatted binary data to bypass validation +type binaryAsHexDisplayValue string + +func (h binaryAsHexDisplayValue) String() string { + return string(h) +} + // binaryHexIterator wraps a row iterator and transforms binary data to hex format type binaryHexIterator struct { inner sql.RowIter schema sql.Schema } -// newBinaryHexIterator creates a new iterator that transforms binary data to hex format. -// It wraps the provided iterator and transforms BINARY and VARBINARY column values -// to hex string representation (e.g., "0x41424344"). +// newBinaryHexIterator creates a new iterator that transforms binary data to hex format func newBinaryHexIterator(inner sql.RowIter, schema sql.Schema) sql.RowIter { return &binaryHexIterator{ inner: inner, @@ -219,26 +224,22 @@ func newBinaryHexIterator(inner sql.RowIter, schema sql.Schema) sql.RowIter { } } -// Next returns the next row from the wrapped iterator with binary data transformed to hex format. -// Binary and VarBinary column values are converted to uppercase hex strings prefixed with "0x". +// Next returns the next row with binary data transformed to hex format func (iter *binaryHexIterator) Next(ctx *sql.Context) (sql.Row, error) { row, err := iter.inner.Next(ctx) if err != nil { return nil, err } - // Transform binary column values to hex string format in place - // Currently supports BINARY and VARBINARY types only. // TODO: Add support for BLOB types (TINYBLOB, BLOB, MEDIUMBLOB, LONGBLOB) and BIT type - // as confirmed by testing MySQL 8.4+ binary-as-hex behavior for i, val := range row { if val != nil && i < len(iter.schema) { switch iter.schema[i].Type.Type() { case sqltypes.Binary, sqltypes.VarBinary: if bytes, ok := val.([]byte); ok { - row[i] = fmt.Sprintf("0x%X", bytes) + row[i] = binaryAsHexDisplayValue(fmt.Sprintf("0x%X", bytes)) } else { - row[i] = fmt.Sprintf("0x%X", []byte(fmt.Sprint(val))) + row[i] = binaryAsHexDisplayValue(fmt.Sprintf("0x%X", []byte(fmt.Sprint(val)))) } } } diff --git a/go/libraries/doltcore/sqle/sqlutil/sql_row.go b/go/libraries/doltcore/sqle/sqlutil/sql_row.go index 4abed78b472..d1633263783 100644 --- a/go/libraries/doltcore/sqle/sqlutil/sql_row.go +++ b/go/libraries/doltcore/sqle/sqlutil/sql_row.go @@ -223,10 +223,18 @@ func keylessDoltRowFromSqlRow(ctx context.Context, vrw types.ValueReadWriter, sq return row.KeylessRow(vrw.Format(), vals[:j]...) } +type binaryAsHexDisplayValue interface { + String() string +} + // SqlColToStr is a utility function for converting a sql column of type interface{} to a string. // NULL values are treated as empty strings. Handle nil separately if you require other behavior. func SqlColToStr(ctx *sql.Context, sqlType sql.Type, col interface{}) (string, error) { if col != nil { + if hexVal, ok := col.(binaryAsHexDisplayValue); ok { + return hexVal.String(), nil + } + switch typedCol := col.(type) { case bool: if typedCol { diff --git a/integration-tests/bats/sql-server.bats b/integration-tests/bats/sql-server.bats index dc5f1060986..f7a7cf26aed 100644 --- a/integration-tests/bats/sql-server.bats +++ b/integration-tests/bats/sql-server.bats @@ -2288,4 +2288,32 @@ EOF stop_sql_server 1 } +@test "sql-server: binary-as-hex works with data at column constraint limits" { + skiponwindows "Missing dependencies" + + cd repo1 + + # Setup: Create table with binary data at column size limits + dolt sql -q "CREATE TABLE constraint_test (pk INT PRIMARY KEY, data VARBINARY(20));" + dolt sql -q "INSERT INTO constraint_test VALUES (1, UNHEX('00112233445566778899AABBCCDDEEFF00112233'));" + dolt sql -q "INSERT INTO constraint_test VALUES (2, X'');" + dolt add . + dolt commit -m "Add constraint test data" + + start_sql_server repo1 + + # Test server connection with binary-as-hex at column constraint limits + run dolt --host 127.0.0.1 --port $PORT --no-tls sql --binary-as-hex -q "USE repo1; SELECT * FROM constraint_test ORDER BY pk" + [ $status -eq 0 ] + [[ "$output" =~ "0x00112233445566778899AABBCCDDEEFF00112233" ]] || false + [[ "$output" =~ "0x" ]] || false # Empty binary shows as "0x" + + # Test CSV format with server connection + run dolt --host 127.0.0.1 --port $PORT --no-tls sql --binary-as-hex --result-format=csv -q "USE repo1; SELECT * FROM constraint_test WHERE pk = 1" + [ $status -eq 0 ] + [[ "$output" =~ "1,0x00112233445566778899AABBCCDDEEFF00112233" ]] || false + + stop_sql_server +} + diff --git a/integration-tests/bats/sql.bats b/integration-tests/bats/sql.bats index 5a2aed36827..e78836d304d 100755 --- a/integration-tests/bats/sql.bats +++ b/integration-tests/bats/sql.bats @@ -2960,6 +2960,33 @@ SQL dolt sql -q "DROP TABLE binary_test" } +@test "sql: binary-as-hex works with data at column constraint limits" { + # Test case for issue #9554: binary data that approaches or reaches column size limits + # should display correctly as hex without validation errors + + # Create table with VARBINARY(20) and insert 20 bytes of data (maximum for column) + dolt sql -q "CREATE TABLE constraint_test (pk INT PRIMARY KEY, data VARBINARY(20));" + dolt sql -q "INSERT INTO constraint_test VALUES (1, UNHEX('00112233445566778899AABBCCDDEEFF00112233'));" + dolt sql -q "INSERT INTO constraint_test VALUES (2, X'');" # Empty binary data + + # Verify the data length is exactly 20 bytes + run dolt sql -q "SELECT LENGTH(data) FROM constraint_test WHERE pk = 1" + [ $status -eq 0 ] + [[ "$output" =~ "20" ]] || false + + # Test binary-as-hex display works without validation errors + # The 20-byte data becomes 42-character hex string "0x00112233445566778899AABBCCDDEEFF00112233" + run dolt sql --binary-as-hex -q "SELECT * FROM constraint_test ORDER BY pk" + [ $status -eq 0 ] + [[ "$output" =~ "0x00112233445566778899AABBCCDDEEFF00112233" ]] || false + [[ "$output" =~ "0x" ]] || false # Empty binary shows as "0x" + + # Test CSV format also works + run dolt sql --binary-as-hex --result-format=csv -q "SELECT * FROM constraint_test WHERE pk = 1" + [ $status -eq 0 ] + [[ "$output" =~ "1,0x00112233445566778899AABBCCDDEEFF00112233" ]] || false +} + @test "sql: interactive binary-as-hex behavior with local database non-filtered selects" { skiponwindows "Missing dependencies" which expect > /dev/null || skip "expect not available" From debe8e876c1a0fb3175527a2bc80dbad575828df Mon Sep 17 00:00:00 2001 From: elianddb Date: Mon, 28 Jul 2025 22:59:55 +0000 Subject: [PATCH 08/17] switch to concrete type --- go/cmd/dolt/commands/engine/sql_print.go | 10 ++-------- go/libraries/doltcore/sqle/sqlutil/sql_row.go | 9 ++++----- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/go/cmd/dolt/commands/engine/sql_print.go b/go/cmd/dolt/commands/engine/sql_print.go index 24eaea0686f..90ad7fe5e5f 100644 --- a/go/cmd/dolt/commands/engine/sql_print.go +++ b/go/cmd/dolt/commands/engine/sql_print.go @@ -203,12 +203,6 @@ func printResultSetSummary(numRows int, numWarnings uint16, warningsList string, return nil } -// binaryAsHexDisplayValue wraps hex-formatted binary data to bypass validation -type binaryAsHexDisplayValue string - -func (h binaryAsHexDisplayValue) String() string { - return string(h) -} // binaryHexIterator wraps a row iterator and transforms binary data to hex format type binaryHexIterator struct { @@ -237,9 +231,9 @@ func (iter *binaryHexIterator) Next(ctx *sql.Context) (sql.Row, error) { switch iter.schema[i].Type.Type() { case sqltypes.Binary, sqltypes.VarBinary: if bytes, ok := val.([]byte); ok { - row[i] = binaryAsHexDisplayValue(fmt.Sprintf("0x%X", bytes)) + row[i] = sqlutil.BinaryAsHexDisplayValue(fmt.Sprintf("0x%X", bytes)) } else { - row[i] = binaryAsHexDisplayValue(fmt.Sprintf("0x%X", []byte(fmt.Sprint(val)))) + row[i] = sqlutil.BinaryAsHexDisplayValue(fmt.Sprintf("0x%X", []byte(fmt.Sprint(val)))) } } } diff --git a/go/libraries/doltcore/sqle/sqlutil/sql_row.go b/go/libraries/doltcore/sqle/sqlutil/sql_row.go index d1633263783..020e47cc446 100644 --- a/go/libraries/doltcore/sqle/sqlutil/sql_row.go +++ b/go/libraries/doltcore/sqle/sqlutil/sql_row.go @@ -223,16 +223,15 @@ func keylessDoltRowFromSqlRow(ctx context.Context, vrw types.ValueReadWriter, sq return row.KeylessRow(vrw.Format(), vals[:j]...) } -type binaryAsHexDisplayValue interface { - String() string -} +// BinaryAsHexDisplayValue is a concrete type for binary-as-hex display values +type BinaryAsHexDisplayValue string // SqlColToStr is a utility function for converting a sql column of type interface{} to a string. // NULL values are treated as empty strings. Handle nil separately if you require other behavior. func SqlColToStr(ctx *sql.Context, sqlType sql.Type, col interface{}) (string, error) { if col != nil { - if hexVal, ok := col.(binaryAsHexDisplayValue); ok { - return hexVal.String(), nil + if hexVal, ok := col.(BinaryAsHexDisplayValue); ok { + return string(hexVal), nil } switch typedCol := col.(type) { From cf1ec5124ab14ce0f913144d388d09c2d92e52ce Mon Sep 17 00:00:00 2001 From: elianddb Date: Mon, 28 Jul 2025 23:08:54 +0000 Subject: [PATCH 09/17] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/cmd/dolt/commands/engine/sql_print.go | 1 - go/libraries/doltcore/sqle/sqlutil/sql_row.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/go/cmd/dolt/commands/engine/sql_print.go b/go/cmd/dolt/commands/engine/sql_print.go index 90ad7fe5e5f..f3df679dfe5 100644 --- a/go/cmd/dolt/commands/engine/sql_print.go +++ b/go/cmd/dolt/commands/engine/sql_print.go @@ -203,7 +203,6 @@ func printResultSetSummary(numRows int, numWarnings uint16, warningsList string, return nil } - // binaryHexIterator wraps a row iterator and transforms binary data to hex format type binaryHexIterator struct { inner sql.RowIter diff --git a/go/libraries/doltcore/sqle/sqlutil/sql_row.go b/go/libraries/doltcore/sqle/sqlutil/sql_row.go index 020e47cc446..3cb72c0825d 100644 --- a/go/libraries/doltcore/sqle/sqlutil/sql_row.go +++ b/go/libraries/doltcore/sqle/sqlutil/sql_row.go @@ -233,7 +233,7 @@ func SqlColToStr(ctx *sql.Context, sqlType sql.Type, col interface{}) (string, e if hexVal, ok := col.(BinaryAsHexDisplayValue); ok { return string(hexVal), nil } - + switch typedCol := col.(type) { case bool: if typedCol { From 14b4292cda778c92dccc4f1d5c609a12b29c6b02 Mon Sep 17 00:00:00 2001 From: elianddb Date: Wed, 30 Jul 2025 11:16:11 -0700 Subject: [PATCH 10/17] add review compile-time interface check, err handling, and test cleanup --- go/cmd/dolt/commands/engine/sql_print.go | 19 ++- go/cmd/dolt/commands/sql.go | 8 +- go/libraries/doltcore/sqle/sqlutil/sql_row.go | 2 +- ...-as-hex-flag-precedence-interactive.expect | 69 ---------- .../binary-as-hex-local-db-interactive.expect | 38 ------ ...ry-as-hex-local-db-skip-interactive.expect | 39 ------ ...y-as-hex-server-default-interactive.expect | 63 --------- .../binary-as-hex-server-interactive.expect | 63 --------- ...binary-as-hex-skip-flag-interactive.expect | 69 ---------- .../bats/helper/common_expect_functions.tcl | 18 +++ integration-tests/bats/sql-server.bats | 120 ------------------ .../bats/sql-shell-binary-as-hex.expect | 102 +++++++++++++++ integration-tests/bats/sql-shell.bats | 66 ++++++++++ integration-tests/bats/sql.bats | 87 ------------- 14 files changed, 202 insertions(+), 561 deletions(-) delete mode 100644 integration-tests/bats/binary-as-hex-flag-precedence-interactive.expect delete mode 100755 integration-tests/bats/binary-as-hex-local-db-interactive.expect delete mode 100755 integration-tests/bats/binary-as-hex-local-db-skip-interactive.expect delete mode 100644 integration-tests/bats/binary-as-hex-server-default-interactive.expect delete mode 100755 integration-tests/bats/binary-as-hex-server-interactive.expect delete mode 100755 integration-tests/bats/binary-as-hex-skip-flag-interactive.expect create mode 100644 integration-tests/bats/sql-shell-binary-as-hex.expect diff --git a/go/cmd/dolt/commands/engine/sql_print.go b/go/cmd/dolt/commands/engine/sql_print.go index f3df679dfe5..d0bcfc5693a 100644 --- a/go/cmd/dolt/commands/engine/sql_print.go +++ b/go/cmd/dolt/commands/engine/sql_print.go @@ -209,6 +209,8 @@ type binaryHexIterator struct { schema sql.Schema } +var _ sql.RowIter = (*binaryHexIterator)(nil) + // newBinaryHexIterator creates a new iterator that transforms binary data to hex format func newBinaryHexIterator(inner sql.RowIter, schema sql.Schema) sql.RowIter { return &binaryHexIterator{ @@ -219,26 +221,29 @@ func newBinaryHexIterator(inner sql.RowIter, schema sql.Schema) sql.RowIter { // Next returns the next row with binary data transformed to hex format func (iter *binaryHexIterator) Next(ctx *sql.Context) (sql.Row, error) { - row, err := iter.inner.Next(ctx) + rowData, err := iter.inner.Next(ctx) if err != nil { return nil, err } // TODO: Add support for BLOB types (TINYBLOB, BLOB, MEDIUMBLOB, LONGBLOB) and BIT type - for i, val := range row { + for i, val := range rowData { if val != nil && i < len(iter.schema) { switch iter.schema[i].Type.Type() { case sqltypes.Binary, sqltypes.VarBinary: - if bytes, ok := val.([]byte); ok { - row[i] = sqlutil.BinaryAsHexDisplayValue(fmt.Sprintf("0x%X", bytes)) - } else { - row[i] = sqlutil.BinaryAsHexDisplayValue(fmt.Sprintf("0x%X", []byte(fmt.Sprint(val)))) + switch v := val.(type) { + case []byte: + rowData[i] = sqlutil.BinaryAsHexDisplayValue(fmt.Sprintf("0x%X", v)) + case string: // handles results from sql-server; mysql wire protocol returns strings + rowData[i] = sqlutil.BinaryAsHexDisplayValue(fmt.Sprintf("0x%X", []byte(v))) + default: + return nil, fmt.Errorf("unexpected type %T for binary column %s", val, iter.schema[i].Name) } } } } - return row, nil + return rowData, nil } // Close closes the wrapped iterator and releases any resources. diff --git a/go/cmd/dolt/commands/sql.go b/go/cmd/dolt/commands/sql.go index e0c4b4fdb2e..2f7e40fb899 100644 --- a/go/cmd/dolt/commands/sql.go +++ b/go/cmd/dolt/commands/sql.go @@ -219,11 +219,9 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE } // Determine binary-as-hex behavior from flags (default false for non-interactive modes) - binaryAsHex := false - if apr.Contains(skipBinaryAsHexFlag) { - binaryAsHex = false - } else if apr.Contains(binaryAsHexFlag) { - binaryAsHex = true + binaryAsHex := apr.Contains(binaryAsHexFlag) + if binaryAsHex && apr.Contains(skipBinaryAsHexFlag) { // We stray from MYSQL here to make usage clear for users + return HandleVErrAndExitCode(errhand.BuildDError("cannot use both --%s and --%s", binaryAsHexFlag, skipBinaryAsHexFlag).Build(), usage) } queryist, sqlCtx, closeFunc, err := cliCtx.QueryEngine(ctx) diff --git a/go/libraries/doltcore/sqle/sqlutil/sql_row.go b/go/libraries/doltcore/sqle/sqlutil/sql_row.go index 3cb72c0825d..44bee564c88 100644 --- a/go/libraries/doltcore/sqle/sqlutil/sql_row.go +++ b/go/libraries/doltcore/sqle/sqlutil/sql_row.go @@ -223,7 +223,7 @@ func keylessDoltRowFromSqlRow(ctx context.Context, vrw types.ValueReadWriter, sq return row.KeylessRow(vrw.Format(), vals[:j]...) } -// BinaryAsHexDisplayValue is a concrete type for binary-as-hex display values +// BinaryAsHexDisplayValue is a wrapper for binary values that should be displayed as hex strings. type BinaryAsHexDisplayValue string // SqlColToStr is a utility function for converting a sql column of type interface{} to a string. diff --git a/integration-tests/bats/binary-as-hex-flag-precedence-interactive.expect b/integration-tests/bats/binary-as-hex-flag-precedence-interactive.expect deleted file mode 100644 index e443aaee711..00000000000 --- a/integration-tests/bats/binary-as-hex-flag-precedence-interactive.expect +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/expect - -set port [lindex $argv 0] -set timeout 10 -log_user 0 - -spawn dolt --host 127.0.0.1 --port $port --no-tls sql --binary-as-hex --skip-binary-as-hex -expect ">" - -send "USE repo1;\r" -expect ">" - -send "SELECT * FROM binary_test;\r" -expect { - "abc" { - expect ">" - } - "0x616263" { - send "exit\r" - expect eof - exit 1 - } - "0x0A000000" { - send "exit\r" - expect eof - exit 1 - } - timeout { - send "exit\r" - expect eof - exit 1 - } -} - -send "SELECT pk, vb FROM binary_test WHERE pk = 2;\r" -expect { - "0x616263" { - send "exit\r" - expect eof - exit 1 - } - "abc" { - expect ">" - send "SELECT pk, vb FROM binary_test WHERE pk = 1;\r" - expect { - "0x0A000000" { - send "exit\r" - expect eof - exit 1 - } - -re "1.*\n" { - expect ">" - send "exit\r" - expect eof - exit 0 - } - timeout { - send "exit\r" - expect eof - exit 1 - } - } - } - timeout { - send "exit\r" - expect eof - exit 1 - } -} \ No newline at end of file diff --git a/integration-tests/bats/binary-as-hex-local-db-interactive.expect b/integration-tests/bats/binary-as-hex-local-db-interactive.expect deleted file mode 100755 index ed89d663bac..00000000000 --- a/integration-tests/bats/binary-as-hex-local-db-interactive.expect +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/expect - -set timeout 10 -log_user 0 - -# Test with --binary-as-hex flag -spawn dolt sql --binary-as-hex -expect ">" - -send "CREATE TABLE test_binary (pk INT PRIMARY KEY, vb VARBINARY(20));\r" -expect ">" - -send "INSERT INTO test_binary VALUES (1, 0x0A000000), (2, 'abc');\r" -expect ">" - -send "SELECT * FROM test_binary;\r" -expect { - "| 1 | 0x0A000000 |" { - expect { - "| 2 | 0x616263 |" { - expect ">" - send "exit\r" - expect eof - exit 0 - } - timeout { - send "exit\r" - expect eof - exit 1 - } - } - } - timeout { - send "exit\r" - expect eof - exit 1 - } -} \ No newline at end of file diff --git a/integration-tests/bats/binary-as-hex-local-db-skip-interactive.expect b/integration-tests/bats/binary-as-hex-local-db-skip-interactive.expect deleted file mode 100755 index febd4204a56..00000000000 --- a/integration-tests/bats/binary-as-hex-local-db-skip-interactive.expect +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/expect - -set timeout 10 -log_user 0 - -# Test with --skip-binary-as-hex flag -spawn dolt sql --skip-binary-as-hex -expect ">" - -send "CREATE TABLE test_binary (pk INT PRIMARY KEY, vb VARBINARY(20));\r" -expect ">" - -send "INSERT INTO test_binary VALUES (1, 0x0A000000), (2, 'abc');\r" -expect ">" - -send "SELECT * FROM test_binary;\r" -expect { - -re {\\| 2.*\\| abc \\|} { - expect ">" - send "exit\r" - expect eof - exit 0 - } - "0x616263" { - send "exit\r" - expect eof - exit 1 - } - "0x0A000000" { - send "exit\r" - expect eof - exit 1 - } - timeout { - send "exit\r" - expect eof - exit 1 - } -} \ No newline at end of file diff --git a/integration-tests/bats/binary-as-hex-server-default-interactive.expect b/integration-tests/bats/binary-as-hex-server-default-interactive.expect deleted file mode 100644 index a7aaada8048..00000000000 --- a/integration-tests/bats/binary-as-hex-server-default-interactive.expect +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/expect - -set port [lindex $argv 0] -set timeout 10 -log_user 0 - -spawn dolt --host 127.0.0.1 --port $port --no-tls sql -expect ">" - -send "USE repo1;\r" -expect ">" - -send "SELECT * FROM binary_test;\r" -expect { - "0x0A000000" { - expect { - "0x616263" { - expect ">" - } - timeout { - send "exit\r" - expect eof - exit 1 - } - } - } - timeout { - send "exit\r" - expect eof - exit 1 - } -} - -send "SELECT pk, vb FROM binary_test WHERE pk = 1;\r" -expect { - "0x0A000000" { - expect ">" - send "SELECT pk, vb FROM binary_test WHERE pk = 2;\r" - expect { - "0x616263" { - expect ">" - send "exit\r" - expect eof - exit 0 - } - "abc" { - send "exit\r" - expect eof - exit 1 - } - timeout { - send "exit\r" - expect eof - exit 1 - } - } - } - timeout { - send "exit\r" - expect eof - exit 1 - } -} \ No newline at end of file diff --git a/integration-tests/bats/binary-as-hex-server-interactive.expect b/integration-tests/bats/binary-as-hex-server-interactive.expect deleted file mode 100755 index c3ca8721c1e..00000000000 --- a/integration-tests/bats/binary-as-hex-server-interactive.expect +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/expect - -set port [lindex $argv 0] -set timeout 10 -log_user 0 - -spawn dolt --host 127.0.0.1 --port $port --no-tls sql --binary-as-hex -expect ">" - -send "USE repo1;\r" -expect ">" - -send "SELECT * FROM binary_test;\r" -expect { - "0x0A000000" { - expect { - "0x616263" { - expect ">" - } - timeout { - send "exit\r" - expect eof - exit 1 - } - } - } - timeout { - send "exit\r" - expect eof - exit 1 - } -} - -send "SELECT pk, vb FROM binary_test WHERE pk = 2;\r" -expect { - "0x616263" { - expect ">" - send "SELECT pk, vb FROM binary_test WHERE pk = 1;\r" - expect { - "0x0A000000" { - expect ">" - send "exit\r" - expect eof - exit 0 - } - timeout { - send "exit\r" - expect eof - exit 1 - } - } - } - "abc" { - send "exit\r" - expect eof - exit 1 - } - timeout { - send "exit\r" - expect eof - exit 1 - } -} \ No newline at end of file diff --git a/integration-tests/bats/binary-as-hex-skip-flag-interactive.expect b/integration-tests/bats/binary-as-hex-skip-flag-interactive.expect deleted file mode 100755 index 627968733ac..00000000000 --- a/integration-tests/bats/binary-as-hex-skip-flag-interactive.expect +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/expect - -set port [lindex $argv 0] -set timeout 10 -log_user 0 - -spawn dolt --host 127.0.0.1 --port $port --no-tls sql --skip-binary-as-hex -expect ">" - -send "USE repo1;\r" -expect ">" - -send "SELECT * FROM binary_test;\r" -expect { - "abc" { - expect ">" - } - "0x616263" { - send "exit\r" - expect eof - exit 1 - } - "0x0A000000" { - send "exit\r" - expect eof - exit 1 - } - timeout { - send "exit\r" - expect eof - exit 1 - } -} - -send "SELECT pk, vb FROM binary_test WHERE pk = 2;\r" -expect { - "0x616263" { - send "exit\r" - expect eof - exit 1 - } - "abc" { - expect ">" - send "SELECT pk, vb FROM binary_test WHERE pk = 1;\r" - expect { - "0x0A000000" { - send "exit\r" - expect eof - exit 1 - } - -re "1.*\n" { - expect ">" - send "exit\r" - expect eof - exit 0 - } - timeout { - send "exit\r" - expect eof - exit 1 - } - } - } - timeout { - send "exit\r" - expect eof - exit 1 - } -} \ No newline at end of file diff --git a/integration-tests/bats/helper/common_expect_functions.tcl b/integration-tests/bats/helper/common_expect_functions.tcl index 6c91d5574d9..592e3d9bd94 100755 --- a/integration-tests/bats/helper/common_expect_functions.tcl +++ b/integration-tests/bats/helper/common_expect_functions.tcl @@ -60,3 +60,21 @@ proc expect_with_defaults_2 {patternA patternB action} { } } +proc expect_without_pattern {bad_pattern action} { + expect { + -re $bad_pattern { + puts "ERROR: Found unexpected pattern: $bad_pattern" + exit 1 + } + timeout { + eval $action + } + eof { + eval $action + } + failed { + puts "<>" + exit 1 + } + } +} diff --git a/integration-tests/bats/sql-server.bats b/integration-tests/bats/sql-server.bats index f7a7cf26aed..7f733daf1d4 100644 --- a/integration-tests/bats/sql-server.bats +++ b/integration-tests/bats/sql-server.bats @@ -2197,123 +2197,3 @@ EOF run grep -F "Dropping persisted '__dolt_local_user__@localhost' because this account name is reserved for Dolt" server_log.txt [ $status -eq 0 ] } - - -@test "sql-server: client binary-as-hex behavior works with server connections" { - skiponwindows "Missing dependencies" - which expect > /dev/null || skip "expect not available" - - cd repo1 - - # Setup: Create table with binary data and commit it - dolt sql -q "DROP TABLE IF EXISTS binary_test; CREATE TABLE binary_test (pk INT PRIMARY KEY, vb VARBINARY(20)); INSERT INTO binary_test VALUES (1, 0x0A000000), (2, 'abc');" - dolt add . - dolt commit -m "Add binary test data" - - start_sql_server repo1 - - # 1. Test default interactive behavior (should show hex by default) - run expect "$BATS_TEST_DIRNAME/binary-as-hex-server-default-interactive.expect" $PORT - [ $status -eq 0 ] - - # 2. Test --binary-as-hex flag in interactive mode (should show hex) - run expect "$BATS_TEST_DIRNAME/binary-as-hex-server-interactive.expect" $PORT - [ $status -eq 0 ] - - # 3. Test --skip-binary-as-hex flag in interactive mode (should show raw) - run expect "$BATS_TEST_DIRNAME/binary-as-hex-skip-flag-interactive.expect" $PORT - [ $status -eq 0 ] - - # 4. Test flag precedence: --skip-binary-as-hex overrides --binary-as-hex - run expect "$BATS_TEST_DIRNAME/binary-as-hex-flag-precedence-interactive.expect" $PORT - [ $status -eq 0 ] - - # 5. Test non-interactive server behavior with -q flag (should show raw by default) - run dolt --host 127.0.0.1 --port $PORT --no-tls sql -q "USE repo1; SELECT vb FROM binary_test WHERE pk = 1" - [ $status -eq 0 ] - ! [[ "$output" =~ "0x0A000000" ]] || false - - # 6. Test non-interactive server behavior with --binary-as-hex flag - run dolt --host 127.0.0.1 --port $PORT --no-tls sql --binary-as-hex -q "USE repo1; SELECT vb FROM binary_test WHERE pk = 1" - [ $status -eq 0 ] - [[ "$output" =~ "0x0A000000" ]] || false - - # 7. Test non-interactive server behavior with printable data - run dolt --host 127.0.0.1 --port $PORT --no-tls sql -q "USE repo1; SELECT vb FROM binary_test WHERE pk = 2" - [ $status -eq 0 ] - [[ "$output" =~ "abc" ]] || false - - # 8. Test non-interactive server flag precedence - run dolt --host 127.0.0.1 --port $PORT --no-tls sql --binary-as-hex --skip-binary-as-hex -q "USE repo1; SELECT vb FROM binary_test WHERE pk = 2" - [ $status -eq 0 ] - [[ "$output" =~ "abc" ]] || false - ! [[ "$output" =~ "0x616263" ]] || false - - # 9. Test non-interactive server behavior with -q flag (should show raw by default) - run dolt --host 127.0.0.1 --port $PORT --no-tls sql -q "USE repo1; SELECT vb FROM binary_test WHERE pk = 1" - [ $status -eq 0 ] - ! [[ "$output" =~ "0x0A000000" ]] || false - - # 10. Test non-interactive server behavior with -q and --binary-as-hex flags - run dolt --host 127.0.0.1 --port $PORT --no-tls sql --binary-as-hex -q "USE repo1; SELECT vb FROM binary_test WHERE pk = 1" - [ $status -eq 0 ] - [[ "$output" =~ "0x0A000000" ]] || false - - # 11. Test non-interactive server -q flag precedence - run dolt --host 127.0.0.1 --port $PORT --no-tls sql --binary-as-hex --skip-binary-as-hex -q "USE repo1; SELECT vb FROM binary_test WHERE pk = 2" - [ $status -eq 0 ] - [[ "$output" =~ "abc" ]] || false - ! [[ "$output" =~ "0x616263" ]] || false - - # 12. Test non-filtered SELECT with default behavior (should show raw by default) - run dolt --host 127.0.0.1 --port $PORT --no-tls sql -q "USE repo1; SELECT * FROM binary_test" - [ $status -eq 0 ] - ! [[ "$output" =~ "0x0A000000" ]] || false - ! [[ "$output" =~ "0x616263" ]] || false - [[ "$output" =~ "abc" ]] || false - - # 13. Test non-filtered SELECT with --binary-as-hex flag - run dolt --host 127.0.0.1 --port $PORT --no-tls sql --binary-as-hex -q "USE repo1; SELECT * FROM binary_test" - [ $status -eq 0 ] - [[ "$output" =~ "0x0A000000" ]] || false - [[ "$output" =~ "0x616263" ]] || false - - # 14. Test non-filtered SELECT with --skip-binary-as-hex flag - run dolt --host 127.0.0.1 --port $PORT --no-tls sql --skip-binary-as-hex -q "USE repo1; SELECT * FROM binary_test" - [ $status -eq 0 ] - ! [[ "$output" =~ "0x0A000000" ]] || false - ! [[ "$output" =~ "0x616263" ]] || false - [[ "$output" =~ "abc" ]] || false - - stop_sql_server 1 -} - -@test "sql-server: binary-as-hex works with data at column constraint limits" { - skiponwindows "Missing dependencies" - - cd repo1 - - # Setup: Create table with binary data at column size limits - dolt sql -q "CREATE TABLE constraint_test (pk INT PRIMARY KEY, data VARBINARY(20));" - dolt sql -q "INSERT INTO constraint_test VALUES (1, UNHEX('00112233445566778899AABBCCDDEEFF00112233'));" - dolt sql -q "INSERT INTO constraint_test VALUES (2, X'');" - dolt add . - dolt commit -m "Add constraint test data" - - start_sql_server repo1 - - # Test server connection with binary-as-hex at column constraint limits - run dolt --host 127.0.0.1 --port $PORT --no-tls sql --binary-as-hex -q "USE repo1; SELECT * FROM constraint_test ORDER BY pk" - [ $status -eq 0 ] - [[ "$output" =~ "0x00112233445566778899AABBCCDDEEFF00112233" ]] || false - [[ "$output" =~ "0x" ]] || false # Empty binary shows as "0x" - - # Test CSV format with server connection - run dolt --host 127.0.0.1 --port $PORT --no-tls sql --binary-as-hex --result-format=csv -q "USE repo1; SELECT * FROM constraint_test WHERE pk = 1" - [ $status -eq 0 ] - [[ "$output" =~ "1,0x00112233445566778899AABBCCDDEEFF00112233" ]] || false - - stop_sql_server -} - - diff --git a/integration-tests/bats/sql-shell-binary-as-hex.expect b/integration-tests/bats/sql-shell-binary-as-hex.expect new file mode 100644 index 00000000000..bd8d092b390 --- /dev/null +++ b/integration-tests/bats/sql-shell-binary-as-hex.expect @@ -0,0 +1,102 @@ +#!/usr/bin/expect +# dolthub/dolt#9554 +# Test script for binary-a-hex flag behavior in dolt sql. +# +# Usage: +# expect binary-hex-test.expect [flags...] +# +# Tracked flags: +# --binary-as-hex: Use binary as hex encoding for VARBINARY and BINARY types. +# --skip-binary-as-hex: Skip binary as hex encoding for VARBINARY and BINARY types. + +source "$env(BATS_CWD)/helper/common_expect_functions.tcl" + +set timeout 10 +set env(NO_COLOR) 1 + +set has_binary_hex 0 +set has_skip_hex 0 + +foreach arg $argv { + if {$arg eq "--binary-as-hex"} {set has_binary_hex 1} + if {$arg eq "--skip-binary-as-hex"} {set has_skip_hex 1} +} + +# In the interactive shell, the default behavior is to use binary as hex output. +if {!$has_skip_hex} { + set has_binary_hex 1 +} + +proc run_query {query expect_proc} { + global has_skip_hex has_binary_hex argv + expect_with_defaults {>} "send {$query\r}" + eval $expect_proc +} + +# Handles the following cases: +# 1. check dolt's ability to detect conflicting flags. +# 2. spawns interactive shell +spawn dolt sql {*}$argv + +if {$has_binary_hex && $has_skip_hex} { + expect { + "cannot use both --binary-as-hex and --skip-binary-as-hex" { + expect eof + exit 3 # differentiate exit err from common_expect_functions.tcl + } + eof { + puts "Process ended without error message." + exit 1 + } + } +} + +run_query "DROP TABLE IF EXISTS test_vbin;" {} +run_query "CREATE TABLE test_vbin (id INT PRIMARY KEY, v VARBINARY(10));" {} +run_query "INSERT INTO test_vbin VALUES (1, 'abc');" {} +run_query "SELECT *, LENGTH(v) FROM test_vbin;" { + if {$has_skip_hex} { + expect_without_pattern {0x[0-9A-F]+} {} + } else { + expect_with_defaults_2 {0x616263} {\| 3 } {} + } +} + +run_query "INSERT INTO test_vbin VALUES (2, UNHEX('0A000000001000112233'));" {} +run_query "INSERT INTO test_vbin VALUES (3, UNHEX(''));" {} +run_query "SELECT *, LENGTH(v) FROM test_vbin;" { + if {$has_skip_hex} { + expect_without_pattern {0x[0-9A-F]+} {} + } else { + expect_with_defaults_2 {0x616263} {\| 3 } {} + expect_with_defaults_2 {0x0A000000001000112233} {\| 10 } {} + expect_with_defaults_2 {0x} {\| 0 } {} + } +} + +run_query "DROP TABLE IF EXISTS test_bin;" {} +run_query "CREATE TABLE test_bin (id INT PRIMARY KEY, b BINARY(10));" {} +run_query "INSERT INTO test_bin VALUES (1, 'abc');" {} +run_query "SELECT *, LENGTH(b) FROM test_bin;" { + if {$has_skip_hex} { + expect_without_pattern {0x[0-9A-F]+} {} + } else { + expect_with_defaults_2 {0x61626300000000000000} {\| 10 } {} + } +} + +run_query "INSERT INTO test_bin VALUES (2, UNHEX('0A000000001000112233'));" {} +run_query "INSERT INTO test_bin VALUES (3, UNHEX(''));" {} +run_query "SELECT *, LENGTH(b) FROM test_bin;" { + if {$has_skip_hex} { + expect_without_pattern {0x[0-9A-F]+} {} + } else { + expect_with_defaults_2 {0x61626300000000000000} {\| 10 } {} + expect_with_defaults_2 {0x0A000000001000112233} {\| 10 } {} + expect_with_defaults_2 {0x00000000000000000000} {\| 10 } {} + } +} + +expect_with_defaults {>} { send "exit\r" } +expect eof +exit 0 diff --git a/integration-tests/bats/sql-shell.bats b/integration-tests/bats/sql-shell.bats index 4f51eb43c24..b629785e568 100644 --- a/integration-tests/bats/sql-shell.bats +++ b/integration-tests/bats/sql-shell.bats @@ -1069,4 +1069,70 @@ expect eof run $BATS_TEST_DIRNAME/sql-shell-commit-time.expect [ "$status" -eq 0 ] +} + +@test "sql-shell: -binary-as-hex, -skip-binary-as-hex flag is respected in server and local contexts" { +# skiponwindows "Missing Dependencies" + which expect > /dev/null || skip "expect is not installed" + + # Default behavior for interactive runs is to output binary as hex + run expect "$BATS_TEST_DIRNAME"/sql-shell-binary-as-hex.expect + [ "$status" -eq 0 ] + + run expect "$BATS_TEST_DIRNAME"/sql-shell-binary-as-hex.expect --binary-as-hex + [ "$status" -eq 0 ] + + run expect "$BATS_TEST_DIRNAME"/sql-shell-binary-as-hex.expect --skip-binary-as-hex + [ "$status" -eq 0 ] + + run expect "$BATS_TEST_DIRNAME"/sql-shell-binary-as-hex.expect --binary-as-hex --skip-binary-as-hex + [ "$status" -eq 3 ] + + # Non-interactive runs should not output binary as hex by default + run dolt sql -q "SELECT * FROM test_vbin" + [ "$status" -eq 0 ] + [[ ! $output =~ 0x[0-9A-F]+ ]] + + run dolt sql -q "SELECT * FROM test_bin" + [ "$status" -eq 0 ] + [[ ! $output =~ 0x[0-9A-F]+ ]] + + run dolt sql -q "SELECT * FROM test_vbin" --binary-as-hex + [ "$status" -eq 0 ] + [[ $output =~ 1.*0x616263 ]] || false + [[ $output =~ 2.*0x0A000000001000112233 ]] || false + [[ $output =~ 3.*0x ]] || false + + run dolt sql -q "SELECT * FROM test_bin" --binary-as-hex + [ $status -eq 0 ] + [[ $output =~ 1.*0x61626300000000000000 ]] || false + [[ $output =~ 2.*0x0A000000001000112233 ]] || false + [[ $output =~ 3.*0x00000000000000000000 ]] || false + + run dolt sql -q "SELECT * FROM test_vbin" --skip-binary-as-hex + [ "$status" -eq 0 ] + [[ ! $output =~ 0x[0-9A-F]+ ]] + + run dolt sql -q "SELECT * FROM test_bin" --skip-binary-as-hex + [ "$status" -eq 0 ] + [[ ! $output =~ 0x[0-9A-F]+ ]] + + run dolt sql -q "" --binary-as-hex --skip-binary-as-hex + [ "$status" -eq 1 ] + [[ "$output" =~ "cannot use both --binary-as-hex and --skip-binary-as-hex" ]] || false + + # Check other formats output is correct + run dolt sql -r csv -q "SELECT * FROM test_vbin WHERE id = 1;" --binary-as-hex + [ "$status" -eq 0 ] + [[ "$output" =~ "1,0x616263" ]] || false + + run dolt sql -r csv -q "SELECT * FROM test_vbin WHERE id = 1;" + [ "$status" -eq 0 ] + [[ "$output" =~ "1,abc" ]] || false + + run dolt sql -r csv -q "SELECT * FROM test_bin" --binary-as-hex + [ "$status" -eq 0 ] + [[ "$output" =~ "1,0x61626300000000000000" ]] || false + [[ "$output" =~ "2,0x0A000000001000112233" ]] || false + [[ "$output" =~ "3,0x00000000000000000000" ]] || false } \ No newline at end of file diff --git a/integration-tests/bats/sql.bats b/integration-tests/bats/sql.bats index e78836d304d..f6c60bf8819 100755 --- a/integration-tests/bats/sql.bats +++ b/integration-tests/bats/sql.bats @@ -2913,90 +2913,3 @@ SQL dolt sql < $BATS_TEST_DIRNAME/helper/with_utf16be_bom.sql dolt table rm t1 } - -@test "sql: client binary-as-hex behavior works with local database" { - # Setup: Create table with binary data - dolt sql -q "CREATE TABLE binary_test (pk INT PRIMARY KEY, vb VARBINARY(20)); INSERT INTO binary_test VALUES (1, 0x0A000000), (2, 'abc');" - - # Client non-interactive mode (using -q flag) should show raw bytes by default - run dolt sql -q "SELECT vb FROM binary_test WHERE pk = 1" - [ $status -eq 0 ] - # Should show raw bytes (invisible chars), not hex format - ! [[ "$output" =~ "0x0A000000" ]] || false - - # Printable data should show as text - run dolt sql -q "SELECT vb FROM binary_test WHERE pk = 2" - [ $status -eq 0 ] - [[ "$output" =~ "abc" ]] || false - - # HEX() function should work consistently - run dolt sql -q "SELECT HEX(vb) FROM binary_test WHERE pk = 1" - [ $status -eq 0 ] - [[ "$output" =~ "0A000000" ]] || false - - # Test --binary-as-hex flag forces hex display in non-interactive mode - run dolt sql --binary-as-hex -q "SELECT vb FROM binary_test WHERE pk = 1" - [ $status -eq 0 ] - [[ "$output" =~ "0x0A000000" ]] || false - - # Test --binary-as-hex with printable data - run dolt sql --binary-as-hex -q "SELECT vb FROM binary_test WHERE pk = 2" - [ $status -eq 0 ] - [[ "$output" =~ "0x616263" ]] || false - - # Test --skip-binary-as-hex flag forces raw display - run dolt sql --skip-binary-as-hex -q "SELECT vb FROM binary_test WHERE pk = 2" - [ $status -eq 0 ] - [[ "$output" =~ "abc" ]] || false - ! [[ "$output" =~ "0x616263" ]] || false - - # Test flag priority: --skip-binary-as-hex overrides --binary-as-hex - run dolt sql --binary-as-hex --skip-binary-as-hex -q "SELECT vb FROM binary_test WHERE pk = 2" - [ $status -eq 0 ] - [[ "$output" =~ "abc" ]] || false - ! [[ "$output" =~ "0x616263" ]] || false - - # Cleanup - dolt sql -q "DROP TABLE binary_test" -} - -@test "sql: binary-as-hex works with data at column constraint limits" { - # Test case for issue #9554: binary data that approaches or reaches column size limits - # should display correctly as hex without validation errors - - # Create table with VARBINARY(20) and insert 20 bytes of data (maximum for column) - dolt sql -q "CREATE TABLE constraint_test (pk INT PRIMARY KEY, data VARBINARY(20));" - dolt sql -q "INSERT INTO constraint_test VALUES (1, UNHEX('00112233445566778899AABBCCDDEEFF00112233'));" - dolt sql -q "INSERT INTO constraint_test VALUES (2, X'');" # Empty binary data - - # Verify the data length is exactly 20 bytes - run dolt sql -q "SELECT LENGTH(data) FROM constraint_test WHERE pk = 1" - [ $status -eq 0 ] - [[ "$output" =~ "20" ]] || false - - # Test binary-as-hex display works without validation errors - # The 20-byte data becomes 42-character hex string "0x00112233445566778899AABBCCDDEEFF00112233" - run dolt sql --binary-as-hex -q "SELECT * FROM constraint_test ORDER BY pk" - [ $status -eq 0 ] - [[ "$output" =~ "0x00112233445566778899AABBCCDDEEFF00112233" ]] || false - [[ "$output" =~ "0x" ]] || false # Empty binary shows as "0x" - - # Test CSV format also works - run dolt sql --binary-as-hex --result-format=csv -q "SELECT * FROM constraint_test WHERE pk = 1" - [ $status -eq 0 ] - [[ "$output" =~ "1,0x00112233445566778899AABBCCDDEEFF00112233" ]] || false -} - -@test "sql: interactive binary-as-hex behavior with local database non-filtered selects" { - skiponwindows "Missing dependencies" - which expect > /dev/null || skip "expect not available" - - # Test interactive shell with --binary-as-hex flag (includes non-filtered SELECT *) - run expect "$BATS_TEST_DIRNAME/binary-as-hex-local-db-interactive.expect" - [ $status -eq 0 ] - - # Test interactive shell with --skip-binary-as-hex flag (includes non-filtered SELECT *) - run expect "$BATS_TEST_DIRNAME/binary-as-hex-local-db-skip-interactive.expect" - [ $status -eq 0 ] -} - From ae9f979d79dffcb6df1603a02ddec54f29569344 Mon Sep 17 00:00:00 2001 From: Elian Date: Wed, 30 Jul 2025 11:19:24 -0700 Subject: [PATCH 11/17] Update sql.go rm extra comments --- go/cmd/dolt/commands/sql.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/cmd/dolt/commands/sql.go b/go/cmd/dolt/commands/sql.go index 2f7e40fb899..6949e2c3f55 100644 --- a/go/cmd/dolt/commands/sql.go +++ b/go/cmd/dolt/commands/sql.go @@ -463,7 +463,6 @@ func execSingleQuery( } if rowIter != nil { - // Apply binary-as-hex formatting if enabled err = engine.PrettyPrintResults(sqlCtx, format, sqlSch, rowIter, false, false, false, binaryAsHex) if err != nil { return errhand.VerboseErrorFromError(err) From 05068bab6c736191132cab2695a29bc3fec67a6f Mon Sep 17 00:00:00 2001 From: elianddb Date: Wed, 30 Jul 2025 11:29:32 -0700 Subject: [PATCH 12/17] add miss dependencies for win and comments --- go/cmd/dolt/commands/engine/sql_print.go | 4 ++-- integration-tests/bats/sql-shell-binary-as-hex.expect | 3 +-- integration-tests/bats/sql-shell.bats | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/go/cmd/dolt/commands/engine/sql_print.go b/go/cmd/dolt/commands/engine/sql_print.go index d0bcfc5693a..4536792649c 100644 --- a/go/cmd/dolt/commands/engine/sql_print.go +++ b/go/cmd/dolt/commands/engine/sql_print.go @@ -232,9 +232,9 @@ func (iter *binaryHexIterator) Next(ctx *sql.Context) (sql.Row, error) { switch iter.schema[i].Type.Type() { case sqltypes.Binary, sqltypes.VarBinary: switch v := val.(type) { - case []byte: + case []byte: // hex fmt is explicitly upper case rowData[i] = sqlutil.BinaryAsHexDisplayValue(fmt.Sprintf("0x%X", v)) - case string: // handles results from sql-server; mysql wire protocol returns strings + case string: // handles results from sql-server; MySQL wire protocol returns strings rowData[i] = sqlutil.BinaryAsHexDisplayValue(fmt.Sprintf("0x%X", []byte(v))) default: return nil, fmt.Errorf("unexpected type %T for binary column %s", val, iter.schema[i].Name) diff --git a/integration-tests/bats/sql-shell-binary-as-hex.expect b/integration-tests/bats/sql-shell-binary-as-hex.expect index bd8d092b390..8bafd18c6d9 100644 --- a/integration-tests/bats/sql-shell-binary-as-hex.expect +++ b/integration-tests/bats/sql-shell-binary-as-hex.expect @@ -97,6 +97,5 @@ run_query "SELECT *, LENGTH(b) FROM test_bin;" { } } -expect_with_defaults {>} { send "exit\r" } -expect eof +run_query "exit;" { expect eof } exit 0 diff --git a/integration-tests/bats/sql-shell.bats b/integration-tests/bats/sql-shell.bats index b629785e568..4ae5cc103f1 100644 --- a/integration-tests/bats/sql-shell.bats +++ b/integration-tests/bats/sql-shell.bats @@ -1072,7 +1072,7 @@ expect eof } @test "sql-shell: -binary-as-hex, -skip-binary-as-hex flag is respected in server and local contexts" { -# skiponwindows "Missing Dependencies" + skiponwindows "Missing Dependencies" which expect > /dev/null || skip "expect is not installed" # Default behavior for interactive runs is to output binary as hex From c1719f12bc626b1868ea525eed3fe31a3ad4bb62 Mon Sep 17 00:00:00 2001 From: elianddb Date: Wed, 30 Jul 2025 12:07:36 -0700 Subject: [PATCH 13/17] rm expect install check in case of malformed installations --- integration-tests/bats/sql-shell.bats | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-tests/bats/sql-shell.bats b/integration-tests/bats/sql-shell.bats index 4ae5cc103f1..ae288ff1f94 100644 --- a/integration-tests/bats/sql-shell.bats +++ b/integration-tests/bats/sql-shell.bats @@ -1073,7 +1073,6 @@ expect eof @test "sql-shell: -binary-as-hex, -skip-binary-as-hex flag is respected in server and local contexts" { skiponwindows "Missing Dependencies" - which expect > /dev/null || skip "expect is not installed" # Default behavior for interactive runs is to output binary as hex run expect "$BATS_TEST_DIRNAME"/sql-shell-binary-as-hex.expect From 5157153557d916210c8badb9fd09591b7396ea69 Mon Sep 17 00:00:00 2001 From: elianddb Date: Wed, 30 Jul 2025 12:12:16 -0700 Subject: [PATCH 14/17] add direct link to issue --- integration-tests/bats/sql-shell-binary-as-hex.expect | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-tests/bats/sql-shell-binary-as-hex.expect b/integration-tests/bats/sql-shell-binary-as-hex.expect index 8bafd18c6d9..81a97b5c516 100644 --- a/integration-tests/bats/sql-shell-binary-as-hex.expect +++ b/integration-tests/bats/sql-shell-binary-as-hex.expect @@ -1,5 +1,6 @@ #!/usr/bin/expect # dolthub/dolt#9554 +# https://github.com/dolthub/dolt/issues/9554 # Test script for binary-a-hex flag behavior in dolt sql. # # Usage: From 65e15a89f2f636837de618a010de77ecf48ffba7 Mon Sep 17 00:00:00 2001 From: elianddb Date: Wed, 30 Jul 2025 12:46:21 -0700 Subject: [PATCH 15/17] add miss || false --- integration-tests/bats/sql-shell.bats | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration-tests/bats/sql-shell.bats b/integration-tests/bats/sql-shell.bats index ae288ff1f94..eb6c61708dc 100644 --- a/integration-tests/bats/sql-shell.bats +++ b/integration-tests/bats/sql-shell.bats @@ -1090,11 +1090,11 @@ expect eof # Non-interactive runs should not output binary as hex by default run dolt sql -q "SELECT * FROM test_vbin" [ "$status" -eq 0 ] - [[ ! $output =~ 0x[0-9A-F]+ ]] + [[ ! $output =~ 0x[0-9A-F]+ ]] || false run dolt sql -q "SELECT * FROM test_bin" [ "$status" -eq 0 ] - [[ ! $output =~ 0x[0-9A-F]+ ]] + [[ ! $output =~ 0x[0-9A-F]+ ]] || false run dolt sql -q "SELECT * FROM test_vbin" --binary-as-hex [ "$status" -eq 0 ] @@ -1110,11 +1110,11 @@ expect eof run dolt sql -q "SELECT * FROM test_vbin" --skip-binary-as-hex [ "$status" -eq 0 ] - [[ ! $output =~ 0x[0-9A-F]+ ]] + [[ ! $output =~ 0x[0-9A-F]+ ]] || false run dolt sql -q "SELECT * FROM test_bin" --skip-binary-as-hex [ "$status" -eq 0 ] - [[ ! $output =~ 0x[0-9A-F]+ ]] + [[ ! $output =~ 0x[0-9A-F]+ ]] || false run dolt sql -q "" --binary-as-hex --skip-binary-as-hex [ "$status" -eq 1 ] From 0ae19fd06f174db4954e34f4c786a475319ddc02 Mon Sep 17 00:00:00 2001 From: elianddb Date: Wed, 30 Jul 2025 12:53:58 -0700 Subject: [PATCH 16/17] add no_lambda tag --- integration-tests/bats/sql-shell.bats | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-tests/bats/sql-shell.bats b/integration-tests/bats/sql-shell.bats index eb6c61708dc..e14534c3816 100644 --- a/integration-tests/bats/sql-shell.bats +++ b/integration-tests/bats/sql-shell.bats @@ -1071,6 +1071,7 @@ expect eof [ "$status" -eq 0 ] } +# bats test_tags=no_lambda @test "sql-shell: -binary-as-hex, -skip-binary-as-hex flag is respected in server and local contexts" { skiponwindows "Missing Dependencies" From 04d13dcadbe694a1a2f00ca2bf80b23cf2019452 Mon Sep 17 00:00:00 2001 From: elianddb Date: Wed, 30 Jul 2025 20:02:16 +0000 Subject: [PATCH 17/17] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/go.sum | 4 ---- 1 file changed, 4 deletions(-) diff --git a/go/go.sum b/go/go.sum index 220e5cc9da6..512fa8a31e3 100644 --- a/go/go.sum +++ b/go/go.sum @@ -240,8 +240,6 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= github.com/dolthub/go-icu-regex v0.0.0-20250327004329-6799764f2dad h1:66ZPawHszNu37VPQckdhX1BPPVzREsGgNxQeefnlm3g= github.com/dolthub/go-icu-regex v0.0.0-20250327004329-6799764f2dad/go.mod h1:ylU4XjUpsMcvl/BKeRRMXSH7e7WBrPXdSLvnRJYrxEA= -github.com/dolthub/go-mysql-server v0.20.1-0.20250729231836-5203d53bc3c8 h1:i4Z3VqPFWy8CNxBW4vEIe9jX/XqEqsy6cLVuuV8reVg= -github.com/dolthub/go-mysql-server v0.20.1-0.20250729231836-5203d53bc3c8/go.mod h1:mqpOPgp1LPDxhE/tZlL1OOiYW12Xd6e89OWm4eN7854= github.com/dolthub/go-mysql-server v0.20.1-0.20250730053026-797f25fcfc4f h1:Cc6mAobw+9VXf5+47ubp1z3YLilGK0k10u8cI0tSfwI= github.com/dolthub/go-mysql-server v0.20.1-0.20250730053026-797f25fcfc4f/go.mod h1:/11huCxrsYYVHJFrUC+RzHtrTig0kJz6lzSTc0xqTIQ= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI= @@ -252,8 +250,6 @@ github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 h1:bMGS25NWAGTE github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71/go.mod h1:2/2zjLQ/JOOSbbSboojeg+cAwcRV0fDLzIiWch/lhqI= github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9XGFa6q5Ap4Z/OhNkAMBaK5YeuEzwJt+NZdhiE= github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY= -github.com/dolthub/vitess v0.0.0-20250729225143-5ab74d1f0182 h1:LrUmlwHlQLSu8OIL60APDGeUoPBoezeo8fTDRAe9EJg= -github.com/dolthub/vitess v0.0.0-20250729225143-5ab74d1f0182/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70= github.com/dolthub/vitess v0.0.0-20250729231828-87bce34800ce h1:WAOnVqPsJF+y5Js5LFGzEUFOb9nnenBIcaOeRni8rZw= github.com/dolthub/vitess v0.0.0-20250729231828-87bce34800ce/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=