dolthub/dolt#9554 - Fix varbinary fields displaying as unicode instead of hex

go/cmd/dolt/commands/blame.go

@@ -130,7 +130,7 @@ func (cmd BlameCmd) Exec(ctx context.Context, commandStr string, args []string,
 		return 1
 	}
 
-	err = engine.PrettyPrintResults(sqlCtx, engine.FormatTabular, schema, ri, false, false, true)
+	err = engine.PrettyPrintResults(sqlCtx, engine.FormatTabular, schema, ri, false, false, true, false)
 	if err != nil {
 		iohelp.WriteLine(cli.CliOut, err.Error())
 		return 1

go/cmd/dolt/commands/cvcmds/verify_constraints.go

@@ -181,7 +181,7 @@ func printViolationsForTable(ctx *sql.Context, dbName, tblName string, tbl *dolt
 
 	limitItr := &sqlLimitIter{itr: sqlItr, limit: 50}
 
-	err = engine.PrettyPrintResults(ctx, engine.FormatTabular, sqlSch, limitItr, false, false, false)
+	err = engine.PrettyPrintResults(ctx, engine.FormatTabular, sqlSch, limitItr, false, false, false, false)
 	if err != nil {
 		return errhand.BuildDError("Error outputting rows").AddCause(err).Build()
 	}

go/cmd/dolt/commands/debug.go

@@ -436,5 +436,5 @@ func execDebugMode(ctx *sql.Context, qryist cli.Queryist, queryFile *os.File, co
 	}()
 	input := bufio.NewReader(transform.NewReader(queryFile, textunicode.BOMOverride(transform.Nop)))
 
-	return execBatchMode(ctx, qryist, input, continueOnErr, format)
+	return execBatchMode(ctx, qryist, input, continueOnErr, format, false)
 }

go/cmd/dolt/commands/engine/sql_print.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/dolthub/go-mysql-server/sql"
 	"github.com/dolthub/go-mysql-server/sql/types"
+	"github.com/dolthub/vitess/go/sqltypes"
 	"github.com/fatih/color"
 
 	"github.com/dolthub/dolt/go/cmd/dolt/cli"
@@ -56,16 +57,16 @@ const (
 )
 
 // PrettyPrintResults prints the result of a query in the format provided
-func PrettyPrintResults(ctx *sql.Context, resultFormat PrintResultFormat, sqlSch sql.Schema, rowIter sql.RowIter, pageResults, showWarnings, printOkResult bool) (rerr error) {
-	return prettyPrintResultsWithSummary(ctx, resultFormat, sqlSch, rowIter, PrintNoSummary, pageResults, showWarnings, printOkResult)
+func PrettyPrintResults(ctx *sql.Context, resultFormat PrintResultFormat, sqlSch sql.Schema, rowIter sql.RowIter, pageResults, showWarnings, printOkResult, binaryAsHex bool) (rerr error) {
+	return prettyPrintResultsWithSummary(ctx, resultFormat, sqlSch, rowIter, PrintNoSummary, pageResults, showWarnings, printOkResult, binaryAsHex)
 }
 
 // PrettyPrintResultsExtended prints the result of a query in the format provided, including row count and timing info
-func PrettyPrintResultsExtended(ctx *sql.Context, resultFormat PrintResultFormat, sqlSch sql.Schema, rowIter sql.RowIter, pageResults, showWarnings, printOkResult bool) (rerr error) {
-	return prettyPrintResultsWithSummary(ctx, resultFormat, sqlSch, rowIter, PrintRowCountAndTiming, pageResults, showWarnings, printOkResult)
+func PrettyPrintResultsExtended(ctx *sql.Context, resultFormat PrintResultFormat, sqlSch sql.Schema, rowIter sql.RowIter, pageResults, showWarnings, printOkResult, binaryAsHex bool) (rerr error) {
+	return prettyPrintResultsWithSummary(ctx, resultFormat, sqlSch, rowIter, PrintRowCountAndTiming, pageResults, showWarnings, printOkResult, binaryAsHex)
 }
 
-func prettyPrintResultsWithSummary(ctx *sql.Context, resultFormat PrintResultFormat, sqlSch sql.Schema, rowIter sql.RowIter, summary PrintSummaryBehavior, pageResults, showWarnings, printOkResult bool) (rerr error) {
+func prettyPrintResultsWithSummary(ctx *sql.Context, resultFormat PrintResultFormat, sqlSch sql.Schema, rowIter sql.RowIter, summary PrintSummaryBehavior, pageResults, showWarnings, printOkResult, binaryAsHex bool) (rerr error) {
 	defer func() {
 		closeErr := rowIter.Close(ctx)
 		if rerr == nil && closeErr != nil {
@@ -126,6 +127,11 @@ func prettyPrintResultsWithSummary(ctx *sql.Context, resultFormat PrintResultFor
 			}
 		}
 
+		// Wrap iterator with binary-to-hex transformation if needed
+		if binaryAsHex {
+			rowIter = newBinaryHexIterator(rowIter, sqlSch)
+		}
+
 		numRows, err = writeResultSet(ctx, rowIter, wr)
 	}
 
@@ -197,6 +203,54 @@ func printResultSetSummary(numRows int, numWarnings uint16, warningsList string,
 	return nil
 }
 
+// binaryHexIterator wraps a row iterator and transforms binary data to hex format
+type binaryHexIterator struct {
+	inner  sql.RowIter
+	schema sql.Schema
+}
+
+var _ sql.RowIter = (*binaryHexIterator)(nil)
+
+// newBinaryHexIterator creates a new iterator that transforms binary data to hex format
+func newBinaryHexIterator(inner sql.RowIter, schema sql.Schema) sql.RowIter {
+	return &binaryHexIterator{
+		inner:  inner,
+		schema: schema,
+	}
+}
+
+// Next returns the next row with binary data transformed to hex format
+func (iter *binaryHexIterator) Next(ctx *sql.Context) (sql.Row, error) {
+	rowData, err := iter.inner.Next(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: Add support for BLOB types (TINYBLOB, BLOB, MEDIUMBLOB, LONGBLOB) and BIT type
+	for i, val := range rowData {
+		if val != nil && i < len(iter.schema) {
+			switch iter.schema[i].Type.Type() {
+			case sqltypes.Binary, sqltypes.VarBinary:
+				switch v := val.(type) {
+				case []byte: // hex fmt is explicitly upper case
+					rowData[i] = sqlutil.BinaryAsHexDisplayValue(fmt.Sprintf("0x%X", v))
+				case string: // handles results from sql-server; MySQL wire protocol returns strings
+					rowData[i] = sqlutil.BinaryAsHexDisplayValue(fmt.Sprintf("0x%X", []byte(v)))
+				default:
+					return nil, fmt.Errorf("unexpected type %T for binary column %s", val, iter.schema[i].Name)
+				}
+			}
+		}
+	}
+
+	return rowData, nil
+}
+
+// Close closes the wrapped iterator and releases any resources.
+func (iter *binaryHexIterator) Close(ctx *sql.Context) error {
+	return iter.inner.Close(ctx)
+}
+
 // writeResultSet drains the iterator given, printing rows from it to the writer given. Returns the number of rows.
 func writeResultSet(ctx *sql.Context, rowIter sql.RowIter, wr table.SqlRowWriter) (int, error) {
 	i := 0

go/cmd/dolt/commands/schcmds/tags.go

@@ -140,7 +140,7 @@ func (cmd TagsCmd) Exec(ctx context.Context, commandStr string, args []string, d
 	}
 
 	sqlCtx := sql.NewContext(ctx)
-	err = engine.PrettyPrintResults(sqlCtx, outputFmt, headerSchema, sql.RowsToRowIter(rows...), false, false, false)
+	err = engine.PrettyPrintResults(sqlCtx, outputFmt, headerSchema, sql.RowsToRowIter(rows...), false, false, false, false)
 
 	return commands.HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
 }

go/cmd/dolt/commands/sql.go

@@ -98,6 +98,11 @@ const (
 	ProfileFlag           = "profile"
 	timeFlag              = "time"
 	outputFlag            = "output"
+	binaryAsHexFlag       = "binary-as-hex"
+	skipBinaryAsHexFlag   = "skip-binary-as-hex"
+	// TODO: Consider simplifying to use MySQL's skip pattern with single flag definition
+	// MySQL handles both --binary-as-hex and --skip-binary-as-hex with one option definition
+	// and uses disabled_my_option to distinguish between enable/disable
 
 	welcomeMsg = `# Welcome to the DoltSQL shell.
 # Statements must be terminated with ';'.
@@ -147,6 +152,9 @@ func (cmd SqlCmd) ArgParser() *argparser.ArgParser {
 	ap.SupportsFlag(BatchFlag, "b", "Use to enable more efficient batch processing for large SQL import scripts. This mode is no longer supported and this flag is a no-op. To speed up your SQL imports, use either LOAD DATA, or structure your SQL import script to insert many rows per statement.")
 	ap.SupportsFlag(continueFlag, "c", "Continue running queries on an error. Used for batch mode only.")
 	ap.SupportsString(fileInputFlag, "f", "input file", "Execute statements from the file given.")
+	ap.SupportsFlag(binaryAsHexFlag, "", "Print binary data as hex. Enabled by default for interactive terminals.")
+	// TODO: MySQL uses a skip- pattern for negating flags and doesn't show them in help
+	ap.SupportsFlag(skipBinaryAsHexFlag, "", "Disable binary data as hex output.")
 	return ap
 }
 
@@ -213,6 +221,12 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE
 		return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
 	}
 
+	// Determine binary-as-hex behavior from flags (default false for non-interactive modes)
+	binaryAsHex := apr.Contains(binaryAsHexFlag)
+	if binaryAsHex && apr.Contains(skipBinaryAsHexFlag) { // We stray from MYSQL here to make usage clear for users
+		return HandleVErrAndExitCode(errhand.BuildDError("cannot use both --%s and --%s", binaryAsHexFlag, skipBinaryAsHexFlag).Build(), usage)
+	}
+
 	queryist, sqlCtx, closeFunc, err := cliCtx.QueryEngine(ctx)
 	if err != nil {
 		return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
@@ -223,17 +237,16 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE
 
 	if query, queryOK := apr.GetValue(QueryFlag); queryOK {
 		if apr.Contains(saveFlag) {
-			return SaveQuery(sqlCtx, queryist, apr, query, format, usage)
+			return SaveQuery(sqlCtx, queryist, apr, query, format, usage, binaryAsHex)
 		}
-		return queryMode(sqlCtx, queryist, apr, query, format, usage)
+		return queryMode(sqlCtx, queryist, apr, query, format, usage, binaryAsHex)
 	} else if savedQueryName, exOk := apr.GetValue(executeFlag); exOk {
-		return executeSavedQuery(sqlCtx, queryist, savedQueryName, format, usage)
+		return executeSavedQuery(sqlCtx, queryist, savedQueryName, format, usage, binaryAsHex)
 	} else if apr.Contains(listSavedFlag) {
 		return listSavedQueries(sqlCtx, queryist, format, usage)
 	} else {
 		// Run in either batch mode for piped input, or shell mode for interactive
 		isTty := false
-
 		fi, err := os.Stdin.Stat()
 		if err != nil {
 			if !osutil.IsWindows {
@@ -265,13 +278,15 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE
 		}
 
 		if isTty {
-			err := execShell(sqlCtx, queryist, format, cliCtx)
+			// In shell mode, default to hex format unless explicitly disabled
+			shellBinaryAsHex := !apr.Contains(skipBinaryAsHexFlag)
+			err := execShell(sqlCtx, queryist, format, cliCtx, shellBinaryAsHex)
 			if err != nil {
 				return sqlHandleVErrAndExitCode(queryist, errhand.VerboseErrorFromError(err), usage)
 			}
 		} else {
 			input = transform.NewReader(input, textunicode.BOMOverride(transform.Nop))
-			err := execBatchMode(sqlCtx, queryist, input, continueOnError, format)
+			err := execBatchMode(sqlCtx, queryist, input, continueOnError, format, binaryAsHex)
 			if err != nil {
 				return sqlHandleVErrAndExitCode(queryist, errhand.VerboseErrorFromError(err), usage)
 			}
@@ -341,10 +356,10 @@ func (cmd SqlCmd) handleLegacyArguments(ap *argparser.ArgParser, commandStr stri
 
 func listSavedQueries(ctx *sql.Context, qryist cli.Queryist, format engine.PrintResultFormat, usage cli.UsagePrinter) int {
 	query := "SELECT * FROM " + doltdb.DoltQueryCatalogTableName
-	return sqlHandleVErrAndExitCode(qryist, execSingleQuery(ctx, qryist, query, format), usage)
+	return sqlHandleVErrAndExitCode(qryist, execSingleQuery(ctx, qryist, query, format, false), usage)
 }
 
-func executeSavedQuery(ctx *sql.Context, qryist cli.Queryist, savedQueryName string, format engine.PrintResultFormat, usage cli.UsagePrinter) int {
+func executeSavedQuery(ctx *sql.Context, qryist cli.Queryist, savedQueryName string, format engine.PrintResultFormat, usage cli.UsagePrinter, binaryAsHex bool) int {
 	var buffer bytes.Buffer
 	buffer.WriteString("SELECT query FROM dolt_query_catalog where id = ?")
 	searchQuery, err := dbr.InterpolateForDialect(buffer.String(), []interface{}{savedQueryName}, dialect.MySQL)
@@ -370,7 +385,7 @@ func executeSavedQuery(ctx *sql.Context, qryist cli.Queryist, savedQueryName str
 	}
 
 	cli.PrintErrf("Executing saved query '%s':\n%s\n", savedQueryName, query)
-	return sqlHandleVErrAndExitCode(qryist, execSingleQuery(ctx, qryist, query, format), usage)
+	return sqlHandleVErrAndExitCode(qryist, execSingleQuery(ctx, qryist, query, format, binaryAsHex), usage)
 }
 
 func queryMode(
@@ -380,23 +395,23 @@ func queryMode(
 	query string,
 	format engine.PrintResultFormat,
 	usage cli.UsagePrinter,
+	binaryAsHex bool,
 ) int {
-
 	_, continueOnError := apr.GetValue(continueFlag)
 
 	input := strings.NewReader(query)
-	err := execBatchMode(ctx, qryist, input, continueOnError, format)
+	err := execBatchMode(ctx, qryist, input, continueOnError, format, binaryAsHex)
 	if err != nil {
 		return sqlHandleVErrAndExitCode(qryist, errhand.VerboseErrorFromError(err), usage)
 	}
 
 	return 0
 }
 
-func SaveQuery(ctx *sql.Context, qryist cli.Queryist, apr *argparser.ArgParseResults, query string, format engine.PrintResultFormat, usage cli.UsagePrinter) int {
+func SaveQuery(ctx *sql.Context, qryist cli.Queryist, apr *argparser.ArgParseResults, query string, format engine.PrintResultFormat, usage cli.UsagePrinter, binaryAsHex bool) int {
 	saveName := apr.GetValueOrDefault(saveFlag, "")
 
-	verr := execSingleQuery(ctx, qryist, query, format)
+	verr := execSingleQuery(ctx, qryist, query, format, binaryAsHex)
 	if verr != nil {
 		return sqlHandleVErrAndExitCode(qryist, verr, usage)
 	}
@@ -435,6 +450,7 @@ func execSingleQuery(
 	qryist cli.Queryist,
 	query string,
 	format engine.PrintResultFormat,
+	binaryAsHex bool,
 ) errhand.VerboseError {
 
 	sqlSch, rowIter, _, err := processQuery(sqlCtx, query, qryist)
@@ -443,7 +459,7 @@ func execSingleQuery(
 	}
 
 	if rowIter != nil {
-		err = engine.PrettyPrintResults(sqlCtx, format, sqlSch, rowIter, false, false, false)
+		err = engine.PrettyPrintResults(sqlCtx, format, sqlSch, rowIter, false, false, false, binaryAsHex)
 		if err != nil {
 			return errhand.VerboseErrorFromError(err)
 		}
@@ -601,7 +617,7 @@ func validateSqlArgs(apr *argparser.ArgParseResults) error {
 }
 
 // execBatchMode runs all the queries in the input reader
-func execBatchMode(ctx *sql.Context, qryist cli.Queryist, input io.Reader, continueOnErr bool, format engine.PrintResultFormat) error {
+func execBatchMode(ctx *sql.Context, qryist cli.Queryist, input io.Reader, continueOnErr bool, format engine.PrintResultFormat, binaryAsHex bool) error {
 	scanner := NewStreamScanner(input)
 	var query string
 	for scanner.Scan() {
@@ -649,7 +665,7 @@ func execBatchMode(ctx *sql.Context, qryist cli.Queryist, input io.Reader, conti
 					fileReadProg.printNewLineIfNeeded()
 				}
 			}
-			err = engine.PrettyPrintResults(ctx, format, sqlSch, rowIter, false, false, false)
+			err = engine.PrettyPrintResults(ctx, format, sqlSch, rowIter, false, false, false, binaryAsHex)
 			if err != nil {
 				err = buildBatchSqlErr(scanner.state.statementStartLine, query, err)
 				if !continueOnErr {
@@ -675,7 +691,7 @@ func buildBatchSqlErr(stmtStartLine int, query string, err error) error {
 
 // execShell starts a SQL shell. Returns when the user exits the shell. The Root of the sqlEngine may
 // be updated by any queries which were processed.
-func execShell(sqlCtx *sql.Context, qryist cli.Queryist, format engine.PrintResultFormat, cliCtx cli.CliContext) error {
+func execShell(sqlCtx *sql.Context, qryist cli.Queryist, format engine.PrintResultFormat, cliCtx cli.CliContext, binaryAsHex bool) error {
 	_ = iohelp.WriteLine(cli.CliOut, welcomeMsg)
 	historyFile := filepath.Join(".sqlhistory") // history file written to working dir
 
@@ -832,9 +848,9 @@ func execShell(sqlCtx *sql.Context, qryist cli.Queryist, format engine.PrintResu
 					} else if rowIter != nil {
 						switch closureFormat {
 						case engine.FormatTabular, engine.FormatVertical:
-							err = engine.PrettyPrintResultsExtended(sqlCtx, closureFormat, sqlSch, rowIter, pagerEnabled, toggleWarnings, true)
+							err = engine.PrettyPrintResultsExtended(sqlCtx, closureFormat, sqlSch, rowIter, pagerEnabled, toggleWarnings, true, binaryAsHex)
 						default:
-							err = engine.PrettyPrintResults(sqlCtx, closureFormat, sqlSch, rowIter, pagerEnabled, toggleWarnings, true)
+							err = engine.PrettyPrintResults(sqlCtx, closureFormat, sqlSch, rowIter, pagerEnabled, toggleWarnings, true, binaryAsHex)
 						}
 					} else {
 						if _, isUseStmt := sqlStmt.(*sqlparser.Use); isUseStmt {

go/cmd/dolt/commands/sqlserver/queryist_utils.go

@@ -23,7 +23,6 @@ import (
 	"strings"
 
 	"github.com/dolthub/go-mysql-server/sql"
-	"github.com/dolthub/go-mysql-server/sql/types"
 	"github.com/dolthub/vitess/go/vt/sqlparser"
 	"github.com/go-sql-driver/mysql"
 	"github.com/gocraft/dbr/v2"
@@ -32,6 +31,7 @@ import (
 	"github.com/dolthub/dolt/go/cmd/dolt/cli"
 	"github.com/dolthub/dolt/go/libraries/doltcore/servercfg"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
+	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
 	"github.com/dolthub/dolt/go/libraries/utils/argparser"
 	"github.com/dolthub/dolt/go/libraries/utils/filesys"
 )
@@ -147,18 +147,18 @@ type MysqlRowWrapper struct {
 var _ sql.RowIter = (*MysqlRowWrapper)(nil)
 
 func NewMysqlRowWrapper(sqlRows *sql2.Rows) (*MysqlRowWrapper, error) {
-	colNames, err := sqlRows.Columns()
+	colTypes, err := sqlRows.ColumnTypes()
 	if err != nil {
 		return nil, err
 	}
-	schema := make(sql.Schema, len(colNames))
-	vRow := make([]*string, len(colNames))
-	iRow := make([]interface{}, len(colNames))
+	schema := make(sql.Schema, len(colTypes))
+	vRow := make([]*string, len(colTypes))
+	iRow := make([]interface{}, len(colTypes))
 	rows := make([]sql.Row, 0)
-	for i, colName := range colNames {
+	for i, colType := range colTypes {
 		schema[i] = &sql.Column{
-			Name:     colName,
-			Type:     types.LongText,
+			Name:     colType.Name(),
+			Type:     sqlutil.DatabaseTypeNameToSqlType(colType.DatabaseTypeName()),
 			Nullable: true,
 		}
 		iRow[i] = &vRow[i]