Do not allow sort-based joins between text and number type columns

enginetest/join_op_tests.go

@@ -2339,6 +2339,30 @@ WHERE
 			},
 		},
 	},
+	{
+		name: "join on string and number columns",
+		setup: [][]string{
+			{
+				"create table t0(c0 varchar(500) primary key, c1 int)",
+				"create table t1(c0 int primary key, c1 varchar(500))",
+				"insert into t0(c0) values (5), (33), (223), ('123a')",
+				"insert into t1(c0) values (5), (33), (223), (123)",
+				"insert into t1(c0) values (-1)",
+				"insert into t0(c0, c1) values (false, -2)",
+			},
+		},
+		tests: []JoinOpTests{
+			{
+				Query:    "select t0.c0, t1.c0 from t0 join t1 on t0.c0 = t1.c0 order by t1.c0;",
+				Expected: []sql.Row{{"5", 5}, {"33", 33}, {"123a", 123}, {"223", 223}},
+			},
+			{
+				// https://github.com/dolthub/dolt/issues/10435
+				Query:    "select * from t1 inner join t0 on (t1.c0 between t0.c1 and t0.c0)",
+				Expected: []sql.Row{{-1, nil, "0", -2}},
+			},
+		},
+	},
 }
 
 var rangeJoinOpTests = []JoinOpTests{

sql/analyzer/indexed_joins.go

@@ -1033,6 +1033,15 @@ func addRangeHeapJoin(m *memo.Memo) error {
 				return nil
 			}
 
+			valType := valueColRef.Type()
+			// TODO: Incompatible sort orders between the value and min columns would be fine if we sorted the tables
+			//  using the same sort order (for example, if value is a number type column and min is a string, we sort
+			//  the right table based on min converted to a number). Incompatible sort orders between value and max
+			//  columns could be fine depending on the heap implementation and if we updated the range heap join iter to
+			//  use a compare expression instead of hard-coding it to use maxColRef.Type().Compare
+			if !compatibleSortOrders(valType, minColRef.Type()) || !compatibleSortOrders(valType, maxColRef.Type()) {
+				return nil
+			}
 			leftIndexScans, err := sortedIndexScansForTableCol(m.Ctx, m.StatsProvider(), leftTab, lIndexes, valueColRef, join.Left.RelProps.FuncDeps().Constants(), lFilters)
 			if err != nil {
 				return err
@@ -1147,7 +1156,7 @@ func addMergeJoins(ctx *sql.Context, m *memo.Memo) error {
 				}
 
 				// check that comparer is not non-decreasing
-				if !canMergeTypes(l.Type(), r.Type()) || !isWeaklyMonotonic(l) || !isWeaklyMonotonic(r) {
+				if !compatibleSortOrders(l.Type(), r.Type()) || !isWeaklyMonotonic(l) || !isWeaklyMonotonic(r) {
 					continue
 				}
 
@@ -1491,15 +1500,18 @@ func makeIndexScan(ctx *sql.Context, statsProv sql.StatsProvider, tab plan.Table
 	}, true, nil
 }
 
-// canMerge checks the types of two columns to see if they can be merged into one another if sorted.
-func canMergeTypes(t1, t2 sql.Type) bool {
-	// TODO: handle other types here. For example, Number and Text types likely can't be merged together. But we need to
-	// add more testing https://github.com/dolthub/dolt/issues/10316
+// compatibleSortOrders checks the types of two columns to see if they can be merged into one another if sorted.
+func compatibleSortOrders(t1, t2 sql.Type) bool {
+	// TODO: handle other types here https://github.com/dolthub/dolt/issues/10316
 	switch {
 	case types.IsEnum(t1):
 		if types.IsEnum(t2) {
 			return types.TypesEqual(t1, t2)
 		}
+	case types.IsNumber(t1):
+		return !types.IsText(t2)
+	case types.IsText(t1):
+		return !types.IsNumber(t2)
 	}
 	return true
 }