Skip to content

Commit

Permalink
expression: disable some json function from pushdown to TiFlash (#56177)
Browse files Browse the repository at this point in the history 
close #56173
  • Loading branch information
@windtalker
windtalker authored Sep 20, 2024
1 parent adbc003 commit ff1f862
Show file tree
Hide file tree
Showing 2 changed files with 118 additions and 28 deletions.
136 changes: 109 additions & 27 deletions pkg/expression/expr_to_pb_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -505,7 +505,7 @@ func TestOtherFunc2Pb(t *testing.T) {
}
}

func TestExprPushDownToFlash(t *testing.T) {
func TestJsonPushDownToFlash(t *testing.T) {
ctx := mock.NewContext()
client := new(mock.Client)
pushDownCtx := NewPushDownContextFromSessionVars(ctx, ctx.GetSessionVars(), client)
Expand All @@ -514,22 +514,10 @@ func TestExprPushDownToFlash(t *testing.T) {

jsonColumn := genColumn(mysql.TypeJSON, 1)
intColumn := genColumn(mysql.TypeLonglong, 2)
realColumn := genColumn(mysql.TypeDouble, 3)
decimalColumn := genColumn(mysql.TypeNewDecimal, 4)
decimalColumn.RetType.SetDecimal(mysql.MaxDecimalScale)
decimalColumn.RetType.SetFlen(mysql.MaxDecimalWidth)
stringColumn := genColumn(mysql.TypeString, 5)
datetimeColumn := genColumn(mysql.TypeDatetime, 6)
binaryStringColumn := genColumn(mysql.TypeString, 7)
binaryStringColumn.RetType.SetCollate(charset.CollationBin)
int32Column := genColumn(mysql.TypeLong, 8)
float32Column := genColumn(mysql.TypeFloat, 9)
enumColumn := genColumn(mysql.TypeEnum, 10)
durationColumn := genColumn(mysql.TypeDuration, 11)
// uint64 col
uintColumn := genColumn(mysql.TypeLonglong, 12)
uintColumn.RetType.AddFlag(mysql.UnsignedFlag)

// functions that can be pushdown to tiflash
// json_length
function, err := NewFunction(mock.NewContext(), ast.JSONLength, types.NewFieldType(mysql.TypeLonglong), jsonColumn)
require.NoError(t, err)
exprs = append(exprs, function)
Expand Down Expand Up @@ -590,8 +578,112 @@ func TestExprPushDownToFlash(t *testing.T) {
require.NoError(t, err)
exprs = append(exprs, function)

// lpad
function, err = NewFunction(mock.NewContext(), ast.Lpad, types.NewFieldType(mysql.TypeString), stringColumn, int32Column, stringColumn)
// CastJsonAsString
function, err = NewFunction(mock.NewContext(), ast.Cast, types.NewFieldType(mysql.TypeString), jsonColumn)
require.NoError(t, err)
exprs = append(exprs, function)

// CastJsonAsJson
function, err = NewFunction(mock.NewContext(), ast.Cast, types.NewFieldType(mysql.TypeJSON), jsonColumn)
require.NoError(t, err)
exprs = append(exprs, function)

// IfNullJson
function, err = NewFunction(mock.NewContext(), ast.Ifnull, types.NewFieldType(mysql.TypeJSON), jsonColumn, jsonColumn)
require.NoError(t, err)
exprs = append(exprs, function)

// IfJson
function, err = NewFunction(mock.NewContext(), ast.If, types.NewFieldType(mysql.TypeJSON), intColumn, jsonColumn, jsonColumn)
require.NoError(t, err)
exprs = append(exprs, function)

// JsonIsNull is not implement, for function json_col is null, it will be converted to cast(json as string) is null
function, err = NewFunction(mock.NewContext(), ast.IsNull, types.NewFieldType(mysql.TypeLonglong), jsonColumn)
require.NoError(t, err)
exprs = append(exprs, function)

// CaseWhenJson
function, err = NewFunction(mock.NewContext(), ast.Case, types.NewFieldType(mysql.TypeJSON), intColumn, jsonColumn, intColumn, jsonColumn, jsonColumn)
require.NoError(t, err)
exprs = append(exprs, function)

// CoalesceJson
function, err = NewFunction(mock.NewContext(), ast.Coalesce, types.NewFieldType(mysql.TypeJSON), jsonColumn, jsonColumn, jsonColumn)
require.NoError(t, err)
exprs = append(exprs, function)

pushed, remained := PushDownExprs(pushDownCtx, exprs, kv.TiFlash)
require.Len(t, pushed, len(exprs))
require.Len(t, remained, 0)

// functions that can not be pushed to tiflash
exprs = exprs[:0]
// LTJson
function, err = NewFunction(mock.NewContext(), ast.LT, types.NewFieldType(mysql.TypeLonglong), jsonColumn, jsonColumn)
require.NoError(t, err)
exprs = append(exprs, function)

// LEJson
function, err = NewFunction(mock.NewContext(), ast.LE, types.NewFieldType(mysql.TypeLonglong), jsonColumn, jsonColumn)
require.NoError(t, err)
exprs = append(exprs, function)

// GTJson
function, err = NewFunction(mock.NewContext(), ast.GT, types.NewFieldType(mysql.TypeLonglong), jsonColumn, jsonColumn)
require.NoError(t, err)
exprs = append(exprs, function)

// GEJson
function, err = NewFunction(mock.NewContext(), ast.GE, types.NewFieldType(mysql.TypeLonglong), jsonColumn, jsonColumn)
require.NoError(t, err)
exprs = append(exprs, function)

// EQJson
function, err = NewFunction(mock.NewContext(), ast.EQ, types.NewFieldType(mysql.TypeLonglong), jsonColumn, jsonColumn)
require.NoError(t, err)
exprs = append(exprs, function)

// NEJson
function, err = NewFunction(mock.NewContext(), ast.NE, types.NewFieldType(mysql.TypeLonglong), jsonColumn, jsonColumn)
require.NoError(t, err)
exprs = append(exprs, function)

// InJson
function, err = NewFunction(mock.NewContext(), ast.In, types.NewFieldType(mysql.TypeLonglong), jsonColumn, jsonColumn, jsonColumn)
require.NoError(t, err)
exprs = append(exprs, function)

pushed, remained = PushDownExprs(pushDownCtx, exprs, kv.TiFlash)
require.Len(t, pushed, 0)
require.Len(t, remained, len(exprs))
}

func TestExprPushDownToFlash(t *testing.T) {
ctx := mock.NewContext()
client := new(mock.Client)
pushDownCtx := NewPushDownContextFromSessionVars(ctx, ctx.GetSessionVars(), client)

exprs := make([]Expression, 0)

intColumn := genColumn(mysql.TypeLonglong, 2)
realColumn := genColumn(mysql.TypeDouble, 3)
decimalColumn := genColumn(mysql.TypeNewDecimal, 4)
decimalColumn.RetType.SetDecimal(mysql.MaxDecimalScale)
decimalColumn.RetType.SetFlen(mysql.MaxDecimalWidth)
stringColumn := genColumn(mysql.TypeString, 5)
datetimeColumn := genColumn(mysql.TypeDatetime, 6)
binaryStringColumn := genColumn(mysql.TypeString, 7)
binaryStringColumn.RetType.SetCollate(charset.CollationBin)
int32Column := genColumn(mysql.TypeLong, 8)
float32Column := genColumn(mysql.TypeFloat, 9)
enumColumn := genColumn(mysql.TypeEnum, 10)
durationColumn := genColumn(mysql.TypeDuration, 11)
// uint64 col
uintColumn := genColumn(mysql.TypeLonglong, 12)
uintColumn.RetType.AddFlag(mysql.UnsignedFlag)

function, err := NewFunction(mock.NewContext(), ast.Lpad, types.NewFieldType(mysql.TypeString), stringColumn, int32Column, stringColumn)
require.NoError(t, err)
exprs = append(exprs, function)

Expand Down Expand Up @@ -706,11 +798,6 @@ func TestExprPushDownToFlash(t *testing.T) {
require.NoError(t, err)
exprs = append(exprs, function)

// CastJsonAsString
function, err = NewFunction(mock.NewContext(), ast.Cast, types.NewFieldType(mysql.TypeString), jsonColumn)
require.NoError(t, err)
exprs = append(exprs, function)

// CastIntAsTime
function, err = NewFunction(mock.NewContext(), ast.Cast, types.NewFieldType(mysql.TypeDatetime), intColumn)
require.NoError(t, err)
Expand Down Expand Up @@ -1370,11 +1457,6 @@ func TestExprPushDownToFlash(t *testing.T) {
require.NoError(t, err)
exprs = append(exprs, function)

// CastJsonAsJson
function, err = NewFunction(mock.NewContext(), ast.Cast, types.NewFieldType(mysql.TypeJSON), jsonColumn)
require.NoError(t, err)
exprs = append(exprs, function)

pushed, remained = PushDownExprs(pushDownCtx, exprs, kv.TiFlash)
require.Len(t, pushed, len(exprs))
require.Len(t, remained, 0)
Expand Down
10 changes: 9 additions & 1 deletion pkg/expression/infer_pushdown.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,15 @@ func scalarExprSupportedByFlash(ctx EvalContext, function *ScalarFunction) bool
tipb.ScalarFuncSig_CoalesceDuration,
tipb.ScalarFuncSig_IfNullDuration,
tipb.ScalarFuncSig_IfDuration,
tipb.ScalarFuncSig_CaseWhenDuration:
tipb.ScalarFuncSig_CaseWhenDuration,
tipb.ScalarFuncSig_LTJson,
tipb.ScalarFuncSig_LEJson,
tipb.ScalarFuncSig_GTJson,
tipb.ScalarFuncSig_GEJson,
tipb.ScalarFuncSig_EQJson,
tipb.ScalarFuncSig_NEJson,
tipb.ScalarFuncSig_JsonIsNull,
tipb.ScalarFuncSig_InJson:
return false
}
return true
Expand Down

0 comments on commit ff1f862

Please sign in to comment.