diff --git a/go/streamlog/streamlog.go b/go/streamlog/streamlog.go index 9c881a877bd..32fdb6be241 100644 --- a/go/streamlog/streamlog.go +++ b/go/streamlog/streamlog.go @@ -42,7 +42,10 @@ var ( QueryLogFormat = flag.String("querylog-format", "text", "format for query logs (\"text\" or \"json\")") // QueryLogFilterTag contains an optional string that must be present in the query for it to be logged - QueryLogFilterTag = flag.String("querylog-filter-tag", "", "string that must be present in the query for it to be logged") + QueryLogFilterTag = flag.String("querylog-filter-tag", "", "string that must be present in the query for it to be logged; if using a value as the tag, you need to disable query normalization") + + // QueryLogRowThreshold only log queries returning or affecting this many rows + QueryLogRowThreshold = flag.Uint64("querylog-row-threshold", 0, "Number of rows a query has to return or affect before being logged; not useful for streaming queries. 0 means all queries will be logged.") sendCount = stats.NewCountersWithSingleLabel("StreamlogSend", "stream log send count", "logger_names") deliveredCount = stats.NewCountersWithMultiLabels( @@ -208,9 +211,19 @@ func GetFormatter(logger *StreamLogger) LogFormatter { // ShouldEmitLog returns whether the log with the given SQL query // should be emitted or filtered -func ShouldEmitLog(sql string) bool { - if *QueryLogFilterTag == "" { - return true +func ShouldEmitLog(sql string, rowsAffected, rowsReturned uint64) bool { + if *QueryLogRowThreshold > maxUint64(rowsAffected, rowsReturned) && *QueryLogFilterTag == "" { + return false + } + if *QueryLogFilterTag != "" { + return strings.Contains(sql, *QueryLogFilterTag) + } + return true +} + +func maxUint64(a, b uint64) uint64 { + if a < b { + return b } - return strings.Contains(sql, *QueryLogFilterTag) + return a } diff --git a/go/vt/vtgate/logstats.go b/go/vt/vtgate/logstats.go index e70f550632e..aeac6fe84c2 100644 --- a/go/vt/vtgate/logstats.go +++ b/go/vt/vtgate/logstats.go @@ -126,7 +126,7 @@ func (stats *LogStats) RemoteAddrUsername() (string, string) { // Logf formats the log record to the given writer, either as // tab-separated list of logged fields or as JSON. func (stats *LogStats) Logf(w io.Writer, params url.Values) error { - if !streamlog.ShouldEmitLog(stats.SQL) { + if !streamlog.ShouldEmitLog(stats.SQL, stats.RowsAffected, stats.RowsReturned) { return nil } diff --git a/go/vt/vtgate/logstats_test.go b/go/vt/vtgate/logstats_test.go index 7a19e75069d..280cf9214c9 100644 --- a/go/vt/vtgate/logstats_test.go +++ b/go/vt/vtgate/logstats_test.go @@ -158,6 +158,35 @@ func TestLogStatsFilter(t *testing.T) { } } +func TestLogStatsRowThreshold(t *testing.T) { + defer func() { *streamlog.QueryLogRowThreshold = 0 }() + + logStats := NewLogStats(context.Background(), "test", "sql1 /* LOG_THIS_QUERY */", map[string]*querypb.BindVariable{"intVal": sqltypes.Int64BindVariable(1)}) + logStats.StartTime = time.Date(2017, time.January, 1, 1, 2, 3, 0, time.UTC) + logStats.EndTime = time.Date(2017, time.January, 1, 1, 2, 4, 1234, time.UTC) + params := map[string][]string{"full": {}} + + got := testFormat(logStats, url.Values(params)) + want := "test\t\t\t''\t''\t2017-01-01 01:02:03.000000\t2017-01-01 01:02:04.000001\t1.000001\t0.000000\t0.000000\t0.000000\t\t\"sql1 /* LOG_THIS_QUERY */\"\tmap[intVal:type:INT64 value:\"1\" ]\t0\t0\t\"\"\t\"\"\t\"\"\t\"\"\t\n" + if got != want { + t.Errorf("logstats format: got:\n%q\nwant:\n%q\n", got, want) + } + + *streamlog.QueryLogRowThreshold = 0 + got = testFormat(logStats, url.Values(params)) + want = "test\t\t\t''\t''\t2017-01-01 01:02:03.000000\t2017-01-01 01:02:04.000001\t1.000001\t0.000000\t0.000000\t0.000000\t\t\"sql1 /* LOG_THIS_QUERY */\"\tmap[intVal:type:INT64 value:\"1\" ]\t0\t0\t\"\"\t\"\"\t\"\"\t\"\"\t\n" + if got != want { + t.Errorf("logstats format: got:\n%q\nwant:\n%q\n", got, want) + } + + *streamlog.QueryLogRowThreshold = 1 + got = testFormat(logStats, url.Values(params)) + want = "" + if got != want { + t.Errorf("logstats format: got:\n%q\nwant:\n%q\n", got, want) + } +} + func TestLogStatsContextHTML(t *testing.T) { html := "HtmlContext" callInfo := &fakecallinfo.FakeCallInfo{ diff --git a/go/vt/vttablet/tabletserver/tabletenv/logstats.go b/go/vt/vttablet/tabletserver/tabletenv/logstats.go index 6fc131d74fa..a0e2e12a89a 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/logstats.go +++ b/go/vt/vttablet/tabletserver/tabletenv/logstats.go @@ -182,7 +182,7 @@ func (stats *LogStats) CallInfo() (string, string) { // Logf formats the log record to the given writer, either as // tab-separated list of logged fields or as JSON. func (stats *LogStats) Logf(w io.Writer, params url.Values) error { - if !streamlog.ShouldEmitLog(stats.OriginalSQL) { + if !streamlog.ShouldEmitLog(stats.OriginalSQL, uint64(stats.RowsAffected), uint64(len(stats.Rows))) { return nil }