diff --git a/plugin/storage/es/spanstore/fixtures/query_01.json b/plugin/storage/es/spanstore/fixtures/query_01.json index e38c1e84ac4..a433bb44062 100644 --- a/plugin/storage/es/spanstore/fixtures/query_01.json +++ b/plugin/storage/es/spanstore/fixtures/query_01.json @@ -4,9 +4,9 @@ { "bool":{ "must":{ - "match":{ + "regexp":{ "tag.bat@foo":{ - "query":"spook" + "value":"spook" } } } @@ -15,9 +15,9 @@ { "bool":{ "must":{ - "match":{ + "regexp":{ "process.tag.bat@foo":{ - "query":"spook" + "value":"spook" } } } @@ -37,9 +37,9 @@ } }, { - "match":{ + "regexp":{ "tags.value":{ - "query":"spook" + "value":"spook" } } } @@ -62,9 +62,9 @@ } }, { - "match":{ + "regexp":{ "process.tags.value":{ - "query":"spook" + "value":"spook" } } } @@ -87,9 +87,9 @@ } }, { - "match":{ + "regexp":{ "logs.fields.value":{ - "query":"spook" + "value":"spook" } } } diff --git a/plugin/storage/es/spanstore/fixtures/query_02.json b/plugin/storage/es/spanstore/fixtures/query_02.json new file mode 100644 index 00000000000..60d6516aef1 --- /dev/null +++ b/plugin/storage/es/spanstore/fixtures/query_02.json @@ -0,0 +1,103 @@ +{ + "bool":{ + "should":[ + { + "bool":{ + "must":{ + "regexp":{ + "tag.bat@foo":{ + "value":"spo.*" + } + } + } + } + }, + { + "bool":{ + "must":{ + "regexp":{ + "process.tag.bat@foo":{ + "value":"spo.*" + } + } + } + } + }, + { + "nested":{ + "path":"tags", + "query":{ + "bool":{ + "must":[ + { + "match":{ + "tags.key":{ + "query":"bat.foo" + } + } + }, + { + "regexp":{ + "tags.value":{ + "value":"spo.*" + } + } + } + ] + } + } + } + }, + { + "nested":{ + "path":"process.tags", + "query":{ + "bool":{ + "must":[ + { + "match":{ + "process.tags.key":{ + "query":"bat.foo" + } + } + }, + { + "regexp":{ + "process.tags.value":{ + "value":"spo.*" + } + } + } + ] + } + } + } + }, + { + "nested":{ + "path":"logs.fields", + "query":{ + "bool":{ + "must":[ + { + "match":{ + "logs.fields.key":{ + "query":"bat.foo" + } + } + }, + { + "regexp":{ + "logs.fields.value":{ + "value":"spo.*" + } + } + } + ] + } + } + } + } + ] + } +} diff --git a/plugin/storage/es/spanstore/fixtures/query_03.json b/plugin/storage/es/spanstore/fixtures/query_03.json new file mode 100644 index 00000000000..05da6e4a0b6 --- /dev/null +++ b/plugin/storage/es/spanstore/fixtures/query_03.json @@ -0,0 +1,103 @@ +{ + "bool":{ + "should":[ + { + "bool":{ + "must":{ + "regexp":{ + "tag.bat@foo":{ + "value":"spo\\*" + } + } + } + } + }, + { + "bool":{ + "must":{ + "regexp":{ + "process.tag.bat@foo":{ + "value":"spo\\*" + } + } + } + } + }, + { + "nested":{ + "path":"tags", + "query":{ + "bool":{ + "must":[ + { + "match":{ + "tags.key":{ + "query":"bat.foo" + } + } + }, + { + "regexp":{ + "tags.value":{ + "value":"spo\\*" + } + } + } + ] + } + } + } + }, + { + "nested":{ + "path":"process.tags", + "query":{ + "bool":{ + "must":[ + { + "match":{ + "process.tags.key":{ + "query":"bat.foo" + } + } + }, + { + "regexp":{ + "process.tags.value":{ + "value":"spo\\*" + } + } + } + ] + } + } + } + }, + { + "nested":{ + "path":"logs.fields", + "query":{ + "bool":{ + "must":[ + { + "match":{ + "logs.fields.key":{ + "query":"bat.foo" + } + } + }, + { + "regexp":{ + "logs.fields.value":{ + "value":"spo\\*" + } + } + } + ] + } + } + } + } + ] + } +} diff --git a/plugin/storage/es/spanstore/reader.go b/plugin/storage/es/spanstore/reader.go index 29124eda749..046ad192068 100644 --- a/plugin/storage/es/spanstore/reader.go +++ b/plugin/storage/es/spanstore/reader.go @@ -622,14 +622,14 @@ func (s *SpanReader) buildNestedQuery(field string, k string, v string) elastic. keyField := fmt.Sprintf("%s.%s", field, tagKeyField) valueField := fmt.Sprintf("%s.%s", field, tagValueField) keyQuery := elastic.NewMatchQuery(keyField, k) - valueQuery := elastic.NewMatchQuery(valueField, v) + valueQuery := elastic.NewRegexpQuery(valueField, v) tagBoolQuery := elastic.NewBoolQuery().Must(keyQuery, valueQuery) return elastic.NewNestedQuery(field, tagBoolQuery) } func (s *SpanReader) buildObjectQuery(field string, k string, v string) elastic.Query { keyField := fmt.Sprintf("%s.%s", field, k) - keyQuery := elastic.NewMatchQuery(keyField, v) + keyQuery := elastic.NewRegexpQuery(keyField, v) return elastic.NewBoolQuery().Must(keyQuery) } diff --git a/plugin/storage/es/spanstore/reader_test.go b/plugin/storage/es/spanstore/reader_test.go index 62f7e63eb58..b2431dd3a31 100644 --- a/plugin/storage/es/spanstore/reader_test.go +++ b/plugin/storage/es/spanstore/reader_test.go @@ -1029,6 +1029,36 @@ func TestSpanReader_buildTagQuery(t *testing.T) { }) } +func TestSpanReader_buildTagRegexQuery(t *testing.T) { + inStr, err := ioutil.ReadFile("fixtures/query_02.json") + require.NoError(t, err) + withSpanReader(func(r *spanReaderTest) { + tagQuery := r.reader.buildTagQuery("bat.foo", "spo.*") + actual, err := tagQuery.Source() + require.NoError(t, err) + + expected := make(map[string]interface{}) + json.Unmarshal(inStr, &expected) + + assert.EqualValues(t, expected, actual) + }) +} + +func TestSpanReader_buildTagRegexEscapedQuery(t *testing.T) { + inStr, err := ioutil.ReadFile("fixtures/query_03.json") + require.NoError(t, err) + withSpanReader(func(r *spanReaderTest) { + tagQuery := r.reader.buildTagQuery("bat.foo", "spo\\*") + actual, err := tagQuery.Source() + require.NoError(t, err) + + expected := make(map[string]interface{}) + json.Unmarshal(inStr, &expected) + + assert.EqualValues(t, expected, actual) + }) +} + func TestSpanReader_GetEmptyIndex(t *testing.T) { withSpanReader(func(r *spanReaderTest) { mockSearchService(r). diff --git a/plugin/storage/integration/elasticsearch_test.go b/plugin/storage/integration/elasticsearch_test.go index de138819330..a5d9ad39a58 100644 --- a/plugin/storage/integration/elasticsearch_test.go +++ b/plugin/storage/integration/elasticsearch_test.go @@ -171,6 +171,8 @@ func testElasticsearchStorage(t *testing.T, allTagsAsFields, archive bool) { s := &ESStorageIntegration{} require.NoError(t, s.initializeES(allTagsAsFields, archive)) + s.Fixtures = loadAndParseQueryTestCases(t, "fixtures/queries_es.json") + if archive { t.Run("ArchiveTrace", s.testArchiveTrace) } else { diff --git a/plugin/storage/integration/fixtures/queries_es.json b/plugin/storage/integration/fixtures/queries_es.json new file mode 100644 index 00000000000..810c912b5db --- /dev/null +++ b/plugin/storage/integration/fixtures/queries_es.json @@ -0,0 +1,34 @@ +[ + { + "Caption": "Tag escaped operator + Operation name + max Duration", + "Query": { + "ServiceName": "query23-service", + "OperationName": "query23-operation", + "Tags": { + "sameplacetag1":"same\\*" + }, + "StartTimeMin": "2017-01-26T15:46:31.639875Z", + "StartTimeMax": "2017-01-26T17:46:31.639875Z", + "DurationMin": 0, + "DurationMax": 1000, + "NumTraces": 1000 + }, + "ExpectedFixtures": ["tags_escaped_operator_trace_1"] + }, + { + "Caption": "Tag wildcard regex", + "Query": { + "ServiceName": "query24-service", + "OperationName": "", + "Tags": { + "sameplacetag1":"same.*" + }, + "StartTimeMin": "2017-01-26T15:46:31.639875Z", + "StartTimeMax": "2017-01-26T17:46:31.639875Z", + "DurationMin": 0, + "DurationMax": 0, + "NumTraces": 1000 + }, + "ExpectedFixtures": ["tags_wildcard_regex_1", "tags_wildcard_regex_2"] + } +] \ No newline at end of file diff --git a/plugin/storage/integration/fixtures/traces/tags_escaped_operator_trace_1.json b/plugin/storage/integration/fixtures/traces/tags_escaped_operator_trace_1.json new file mode 100644 index 00000000000..6079e9beea5 --- /dev/null +++ b/plugin/storage/integration/fixtures/traces/tags_escaped_operator_trace_1.json @@ -0,0 +1,33 @@ +{ + "spans": [ + { + "traceId": "AAAAAAAAAAAAAAAAAAAFEh==", + "spanId": "AAAAAAAAAAU=", + "operationName": "query23-operation", + "references": [], + "startTime": "2017-01-26T16:46:31.639875Z", + "duration": "1000ns", + "tags": [ + { + "key": "sameplacetag1", + "vType": "STRING", + "vStr": "same*" + } + ], + "process": { + "serviceName": "query23-service", + "tags": [] + }, + "logs": [ + { + "timestamp": "2017-01-26T16:46:31.639875Z", + "fields": [] + }, + { + "timestamp": "2017-01-26T16:46:31.639875Z", + "fields": [] + } + ] + } + ] +} diff --git a/plugin/storage/integration/fixtures/traces/tags_escaped_operator_trace_2.json b/plugin/storage/integration/fixtures/traces/tags_escaped_operator_trace_2.json new file mode 100644 index 00000000000..2a0da0ddc5a --- /dev/null +++ b/plugin/storage/integration/fixtures/traces/tags_escaped_operator_trace_2.json @@ -0,0 +1,33 @@ +{ + "spans": [ + { + "traceId": "AAAAAAAAAAAAAAAAAABZEh==", + "spanId": "AAAAAAAAAAU=", + "operationName": "query23-operation", + "references": [], + "startTime": "2017-01-26T16:46:31.639875Z", + "duration": "1000ns", + "tags": [ + { + "key": "sameplacetag1", + "vType": "STRING", + "vStr": "sameplacedifferentvalue" + } + ], + "process": { + "serviceName": "query23-service", + "tags": [] + }, + "logs": [ + { + "timestamp": "2017-01-26T16:46:31.639875Z", + "fields": [] + }, + { + "timestamp": "2017-01-26T16:46:31.639875Z", + "fields": [] + } + ] + } + ] +} diff --git a/plugin/storage/integration/fixtures/traces/tags_wildcard_regex_1.json b/plugin/storage/integration/fixtures/traces/tags_wildcard_regex_1.json new file mode 100644 index 00000000000..9eaa2731950 --- /dev/null +++ b/plugin/storage/integration/fixtures/traces/tags_wildcard_regex_1.json @@ -0,0 +1,25 @@ +{ + "spans": [ + { + "traceId": "AAAAAAAAAAAAAAAAAAAKEg==", + "spanId": "AAAAAAAAAAQ=", + "operationName": "", + "references": [ + ], + "tags": [ + { + "key": "sameplacetag1", + "vType": "STRING", + "vStr": "sameplacevalue1" + } + ], + "startTime": "2017-01-26T16:46:31.639875Z", + "duration": "2000ns", + "process": { + "serviceName": "query24-service", + "tags": [] + }, + "logs": [] + } + ] +} diff --git a/plugin/storage/integration/fixtures/traces/tags_wildcard_regex_2.json b/plugin/storage/integration/fixtures/traces/tags_wildcard_regex_2.json new file mode 100644 index 00000000000..887978c6c4c --- /dev/null +++ b/plugin/storage/integration/fixtures/traces/tags_wildcard_regex_2.json @@ -0,0 +1,25 @@ +{ + "spans": [ + { + "traceId": "AAAAAAAAAAAAAAAAAAASEg==", + "spanId": "AAAAAAAAAAQ=", + "operationName": "", + "references": [ + ], + "tags": [ + { + "key": "sameplacetag1", + "vType": "STRING", + "vStr": "sameplacevalue2" + } + ], + "startTime": "2017-01-26T16:46:31.639875Z", + "duration": "2000ns", + "process": { + "serviceName": "query24-service", + "tags": [] + }, + "logs": [] + } + ] +} diff --git a/plugin/storage/integration/integration_test.go b/plugin/storage/integration/integration_test.go index 3fa9646e4b2..03a0c7d29f5 100644 --- a/plugin/storage/integration/integration_test.go +++ b/plugin/storage/integration/integration_test.go @@ -61,6 +61,7 @@ type StorageIntegration struct { SpanReader spanstore.Reader DependencyWriter dependencystore.Writer DependencyReader dependencystore.Reader + Fixtures []*QueryFixtures // TODO: remove this flag after all storage plugins returns spanKind with operationNames notSupportSpanKindWithOperation bool @@ -208,13 +209,13 @@ func (s *StorageIntegration) testFindTraces(t *testing.T) { defer s.cleanUp(t) // Note: all cases include ServiceName + StartTime range - queryTestCases := loadAndParseQueryTestCases(t) + s.Fixtures = append(s.Fixtures, loadAndParseQueryTestCases(t, "fixtures/queries.json")...) // Each query test case only specifies matching traces, but does not provide counterexamples. // To improve coverage we get all possible traces and store all of them before running queries. allTraceFixtures := make(map[string]*model.Trace) - expectedTracesPerTestCase := make([][]*model.Trace, 0, len(queryTestCases)) - for _, queryTestCase := range queryTestCases { + expectedTracesPerTestCase := make([][]*model.Trace, 0, len(s.Fixtures)) + for _, queryTestCase := range s.Fixtures { var expected []*model.Trace for _, traceFixture := range queryTestCase.ExpectedFixtures { trace, ok := allTraceFixtures[traceFixture] @@ -229,7 +230,7 @@ func (s *StorageIntegration) testFindTraces(t *testing.T) { expectedTracesPerTestCase = append(expectedTracesPerTestCase, expected) } s.refresh(t) - for i, queryTestCase := range queryTestCases { + for i, queryTestCase := range s.Fixtures { t.Run(queryTestCase.Caption, func(t *testing.T) { expected := expectedTracesPerTestCase[i] actual := s.findTracesByQuery(t, queryTestCase.Query, expected) @@ -304,9 +305,9 @@ func loadAndParseJSONPB(t *testing.T, path string, object proto.Message) { require.NoError(t, err, "Not expecting error when unmarshaling fixture %s", path) } -func loadAndParseQueryTestCases(t *testing.T) []*QueryFixtures { +func loadAndParseQueryTestCases(t *testing.T, queriesFile string) []*QueryFixtures { var queries []*QueryFixtures - loadAndParseJSON(t, "fixtures/queries.json", &queries) + loadAndParseJSON(t, queriesFile, &queries) return queries }