diff --git a/cmd/anonymizer/app/uiconv/extractor.go b/cmd/anonymizer/app/uiconv/extractor.go new file mode 100644 index 00000000000..8679766318f --- /dev/null +++ b/cmd/anonymizer/app/uiconv/extractor.go @@ -0,0 +1,94 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package uiconv + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "go.uber.org/zap" + + uimodel "github.com/jaegertracing/jaeger/model/json" +) + +// Extractor reads the spans from reader, filters by traceID, and stores as JSON into uiFile. +type Extractor struct { + uiFile *os.File + traceID string + reader *Reader + logger *zap.Logger +} + +// NewExtractor creates Extractor. +func NewExtractor(uiFile string, traceID string, reader *Reader, logger *zap.Logger) (*Extractor, error) { + f, err := os.OpenFile(uiFile, os.O_CREATE|os.O_WRONLY, os.ModePerm) + if err != nil { + return nil, fmt.Errorf("cannot create output file: %w", err) + } + logger.Sugar().Infof("Writing spans to UI file %s", uiFile) + + return &Extractor{ + uiFile: f, + traceID: traceID, + reader: reader, + logger: logger, + }, nil +} + +// Run executes the extraction. +func (e *Extractor) Run() error { + e.logger.Info("Parsing captured file for trace", zap.String("trace_id", e.traceID)) + + var ( + spans []uimodel.Span + span *uimodel.Span + err error + ) + for span, err = e.reader.NextSpan(); err == nil; span, err = e.reader.NextSpan() { + if string(span.TraceID) == e.traceID { + spans = append(spans, *span) + } + } + if err != io.EOF { + return fmt.Errorf("failed when scanning the file: %w", err) + } + trace := uimodel.Trace{ + TraceID: uimodel.TraceID(e.traceID), + Spans: spans, + Processes: make(map[uimodel.ProcessID]uimodel.Process), + } + // (ys) The following is not exactly correct because it does not dedupe the processes, + // but I don't think it affects the UI. + for i := range spans { + span := &spans[i] + pid := uimodel.ProcessID(fmt.Sprintf("p%d", i)) + trace.Processes[pid] = *span.Process + span.Process = nil + span.ProcessID = pid + } + jsonBytes, err := json.Marshal(trace) + if err != nil { + return fmt.Errorf("failed to marshal UI trace: %w", err) + } + e.uiFile.Write([]byte(`{"data": [`)) + e.uiFile.Write(jsonBytes) + e.uiFile.Write([]byte(`]}`)) + e.uiFile.Sync() + e.uiFile.Close() + e.logger.Sugar().Infof("Wrote spans to UI file %s", e.uiFile.Name()) + return nil +} diff --git a/cmd/anonymizer/app/uiconv/extractor_test.go b/cmd/anonymizer/app/uiconv/extractor_test.go new file mode 100644 index 00000000000..369ac11caba --- /dev/null +++ b/cmd/anonymizer/app/uiconv/extractor_test.go @@ -0,0 +1,118 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package uiconv + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/go-openapi/swag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/jaegertracing/jaeger/model" +) + +type UITrace struct { + Data []model.Trace +} + +func TestExtractor_TraceSuccess(t *testing.T) { + inputFile := "fixtures/trace_success.json" + outputFile := "fixtures/trace_success_ui_anonymized.json" + defer os.Remove(outputFile) + + reader, err := NewReader( + inputFile, + zap.NewNop(), + ) + require.NoError(t, err) + + extractor, err := NewExtractor( + outputFile, + "2be38093ead7a083", + reader, + zap.NewNop(), + ) + require.NoError(t, err) + + err = extractor.Run() + require.NoError(t, err) + + var trace UITrace + loadJSON(t, outputFile, &trace) + + for i := range trace.Data { + for j := range trace.Data[i].Spans { + assert.Equal(t, "span.kind", trace.Data[i].Spans[j].Tags[0].Key) + } + } +} + +func TestExtractor_TraceOutputFileError(t *testing.T) { + inputFile := "fixtures/trace_success.json" + outputFile := "fixtures/trace_success_ui_anonymized.json" + defer os.Remove(outputFile) + + reader, err := NewReader( + inputFile, + zap.NewNop(), + ) + require.NoError(t, err) + + err = os.Chmod("fixtures", 0000) + require.NoError(t, err) + defer os.Chmod("fixtures", 0755) + + _, err = NewExtractor( + outputFile, + "2be38093ead7a083", + reader, + zap.NewNop(), + ) + require.Contains(t, err.Error(), "cannot create output file") +} + +func TestExtractor_TraceScanError(t *testing.T) { + inputFile := "fixtures/trace_scan_error.json" + outputFile := "fixtures/trace_scan_error_ui_anonymized.json" + defer os.Remove(outputFile) + + reader, err := NewReader( + inputFile, + zap.NewNop(), + ) + require.NoError(t, err) + + extractor, err := NewExtractor( + outputFile, + "2be38093ead7a083", + reader, + zap.NewNop(), + ) + require.NoError(t, err) + + err = extractor.Run() + require.Contains(t, err.Error(), "failed when scanning the file") +} + +func loadJSON(t *testing.T, fileName string, i interface{}) { + b, err := ioutil.ReadFile(fileName) + require.NoError(t, err) + err = swag.ReadJSON(b, i) + require.NoError(t, err, "Failed to parse json fixture file %s", fileName) +} diff --git a/cmd/anonymizer/app/uiconv/fixtures/trace_empty.json b/cmd/anonymizer/app/uiconv/fixtures/trace_empty.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cmd/anonymizer/app/uiconv/fixtures/trace_invalid_json.json b/cmd/anonymizer/app/uiconv/fixtures/trace_invalid_json.json new file mode 100644 index 00000000000..cc2337d97a1 --- /dev/null +++ b/cmd/anonymizer/app/uiconv/fixtures/trace_invalid_json.json @@ -0,0 +1,2 @@ +[{"traceID":"2be38093ead7a083","spanID":"7bd66f09ba90ea3d","duration": "invalid"} +] \ No newline at end of file diff --git a/cmd/anonymizer/app/uiconv/fixtures/trace_scan_error.json b/cmd/anonymizer/app/uiconv/fixtures/trace_scan_error.json new file mode 100644 index 00000000000..963a379ab25 --- /dev/null +++ b/cmd/anonymizer/app/uiconv/fixtures/trace_scan_error.json @@ -0,0 +1,2 @@ +[{"traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34","duration":267037}, +] \ No newline at end of file diff --git a/cmd/anonymizer/app/uiconv/fixtures/trace_success.json b/cmd/anonymizer/app/uiconv/fixtures/trace_success.json new file mode 100644 index 00000000000..7caed81fe70 --- /dev/null +++ b/cmd/anonymizer/app/uiconv/fixtures/trace_success.json @@ -0,0 +1,3 @@ +[{"traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34","flags":1,"operationName":"a071653098f9250d","references":[{"refType":"CHILD_OF","traceID":"2be38093ead7a083","spanID":"492770a15935810f"}],"startTime":1605223981761425,"duration":267037,"tags":[{"key":"span.kind","type":"string","value":"server"}],"logs":[],"process":{"serviceName":"16af988c443cff37","tags":[]},"warnings":null}, + {"traceID":"2be38093ead7a083","spanID":"7bd66f09ba90ea3d","flags":1,"operationName":"471418097747d04a","references":[{"refType":"CHILD_OF","traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34"}],"startTime":1605223981965074,"duration":32782,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"error","type":"bool","value":"true"}],"logs":[],"process":{"serviceName":"3c220036602f839e","tags":[]},"warnings":null} +] \ No newline at end of file diff --git a/cmd/anonymizer/app/uiconv/fixtures/trace_wrong_format.json b/cmd/anonymizer/app/uiconv/fixtures/trace_wrong_format.json new file mode 100644 index 00000000000..672ae56f024 --- /dev/null +++ b/cmd/anonymizer/app/uiconv/fixtures/trace_wrong_format.json @@ -0,0 +1 @@ +{"traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34","duration":267037} diff --git a/cmd/anonymizer/app/uiconv/module.go b/cmd/anonymizer/app/uiconv/module.go new file mode 100644 index 00000000000..bd19620dd0c --- /dev/null +++ b/cmd/anonymizer/app/uiconv/module.go @@ -0,0 +1,40 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package uiconv + +import ( + "go.uber.org/zap" +) + +// Config for the extractor. +type Config struct { + CapturedFile string `yaml:"captured_file"` + UIFile string `yaml:"ui_file"` + TraceID string `yaml:"trace_id"` +} + +// Extract reads anonymized file, finds spans for a given trace, +// and writes out that trace in the UI format. +func Extract(config Config, logger *zap.Logger) error { + reader, err := NewReader(config.CapturedFile, logger) + if err != nil { + return err + } + ext, err := NewExtractor(config.UIFile, config.TraceID, reader, logger) + if err != nil { + return err + } + return ext.Run() +} diff --git a/cmd/anonymizer/app/uiconv/module_test.go b/cmd/anonymizer/app/uiconv/module_test.go new file mode 100644 index 00000000000..cdaee6c9fb1 --- /dev/null +++ b/cmd/anonymizer/app/uiconv/module_test.go @@ -0,0 +1,80 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package uiconv + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func TestModule_TraceSuccess(t *testing.T) { + inputFile := "fixtures/trace_success.json" + outputFile := "fixtures/trace_success_ui_anonymized.json" + defer os.Remove(outputFile) + + config := Config{ + CapturedFile: inputFile, + UIFile: outputFile, + TraceID: "2be38093ead7a083", + } + err := Extract(config, zap.NewNop()) + require.NoError(t, err) + + var trace UITrace + loadJSON(t, outputFile, &trace) + + for i := range trace.Data { + for j := range trace.Data[i].Spans { + assert.Equal(t, "span.kind", trace.Data[i].Spans[j].Tags[0].Key) + } + } +} + +func TestModule_TraceNonExistent(t *testing.T) { + inputFile := "fixtures/trace_non_existent.json" + outputFile := "fixtures/trace_non_existent_ui_anonymized.json" + defer os.Remove(outputFile) + + config := Config{ + CapturedFile: inputFile, + UIFile: outputFile, + TraceID: "2be38093ead7a083", + } + err := Extract(config, zap.NewNop()) + require.Contains(t, err.Error(), "cannot open captured file") +} + +func TestModule_TraceOutputFileError(t *testing.T) { + inputFile := "fixtures/trace_success.json" + outputFile := "fixtures/trace_success_ui_anonymized.json" + defer os.Remove(outputFile) + + config := Config{ + CapturedFile: inputFile, + UIFile: outputFile, + TraceID: "2be38093ead7a083", + } + + err := os.Chmod("fixtures", 0550) + require.NoError(t, err) + defer os.Chmod("fixtures", 0755) + + err = Extract(config, zap.NewNop()) + require.Contains(t, err.Error(), "cannot create output file") +} diff --git a/cmd/anonymizer/app/uiconv/reader.go b/cmd/anonymizer/app/uiconv/reader.go new file mode 100644 index 00000000000..ee26edb5093 --- /dev/null +++ b/cmd/anonymizer/app/uiconv/reader.go @@ -0,0 +1,90 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package uiconv + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "os" + + "go.uber.org/zap" + + uimodel "github.com/jaegertracing/jaeger/model/json" +) + +// Reader loads previously captured spans from a file. +type Reader struct { + logger *zap.Logger + capturedFile *os.File + reader *bufio.Reader + spansRead int + eofReached bool +} + +// NewReader creates a Reader. +func NewReader(capturedFile string, logger *zap.Logger) (*Reader, error) { + cf, err := os.OpenFile(capturedFile, os.O_RDONLY, os.ModePerm) + if err != nil { + return nil, fmt.Errorf("cannot open captured file: %w", err) + } + logger.Sugar().Infof("Reading captured spans from file %s", capturedFile) + + return &Reader{ + logger: logger, + capturedFile: cf, + reader: bufio.NewReader(cf), + }, nil +} + +// NextSpan reads the next span from the capture file, or returns io.EOF. +func (r *Reader) NextSpan() (*uimodel.Span, error) { + if r.eofReached { + return nil, io.EOF + } + if r.spansRead == 0 { + b, err := r.reader.ReadByte() + if err != nil { + r.eofReached = true + return nil, fmt.Errorf("cannot read file: %w", err) + } + if b != '[' { + r.eofReached = true + return nil, fmt.Errorf("file must begin with '['") + } + } + s, err := r.reader.ReadString('\n') + if err != nil { + r.eofReached = true + return nil, fmt.Errorf("cannot read file: %w", err) + } + if s[len(s)-2] == ',' { // all but last span lines end with ,\n + s = s[0 : len(s)-2] + } else { + r.eofReached = true + } + var span uimodel.Span + err = json.Unmarshal([]byte(s), &span) + if err != nil { + r.eofReached = true + return nil, fmt.Errorf("cannot unmarshal span: %w; %s", err, s) + } + r.spansRead++ + if r.spansRead%1000 == 0 { + r.logger.Info("Scan progress", zap.Int("span_count", r.spansRead)) + } + return &span, nil +} diff --git a/cmd/anonymizer/app/uiconv/reader_test.go b/cmd/anonymizer/app/uiconv/reader_test.go new file mode 100644 index 00000000000..c9e230c876c --- /dev/null +++ b/cmd/anonymizer/app/uiconv/reader_test.go @@ -0,0 +1,103 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package uiconv + +import ( + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func TestReader_TraceSuccess(t *testing.T) { + inputFile := "fixtures/trace_success.json" + r, err := NewReader( + inputFile, + zap.NewNop(), + ) + require.NoError(t, err) + + s1, err := r.NextSpan() + require.NoError(t, err) + assert.Equal(t, "a071653098f9250d", s1.OperationName) + assert.Equal(t, 1, r.spansRead) + assert.Equal(t, false, r.eofReached) + + r.spansRead = 999 + + s2, err := r.NextSpan() + require.NoError(t, err) + assert.Equal(t, "471418097747d04a", s2.OperationName) + assert.Equal(t, 1000, r.spansRead) + assert.Equal(t, true, r.eofReached) + + _, err = r.NextSpan() + require.Equal(t, io.EOF, err) + assert.Equal(t, 1000, r.spansRead) + assert.Equal(t, true, r.eofReached) +} + +func TestReader_TraceNonExistent(t *testing.T) { + inputFile := "fixtures/trace_non_existent.json" + _, err := NewReader( + inputFile, + zap.NewNop(), + ) + require.Contains(t, err.Error(), "cannot open captured file") +} + +func TestReader_TraceEmpty(t *testing.T) { + inputFile := "fixtures/trace_empty.json" + r, err := NewReader( + inputFile, + zap.NewNop(), + ) + require.NoError(t, err) + + _, err = r.NextSpan() + require.Contains(t, err.Error(), "cannot read file") + assert.Equal(t, 0, r.spansRead) + assert.Equal(t, true, r.eofReached) +} + +func TestReader_TraceWrongFormat(t *testing.T) { + inputFile := "fixtures/trace_wrong_format.json" + r, err := NewReader( + inputFile, + zap.NewNop(), + ) + require.NoError(t, err) + + _, err = r.NextSpan() + require.Equal(t, "file must begin with '['", err.Error()) + assert.Equal(t, 0, r.spansRead) + assert.Equal(t, true, r.eofReached) +} + +func TestReader_TraceInvalidJson(t *testing.T) { + inputFile := "fixtures/trace_invalid_json.json" + r, err := NewReader( + inputFile, + zap.NewNop(), + ) + require.NoError(t, err) + + _, err = r.NextSpan() + require.Contains(t, err.Error(), "cannot unmarshal span") + assert.Equal(t, 0, r.spansRead) + assert.Equal(t, true, r.eofReached) +} diff --git a/cmd/anonymizer/app/writer/writer.go b/cmd/anonymizer/app/writer/writer.go index 5d04e61df53..dee49e1735d 100644 --- a/cmd/anonymizer/app/writer/writer.go +++ b/cmd/anonymizer/app/writer/writer.go @@ -127,7 +127,7 @@ func (w *Writer) WriteSpan(msg *model.Span) error { w.logger.Info("progress", zap.Int("numSpans", w.spanCount)) } - if w.spanCount >= w.config.MaxSpansCount { + if w.config.MaxSpansCount > 0 && w.spanCount >= w.config.MaxSpansCount { w.logger.Info("Saved enough spans, exiting...") w.Close() os.Exit(0) diff --git a/cmd/anonymizer/main.go b/cmd/anonymizer/main.go index a4dd00e4a9f..d80d609adaa 100644 --- a/cmd/anonymizer/main.go +++ b/cmd/anonymizer/main.go @@ -21,10 +21,11 @@ import ( "github.com/spf13/cobra" "go.uber.org/zap" - app "github.com/jaegertracing/jaeger/cmd/anonymizer/app" + "github.com/jaegertracing/jaeger/cmd/anonymizer/app" "github.com/jaegertracing/jaeger/cmd/anonymizer/app/anonymizer" - query "github.com/jaegertracing/jaeger/cmd/anonymizer/app/query" - writer "github.com/jaegertracing/jaeger/cmd/anonymizer/app/writer" + "github.com/jaegertracing/jaeger/cmd/anonymizer/app/query" + "github.com/jaegertracing/jaeger/cmd/anonymizer/app/uiconv" + "github.com/jaegertracing/jaeger/cmd/anonymizer/app/writer" "github.com/jaegertracing/jaeger/pkg/version" ) @@ -41,9 +42,9 @@ func main() { prefix := options.OutputDir + "/" + options.TraceID conf := writer.Config{ MaxSpansCount: options.MaxSpansCount, - CapturedFile: prefix + ".original", - AnonymizedFile: prefix + ".anonymized", - MappingFile: prefix + ".mapping", + CapturedFile: prefix + ".original.json", + AnonymizedFile: prefix + ".anonymized.json", + MappingFile: prefix + ".mapping.json", AnonymizerOpts: anonymizer.Options{ HashStandardTags: options.HashStandardTags, HashCustomTags: options.HashCustomTags, @@ -71,6 +72,16 @@ func main() { writer.WriteSpan(&span) } writer.Close() + + uiCfg := uiconv.Config{ + CapturedFile: conf.AnonymizedFile, + UIFile: prefix + ".anonymized-ui-trace.json", + TraceID: options.TraceID, + } + if err := uiconv.Extract(uiCfg, logger); err != nil { + logger.Fatal("error while extracing UI trace", zap.Error(err)) + } + logger.Sugar().Infof("Wrote UI-compatible anonymized file to %s", uiCfg.UIFile) }, }