From 160869ceb3a218bb47fab836c671f6a6989abbda Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Thu, 20 Jun 2019 18:57:45 +0200 Subject: [PATCH 1/3] Use json number when unmarshalling data from ES Signed-off-by: Pavol Loffay --- .../storage/es/spanstore/dbmodel/to_domain.go | 20 ++++++++++++----- .../es/spanstore/dbmodel/to_domain_test.go | 22 ++++++++++++++----- plugin/storage/es/spanstore/reader.go | 6 ++++- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/plugin/storage/es/spanstore/dbmodel/to_domain.go b/plugin/storage/es/spanstore/dbmodel/to_domain.go index 83d75efd8ee..5f2c775bd83 100644 --- a/plugin/storage/es/spanstore/dbmodel/to_domain.go +++ b/plugin/storage/es/spanstore/dbmodel/to_domain.go @@ -16,6 +16,7 @@ package dbmodel import ( "encoding/hex" + "encoding/json" "fmt" "strconv" "strings" @@ -161,12 +162,9 @@ func (td ToDomain) convertTagFields(tagsMap map[string]interface{}) ([]model.Key func (td ToDomain) convertTagField(k string, v interface{}) (model.KeyValue, error) { dKey := td.ReplaceDotReplacement(k) - // The number is always a float64 therefore type assertion on int (v.(int/64/32)) does not work. - // If 1.0, 2.0.. was stored as float it will be read as int - if pInt, err := strconv.ParseInt(fmt.Sprintf("%v", v), 10, 64); err == nil { - return model.Int64(dKey, pInt), nil - } switch val := v.(type) { + case int64: + return model.Int64(dKey, val), nil case float64: return model.Float64(dKey, val), nil case bool: @@ -176,6 +174,18 @@ func (td ToDomain) convertTagField(k string, v interface{}) (model.KeyValue, err // the binary is never returned, ES returns it as string with base64 encoding case []byte: return model.Binary(dKey, val), nil + // in spans are decoded using json.UseNumber() to preserve the type + // however note that float(1) will be parsed as int as ES does not store decimal point + case json.Number: + n, err := val.Int64() + if err == nil { + return model.Int64(dKey, n), nil + } + f, err := val.Float64() + if err == nil { + return model.Float64(dKey, f), nil + } + return model.String("", ""), err default: return model.String("", ""), fmt.Errorf("invalid tag type in %+v", v) } diff --git a/plugin/storage/es/spanstore/dbmodel/to_domain_test.go b/plugin/storage/es/spanstore/dbmodel/to_domain_test.go index d1578ea56d3..ace599a069d 100644 --- a/plugin/storage/es/spanstore/dbmodel/to_domain_test.go +++ b/plugin/storage/es/spanstore/dbmodel/to_domain_test.go @@ -319,10 +319,12 @@ func TestTagsMap(t *testing.T) { {fieldTags: map[string]interface{}{"int.int": int64(1)}, expected: []model.KeyValue{model.Int64("int.int", 1)}}, {fieldTags: map[string]interface{}{"int:int": int64(2)}, expected: []model.KeyValue{model.Int64("int.int", 2)}}, {fieldTags: map[string]interface{}{"float": float64(1.1)}, expected: []model.KeyValue{model.Float64("float", 1.1)}}, - // we are not able to reproduce type for float 123 or any N.0 number therefore returning int - {fieldTags: map[string]interface{}{"float": float64(123)}, expected: []model.KeyValue{model.Int64("float", 123)}}, - {fieldTags: map[string]interface{}{"float": float64(123.0)}, expected: []model.KeyValue{model.Int64("float", 123)}}, - {fieldTags: map[string]interface{}{"float:float": float64(123)}, expected: []model.KeyValue{model.Int64("float.float", 123)}}, + {fieldTags: map[string]interface{}{"float": float64(123)}, expected: []model.KeyValue{model.Float64("float", float64(123))}}, + {fieldTags: map[string]interface{}{"float": float64(123.0)}, expected: []model.KeyValue{model.Float64("float", float64(123.0))}}, + {fieldTags: map[string]interface{}{"float:float": float64(123)}, expected: []model.KeyValue{model.Float64("float.float", float64(123))}}, + {fieldTags: map[string]interface{}{"json_number:int": json.Number("123")}, expected: []model.KeyValue{model.Int64("json_number.int", 123)}}, + {fieldTags: map[string]interface{}{"json_number:float": json.Number("123.0")}, expected: []model.KeyValue{model.Float64("json_number.float", float64(123.0))}}, + {fieldTags: map[string]interface{}{"json_number:err": json.Number("foo")}, err: fmt.Errorf("strconv.ParseFloat: parsing \"foo\": invalid syntax")}, {fieldTags: map[string]interface{}{"str": "foo"}, expected: []model.KeyValue{model.String("str", "foo")}}, {fieldTags: map[string]interface{}{"str:str": "foo"}, expected: []model.KeyValue{model.String("str.str", "foo")}}, {fieldTags: map[string]interface{}{"binary": []byte("foo")}, expected: []model.KeyValue{model.Binary("binary", []byte("foo"))}}, @@ -333,8 +335,16 @@ func TestTagsMap(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("%d, %s", i, test.fieldTags), func(t *testing.T) { tags, err := converter.convertTagFields(test.fieldTags) - assert.Equal(t, test.expected, tags) - assert.Equal(t, test.err, err) + if err != nil { + fmt.Println(err.Error()) + } + if test.err != nil { + assert.Equal(t, test.err.Error(), err.Error()) + require.Nil(t, tags) + } else { + require.NoError(t, err) + assert.Equal(t, test.expected, tags) + } }) } } diff --git a/plugin/storage/es/spanstore/reader.go b/plugin/storage/es/spanstore/reader.go index 494c445383f..8cb6abca739 100644 --- a/plugin/storage/es/spanstore/reader.go +++ b/plugin/storage/es/spanstore/reader.go @@ -15,6 +15,7 @@ package spanstore import ( + "bytes" "context" "encoding/json" "fmt" @@ -218,7 +219,10 @@ func (s *SpanReader) unmarshalJSONSpan(esSpanRaw *elastic.SearchHit) (*dbmodel.S esSpanInByteArray := esSpanRaw.Source var jsonSpan dbmodel.Span - if err := json.Unmarshal(*esSpanInByteArray, &jsonSpan); err != nil { + + d := json.NewDecoder(bytes.NewReader(*esSpanInByteArray)) + d.UseNumber() + if err := d.Decode(&jsonSpan); err != nil { return nil, err } return &jsonSpan, nil From 2111713360093718919e6b38392587414c28be5b Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Fri, 21 Jun 2019 12:11:09 +0200 Subject: [PATCH 2/3] Wrap error Signed-off-by: Pavol Loffay --- plugin/storage/es/spanstore/dbmodel/to_domain.go | 3 ++- plugin/storage/es/spanstore/dbmodel/to_domain_test.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/plugin/storage/es/spanstore/dbmodel/to_domain.go b/plugin/storage/es/spanstore/dbmodel/to_domain.go index 5f2c775bd83..c15cdfdbf4d 100644 --- a/plugin/storage/es/spanstore/dbmodel/to_domain.go +++ b/plugin/storage/es/spanstore/dbmodel/to_domain.go @@ -18,6 +18,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "github.com/pkg/errors" "strconv" "strings" @@ -185,7 +186,7 @@ func (td ToDomain) convertTagField(k string, v interface{}) (model.KeyValue, err if err == nil { return model.Float64(dKey, f), nil } - return model.String("", ""), err + return model.String("", ""), errors.Wrapf(err, "invalid tag type in %+v", v) default: return model.String("", ""), fmt.Errorf("invalid tag type in %+v", v) } diff --git a/plugin/storage/es/spanstore/dbmodel/to_domain_test.go b/plugin/storage/es/spanstore/dbmodel/to_domain_test.go index ace599a069d..f723929edc2 100644 --- a/plugin/storage/es/spanstore/dbmodel/to_domain_test.go +++ b/plugin/storage/es/spanstore/dbmodel/to_domain_test.go @@ -324,7 +324,7 @@ func TestTagsMap(t *testing.T) { {fieldTags: map[string]interface{}{"float:float": float64(123)}, expected: []model.KeyValue{model.Float64("float.float", float64(123))}}, {fieldTags: map[string]interface{}{"json_number:int": json.Number("123")}, expected: []model.KeyValue{model.Int64("json_number.int", 123)}}, {fieldTags: map[string]interface{}{"json_number:float": json.Number("123.0")}, expected: []model.KeyValue{model.Float64("json_number.float", float64(123.0))}}, - {fieldTags: map[string]interface{}{"json_number:err": json.Number("foo")}, err: fmt.Errorf("strconv.ParseFloat: parsing \"foo\": invalid syntax")}, + {fieldTags: map[string]interface{}{"json_number:err": json.Number("foo")}, err: fmt.Errorf("invalid tag type in foo: strconv.ParseFloat: parsing \"foo\": invalid syntax")}, {fieldTags: map[string]interface{}{"str": "foo"}, expected: []model.KeyValue{model.String("str", "foo")}}, {fieldTags: map[string]interface{}{"str:str": "foo"}, expected: []model.KeyValue{model.String("str.str", "foo")}}, {fieldTags: map[string]interface{}{"binary": []byte("foo")}, expected: []model.KeyValue{model.Binary("binary", []byte("foo"))}}, From a11a7fe4a0d69679f1b0aa9ecf5ed99ecde7f0a7 Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Fri, 21 Jun 2019 14:21:38 +0200 Subject: [PATCH 3/3] fmt Signed-off-by: Pavol Loffay --- plugin/storage/es/spanstore/dbmodel/to_domain.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugin/storage/es/spanstore/dbmodel/to_domain.go b/plugin/storage/es/spanstore/dbmodel/to_domain.go index c15cdfdbf4d..01d3d4d08eb 100644 --- a/plugin/storage/es/spanstore/dbmodel/to_domain.go +++ b/plugin/storage/es/spanstore/dbmodel/to_domain.go @@ -18,10 +18,11 @@ import ( "encoding/hex" "encoding/json" "fmt" - "github.com/pkg/errors" "strconv" "strings" + "github.com/pkg/errors" + "github.com/jaegertracing/jaeger/model" )