diff --git a/internal/shared/attrdedup/dedup.go.tmpl b/internal/shared/attrnorm/dedup.go.tmpl similarity index 98% rename from internal/shared/attrdedup/dedup.go.tmpl rename to internal/shared/attrnorm/dedup.go.tmpl index c5ef89dd7af..fe9ddc96db7 100644 --- a/internal/shared/attrdedup/dedup.go.tmpl +++ b/internal/shared/attrnorm/dedup.go.tmpl @@ -2,10 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 // DO NOT MODIFY. Generated by gotmpl. -// source: internal/shared/attrdedup/dedup.go.tmpl +// source: internal/shared/attrnorm/dedup.go.tmpl -// Package attrdedup deduplicates attribute map values. -package attrdedup +// Package attrnorm normalizes attribute values. +package attrnorm import ( "reflect" diff --git a/internal/shared/attrdedup/dedup_test.go.tmpl b/internal/shared/attrnorm/dedup_test.go.tmpl similarity index 99% rename from internal/shared/attrdedup/dedup_test.go.tmpl rename to internal/shared/attrnorm/dedup_test.go.tmpl index 82fab043ab4..0e6b10b193a 100644 --- a/internal/shared/attrdedup/dedup_test.go.tmpl +++ b/internal/shared/attrnorm/dedup_test.go.tmpl @@ -2,9 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 // DO NOT MODIFY. Generated by gotmpl. -// source: internal/shared/attrdedup/dedup_test.go.tmpl +// source: internal/shared/attrnorm/dedup_test.go.tmpl -package attrdedup +package attrnorm import ( "testing" diff --git a/internal/shared/attrnorm/truncate.go.tmpl b/internal/shared/attrnorm/truncate.go.tmpl new file mode 100644 index 00000000000..9955e907268 --- /dev/null +++ b/internal/shared/attrnorm/truncate.go.tmpl @@ -0,0 +1,232 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// DO NOT MODIFY. Generated by gotmpl. +// source: internal/shared/attrnorm/truncate.go.tmpl + +package attrnorm + +import ( + "slices" + "strings" + "unicode/utf8" + + "go.opentelemetry.io/otel/attribute" +) + +// Truncate returns a truncated version of attr. Only string, string slice, +// byte slice, slice, and map attribute values are truncated. String values are +// truncated to at most a length of limit. Each string slice value is truncated +// in this fashion (the slice length itself is unaffected), and byte slice +// values are truncated to at most limit bytes. For slice and map attribute +// values, the limit is applied recursively to contained values. +// +// No truncation is performed for a negative limit. +func Truncate(limit int, attr attribute.KeyValue) attribute.KeyValue { + if limit < 0 { + return attr + } + switch attr.Value.Type() { + case attribute.STRING: + v := attr.Value.AsString() + return attr.Key.String(truncate(limit, v)) + case attribute.STRINGSLICE: + v := attr.Value.AsStringSlice() + for i := range v { + v[i] = truncate(limit, v[i]) + } + return attr.Key.StringSlice(v) + case attribute.BYTESLICE: + v := attr.Value.AsString() + if len(v) > limit { + return attr.Key.ByteSlice([]byte(v[:limit])) + } + return attr + case attribute.SLICE: + v := attr.Value.AsSlice() + if !slices.ContainsFunc(v, func(e attribute.Value) bool { return needsTruncation(limit, e) }) { + return attr + } + newV := make([]attribute.Value, len(v)) + for i, elem := range v { + newV[i] = TruncateValue(limit, elem) + } + return attr.Key.Slice(newV...) + case attribute.MAP: + v := attr.Value.AsMap() + if !slices.ContainsFunc(v, func(kv attribute.KeyValue) bool { return needsTruncation(limit, kv.Value) }) { + return attr + } + newV := make([]attribute.KeyValue, len(v)) + for i, elem := range v { + elem.Value = TruncateValue(limit, elem.Value) + newV[i] = elem + } + return attr.Key.Map(newV...) + } + return attr +} + +// TruncateValue returns a truncated version of v. Only string, string +// slice, byte slice, and (recursively) slice and map values are modified. +// +// No truncation is performed for a negative limit. +func TruncateValue(limit int, v attribute.Value) attribute.Value { + if limit < 0 { + return v + } + + switch v.Type() { + case attribute.STRING: + return attribute.StringValue(truncate(limit, v.AsString())) + case attribute.STRINGSLICE: + ss := v.AsStringSlice() + for i := range ss { + ss[i] = truncate(limit, ss[i]) + } + return attribute.StringSliceValue(ss) + case attribute.BYTESLICE: + // len(v.AsString()) is identical to len(v.AsByteSlice()) but + // avoids allocating the full slice before truncation. + s := v.AsString() + if limit >= 0 && len(s) > limit { + return attribute.ByteSliceValue([]byte(s[:limit])) + } + case attribute.SLICE: + sl := v.AsSlice() + if !slices.ContainsFunc(sl, func(e attribute.Value) bool { return needsTruncation(limit, e) }) { + return v + } + newSl := make([]attribute.Value, len(sl)) + for i, elem := range sl { + newSl[i] = TruncateValue(limit, elem) + } + return attribute.SliceValue(newSl...) + case attribute.MAP: + m := v.AsMap() + if !slices.ContainsFunc(m, func(kv attribute.KeyValue) bool { return needsTruncation(limit, kv.Value) }) { + return v + } + newM := make([]attribute.KeyValue, len(m)) + for i, elem := range m { + elem.Value = TruncateValue(limit, elem.Value) + newM[i] = elem + } + return attribute.MapValue(newM...) + } + return v +} + +// stringNeedsTruncation reports whether s would be modified by truncate for the +// given limit. +func stringNeedsTruncation(limit int, s string) bool { + if limit < 0 || len(s) <= limit { + return false + } + return utf8.RuneCountInString(s) > limit || !utf8.ValidString(s) +} + +// needsTruncation reports whether v would be modified by TruncateValue for the +// given limit. +func needsTruncation(limit int, v attribute.Value) bool { + switch v.Type() { + case attribute.STRING: + return stringNeedsTruncation(limit, v.AsString()) + case attribute.BYTESLICE: + // len(v.AsString()) is identical to len(v.AsByteSlice()) but + // avoids memory allocation. + if limit >= 0 && len(v.AsString()) > limit { + return true + } + case attribute.STRINGSLICE: + for _, s := range v.AsStringSlice() { + if stringNeedsTruncation(limit, s) { + return true + } + } + case attribute.SLICE: + return slices.ContainsFunc(v.AsSlice(), func(e attribute.Value) bool { return needsTruncation(limit, e) }) + case attribute.MAP: + return slices.ContainsFunc( + v.AsMap(), + func(kv attribute.KeyValue) bool { return needsTruncation(limit, kv.Value) }, + ) + } + return false +} + +// truncate returns a truncated version of s such that it contains less than +// the limit number of characters. Truncation is applied by returning the limit +// number of valid characters contained in s. +// +// If limit is negative, it returns the original string. +// +// UTF-8 is supported. When truncating, all invalid characters are dropped +// before applying truncation. +// +// If s already contains less than the limit number of bytes, it is returned +// unchanged. No invalid characters are removed. +func truncate(limit int, s string) string { + // This prioritize performance in the following order based on the most + // common expected use-cases. + // + // - Short values less than the default limit (128). + // - Strings with valid encodings that exceed the limit. + // - No limit. + // - Strings with invalid encodings that exceed the limit. + if limit < 0 || len(s) <= limit { + return s + } + + // Optimistically, assume all valid UTF-8. + var b strings.Builder + count := 0 + for i, c := range s { + if c != utf8.RuneError { + count++ + if count > limit { + return s[:i] + } + continue + } + + _, size := utf8.DecodeRuneInString(s[i:]) + if size == 1 { + // Invalid encoding. + b.Grow(len(s) - 1) + _, _ = b.WriteString(s[:i]) + s = s[i:] + break + } + } + + // Fast-path, no invalid input. + if b.Cap() == 0 { + return s + } + + // Truncate while validating UTF-8. + for i := 0; i < len(s) && count < limit; { + c := s[i] + if c < utf8.RuneSelf { + // Optimization for single byte runes (common case). + _ = b.WriteByte(c) + i++ + count++ + continue + } + + _, size := utf8.DecodeRuneInString(s[i:]) + if size == 1 { + // We checked for all 1-byte runes above, this is a RuneError. + i++ + continue + } + + _, _ = b.WriteString(s[i : i+size]) + i += size + count++ + } + + return b.String() +} diff --git a/internal/shared/attrnorm/truncate_test.go.tmpl b/internal/shared/attrnorm/truncate_test.go.tmpl new file mode 100644 index 00000000000..ad19bbbc09c --- /dev/null +++ b/internal/shared/attrnorm/truncate_test.go.tmpl @@ -0,0 +1,604 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// DO NOT MODIFY. Generated by gotmpl. +// source: internal/shared/attrnorm/truncate_test.go.tmpl + +package attrnorm + +import ( + "bytes" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/otel/attribute" +) + +func TestTruncateAttr(t *testing.T) { + const key = "key" + + strAttr := attribute.String(key, "value") + bytesAttr := attribute.ByteSlice(key, []byte("value")) + strSliceAttr := attribute.StringSlice(key, []string{"value-0", "value-1"}) + + tests := []struct { + limit int + attr, want attribute.KeyValue + }{ + { + limit: -1, + attr: strAttr, + want: strAttr, + }, + { + limit: -1, + attr: strSliceAttr, + want: strSliceAttr, + }, + { + limit: -1, + attr: bytesAttr, + want: bytesAttr, + }, + { + limit: 0, + attr: attribute.Bool(key, true), + want: attribute.Bool(key, true), + }, + { + limit: 0, + attr: attribute.BoolSlice(key, []bool{true, false}), + want: attribute.BoolSlice(key, []bool{true, false}), + }, + { + limit: 0, + attr: attribute.Int(key, 42), + want: attribute.Int(key, 42), + }, + { + limit: 0, + attr: attribute.IntSlice(key, []int{42, -1}), + want: attribute.IntSlice(key, []int{42, -1}), + }, + { + limit: 0, + attr: attribute.Int64(key, 42), + want: attribute.Int64(key, 42), + }, + { + limit: 0, + attr: attribute.Int64Slice(key, []int64{42, -1}), + want: attribute.Int64Slice(key, []int64{42, -1}), + }, + { + limit: 0, + attr: attribute.Float64(key, 42), + want: attribute.Float64(key, 42), + }, + { + limit: 0, + attr: attribute.Float64Slice(key, []float64{42, -1}), + want: attribute.Float64Slice(key, []float64{42, -1}), + }, + { + limit: 0, + attr: strAttr, + want: attribute.String(key, ""), + }, + { + limit: 0, + attr: strSliceAttr, + want: attribute.StringSlice(key, []string{"", ""}), + }, + { + limit: 0, + attr: attribute.Stringer(key, bytes.NewBufferString("value")), + want: attribute.String(key, ""), + }, + { + limit: 0, + attr: bytesAttr, + want: attribute.ByteSlice(key, []byte{}), + }, + { + limit: 1, + attr: strAttr, + want: attribute.String(key, "v"), + }, + { + limit: 1, + attr: strSliceAttr, + want: attribute.StringSlice(key, []string{"v", "v"}), + }, + { + limit: 1, + attr: bytesAttr, + want: attribute.ByteSlice(key, []byte("v")), + }, + { + limit: 5, + attr: strAttr, + want: strAttr, + }, + { + limit: 5, + attr: bytesAttr, + want: bytesAttr, + }, + { + limit: 7, + attr: strSliceAttr, + want: strSliceAttr, + }, + { + limit: 6, + attr: attribute.StringSlice(key, []string{"value", "value-1"}), + want: attribute.StringSlice(key, []string{"value", "value-"}), + }, + { + limit: 128, + attr: strAttr, + want: strAttr, + }, + { + limit: 128, + attr: strSliceAttr, + want: strSliceAttr, + }, + { + limit: 128, + attr: bytesAttr, + want: bytesAttr, + }, + { + // Multi-byte string: byte length (9) exceeds limit (5) but rune count (3) does not. + // Must not be truncated. + limit: 5, + attr: attribute.String(key, "日本語"), + want: attribute.String(key, "日本語"), + }, + { + // Multi-byte string: both byte length and rune count exceed limit. + // Must be truncated to limit runes. + limit: 2, + attr: attribute.String(key, "日本語"), + want: attribute.String(key, "日本"), + }, + { + // STRINGSLICE with multi-byte elements: byte lengths exceed limit but rune counts do not. + // Must not be truncated. + limit: 1, + attr: attribute.StringSlice(key, []string{"日", "本"}), + want: attribute.StringSlice(key, []string{"日", "本"}), + }, + // SLICE cases + { + limit: -1, + attr: attribute.Slice(key, attribute.StringValue("value")), + want: attribute.Slice(key, attribute.StringValue("value")), + }, + { + limit: 0, + attr: attribute.Slice(key, attribute.BoolValue(true), attribute.StringValue("value")), + want: attribute.Slice(key, attribute.BoolValue(true), attribute.StringValue("")), + }, + { + limit: 5, + attr: attribute.Slice(key, attribute.StringValue("value"), attribute.StringValue("toolong")), + want: attribute.Slice(key, attribute.StringValue("value"), attribute.StringValue("toolo")), + }, + { + // Nested SLICE: recursive truncation. + limit: 1, + attr: attribute.Slice(key, attribute.SliceValue(attribute.StringValue("value"))), + want: attribute.Slice(key, attribute.SliceValue(attribute.StringValue("v"))), + }, + { + // STRINGSLICE within SLICE: each string element is truncated. + limit: 2, + attr: attribute.Slice(key, attribute.StringSliceValue([]string{"abc", "de"})), + want: attribute.Slice(key, attribute.StringSliceValue([]string{"ab", "de"})), + }, + { + // STRINGSLICE within SLICE where all strings fit: no change. + // Exercises needsTruncation(STRINGSLICE) exhausting the loop without + // finding an over-limit string, returning false. + limit: 7, + attr: attribute.Slice(key, attribute.StringSliceValue([]string{"value-0", "value-1"})), + want: attribute.Slice(key, attribute.StringSliceValue([]string{"value-0", "value-1"})), + }, + { + // Mixed SLICE: STRINGSLICE (all strings fit) + STRING (too long). + // Exercises recursive truncation over mixed slice elements: the + // STRINGSLICE element remains unchanged because each string fits + // within the limit, while the sibling STRING element is truncated. + limit: 3, + attr: attribute.Slice( + key, + attribute.StringSliceValue([]string{"ab", "cd"}), + attribute.StringValue("toolong"), + ), + want: attribute.Slice( + key, + attribute.StringSliceValue([]string{"ab", "cd"}), + attribute.StringValue("too"), + ), + }, + { + // Nested SLICE (no truncation needed) alongside STRING (needs truncation). + // Exercises the TruncateValue SLICE branch early-return path: TruncateValue + // is called recursively on the nested SLICE but returns it unchanged because + // none of its elements require truncation. + limit: 3, + attr: attribute.Slice( + key, + attribute.SliceValue(attribute.BoolValue(true)), + attribute.StringValue("toolong"), + ), + want: attribute.Slice( + key, + attribute.SliceValue(attribute.BoolValue(true)), + attribute.StringValue("too"), + ), + }, + { + // Multi-byte string whose byte length exceeds the limit but rune count + // does not: must not be truncated (guards use rune count, not byte length). + limit: 3, + attr: attribute.Slice(key, attribute.StringValue("日本語")), // 3 runes, 9 bytes + want: attribute.Slice(key, attribute.StringValue("日本語")), + }, + { + // SLICE with invalid UTF-8 where rune count equals the limit: + // invalid byte is dropped. + limit: 2, + attr: attribute.Slice(key, attribute.StringValue("日\x80")), // 2 runes (日 + invalid byte), 4 bytes + want: attribute.Slice(key, attribute.StringValue("日")), + }, + { + // BYTESLICE within SLICE: each byte slice is truncated. + limit: 2, + attr: attribute.Slice(key, attribute.ByteSliceValue([]byte{1, 2, 3})), + want: attribute.Slice(key, attribute.ByteSliceValue([]byte{1, 2})), + }, + { + // BYTESLICE within SLICE: no truncation needed. + limit: 5, + attr: attribute.Slice(key, attribute.ByteSliceValue([]byte{1, 2})), + want: attribute.Slice(key, attribute.ByteSliceValue([]byte{1, 2})), + }, + { + // Mixed SLICE: BYTESLICE + STRING (both need truncation). + limit: 2, + attr: attribute.Slice( + key, + attribute.ByteSliceValue([]byte{1, 2, 3}), + attribute.StringValue("abc"), + ), + want: attribute.Slice( + key, + attribute.ByteSliceValue([]byte{1, 2}), + attribute.StringValue("ab"), + ), + }, + // MAP cases + { + limit: -1, + attr: attribute.Map(key, attribute.String("value", "value")), + want: attribute.Map(key, attribute.String("value", "value")), + }, + { + limit: 0, + attr: attribute.Map(key, attribute.Bool("ok", true), attribute.String("value", "value")), + want: attribute.Map(key, attribute.Bool("ok", true), attribute.String("value", "")), + }, + { + limit: 5, + attr: attribute.Map( + key, + attribute.String("short", "value"), + attribute.String("long", "toolong"), + ), + want: attribute.Map( + key, + attribute.String("short", "value"), + attribute.String("long", "toolo"), + ), + }, + { + // STRINGSLICE within MAP: each string element is truncated. + limit: 2, + attr: attribute.Map(key, attribute.StringSlice("strings", []string{"abc", "de"})), + want: attribute.Map(key, attribute.StringSlice("strings", []string{"ab", "de"})), + }, + { + // BYTESLICE within MAP: each byte slice is truncated. + limit: 2, + attr: attribute.Map(key, attribute.ByteSlice("bytes", []byte{1, 2, 3})), + want: attribute.Map(key, attribute.ByteSlice("bytes", []byte{1, 2})), + }, + { + // Nested MAP: recursive truncation. + limit: 1, + attr: attribute.Map(key, attribute.Map("map", attribute.String("nested", "value"))), + want: attribute.Map(key, attribute.Map("map", attribute.String("nested", "v"))), + }, + { + // SLICE within MAP: recursive truncation. + limit: 2, + attr: attribute.Map( + key, + attribute.Slice( + "slice", + attribute.StringValue("abc"), + attribute.MapValue(attribute.String("nested", "abc")), + ), + ), + want: attribute.Map( + key, + attribute.Slice( + "slice", + attribute.StringValue("ab"), + attribute.MapValue(attribute.String("nested", "ab")), + ), + ), + }, + { + // MAP within SLICE: recursive truncation. + limit: 2, + attr: attribute.Slice(key, attribute.MapValue(attribute.String("nested", "value"))), + want: attribute.Slice(key, attribute.MapValue(attribute.String("nested", "va"))), + }, + { + // Multi-byte string whose byte length exceeds the limit but rune count + // does not: must not be truncated. + limit: 3, + attr: attribute.Map(key, attribute.String("string", "日本語")), // 3 runes, 9 bytes + want: attribute.Map(key, attribute.String("string", "日本語")), + }, + { + // MAP with invalid UTF-8 where rune count equals the limit: + // invalid byte is dropped. + limit: 2, + attr: attribute.Map(key, attribute.String("string", "日\x80")), // 2 runes, 4 bytes + want: attribute.Map(key, attribute.String("string", "日")), + }, + { + // Duplicate MAP entries are truncated but not dropped. + limit: 2, + attr: attribute.Map( + key, + attribute.String("dup", "abc"), + attribute.String("dup", "de"), + ), + want: attribute.Map( + key, + attribute.String("dup", "ab"), + attribute.String("dup", "de"), + ), + }, + } + + for _, test := range tests { + name := fmt.Sprintf("%s->%s(limit:%d)", test.attr.Key, test.attr.Value.String(), test.limit) + t.Run(name, func(t *testing.T) { + assert.Equal(t, test.want, Truncate(test.limit, test.attr)) + }) + } +} + +func TestTruncateValue(t *testing.T) { + tests := []struct { + name string + limit int + value, want attribute.Value + }{ + { + name: "NegativeLimit", + limit: -1, + value: attribute.StringValue("value"), + want: attribute.StringValue("value"), + }, + { + name: "String", + limit: 2, + value: attribute.StringValue("value"), + want: attribute.StringValue("va"), + }, + { + name: "Map", + limit: 2, + value: attribute.MapValue(attribute.String("key", "value")), + want: attribute.MapValue(attribute.String("key", "va")), + }, + { + name: "UnchangedMap", + limit: 5, + value: attribute.MapValue(attribute.String("key", "value")), + want: attribute.MapValue(attribute.String("key", "value")), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.want, TruncateValue(test.limit, test.value)) + }) + } +} + +func TestTruncateString(t *testing.T) { + type group struct { + limit int + input string + expected string + } + + tests := []struct { + name string + groups []group + }{ + // Edge case: limit is negative, no truncation should occur + { + name: "NoTruncation", + groups: []group{ + {-1, "No truncation!", "No truncation!"}, + }, + }, + + // Edge case: string is already shorter than the limit, no truncation + // should occur + { + name: "ShortText", + groups: []group{ + {10, "Short text", "Short text"}, + {15, "Short text", "Short text"}, + {100, "Short text", "Short text"}, + }, + }, + + // Edge case: truncation happens with ASCII characters only + { + name: "ASCIIOnly", + groups: []group{ + {1, "Hello World!", "H"}, + {5, "Hello World!", "Hello"}, + {12, "Hello World!", "Hello World!"}, + }, + }, + + // Truncation including multi-byte characters (UTF-8) + { + name: "ValidUTF-8", + groups: []group{ + {7, "Hello, 世界", "Hello, "}, + {8, "Hello, 世界", "Hello, 世"}, + {2, "こんにちは", "こん"}, + {3, "こんにちは", "こんに"}, + {5, "こんにちは", "こんにちは"}, + {12, "こんにちは", "こんにちは"}, + }, + }, + + // Truncation with invalid UTF-8 characters + { + name: "InvalidUTF-8", + groups: []group{ + {11, "Invalid\x80text", "Invalidtext"}, + // Do not modify invalid text if equal to limit. + {11, "Valid text\x80", "Valid text\x80"}, + // Do not modify invalid text if under limit. + {15, "Valid text\x80", "Valid text\x80"}, + {5, "Hello\x80World", "Hello"}, + {11, "Hello\x80World\x80!", "HelloWorld!"}, + {15, "Hello\x80World\x80Test", "HelloWorldTest"}, + {15, "Hello\x80\x80\x80World\x80Test", "HelloWorldTest"}, + {15, "\x80\x80\x80Hello\x80\x80\x80World\x80Test\x80\x80", "HelloWorldTest"}, + }, + }, + + // Truncation with mixed validn and invalid UTF-8 characters + { + name: "MixedUTF-8", + groups: []group{ + {6, "€"[0:2] + "hello€€", "hello€"}, + {6, "€" + "€"[0:2] + "hello", "€hello"}, + {11, "Valid text\x80📜", "Valid text📜"}, + {11, "Valid text📜\x80", "Valid text📜"}, + {14, "😊 Hello\x80World🌍🚀", "😊 HelloWorld🌍🚀"}, + {14, "😊\x80 Hello\x80World🌍🚀", "😊 HelloWorld🌍🚀"}, + {14, "😊\x80 Hello\x80World🌍\x80🚀", "😊 HelloWorld🌍🚀"}, + {14, "😊\x80 Hello\x80World🌍\x80🚀\x80", "😊 HelloWorld🌍🚀"}, + {14, "\x80😊\x80 Hello\x80World🌍\x80🚀\x80", "😊 HelloWorld🌍🚀"}, + }, + }, + + // Edge case: empty string, should return empty string + { + name: "Empty", + groups: []group{ + {5, "", ""}, + }, + }, + + // Edge case: limit is 0, should return an empty string + { + name: "Zero", + groups: []group{ + {0, "Some text", ""}, + {0, "", ""}, + }, + }, + } + + for _, tt := range tests { + for _, g := range tt.groups { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := truncate(g.limit, g.input) + assert.Equalf( + t, g.expected, got, + "input: %q([]rune%v))\ngot: %q([]rune%v)\nwant %q([]rune%v)", + g.input, []rune(g.input), + got, []rune(got), + g.expected, []rune(g.expected), + ) + }) + } + } +} + +func BenchmarkTruncateAttr(b *testing.B) { + const key = "key" + + strAttr := attribute.String(key, "value") + bytesAttr := attribute.ByteSlice(key, []byte("value")) + strSliceAttr := attribute.StringSlice(key, []string{"value-0", "value-1"}) + + run := func(limit int, attr attribute.KeyValue) func(b *testing.B) { + return func(b *testing.B) { + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + var out attribute.KeyValue + for pb.Next() { + out = Truncate(limit, attr) + } + _ = out + }) + } + } + + b.Run("String", run(3, strAttr)) + b.Run("StringSlice", run(3, strSliceAttr)) + b.Run("ByteSlice", run(3, bytesAttr)) + b.Run("String/Limit0", run(0, strAttr)) + b.Run("StringSlice/Limit0", run(0, strSliceAttr)) + b.Run("ByteSlice/Limit0", run(0, bytesAttr)) + b.Run("String/Unlimited", run(-1, strAttr)) + b.Run("StringSlice/Unlimited", run(-1, strSliceAttr)) + b.Run("ByteSlice/Unlimited", run(-1, bytesAttr)) +} + +func BenchmarkTruncate(b *testing.B) { + run := func(limit int, input string) func(b *testing.B) { + return func(b *testing.B) { + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + var out string + for pb.Next() { + out = truncate(limit, input) + } + _ = out + }) + } + } + b.Run("Unlimited", run(-1, "hello 😊 world 🌍🚀")) + b.Run("Zero", run(0, "Some text")) + b.Run("Short", run(10, "Short Text")) + b.Run("ASCII", run(5, "Hello, World!")) + b.Run("ValidUTF-8", run(10, "hello 😊 world 🌍🚀")) + b.Run("InvalidUTF-8", run(6, "€"[0:2]+"hello€€")) + b.Run("MixedUTF-8", run(14, "\x80😊\x80 Hello\x80World🌍\x80🚀\x80")) +} diff --git a/sdk/internal/attrdedup/dedup.go b/sdk/internal/attrnorm/dedup.go similarity index 98% rename from sdk/internal/attrdedup/dedup.go rename to sdk/internal/attrnorm/dedup.go index c5ef89dd7af..fe9ddc96db7 100644 --- a/sdk/internal/attrdedup/dedup.go +++ b/sdk/internal/attrnorm/dedup.go @@ -2,10 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 // DO NOT MODIFY. Generated by gotmpl. -// source: internal/shared/attrdedup/dedup.go.tmpl +// source: internal/shared/attrnorm/dedup.go.tmpl -// Package attrdedup deduplicates attribute map values. -package attrdedup +// Package attrnorm normalizes attribute values. +package attrnorm import ( "reflect" diff --git a/sdk/internal/attrdedup/dedup_benchmark_test.go b/sdk/internal/attrnorm/dedup_benchmark_test.go similarity index 89% rename from sdk/internal/attrdedup/dedup_benchmark_test.go rename to sdk/internal/attrnorm/dedup_benchmark_test.go index 6d4c1fe37c2..ef3e4e3fde0 100644 --- a/sdk/internal/attrdedup/dedup_benchmark_test.go +++ b/sdk/internal/attrnorm/dedup_benchmark_test.go @@ -1,13 +1,13 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package attrdedup_test +package attrnorm_test import ( "testing" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/sdk/internal/attrdedup" + "go.opentelemetry.io/otel/sdk/internal/attrnorm" ) func BenchmarkValue(b *testing.B) { @@ -47,7 +47,7 @@ func BenchmarkValue(b *testing.B) { b.Run(value.name, func(b *testing.B) { b.ReportAllocs() for b.Loop() { - _, _ = attrdedup.Value(value.value) + _, _ = attrnorm.Value(value.value) } }) } diff --git a/sdk/metric/internal/attrdedup/dedup_test.go b/sdk/internal/attrnorm/dedup_test.go similarity index 99% rename from sdk/metric/internal/attrdedup/dedup_test.go rename to sdk/internal/attrnorm/dedup_test.go index 82fab043ab4..0e6b10b193a 100644 --- a/sdk/metric/internal/attrdedup/dedup_test.go +++ b/sdk/internal/attrnorm/dedup_test.go @@ -2,9 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 // DO NOT MODIFY. Generated by gotmpl. -// source: internal/shared/attrdedup/dedup_test.go.tmpl +// source: internal/shared/attrnorm/dedup_test.go.tmpl -package attrdedup +package attrnorm import ( "testing" diff --git a/sdk/internal/attrnorm/truncate.go b/sdk/internal/attrnorm/truncate.go new file mode 100644 index 00000000000..9955e907268 --- /dev/null +++ b/sdk/internal/attrnorm/truncate.go @@ -0,0 +1,232 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// DO NOT MODIFY. Generated by gotmpl. +// source: internal/shared/attrnorm/truncate.go.tmpl + +package attrnorm + +import ( + "slices" + "strings" + "unicode/utf8" + + "go.opentelemetry.io/otel/attribute" +) + +// Truncate returns a truncated version of attr. Only string, string slice, +// byte slice, slice, and map attribute values are truncated. String values are +// truncated to at most a length of limit. Each string slice value is truncated +// in this fashion (the slice length itself is unaffected), and byte slice +// values are truncated to at most limit bytes. For slice and map attribute +// values, the limit is applied recursively to contained values. +// +// No truncation is performed for a negative limit. +func Truncate(limit int, attr attribute.KeyValue) attribute.KeyValue { + if limit < 0 { + return attr + } + switch attr.Value.Type() { + case attribute.STRING: + v := attr.Value.AsString() + return attr.Key.String(truncate(limit, v)) + case attribute.STRINGSLICE: + v := attr.Value.AsStringSlice() + for i := range v { + v[i] = truncate(limit, v[i]) + } + return attr.Key.StringSlice(v) + case attribute.BYTESLICE: + v := attr.Value.AsString() + if len(v) > limit { + return attr.Key.ByteSlice([]byte(v[:limit])) + } + return attr + case attribute.SLICE: + v := attr.Value.AsSlice() + if !slices.ContainsFunc(v, func(e attribute.Value) bool { return needsTruncation(limit, e) }) { + return attr + } + newV := make([]attribute.Value, len(v)) + for i, elem := range v { + newV[i] = TruncateValue(limit, elem) + } + return attr.Key.Slice(newV...) + case attribute.MAP: + v := attr.Value.AsMap() + if !slices.ContainsFunc(v, func(kv attribute.KeyValue) bool { return needsTruncation(limit, kv.Value) }) { + return attr + } + newV := make([]attribute.KeyValue, len(v)) + for i, elem := range v { + elem.Value = TruncateValue(limit, elem.Value) + newV[i] = elem + } + return attr.Key.Map(newV...) + } + return attr +} + +// TruncateValue returns a truncated version of v. Only string, string +// slice, byte slice, and (recursively) slice and map values are modified. +// +// No truncation is performed for a negative limit. +func TruncateValue(limit int, v attribute.Value) attribute.Value { + if limit < 0 { + return v + } + + switch v.Type() { + case attribute.STRING: + return attribute.StringValue(truncate(limit, v.AsString())) + case attribute.STRINGSLICE: + ss := v.AsStringSlice() + for i := range ss { + ss[i] = truncate(limit, ss[i]) + } + return attribute.StringSliceValue(ss) + case attribute.BYTESLICE: + // len(v.AsString()) is identical to len(v.AsByteSlice()) but + // avoids allocating the full slice before truncation. + s := v.AsString() + if limit >= 0 && len(s) > limit { + return attribute.ByteSliceValue([]byte(s[:limit])) + } + case attribute.SLICE: + sl := v.AsSlice() + if !slices.ContainsFunc(sl, func(e attribute.Value) bool { return needsTruncation(limit, e) }) { + return v + } + newSl := make([]attribute.Value, len(sl)) + for i, elem := range sl { + newSl[i] = TruncateValue(limit, elem) + } + return attribute.SliceValue(newSl...) + case attribute.MAP: + m := v.AsMap() + if !slices.ContainsFunc(m, func(kv attribute.KeyValue) bool { return needsTruncation(limit, kv.Value) }) { + return v + } + newM := make([]attribute.KeyValue, len(m)) + for i, elem := range m { + elem.Value = TruncateValue(limit, elem.Value) + newM[i] = elem + } + return attribute.MapValue(newM...) + } + return v +} + +// stringNeedsTruncation reports whether s would be modified by truncate for the +// given limit. +func stringNeedsTruncation(limit int, s string) bool { + if limit < 0 || len(s) <= limit { + return false + } + return utf8.RuneCountInString(s) > limit || !utf8.ValidString(s) +} + +// needsTruncation reports whether v would be modified by TruncateValue for the +// given limit. +func needsTruncation(limit int, v attribute.Value) bool { + switch v.Type() { + case attribute.STRING: + return stringNeedsTruncation(limit, v.AsString()) + case attribute.BYTESLICE: + // len(v.AsString()) is identical to len(v.AsByteSlice()) but + // avoids memory allocation. + if limit >= 0 && len(v.AsString()) > limit { + return true + } + case attribute.STRINGSLICE: + for _, s := range v.AsStringSlice() { + if stringNeedsTruncation(limit, s) { + return true + } + } + case attribute.SLICE: + return slices.ContainsFunc(v.AsSlice(), func(e attribute.Value) bool { return needsTruncation(limit, e) }) + case attribute.MAP: + return slices.ContainsFunc( + v.AsMap(), + func(kv attribute.KeyValue) bool { return needsTruncation(limit, kv.Value) }, + ) + } + return false +} + +// truncate returns a truncated version of s such that it contains less than +// the limit number of characters. Truncation is applied by returning the limit +// number of valid characters contained in s. +// +// If limit is negative, it returns the original string. +// +// UTF-8 is supported. When truncating, all invalid characters are dropped +// before applying truncation. +// +// If s already contains less than the limit number of bytes, it is returned +// unchanged. No invalid characters are removed. +func truncate(limit int, s string) string { + // This prioritize performance in the following order based on the most + // common expected use-cases. + // + // - Short values less than the default limit (128). + // - Strings with valid encodings that exceed the limit. + // - No limit. + // - Strings with invalid encodings that exceed the limit. + if limit < 0 || len(s) <= limit { + return s + } + + // Optimistically, assume all valid UTF-8. + var b strings.Builder + count := 0 + for i, c := range s { + if c != utf8.RuneError { + count++ + if count > limit { + return s[:i] + } + continue + } + + _, size := utf8.DecodeRuneInString(s[i:]) + if size == 1 { + // Invalid encoding. + b.Grow(len(s) - 1) + _, _ = b.WriteString(s[:i]) + s = s[i:] + break + } + } + + // Fast-path, no invalid input. + if b.Cap() == 0 { + return s + } + + // Truncate while validating UTF-8. + for i := 0; i < len(s) && count < limit; { + c := s[i] + if c < utf8.RuneSelf { + // Optimization for single byte runes (common case). + _ = b.WriteByte(c) + i++ + count++ + continue + } + + _, size := utf8.DecodeRuneInString(s[i:]) + if size == 1 { + // We checked for all 1-byte runes above, this is a RuneError. + i++ + continue + } + + _, _ = b.WriteString(s[i : i+size]) + i += size + count++ + } + + return b.String() +} diff --git a/sdk/internal/attrnorm/truncate_test.go b/sdk/internal/attrnorm/truncate_test.go new file mode 100644 index 00000000000..ad19bbbc09c --- /dev/null +++ b/sdk/internal/attrnorm/truncate_test.go @@ -0,0 +1,604 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// DO NOT MODIFY. Generated by gotmpl. +// source: internal/shared/attrnorm/truncate_test.go.tmpl + +package attrnorm + +import ( + "bytes" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/otel/attribute" +) + +func TestTruncateAttr(t *testing.T) { + const key = "key" + + strAttr := attribute.String(key, "value") + bytesAttr := attribute.ByteSlice(key, []byte("value")) + strSliceAttr := attribute.StringSlice(key, []string{"value-0", "value-1"}) + + tests := []struct { + limit int + attr, want attribute.KeyValue + }{ + { + limit: -1, + attr: strAttr, + want: strAttr, + }, + { + limit: -1, + attr: strSliceAttr, + want: strSliceAttr, + }, + { + limit: -1, + attr: bytesAttr, + want: bytesAttr, + }, + { + limit: 0, + attr: attribute.Bool(key, true), + want: attribute.Bool(key, true), + }, + { + limit: 0, + attr: attribute.BoolSlice(key, []bool{true, false}), + want: attribute.BoolSlice(key, []bool{true, false}), + }, + { + limit: 0, + attr: attribute.Int(key, 42), + want: attribute.Int(key, 42), + }, + { + limit: 0, + attr: attribute.IntSlice(key, []int{42, -1}), + want: attribute.IntSlice(key, []int{42, -1}), + }, + { + limit: 0, + attr: attribute.Int64(key, 42), + want: attribute.Int64(key, 42), + }, + { + limit: 0, + attr: attribute.Int64Slice(key, []int64{42, -1}), + want: attribute.Int64Slice(key, []int64{42, -1}), + }, + { + limit: 0, + attr: attribute.Float64(key, 42), + want: attribute.Float64(key, 42), + }, + { + limit: 0, + attr: attribute.Float64Slice(key, []float64{42, -1}), + want: attribute.Float64Slice(key, []float64{42, -1}), + }, + { + limit: 0, + attr: strAttr, + want: attribute.String(key, ""), + }, + { + limit: 0, + attr: strSliceAttr, + want: attribute.StringSlice(key, []string{"", ""}), + }, + { + limit: 0, + attr: attribute.Stringer(key, bytes.NewBufferString("value")), + want: attribute.String(key, ""), + }, + { + limit: 0, + attr: bytesAttr, + want: attribute.ByteSlice(key, []byte{}), + }, + { + limit: 1, + attr: strAttr, + want: attribute.String(key, "v"), + }, + { + limit: 1, + attr: strSliceAttr, + want: attribute.StringSlice(key, []string{"v", "v"}), + }, + { + limit: 1, + attr: bytesAttr, + want: attribute.ByteSlice(key, []byte("v")), + }, + { + limit: 5, + attr: strAttr, + want: strAttr, + }, + { + limit: 5, + attr: bytesAttr, + want: bytesAttr, + }, + { + limit: 7, + attr: strSliceAttr, + want: strSliceAttr, + }, + { + limit: 6, + attr: attribute.StringSlice(key, []string{"value", "value-1"}), + want: attribute.StringSlice(key, []string{"value", "value-"}), + }, + { + limit: 128, + attr: strAttr, + want: strAttr, + }, + { + limit: 128, + attr: strSliceAttr, + want: strSliceAttr, + }, + { + limit: 128, + attr: bytesAttr, + want: bytesAttr, + }, + { + // Multi-byte string: byte length (9) exceeds limit (5) but rune count (3) does not. + // Must not be truncated. + limit: 5, + attr: attribute.String(key, "日本語"), + want: attribute.String(key, "日本語"), + }, + { + // Multi-byte string: both byte length and rune count exceed limit. + // Must be truncated to limit runes. + limit: 2, + attr: attribute.String(key, "日本語"), + want: attribute.String(key, "日本"), + }, + { + // STRINGSLICE with multi-byte elements: byte lengths exceed limit but rune counts do not. + // Must not be truncated. + limit: 1, + attr: attribute.StringSlice(key, []string{"日", "本"}), + want: attribute.StringSlice(key, []string{"日", "本"}), + }, + // SLICE cases + { + limit: -1, + attr: attribute.Slice(key, attribute.StringValue("value")), + want: attribute.Slice(key, attribute.StringValue("value")), + }, + { + limit: 0, + attr: attribute.Slice(key, attribute.BoolValue(true), attribute.StringValue("value")), + want: attribute.Slice(key, attribute.BoolValue(true), attribute.StringValue("")), + }, + { + limit: 5, + attr: attribute.Slice(key, attribute.StringValue("value"), attribute.StringValue("toolong")), + want: attribute.Slice(key, attribute.StringValue("value"), attribute.StringValue("toolo")), + }, + { + // Nested SLICE: recursive truncation. + limit: 1, + attr: attribute.Slice(key, attribute.SliceValue(attribute.StringValue("value"))), + want: attribute.Slice(key, attribute.SliceValue(attribute.StringValue("v"))), + }, + { + // STRINGSLICE within SLICE: each string element is truncated. + limit: 2, + attr: attribute.Slice(key, attribute.StringSliceValue([]string{"abc", "de"})), + want: attribute.Slice(key, attribute.StringSliceValue([]string{"ab", "de"})), + }, + { + // STRINGSLICE within SLICE where all strings fit: no change. + // Exercises needsTruncation(STRINGSLICE) exhausting the loop without + // finding an over-limit string, returning false. + limit: 7, + attr: attribute.Slice(key, attribute.StringSliceValue([]string{"value-0", "value-1"})), + want: attribute.Slice(key, attribute.StringSliceValue([]string{"value-0", "value-1"})), + }, + { + // Mixed SLICE: STRINGSLICE (all strings fit) + STRING (too long). + // Exercises recursive truncation over mixed slice elements: the + // STRINGSLICE element remains unchanged because each string fits + // within the limit, while the sibling STRING element is truncated. + limit: 3, + attr: attribute.Slice( + key, + attribute.StringSliceValue([]string{"ab", "cd"}), + attribute.StringValue("toolong"), + ), + want: attribute.Slice( + key, + attribute.StringSliceValue([]string{"ab", "cd"}), + attribute.StringValue("too"), + ), + }, + { + // Nested SLICE (no truncation needed) alongside STRING (needs truncation). + // Exercises the TruncateValue SLICE branch early-return path: TruncateValue + // is called recursively on the nested SLICE but returns it unchanged because + // none of its elements require truncation. + limit: 3, + attr: attribute.Slice( + key, + attribute.SliceValue(attribute.BoolValue(true)), + attribute.StringValue("toolong"), + ), + want: attribute.Slice( + key, + attribute.SliceValue(attribute.BoolValue(true)), + attribute.StringValue("too"), + ), + }, + { + // Multi-byte string whose byte length exceeds the limit but rune count + // does not: must not be truncated (guards use rune count, not byte length). + limit: 3, + attr: attribute.Slice(key, attribute.StringValue("日本語")), // 3 runes, 9 bytes + want: attribute.Slice(key, attribute.StringValue("日本語")), + }, + { + // SLICE with invalid UTF-8 where rune count equals the limit: + // invalid byte is dropped. + limit: 2, + attr: attribute.Slice(key, attribute.StringValue("日\x80")), // 2 runes (日 + invalid byte), 4 bytes + want: attribute.Slice(key, attribute.StringValue("日")), + }, + { + // BYTESLICE within SLICE: each byte slice is truncated. + limit: 2, + attr: attribute.Slice(key, attribute.ByteSliceValue([]byte{1, 2, 3})), + want: attribute.Slice(key, attribute.ByteSliceValue([]byte{1, 2})), + }, + { + // BYTESLICE within SLICE: no truncation needed. + limit: 5, + attr: attribute.Slice(key, attribute.ByteSliceValue([]byte{1, 2})), + want: attribute.Slice(key, attribute.ByteSliceValue([]byte{1, 2})), + }, + { + // Mixed SLICE: BYTESLICE + STRING (both need truncation). + limit: 2, + attr: attribute.Slice( + key, + attribute.ByteSliceValue([]byte{1, 2, 3}), + attribute.StringValue("abc"), + ), + want: attribute.Slice( + key, + attribute.ByteSliceValue([]byte{1, 2}), + attribute.StringValue("ab"), + ), + }, + // MAP cases + { + limit: -1, + attr: attribute.Map(key, attribute.String("value", "value")), + want: attribute.Map(key, attribute.String("value", "value")), + }, + { + limit: 0, + attr: attribute.Map(key, attribute.Bool("ok", true), attribute.String("value", "value")), + want: attribute.Map(key, attribute.Bool("ok", true), attribute.String("value", "")), + }, + { + limit: 5, + attr: attribute.Map( + key, + attribute.String("short", "value"), + attribute.String("long", "toolong"), + ), + want: attribute.Map( + key, + attribute.String("short", "value"), + attribute.String("long", "toolo"), + ), + }, + { + // STRINGSLICE within MAP: each string element is truncated. + limit: 2, + attr: attribute.Map(key, attribute.StringSlice("strings", []string{"abc", "de"})), + want: attribute.Map(key, attribute.StringSlice("strings", []string{"ab", "de"})), + }, + { + // BYTESLICE within MAP: each byte slice is truncated. + limit: 2, + attr: attribute.Map(key, attribute.ByteSlice("bytes", []byte{1, 2, 3})), + want: attribute.Map(key, attribute.ByteSlice("bytes", []byte{1, 2})), + }, + { + // Nested MAP: recursive truncation. + limit: 1, + attr: attribute.Map(key, attribute.Map("map", attribute.String("nested", "value"))), + want: attribute.Map(key, attribute.Map("map", attribute.String("nested", "v"))), + }, + { + // SLICE within MAP: recursive truncation. + limit: 2, + attr: attribute.Map( + key, + attribute.Slice( + "slice", + attribute.StringValue("abc"), + attribute.MapValue(attribute.String("nested", "abc")), + ), + ), + want: attribute.Map( + key, + attribute.Slice( + "slice", + attribute.StringValue("ab"), + attribute.MapValue(attribute.String("nested", "ab")), + ), + ), + }, + { + // MAP within SLICE: recursive truncation. + limit: 2, + attr: attribute.Slice(key, attribute.MapValue(attribute.String("nested", "value"))), + want: attribute.Slice(key, attribute.MapValue(attribute.String("nested", "va"))), + }, + { + // Multi-byte string whose byte length exceeds the limit but rune count + // does not: must not be truncated. + limit: 3, + attr: attribute.Map(key, attribute.String("string", "日本語")), // 3 runes, 9 bytes + want: attribute.Map(key, attribute.String("string", "日本語")), + }, + { + // MAP with invalid UTF-8 where rune count equals the limit: + // invalid byte is dropped. + limit: 2, + attr: attribute.Map(key, attribute.String("string", "日\x80")), // 2 runes, 4 bytes + want: attribute.Map(key, attribute.String("string", "日")), + }, + { + // Duplicate MAP entries are truncated but not dropped. + limit: 2, + attr: attribute.Map( + key, + attribute.String("dup", "abc"), + attribute.String("dup", "de"), + ), + want: attribute.Map( + key, + attribute.String("dup", "ab"), + attribute.String("dup", "de"), + ), + }, + } + + for _, test := range tests { + name := fmt.Sprintf("%s->%s(limit:%d)", test.attr.Key, test.attr.Value.String(), test.limit) + t.Run(name, func(t *testing.T) { + assert.Equal(t, test.want, Truncate(test.limit, test.attr)) + }) + } +} + +func TestTruncateValue(t *testing.T) { + tests := []struct { + name string + limit int + value, want attribute.Value + }{ + { + name: "NegativeLimit", + limit: -1, + value: attribute.StringValue("value"), + want: attribute.StringValue("value"), + }, + { + name: "String", + limit: 2, + value: attribute.StringValue("value"), + want: attribute.StringValue("va"), + }, + { + name: "Map", + limit: 2, + value: attribute.MapValue(attribute.String("key", "value")), + want: attribute.MapValue(attribute.String("key", "va")), + }, + { + name: "UnchangedMap", + limit: 5, + value: attribute.MapValue(attribute.String("key", "value")), + want: attribute.MapValue(attribute.String("key", "value")), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.want, TruncateValue(test.limit, test.value)) + }) + } +} + +func TestTruncateString(t *testing.T) { + type group struct { + limit int + input string + expected string + } + + tests := []struct { + name string + groups []group + }{ + // Edge case: limit is negative, no truncation should occur + { + name: "NoTruncation", + groups: []group{ + {-1, "No truncation!", "No truncation!"}, + }, + }, + + // Edge case: string is already shorter than the limit, no truncation + // should occur + { + name: "ShortText", + groups: []group{ + {10, "Short text", "Short text"}, + {15, "Short text", "Short text"}, + {100, "Short text", "Short text"}, + }, + }, + + // Edge case: truncation happens with ASCII characters only + { + name: "ASCIIOnly", + groups: []group{ + {1, "Hello World!", "H"}, + {5, "Hello World!", "Hello"}, + {12, "Hello World!", "Hello World!"}, + }, + }, + + // Truncation including multi-byte characters (UTF-8) + { + name: "ValidUTF-8", + groups: []group{ + {7, "Hello, 世界", "Hello, "}, + {8, "Hello, 世界", "Hello, 世"}, + {2, "こんにちは", "こん"}, + {3, "こんにちは", "こんに"}, + {5, "こんにちは", "こんにちは"}, + {12, "こんにちは", "こんにちは"}, + }, + }, + + // Truncation with invalid UTF-8 characters + { + name: "InvalidUTF-8", + groups: []group{ + {11, "Invalid\x80text", "Invalidtext"}, + // Do not modify invalid text if equal to limit. + {11, "Valid text\x80", "Valid text\x80"}, + // Do not modify invalid text if under limit. + {15, "Valid text\x80", "Valid text\x80"}, + {5, "Hello\x80World", "Hello"}, + {11, "Hello\x80World\x80!", "HelloWorld!"}, + {15, "Hello\x80World\x80Test", "HelloWorldTest"}, + {15, "Hello\x80\x80\x80World\x80Test", "HelloWorldTest"}, + {15, "\x80\x80\x80Hello\x80\x80\x80World\x80Test\x80\x80", "HelloWorldTest"}, + }, + }, + + // Truncation with mixed validn and invalid UTF-8 characters + { + name: "MixedUTF-8", + groups: []group{ + {6, "€"[0:2] + "hello€€", "hello€"}, + {6, "€" + "€"[0:2] + "hello", "€hello"}, + {11, "Valid text\x80📜", "Valid text📜"}, + {11, "Valid text📜\x80", "Valid text📜"}, + {14, "😊 Hello\x80World🌍🚀", "😊 HelloWorld🌍🚀"}, + {14, "😊\x80 Hello\x80World🌍🚀", "😊 HelloWorld🌍🚀"}, + {14, "😊\x80 Hello\x80World🌍\x80🚀", "😊 HelloWorld🌍🚀"}, + {14, "😊\x80 Hello\x80World🌍\x80🚀\x80", "😊 HelloWorld🌍🚀"}, + {14, "\x80😊\x80 Hello\x80World🌍\x80🚀\x80", "😊 HelloWorld🌍🚀"}, + }, + }, + + // Edge case: empty string, should return empty string + { + name: "Empty", + groups: []group{ + {5, "", ""}, + }, + }, + + // Edge case: limit is 0, should return an empty string + { + name: "Zero", + groups: []group{ + {0, "Some text", ""}, + {0, "", ""}, + }, + }, + } + + for _, tt := range tests { + for _, g := range tt.groups { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := truncate(g.limit, g.input) + assert.Equalf( + t, g.expected, got, + "input: %q([]rune%v))\ngot: %q([]rune%v)\nwant %q([]rune%v)", + g.input, []rune(g.input), + got, []rune(got), + g.expected, []rune(g.expected), + ) + }) + } + } +} + +func BenchmarkTruncateAttr(b *testing.B) { + const key = "key" + + strAttr := attribute.String(key, "value") + bytesAttr := attribute.ByteSlice(key, []byte("value")) + strSliceAttr := attribute.StringSlice(key, []string{"value-0", "value-1"}) + + run := func(limit int, attr attribute.KeyValue) func(b *testing.B) { + return func(b *testing.B) { + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + var out attribute.KeyValue + for pb.Next() { + out = Truncate(limit, attr) + } + _ = out + }) + } + } + + b.Run("String", run(3, strAttr)) + b.Run("StringSlice", run(3, strSliceAttr)) + b.Run("ByteSlice", run(3, bytesAttr)) + b.Run("String/Limit0", run(0, strAttr)) + b.Run("StringSlice/Limit0", run(0, strSliceAttr)) + b.Run("ByteSlice/Limit0", run(0, bytesAttr)) + b.Run("String/Unlimited", run(-1, strAttr)) + b.Run("StringSlice/Unlimited", run(-1, strSliceAttr)) + b.Run("ByteSlice/Unlimited", run(-1, bytesAttr)) +} + +func BenchmarkTruncate(b *testing.B) { + run := func(limit int, input string) func(b *testing.B) { + return func(b *testing.B) { + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + var out string + for pb.Next() { + out = truncate(limit, input) + } + _ = out + }) + } + } + b.Run("Unlimited", run(-1, "hello 😊 world 🌍🚀")) + b.Run("Zero", run(0, "Some text")) + b.Run("Short", run(10, "Short Text")) + b.Run("ASCII", run(5, "Hello, World!")) + b.Run("ValidUTF-8", run(10, "hello 😊 world 🌍🚀")) + b.Run("InvalidUTF-8", run(6, "€"[0:2]+"hello€€")) + b.Run("MixedUTF-8", run(14, "\x80😊\x80 Hello\x80World🌍\x80🚀\x80")) +} diff --git a/sdk/internal/gen.go b/sdk/internal/gen.go index 7a8511148a0..e58925ad61e 100644 --- a/sdk/internal/gen.go +++ b/sdk/internal/gen.go @@ -6,5 +6,7 @@ package internal //go:generate gotmpl --body=../../internal/shared/x/x.go.tmpl "--data={ \"pkg\": \"go.opentelemetry.io/otel/sdk\" }" --out=x/x.go //go:generate gotmpl --body=../../internal/shared/x/x_test.go.tmpl "--data={}" --out=x/x_test.go -//go:generate gotmpl --body=../../internal/shared/attrdedup/dedup.go.tmpl "--data={}" --out=attrdedup/dedup.go -//go:generate gotmpl --body=../../internal/shared/attrdedup/dedup_test.go.tmpl "--data={}" --out=attrdedup/dedup_test.go +//go:generate gotmpl --body=../../internal/shared/attrnorm/dedup.go.tmpl "--data={}" --out=attrnorm/dedup.go +//go:generate gotmpl --body=../../internal/shared/attrnorm/dedup_test.go.tmpl "--data={}" --out=attrnorm/dedup_test.go +//go:generate gotmpl --body=../../internal/shared/attrnorm/truncate.go.tmpl "--data={}" --out=attrnorm/truncate.go +//go:generate gotmpl --body=../../internal/shared/attrnorm/truncate_test.go.tmpl "--data={}" --out=attrnorm/truncate_test.go diff --git a/sdk/metric/internal/attrdedup/dedup.go b/sdk/log/internal/attrnorm/dedup.go similarity index 98% rename from sdk/metric/internal/attrdedup/dedup.go rename to sdk/log/internal/attrnorm/dedup.go index c5ef89dd7af..fe9ddc96db7 100644 --- a/sdk/metric/internal/attrdedup/dedup.go +++ b/sdk/log/internal/attrnorm/dedup.go @@ -2,10 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 // DO NOT MODIFY. Generated by gotmpl. -// source: internal/shared/attrdedup/dedup.go.tmpl +// source: internal/shared/attrnorm/dedup.go.tmpl -// Package attrdedup deduplicates attribute map values. -package attrdedup +// Package attrnorm normalizes attribute values. +package attrnorm import ( "reflect" diff --git a/sdk/internal/attrdedup/dedup_test.go b/sdk/log/internal/attrnorm/dedup_test.go similarity index 99% rename from sdk/internal/attrdedup/dedup_test.go rename to sdk/log/internal/attrnorm/dedup_test.go index 82fab043ab4..0e6b10b193a 100644 --- a/sdk/internal/attrdedup/dedup_test.go +++ b/sdk/log/internal/attrnorm/dedup_test.go @@ -2,9 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 // DO NOT MODIFY. Generated by gotmpl. -// source: internal/shared/attrdedup/dedup_test.go.tmpl +// source: internal/shared/attrnorm/dedup_test.go.tmpl -package attrdedup +package attrnorm import ( "testing" diff --git a/sdk/log/internal/attrnorm/truncate.go b/sdk/log/internal/attrnorm/truncate.go new file mode 100644 index 00000000000..9955e907268 --- /dev/null +++ b/sdk/log/internal/attrnorm/truncate.go @@ -0,0 +1,232 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// DO NOT MODIFY. Generated by gotmpl. +// source: internal/shared/attrnorm/truncate.go.tmpl + +package attrnorm + +import ( + "slices" + "strings" + "unicode/utf8" + + "go.opentelemetry.io/otel/attribute" +) + +// Truncate returns a truncated version of attr. Only string, string slice, +// byte slice, slice, and map attribute values are truncated. String values are +// truncated to at most a length of limit. Each string slice value is truncated +// in this fashion (the slice length itself is unaffected), and byte slice +// values are truncated to at most limit bytes. For slice and map attribute +// values, the limit is applied recursively to contained values. +// +// No truncation is performed for a negative limit. +func Truncate(limit int, attr attribute.KeyValue) attribute.KeyValue { + if limit < 0 { + return attr + } + switch attr.Value.Type() { + case attribute.STRING: + v := attr.Value.AsString() + return attr.Key.String(truncate(limit, v)) + case attribute.STRINGSLICE: + v := attr.Value.AsStringSlice() + for i := range v { + v[i] = truncate(limit, v[i]) + } + return attr.Key.StringSlice(v) + case attribute.BYTESLICE: + v := attr.Value.AsString() + if len(v) > limit { + return attr.Key.ByteSlice([]byte(v[:limit])) + } + return attr + case attribute.SLICE: + v := attr.Value.AsSlice() + if !slices.ContainsFunc(v, func(e attribute.Value) bool { return needsTruncation(limit, e) }) { + return attr + } + newV := make([]attribute.Value, len(v)) + for i, elem := range v { + newV[i] = TruncateValue(limit, elem) + } + return attr.Key.Slice(newV...) + case attribute.MAP: + v := attr.Value.AsMap() + if !slices.ContainsFunc(v, func(kv attribute.KeyValue) bool { return needsTruncation(limit, kv.Value) }) { + return attr + } + newV := make([]attribute.KeyValue, len(v)) + for i, elem := range v { + elem.Value = TruncateValue(limit, elem.Value) + newV[i] = elem + } + return attr.Key.Map(newV...) + } + return attr +} + +// TruncateValue returns a truncated version of v. Only string, string +// slice, byte slice, and (recursively) slice and map values are modified. +// +// No truncation is performed for a negative limit. +func TruncateValue(limit int, v attribute.Value) attribute.Value { + if limit < 0 { + return v + } + + switch v.Type() { + case attribute.STRING: + return attribute.StringValue(truncate(limit, v.AsString())) + case attribute.STRINGSLICE: + ss := v.AsStringSlice() + for i := range ss { + ss[i] = truncate(limit, ss[i]) + } + return attribute.StringSliceValue(ss) + case attribute.BYTESLICE: + // len(v.AsString()) is identical to len(v.AsByteSlice()) but + // avoids allocating the full slice before truncation. + s := v.AsString() + if limit >= 0 && len(s) > limit { + return attribute.ByteSliceValue([]byte(s[:limit])) + } + case attribute.SLICE: + sl := v.AsSlice() + if !slices.ContainsFunc(sl, func(e attribute.Value) bool { return needsTruncation(limit, e) }) { + return v + } + newSl := make([]attribute.Value, len(sl)) + for i, elem := range sl { + newSl[i] = TruncateValue(limit, elem) + } + return attribute.SliceValue(newSl...) + case attribute.MAP: + m := v.AsMap() + if !slices.ContainsFunc(m, func(kv attribute.KeyValue) bool { return needsTruncation(limit, kv.Value) }) { + return v + } + newM := make([]attribute.KeyValue, len(m)) + for i, elem := range m { + elem.Value = TruncateValue(limit, elem.Value) + newM[i] = elem + } + return attribute.MapValue(newM...) + } + return v +} + +// stringNeedsTruncation reports whether s would be modified by truncate for the +// given limit. +func stringNeedsTruncation(limit int, s string) bool { + if limit < 0 || len(s) <= limit { + return false + } + return utf8.RuneCountInString(s) > limit || !utf8.ValidString(s) +} + +// needsTruncation reports whether v would be modified by TruncateValue for the +// given limit. +func needsTruncation(limit int, v attribute.Value) bool { + switch v.Type() { + case attribute.STRING: + return stringNeedsTruncation(limit, v.AsString()) + case attribute.BYTESLICE: + // len(v.AsString()) is identical to len(v.AsByteSlice()) but + // avoids memory allocation. + if limit >= 0 && len(v.AsString()) > limit { + return true + } + case attribute.STRINGSLICE: + for _, s := range v.AsStringSlice() { + if stringNeedsTruncation(limit, s) { + return true + } + } + case attribute.SLICE: + return slices.ContainsFunc(v.AsSlice(), func(e attribute.Value) bool { return needsTruncation(limit, e) }) + case attribute.MAP: + return slices.ContainsFunc( + v.AsMap(), + func(kv attribute.KeyValue) bool { return needsTruncation(limit, kv.Value) }, + ) + } + return false +} + +// truncate returns a truncated version of s such that it contains less than +// the limit number of characters. Truncation is applied by returning the limit +// number of valid characters contained in s. +// +// If limit is negative, it returns the original string. +// +// UTF-8 is supported. When truncating, all invalid characters are dropped +// before applying truncation. +// +// If s already contains less than the limit number of bytes, it is returned +// unchanged. No invalid characters are removed. +func truncate(limit int, s string) string { + // This prioritize performance in the following order based on the most + // common expected use-cases. + // + // - Short values less than the default limit (128). + // - Strings with valid encodings that exceed the limit. + // - No limit. + // - Strings with invalid encodings that exceed the limit. + if limit < 0 || len(s) <= limit { + return s + } + + // Optimistically, assume all valid UTF-8. + var b strings.Builder + count := 0 + for i, c := range s { + if c != utf8.RuneError { + count++ + if count > limit { + return s[:i] + } + continue + } + + _, size := utf8.DecodeRuneInString(s[i:]) + if size == 1 { + // Invalid encoding. + b.Grow(len(s) - 1) + _, _ = b.WriteString(s[:i]) + s = s[i:] + break + } + } + + // Fast-path, no invalid input. + if b.Cap() == 0 { + return s + } + + // Truncate while validating UTF-8. + for i := 0; i < len(s) && count < limit; { + c := s[i] + if c < utf8.RuneSelf { + // Optimization for single byte runes (common case). + _ = b.WriteByte(c) + i++ + count++ + continue + } + + _, size := utf8.DecodeRuneInString(s[i:]) + if size == 1 { + // We checked for all 1-byte runes above, this is a RuneError. + i++ + continue + } + + _, _ = b.WriteString(s[i : i+size]) + i += size + count++ + } + + return b.String() +} diff --git a/sdk/log/internal/attrnorm/truncate_test.go b/sdk/log/internal/attrnorm/truncate_test.go new file mode 100644 index 00000000000..ad19bbbc09c --- /dev/null +++ b/sdk/log/internal/attrnorm/truncate_test.go @@ -0,0 +1,604 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// DO NOT MODIFY. Generated by gotmpl. +// source: internal/shared/attrnorm/truncate_test.go.tmpl + +package attrnorm + +import ( + "bytes" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/otel/attribute" +) + +func TestTruncateAttr(t *testing.T) { + const key = "key" + + strAttr := attribute.String(key, "value") + bytesAttr := attribute.ByteSlice(key, []byte("value")) + strSliceAttr := attribute.StringSlice(key, []string{"value-0", "value-1"}) + + tests := []struct { + limit int + attr, want attribute.KeyValue + }{ + { + limit: -1, + attr: strAttr, + want: strAttr, + }, + { + limit: -1, + attr: strSliceAttr, + want: strSliceAttr, + }, + { + limit: -1, + attr: bytesAttr, + want: bytesAttr, + }, + { + limit: 0, + attr: attribute.Bool(key, true), + want: attribute.Bool(key, true), + }, + { + limit: 0, + attr: attribute.BoolSlice(key, []bool{true, false}), + want: attribute.BoolSlice(key, []bool{true, false}), + }, + { + limit: 0, + attr: attribute.Int(key, 42), + want: attribute.Int(key, 42), + }, + { + limit: 0, + attr: attribute.IntSlice(key, []int{42, -1}), + want: attribute.IntSlice(key, []int{42, -1}), + }, + { + limit: 0, + attr: attribute.Int64(key, 42), + want: attribute.Int64(key, 42), + }, + { + limit: 0, + attr: attribute.Int64Slice(key, []int64{42, -1}), + want: attribute.Int64Slice(key, []int64{42, -1}), + }, + { + limit: 0, + attr: attribute.Float64(key, 42), + want: attribute.Float64(key, 42), + }, + { + limit: 0, + attr: attribute.Float64Slice(key, []float64{42, -1}), + want: attribute.Float64Slice(key, []float64{42, -1}), + }, + { + limit: 0, + attr: strAttr, + want: attribute.String(key, ""), + }, + { + limit: 0, + attr: strSliceAttr, + want: attribute.StringSlice(key, []string{"", ""}), + }, + { + limit: 0, + attr: attribute.Stringer(key, bytes.NewBufferString("value")), + want: attribute.String(key, ""), + }, + { + limit: 0, + attr: bytesAttr, + want: attribute.ByteSlice(key, []byte{}), + }, + { + limit: 1, + attr: strAttr, + want: attribute.String(key, "v"), + }, + { + limit: 1, + attr: strSliceAttr, + want: attribute.StringSlice(key, []string{"v", "v"}), + }, + { + limit: 1, + attr: bytesAttr, + want: attribute.ByteSlice(key, []byte("v")), + }, + { + limit: 5, + attr: strAttr, + want: strAttr, + }, + { + limit: 5, + attr: bytesAttr, + want: bytesAttr, + }, + { + limit: 7, + attr: strSliceAttr, + want: strSliceAttr, + }, + { + limit: 6, + attr: attribute.StringSlice(key, []string{"value", "value-1"}), + want: attribute.StringSlice(key, []string{"value", "value-"}), + }, + { + limit: 128, + attr: strAttr, + want: strAttr, + }, + { + limit: 128, + attr: strSliceAttr, + want: strSliceAttr, + }, + { + limit: 128, + attr: bytesAttr, + want: bytesAttr, + }, + { + // Multi-byte string: byte length (9) exceeds limit (5) but rune count (3) does not. + // Must not be truncated. + limit: 5, + attr: attribute.String(key, "日本語"), + want: attribute.String(key, "日本語"), + }, + { + // Multi-byte string: both byte length and rune count exceed limit. + // Must be truncated to limit runes. + limit: 2, + attr: attribute.String(key, "日本語"), + want: attribute.String(key, "日本"), + }, + { + // STRINGSLICE with multi-byte elements: byte lengths exceed limit but rune counts do not. + // Must not be truncated. + limit: 1, + attr: attribute.StringSlice(key, []string{"日", "本"}), + want: attribute.StringSlice(key, []string{"日", "本"}), + }, + // SLICE cases + { + limit: -1, + attr: attribute.Slice(key, attribute.StringValue("value")), + want: attribute.Slice(key, attribute.StringValue("value")), + }, + { + limit: 0, + attr: attribute.Slice(key, attribute.BoolValue(true), attribute.StringValue("value")), + want: attribute.Slice(key, attribute.BoolValue(true), attribute.StringValue("")), + }, + { + limit: 5, + attr: attribute.Slice(key, attribute.StringValue("value"), attribute.StringValue("toolong")), + want: attribute.Slice(key, attribute.StringValue("value"), attribute.StringValue("toolo")), + }, + { + // Nested SLICE: recursive truncation. + limit: 1, + attr: attribute.Slice(key, attribute.SliceValue(attribute.StringValue("value"))), + want: attribute.Slice(key, attribute.SliceValue(attribute.StringValue("v"))), + }, + { + // STRINGSLICE within SLICE: each string element is truncated. + limit: 2, + attr: attribute.Slice(key, attribute.StringSliceValue([]string{"abc", "de"})), + want: attribute.Slice(key, attribute.StringSliceValue([]string{"ab", "de"})), + }, + { + // STRINGSLICE within SLICE where all strings fit: no change. + // Exercises needsTruncation(STRINGSLICE) exhausting the loop without + // finding an over-limit string, returning false. + limit: 7, + attr: attribute.Slice(key, attribute.StringSliceValue([]string{"value-0", "value-1"})), + want: attribute.Slice(key, attribute.StringSliceValue([]string{"value-0", "value-1"})), + }, + { + // Mixed SLICE: STRINGSLICE (all strings fit) + STRING (too long). + // Exercises recursive truncation over mixed slice elements: the + // STRINGSLICE element remains unchanged because each string fits + // within the limit, while the sibling STRING element is truncated. + limit: 3, + attr: attribute.Slice( + key, + attribute.StringSliceValue([]string{"ab", "cd"}), + attribute.StringValue("toolong"), + ), + want: attribute.Slice( + key, + attribute.StringSliceValue([]string{"ab", "cd"}), + attribute.StringValue("too"), + ), + }, + { + // Nested SLICE (no truncation needed) alongside STRING (needs truncation). + // Exercises the TruncateValue SLICE branch early-return path: TruncateValue + // is called recursively on the nested SLICE but returns it unchanged because + // none of its elements require truncation. + limit: 3, + attr: attribute.Slice( + key, + attribute.SliceValue(attribute.BoolValue(true)), + attribute.StringValue("toolong"), + ), + want: attribute.Slice( + key, + attribute.SliceValue(attribute.BoolValue(true)), + attribute.StringValue("too"), + ), + }, + { + // Multi-byte string whose byte length exceeds the limit but rune count + // does not: must not be truncated (guards use rune count, not byte length). + limit: 3, + attr: attribute.Slice(key, attribute.StringValue("日本語")), // 3 runes, 9 bytes + want: attribute.Slice(key, attribute.StringValue("日本語")), + }, + { + // SLICE with invalid UTF-8 where rune count equals the limit: + // invalid byte is dropped. + limit: 2, + attr: attribute.Slice(key, attribute.StringValue("日\x80")), // 2 runes (日 + invalid byte), 4 bytes + want: attribute.Slice(key, attribute.StringValue("日")), + }, + { + // BYTESLICE within SLICE: each byte slice is truncated. + limit: 2, + attr: attribute.Slice(key, attribute.ByteSliceValue([]byte{1, 2, 3})), + want: attribute.Slice(key, attribute.ByteSliceValue([]byte{1, 2})), + }, + { + // BYTESLICE within SLICE: no truncation needed. + limit: 5, + attr: attribute.Slice(key, attribute.ByteSliceValue([]byte{1, 2})), + want: attribute.Slice(key, attribute.ByteSliceValue([]byte{1, 2})), + }, + { + // Mixed SLICE: BYTESLICE + STRING (both need truncation). + limit: 2, + attr: attribute.Slice( + key, + attribute.ByteSliceValue([]byte{1, 2, 3}), + attribute.StringValue("abc"), + ), + want: attribute.Slice( + key, + attribute.ByteSliceValue([]byte{1, 2}), + attribute.StringValue("ab"), + ), + }, + // MAP cases + { + limit: -1, + attr: attribute.Map(key, attribute.String("value", "value")), + want: attribute.Map(key, attribute.String("value", "value")), + }, + { + limit: 0, + attr: attribute.Map(key, attribute.Bool("ok", true), attribute.String("value", "value")), + want: attribute.Map(key, attribute.Bool("ok", true), attribute.String("value", "")), + }, + { + limit: 5, + attr: attribute.Map( + key, + attribute.String("short", "value"), + attribute.String("long", "toolong"), + ), + want: attribute.Map( + key, + attribute.String("short", "value"), + attribute.String("long", "toolo"), + ), + }, + { + // STRINGSLICE within MAP: each string element is truncated. + limit: 2, + attr: attribute.Map(key, attribute.StringSlice("strings", []string{"abc", "de"})), + want: attribute.Map(key, attribute.StringSlice("strings", []string{"ab", "de"})), + }, + { + // BYTESLICE within MAP: each byte slice is truncated. + limit: 2, + attr: attribute.Map(key, attribute.ByteSlice("bytes", []byte{1, 2, 3})), + want: attribute.Map(key, attribute.ByteSlice("bytes", []byte{1, 2})), + }, + { + // Nested MAP: recursive truncation. + limit: 1, + attr: attribute.Map(key, attribute.Map("map", attribute.String("nested", "value"))), + want: attribute.Map(key, attribute.Map("map", attribute.String("nested", "v"))), + }, + { + // SLICE within MAP: recursive truncation. + limit: 2, + attr: attribute.Map( + key, + attribute.Slice( + "slice", + attribute.StringValue("abc"), + attribute.MapValue(attribute.String("nested", "abc")), + ), + ), + want: attribute.Map( + key, + attribute.Slice( + "slice", + attribute.StringValue("ab"), + attribute.MapValue(attribute.String("nested", "ab")), + ), + ), + }, + { + // MAP within SLICE: recursive truncation. + limit: 2, + attr: attribute.Slice(key, attribute.MapValue(attribute.String("nested", "value"))), + want: attribute.Slice(key, attribute.MapValue(attribute.String("nested", "va"))), + }, + { + // Multi-byte string whose byte length exceeds the limit but rune count + // does not: must not be truncated. + limit: 3, + attr: attribute.Map(key, attribute.String("string", "日本語")), // 3 runes, 9 bytes + want: attribute.Map(key, attribute.String("string", "日本語")), + }, + { + // MAP with invalid UTF-8 where rune count equals the limit: + // invalid byte is dropped. + limit: 2, + attr: attribute.Map(key, attribute.String("string", "日\x80")), // 2 runes, 4 bytes + want: attribute.Map(key, attribute.String("string", "日")), + }, + { + // Duplicate MAP entries are truncated but not dropped. + limit: 2, + attr: attribute.Map( + key, + attribute.String("dup", "abc"), + attribute.String("dup", "de"), + ), + want: attribute.Map( + key, + attribute.String("dup", "ab"), + attribute.String("dup", "de"), + ), + }, + } + + for _, test := range tests { + name := fmt.Sprintf("%s->%s(limit:%d)", test.attr.Key, test.attr.Value.String(), test.limit) + t.Run(name, func(t *testing.T) { + assert.Equal(t, test.want, Truncate(test.limit, test.attr)) + }) + } +} + +func TestTruncateValue(t *testing.T) { + tests := []struct { + name string + limit int + value, want attribute.Value + }{ + { + name: "NegativeLimit", + limit: -1, + value: attribute.StringValue("value"), + want: attribute.StringValue("value"), + }, + { + name: "String", + limit: 2, + value: attribute.StringValue("value"), + want: attribute.StringValue("va"), + }, + { + name: "Map", + limit: 2, + value: attribute.MapValue(attribute.String("key", "value")), + want: attribute.MapValue(attribute.String("key", "va")), + }, + { + name: "UnchangedMap", + limit: 5, + value: attribute.MapValue(attribute.String("key", "value")), + want: attribute.MapValue(attribute.String("key", "value")), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.want, TruncateValue(test.limit, test.value)) + }) + } +} + +func TestTruncateString(t *testing.T) { + type group struct { + limit int + input string + expected string + } + + tests := []struct { + name string + groups []group + }{ + // Edge case: limit is negative, no truncation should occur + { + name: "NoTruncation", + groups: []group{ + {-1, "No truncation!", "No truncation!"}, + }, + }, + + // Edge case: string is already shorter than the limit, no truncation + // should occur + { + name: "ShortText", + groups: []group{ + {10, "Short text", "Short text"}, + {15, "Short text", "Short text"}, + {100, "Short text", "Short text"}, + }, + }, + + // Edge case: truncation happens with ASCII characters only + { + name: "ASCIIOnly", + groups: []group{ + {1, "Hello World!", "H"}, + {5, "Hello World!", "Hello"}, + {12, "Hello World!", "Hello World!"}, + }, + }, + + // Truncation including multi-byte characters (UTF-8) + { + name: "ValidUTF-8", + groups: []group{ + {7, "Hello, 世界", "Hello, "}, + {8, "Hello, 世界", "Hello, 世"}, + {2, "こんにちは", "こん"}, + {3, "こんにちは", "こんに"}, + {5, "こんにちは", "こんにちは"}, + {12, "こんにちは", "こんにちは"}, + }, + }, + + // Truncation with invalid UTF-8 characters + { + name: "InvalidUTF-8", + groups: []group{ + {11, "Invalid\x80text", "Invalidtext"}, + // Do not modify invalid text if equal to limit. + {11, "Valid text\x80", "Valid text\x80"}, + // Do not modify invalid text if under limit. + {15, "Valid text\x80", "Valid text\x80"}, + {5, "Hello\x80World", "Hello"}, + {11, "Hello\x80World\x80!", "HelloWorld!"}, + {15, "Hello\x80World\x80Test", "HelloWorldTest"}, + {15, "Hello\x80\x80\x80World\x80Test", "HelloWorldTest"}, + {15, "\x80\x80\x80Hello\x80\x80\x80World\x80Test\x80\x80", "HelloWorldTest"}, + }, + }, + + // Truncation with mixed validn and invalid UTF-8 characters + { + name: "MixedUTF-8", + groups: []group{ + {6, "€"[0:2] + "hello€€", "hello€"}, + {6, "€" + "€"[0:2] + "hello", "€hello"}, + {11, "Valid text\x80📜", "Valid text📜"}, + {11, "Valid text📜\x80", "Valid text📜"}, + {14, "😊 Hello\x80World🌍🚀", "😊 HelloWorld🌍🚀"}, + {14, "😊\x80 Hello\x80World🌍🚀", "😊 HelloWorld🌍🚀"}, + {14, "😊\x80 Hello\x80World🌍\x80🚀", "😊 HelloWorld🌍🚀"}, + {14, "😊\x80 Hello\x80World🌍\x80🚀\x80", "😊 HelloWorld🌍🚀"}, + {14, "\x80😊\x80 Hello\x80World🌍\x80🚀\x80", "😊 HelloWorld🌍🚀"}, + }, + }, + + // Edge case: empty string, should return empty string + { + name: "Empty", + groups: []group{ + {5, "", ""}, + }, + }, + + // Edge case: limit is 0, should return an empty string + { + name: "Zero", + groups: []group{ + {0, "Some text", ""}, + {0, "", ""}, + }, + }, + } + + for _, tt := range tests { + for _, g := range tt.groups { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := truncate(g.limit, g.input) + assert.Equalf( + t, g.expected, got, + "input: %q([]rune%v))\ngot: %q([]rune%v)\nwant %q([]rune%v)", + g.input, []rune(g.input), + got, []rune(got), + g.expected, []rune(g.expected), + ) + }) + } + } +} + +func BenchmarkTruncateAttr(b *testing.B) { + const key = "key" + + strAttr := attribute.String(key, "value") + bytesAttr := attribute.ByteSlice(key, []byte("value")) + strSliceAttr := attribute.StringSlice(key, []string{"value-0", "value-1"}) + + run := func(limit int, attr attribute.KeyValue) func(b *testing.B) { + return func(b *testing.B) { + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + var out attribute.KeyValue + for pb.Next() { + out = Truncate(limit, attr) + } + _ = out + }) + } + } + + b.Run("String", run(3, strAttr)) + b.Run("StringSlice", run(3, strSliceAttr)) + b.Run("ByteSlice", run(3, bytesAttr)) + b.Run("String/Limit0", run(0, strAttr)) + b.Run("StringSlice/Limit0", run(0, strSliceAttr)) + b.Run("ByteSlice/Limit0", run(0, bytesAttr)) + b.Run("String/Unlimited", run(-1, strAttr)) + b.Run("StringSlice/Unlimited", run(-1, strSliceAttr)) + b.Run("ByteSlice/Unlimited", run(-1, bytesAttr)) +} + +func BenchmarkTruncate(b *testing.B) { + run := func(limit int, input string) func(b *testing.B) { + return func(b *testing.B) { + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + var out string + for pb.Next() { + out = truncate(limit, input) + } + _ = out + }) + } + } + b.Run("Unlimited", run(-1, "hello 😊 world 🌍🚀")) + b.Run("Zero", run(0, "Some text")) + b.Run("Short", run(10, "Short Text")) + b.Run("ASCII", run(5, "Hello, World!")) + b.Run("ValidUTF-8", run(10, "hello 😊 world 🌍🚀")) + b.Run("InvalidUTF-8", run(6, "€"[0:2]+"hello€€")) + b.Run("MixedUTF-8", run(14, "\x80😊\x80 Hello\x80World🌍\x80🚀\x80")) +} diff --git a/sdk/log/internal/gen.go b/sdk/log/internal/gen.go index 64ab0d16411..d07ef94e21a 100644 --- a/sdk/log/internal/gen.go +++ b/sdk/log/internal/gen.go @@ -6,8 +6,9 @@ package internal //go:generate gotmpl --body=../../../internal/shared/x/x.go.tmpl "--data={ \"pkg\": \"go.opentelemetry.io/otel/sdk/log\" }" --out=x/x.go //go:generate gotmpl --body=../../../internal/shared/x/x_test.go.tmpl "--data={}" --out=x/x_test.go - +//go:generate gotmpl --body=../../../internal/shared/attrnorm/dedup.go.tmpl "--data={}" --out=attrnorm/dedup.go +//go:generate gotmpl --body=../../../internal/shared/attrnorm/dedup_test.go.tmpl "--data={}" --out=attrnorm/dedup_test.go +//go:generate gotmpl --body=../../../internal/shared/attrnorm/truncate.go.tmpl "--data={}" --out=attrnorm/truncate.go +//go:generate gotmpl --body=../../../internal/shared/attrnorm/truncate_test.go.tmpl "--data={}" --out=attrnorm/truncate_test.go //go:generate gotmpl --body=../../../internal/shared/counter/counter.go.tmpl "--data={ \"pkg\": \"go.opentelemetry.io/otel/sdk/log\" }" --out=counter/counter.go //go:generate gotmpl --body=../../../internal/shared/counter/counter_test.go.tmpl "--data={}" --out=counter/counter_test.go -//go:generate gotmpl --body=../../../internal/shared/attrdedup/dedup.go.tmpl "--data={}" --out=attrdedup/dedup.go -//go:generate gotmpl --body=../../../internal/shared/attrdedup/dedup_test.go.tmpl "--data={}" --out=attrdedup/dedup_test.go diff --git a/sdk/log/provider.go b/sdk/log/provider.go index b71cb2f4162..6689c54c57a 100644 --- a/sdk/log/provider.go +++ b/sdk/log/provider.go @@ -15,7 +15,7 @@ import ( "go.opentelemetry.io/otel/log/embedded" "go.opentelemetry.io/otel/log/noop" "go.opentelemetry.io/otel/sdk/instrumentation" - "go.opentelemetry.io/otel/sdk/log/internal/attrdedup" + "go.opentelemetry.io/otel/sdk/log/internal/attrnorm" "go.opentelemetry.io/otel/sdk/resource" ) @@ -121,7 +121,7 @@ func (p *LoggerProvider) Logger(name string, opts ...log.LoggerOption) log.Logge cfg := log.NewLoggerConfig(opts...) attrs := cfg.InstrumentationAttributes() if !p.allowDupKeys { - attrs, _ = attrdedup.Set(attrs) + attrs, _ = attrnorm.Set(attrs) } scope := instrumentation.Scope{ Name: name, diff --git a/sdk/log/record.go b/sdk/log/record.go index 2949fe8b1a3..6f64e04e3d7 100644 --- a/sdk/log/record.go +++ b/sdk/log/record.go @@ -5,16 +5,14 @@ package log import ( "slices" - "strings" "sync" "time" - "unicode/utf8" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/internal/global" "go.opentelemetry.io/otel/log" "go.opentelemetry.io/otel/sdk/instrumentation" - "go.opentelemetry.io/otel/sdk/log/internal/attrdedup" + "go.opentelemetry.io/otel/sdk/log/internal/attrnorm" "go.opentelemetry.io/otel/sdk/resource" "go.opentelemetry.io/otel/trace" ) @@ -195,7 +193,7 @@ func (r *Record) Body() attribute.Value { // SetBody sets the body of the log record. func (r *Record) SetBody(v attribute.Value) { if !r.allowDupKeys { - r.body, _ = attrdedup.Value(v) + r.body, _ = attrnorm.Value(v) } else { r.body = v } @@ -479,175 +477,11 @@ func (r *Record) Clone() Record { func (r *Record) applyAttrLimitsAndDedup(attr attribute.KeyValue) attribute.KeyValue { if !r.allowDupKeys { var changed bool - attr, changed = attrdedup.KeyValue(attr) + attr, changed = attrnorm.KeyValue(attr) if changed { logKeyValuePairDropped() } } - attr.Value = truncateValue(r.attributeValueLengthLimit, attr.Value) + attr.Value = attrnorm.TruncateValue(r.attributeValueLengthLimit, attr.Value) return attr } - -// truncateValue returns a truncated version of v. Only string, string slice, -// byte slice, and (recursively) slice and map values are modified. -// -// No truncation is performed for a negative limit. -func truncateValue(limit int, v attribute.Value) attribute.Value { - if limit < 0 { - return v - } - - switch v.Type() { - case attribute.STRING: - return attribute.StringValue(truncate(limit, v.AsString())) - case attribute.STRINGSLICE: - ss := v.AsStringSlice() - for i := range ss { - ss[i] = truncate(limit, ss[i]) - } - return attribute.StringSliceValue(ss) - case attribute.BYTESLICE: - // len(v.AsString()) is identical to len(v.AsByteSlice()) but - // avoids allocating the full slice before truncation. - s := v.AsString() - if limit >= 0 && len(s) > limit { - return attribute.ByteSliceValue([]byte(s[:limit])) - } - case attribute.SLICE: - sl := v.AsSlice() - if !slices.ContainsFunc(sl, func(e attribute.Value) bool { return needsTruncation(limit, e) }) { - return v - } - newSl := make([]attribute.Value, len(sl)) - for i, elem := range sl { - newSl[i] = truncateValue(limit, elem) - } - return attribute.SliceValue(newSl...) - case attribute.MAP: - m := v.AsMap() - if !slices.ContainsFunc(m, func(kv attribute.KeyValue) bool { return needsTruncation(limit, kv.Value) }) { - return v - } - newM := make([]attribute.KeyValue, len(m)) - for i, elem := range m { - elem.Value = truncateValue(limit, elem.Value) - newM[i] = elem - } - return attribute.MapValue(newM...) - } - return v -} - -// stringNeedsTruncation reports whether s would be modified by truncate for the -// given limit. -func stringNeedsTruncation(limit int, s string) bool { - if limit < 0 || len(s) <= limit { - return false - } - return utf8.RuneCountInString(s) > limit || !utf8.ValidString(s) -} - -// needsTruncation reports whether v would be modified by truncateValue for the -// given limit. -func needsTruncation(limit int, v attribute.Value) bool { - switch v.Type() { - case attribute.STRING: - return stringNeedsTruncation(limit, v.AsString()) - case attribute.BYTESLICE: - // len(v.AsString()) is identical to len(v.AsByteSlice()) but - // avoids memory allocation. - if limit >= 0 && len(v.AsString()) > limit { - return true - } - case attribute.STRINGSLICE: - for _, s := range v.AsStringSlice() { - if stringNeedsTruncation(limit, s) { - return true - } - } - case attribute.SLICE: - return slices.ContainsFunc(v.AsSlice(), func(e attribute.Value) bool { return needsTruncation(limit, e) }) - case attribute.MAP: - return slices.ContainsFunc( - v.AsMap(), - func(kv attribute.KeyValue) bool { return needsTruncation(limit, kv.Value) }, - ) - } - return false -} - -// truncate returns a truncated version of s such that it contains less than -// the limit number of characters. Truncation is applied by returning the limit -// number of valid characters contained in s. -// -// If limit is negative, it returns the original string. -// -// UTF-8 is supported. When truncating, all invalid characters are dropped -// before applying truncation. -// -// If s already contains less than the limit number of bytes, it is returned -// unchanged. No invalid characters are removed. -func truncate(limit int, s string) string { - // This prioritize performance in the following order based on the most - // common expected use-cases. - // - // - Short values less than the default limit (128). - // - Strings with valid encodings that exceed the limit. - // - No limit. - // - Strings with invalid encodings that exceed the limit. - if limit < 0 || len(s) <= limit { - return s - } - - // Optimistically, assume all valid UTF-8. - var b strings.Builder - count := 0 - for i, c := range s { - if c != utf8.RuneError { - count++ - if count > limit { - return s[:i] - } - continue - } - - _, size := utf8.DecodeRuneInString(s[i:]) - if size == 1 { - // Invalid encoding. - b.Grow(len(s) - 1) - _, _ = b.WriteString(s[:i]) - s = s[i:] - break - } - } - - // Fast-path, no invalid input. - if b.Cap() == 0 { - return s - } - - // Truncate while validating UTF-8. - for i := 0; i < len(s) && count < limit; { - c := s[i] - if c < utf8.RuneSelf { - // Optimization for single byte runes (common case). - _ = b.WriteByte(c) - i++ - count++ - continue - } - - _, size := utf8.DecodeRuneInString(s[i:]) - if size == 1 { - // We checked for all 1-byte runes above, this is a RuneError. - i++ - continue - } - - _, _ = b.WriteString(s[i : i+size]) - i += size - count++ - } - - return b.String() -} diff --git a/sdk/log/record_test.go b/sdk/log/record_test.go index 7699a37b580..22a60d44259 100644 --- a/sdk/log/record_test.go +++ b/sdk/log/record_test.go @@ -1465,128 +1465,6 @@ func assertKV(t *testing.T, r Record, kv attribute.KeyValue) { assert.Truef(t, keyValueEqual(kv, kvs[0]), "%s != %s", kv, kvs[0]) } -func TestTruncate(t *testing.T) { - type group struct { - limit int - input string - expected string - } - - tests := []struct { - name string - groups []group - }{ - // Edge case: limit is negative, no truncation should occur - { - name: "NoTruncation", - groups: []group{ - {-1, "No truncation!", "No truncation!"}, - }, - }, - - // Edge case: string is already shorter than the limit, no truncation - // should occur - { - name: "ShortText", - groups: []group{ - {10, "Short text", "Short text"}, - {15, "Short text", "Short text"}, - {100, "Short text", "Short text"}, - }, - }, - - // Edge case: truncation happens with ASCII characters only - { - name: "ASCIIOnly", - groups: []group{ - {1, "Hello World!", "H"}, - {5, "Hello World!", "Hello"}, - {12, "Hello World!", "Hello World!"}, - }, - }, - - // Truncation including multi-byte characters (UTF-8) - { - name: "ValidUTF-8", - groups: []group{ - {7, "Hello, 世界", "Hello, "}, - {8, "Hello, 世界", "Hello, 世"}, - {2, "こんにちは", "こん"}, - {3, "こんにちは", "こんに"}, - {5, "こんにちは", "こんにちは"}, - {12, "こんにちは", "こんにちは"}, - }, - }, - - // Truncation with invalid UTF-8 characters - { - name: "InvalidUTF-8", - groups: []group{ - {11, "Invalid\x80text", "Invalidtext"}, - // Do not modify invalid text if equal to limit. - {11, "Valid text\x80", "Valid text\x80"}, - // Do not modify invalid text if under limit. - {15, "Valid text\x80", "Valid text\x80"}, - {5, "Hello\x80World", "Hello"}, - {11, "Hello\x80World\x80!", "HelloWorld!"}, - {15, "Hello\x80World\x80Test", "HelloWorldTest"}, - {15, "Hello\x80\x80\x80World\x80Test", "HelloWorldTest"}, - {15, "\x80\x80\x80Hello\x80\x80\x80World\x80Test\x80\x80", "HelloWorldTest"}, - }, - }, - - // Truncation with mixed validn and invalid UTF-8 characters - { - name: "MixedUTF-8", - groups: []group{ - {6, "€"[0:2] + "hello€€", "hello€"}, - {6, "€" + "€"[0:2] + "hello", "€hello"}, - {11, "Valid text\x80📜", "Valid text📜"}, - {11, "Valid text📜\x80", "Valid text📜"}, - {14, "😊 Hello\x80World🌍🚀", "😊 HelloWorld🌍🚀"}, - {14, "😊\x80 Hello\x80World🌍🚀", "😊 HelloWorld🌍🚀"}, - {14, "😊\x80 Hello\x80World🌍\x80🚀", "😊 HelloWorld🌍🚀"}, - {14, "😊\x80 Hello\x80World🌍\x80🚀\x80", "😊 HelloWorld🌍🚀"}, - {14, "\x80😊\x80 Hello\x80World🌍\x80🚀\x80", "😊 HelloWorld🌍🚀"}, - }, - }, - - // Edge case: empty string, should return empty string - { - name: "Empty", - groups: []group{ - {5, "", ""}, - }, - }, - - // Edge case: limit is 0, should return an empty string - { - name: "Zero", - groups: []group{ - {0, "Some text", ""}, - {0, "", ""}, - }, - }, - } - - for _, tt := range tests { - for _, g := range tt.groups { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - got := truncate(g.limit, g.input) - assert.Equalf( - t, g.expected, got, - "input: %q([]rune%v))\ngot: %q([]rune%v)\nwant %q([]rune%v)", - g.input, []rune(g.input), - got, []rune(got), - g.expected, []rune(g.expected), - ) - }) - } - } -} - func TestRecordAddAttributesDoesNotMutateInput(t *testing.T) { attrs := []attribute.KeyValue{ attribute.String("attr1", "very long value that will be truncated"), @@ -1719,28 +1597,6 @@ func printKVs(kvs []attribute.KeyValue) string { return sb.String() } -func BenchmarkTruncate(b *testing.B) { - run := func(limit int, input string) func(b *testing.B) { - return func(b *testing.B) { - b.ReportAllocs() - b.RunParallel(func(pb *testing.PB) { - var out string - for pb.Next() { - out = truncate(limit, input) - } - _ = out - }) - } - } - b.Run("Unlimited", run(-1, "hello 😊 world 🌍🚀")) - b.Run("Zero", run(0, "Some text")) - b.Run("Short", run(10, "Short Text")) - b.Run("ASCII", run(5, "Hello, World!")) - b.Run("ValidUTF-8", run(10, "hello 😊 world 🌍🚀")) - b.Run("InvalidUTF-8", run(6, "€"[0:2]+"hello€€")) - b.Run("MixedUTF-8", run(14, "\x80😊\x80 Hello\x80World🌍\x80🚀\x80")) -} - func BenchmarkWalkAttributes(b *testing.B) { for _, tt := range []struct { attrCount int diff --git a/sdk/metric/instrument.go b/sdk/metric/instrument.go index 46451a47239..31f7afef903 100644 --- a/sdk/metric/instrument.go +++ b/sdk/metric/instrument.go @@ -16,7 +16,7 @@ import ( "go.opentelemetry.io/otel/metric/embedded" "go.opentelemetry.io/otel/sdk/instrumentation" "go.opentelemetry.io/otel/sdk/metric/internal/aggregate" - "go.opentelemetry.io/otel/sdk/metric/internal/attrdedup" + "go.opentelemetry.io/otel/sdk/metric/internal/attrnorm" ) var zeroScope instrumentation.Scope @@ -207,11 +207,11 @@ func extractRawKVs[T any](opts []T) []attribute.KeyValue { } func resolveAttributes(configAttrs attribute.Set, rawKVs []attribute.KeyValue) attribute.Set { - configAttrs, _ = attrdedup.Set(configAttrs) + configAttrs, _ = attrnorm.Set(configAttrs) if len(rawKVs) == 0 { return configAttrs } - rawKVs, _ = attrdedup.KeyValues(rawKVs) + rawKVs, _ = attrnorm.KeyValues(rawKVs) merged := make([]attribute.KeyValue, 0, configAttrs.Len()+len(rawKVs)) merged = append(merged, configAttrs.ToSlice()...) // rawKVs are appended after configAttrs, meaning they will override any duplicate keys in configAttrs. diff --git a/sdk/log/internal/attrdedup/dedup.go b/sdk/metric/internal/attrnorm/dedup.go similarity index 98% rename from sdk/log/internal/attrdedup/dedup.go rename to sdk/metric/internal/attrnorm/dedup.go index c5ef89dd7af..fe9ddc96db7 100644 --- a/sdk/log/internal/attrdedup/dedup.go +++ b/sdk/metric/internal/attrnorm/dedup.go @@ -2,10 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 // DO NOT MODIFY. Generated by gotmpl. -// source: internal/shared/attrdedup/dedup.go.tmpl +// source: internal/shared/attrnorm/dedup.go.tmpl -// Package attrdedup deduplicates attribute map values. -package attrdedup +// Package attrnorm normalizes attribute values. +package attrnorm import ( "reflect" diff --git a/sdk/log/internal/attrdedup/dedup_test.go b/sdk/metric/internal/attrnorm/dedup_test.go similarity index 99% rename from sdk/log/internal/attrdedup/dedup_test.go rename to sdk/metric/internal/attrnorm/dedup_test.go index 82fab043ab4..0e6b10b193a 100644 --- a/sdk/log/internal/attrdedup/dedup_test.go +++ b/sdk/metric/internal/attrnorm/dedup_test.go @@ -2,9 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 // DO NOT MODIFY. Generated by gotmpl. -// source: internal/shared/attrdedup/dedup_test.go.tmpl +// source: internal/shared/attrnorm/dedup_test.go.tmpl -package attrdedup +package attrnorm import ( "testing" diff --git a/sdk/metric/internal/gen.go b/sdk/metric/internal/gen.go index 500379796a3..5f077dfe93b 100644 --- a/sdk/metric/internal/gen.go +++ b/sdk/metric/internal/gen.go @@ -6,5 +6,5 @@ package internal //go:generate gotmpl --body=../../../internal/shared/x/x.go.tmpl "--data={ \"pkg\": \"go.opentelemetry.io/otel/sdk/metric\" }" --out=x/x.go //go:generate gotmpl --body=../../../internal/shared/x/x_test.go.tmpl "--data={}" --out=x/x_test.go -//go:generate gotmpl --body=../../../internal/shared/attrdedup/dedup.go.tmpl "--data={}" --out=attrdedup/dedup.go -//go:generate gotmpl --body=../../../internal/shared/attrdedup/dedup_test.go.tmpl "--data={}" --out=attrdedup/dedup_test.go +//go:generate gotmpl --body=../../../internal/shared/attrnorm/dedup.go.tmpl "--data={}" --out=attrnorm/dedup.go +//go:generate gotmpl --body=../../../internal/shared/attrnorm/dedup_test.go.tmpl "--data={}" --out=attrnorm/dedup_test.go diff --git a/sdk/metric/provider.go b/sdk/metric/provider.go index f2b21555f76..9f9f01f4323 100644 --- a/sdk/metric/provider.go +++ b/sdk/metric/provider.go @@ -12,7 +12,7 @@ import ( "go.opentelemetry.io/otel/metric/embedded" "go.opentelemetry.io/otel/metric/noop" "go.opentelemetry.io/otel/sdk/instrumentation" - "go.opentelemetry.io/otel/sdk/metric/internal/attrdedup" + "go.opentelemetry.io/otel/sdk/metric/internal/attrnorm" ) // MeterProvider handles the creation and coordination of Meters. All Meters @@ -77,7 +77,7 @@ func (mp *MeterProvider) Meter(name string, options ...metric.MeterOption) metri } c := metric.NewMeterConfig(options...) - attrs, _ := attrdedup.Set(c.InstrumentationAttributes()) + attrs, _ := attrnorm.Set(c.InstrumentationAttributes()) s := instrumentation.Scope{ Name: name, Version: c.InstrumentationVersion(), diff --git a/sdk/resource/resource.go b/sdk/resource/resource.go index 3b1f9f40a66..68ab274d2f1 100644 --- a/sdk/resource/resource.go +++ b/sdk/resource/resource.go @@ -11,7 +11,7 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/sdk/internal/attrdedup" + "go.opentelemetry.io/otel/sdk/internal/attrnorm" "go.opentelemetry.io/otel/sdk/internal/x" ) @@ -90,7 +90,7 @@ func NewSchemaless(attrs ...attribute.KeyValue) *Resource { return &Resource{} } - attrs, _ = attrdedup.KeyValues(attrs) + attrs, _ = attrnorm.KeyValues(attrs) // Ensure attributes comply with the specification: // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.20.0/specification/common/README.md#attribute diff --git a/sdk/trace/provider.go b/sdk/trace/provider.go index 9d66d34e5d7..6b2368413a1 100644 --- a/sdk/trace/provider.go +++ b/sdk/trace/provider.go @@ -13,7 +13,7 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/internal/global" "go.opentelemetry.io/otel/sdk/instrumentation" - "go.opentelemetry.io/otel/sdk/internal/attrdedup" + "go.opentelemetry.io/otel/sdk/internal/attrnorm" "go.opentelemetry.io/otel/sdk/resource" "go.opentelemetry.io/otel/sdk/trace/internal/observ" "go.opentelemetry.io/otel/trace" @@ -150,7 +150,7 @@ func (p *TracerProvider) Tracer(name string, opts ...trace.TracerOption) trace.T return noop.NewTracerProvider().Tracer(name, opts...) } c := trace.NewTracerConfig(opts...) - attrs, _ := attrdedup.Set(c.InstrumentationAttributes()) + attrs, _ := attrnorm.Set(c.InstrumentationAttributes()) if name == "" { name = defaultTracerName } diff --git a/sdk/trace/span.go b/sdk/trace/span.go index 5c8deb3debe..0aae5f5bb2f 100644 --- a/sdk/trace/span.go +++ b/sdk/trace/span.go @@ -10,16 +10,14 @@ import ( "runtime" rt "runtime/trace" "slices" - "strings" "sync" "time" - "unicode/utf8" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/internal/global" "go.opentelemetry.io/otel/sdk/instrumentation" - "go.opentelemetry.io/otel/sdk/internal/attrdedup" + "go.opentelemetry.io/otel/sdk/internal/attrnorm" "go.opentelemetry.io/otel/sdk/resource" semconv "go.opentelemetry.io/otel/semconv/v1.43.0" "go.opentelemetry.io/otel/trace" @@ -271,7 +269,7 @@ func (s *recordingSpan) SetAttributes(attributes ...attribute.KeyValue) { continue } a = dedupAttr(a) - a = truncateAttr(s.tracer.provider.spanLimits.AttributeValueLengthLimit, a) + a = attrnorm.Truncate(s.tracer.provider.spanLimits.AttributeValueLengthLimit, a) s.attributes = append(s.attributes, a) } } @@ -332,7 +330,7 @@ func (s *recordingSpan) addOverCapAttrs(limit int, attrs []attribute.KeyValue) { if idx, ok := exists[a.Key]; ok { // Perform all updates before dropping, even when at capacity. a = dedupAttr(a) - a = truncateAttr(s.tracer.provider.spanLimits.AttributeValueLengthLimit, a) + a = attrnorm.Truncate(s.tracer.provider.spanLimits.AttributeValueLengthLimit, a) s.attributes[idx] = a continue } @@ -343,7 +341,7 @@ func (s *recordingSpan) addOverCapAttrs(limit int, attrs []attribute.KeyValue) { s.addDroppedAttr(1) } else { a = dedupAttr(a) - a = truncateAttr(s.tracer.provider.spanLimits.AttributeValueLengthLimit, a) + a = attrnorm.Truncate(s.tracer.provider.spanLimits.AttributeValueLengthLimit, a) s.attributes = append(s.attributes, a) exists[a.Key] = len(s.attributes) - 1 } @@ -353,227 +351,13 @@ func (s *recordingSpan) addOverCapAttrs(limit int, attrs []attribute.KeyValue) { func dedupAttr(attr attribute.KeyValue) attribute.KeyValue { switch attr.Value.Type() { case attribute.SLICE, attribute.MAP: - attr, _ = attrdedup.KeyValue(attr) + attr, _ = attrnorm.KeyValue(attr) return attr default: return attr } } -// truncateAttr returns a truncated version of attr. Only string, string -// slice, byte slice, slice, and map attribute values are truncated. String -// values are truncated to at most a length of limit. Each string slice value -// is truncated in this fashion (the slice length itself is unaffected), and -// byte slice values are truncated to at most limit bytes. For slice and map -// attribute values, the limit is applied recursively to contained values. -// -// No truncation is performed for a negative limit. -func truncateAttr(limit int, attr attribute.KeyValue) attribute.KeyValue { - if limit < 0 { - return attr - } - switch attr.Value.Type() { - case attribute.STRING: - v := attr.Value.AsString() - return attr.Key.String(truncate(limit, v)) - case attribute.STRINGSLICE: - v := attr.Value.AsStringSlice() - for i := range v { - v[i] = truncate(limit, v[i]) - } - return attr.Key.StringSlice(v) - case attribute.BYTESLICE: - v := attr.Value.AsString() - if len(v) > limit { - return attr.Key.ByteSlice([]byte(v[:limit])) - } - return attr - case attribute.SLICE: - v := attr.Value.AsSlice() - if !slices.ContainsFunc(v, func(e attribute.Value) bool { return needsTruncation(limit, e) }) { - return attr - } - newV := make([]attribute.Value, len(v)) - for i, elem := range v { - newV[i] = truncateValue(limit, elem) - } - return attr.Key.Slice(newV...) - case attribute.MAP: - v := attr.Value.AsMap() - if !slices.ContainsFunc(v, func(kv attribute.KeyValue) bool { return needsTruncation(limit, kv.Value) }) { - return attr - } - newV := make([]attribute.KeyValue, len(v)) - for i, elem := range v { - elem.Value = truncateValue(limit, elem.Value) - newV[i] = elem - } - return attr.Key.Map(newV...) - } - return attr -} - -// truncateValue returns a truncated version of v. Only string, string slice, -// byte slice, and (recursively) slice and map values are modified. -// -// No truncation is performed for a negative limit. -func truncateValue(limit int, v attribute.Value) attribute.Value { - switch v.Type() { - case attribute.STRING: - return attribute.StringValue(truncate(limit, v.AsString())) - case attribute.STRINGSLICE: - ss := v.AsStringSlice() - for i := range ss { - ss[i] = truncate(limit, ss[i]) - } - return attribute.StringSliceValue(ss) - - case attribute.BYTESLICE: - // len(v.AsString()) is identical to len(v.AsByteSlice()) but - // avoids allocating the full slice before truncation. - s := v.AsString() - if limit >= 0 && len(s) > limit { - return attribute.ByteSliceValue([]byte(s[:limit])) - } - case attribute.SLICE: - sl := v.AsSlice() - if !slices.ContainsFunc(sl, func(e attribute.Value) bool { return needsTruncation(limit, e) }) { - return v - } - newSl := make([]attribute.Value, len(sl)) - for i, elem := range sl { - newSl[i] = truncateValue(limit, elem) - } - return attribute.SliceValue(newSl...) - case attribute.MAP: - m := v.AsMap() - if !slices.ContainsFunc(m, func(kv attribute.KeyValue) bool { return needsTruncation(limit, kv.Value) }) { - return v - } - newM := make([]attribute.KeyValue, len(m)) - for i, elem := range m { - elem.Value = truncateValue(limit, elem.Value) - newM[i] = elem - } - return attribute.MapValue(newM...) - } - return v -} - -// stringNeedsTruncation reports whether s would be modified by truncate for the -// given limit. -func stringNeedsTruncation(limit int, s string) bool { - if limit < 0 || len(s) <= limit { - return false - } - return utf8.RuneCountInString(s) > limit || !utf8.ValidString(s) -} - -// needsTruncation reports whether v would be modified by truncateValue for the -// given limit. -func needsTruncation(limit int, v attribute.Value) bool { - switch v.Type() { - case attribute.STRING: - return stringNeedsTruncation(limit, v.AsString()) - case attribute.BYTESLICE: - // len(v.AsString()) is identical to len(v.AsByteSlice()) but - // avoids memory allocation. - if limit >= 0 && len(v.AsString()) > limit { - return true - } - case attribute.STRINGSLICE: - for _, s := range v.AsStringSlice() { - if stringNeedsTruncation(limit, s) { - return true - } - } - case attribute.SLICE: - return slices.ContainsFunc(v.AsSlice(), func(e attribute.Value) bool { return needsTruncation(limit, e) }) - case attribute.MAP: - return slices.ContainsFunc( - v.AsMap(), - func(kv attribute.KeyValue) bool { return needsTruncation(limit, kv.Value) }, - ) - } - return false -} - -// truncate returns a truncated version of s such that it contains less than -// the limit number of characters. Truncation is applied by returning the limit -// number of valid characters contained in s. -// -// If limit is negative, it returns the original string. -// -// UTF-8 is supported. When truncating, all invalid characters are dropped -// before applying truncation. -// -// If s already contains less than the limit number of bytes, it is returned -// unchanged. No invalid characters are removed. -func truncate(limit int, s string) string { - // This prioritize performance in the following order based on the most - // common expected use-cases. - // - // - Short values less than the default limit (128). - // - Strings with valid encodings that exceed the limit. - // - No limit. - // - Strings with invalid encodings that exceed the limit. - if limit < 0 || len(s) <= limit { - return s - } - - // Optimistically, assume all valid UTF-8. - var b strings.Builder - count := 0 - for i, c := range s { - if c != utf8.RuneError { - count++ - if count > limit { - return s[:i] - } - continue - } - - _, size := utf8.DecodeRuneInString(s[i:]) - if size == 1 { - // Invalid encoding. - b.Grow(len(s) - 1) - _, _ = b.WriteString(s[:i]) - s = s[i:] - break - } - } - - // Fast-path, no invalid input. - if b.Cap() == 0 { - return s - } - - // Truncate while validating UTF-8. - for i := 0; i < len(s) && count < limit; { - c := s[i] - if c < utf8.RuneSelf { - // Optimization for single byte runes (common case). - _ = b.WriteByte(c) - i++ - count++ - continue - } - - _, size := utf8.DecodeRuneInString(s[i:]) - if size == 1 { - // We checked for all 1-byte runes above, this is a RuneError. - i++ - continue - } - - _, _ = b.WriteString(s[i : i+size]) - i += size - count++ - } - - return b.String() -} - // End ends the span. This method does nothing if the span is already ended or // is not being recorded. // @@ -732,7 +516,7 @@ func (s *recordingSpan) AddEvent(name string, o ...trace.EventOption) { // This method assumes s.mu.Lock is held by the caller. func (s *recordingSpan) addEvent(name string, o ...trace.EventOption) { c := trace.NewEventConfig(o...) - attrs, _ := attrdedup.KeyValues(c.Attributes()) + attrs, _ := attrnorm.KeyValues(c.Attributes()) e := Event{Name: name, Attributes: attrs, Time: c.Timestamp()} // Discard attributes over limit. @@ -906,7 +690,7 @@ func (s *recordingSpan) AddLink(link trace.Link) { return } - attrs, _ := attrdedup.KeyValues(link.Attributes) + attrs, _ := attrnorm.KeyValues(link.Attributes) l := Link{SpanContext: link.SpanContext, Attributes: attrs} // Discard attributes over limit. diff --git a/sdk/trace/span_test.go b/sdk/trace/span_test.go index 8d7da770542..ec7ca5d60d4 100644 --- a/sdk/trace/span_test.go +++ b/sdk/trace/span_test.go @@ -4,7 +4,6 @@ package trace import ( - "bytes" "fmt" "testing" @@ -90,554 +89,6 @@ func TestSetStatus(t *testing.T) { } } -func TestTruncateAttr(t *testing.T) { - const key = "key" - - strAttr := attribute.String(key, "value") - bytesAttr := attribute.ByteSlice(key, []byte("value")) - strSliceAttr := attribute.StringSlice(key, []string{"value-0", "value-1"}) - - tests := []struct { - limit int - attr, want attribute.KeyValue - }{ - { - limit: -1, - attr: strAttr, - want: strAttr, - }, - { - limit: -1, - attr: strSliceAttr, - want: strSliceAttr, - }, - { - limit: -1, - attr: bytesAttr, - want: bytesAttr, - }, - { - limit: 0, - attr: attribute.Bool(key, true), - want: attribute.Bool(key, true), - }, - { - limit: 0, - attr: attribute.BoolSlice(key, []bool{true, false}), - want: attribute.BoolSlice(key, []bool{true, false}), - }, - { - limit: 0, - attr: attribute.Int(key, 42), - want: attribute.Int(key, 42), - }, - { - limit: 0, - attr: attribute.IntSlice(key, []int{42, -1}), - want: attribute.IntSlice(key, []int{42, -1}), - }, - { - limit: 0, - attr: attribute.Int64(key, 42), - want: attribute.Int64(key, 42), - }, - { - limit: 0, - attr: attribute.Int64Slice(key, []int64{42, -1}), - want: attribute.Int64Slice(key, []int64{42, -1}), - }, - { - limit: 0, - attr: attribute.Float64(key, 42), - want: attribute.Float64(key, 42), - }, - { - limit: 0, - attr: attribute.Float64Slice(key, []float64{42, -1}), - want: attribute.Float64Slice(key, []float64{42, -1}), - }, - { - limit: 0, - attr: strAttr, - want: attribute.String(key, ""), - }, - { - limit: 0, - attr: strSliceAttr, - want: attribute.StringSlice(key, []string{"", ""}), - }, - { - limit: 0, - attr: attribute.Stringer(key, bytes.NewBufferString("value")), - want: attribute.String(key, ""), - }, - { - limit: 0, - attr: bytesAttr, - want: attribute.ByteSlice(key, []byte{}), - }, - { - limit: 1, - attr: strAttr, - want: attribute.String(key, "v"), - }, - { - limit: 1, - attr: strSliceAttr, - want: attribute.StringSlice(key, []string{"v", "v"}), - }, - { - limit: 1, - attr: bytesAttr, - want: attribute.ByteSlice(key, []byte("v")), - }, - { - limit: 5, - attr: strAttr, - want: strAttr, - }, - { - limit: 5, - attr: bytesAttr, - want: bytesAttr, - }, - { - limit: 7, - attr: strSliceAttr, - want: strSliceAttr, - }, - { - limit: 6, - attr: attribute.StringSlice(key, []string{"value", "value-1"}), - want: attribute.StringSlice(key, []string{"value", "value-"}), - }, - { - limit: 128, - attr: strAttr, - want: strAttr, - }, - { - limit: 128, - attr: strSliceAttr, - want: strSliceAttr, - }, - { - limit: 128, - attr: bytesAttr, - want: bytesAttr, - }, - { - // Multi-byte string: byte length (9) exceeds limit (5) but rune count (3) does not. - // Must not be truncated. - limit: 5, - attr: attribute.String(key, "日本語"), - want: attribute.String(key, "日本語"), - }, - { - // Multi-byte string: both byte length and rune count exceed limit. - // Must be truncated to limit runes. - limit: 2, - attr: attribute.String(key, "日本語"), - want: attribute.String(key, "日本"), - }, - { - // STRINGSLICE with multi-byte elements: byte lengths exceed limit but rune counts do not. - // Must not be truncated. - limit: 1, - attr: attribute.StringSlice(key, []string{"日", "本"}), - want: attribute.StringSlice(key, []string{"日", "本"}), - }, - // SLICE cases - { - limit: -1, - attr: attribute.Slice(key, attribute.StringValue("value")), - want: attribute.Slice(key, attribute.StringValue("value")), - }, - { - limit: 0, - attr: attribute.Slice(key, attribute.BoolValue(true), attribute.StringValue("value")), - want: attribute.Slice(key, attribute.BoolValue(true), attribute.StringValue("")), - }, - { - limit: 5, - attr: attribute.Slice(key, attribute.StringValue("value"), attribute.StringValue("toolong")), - want: attribute.Slice(key, attribute.StringValue("value"), attribute.StringValue("toolo")), - }, - { - // Nested SLICE: recursive truncation. - limit: 1, - attr: attribute.Slice(key, attribute.SliceValue(attribute.StringValue("value"))), - want: attribute.Slice(key, attribute.SliceValue(attribute.StringValue("v"))), - }, - { - // STRINGSLICE within SLICE: each string element is truncated. - limit: 2, - attr: attribute.Slice(key, attribute.StringSliceValue([]string{"abc", "de"})), - want: attribute.Slice(key, attribute.StringSliceValue([]string{"ab", "de"})), - }, - { - // STRINGSLICE within SLICE where all strings fit: no change. - // Exercises needsTruncation(STRINGSLICE) exhausting the loop without - // finding an over-limit string, returning false. - limit: 7, - attr: attribute.Slice(key, attribute.StringSliceValue([]string{"value-0", "value-1"})), - want: attribute.Slice(key, attribute.StringSliceValue([]string{"value-0", "value-1"})), - }, - { - // Mixed SLICE: STRINGSLICE (all strings fit) + STRING (too long). - // Exercises recursive truncation over mixed slice elements: the - // STRINGSLICE element remains unchanged because each string fits - // within the limit, while the sibling STRING element is truncated. - limit: 3, - attr: attribute.Slice( - key, - attribute.StringSliceValue([]string{"ab", "cd"}), - attribute.StringValue("toolong"), - ), - want: attribute.Slice( - key, - attribute.StringSliceValue([]string{"ab", "cd"}), - attribute.StringValue("too"), - ), - }, - { - // Nested SLICE (no truncation needed) alongside STRING (needs truncation). - // Exercises the truncateValue SLICE branch early-return path: truncateValue - // is called recursively on the nested SLICE but returns it unchanged because - // none of its elements require truncation. - limit: 3, - attr: attribute.Slice( - key, - attribute.SliceValue(attribute.BoolValue(true)), - attribute.StringValue("toolong"), - ), - want: attribute.Slice( - key, - attribute.SliceValue(attribute.BoolValue(true)), - attribute.StringValue("too"), - ), - }, - { - // Multi-byte string whose byte length exceeds the limit but rune count - // does not: must not be truncated (guards use rune count, not byte length). - limit: 3, - attr: attribute.Slice(key, attribute.StringValue("日本語")), // 3 runes, 9 bytes - want: attribute.Slice(key, attribute.StringValue("日本語")), - }, - { - // SLICE with invalid UTF-8 where rune count equals the limit: - // invalid byte is dropped. - limit: 2, - attr: attribute.Slice(key, attribute.StringValue("日\x80")), // 2 runes (日 + invalid byte), 4 bytes - want: attribute.Slice(key, attribute.StringValue("日")), - }, - { - // BYTESLICE within SLICE: each byte slice is truncated. - limit: 2, - attr: attribute.Slice(key, attribute.ByteSliceValue([]byte{1, 2, 3})), - want: attribute.Slice(key, attribute.ByteSliceValue([]byte{1, 2})), - }, - { - // BYTESLICE within SLICE: no truncation needed. - limit: 5, - attr: attribute.Slice(key, attribute.ByteSliceValue([]byte{1, 2})), - want: attribute.Slice(key, attribute.ByteSliceValue([]byte{1, 2})), - }, - { - // Mixed SLICE: BYTESLICE + STRING (both need truncation). - limit: 2, - attr: attribute.Slice( - key, - attribute.ByteSliceValue([]byte{1, 2, 3}), - attribute.StringValue("abc"), - ), - want: attribute.Slice( - key, - attribute.ByteSliceValue([]byte{1, 2}), - attribute.StringValue("ab"), - ), - }, - // MAP cases - { - limit: -1, - attr: attribute.Map(key, attribute.String("value", "value")), - want: attribute.Map(key, attribute.String("value", "value")), - }, - { - limit: 0, - attr: attribute.Map(key, attribute.Bool("ok", true), attribute.String("value", "value")), - want: attribute.Map(key, attribute.Bool("ok", true), attribute.String("value", "")), - }, - { - limit: 5, - attr: attribute.Map( - key, - attribute.String("short", "value"), - attribute.String("long", "toolong"), - ), - want: attribute.Map( - key, - attribute.String("short", "value"), - attribute.String("long", "toolo"), - ), - }, - { - // STRINGSLICE within MAP: each string element is truncated. - limit: 2, - attr: attribute.Map(key, attribute.StringSlice("strings", []string{"abc", "de"})), - want: attribute.Map(key, attribute.StringSlice("strings", []string{"ab", "de"})), - }, - { - // BYTESLICE within MAP: each byte slice is truncated. - limit: 2, - attr: attribute.Map(key, attribute.ByteSlice("bytes", []byte{1, 2, 3})), - want: attribute.Map(key, attribute.ByteSlice("bytes", []byte{1, 2})), - }, - { - // Nested MAP: recursive truncation. - limit: 1, - attr: attribute.Map(key, attribute.Map("map", attribute.String("nested", "value"))), - want: attribute.Map(key, attribute.Map("map", attribute.String("nested", "v"))), - }, - { - // SLICE within MAP: recursive truncation. - limit: 2, - attr: attribute.Map( - key, - attribute.Slice( - "slice", - attribute.StringValue("abc"), - attribute.MapValue(attribute.String("nested", "abc")), - ), - ), - want: attribute.Map( - key, - attribute.Slice( - "slice", - attribute.StringValue("ab"), - attribute.MapValue(attribute.String("nested", "ab")), - ), - ), - }, - { - // MAP within SLICE: recursive truncation. - limit: 2, - attr: attribute.Slice(key, attribute.MapValue(attribute.String("nested", "value"))), - want: attribute.Slice(key, attribute.MapValue(attribute.String("nested", "va"))), - }, - { - // Multi-byte string whose byte length exceeds the limit but rune count - // does not: must not be truncated. - limit: 3, - attr: attribute.Map(key, attribute.String("string", "日本語")), // 3 runes, 9 bytes - want: attribute.Map(key, attribute.String("string", "日本語")), - }, - { - // MAP with invalid UTF-8 where rune count equals the limit: - // invalid byte is dropped. - limit: 2, - attr: attribute.Map(key, attribute.String("string", "日\x80")), // 2 runes, 4 bytes - want: attribute.Map(key, attribute.String("string", "日")), - }, - { - // Duplicate MAP entries are truncated but not dropped. - limit: 2, - attr: attribute.Map( - key, - attribute.String("dup", "abc"), - attribute.String("dup", "de"), - ), - want: attribute.Map( - key, - attribute.String("dup", "ab"), - attribute.String("dup", "de"), - ), - }, - } - - for _, test := range tests { - name := fmt.Sprintf("%s->%s(limit:%d)", test.attr.Key, test.attr.Value.String(), test.limit) - t.Run(name, func(t *testing.T) { - assert.Equal(t, test.want, truncateAttr(test.limit, test.attr)) - }) - } -} - -func TestTruncate(t *testing.T) { - type group struct { - limit int - input string - expected string - } - - tests := []struct { - name string - groups []group - }{ - // Edge case: limit is negative, no truncation should occur - { - name: "NoTruncation", - groups: []group{ - {-1, "No truncation!", "No truncation!"}, - }, - }, - - // Edge case: string is already shorter than the limit, no truncation - // should occur - { - name: "ShortText", - groups: []group{ - {10, "Short text", "Short text"}, - {15, "Short text", "Short text"}, - {100, "Short text", "Short text"}, - }, - }, - - // Edge case: truncation happens with ASCII characters only - { - name: "ASCIIOnly", - groups: []group{ - {1, "Hello World!", "H"}, - {5, "Hello World!", "Hello"}, - {12, "Hello World!", "Hello World!"}, - }, - }, - - // Truncation including multi-byte characters (UTF-8) - { - name: "ValidUTF-8", - groups: []group{ - {7, "Hello, 世界", "Hello, "}, - {8, "Hello, 世界", "Hello, 世"}, - {2, "こんにちは", "こん"}, - {3, "こんにちは", "こんに"}, - {5, "こんにちは", "こんにちは"}, - {12, "こんにちは", "こんにちは"}, - }, - }, - - // Truncation with invalid UTF-8 characters - { - name: "InvalidUTF-8", - groups: []group{ - {11, "Invalid\x80text", "Invalidtext"}, - // Do not modify invalid text if equal to limit. - {11, "Valid text\x80", "Valid text\x80"}, - // Do not modify invalid text if under limit. - {15, "Valid text\x80", "Valid text\x80"}, - {5, "Hello\x80World", "Hello"}, - {11, "Hello\x80World\x80!", "HelloWorld!"}, - {15, "Hello\x80World\x80Test", "HelloWorldTest"}, - {15, "Hello\x80\x80\x80World\x80Test", "HelloWorldTest"}, - {15, "\x80\x80\x80Hello\x80\x80\x80World\x80Test\x80\x80", "HelloWorldTest"}, - }, - }, - - // Truncation with mixed validn and invalid UTF-8 characters - { - name: "MixedUTF-8", - groups: []group{ - {6, "€"[0:2] + "hello€€", "hello€"}, - {6, "€" + "€"[0:2] + "hello", "€hello"}, - {11, "Valid text\x80📜", "Valid text📜"}, - {11, "Valid text📜\x80", "Valid text📜"}, - {14, "😊 Hello\x80World🌍🚀", "😊 HelloWorld🌍🚀"}, - {14, "😊\x80 Hello\x80World🌍🚀", "😊 HelloWorld🌍🚀"}, - {14, "😊\x80 Hello\x80World🌍\x80🚀", "😊 HelloWorld🌍🚀"}, - {14, "😊\x80 Hello\x80World🌍\x80🚀\x80", "😊 HelloWorld🌍🚀"}, - {14, "\x80😊\x80 Hello\x80World🌍\x80🚀\x80", "😊 HelloWorld🌍🚀"}, - }, - }, - - // Edge case: empty string, should return empty string - { - name: "Empty", - groups: []group{ - {5, "", ""}, - }, - }, - - // Edge case: limit is 0, should return an empty string - { - name: "Zero", - groups: []group{ - {0, "Some text", ""}, - {0, "", ""}, - }, - }, - } - - for _, tt := range tests { - for _, g := range tt.groups { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - got := truncate(g.limit, g.input) - assert.Equalf( - t, g.expected, got, - "input: %q([]rune%v))\ngot: %q([]rune%v)\nwant %q([]rune%v)", - g.input, []rune(g.input), - got, []rune(got), - g.expected, []rune(g.expected), - ) - }) - } - } -} - -func BenchmarkTruncateAttr(b *testing.B) { - const key = "key" - - strAttr := attribute.String(key, "value") - bytesAttr := attribute.ByteSlice(key, []byte("value")) - strSliceAttr := attribute.StringSlice(key, []string{"value-0", "value-1"}) - - run := func(limit int, attr attribute.KeyValue) func(b *testing.B) { - return func(b *testing.B) { - b.ReportAllocs() - b.RunParallel(func(pb *testing.PB) { - var out attribute.KeyValue - for pb.Next() { - out = truncateAttr(limit, attr) - } - _ = out - }) - } - } - - b.Run("String", run(3, strAttr)) - b.Run("StringSlice", run(3, strSliceAttr)) - b.Run("ByteSlice", run(3, bytesAttr)) - b.Run("String/Limit0", run(0, strAttr)) - b.Run("StringSlice/Limit0", run(0, strSliceAttr)) - b.Run("ByteSlice/Limit0", run(0, bytesAttr)) - b.Run("String/Unlimited", run(-1, strAttr)) - b.Run("StringSlice/Unlimited", run(-1, strSliceAttr)) - b.Run("ByteSlice/Unlimited", run(-1, bytesAttr)) -} - -func BenchmarkTruncate(b *testing.B) { - run := func(limit int, input string) func(b *testing.B) { - return func(b *testing.B) { - b.ReportAllocs() - b.RunParallel(func(pb *testing.PB) { - var out string - for pb.Next() { - out = truncate(limit, input) - } - _ = out - }) - } - } - b.Run("Unlimited", run(-1, "hello 😊 world 🌍🚀")) - b.Run("Zero", run(0, "Some text")) - b.Run("Short", run(10, "Short Text")) - b.Run("ASCII", run(5, "Hello, World!")) - b.Run("ValidUTF-8", run(10, "hello 😊 world 🌍🚀")) - b.Run("InvalidUTF-8", run(6, "€"[0:2]+"hello€€")) - b.Run("MixedUTF-8", run(14, "\x80😊\x80 Hello\x80World🌍\x80🚀\x80")) -} - func TestLogDropAttrs(t *testing.T) { orig := logDropAttrs t.Cleanup(func() { logDropAttrs = orig })