From 81d468ff91dce9679dd20664c8eb9d4c7a75b836 Mon Sep 17 00:00:00 2001 From: n4mine Date: Wed, 7 Mar 2018 19:50:46 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20common=20utils.=20=E5=8C=85=E6=8B=ACMd5?= =?UTF-8?q?,=20PK,=20PK2,=20UUID,=20SortedTags,=20DictedTagstring=E7=AD=89?= =?UTF-8?q?=E5=87=BD=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/utils/func.go | 56 ++++++++- common/utils/func_test.go | 123 +++++++++++++++++++ common/utils/md5.go | 9 +- common/utils/md5_test.go | 33 +++++ common/utils/objpool.go | 8 ++ common/utils/tags.go | 31 +++-- common/utils/tags_test.go | 248 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 488 insertions(+), 20 deletions(-) create mode 100644 common/utils/func_test.go create mode 100644 common/utils/md5_test.go create mode 100644 common/utils/objpool.go create mode 100644 common/utils/tags_test.go diff --git a/common/utils/func.go b/common/utils/func.go index 62e1dbe3b..4593b1f03 100644 --- a/common/utils/func.go +++ b/common/utils/func.go @@ -15,25 +15,69 @@ package utils import ( - "fmt" + "bytes" + "strconv" ) func PK(endpoint, metric string, tags map[string]string) string { + ret := bufferPool.Get().(*bytes.Buffer) + ret.Reset() + defer bufferPool.Put(ret) + if tags == nil || len(tags) == 0 { - return fmt.Sprintf("%s/%s", endpoint, metric) + ret.WriteString(endpoint) + ret.WriteString("/") + ret.WriteString(metric) + + return ret.String() } - return fmt.Sprintf("%s/%s/%s", endpoint, metric, SortedTags(tags)) + ret.WriteString(endpoint) + ret.WriteString("/") + ret.WriteString(metric) + ret.WriteString("/") + ret.WriteString(SortedTags(tags)) + return ret.String() } func PK2(endpoint, counter string) string { - return fmt.Sprintf("%s/%s", endpoint, counter) + ret := bufferPool.Get().(*bytes.Buffer) + ret.Reset() + defer bufferPool.Put(ret) + + ret.WriteString(endpoint) + ret.WriteString("/") + ret.WriteString(counter) + + return ret.String() } func UUID(endpoint, metric string, tags map[string]string, dstype string, step int) string { + ret := bufferPool.Get().(*bytes.Buffer) + ret.Reset() + defer bufferPool.Put(ret) + if tags == nil || len(tags) == 0 { - return fmt.Sprintf("%s/%s/%s/%d", endpoint, metric, dstype, step) + ret.WriteString(endpoint) + ret.WriteString("/") + ret.WriteString(metric) + ret.WriteString("/") + ret.WriteString(dstype) + ret.WriteString("/") + ret.WriteString(strconv.Itoa(step)) + + return ret.String() } - return fmt.Sprintf("%s/%s/%s/%s/%d", endpoint, metric, SortedTags(tags), dstype, step) + ret.WriteString(endpoint) + ret.WriteString("/") + ret.WriteString(metric) + ret.WriteString("/") + ret.WriteString(SortedTags(tags)) + ret.WriteString("/") + ret.WriteString(dstype) + ret.WriteString("/") + ret.WriteString(strconv.Itoa(step)) + + return ret.String() } func Checksum(endpoint string, metric string, tags map[string]string) string { diff --git a/common/utils/func_test.go b/common/utils/func_test.go new file mode 100644 index 000000000..1debfd918 --- /dev/null +++ b/common/utils/func_test.go @@ -0,0 +1,123 @@ +package utils + +import ( + "fmt" + "testing" +) + +func origPK(endpoint, metric string, tags map[string]string) string { + if tags == nil || len(tags) == 0 { + return fmt.Sprintf("%s/%s", endpoint, metric) + } + return fmt.Sprintf("%s/%s/%s", endpoint, metric, SortedTags(tags)) +} + +func origPK2(endpoint, counter string) string { + return fmt.Sprintf("%s/%s", endpoint, counter) +} + +func origUUID(endpoint, metric string, tags map[string]string, dstype string, step int) string { + if tags == nil || len(tags) == 0 { + return fmt.Sprintf("%s/%s/%s/%d", endpoint, metric, dstype, step) + } + return fmt.Sprintf("%s/%s/%s/%s/%d", endpoint, metric, SortedTags(tags), dstype, step) +} + +var pkCase = []struct { + endpoint string + metric string + tags map[string]string + except string +}{ + {"endpoint1", "metric1", nil, "endpoint1/metric1"}, + {"endpoint1", "metric1", map[string]string{}, "endpoint1/metric1"}, + {"endpoint1", "metric1", map[string]string{"k1": "v1", "k2": "v2"}, "endpoint1/metric1/k1=v1,k2=v2"}, + {"endpoint1", "metric1", map[string]string{"k2": "v2", "k1": "v1"}, "endpoint1/metric1/k1=v1,k2=v2"}, +} + +var pk2Case = []struct { + endpoint string + counter string + except string +}{ + {"endpoint1", "counter1", "endpoint1/counter1"}, +} + +var uuidCase = []struct { + endpoint string + metric string + tags map[string]string + dstype string + step int + except string +}{ + {"endpoint1", "metric1", nil, "ds", 10, "endpoint1/metric1/ds/10"}, + {"endpoint1", "metric1", map[string]string{}, "ds", 10, "endpoint1/metric1/ds/10"}, + {"endpoint1", "metric1", map[string]string{"k1": "v1", "k2": "v2"}, "ds", 10, "endpoint1/metric1/k1=v1,k2=v2/ds/10"}, + {"endpoint1", "metric1", map[string]string{"k2": "v2", "k1": "v1"}, "ds", 10, "endpoint1/metric1/k1=v1,k2=v2/ds/10"}, +} + +func Test_PK(t *testing.T) { + for _, pk := range pkCase { + if PK(pk.endpoint, pk.metric, pk.tags) != origPK(pk.endpoint, pk.metric, pk.tags) || PK(pk.endpoint, pk.metric, pk.tags) != pk.except { + t.Error("not except") + } + } +} + +func Test_PK2(t *testing.T) { + for _, pk2 := range pk2Case { + if PK2(pk2.endpoint, pk2.counter) != origPK2(pk2.endpoint, pk2.counter) || PK2(pk2.endpoint, pk2.counter) != pk2.except { + t.Error("not except") + } + } +} + +func Test_UUID(t *testing.T) { + for _, uuid := range uuidCase { + if UUID(uuid.endpoint, uuid.metric, uuid.tags, uuid.dstype, uuid.step) != origUUID(uuid.endpoint, uuid.metric, uuid.tags, uuid.dstype, uuid.step) || + UUID(uuid.endpoint, uuid.metric, uuid.tags, uuid.dstype, uuid.step) != uuid.except { + t.Error("not except") + } + } +} + +var ( + testTags = map[string]string{"k1": "v1", "k2": "v2"} +) + +func Benchmark_PK(b *testing.B) { + for i := 0; i < b.N; i++ { + PK("endpoint1", "metric1", testTags) + } +} + +func Benchmark_PK_orig(b *testing.B) { + for i := 0; i < b.N; i++ { + origPK("endpoint1", "metric1", testTags) + } +} + +func Benchmark_PK2(b *testing.B) { + for i := 0; i < b.N; i++ { + PK2("endpoint1", "counter1") + } +} + +func Benchmark_PK2_orig(b *testing.B) { + for i := 0; i < b.N; i++ { + origPK2("endpoint1", "counter1") + } +} + +func Benchmark_UUID(b *testing.B) { + for i := 0; i < b.N; i++ { + UUID("endpoint1", "metric1", testTags, "dt", 10) + } +} + +func Benchmark_UUID_orig(b *testing.B) { + for i := 0; i < b.N; i++ { + origUUID("endpoint1", "metric1", testTags, "dt", 10) + } +} diff --git a/common/utils/md5.go b/common/utils/md5.go index cc7c40299..ab029d54a 100644 --- a/common/utils/md5.go +++ b/common/utils/md5.go @@ -16,13 +16,10 @@ package utils import ( "crypto/md5" - "fmt" - "io" + "encoding/hex" ) func Md5(raw string) string { - h := md5.New() - io.WriteString(h, raw) - - return fmt.Sprintf("%x", h.Sum(nil)) + h := md5.Sum([]byte(raw)) + return hex.EncodeToString(h[:]) } diff --git a/common/utils/md5_test.go b/common/utils/md5_test.go new file mode 100644 index 000000000..c60b6fe66 --- /dev/null +++ b/common/utils/md5_test.go @@ -0,0 +1,33 @@ +package utils + +import ( + "crypto/md5" + "fmt" + "io" + "testing" +) + +func origMd5(raw string) string { + h := md5.New() + io.WriteString(h, raw) + + return fmt.Sprintf("%x", h.Sum(nil)) +} + +func Test_Md5(t *testing.T) { + if Md5("1234567890123") != origMd5("1234567890123") { + t.Error("not expect") + } +} + +func Benchmark_Md5(b *testing.B) { + for i := 0; i < b.N; i++ { + Md5("1234567890123") + } +} + +func Benchmark_Md5_orig(b *testing.B) { + for i := 0; i < b.N; i++ { + origMd5("1234567890123") + } +} diff --git a/common/utils/objpool.go b/common/utils/objpool.go new file mode 100644 index 000000000..a1743ad57 --- /dev/null +++ b/common/utils/objpool.go @@ -0,0 +1,8 @@ +package utils + +import ( + "bytes" + "sync" +) + +var bufferPool = sync.Pool{New: func() interface{} { return new(bytes.Buffer) }} diff --git a/common/utils/tags.go b/common/utils/tags.go index ace771752..2ad8ed8e3 100644 --- a/common/utils/tags.go +++ b/common/utils/tags.go @@ -15,6 +15,7 @@ package utils import ( + "bytes" "fmt" "sort" "strings" @@ -31,10 +32,17 @@ func SortedTags(tags map[string]string) string { return "" } + ret := bufferPool.Get().(*bytes.Buffer) + ret.Reset() + defer bufferPool.Put(ret) + if size == 1 { for k, v := range tags { - return fmt.Sprintf("%s=%s", k, v) + ret.WriteString(k) + ret.WriteString("=") + ret.WriteString(v) } + return ret.String() } keys := make([]string, size) @@ -46,26 +54,33 @@ func SortedTags(tags map[string]string) string { sort.Strings(keys) - ret := make([]string, size) for j, key := range keys { - ret[j] = fmt.Sprintf("%s=%s", key, tags[key]) + ret.WriteString(key) + ret.WriteString("=") + ret.WriteString(tags[key]) + if j != size-1 { + ret.WriteString(",") + } } - return strings.Join(ret, ",") + return ret.String() } func DictedTagstring(s string) map[string]string { if s == "" { return map[string]string{} } - s = strings.Replace(s, " ", "", -1) + + if strings.ContainsRune(s, ' ') { + s = strings.Replace(s, " ", "", -1) + } tag_dict := make(map[string]string) tags := strings.Split(s, ",") for _, tag := range tags { - tag_pair := strings.SplitN(tag, "=", 2) - if len(tag_pair) == 2 { - tag_dict[tag_pair[0]] = tag_pair[1] + idx := strings.IndexRune(tag, '=') + if idx != -1 { + tag_dict[tag[:idx]] = tag[idx+1:] } } return tag_dict diff --git a/common/utils/tags_test.go b/common/utils/tags_test.go new file mode 100644 index 000000000..de67c7b0f --- /dev/null +++ b/common/utils/tags_test.go @@ -0,0 +1,248 @@ +package utils + +import ( + "fmt" + "sort" + "strings" + "testing" +) + +var testCases4map2string = []struct { + tags map[string]string + expect string +}{ + {map[string]string{"1": "1"}, "1=1"}, + {map[string]string{"1": "1", "2": "2"}, "1=1,2=2"}, + {map[string]string{"1": "1", "2": "2", "0": "0"}, "0=0,1=1,2=2"}, +} + +var testCases4string2map = []struct { + tags string + expect map[string]string +}{ + {"1=1", map[string]string{"1": "1"}}, + {"1=1,2=2", map[string]string{"1": "1", "2": "2"}}, + {"0=0,1=1,2=2", map[string]string{"1": "1", "2": "2", "0": "0"}}, + {"0,1=1,2=2", map[string]string{"1": "1", "2": "2"}}, + {"0=,1=1,2=2", map[string]string{"0": "", "1": "1", "2": "2"}}, + {"=0,1=1,2=2", map[string]string{"": "0", "1": "1", "2": "2"}}, + {"0=0, 1=1, 2=2", map[string]string{"0": "0", "1": "1", "2": "2"}}, +} + +func origSortedTags(tags map[string]string) string { + if tags == nil { + return "" + } + + size := len(tags) + + if size == 0 { + return "" + } + + if size == 1 { + for k, v := range tags { + return fmt.Sprintf("%s=%s", k, v) + } + } + + keys := make([]string, size) + i := 0 + for k := range tags { + keys[i] = k + i++ + } + + sort.Strings(keys) + + ret := make([]string, size) + for j, key := range keys { + ret[j] = fmt.Sprintf("%s=%s", key, tags[key]) + } + + return strings.Join(ret, ",") +} + +func origDictedTagstring(s string) map[string]string { + if s == "" { + return map[string]string{} + } + s = strings.Replace(s, " ", "", -1) + + tag_dict := make(map[string]string) + tags := strings.Split(s, ",") + for _, tag := range tags { + tag_pair := strings.SplitN(tag, "=", 2) + if len(tag_pair) == 2 { + tag_dict[tag_pair[0]] = tag_pair[1] + } + } + return tag_dict +} + +func Test_SortedTags(t *testing.T) { + for _, testCase := range testCases4map2string { + if r := SortedTags(testCase.tags); r != testCase.expect || SortedTags(testCase.tags) != origSortedTags(testCase.tags) { + t.Errorf("expect %v, got %v\n", testCase.expect, r) + } + } +} +func Test_DictedTagstring(t *testing.T) { + for _, testCase := range testCases4string2map { + r := DictedTagstring(testCase.tags) + origR := origDictedTagstring(testCase.tags) + + if len(r) != len(testCase.expect) || len(r) != len(origR) { + t.FailNow() + } + for k, v := range r { + if expectV, exist := testCase.expect[k]; !exist || v != expectV || v != origR[k] { + t.Errorf("expect %v, got %v\n", testCase.expect, r) + } + } + } +} + +func Benchmark_SortedTags_1pair(b *testing.B) { + for i := 0; i < b.N; i++ { + SortedTags(map[string]string{"1": "1"}) + } +} + +func Benchmark_SortedTags_1pair_orig(b *testing.B) { + for i := 0; i < b.N; i++ { + origSortedTags(map[string]string{"1": "1"}) + } +} + +func Benchmark_SortedTags_2pairs(b *testing.B) { + for i := 0; i < b.N; i++ { + SortedTags(map[string]string{"1": "1", "2": "2"}) + } +} + +func Benchmark_SortedTags_2pairs_orig(b *testing.B) { + for i := 0; i < b.N; i++ { + origSortedTags(map[string]string{"1": "1", "2": "2"}) + } +} + +func Benchmark_SortedTags_3pairs(b *testing.B) { + for i := 0; i < b.N; i++ { + SortedTags(map[string]string{"1": "1", "2": "2", "3": "3"}) + } +} + +func Benchmark_SortedTags_3pairs_orig(b *testing.B) { + for i := 0; i < b.N; i++ { + origSortedTags(map[string]string{"1": "1", "2": "2", "3": "3"}) + } +} + +func Benchmark_SortedTags_4pairs(b *testing.B) { + for i := 0; i < b.N; i++ { + SortedTags(map[string]string{"1": "1", "2": "2", "3": "3", "4": "4"}) + } +} + +func Benchmark_SortedTags_4pairs_orig(b *testing.B) { + for i := 0; i < b.N; i++ { + origSortedTags(map[string]string{"1": "1", "2": "2", "3": "3", "4": "4"}) + } +} + +func Benchmark_SortedTags_5pairs(b *testing.B) { + for i := 0; i < b.N; i++ { + SortedTags(map[string]string{"1": "1", "2": "2", "3": "3", "4": "4", "5": "5"}) + } +} + +func Benchmark_SortedTags_5pairs_orig(b *testing.B) { + for i := 0; i < b.N; i++ { + origSortedTags(map[string]string{"1": "1", "2": "2", "3": "3", "4": "4", "5": "5"}) + } +} + +func Benchmark_SortedTags_6pairs(b *testing.B) { + for i := 0; i < b.N; i++ { + SortedTags(map[string]string{"1": "1", "2": "2", "3": "3", "4": "4", "5": "5", "0": "0"}) + } +} + +func Benchmark_SortedTags_6pairs_orig(b *testing.B) { + for i := 0; i < b.N; i++ { + origSortedTags(map[string]string{"1": "1", "2": "2", "3": "3", "4": "4", "5": "5", "0": "0"}) + } +} + +func Benchmark_DictedTagstring_1pair(b *testing.B) { + for i := 0; i < b.N; i++ { + DictedTagstring("1=1") + } +} + +func Benchmark_DictedTagstring_1pair_orig(b *testing.B) { + for i := 0; i < b.N; i++ { + origDictedTagstring("1=1") + } +} + +func Benchmark_DictedTagstring_2pairs(b *testing.B) { + for i := 0; i < b.N; i++ { + DictedTagstring("1=1,2=2") + } +} + +func Benchmark_DictedTagstring_2pairs_orig(b *testing.B) { + for i := 0; i < b.N; i++ { + origDictedTagstring("1=1,2=2") + } +} + +func Benchmark_DictedTagstring_3pairs(b *testing.B) { + for i := 0; i < b.N; i++ { + DictedTagstring("1=1,2=2,3=3") + } +} + +func Benchmark_DictedTagstring_3pairs_orig(b *testing.B) { + for i := 0; i < b.N; i++ { + origDictedTagstring("1=1,2=2,3=3") + } +} + +func Benchmark_DictedTagstring_4pairs(b *testing.B) { + for i := 0; i < b.N; i++ { + DictedTagstring("1=1,2=2,3=3,4=4") + } +} + +func Benchmark_DictedTagstring_4pairs_orig(b *testing.B) { + for i := 0; i < b.N; i++ { + origDictedTagstring("1=1,2=2,3=3,4=4") + } +} + +func Benchmark_DictedTagstring_5pairs(b *testing.B) { + for i := 0; i < b.N; i++ { + DictedTagstring("1=1,2=2,3=3,4=4,5=5") + } +} + +func Benchmark_DictedTagstring_5pairs_orig(b *testing.B) { + for i := 0; i < b.N; i++ { + origDictedTagstring("1=1,2=2,3=3,4=4,5=5") + } +} + +func Benchmark_DictedTagstring_6pairs(b *testing.B) { + for i := 0; i < b.N; i++ { + DictedTagstring("1=1,2=2,3=3,4=4,5=5,6=6") + } +} + +func Benchmark_DictedTagstring_6pairs_orig(b *testing.B) { + for i := 0; i < b.N; i++ { + origDictedTagstring("1=1,2=2,3=3,4=4,5=5,6=6") + } +}