From 52cf43a31d7bb170326ce7ef0b1f394d1721e8e2 Mon Sep 17 00:00:00 2001 From: Gaius Date: Fri, 29 Oct 2021 10:34:28 +0800 Subject: [PATCH] feat: clear hashcircler and maputils package (#768) * feat: clear hashcircler and maputils package Signed-off-by: Gaius --- cdn/supervisor/cdn/downloader.go | 4 +- pkg/source/httpprotocol/http_source_client.go | 6 +- pkg/structure/hashcircler/hash_circler.go | 176 ------------------ .../hashcircler/hash_circler_test.go | 171 ----------------- .../maputils/map_utils.go} | 9 +- 5 files changed, 9 insertions(+), 357 deletions(-) delete mode 100644 pkg/structure/hashcircler/hash_circler.go delete mode 100644 pkg/structure/hashcircler/hash_circler_test.go rename pkg/{structure/maputils/maputils.go => util/maputils/map_utils.go} (78%) diff --git a/cdn/supervisor/cdn/downloader.go b/cdn/supervisor/cdn/downloader.go index a355457f424..ce2accc33f2 100644 --- a/cdn/supervisor/cdn/downloader.go +++ b/cdn/supervisor/cdn/downloader.go @@ -23,7 +23,7 @@ import ( "d7y.io/dragonfly/v2/cdn/types" "d7y.io/dragonfly/v2/pkg/source" - "d7y.io/dragonfly/v2/pkg/structure/maputils" + "d7y.io/dragonfly/v2/pkg/util/maputils" "d7y.io/dragonfly/v2/pkg/util/rangeutils" "github.com/pkg/errors" ) @@ -31,7 +31,7 @@ import ( const RangeHeaderName = "Range" func (cm *Manager) download(ctx context.Context, task *types.SeedTask, detectResult *cacheResult) (io.ReadCloser, error) { - headers := maputils.DeepCopyMap(nil, task.Header) + headers := maputils.DeepCopy(task.Header) if detectResult.breakPoint > 0 { breakRange, err := rangeutils.GetBreakRange(detectResult.breakPoint, task.SourceFileLength) if err != nil { diff --git a/pkg/source/httpprotocol/http_source_client.go b/pkg/source/httpprotocol/http_source_client.go index 730d77172c8..c630f66bef5 100644 --- a/pkg/source/httpprotocol/http_source_client.go +++ b/pkg/source/httpprotocol/http_source_client.go @@ -27,12 +27,12 @@ import ( "time" "d7y.io/dragonfly/v2/cdn/types" + "d7y.io/dragonfly/v2/pkg/util/maputils" "d7y.io/dragonfly/v2/pkg/util/rangeutils" "github.com/go-http-utils/headers" "d7y.io/dragonfly/v2/pkg/source" - "d7y.io/dragonfly/v2/pkg/structure/maputils" "d7y.io/dragonfly/v2/pkg/util/stringutils" "d7y.io/dragonfly/v2/pkg/util/timeutils" ) @@ -119,7 +119,7 @@ func (client *httpSourceClient) GetContentLength(ctx context.Context, url string } func (client *httpSourceClient) IsSupportRange(ctx context.Context, url string, header source.RequestHeader) (bool, error) { - copied := maputils.DeepCopyMap(nil, header) + copied := maputils.DeepCopy(header) copied[headers.Range] = "bytes=0-0" resp, err := client.doRequest(ctx, http.MethodGet, url, copied) @@ -140,7 +140,7 @@ func (client *httpSourceClient) IsExpired(ctx context.Context, url string, heade } // set header: header is a reference to map, should not change it - copied := maputils.DeepCopyMap(nil, header) + copied := maputils.DeepCopy(header) if lastModified > 0 { copied[headers.IfModifiedSince] = expireInfo[headers.LastModified] } diff --git a/pkg/structure/hashcircler/hash_circler.go b/pkg/structure/hashcircler/hash_circler.go deleted file mode 100644 index 86e6b0d4cd1..00000000000 --- a/pkg/structure/hashcircler/hash_circler.go +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright 2020 The Dragonfly Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package hashcircler - -import ( - "fmt" - "hash/fnv" - "sync" - - "github.com/HuKeping/rbtree" - "github.com/pkg/errors" -) - -// HashCircler hashes input string to target key, and the key could be enabled or disabled. -// And the keys array is preset, only the keys could be enable or disable. -type HashCircler interface { - // Add adds the target key in hash circle. - Add(key string) - - // Hash hashes the input and output the target key which hashes. - Hash(input string) (key string, err error) - - // Delete deletes the target key - Delete(key string) -} - -var ( - ErrKeyNotPresent = errors.New("key is not present") -) - -// consistentHashCircler is an implementation of HashCircler. And the keys is preset. -type consistentHashCircler struct { - sync.RWMutex - hashFunc func(string) uint64 - - keysMap map[uint64]string - replicationPerKey int - rb *rbtree.Rbtree -} - -// NewConsistentHashCircler constructs an instance of HashCircler from keys. And this is thread safety. -// if hashFunc is nil, it will be set to default hash func. -func NewConsistentHashCircler(keys []string, hashFunc func(string) uint64) (HashCircler, error) { - if hashFunc == nil { - hashFunc = fnvHashFunc - } - - if len(keys) == 0 { - return nil, fmt.Errorf("empty keys") - } - - hc := &consistentHashCircler{ - hashFunc: hashFunc, - keysMap: make(map[uint64]string), - replicationPerKey: 16, - rb: rbtree.New(), - } - - for _, k := range keys { - hc.Add(k) - } - - return hc, nil -} - -func (h *consistentHashCircler) Add(key string) { - h.Lock() - defer h.Unlock() - - for i := 0; i < h.replicationPerKey; i++ { - m := h.hashFunc(fmt.Sprintf("%s-%d", key, i)) - if _, exist := h.keysMap[m]; exist { - continue - } - h.keysMap[m] = key - h.addToRbTree(m, key) - } - - return -} - -func (h *consistentHashCircler) Hash(input string) (key string, err error) { - h.RLock() - defer h.RUnlock() - - if len(h.keysMap) == 0 { - return "", ErrKeyNotPresent - } - - index := h.hashFunc(input) - - return h.searchFromRbTree(index), nil -} - -func (h *consistentHashCircler) Delete(key string) { - h.Lock() - defer h.Unlock() - - for i := 0; i < h.replicationPerKey; i++ { - m := h.hashFunc(fmt.Sprintf("%s-%d", key, i)) - delete(h.keysMap, m) - h.deleteFromRbTree(m) - } - - return -} - -func fnvHashFunc(input string) uint64 { - h := fnv.New64a() - h.Write([]byte(input)) - return h.Sum64() -} - -func (h *consistentHashCircler) addToRbTree(index uint64, key string) { - i := &item{ - index: index, - key: key, - } - - h.rb.Insert(i) -} - -func (h *consistentHashCircler) deleteFromRbTree(index uint64) { - i := &item{ - index: index, - } - - h.rb.Delete(i) -} - -func (h *consistentHashCircler) searchFromRbTree(index uint64) string { - comp := &item{ - index: index, - } - - target := "" - - // find the key which index of item greater or equal than input index. - h.rb.Ascend(comp, func(i rbtree.Item) bool { - o := i.(*item) - target = o.key - return false - }) - - // if not found the target, return the max item. - if target == "" { - i := h.rb.Max() - target = i.(*item).key - } - - return target -} - -type item struct { - index uint64 - key string -} - -func (i *item) Less(than rbtree.Item) bool { - other := than.(*item) - return i.index < other.index -} diff --git a/pkg/structure/hashcircler/hash_circler_test.go b/pkg/structure/hashcircler/hash_circler_test.go deleted file mode 100644 index 29a6aa8175c..00000000000 --- a/pkg/structure/hashcircler/hash_circler_test.go +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright 2020 The Dragonfly Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package hashcircler - -import ( - "math" - "math/rand" - "testing" - - "github.com/stretchr/testify/suite" -) - -func TestSuite(t *testing.T) { - suite.Run(t, &hashCirclerSuite{ - hashMap: map[string]uint64{}, - }) -} - -type hashCirclerSuite struct { - hashMap map[string]uint64 - suite.Suite -} - -func (suite *hashCirclerSuite) registerHashKV(key string, value uint64) { - suite.hashMap[key] = value -} - -func (suite *hashCirclerSuite) unRegisterHashKV(key string) { - delete(suite.hashMap, key) -} - -func (suite *hashCirclerSuite) cleanHashMap() { - suite.hashMap = make(map[string]uint64) -} - -func (suite *hashCirclerSuite) hash(input string) uint64 { - v, ok := suite.hashMap[input] - if ok { - return v - } - - return 0 -} - -func (suite *hashCirclerSuite) TestHashCircler() { - defer suite.cleanHashMap() - - rangeSize := uint64(math.MaxUint64 / 5) - suite.registerHashKV("v1", rand.Uint64()%rangeSize) - suite.registerHashKV("v2", rand.Uint64()%rangeSize) - suite.registerHashKV("v3", rand.Uint64()%rangeSize+rangeSize) - suite.registerHashKV("v4", rand.Uint64()%rangeSize+rangeSize) - suite.registerHashKV("v5", rand.Uint64()%rangeSize+rangeSize*2) - suite.registerHashKV("v6", rand.Uint64()%rangeSize+rangeSize*2) - suite.registerHashKV("v7", rand.Uint64()%rangeSize+rangeSize*3) - suite.registerHashKV("v8", rand.Uint64()%rangeSize+rangeSize*3) - suite.registerHashKV("v9", rand.Uint64()%rangeSize+rangeSize*4) - suite.registerHashKV("v10", rand.Uint64()%rangeSize+rangeSize*4) - - arr := []string{ - "key1", "key2", "key3", "key4", "key5", - } - - inputStrs := []string{ - "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", - } - - hasher, err := NewConsistentHashCircler(arr, nil) - suite.Nil(err) - - originKeys := make([]string, len(inputStrs)) - - for i := 0; i < 10; i++ { - k, err := hasher.Hash(inputStrs[i]) - suite.Nil(err) - originKeys[i] = k - } - - // disable arr[0] - hasher.Delete(arr[0]) - for i := 0; i < 10; i++ { - k, err := hasher.Hash(inputStrs[i]) - suite.Nil(err) - suite.NotEqual(k, arr[0]) - if originKeys[i] != arr[0] { - suite.Equal(k, originKeys[i]) - } - } - - hasher.Delete(arr[1]) - hasher.Delete(arr[2]) - hasher.Delete(arr[4]) - - for i := 0; i < 10; i++ { - k, err := hasher.Hash(inputStrs[i]) - suite.Nil(err) - suite.Equal(k, arr[3]) - } - - hasher.Add(arr[1]) - - for i := 0; i < 10; i++ { - k, err := hasher.Hash(inputStrs[i]) - suite.Nil(err) - if originKeys[i] == arr[1] || originKeys[i] == arr[3] { - suite.Equal(k, originKeys[i]) - } - suite.Equal(true, k == arr[3] || k == arr[1]) - } - - hasher.Add(arr[1]) - hasher.Add(arr[2]) - - for i := 0; i < 10; i++ { - k, err := hasher.Hash(inputStrs[i]) - suite.Nil(err) - - if originKeys[i] == arr[1] || originKeys[i] == arr[2] || originKeys[i] == arr[3] { - suite.Equal(k, originKeys[i]) - } - - suite.Equal(true, k != arr[0] && k != arr[4]) - } - - hasher.Delete(arr[0]) - hasher.Delete(arr[1]) - hasher.Delete(arr[2]) - for i := 0; i < 10; i++ { - k, err := hasher.Hash(inputStrs[i]) - suite.Nil(err) - suite.Equal(k, arr[3]) - } - - hasher.Delete(arr[3]) - for i := 0; i < 10; i++ { - _, err = hasher.Hash(inputStrs[i]) - suite.NotNil(err) - } - - hasher.Add(arr[0]) - for i := 0; i < 10; i++ { - k, err := hasher.Hash(inputStrs[i]) - suite.Nil(err) - suite.Equal(k, arr[0]) - } - - hasher.Add(arr[1]) - hasher.Add(arr[2]) - hasher.Add(arr[3]) - hasher.Add(arr[4]) - - for i := 0; i < 10; i++ { - k, err := hasher.Hash(inputStrs[i]) - suite.Nil(err) - suite.Equal(k, originKeys[i]) - } -} diff --git a/pkg/structure/maputils/maputils.go b/pkg/util/maputils/map_utils.go similarity index 78% rename from pkg/structure/maputils/maputils.go rename to pkg/util/maputils/map_utils.go index 3c8df7629d1..462eb3f837b 100644 --- a/pkg/structure/maputils/maputils.go +++ b/pkg/util/maputils/map_utils.go @@ -16,11 +16,10 @@ package maputils -// DeepCopyMap copies the src to dst and return a non-nil dst map. -func DeepCopyMap(dst, src map[string]string) map[string]string { - if dst == nil { - dst = make(map[string]string) - } +// DeepCopy copies the src to dst and return a non-nil dst map. +func DeepCopy(src map[string]string) map[string]string { + dst := make(map[string]string) + for k, v := range src { dst[k] = v }