-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add concurrent map and fix reserve time and init transport
- Loading branch information
Showing
4 changed files
with
174 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,156 @@ | ||
package main | ||
|
||
import ( | ||
"encoding/json" | ||
"sync" | ||
) | ||
|
||
var SHARD_COUNT = 32 | ||
|
||
// A "thread" safe map of type string:Anything. | ||
// To avoid lock bottlenecks this map is dived to several (SHARD_COUNT) map shards. | ||
type ConcurrentMap []*ConcurrentMapShared | ||
|
||
// A "thread" safe string to anything map. | ||
type ConcurrentMapShared struct { | ||
items map[string]interface{} | ||
sync.RWMutex // Read Write mutex, guards access to internal map. | ||
} | ||
|
||
// Creates a new concurrent map. | ||
func NewConcurrentMap() ConcurrentMap { | ||
m := make(ConcurrentMap, SHARD_COUNT) | ||
for i := 0; i < SHARD_COUNT; i++ { | ||
m[i] = &ConcurrentMapShared{items: make(map[string]interface{})} | ||
} | ||
return m | ||
} | ||
|
||
// Returns shard under given key | ||
func (m ConcurrentMap) GetShard(key string) *ConcurrentMapShared { | ||
return m[uint(fnv32(key))%uint(SHARD_COUNT)] | ||
} | ||
|
||
// Sets the given value under the specified key. | ||
func (m ConcurrentMap) Set(key string, value interface{}) { | ||
// Get map shard. | ||
shard := m.GetShard(key) | ||
shard.Lock() | ||
shard.items[key] = value | ||
shard.Unlock() | ||
} | ||
|
||
// Retrieves an element from map under given key. | ||
func (m ConcurrentMap) Get(key string) (interface{}, bool) { | ||
// Get shard | ||
shard := m.GetShard(key) | ||
shard.RLock() | ||
// Get item from shard. | ||
val, ok := shard.items[key] | ||
shard.RUnlock() | ||
return val, ok | ||
} | ||
|
||
// Used by the Iter & IterBuffered functions to wrap two variables together over a channel, | ||
type Tuple struct { | ||
Key string | ||
Val interface{} | ||
} | ||
|
||
// Returns a buffered iterator which could be used in a for range loop. | ||
func (m ConcurrentMap) IterBuffered() <-chan Tuple { | ||
chans := snapshot(m) | ||
total := 0 | ||
for _, c := range chans { | ||
total += cap(c) | ||
} | ||
ch := make(chan Tuple, total) | ||
go fanIn(chans, ch) | ||
return ch | ||
} | ||
|
||
// Returns a array of channels that contains elements in each shard, | ||
// which likely takes a snapshot of `m`. | ||
// It returns once the size of each buffered channel is determined, | ||
// before all the channels are populated using goroutines. | ||
func snapshot(m ConcurrentMap) (chans []chan Tuple) { | ||
chans = make([]chan Tuple, SHARD_COUNT) | ||
wg := sync.WaitGroup{} | ||
wg.Add(SHARD_COUNT) | ||
// Foreach shard. | ||
for index, shard := range m { | ||
go func(index int, shard *ConcurrentMapShared) { | ||
// Foreach key, value pair. | ||
shard.RLock() | ||
chans[index] = make(chan Tuple, len(shard.items)) | ||
wg.Done() | ||
for key, val := range shard.items { | ||
chans[index] <- Tuple{key, val} | ||
} | ||
shard.RUnlock() | ||
close(chans[index]) | ||
}(index, shard) | ||
} | ||
wg.Wait() | ||
return chans | ||
} | ||
|
||
// fanIn reads elements from channels `chans` into channel `out` | ||
func fanIn(chans []chan Tuple, out chan Tuple) { | ||
wg := sync.WaitGroup{} | ||
wg.Add(len(chans)) | ||
for _, ch := range chans { | ||
go func(ch chan Tuple) { | ||
for t := range ch { | ||
out <- t | ||
} | ||
wg.Done() | ||
}(ch) | ||
} | ||
wg.Wait() | ||
close(out) | ||
} | ||
|
||
// Reviles ConcurrentMap "private" variables to json marshal. | ||
func (m ConcurrentMap) MarshalJSON() ([]byte, error) { | ||
// Create a temporary map, which will hold all item spread across shards. | ||
tmp := make(map[string]interface{}) | ||
|
||
// Insert items to temporary map. | ||
for item := range m.IterBuffered() { | ||
tmp[item.Key] = item.Val | ||
} | ||
return json.Marshal(tmp) | ||
} | ||
|
||
func fnv32(key string) uint32 { | ||
hash := uint32(2166136261) | ||
const prime32 = uint32(16777619) | ||
for i := 0; i < len(key); i++ { | ||
hash *= prime32 | ||
hash ^= uint32(key[i]) | ||
} | ||
return hash | ||
} | ||
|
||
// Concurrent map uses Interface{} as its value, therefor JSON Unmarshal | ||
// will probably won't know which to type to unmarshal into, in such case | ||
// we'll end up with a value of type map[string]interface{}, In most cases this isn't | ||
// out value type, this is why we've decided to remove this functionality. | ||
|
||
// func (m *ConcurrentMap) UnmarshalJSON(b []byte) (err error) { | ||
// // Reverse process of Marshal. | ||
|
||
// tmp := make(map[string]interface{}) | ||
|
||
// // Unmarshal into a single map. | ||
// if err := json.Unmarshal(b, &tmp); err != nil { | ||
// return nil | ||
// } | ||
|
||
// // foreach key,value pair in temporary map insert into our concurrent map. | ||
// for key, val := range tmp { | ||
// m.Set(key, val) | ||
// } | ||
// return nil | ||
// } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -13,6 +13,7 @@ userinfo: | |
openid: "" | ||
h_5_source: "" | ||
device_token: "" | ||
s_id: "" | ||
|
||
headers: | ||
ddmc_city_number: "" | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters