Skip to content

Commit

Permalink
Benchmark skiplist against map
Browse files Browse the repository at this point in the history
  • Loading branch information
@jchiu255
jchiu255 committed Mar 8, 2017
1 parent 04490f9 commit 1a0c61c
Show file tree
Hide file tree
Showing 5 changed files with 95 additions and 185 deletions.
55 changes: 12 additions & 43 deletions skiplist/README.md
View file
Original file line number Diff line number Diff line change
@@ -1,46 +1,15 @@
Ran on macbook.
Ran on Macbook.

```
BenchmarkWrite/branch_2-4 1000000 44162 ns/op
BenchmarkWrite/branch_3-4 1000000 6306 ns/op
BenchmarkWrite/branch_4-4 1000000 6922 ns/op
BenchmarkWrite/branch_5-4 1000000 9222 ns/op
BenchmarkWrite/branch_6-4 1000000 7646 ns/op
BenchmarkWrite/branch_7-4 500000 6604 ns/op
BenchmarkWriteParallel/branch_2-4 500000 24437 ns/op
BenchmarkWriteParallel/branch_3-4 1000000 7390 ns/op
BenchmarkWriteParallel/branch_4-4 1000000 8112 ns/op
BenchmarkWriteParallel/branch_5-4 500000 7885 ns/op
BenchmarkWriteParallel/branch_6-4 500000 8459 ns/op
BenchmarkWriteParallel/branch_7-4 500000 8593 ns/op
BenchmarkRead/branch_2-4 200000 8491 ns/op
BenchmarkRead/branch_3-4 500000 3108 ns/op
BenchmarkRead/branch_4-4 500000 3223 ns/op
BenchmarkRead/branch_5-4 500000 3586 ns/op
BenchmarkRead/branch_6-4 500000 4118 ns/op
BenchmarkRead/branch_7-4 300000 4729 ns/op
BenchmarkReadParallel/branch_2-4 500000 2680 ns/op
BenchmarkReadParallel/branch_3-4 1000000 1909 ns/op
BenchmarkReadParallel/branch_4-4 1000000 1928 ns/op
BenchmarkReadParallel/branch_5-4 1000000 1663 ns/op
BenchmarkReadParallel/branch_6-4 1000000 1400 ns/op
BenchmarkReadParallel/branch_7-4 1000000 1733 ns/op
BenchmarkReadWrite/frac_0-4 1000000 6719 ns/op
BenchmarkReadWrite/frac_1-4 1000000 6177 ns/op
BenchmarkReadWrite/frac_2-4 1000000 6239 ns/op
BenchmarkReadWrite/frac_3-4 1000000 5616 ns/op
BenchmarkReadWrite/frac_4-4 1000000 5103 ns/op
BenchmarkReadWrite/frac_5-4 1000000 4962 ns/op
BenchmarkReadWrite/frac_6-4 1000000 4172 ns/op
BenchmarkReadWrite/frac_7-4 1000000 3175 ns/op
BenchmarkReadWrite/frac_8-4 1000000 2070 ns/op
BenchmarkReadWrite/frac_9-4 1000000 1201 ns/op
BenchmarkReadWrite/frac_10-4 5000000 456 ns/o
```

For comparison with lock-free skiplist

```
BenchmarkReadParallel/branch_4-4 1000000 1358 ns/op
BenchmarkWriteParallelAlt-4 100 19644625 ns/op
BenchmarkReadWrite/frac_0-4 500000 7073 ns/op
BenchmarkReadWrite/frac_1-4 1000000 8509 ns/op
BenchmarkReadWrite/frac_2-4 1000000 7978 ns/op
BenchmarkReadWrite/frac_3-4 500000 5832 ns/op
BenchmarkReadWrite/frac_4-4 1000000 5714 ns/op
BenchmarkReadWrite/frac_5-4 1000000 5011 ns/op
BenchmarkReadWrite/frac_6-4 1000000 4417 ns/op
BenchmarkReadWrite/frac_7-4 1000000 3434 ns/op
BenchmarkReadWrite/frac_8-4 1000000 2393 ns/op
BenchmarkReadWrite/frac_9-4 1000000 1437 ns/op
BenchmarkReadWrite/frac_10-4 3000000 483 ns/op
```
79 changes: 0 additions & 79 deletions skiplist/skiplist_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,85 +148,6 @@ func randomKey() []byte {
return bs
}

func BenchmarkWrite(b *testing.B) {
maxDepth := 10
for branch := 2; branch <= 7; branch++ {
b.Run(fmt.Sprintf("branch_%d", branch), func(b *testing.B) {
list := NewSkiplist(maxDepth, branch, DefaultComparator)
for i := 0; i < b.N; i++ {
list.Insert(randomKey())
}
})
}
}

func BenchmarkWriteParallel(b *testing.B) {
maxDepth := 10
for branch := 2; branch <= 7; branch++ {
b.Run(fmt.Sprintf("branch_%d", branch), func(b *testing.B) {
list := NewSkiplist(maxDepth, branch, DefaultComparator)
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
list.InsertConcurrently(randomKey())
}
})
})
}
}

// Alternate version to WriteParallel that fixes the number of writes to be decently large. This
// is needed for the lock-free skiplist benchmark and we need this here for a fairer comparison.
// If we do intend to keep multiple skiplist implementations in the codebase, we can refactor
// this benchmarking code.
func BenchmarkWriteParallelAlt(b *testing.B) {
maxDepth := 10
branch := 3
b.ResetTimer()
for i := 0; i < b.N; i++ {
list := NewSkiplist(maxDepth, branch, DefaultComparator)
for i := 0; i < 10000; i++ {
list.InsertConcurrently(randomKey())
}
}
}

func BenchmarkRead(b *testing.B) {
maxDepth := 10
for branch := 2; branch <= 7; branch++ {
b.Run(fmt.Sprintf("branch_%d", branch), func(b *testing.B) {
list := NewSkiplist(maxDepth, branch, DefaultComparator)
for i := 0; i < 100000; i++ {
list.Insert(randomKey())
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
it := list.Iterator()
it.Seek(randomKey())
}
})
}
}

// For comparison with lock-free skiplist.
func BenchmarkReadParallel(b *testing.B) {
maxDepth := 10
for branch := 2; branch <= 7; branch++ {
b.Run(fmt.Sprintf("branch_%d", branch), func(b *testing.B) {
list := NewSkiplist(maxDepth, branch, DefaultComparator)
for i := 0; i < 100000; i++ {
list.Insert(randomKey())
}
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
it := list.Iterator()
it.Seek(randomKey())
}
})
})
}
}

// Standard test. Some fraction is read. Some fraction is write. Writes have
// to go through mutex lock.
func BenchmarkReadWrite(b *testing.B) {
Expand Down
32 changes: 27 additions & 5 deletions skl/README.md
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,31 @@
This is better than `skiplist` and `slist`.
This is much better than `skiplist` and `slist`.

WriteParallel is about 3X faster and ReadParallel is 2X faster than `skiplist` and 20X faster
than `slist`.
```
BenchmarkReadWrite/frac_0-4 1000000 1516 ns/op
BenchmarkReadWrite/frac_1-4 1000000 1456 ns/op
BenchmarkReadWrite/frac_2-4 1000000 1354 ns/op
BenchmarkReadWrite/frac_3-4 1000000 1295 ns/op
BenchmarkReadWrite/frac_4-4 1000000 1142 ns/op
BenchmarkReadWrite/frac_5-4 1000000 1077 ns/op
BenchmarkReadWrite/frac_6-4 1000000 1003 ns/op
BenchmarkReadWrite/frac_7-4 2000000 1054 ns/op
BenchmarkReadWrite/frac_8-4 2000000 929 ns/op
BenchmarkReadWrite/frac_9-4 3000000 815 ns/op
BenchmarkReadWrite/frac_10-4 5000000 472 ns/op
```

But compared to a simple map with read-write lock, it is still slower.

```
BenchmarkWriteParallelAlt-4 200 7700008 ns/op
BenchmarkReadParallel-4 2000000 787 ns/op
BenchmarkReadWriteMap/frac_0-4 2000000 883 ns/op
BenchmarkReadWriteMap/frac_1-4 2000000 830 ns/op
BenchmarkReadWriteMap/frac_2-4 2000000 658 ns/op
BenchmarkReadWriteMap/frac_3-4 2000000 662 ns/op
BenchmarkReadWriteMap/frac_4-4 2000000 657 ns/op
BenchmarkReadWriteMap/frac_5-4 2000000 650 ns/op
BenchmarkReadWriteMap/frac_6-4 3000000 592 ns/op
BenchmarkReadWriteMap/frac_7-4 3000000 573 ns/op
BenchmarkReadWriteMap/frac_8-4 3000000 539 ns/op
BenchmarkReadWriteMap/frac_9-4 3000000 521 ns/op
BenchmarkReadWriteMap/frac_10-4 3000000 479 ns/op
```
66 changes: 48 additions & 18 deletions skl/skl_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,28 +262,58 @@ func randomKey() []byte {
return b
}

// Alternate version to WriteParallel that fixes the number of writes to be decently large.
func BenchmarkWriteParallelAlt(b *testing.B) {
// Standard test. Some fraction is read. Some fraction is write. Writes have
// to go through mutex lock.
func BenchmarkReadWrite(b *testing.B) {
value := newValue(123)
b.ResetTimer()
for i := 0; i < b.N; i++ {
list := NewSkiplist()
for i := 0; i < 10000; i++ {
list.Put(randomKey(), value, true)
}
for i := 0; i <= 10; i++ {
readFrac := float32(i) / 10.0
b.Run(fmt.Sprintf("frac_%d", i), func(b *testing.B) {
list := NewSkiplist()
b.ResetTimer()
var count int
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
if rand.Float32() < readFrac {
if list.Get(randomKey()) != nil {
count++
}
} else {
list.Put(randomKey(), value, true)
}
}
})
})
}
}

func BenchmarkReadParallel(b *testing.B) {
// Standard test. Some fraction is read. Some fraction is write. Writes have
// to go through mutex lock.
func BenchmarkReadWriteMap(b *testing.B) {
value := newValue(123)
list := NewSkiplist()
for i := 0; i < 100000; i++ {
list.Put(randomKey(), value, true)
for i := 0; i <= 10; i++ {
readFrac := float32(i) / 10.0
b.Run(fmt.Sprintf("frac_%d", i), func(b *testing.B) {
m := make(map[string]unsafe.Pointer)
var mutex sync.RWMutex
b.ResetTimer()
var count int
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
if rand.Float32() < readFrac {
mutex.RLock()
_, ok := m[string(randomKey())]
mutex.RUnlock()
if ok {
count++
}
} else {
mutex.Lock()
m[string(randomKey())] = value
mutex.Unlock()
}
}
})
})
}
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
list.Get(randomKey())
}
})
}
48 changes: 8 additions & 40 deletions slist/slist_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,34 +223,26 @@ func randomKey() []byte {
return b
}

func BenchmarkReadParallel(b *testing.B) {
value := newValue(123)
list := NewSkiplist()
for i := 0; i < 100000; i++ {
list.Put(randomKey(), value, true)
}
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
list.Get(randomKey())
}
})
}

// Standard test. Some fraction is read. Some fraction is write. Writes have
// to go through mutex lock.
// NOTE: For this implementation, this test doesn't seem to terminate sometimes.
// I think the reason is that this skiplist likes to restart and there is high variance in
// the running time. As a result, the benchmark framework is unsure and keeps retrying.
// We used to have a read-parallel and write-parallel test. But let's just delete this lib soon.
func BenchmarkReadWrite(b *testing.B) {
value := newValue(123)
for i := 0; i <= 10; i++ {
readFrac := float32(i) / 10.0
b.Run(fmt.Sprintf("frac_%d", i), func(b *testing.B) {
list := NewSkiplist()
b.ResetTimer()
var count int
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
if rand.Float32() < readFrac {
it := list.NewIterator()
it.Seek(randomKey())
if list.Get(randomKey()) != nil {
count++
}
} else {
list.Put(randomKey(), value, true)
}
Expand All @@ -259,27 +251,3 @@ func BenchmarkReadWrite(b *testing.B) {
})
}
}

// Test takes a long time because early parallel writes tend to lead to a lot of starts and
// variance in the running time.
func BenchmarkWriteParallel(b *testing.B) {
value := newValue(123)
list := NewSkiplist()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
list.Put(randomKey(), value, true)
}
})
}

// Alternate version to WriteParallel that fixes the number of writes to be decently large.
func BenchmarkWriteParallelAlt(b *testing.B) {
value := newValue(123)
b.ResetTimer()
for i := 0; i < b.N; i++ {
list := NewSkiplist()
for i := 0; i < 10000; i++ {
list.Put(randomKey(), value, true)
}
}
}

0 comments on commit 1a0c61c

Please sign in to comment.