-
Notifications
You must be signed in to change notification settings - Fork 263
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use 8 bytes to store int64 components of database keys (#107)
* Introduce KeyFormat that uses a full 8 bytes for int64 values and avoids string manipulatio/scanning * Add node key and orphan key benchmark * Fix various issue from PR: punctuation, add overflow test, and improve scan function
- Loading branch information
Showing
8 changed files
with
318 additions
and
57 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,144 @@ | ||
| package iavl | ||
|
|
||
| import ( | ||
| "encoding/binary" | ||
| "fmt" | ||
| ) | ||
|
|
||
| // Provides a fixed-width lexicographically sortable []byte key format | ||
| type KeyFormat struct { | ||
| prefix byte | ||
| layout []int | ||
| length int | ||
| } | ||
|
|
||
| // Create a []byte key format based on a single byte prefix and fixed width key segments each of whose length is | ||
| // specified by by the corresponding element of layout. | ||
| // | ||
| // For example, to store keys that could index some objects by a version number and their SHA256 hash using the form: | ||
| // 'c<version uint64><hash [32]byte>' then you would define the KeyFormat with: | ||
| // | ||
| // var keyFormat = NewKeyFormat('c', 8, 32) | ||
| // | ||
| // Then you can create a key with: | ||
| // | ||
| // func ObjectKey(version uint64, objectBytes []byte) []byte { | ||
| // hasher := sha256.New() | ||
| // hasher.Sum(nil) | ||
| // return keyFormat.Key(version, hasher.Sum(nil)) | ||
| // } | ||
| func NewKeyFormat(prefix byte, layout ...int) *KeyFormat { | ||
| // For prefix byte | ||
| length := 1 | ||
| for _, l := range layout { | ||
| length += int(l) | ||
| } | ||
| return &KeyFormat{ | ||
| prefix: prefix, | ||
| layout: layout, | ||
| length: length, | ||
| } | ||
| } | ||
|
|
||
| // Format the byte segments into the key format - will panic if the segment lengths do not match the layout. | ||
| func (kf *KeyFormat) KeyBytes(segments ...[]byte) []byte { | ||
| key := make([]byte, kf.length) | ||
| key[0] = kf.prefix | ||
| n := 1 | ||
| for i, s := range segments { | ||
| l := kf.layout[i] | ||
| if len(s) > l { | ||
| panic(fmt.Errorf("length of segment %X provided to KeyFormat.KeyBytes() is longer than the %d bytes "+ | ||
| "required by layout for segment %d", s, l, i)) | ||
| } | ||
| n += l | ||
| // Big endian so pad on left if not given the full width for this segment | ||
| copy(key[n-len(s):n], s) | ||
| } | ||
| return key[:n] | ||
| } | ||
|
|
||
| // Format the args passed into the key format - will panic if the arguments passed do not match the length | ||
| // of the segment to which they correspond. When called with no arguments returns the raw prefix (useful as a start | ||
| // element of the entire keys space when sorted lexicographically). | ||
| func (kf *KeyFormat) Key(args ...interface{}) []byte { | ||
| if len(args) > len(kf.layout) { | ||
| panic(fmt.Errorf("KeyFormat.Key() is provided with %d args but format only has %d segments", | ||
| len(args), len(kf.layout))) | ||
| } | ||
| segments := make([][]byte, len(args)) | ||
| for i, a := range args { | ||
| segments[i] = format(a) | ||
| } | ||
| return kf.KeyBytes(segments...) | ||
| } | ||
|
|
||
| // Reads out the bytes associated with each segment of the key format from key. | ||
| func (kf *KeyFormat) ScanBytes(key []byte) [][]byte { | ||
| segments := make([][]byte, len(kf.layout)) | ||
| n := 1 | ||
| for i, l := range kf.layout { | ||
| n += l | ||
| if n > len(key) { | ||
| return segments[:i] | ||
| } | ||
| segments[i] = key[n-l : n] | ||
| } | ||
| return segments | ||
| } | ||
|
|
||
| // Extracts the segments into the values pointed to by each of args. Each arg must be a pointer to int64, uint64, or | ||
| // []byte, and the width of the args must match layout. | ||
| func (kf *KeyFormat) Scan(key []byte, args ...interface{}) { | ||
| segments := kf.ScanBytes(key) | ||
| if len(args) > len(segments) { | ||
| panic(fmt.Errorf("KeyFormat.Scan() is provided with %d args but format only has %d segments in key %X", | ||
| len(args), len(segments), key)) | ||
| } | ||
| for i, a := range args { | ||
| scan(a, segments[i]) | ||
| } | ||
| } | ||
|
|
||
| // Return the prefix as a string. | ||
| func (kf *KeyFormat) Prefix() string { | ||
| return string([]byte{kf.prefix}) | ||
| } | ||
|
|
||
| func scan(a interface{}, value []byte) { | ||
| switch v := a.(type) { | ||
| case *int64: | ||
| // Negative values will be mapped correctly when read in as uint64 and then type converted | ||
| *v = int64(binary.BigEndian.Uint64(value)) | ||
| case *uint64: | ||
| *v = binary.BigEndian.Uint64(value) | ||
| case *[]byte: | ||
| *v = value | ||
| default: | ||
| panic(fmt.Errorf("KeyFormat scan() does not support scanning value of type %T: %v", a, a)) | ||
| } | ||
| } | ||
|
|
||
| func format(a interface{}) []byte { | ||
| switch v := a.(type) { | ||
| case uint64: | ||
| return formatUint64(v) | ||
| case int64: | ||
| return formatUint64(uint64(v)) | ||
| // Provide formatting from int,uint as a convenience to avoid casting arguments | ||
| case uint: | ||
| return formatUint64(uint64(v)) | ||
| case int: | ||
| return formatUint64(uint64(v)) | ||
| case []byte: | ||
| return v | ||
| default: | ||
| panic(fmt.Errorf("KeyFormat format() does not support formatting value of type %T: %v", a, a)) | ||
| } | ||
| } | ||
|
|
||
| func formatUint64(v uint64) []byte { | ||
| bs := make([]byte, 8) | ||
| binary.BigEndian.PutUint64(bs, v) | ||
| return bs | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| package iavl | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func TestKeyFormatBytes(t *testing.T) { | ||
| kf := NewKeyFormat(byte('e'), 8, 8, 8) | ||
| assert.Equal(t, []byte{'e', 0, 0, 0, 0, 0, 1, 2, 3}, kf.KeyBytes([]byte{1, 2, 3})) | ||
| assert.Equal(t, []byte{'e', 1, 2, 3, 4, 5, 6, 7, 8}, kf.KeyBytes([]byte{1, 2, 3, 4, 5, 6, 7, 8})) | ||
| assert.Equal(t, []byte{'e', 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 0, 0, 1, 1, 2, 2, 3, 3}, | ||
| kf.KeyBytes([]byte{1, 2, 3, 4, 5, 6, 7, 8}, []byte{1, 2, 3, 4, 5, 6, 7, 8}, []byte{1, 1, 2, 2, 3, 3})) | ||
| assert.Equal(t, []byte{'e'}, kf.KeyBytes()) | ||
| } | ||
|
|
||
| func TestKeyFormat(t *testing.T) { | ||
| kf := NewKeyFormat(byte('e'), 8, 8, 8) | ||
| key := []byte{'e', 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 0, 0, 1, 144} | ||
| var a, b, c int64 = 100, 200, 400 | ||
| assert.Equal(t, key, kf.Key(a, b, c)) | ||
|
|
||
| var ao, bo, co = new(int64), new(int64), new(int64) | ||
| kf.Scan(key, ao, bo, co) | ||
| assert.Equal(t, a, *ao) | ||
| assert.Equal(t, b, *bo) | ||
| assert.Equal(t, c, *co) | ||
|
|
||
| bs := new([]byte) | ||
| kf.Scan(key, ao, bo, bs) | ||
| assert.Equal(t, a, *ao) | ||
| assert.Equal(t, b, *bo) | ||
| assert.Equal(t, []byte{0, 0, 0, 0, 0, 0, 1, 144}, *bs) | ||
|
|
||
| assert.Equal(t, []byte{'e', 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 200}, kf.Key(a, b)) | ||
| } | ||
|
|
||
| func TestNegativeKeys(t *testing.T) { | ||
| kf := NewKeyFormat(byte('e'), 8, 8) | ||
|
|
||
| var a, b int64 = -100, -200 | ||
| // One's complement plus one | ||
| key := []byte{'e', | ||
| 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, byte(0xff + a + 1), | ||
| 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, byte(0xff + b + 1)} | ||
| assert.Equal(t, key, kf.Key(a, b)) | ||
|
|
||
| var ao, bo = new(int64), new(int64) | ||
| kf.Scan(key, ao, bo) | ||
| assert.Equal(t, a, *ao) | ||
| assert.Equal(t, b, *bo) | ||
| } | ||
|
|
||
| func TestOverflow(t *testing.T) { | ||
| kf := NewKeyFormat(byte('o'), 8, 8) | ||
|
|
||
| var a int64 = 1 << 62 | ||
| var b uint64 = 1 << 63 | ||
| key := []byte{'o', | ||
| 0x40, 0, 0, 0, 0, 0, 0, 0, | ||
| 0x80, 0, 0, 0, 0, 0, 0, 0, | ||
| } | ||
| assert.Equal(t, key, kf.Key(a, b)) | ||
|
|
||
| var ao, bo = new(int64), new(int64) | ||
| kf.Scan(key, ao, bo) | ||
| assert.Equal(t, a, *ao) | ||
| assert.Equal(t, int64(b), *bo) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.