From cd9840850ba68bc338f6566e2fd788a58786459b Mon Sep 17 00:00:00 2001 From: Ibrahim Jarif Date: Mon, 2 Nov 2020 14:43:54 +0530 Subject: [PATCH] fix(table): Add onDisk size (#1569) This PR adds onDisk size to the table index --- fb/TableIndex.go | 32 ++++++++++++++++---------------- fb/flatbuffer.fbs | 4 ++-- levels.go | 4 ++-- table/builder.go | 29 +++++++++++++++++------------ table/table.go | 20 +++++--------------- table/table_test.go | 25 ------------------------- 6 files changed, 42 insertions(+), 72 deletions(-) diff --git a/fb/TableIndex.go b/fb/TableIndex.go index 72bea12bd..6c5d794f2 100644 --- a/fb/TableIndex.go +++ b/fb/TableIndex.go @@ -80,28 +80,28 @@ func (rcv *TableIndex) MutateBloomFilter(j int, n byte) bool { return false } -func (rcv *TableIndex) EstimatedSize() uint32 { +func (rcv *TableIndex) MaxVersion() uint64 { o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) if o != 0 { - return rcv._tab.GetUint32(o + rcv._tab.Pos) + return rcv._tab.GetUint64(o + rcv._tab.Pos) } return 0 } -func (rcv *TableIndex) MutateEstimatedSize(n uint32) bool { - return rcv._tab.MutateUint32Slot(8, n) +func (rcv *TableIndex) MutateMaxVersion(n uint64) bool { + return rcv._tab.MutateUint64Slot(8, n) } -func (rcv *TableIndex) MaxVersion() uint64 { +func (rcv *TableIndex) KeyCount() uint32 { o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) if o != 0 { - return rcv._tab.GetUint64(o + rcv._tab.Pos) + return rcv._tab.GetUint32(o + rcv._tab.Pos) } return 0 } -func (rcv *TableIndex) MutateMaxVersion(n uint64) bool { - return rcv._tab.MutateUint64Slot(10, n) +func (rcv *TableIndex) MutateKeyCount(n uint32) bool { + return rcv._tab.MutateUint32Slot(10, n) } func (rcv *TableIndex) UncompressedSize() uint32 { @@ -116,7 +116,7 @@ func (rcv *TableIndex) MutateUncompressedSize(n uint32) bool { return rcv._tab.MutateUint32Slot(12, n) } -func (rcv *TableIndex) KeyCount() uint32 { +func (rcv *TableIndex) OnDiskSize() uint32 { o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) if o != 0 { return rcv._tab.GetUint32(o + rcv._tab.Pos) @@ -124,7 +124,7 @@ func (rcv *TableIndex) KeyCount() uint32 { return 0 } -func (rcv *TableIndex) MutateKeyCount(n uint32) bool { +func (rcv *TableIndex) MutateOnDiskSize(n uint32) bool { return rcv._tab.MutateUint32Slot(14, n) } @@ -143,17 +143,17 @@ func TableIndexAddBloomFilter(builder *flatbuffers.Builder, bloomFilter flatbuff func TableIndexStartBloomFilterVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { return builder.StartVector(1, numElems, 1) } -func TableIndexAddEstimatedSize(builder *flatbuffers.Builder, estimatedSize uint32) { - builder.PrependUint32Slot(2, estimatedSize, 0) -} func TableIndexAddMaxVersion(builder *flatbuffers.Builder, maxVersion uint64) { - builder.PrependUint64Slot(3, maxVersion, 0) + builder.PrependUint64Slot(2, maxVersion, 0) +} +func TableIndexAddKeyCount(builder *flatbuffers.Builder, keyCount uint32) { + builder.PrependUint32Slot(3, keyCount, 0) } func TableIndexAddUncompressedSize(builder *flatbuffers.Builder, uncompressedSize uint32) { builder.PrependUint32Slot(4, uncompressedSize, 0) } -func TableIndexAddKeyCount(builder *flatbuffers.Builder, keyCount uint32) { - builder.PrependUint32Slot(5, keyCount, 0) +func TableIndexAddOnDiskSize(builder *flatbuffers.Builder, onDiskSize uint32) { + builder.PrependUint32Slot(5, onDiskSize, 0) } func TableIndexEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() diff --git a/fb/flatbuffer.fbs b/fb/flatbuffer.fbs index 580f183ba..c76fa7df7 100644 --- a/fb/flatbuffer.fbs +++ b/fb/flatbuffer.fbs @@ -19,10 +19,10 @@ namespace fb; table TableIndex { offsets:[BlockOffset]; bloom_filter:[ubyte]; - estimated_size:uint32; max_version:uint64; - uncompressed_size:uint32; key_count:uint32; + uncompressed_size:uint32; + on_disk_size:uint32; } table BlockOffset { diff --git a/levels.go b/levels.go index 27c00af72..3cdb38e3f 100644 --- a/levels.go +++ b/levels.go @@ -1459,7 +1459,7 @@ type TableInfo struct { Left []byte Right []byte KeyCount uint32 // Number of keys in the table - EstimatedSz uint32 + OnDiskSize uint32 UncompressedSize uint32 MaxVersion uint64 IndexSz int @@ -1476,7 +1476,7 @@ func (s *levelsController) getTableInfo() (result []TableInfo) { Left: t.Smallest(), Right: t.Biggest(), KeyCount: t.KeyCount(), - EstimatedSz: t.EstimatedSize(), + OnDiskSize: t.OnDiskSize(), IndexSz: t.IndexSize(), BloomFilterSize: t.BloomFilterSize(), UncompressedSize: t.UncompressedSize(), diff --git a/table/builder.go b/table/builder.go index b42eec8ea..639967bdd 100644 --- a/table/builder.go +++ b/table/builder.go @@ -85,12 +85,12 @@ type Builder struct { baseKey []byte // Base key for the current block. baseOffset uint32 // Offset for the current block. - entryOffsets []uint32 // Offsets of entries present in current block. - offsets *z.Buffer - estimatedSize uint32 - keyHashes []uint32 // Used for building the bloomfilter. - opt *Options - maxVersion uint64 + entryOffsets []uint32 // Offsets of entries present in current block. + offsets *z.Buffer + onDiskSize uint32 + keyHashes []uint32 // Used for building the bloomfilter. + opt *Options + maxVersion uint64 // Used to concurrently compress/encrypt blocks. wg sync.WaitGroup @@ -230,10 +230,9 @@ func (b *Builder) addHelper(key []byte, v y.ValueStruct, vpLen uint32) { } b.sz += v.Encode(b.buf[b.sz:]) - // Size of KV on SST. - sstSz := uint32(headerSize) + uint32(len(diffKey)) + v.EncodedSize() - // Total estimated size = size on SST + size on vlog (length of value pointer). - b.estimatedSize += (sstSz + vpLen) + // Add the vpLen to the onDisk size. We'll add the size of the block to + // onDisk size in Finish() function. + b.onDiskSize += vpLen } // grow increases the size of b.buf by atleast 50%. @@ -438,6 +437,8 @@ func (b *Builder) Finish(allocate bool) []byte { b.sz = dstLen } + // b.sz is the total size of the compressed table without the index. + b.onDiskSize += b.sz var f y.Filter if b.opt.BloomFalsePositive > 0 { bits := y.BloomBitsPerKey(len(b.keyHashes), b.opt.BloomFalsePositive) @@ -566,13 +567,17 @@ func (b *Builder) buildIndex(bloom []byte, tableSz uint32) []byte { fb.TableIndexStart(builder) fb.TableIndexAddOffsets(builder, boEnd) fb.TableIndexAddBloomFilter(builder, bfoff) - fb.TableIndexAddEstimatedSize(builder, b.estimatedSize) fb.TableIndexAddMaxVersion(builder, b.maxVersion) fb.TableIndexAddUncompressedSize(builder, tableSz) fb.TableIndexAddKeyCount(builder, uint32(len(b.keyHashes))) + fb.TableIndexAddOnDiskSize(builder, b.onDiskSize) builder.Finish(fb.TableIndexEnd(builder)) - return builder.FinishedBytes() + buf := builder.FinishedBytes() + index := fb.GetRootAsTableIndex(buf, 0) + // Mutate the ondisk size to include the size of the index as well. + y.AssertTrue(index.MutateOnDiskSize(index.OnDiskSize() + uint32(len(buf)))) + return buf } // writeBlockOffsets writes all the blockOffets in b.offsets and returns the diff --git a/table/table.go b/table/table.go index 5a9f21843..d0a02d419 100644 --- a/table/table.go +++ b/table/table.go @@ -108,7 +108,7 @@ type Table struct { Checksum []byte CreatedAt time.Time // Stores the total size of key-values stored in this table (including the size on vlog). - estimatedSize uint32 + onDiskSize uint32 indexStart int indexLen int hasBloomFilter bool @@ -388,17 +388,7 @@ func (t *Table) initIndex() (*fb.BlockOffset, error) { t._index = index } - if t.opt.Compression == options.None { - t.estimatedSize = index.EstimatedSize() - } else { - // TODO(ibrahim): This estimatedSize doesn't make any sense. If it is tracking the size of - // values including in value log, then index.EstimatedSize() should be used irrespective of - // compression or not. - // - // Due to compression the real size on disk is much - // smaller than what we estimate from index.EstimatedSize. - t.estimatedSize = uint32(t.tableSize) - } + t.onDiskSize = index.OnDiskSize() t.hasBloomFilter = len(index.BloomFilterBytes()) > 0 var bo fb.BlockOffset @@ -580,7 +570,7 @@ func (t *Table) indexKey() uint64 { return t.id } -// UncompressedSize is the size of table index in bytes. +// UncompressedSize is the size uncompressed data stored in this file. func (t *Table) UncompressedSize() uint32 { return t.fetchIndex().UncompressedSize() } @@ -600,9 +590,9 @@ func (t *Table) BloomFilterSize() int { return t.fetchIndex().BloomFilterLength() } -// EstimatedSize returns the total size of key-values stored in this table (including the +// OnDiskSize returns the total size of key-values stored in this table (including the // disk space occupied on the value log). -func (t *Table) EstimatedSize() uint32 { return t.estimatedSize } +func (t *Table) OnDiskSize() uint32 { return t.onDiskSize } // Size is its file size in bytes func (t *Table) Size() int64 { return int64(t.tableSize) } diff --git a/table/table_test.go b/table/table_test.go index 398443f22..8310c9779 100644 --- a/table/table_test.go +++ b/table/table_test.go @@ -865,31 +865,6 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestOpenKVSize(t *testing.T) { - t.Run("compression", func(t *testing.T) { - // When compression is on - opts := getTestTableOptions() - opts.Compression = options.ZSTD - table := buildTestTable(t, "foo", 1000, opts) - defer table.DecrRef() - - // The estimated size is same as table size in case compression is enabled. - require.Equal(t, uint32(table.tableSize), table.EstimatedSize()) - }) - - t.Run("no compressin", func(t *testing.T) { - // When compression is off - opts := getTestTableOptions() - opts.Compression = options.None - table := buildTestTable(t, "foo", 1, opts) - defer table.DecrRef() - - stat, err := table.Fd.Stat() - require.NoError(t, err) - require.Less(t, table.EstimatedSize(), uint32(stat.Size())) - }) -} - // Run this test with command "go test -race -run TestDoesNotHaveRace" func TestDoesNotHaveRace(t *testing.T) { opts := getTestTableOptions()