Skip to content

Commit 72a4c72

Browse files
committed
compute block length accounting for compression
Taken from PR #1698 and commit da5f789
1 parent c65a8ac commit 72a4c72

File tree

2 files changed

+11
-2
lines changed

2 files changed

+11
-2
lines changed

Diff for: go.sum

-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE
55
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
66
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
77
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
8-
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
98
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
109
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
1110
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=

Diff for: table/builder.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,16 @@ func NewTableBuilder(opts Options) *Builder {
156156
return b
157157
}
158158

159+
func maxEncodedLen(ctype options.CompressionType, sz int) int {
160+
switch ctype {
161+
case options.Snappy:
162+
return snappy.MaxEncodedLen(sz)
163+
case options.ZSTD:
164+
return y.ZSTDCompressBound(sz)
165+
}
166+
return sz
167+
}
168+
159169
func (b *Builder) handleBlock() {
160170
defer b.wg.Done()
161171

@@ -178,7 +188,7 @@ func (b *Builder) handleBlock() {
178188
// BlockBuf should always less than or equal to allocated space. If the blockBuf is greater
179189
// than allocated space that means the data from this block cannot be stored in its
180190
// existing location.
181-
allocatedSpace := (item.end) + padding + 1
191+
allocatedSpace := maxEncodedLen(b.opts.Compression, (item.end)) + padding + 1
182192
y.AssertTrue(len(blockBuf) <= allocatedSpace)
183193

184194
// blockBuf was allocated on allocator. So, we don't need to copy it over.

0 commit comments

Comments
 (0)