Skip to content

Commit 7e35fc9

Browse files
End-reyEnd-rey
committed
fstree: optimize GetRange operation
Since #3383 and #3431, it is now not necessary to unmarshal the object to get its payload, and we can read the payload from reader to find the data. ``` goos: linux goarch: amd64 pkg: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree cpu: AMD Ryzen 7 PRO 4750U with Radeon Graphics │ old.txt │ new.txt │ │ sec/op │ sec/op vs base │ FSTree_GetRange/size=10MB,off=1MB,len=4KB/regular-16 7457.9µ ± 8% 262.2µ ± 8% -96.48% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=1MB,len=4KB/compressed-16 7440.2µ ± 5% 261.5µ ± 10% -96.49% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=1MB,len=4KB/combined-16 9840.3µ ± 6% 355.8µ ± 4% -96.38% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=10MB/regular-16 8.242m ± 8% 3.977m ± 1% -51.75% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=10MB/compressed-16 7.814m ± 4% 3.979m ± 2% -49.07% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=10MB/combined-16 9.821m ± 4% 3.765m ± 3% -61.67% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=Empty/regular-16 10.432m ± 6% 3.987m ± 2% -61.78% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=Empty/compressed-16 10.721m ± 9% 3.982m ± 2% -62.85% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=Empty/combined-16 9.766m ± 5% 3.680m ± 6% -62.32% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=4KB/regular-16 77.48µ ± 2% 90.66µ ± 2% +17.00% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=4KB/compressed-16 77.42µ ± 5% 91.58µ ± 3% +18.29% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=4KB/combined-16 79.90µ ± 3% 90.45µ ± 1% +13.21% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=Empty/regular-16 80.28µ ± 2% 89.15µ ± 2% +11.05% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=Empty/compressed-16 79.36µ ± 3% 89.60µ ± 3% +12.91% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=Empty/combined-16 77.83µ ± 6% 91.00µ ± 1% +16.93% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=1KB,len=1KB/regular-16 79.47µ ± 2% 86.42µ ± 2% +8.75% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=1KB,len=1KB/compressed-16 79.30µ ± 4% 88.33µ ± 2% +11.40% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=1KB,len=1KB/combined-16 79.86µ ± 2% 90.45µ ± 1% +13.27% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=9MB,len=4KB/regular-16 10.294m ± 6% 1.856m ± 3% -81.97% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=9MB,len=4KB/compressed-16 10.059m ± 6% 1.848m ± 4% -81.63% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=9MB,len=4KB/combined-16 9.650m ± 6% 1.723m ± 6% -82.15% (p=0.000 n=10) geomean 1.199m 478.5µ -60.09% │ old.txt │ new.txt │ │ B/op │ B/op vs base │ FSTree_GetRange/size=10MB,off=1MB,len=4KB/regular-16 20495.52Ki ± 0% 43.91Ki ± 0% -99.79% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=1MB,len=4KB/compressed-16 20495.52Ki ± 0% 43.90Ki ± 0% -99.79% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=1MB,len=4KB/combined-16 20495.96Ki ± 0% 43.81Ki ± 0% -99.79% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=10MB/regular-16 20.02Mi ± 0% 10.04Mi ± 0% -49.84% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=10MB/compressed-16 20.02Mi ± 0% 10.04Mi ± 0% -49.84% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=10MB/combined-16 20.02Mi ± 0% 10.04Mi ± 0% -49.84% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=Empty/regular-16 20.02Mi ± 0% 10.04Mi ± 0% -49.85% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=Empty/compressed-16 20.02Mi ± 0% 10.04Mi ± 0% -49.85% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=Empty/combined-16 20.02Mi ± 0% 10.04Mi ± 0% -49.84% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=4KB/regular-16 17.62Ki ± 0% 48.31Ki ± 0% +174.24% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=4KB/compressed-16 17.62Ki ± 0% 48.31Ki ± 0% +174.24% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=4KB/combined-16 17.90Ki ± 0% 48.08Ki ± 0% +168.60% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=Empty/regular-16 17.77Ki ± 0% 47.95Ki ± 0% +169.80% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=Empty/compressed-16 17.77Ki ± 0% 47.95Ki ± 0% +169.80% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=Empty/combined-16 17.91Ki ± 0% 48.13Ki ± 0% +168.81% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=1KB,len=1KB/regular-16 17.92Ki ± 0% 44.77Ki ± 0% +149.83% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=1KB,len=1KB/compressed-16 17.92Ki ± 0% 44.77Ki ± 0% +149.83% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=1KB,len=1KB/combined-16 17.95Ki ± 1% 45.13Ki ± 0% +151.45% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=9MB,len=4KB/regular-16 20495.92Ki ± 0% 43.66Ki ± 0% -99.79% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=9MB,len=4KB/compressed-16 20495.92Ki ± 0% 43.66Ki ± 0% -99.79% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=9MB,len=4KB/combined-16 20495.94Ki ± 0% 43.75Ki ± 0% -99.79% (p=0.000 n=10) geomean 999.8Ki 214.7Ki -78.52% │ old.txt │ new.txt │ │ allocs/op │ allocs/op vs base │ FSTree_GetRange/size=10MB,off=1MB,len=4KB/regular-16 127.0 ± 0% 145.0 ± 0% +14.17% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=1MB,len=4KB/compressed-16 127.0 ± 0% 145.0 ± 0% +14.17% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=1MB,len=4KB/combined-16 139.0 ± 2% 141.0 ± 1% +1.44% (p=0.010 n=10) FSTree_GetRange/size=10MB,off=Empty,len=10MB/regular-16 149.0 ± 0% 148.0 ± 0% -0.67% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=10MB/compressed-16 149.0 ± 0% 148.0 ± 0% -0.67% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=10MB/combined-16 138.5 ± 3% 140.0 ± 2% +1.08% (p=0.027 n=10) FSTree_GetRange/size=10MB,off=Empty,len=Empty/regular-16 135.0 ± 0% 137.0 ± 0% +1.48% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=Empty/compressed-16 135.0 ± 0% 137.0 ± 0% +1.48% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=Empty/combined-16 137.0 ± 2% 141.0 ± 1% +2.92% (p=0.001 n=10) FSTree_GetRange/size=4KB,off=Empty,len=4KB/regular-16 131.0 ± 0% 145.0 ± 0% +10.69% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=4KB/compressed-16 131.0 ± 0% 145.0 ± 0% +10.69% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=4KB/combined-16 137.0 ± 1% 139.0 ± 1% +1.46% (p=0.007 n=10) FSTree_GetRange/size=4KB,off=Empty,len=Empty/regular-16 135.0 ± 0% 137.0 ± 0% +1.48% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=Empty/compressed-16 135.0 ± 0% 137.0 ± 0% +1.48% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=Empty/combined-16 137.5 ± 0% 140.5 ± 1% +2.18% (p=0.001 n=10) FSTree_GetRange/size=4KB,off=1KB,len=1KB/regular-16 139.0 ± 0% 129.0 ± 0% -7.19% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=1KB,len=1KB/compressed-16 139.0 ± 0% 129.0 ± 0% -7.19% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=1KB,len=1KB/combined-16 139.0 ± 2% 140.5 ± 2% ~ (p=0.058 n=10) FSTree_GetRange/size=10MB,off=9MB,len=4KB/regular-16 139.0 ± 0% 138.0 ± 0% -0.72% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=9MB,len=4KB/compressed-16 139.0 ± 0% 138.0 ± 0% -0.72% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=9MB,len=4KB/combined-16 138.0 ± 2% 140.5 ± 2% +1.81% (p=0.002 n=10) geomean 136.8 139.9 +2.26% ``` Closes #1724. Signed-off-by: Andrey Butusov <[email protected]>
1 parent 4ef414f commit 7e35fc9

File tree

6 files changed

+44
-11
lines changed

6 files changed

+44
-11
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ Changelog for NeoFS Node
2323
- SN now verifies object header from proxy GET/HEAD responses against requested ID (#3406)
2424
- SN now verifies payload from proxy GET responses against in-header checksum (#3406)
2525
- Write cache initialization happens much faster now, some redundant checks were removed (#3417)
26+
- Optimize `GetRange` operation for FSTree (#3438)
2627

2728
### Removed
2829
- Short header support in HEAD's request and response (#3424)

‎pkg/local_object_storage/blobstor/fstree/fstree.go‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -503,25 +503,44 @@ func (t *FSTree) GetStream(addr oid.Address) (*objectSDK.Object, io.ReadCloser,
503503

504504
// GetRange implements common.Storage.
505505
func (t *FSTree) GetRange(addr oid.Address, from uint64, length uint64) ([]byte, error) {
506-
obj, err := t.Get(addr)
506+
if length > math.MaxInt {
507+
return nil, logicerr.Wrap(apistatus.ErrObjectOutOfRange)
508+
}
509+
510+
header, reader, err := t.getObjectStream(addr)
507511
if err != nil {
508512
return nil, err
509513
}
514+
defer reader.Close()
510515

511-
payload := obj.Payload()
512-
pLen := uint64(len(payload))
516+
pLen := header.PayloadSize()
513517
var to uint64
514518
if length != 0 {
515519
to = from + length
516520
} else {
517521
to = pLen
518522
}
519-
520523
if to < from || pLen < from || pLen < to {
521-
return nil, logicerr.Wrap(apistatus.ObjectOutOfRange{})
524+
return nil, logicerr.Wrap(apistatus.ErrObjectOutOfRange)
525+
}
526+
527+
if from > 0 {
528+
n, err := reader.Seek(int64(from), io.SeekStart)
529+
if err != nil {
530+
return nil, fmt.Errorf("seek to %d in stream: %w", from, err)
531+
}
532+
if n < int64(from) {
533+
return nil, logicerr.Wrap(apistatus.ErrObjectOutOfRange)
534+
}
535+
}
536+
537+
payload := make([]byte, to-from)
538+
_, err = io.ReadFull(reader, payload)
539+
if err != nil {
540+
return nil, fmt.Errorf("read %d bytes from stream: %w", length, err)
522541
}
523542

524-
return payload[from:to], nil
543+
return payload, nil
525544
}
526545

527546
// Type is fstree storage type used in logs and configuration.

‎pkg/local_object_storage/blobstor/fstree/head.go‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ func (t *FSTree) Head(addr oid.Address) (*objectSDK.Object, error) {
4141

4242
// getObjectStream reads an object from the storage by address as a stream.
4343
// It returns the object with header only, and a reader for the payload.
44-
func (t *FSTree) getObjectStream(addr oid.Address) (*objectSDK.Object, io.ReadCloser, error) {
44+
func (t *FSTree) getObjectStream(addr oid.Address) (*objectSDK.Object, io.ReadSeekCloser, error) {
4545
p := t.treePath(addr)
4646

4747
f, err := os.Open(p)
@@ -68,7 +68,7 @@ func (t *FSTree) getObjectStream(addr oid.Address) (*objectSDK.Object, io.ReadCl
6868

6969
// extractHeaderOnly reads the header of an object from a file.
7070
// The caller is responsible for closing the returned io.ReadCloser if it is not nil.
71-
func (t *FSTree) extractHeaderAndStream(id oid.ID, f *os.File) (*objectSDK.Object, io.ReadCloser, error) {
71+
func (t *FSTree) extractHeaderAndStream(id oid.ID, f *os.File) (*objectSDK.Object, io.ReadSeekCloser, error) {
7272
buf := make([]byte, objectSDK.MaxHeaderLen, 2*objectSDK.MaxHeaderLen)
7373
n, err := io.ReadFull(f, buf)
7474
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
@@ -124,7 +124,7 @@ func (t *FSTree) extractHeaderAndStream(id oid.ID, f *os.File) (*objectSDK.Objec
124124

125125
// readHeaderAndPayload reads an object header from the file and returns reader for payload.
126126
// This function takes ownership of the io.ReadCloser and will close it if it does not return it.
127-
func (t *FSTree) readHeaderAndPayload(f io.ReadCloser, initial []byte) (*objectSDK.Object, io.ReadCloser, error) {
127+
func (t *FSTree) readHeaderAndPayload(f io.ReadCloser, initial []byte) (*objectSDK.Object, io.ReadSeekCloser, error) {
128128
var err error
129129
if len(initial) < objectSDK.MaxHeaderLen {
130130
_ = f.Close()
@@ -137,7 +137,10 @@ func (t *FSTree) readHeaderAndPayload(f io.ReadCloser, initial []byte) (*objectS
137137
if err != nil {
138138
return nil, nil, fmt.Errorf("unmarshal object: %w", err)
139139
}
140-
return obj.CutPayload(), io.NopCloser(bytes.NewReader(obj.Payload())), nil
140+
return obj.CutPayload(), &payloadReader{
141+
Reader: bytes.NewReader(obj.Payload()),
142+
close: func() error { return nil },
143+
}, nil
141144
}
142145

143146
return t.readUntilPayload(f, initial)
@@ -146,7 +149,7 @@ func (t *FSTree) readHeaderAndPayload(f io.ReadCloser, initial []byte) (*objectS
146149
// readUntilPayload reads an object from the file until the payload field is reached
147150
// and returns the object along with a reader for the remaining data.
148151
// This function takes ownership of the io.ReadCloser and will close it if it does not return it.
149-
func (t *FSTree) readUntilPayload(f io.ReadCloser, initial []byte) (*objectSDK.Object, io.ReadCloser, error) {
152+
func (t *FSTree) readUntilPayload(f io.ReadCloser, initial []byte) (*objectSDK.Object, io.ReadSeekCloser, error) {
150153
reader := f
151154

152155
if t.IsCompressed(initial) {
@@ -249,3 +252,10 @@ type payloadReader struct {
249252
func (p *payloadReader) Close() error {
250253
return p.close()
251254
}
255+
256+
func (p *payloadReader) Seek(offset int64, whence int) (int64, error) {
257+
if seeker, ok := p.Reader.(io.Seeker); ok {
258+
return seeker.Seek(offset, whence)
259+
}
260+
return io.CopyN(io.Discard, p.Reader, offset)
261+
}

‎pkg/local_object_storage/blobstor/fstree/head_bench_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ func generateTestObject(payloadSize int) *objectSDK.Object {
105105
} else {
106106
obj.SetPayload(nil)
107107
}
108+
obj.SetPayloadSize(uint64(payloadSize))
108109

109110
return &obj
110111
}

‎pkg/local_object_storage/blobstor/internal/storagetest/common.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ func NewObject(sz uint64) *objectSDK.Object {
100100
if ln := uint64(len(data)); ln > sz {
101101
raw.SetPayload(raw.Payload()[:sz-(ln-sz)])
102102
}
103+
raw.SetPayloadSize(uint64(len(raw.Payload())))
103104

104105
return raw
105106
}

‎pkg/local_object_storage/engine/error_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ func TestBlobstorFailback(t *testing.T) {
137137
for _, size := range []int{15, errSmallSize + 1} {
138138
obj := generateObjectWithCID(cidtest.ID())
139139
obj.SetPayload(make([]byte, size))
140+
obj.SetPayloadSize(uint64(size))
140141

141142
e.mtx.RLock()
142143
err = e.shards[id[0].String()].Shard.Put(obj, nil, 0)

0 commit comments

Comments
 (0)