Skip to content

Commit 766d39b

Browse files
End-reyEnd-rey
committed
fstree: optimize GetRange operation
Since #3383 and #3431, it is now not necessary to unmarshal the object to get its payload, and we can read the payload from reader to find the data. ``` goos: linux goarch: amd64 pkg: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree cpu: AMD Ryzen 7 PRO 4750U with Radeon Graphics │ old.txt │ new.txt │ │ sec/op │ sec/op vs base │ FSTree_GetRange/size=10MB,off=1MB,len=4KB/regular-16 8027.9µ ± 9% 264.8µ ± 5% -96.70% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=1MB,len=4KB/compressed-16 7608.3µ ± 5% 270.5µ ± 6% -96.45% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=1MB,len=4KB/combined-16 7391.8µ ± 6% 360.6µ ± 8% -95.12% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=10MB/regular-16 7.876m ± 4% 4.134m ± 3% -47.52% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=10MB/compressed-16 7.989m ± 6% 4.170m ± 3% -47.80% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=10MB/combined-16 7.530m ± 2% 4.038m ± 8% -46.37% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=Empty/regular-16 8.220m ± 6% 4.164m ± 4% -49.35% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=Empty/compressed-16 8.183m ± 6% 4.703m ± 6% -42.53% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=Empty/combined-16 7.508m ± 5% 4.325m ± 5% -42.40% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=1KB,len=4KB/regular-16 822.54µ ± 6% 83.04µ ± 4% -89.90% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=1KB,len=4KB/compressed-16 884.94µ ± 7% 92.61µ ± 5% -89.53% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=1KB,len=4KB/combined-16 1346.99µ ± 11% 84.62µ ± 10% -93.72% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=Empty,len=1MB/regular-16 908.2µ ± 15% 817.8µ ± 9% ~ (p=0.165 n=10) FSTree_GetRange/size=1MB,off=Empty,len=1MB/compressed-16 946.3µ ± 3% 806.2µ ± 4% -14.80% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=Empty,len=1MB/combined-16 1290.4µ ± 9% 914.6µ ± 11% -29.12% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=Empty,len=Empty/regular-16 955.0µ ± 3% 863.4µ ± 4% -9.60% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=Empty,len=Empty/compressed-16 948.7µ ± 4% 869.5µ ± 5% -8.35% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=Empty,len=Empty/combined-16 1318.8µ ± 13% 955.9µ ± 7% -27.52% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=4KB/regular-16 79.55µ ± 5% 89.11µ ± 2% +12.01% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=4KB/compressed-16 78.48µ ± 5% 87.90µ ± 2% +12.01% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=4KB/combined-16 79.15µ ± 4% 92.59µ ± 3% +16.98% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=Empty/regular-16 81.78µ ± 4% 92.22µ ± 4% +12.77% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=Empty/compressed-16 79.80µ ± 11% 90.99µ ± 3% +14.02% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=Empty/combined-16 92.39µ ± 9% 92.73µ ± 2% ~ (p=0.280 n=10) FSTree_GetRange/size=4KB,off=1KB,len=1KB/regular-16 92.11µ ± 1% 89.30µ ± 2% -3.06% (p=0.002 n=10) FSTree_GetRange/size=4KB,off=1KB,len=1KB/compressed-16 80.11µ ± 17% 91.34µ ± 2% ~ (p=0.105 n=10) FSTree_GetRange/size=4KB,off=1KB,len=1KB/combined-16 79.40µ ± 5% 90.61µ ± 4% +14.12% (p=0.000 n=10) geomean 871.6µ 399.9µ -54.12% │ old.txt │ new.txt │ │ B/op │ B/op vs base │ FSTree_GetRange/size=10MB,off=1MB,len=4KB/regular-16 20495.77Ki ± 0% 43.65Ki ± 0% -99.79% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=1MB,len=4KB/compressed-16 20495.77Ki ± 0% 43.65Ki ± 0% -99.79% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=1MB,len=4KB/combined-16 20496.00Ki ± 0% 43.81Ki ± 0% -99.79% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=10MB/regular-16 20.02Mi ± 0% 10.04Mi ± 0% -49.84% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=10MB/compressed-16 20.02Mi ± 0% 10.04Mi ± 0% -49.84% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=10MB/combined-16 20.02Mi ± 0% 10.04Mi ± 0% -49.84% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=Empty/regular-16 20.02Mi ± 0% 10.04Mi ± 0% -49.84% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=Empty/compressed-16 20.02Mi ± 0% 10.04Mi ± 0% -49.84% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=Empty/combined-16 20.02Mi ± 0% 10.04Mi ± 0% -49.85% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=1KB,len=4KB/regular-16 2063.83Ki ± 0% 43.65Ki ± 0% -97.89% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=1KB,len=4KB/compressed-16 2063.83Ki ± 0% 43.65Ki ± 0% -97.88% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=1KB,len=4KB/combined-16 2063.98Ki ± 0% 43.89Ki ± 0% -97.87% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=Empty,len=1MB/regular-16 2.015Mi ± 0% 1.039Mi ± 0% -48.45% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=Empty,len=1MB/compressed-16 2.015Mi ± 0% 1.039Mi ± 0% -48.45% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=Empty,len=1MB/combined-16 2.016Mi ± 0% 1.039Mi ± 0% -48.46% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=Empty,len=Empty/regular-16 2.015Mi ± 0% 1.039Mi ± 0% -48.44% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=Empty,len=Empty/compressed-16 2.015Mi ± 0% 1.039Mi ± 0% -48.44% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=Empty,len=Empty/combined-16 2.016Mi ± 0% 1.039Mi ± 0% -48.46% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=4KB/regular-16 17.97Ki ± 0% 47.68Ki ± 0% +165.35% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=4KB/compressed-16 17.97Ki ± 0% 47.68Ki ± 0% +165.35% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=4KB/combined-16 17.91Ki ± 0% 48.13Ki ± 0% +168.71% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=Empty/regular-16 18.12Ki ± 0% 48.09Ki ± 0% +165.42% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=Empty/compressed-16 18.12Ki ± 0% 48.09Ki ± 0% +165.42% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=Empty/combined-16 17.90Ki ± 1% 48.12Ki ± 0% +168.78% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=1KB,len=1KB/regular-16 18.15Ki ± 0% 45.07Ki ± 0% +148.34% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=1KB,len=1KB/compressed-16 18.15Ki ± 0% 45.07Ki ± 0% +148.34% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=1KB,len=1KB/combined-16 17.92Ki ± 1% 45.11Ki ± 0% +151.67% (p=0.000 n=10) geomean 913.5Ki 306.3Ki -66.47% │ old.txt │ new.txt │ │ allocs/op │ allocs/op vs base │ FSTree_GetRange/size=10MB,off=1MB,len=4KB/regular-16 135.0 ± 0% 138.0 ± 0% +2.22% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=1MB,len=4KB/compressed-16 135.0 ± 0% 138.0 ± 0% +2.22% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=1MB,len=4KB/combined-16 140.0 ± 1% 140.5 ± 1% +0.36% (p=0.038 n=10) FSTree_GetRange/size=10MB,off=Empty,len=10MB/regular-16 142.0 ± 0% 145.0 ± 0% +2.11% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=10MB/compressed-16 142.0 ± 0% 145.0 ± 0% +2.11% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=10MB/combined-16 138.0 ± 1% 140.5 ± 2% ~ (p=0.063 n=10) FSTree_GetRange/size=10MB,off=Empty,len=Empty/regular-16 139.0 ± 0% 148.0 ± 0% +6.47% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=Empty/compressed-16 139.0 ± 0% 148.0 ± 0% +6.47% (p=0.000 n=10) FSTree_GetRange/size=10MB,off=Empty,len=Empty/combined-16 139.0 ± 2% 139.0 ± 1% ~ (p=0.773 n=10) FSTree_GetRange/size=1MB,off=1KB,len=4KB/regular-16 135.0 ± 0% 134.0 ± 0% -0.74% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=1KB,len=4KB/compressed-16 135.0 ± 0% 134.0 ± 0% -0.74% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=1KB,len=4KB/combined-16 139.5 ± 2% 140.5 ± 2% +0.72% (p=0.030 n=10) FSTree_GetRange/size=1MB,off=Empty,len=1MB/regular-16 127.0 ± 0% 141.0 ± 0% +11.02% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=Empty,len=1MB/compressed-16 127.0 ± 0% 141.0 ± 0% +11.02% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=Empty,len=1MB/combined-16 138.0 ± 2% 139.5 ± 1% ~ (p=0.195 n=10) FSTree_GetRange/size=1MB,off=Empty,len=Empty/regular-16 127.0 ± 0% 144.0 ± 0% +13.39% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=Empty,len=Empty/compressed-16 127.0 ± 0% 144.0 ± 0% +13.39% (p=0.000 n=10) FSTree_GetRange/size=1MB,off=Empty,len=Empty/combined-16 139.0 ± 1% 140.0 ± 2% ~ (p=0.146 n=10) FSTree_GetRange/size=4KB,off=Empty,len=4KB/regular-16 139.0 ± 0% 129.0 ± 0% -7.19% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=4KB/compressed-16 139.0 ± 0% 129.0 ± 0% -7.19% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=4KB/combined-16 138.0 ± 1% 140.5 ± 1% +1.81% (p=0.001 n=10) FSTree_GetRange/size=4KB,off=Empty,len=Empty/regular-16 142.0 ± 0% 140.0 ± 0% -1.41% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=Empty/compressed-16 142.0 ± 0% 140.0 ± 0% -1.41% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=Empty,len=Empty/combined-16 137.5 ± 3% 140.0 ± 2% ~ (p=0.091 n=10) FSTree_GetRange/size=4KB,off=1KB,len=1KB/regular-16 143.0 ± 0% 140.0 ± 0% -2.10% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=1KB,len=1KB/compressed-16 143.0 ± 0% 140.0 ± 0% -2.10% (p=0.000 n=10) FSTree_GetRange/size=4KB,off=1KB,len=1KB/combined-16 138.0 ± 1% 140.5 ± 2% +1.81% (p=0.030 n=10) geomean 137.1 139.9 +2.01% ``` Closes #1724. Signed-off-by: Andrey Butusov <[email protected]>
1 parent fd0307d commit 766d39b

File tree

6 files changed

+44
-11
lines changed

6 files changed

+44
-11
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ Changelog for NeoFS Node
2626
- SN now verifies payload from proxy GET responses against in-header checksum (#3406)
2727
- Write cache initialization happens much faster now, some redundant checks were removed (#3417)
2828
- Metabase no longer stores object headers (#3430)
29+
- Optimize `GetRange` operation for FSTree (#3438)
2930

3031
### Removed
3132
- Short header support in HEAD's request and response (#3424)

‎pkg/local_object_storage/blobstor/fstree/fstree.go‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -503,25 +503,37 @@ func (t *FSTree) GetStream(addr oid.Address) (*objectSDK.Object, io.ReadCloser,
503503

504504
// GetRange implements common.Storage.
505505
func (t *FSTree) GetRange(addr oid.Address, from uint64, length uint64) ([]byte, error) {
506-
obj, err := t.Get(addr)
506+
header, reader, err := t.getObjectStream(addr)
507507
if err != nil {
508508
return nil, err
509509
}
510+
defer reader.Close()
510511

511-
payload := obj.Payload()
512-
pLen := uint64(len(payload))
512+
pLen := header.PayloadSize()
513513
var to uint64
514514
if length != 0 {
515515
to = from + length
516516
} else {
517517
to = pLen
518518
}
519-
520519
if to < from || pLen < from || pLen < to {
521-
return nil, logicerr.Wrap(apistatus.ObjectOutOfRange{})
520+
return nil, logicerr.Wrap(apistatus.ErrObjectOutOfRange)
521+
}
522+
523+
if from > 0 {
524+
_, err = reader.Seek(int64(from), io.SeekStart)
525+
if err != nil {
526+
return nil, fmt.Errorf("seek to %d in stream: %w", from, err)
527+
}
528+
}
529+
530+
payload := make([]byte, to-from)
531+
_, err = io.ReadFull(reader, payload)
532+
if err != nil {
533+
return nil, fmt.Errorf("read %d bytes from stream: %w", length, err)
522534
}
523535

524-
return payload[from:to], nil
536+
return payload, nil
525537
}
526538

527539
// Type is fstree storage type used in logs and configuration.

‎pkg/local_object_storage/blobstor/fstree/head.go‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ func (t *FSTree) Head(addr oid.Address) (*objectSDK.Object, error) {
4141

4242
// getObjectStream reads an object from the storage by address as a stream.
4343
// It returns the object with header only, and a reader for the payload.
44-
func (t *FSTree) getObjectStream(addr oid.Address) (*objectSDK.Object, io.ReadCloser, error) {
44+
func (t *FSTree) getObjectStream(addr oid.Address) (*objectSDK.Object, io.ReadSeekCloser, error) {
4545
p := t.treePath(addr)
4646

4747
f, err := os.Open(p)
@@ -68,7 +68,7 @@ func (t *FSTree) getObjectStream(addr oid.Address) (*objectSDK.Object, io.ReadCl
6868

6969
// extractHeaderOnly reads the header of an object from a file.
7070
// The caller is responsible for closing the returned io.ReadCloser if it is not nil.
71-
func (t *FSTree) extractHeaderAndStream(id oid.ID, f *os.File) (*objectSDK.Object, io.ReadCloser, error) {
71+
func (t *FSTree) extractHeaderAndStream(id oid.ID, f *os.File) (*objectSDK.Object, io.ReadSeekCloser, error) {
7272
buf := make([]byte, objectSDK.MaxHeaderLen, 2*objectSDK.MaxHeaderLen)
7373
n, err := io.ReadFull(f, buf)
7474
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
@@ -124,7 +124,7 @@ func (t *FSTree) extractHeaderAndStream(id oid.ID, f *os.File) (*objectSDK.Objec
124124

125125
// readHeaderAndPayload reads an object header from the file and returns reader for payload.
126126
// This function takes ownership of the io.ReadCloser and will close it if it does not return it.
127-
func (t *FSTree) readHeaderAndPayload(f io.ReadCloser, initial []byte) (*objectSDK.Object, io.ReadCloser, error) {
127+
func (t *FSTree) readHeaderAndPayload(f io.ReadCloser, initial []byte) (*objectSDK.Object, io.ReadSeekCloser, error) {
128128
var err error
129129
if len(initial) < objectSDK.MaxHeaderLen {
130130
_ = f.Close()
@@ -137,7 +137,10 @@ func (t *FSTree) readHeaderAndPayload(f io.ReadCloser, initial []byte) (*objectS
137137
if err != nil {
138138
return nil, nil, fmt.Errorf("unmarshal object: %w", err)
139139
}
140-
return obj.CutPayload(), io.NopCloser(bytes.NewReader(obj.Payload())), nil
140+
return obj.CutPayload(), &payloadReader{
141+
Reader: bytes.NewReader(obj.Payload()),
142+
close: func() error { return nil },
143+
}, nil
141144
}
142145

143146
return t.readUntilPayload(f, initial)
@@ -146,7 +149,7 @@ func (t *FSTree) readHeaderAndPayload(f io.ReadCloser, initial []byte) (*objectS
146149
// readUntilPayload reads an object from the file until the payload field is reached
147150
// and returns the object along with a reader for the remaining data.
148151
// This function takes ownership of the io.ReadCloser and will close it if it does not return it.
149-
func (t *FSTree) readUntilPayload(f io.ReadCloser, initial []byte) (*objectSDK.Object, io.ReadCloser, error) {
152+
func (t *FSTree) readUntilPayload(f io.ReadCloser, initial []byte) (*objectSDK.Object, io.ReadSeekCloser, error) {
150153
reader := f
151154

152155
if t.IsCompressed(initial) {
@@ -249,3 +252,17 @@ type payloadReader struct {
249252
func (p *payloadReader) Close() error {
250253
return p.close()
251254
}
255+
256+
// Seek implements io.Seeker interface for payloadReader.
257+
// If the Reader does not support seeking, it will discard the data until the offset
258+
// is reached, but only if whence is io.SeekStart. If whence is not io.SeekStart,
259+
// it returns an error indicating that seeking is not supported.
260+
func (p *payloadReader) Seek(offset int64, whence int) (int64, error) {
261+
if seeker, ok := p.Reader.(io.Seeker); ok {
262+
return seeker.Seek(offset, whence)
263+
}
264+
if whence == io.SeekStart {
265+
return io.CopyN(io.Discard, p.Reader, offset)
266+
}
267+
return 0, errors.New("payload reader does not support seeking")
268+
}

‎pkg/local_object_storage/blobstor/fstree/head_bench_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ func generateTestObject(payloadSize int) *objectSDK.Object {
105105
} else {
106106
obj.SetPayload(nil)
107107
}
108+
obj.SetPayloadSize(uint64(payloadSize))
108109

109110
return &obj
110111
}

‎pkg/local_object_storage/blobstor/internal/storagetest/common.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ func NewObject(sz uint64) *objectSDK.Object {
100100
if ln := uint64(len(data)); ln > sz {
101101
raw.SetPayload(raw.Payload()[:sz-(ln-sz)])
102102
}
103+
raw.SetPayloadSize(uint64(len(raw.Payload())))
103104

104105
return raw
105106
}

‎pkg/local_object_storage/engine/error_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ func TestBlobstorFailback(t *testing.T) {
137137
for _, size := range []int{15, errSmallSize + 1} {
138138
obj := generateObjectWithCID(cidtest.ID())
139139
obj.SetPayload(make([]byte, size))
140+
obj.SetPayloadSize(uint64(size))
140141

141142
e.mtx.RLock()
142143
err = e.shards[id[0].String()].Shard.Put(obj, nil)

0 commit comments

Comments
 (0)