Skip to content

Commit cd3e5d3

Browse files
cthulhu-ridercthulhu-rider
committed
sn/object: Initial support for erasure coding policies by PUT server
This extends PUT server logic to comply with EC policies - independently or together with REP ones. This logic will remain inactive until it becomes possible to create containers with EC policies. This works for small objects only: encoding and distribution will be the same for big objects, but SNs will deny them without proper adaptation. Closes #3419. Refs #3420. Signed-off-by: Leonard Lyubich <[email protected]>
1 parent 22e3dfc commit cd3e5d3

File tree

23 files changed

+1377
-122
lines changed

23 files changed

+1377
-122
lines changed

‎cmd/neofs-node/object.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010

1111
"github.com/google/uuid"
1212
lru "github.com/hashicorp/golang-lru/v2"
13+
iec "github.com/nspcc-dev/neofs-node/internal/ec"
1314
coreclient "github.com/nspcc-dev/neofs-node/pkg/core/client"
1415
containercore "github.com/nspcc-dev/neofs-node/pkg/core/container"
1516
"github.com/nspcc-dev/neofs-node/pkg/core/netmap"
@@ -804,6 +805,7 @@ type containerNodesSorter struct {
804805

805806
func (x *containerNodesSorter) Unsorted() [][]netmapsdk.NodeInfo { return x.policy.nodeSets }
806807
func (x *containerNodesSorter) PrimaryCounts() []uint { return x.policy.repCounts }
808+
func (x *containerNodesSorter) ECRules() []iec.Rule { return nil }
807809
func (x *containerNodesSorter) SortForObject(obj oid.ID) ([][]netmapsdk.NodeInfo, error) {
808810
cacheKey := objectNodesCacheKey{epoch: x.curEpoch}
809811
cacheKey.addr.SetContainer(x.cnrID)

‎go.mod‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,12 @@ require (
1313
github.com/google/uuid v1.6.0
1414
github.com/hashicorp/golang-lru/v2 v2.0.7
1515
github.com/klauspost/compress v1.17.11
16+
github.com/klauspost/reedsolomon v1.12.4
1617
github.com/mitchellh/go-homedir v1.1.0
1718
github.com/mitchellh/mapstructure v1.5.0
1819
github.com/mr-tron/base58 v1.2.0
1920
github.com/multiformats/go-multiaddr v0.12.2
21+
github.com/mxschmitt/golang-combinations v1.2.0
2022
github.com/nspcc-dev/hrw/v2 v2.0.3
2123
github.com/nspcc-dev/locode-db v0.6.0
2224
github.com/nspcc-dev/neo-go v0.111.0
@@ -62,7 +64,7 @@ require (
6264
github.com/holiman/uint256 v1.3.2 // indirect
6365
github.com/inconshreveable/mousetrap v1.1.0 // indirect
6466
github.com/ipfs/go-cid v0.4.1 // indirect
65-
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
67+
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
6668
github.com/magiconair/properties v1.8.7 // indirect
6769
github.com/mattn/go-runewidth v0.0.15 // indirect
6870
github.com/minio/sha256-simd v1.0.1 // indirect

‎go.sum‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -121,8 +121,10 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU
121121
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
122122
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
123123
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
124-
github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
125-
github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
124+
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
125+
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
126+
github.com/klauspost/reedsolomon v1.12.4 h1:5aDr3ZGoJbgu/8+j45KtUJxzYm8k08JGtB9Wx1VQ4OA=
127+
github.com/klauspost/reedsolomon v1.12.4/go.mod h1:d3CzOMOt0JXGIFZm1StgkyF14EYr3xneR2rNWo7NcMU=
126128
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
127129
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
128130
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -185,6 +187,8 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n
185187
github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
186188
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
187189
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
190+
github.com/mxschmitt/golang-combinations v1.2.0 h1:V5E7MncIK8Yr1SL/SpdqMuSquFsfoIs5auI7Y3n8z14=
191+
github.com/mxschmitt/golang-combinations v1.2.0/go.mod h1:RCm5eR03B+JrBOMRDLsKZWShluXdrHu+qwhPEJ0miBM=
188192
github.com/nspcc-dev/bbolt v0.0.0-20250612101626-5df2544a4a22 h1:M5Nmg1iCnbZngzIBDIlMr9vW+okFfcSMBvBlXG8r+14=
189193
github.com/nspcc-dev/bbolt v0.0.0-20250612101626-5df2544a4a22/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk=
190194
github.com/nspcc-dev/dbft v0.4.0 h1:4/atD4GrrMEtrYBDiZPrPzdKZ6ws7PR/cg0M4DEdVeI=
@@ -358,7 +362,6 @@ golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7w
358362
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
359363
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
360364
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
361-
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
362365
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
363366
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
364367
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

‎internal/ec/ec.go‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package ec
2+
3+
import (
4+
"fmt"
5+
"slices"
6+
"strconv"
7+
8+
"github.com/klauspost/reedsolomon"
9+
)
10+
11+
// Erasure coding attributes.
12+
const (
13+
AttributePrefix = "__NEOFS__EC_"
14+
AttributeRuleIdx = AttributePrefix + "RULE_IDX"
15+
AttributePartIdx = AttributePrefix + "PART_IDX"
16+
)
17+
18+
// Rule represents erasure coding rule for object payload's encoding and placement.
19+
type Rule struct {
20+
DataPartNum uint8
21+
ParityPartNum uint8
22+
}
23+
24+
// String implements [fmt.Stringer].
25+
func (x Rule) String() string {
26+
return strconv.FormatUint(uint64(x.DataPartNum), 10) + "/" + strconv.FormatUint(uint64(x.ParityPartNum), 10)
27+
}
28+
29+
// Encode encodes given data according to specified EC rule and returns coded
30+
// parts. First [Rule.DataPartNum] elements are data parts, other
31+
// [Rule.ParityPartNum] ones are parity blocks.
32+
//
33+
// All parts are the same length. If data len is not divisible by
34+
// [Rule.DataPartNum], last data part is aligned with zeros.
35+
//
36+
// If data is empty, all parts are nil.
37+
func Encode(rule Rule, data []byte) ([][]byte, error) {
38+
if len(data) == 0 {
39+
return make([][]byte, rule.DataPartNum+rule.ParityPartNum), nil
40+
}
41+
42+
// TODO: Explore reedsolomon.Option for performance improvement. https://github.com/nspcc-dev/neofs-node/issues/3501
43+
enc, err := reedsolomon.New(int(rule.DataPartNum), int(rule.ParityPartNum))
44+
if err != nil { // should never happen with correct rule
45+
return nil, fmt.Errorf("init Reed-Solomon encoder: %w", err)
46+
}
47+
48+
parts, err := enc.Split(data)
49+
if err != nil {
50+
return nil, fmt.Errorf("split data: %w", err)
51+
}
52+
53+
if err := enc.Encode(parts); err != nil {
54+
return nil, fmt.Errorf("calculate Reed-Solomon parity: %w", err)
55+
}
56+
57+
return parts, nil
58+
}
59+
60+
// Decode decodes source data of known len from EC parts obtained by applying
61+
// specified rule.
62+
func Decode(rule Rule, dataLen uint64, parts [][]byte) ([]byte, error) {
63+
// TODO: Explore reedsolomon.Option for performance improvement. https://github.com/nspcc-dev/neofs-node/issues/3501
64+
dec, err := reedsolomon.New(int(rule.DataPartNum), int(rule.ParityPartNum))
65+
if err != nil { // should never happen with correct rule
66+
return nil, fmt.Errorf("init Reed-Solomon decoder: %w", err)
67+
}
68+
69+
required := make([]bool, rule.DataPartNum+rule.ParityPartNum)
70+
for i := range rule.DataPartNum {
71+
required[i] = true
72+
}
73+
74+
if err := dec.ReconstructSome(parts, required); err != nil {
75+
return nil, fmt.Errorf("restore Reed-Solomon: %w", err)
76+
}
77+
78+
// TODO: last part may be shorter, do not overallocate buffer.
79+
return slices.Concat(parts[:rule.DataPartNum]...)[:dataLen], nil
80+
}

‎internal/ec/ec_test.go‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
package ec_test
2+
3+
import (
4+
"testing"
5+
6+
"github.com/klauspost/reedsolomon"
7+
iec "github.com/nspcc-dev/neofs-node/internal/ec"
8+
islices "github.com/nspcc-dev/neofs-node/internal/slices"
9+
"github.com/nspcc-dev/neofs-node/internal/testutil"
10+
"github.com/stretchr/testify/require"
11+
)
12+
13+
func TestRule_String(t *testing.T) {
14+
r := iec.Rule{
15+
DataPartNum: 12,
16+
ParityPartNum: 23,
17+
}
18+
require.Equal(t, "12/23", r.String())
19+
}
20+
21+
func testEncode(t *testing.T, rule iec.Rule, data []byte) {
22+
ln := uint64(len(data))
23+
24+
parts, err := iec.Encode(rule, data)
25+
require.NoError(t, err)
26+
27+
res, err := iec.Decode(rule, ln, parts)
28+
require.NoError(t, err)
29+
require.Equal(t, data, res)
30+
31+
for lostCount := 1; lostCount <= int(rule.ParityPartNum); lostCount++ {
32+
for _, lostIdxs := range islices.IndexCombos(len(parts), lostCount) {
33+
res, err := iec.Decode(rule, ln, islices.NilTwoDimSliceElements(parts, lostIdxs))
34+
require.NoError(t, err)
35+
require.Equal(t, data, res)
36+
}
37+
}
38+
39+
for _, lostIdxs := range islices.IndexCombos(len(parts), int(rule.ParityPartNum)+1) {
40+
_, err := iec.Decode(rule, ln, islices.NilTwoDimSliceElements(parts, lostIdxs))
41+
require.ErrorContains(t, err, "restore Reed-Solomon")
42+
require.ErrorIs(t, err, reedsolomon.ErrTooFewShards)
43+
}
44+
}
45+
46+
func TestEncode(t *testing.T) {
47+
rules := []iec.Rule{
48+
{DataPartNum: 3, ParityPartNum: 1},
49+
{DataPartNum: 12, ParityPartNum: 4},
50+
}
51+
52+
data := testutil.RandByteSlice(4 << 10)
53+
54+
t.Run("empty", func(t *testing.T) {
55+
for _, rule := range rules {
56+
t.Run(rule.String(), func(t *testing.T) {
57+
test := func(t *testing.T, data []byte) {
58+
res, err := iec.Encode(rule, []byte{})
59+
require.NoError(t, err)
60+
61+
total := int(rule.DataPartNum + rule.ParityPartNum)
62+
require.Len(t, res, total)
63+
require.EqualValues(t, total, islices.CountNilsInTwoDimSlice(res))
64+
}
65+
test(t, nil)
66+
test(t, []byte{})
67+
})
68+
}
69+
})
70+
71+
for _, rule := range rules {
72+
t.Run(rule.String(), func(t *testing.T) {
73+
testEncode(t, rule, data)
74+
})
75+
}
76+
}

0 commit comments

Comments
 (0)