diff --git a/client/object_get.go b/client/object_get.go index 28c3befb..927ddfd5 100644 --- a/client/object_get.go +++ b/client/object_get.go @@ -1,9 +1,12 @@ package client import ( + "bytes" "context" + "crypto/sha256" "errors" "fmt" + "hash" "io" "time" @@ -11,6 +14,7 @@ import ( apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" neofscrypto "github.com/nspcc-dev/neofs-sdk-go/crypto" + neofsproto "github.com/nspcc-dev/neofs-sdk-go/internal/proto" "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" protoobject "github.com/nspcc-dev/neofs-sdk-go/proto/object" @@ -54,6 +58,13 @@ func (x *prmObjectRead) WithBearerToken(t bearer.Token) { // PrmObjectGet groups optional parameters of ObjectGetInit operation. type PrmObjectGet struct { prmObjectRead + skipChecksumVerification bool +} + +// SkipChecksumVerification allows to skip verification of received header +// against object ID and payload against its in-header checksum. +func (x *PrmObjectGet) SkipChecksumVerification() { + x.skipChecksumVerification = true } // used part of [protoobject.ObjectService_GetClient] simplifying test @@ -84,6 +95,13 @@ type PayloadReader struct { statisticCallback shortStatisticCallback startTime time.Time // if statisticCallback is set only + + requestedOID oid.ID + + verifyChecksums bool + + payloadHashCheck []byte + payloadHashGot hash.Hash } // readHeader reads header of the object. Result means success. @@ -99,11 +117,6 @@ func (x *PayloadReader) readHeader(dst *object.Object) bool { return false } - if x.err = neofscrypto.VerifyResponseWithBuffer[*protoobject.GetResponse_Body](resp, nil); x.err != nil { - x.err = fmt.Errorf("%w: %w", errResponseSignatures, x.err) - return false - } - if x.err = apistatus.ToError(resp.GetMetaHeader().GetStatus()); x.err != nil { return false } @@ -154,7 +167,26 @@ func (x *PayloadReader) readHeader(dst *object.Object) bool { Signature: partInit.Signature, Header: partInit.Header, }) - return x.err == nil + if x.err != nil { + return false + } + + if x.verifyChecksums { + hb := neofsproto.MarshalMessage(partInit.Header) + if oid.NewFromObjectHeaderBinary(hb) != x.requestedOID { + x.err = errors.New("received header mismatches ID") + return false + } + + if partInit.Header.PayloadHash == nil { + x.err = errors.New("missing payload hash in header") + return false + } + x.payloadHashGot = sha256.New() + x.payloadHashCheck = partInit.Header.PayloadHash.Sum + } + + return true } func (x *PayloadReader) readChunk(buf []byte) (int, bool) { @@ -183,11 +215,6 @@ func (x *PayloadReader) readChunk(buf []byte) (int, bool) { return read, false } - if x.err = neofscrypto.VerifyResponseWithBuffer[*protoobject.GetResponse_Body](resp, nil); x.err != nil { - x.err = fmt.Errorf("%w: %w", errResponseSignatures, x.err) - return read, false - } - if x.err = apistatus.ToError(resp.GetMetaHeader().GetStatus()); x.err != nil { return read, false } @@ -210,6 +237,9 @@ func (x *PayloadReader) readChunk(buf []byte) (int, bool) { continue } + if x.verifyChecksums { + x.payloadHashGot.Write(chunk) // never returns an error according to docs + } lastRead = copy(buf[read:], chunk) read += lastRead @@ -233,6 +263,9 @@ func (x *PayloadReader) close(ignoreEOF bool) error { if x.remainingPayloadLen > 0 { return io.ErrUnexpectedEOF } + if x.verifyChecksums && !bytes.Equal(x.payloadHashGot.Sum(nil), x.payloadHashCheck) { + return errors.New("received payload mismatches checksum from header") + } } return x.err } @@ -352,6 +385,8 @@ func (c *Client) ObjectGetInit(ctx context.Context, containerID cid.ID, objectID } var r PayloadReader + r.requestedOID = objectID + r.verifyChecksums = !prm.skipChecksumVerification r.cancelCtxStream = cancel r.stream = stream r.singleMsgTimeout = c.streamTimeout @@ -373,6 +408,13 @@ func (c *Client) ObjectGetInit(ctx context.Context, containerID cid.ID, objectID // PrmObjectHead groups optional parameters of ObjectHead operation. type PrmObjectHead struct { prmObjectRead + skipChecksumVerification bool +} + +// SkipChecksumVerification allows to skip verification of received header +// against object ID. +func (x *PrmObjectHead) SkipChecksumVerification() { + x.skipChecksumVerification = true } // ObjectHead reads object header through a remote server using NeoFS API protocol. @@ -450,11 +492,6 @@ func (c *Client) ObjectHead(ctx context.Context, containerID cid.ID, objectID oi return nil, err } - if err = neofscrypto.VerifyResponseWithBuffer[*protoobject.HeadResponse_Body](resp, *buf); err != nil { - err = fmt.Errorf("%w: %w", errResponseSignatures, err) - return nil, err - } - if err = apistatus.ToError(resp.GetMetaHeader().GetStatus()); err != nil { return nil, err } @@ -496,6 +533,14 @@ func (c *Client) ObjectHead(ctx context.Context, containerID cid.ID, objectID oi }); err != nil { return nil, fmt.Errorf("invalid header response: %w", err) } + + if !prm.skipChecksumVerification { + hb := neofsproto.MarshalMessage(v.Header.Header) + if oid.NewFromObjectHeaderBinary(hb) != objectID { + return nil, errors.New("received header mismatches ID") + } + } + return &obj, nil } } diff --git a/client/object_get_test.go b/client/object_get_test.go index 7a7cc4a5..32085e9e 100644 --- a/client/object_get_test.go +++ b/client/object_get_test.go @@ -14,8 +14,10 @@ import ( bearertest "github.com/nspcc-dev/neofs-sdk-go/bearer/test" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test" + neofsproto "github.com/nspcc-dev/neofs-sdk-go/internal/proto" "github.com/nspcc-dev/neofs-sdk-go/internal/testutil" "github.com/nspcc-dev/neofs-sdk-go/object" + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test" protoobject "github.com/nspcc-dev/neofs-sdk-go/proto/object" protorefs "github.com/nspcc-dev/neofs-sdk-go/proto/refs" @@ -25,9 +27,11 @@ import ( "github.com/nspcc-dev/neofs-sdk-go/stat" usertest "github.com/nspcc-dev/neofs-sdk-go/user/test" "github.com/stretchr/testify/require" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" ) func setPayloadLengthInHeadingGetResponse(b *protoobject.GetResponse_Body, ln uint64) *protoobject.GetResponse_Body { @@ -71,6 +75,16 @@ func checkSuccessfulGetObjectTransport(t testing.TB, hb *protoobject.GetResponse })) } +func getObjectIDForHeaderResponseBody(resp *protoobject.HeadResponse_Body) oid.ID { + h := resp.GetHeader().GetHeader() + return oid.NewFromObjectHeaderBinary(neofsproto.MarshalMessage(h)) +} + +func getObjectIDForGetResponseBody(resp *protoobject.GetResponse_Body) oid.ID { + h := resp.GetInit().GetHeader() + return oid.NewFromObjectHeaderBinary(neofsproto.MarshalMessage(h)) +} + type testCommonReadObjectRequestServerSettings struct { testObjectSessionServerSettings testBearerTokenServerSettings @@ -422,7 +436,7 @@ func TestClient_ObjectHead(t *testing.T) { ctx := context.Background() var anyValidOpts PrmObjectHead anyCID := cidtest.ID() - anyOID := oidtest.ID() + anyOID := getObjectIDForHeaderResponseBody(validMinObjectHeadResponseBody) anyValidSigner := usertest.User() t.Run("messages", func(t *testing.T) { @@ -541,7 +555,8 @@ func TestClient_ObjectHead(t *testing.T) { c := newTestObjectClient(t, srv) srv.respondWithBody(tc.body) - hdr, err := c.ObjectHead(ctx, anyCID, anyOID, anyValidSigner, anyValidOpts) + id := getObjectIDForHeaderResponseBody(tc.body) + hdr, err := c.ObjectHead(ctx, anyCID, id, anyValidSigner, anyValidOpts) if err != nil { tc.assert(t, tc.body, object.Object{}, err) } else { @@ -556,19 +571,32 @@ func TestClient_ObjectHead(t *testing.T) { return err }) }) + t.Run("unsigned response", func(t *testing.T) { + srv := newTestHeadObjectServer() + c := newTestObjectClient(t, srv) + + srv.respondWithoutSigning() + _, err := c.ObjectHead(ctx, anyCID, anyOID, anyValidSigner, anyValidOpts) + require.NoError(t, err) + }) }) t.Run("invalid", func(t *testing.T) { t.Run("format", func(t *testing.T) { - testIncorrectUnaryRPCResponseFormat(t, "object.ObjectService", "Head", func(c *Client) error { - _, err := c.ObjectHead(ctx, anyCID, anyOID, anyValidSigner, anyValidOpts) - return err - }) - }) - t.Run("verification header", func(t *testing.T) { - testInvalidResponseVerificationHeader(t, newTestHeadObjectServer, newTestObjectClient, func(c *Client) error { - _, err := c.ObjectHead(ctx, anyCID, anyOID, anyValidSigner, anyValidOpts) - return err - }) + svc := testService{ + desc: &grpc.ServiceDesc{ServiceName: "neo.fs.v2.object.ObjectService", Methods: []grpc.MethodDesc{ + { + MethodName: "Head", + Handler: func(srv any, ctx context.Context, dec func(any) error, interceptor grpc.UnaryServerInterceptor) (any, error) { + return timestamppb.Now(), nil // any completely different message + }, + }, + }}, + impl: nil, // disables interface assert + } + c := newClient(t, svc) + + _, err := c.ObjectHead(ctx, anyCID, anyOID, anyValidSigner, anyValidOpts) + require.EqualError(t, err, "unexpected header type ") }) t.Run("payloads", func(t *testing.T) { type testcase = invalidResponseBodyTestcase[protoobject.HeadResponse_Body] @@ -721,11 +749,30 @@ func TestClient_ObjectHead(t *testing.T) { }, ) }) + t.Run("checksum", func(t *testing.T) { + srv := newTestHeadObjectServer() + c := newTestObjectClient(t, srv) + + _, err := c.ObjectHead(ctx, anyCID, anyOID, anyValidSigner, anyValidOpts) + require.NoError(t, err) + + otherID := oidtest.OtherID(anyOID) + _, err = c.ObjectHead(ctx, anyCID, otherID, anyValidSigner, anyValidOpts) + require.EqualError(t, err, "received header mismatches ID") + + t.Run("skip", func(t *testing.T) { + opts := anyValidOpts + opts.SkipChecksumVerification() + _, err = c.ObjectHead(ctx, anyCID, otherID, anyValidSigner, opts) + require.NoError(t, err) + }) + }) } func TestClient_ObjectGetInit(t *testing.T) { ctx := context.Background() var anyValidOpts PrmObjectGet + anyValidOpts.SkipChecksumVerification() anyCID := cidtest.ID() anyOID := oidtest.ID() anyValidSigner := usertest.User() @@ -743,7 +790,7 @@ func TestClient_ObjectGetInit(t *testing.T) { srv.checkRequestObjectAddress(anyCID, anyOID) srv.authenticateRequest(anyValidSigner) - _, r, err := c.ObjectGetInit(ctx, anyCID, anyOID, anyValidSigner, PrmObjectGet{}) + _, r, err := c.ObjectGetInit(ctx, anyCID, anyOID, anyValidSigner, anyValidOpts) require.NoError(t, err) _, err = io.Copy(io.Discard, r) require.NoError(t, err) @@ -992,40 +1039,43 @@ func TestClient_ObjectGetInit(t *testing.T) { }) }) }) - }) - t.Run("invalid", func(t *testing.T) { - t.Run("format", func(t *testing.T) { - testIncorrectUnaryRPCResponseFormat(t, "object.ObjectService", "Get", func(c *Client) error { - _, _, err := c.ObjectGetInit(ctx, anyCID, anyOID, anyValidSigner, anyValidOpts) - return err - }) - }) - t.Run("verification header", func(t *testing.T) { - t.Run("heading message", func(t *testing.T) { - srv := newTestGetObjectServer() - srv.respondWithoutSigning(0) - c := newTestObjectClient(t, srv) - _, _, err := c.ObjectGetInit(ctx, anyCID, anyOID, anyValidSigner, anyValidOpts) - require.ErrorContains(t, err, "invalid response signature") - }) - t.Run("payload chunk message", func(t *testing.T) { - srv := newTestGetObjectServer() - c := newTestObjectClient(t, srv) - - const n = 10 - chunks := make([][]byte, n) - for i := range chunks { - chunks[i] = []byte(fmt.Sprintf("chunk#%d", i)) - } + t.Run("unsigned response", func(t *testing.T) { + srv := newTestGetObjectServer() + c := newTestObjectClient(t, srv) + + const n = 10 + chunks := make([][]byte, n) + for i := range chunks { + chunks[i] = []byte(fmt.Sprintf("chunk#%d", i)) + } - srv.respondWithObject(validFullHeadingObjectGetResponseBody.GetInit(), chunks) - srv.respondWithoutSigning(n) // remember that 1st message is heading + srv.respondWithObject(validFullHeadingObjectGetResponseBody.GetInit(), chunks) + for i := range uint(n + 1) { + srv.respondWithoutSigning(i) _, r, err := c.ObjectGetInit(ctx, anyCID, anyOID, anyValidSigner, anyValidOpts) require.NoError(t, err) read, err := io.ReadAll(r) - require.ErrorContains(t, err, "invalid response signature") - require.Equal(t, join(chunks[:n-1]), read) - }) + require.NoError(t, err) + require.Equal(t, join(chunks), read) + } + }) + }) + t.Run("invalid", func(t *testing.T) { + t.Run("format", func(t *testing.T) { + svc := testService{ + desc: &grpc.ServiceDesc{ServiceName: "neo.fs.v2.object.ObjectService", Methods: []grpc.MethodDesc{ + { + MethodName: "Get", + Handler: func(srv any, ctx context.Context, dec func(any) error, interceptor grpc.UnaryServerInterceptor) (any, error) { + return timestamppb.Now(), nil // any completely different message + }, + }, + }}, + impl: nil, // disables interface assert + } + c := newClient(t, svc) + _, _, err := c.ObjectGetInit(ctx, anyCID, anyOID, anyValidSigner, anyValidOpts) + require.EqualError(t, err, "read header: unexpected message instead of heading part: ") }) t.Run("payloads", func(t *testing.T) { t.Run("split info", func(t *testing.T) { @@ -1380,6 +1430,61 @@ func TestClient_ObjectGetInit(t *testing.T) { require.Greater(t, collected[1].dur, sleepDur) }) }) + t.Run("checksum", func(t *testing.T) { + srv := newTestGetObjectServer() + c := newTestObjectClient(t, srv) + + hb := proto.Clone(validFullHeadingObjectGetResponseBody).(*protoobject.GetResponse_Body) + hb = setPayloadLengthInHeadingGetResponse(hb, 13) + hb.GetInit().Header.PayloadHash = &protorefs.Checksum{ + Type: protorefs.ChecksumType_SHA256, + Sum: []byte{49, 95, 91, 219, 118, 208, 120, 196, 59, 138, 192, 6, 78, 74, 1, 100, 97, 43, 31, 206, 119, 200, + 105, 52, 91, 252, 148, 199, 88, 148, 237, 211}, + } + + id := getObjectIDForGetResponseBody(hb) + + payloadChunks := [][]byte{[]byte("Hello, "), []byte("world!")} + fullPayload := join(payloadChunks) + srv.respondWithObject(hb.GetInit(), payloadChunks) + + var opts PrmObjectGet + h, r, err := c.ObjectGetInit(ctx, anyCID, id, anyValidSigner, opts) + require.NoError(t, err) + checkSuccessfulGetObjectTransport(t, hb, fullPayload, h, r, err) + + t.Run("ID mismatch", func(t *testing.T) { + otherID := oidtest.OtherID(id) + _, _, err := c.ObjectGetInit(ctx, anyCID, otherID, anyValidSigner, opts) + require.EqualError(t, err, "read header: received header mismatches ID") + + t.Run("skip", func(t *testing.T) { + opts := opts + opts.SkipChecksumVerification() + h, r, err := c.ObjectGetInit(ctx, anyCID, id, anyValidSigner, opts) + require.NoError(t, err) + checkSuccessfulGetObjectTransport(t, hb, fullPayload, h, r, err) + }) + }) + + t.Run("payload checksum mismatch", func(t *testing.T) { + hb.GetInit().Header.PayloadHash.Sum[0]++ + srv.respondWithObject(hb.GetInit(), payloadChunks) + + _, r, err := c.ObjectGetInit(ctx, anyCID, getObjectIDForGetResponseBody(hb), anyValidSigner, opts) + require.NoError(t, err) + _, err = io.ReadAll(r) + require.EqualError(t, err, "received payload mismatches checksum from header") + + t.Run("skip", func(t *testing.T) { + opts := opts + opts.SkipChecksumVerification() + h, r, err := c.ObjectGetInit(ctx, anyCID, id, anyValidSigner, opts) + require.NoError(t, err) + checkSuccessfulGetObjectTransport(t, hb, fullPayload, h, r, err) + }) + }) + }) } func TestClient_ObjectRangeInit(t *testing.T) { diff --git a/proto/netmap/types.pb.go b/proto/netmap/types.pb.go index 0ef0ef02..b627f399 100644 --- a/proto/netmap/types.pb.go +++ b/proto/netmap/types.pb.go @@ -953,7 +953,7 @@ func (x *NodeInfo_Attribute) GetParents() []string { // Number of EigenTrust algorithm iterations to pass in the Reputation system. // Value: little-endian integer. Default: 0. // - **EpochDuration** \ -// NeoFS epoch duration measured in FS chain blocks. +// NeoFS epoch duration measured in seconds. // Value: little-endian integer. Default: 0. // - **HomomorphicHashingDisabled** \ // Flag of disabling the homomorphic hashing of objects' payload. diff --git a/proto/object/service.pb.go b/proto/object/service.pb.go index 76279941..fac490cc 100644 --- a/proto/object/service.pb.go +++ b/proto/object/service.pb.go @@ -102,6 +102,8 @@ type GetResponse struct { // Carries response verification information. This header is used to // authenticate the nodes of the message route and check the correctness of // transmission. + // DEPRECATED: Verify header and payload checksums instead. Servers MUST + // attach it for requests with `meta_header.version` <= 2.17. VerifyHeader *session.ResponseVerificationHeader `protobuf:"bytes,3,opt,name=verify_header,json=verifyHeader,proto3" json:"verify_header,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -566,6 +568,8 @@ type HeadResponse struct { // Carries response verification information. This header is used to // authenticate the nodes of the message route and check the correctness of // transmission. + // DEPRECATED: Verify header and payload checksums instead. Servers MUST + // attach it for requests with `meta_header.version` <= 2.17. VerifyHeader *session.ResponseVerificationHeader `protobuf:"bytes,3,opt,name=verify_header,json=verifyHeader,proto3" json:"verify_header,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache