Skip to content

Commit ed36a09

Browse files
rvaggrvagg
committed
fix: use fixed car BlockReader#SkipNext SourceOffset
Ref: ipld/go-car#491
1 parent 51bc49f commit ed36a09

File tree

3 files changed

+21
-56
lines changed

3 files changed

+21
-56
lines changed

Diff for: ‎go.mod

+5-5
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,8 @@ require (
6767
github.com/ipfs/go-metrics-interface v0.0.1
6868
github.com/ipfs/go-unixfs v0.4.5 // indirect
6969
github.com/ipld/go-car v0.6.1
70-
github.com/ipld/go-car/v2 v2.10.1
71-
github.com/ipld/go-ipld-prime v0.20.0
70+
github.com/ipld/go-car/v2 v2.11.1-0.20230901052914-ceb1d88d7b17
71+
github.com/ipld/go-ipld-prime v0.21.0
7272
github.com/ipld/go-ipld-selector-text-lite v0.0.1
7373
github.com/ipni/index-provider v0.13.4
7474
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
@@ -96,7 +96,7 @@ require (
9696
github.com/stretchr/testify v1.8.4
9797
github.com/urfave/cli/v2 v2.25.5
9898
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc
99-
github.com/whyrusleeping/cbor-gen v0.0.0-20230418232409-daab9ece03a0
99+
github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25
100100
go.opencensus.io v0.24.0
101101
go.opentelemetry.io/otel v1.16.0
102102
go.opentelemetry.io/otel/exporters/jaeger v1.14.0 // indirect
@@ -220,10 +220,10 @@ require (
220220
github.com/ipfs/go-ipfs-posinfo v0.0.1 // indirect
221221
github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
222222
github.com/ipfs/go-ipfs-util v0.0.3 // indirect
223-
github.com/ipfs/go-ipld-cbor v0.0.6
223+
github.com/ipfs/go-ipld-cbor v0.1.0
224224
github.com/ipfs/go-log v1.0.5 // indirect
225225
github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
226-
github.com/ipfs/go-unixfsnode v1.7.1
226+
github.com/ipfs/go-unixfsnode v1.7.4
227227
github.com/ipfs/go-verifcid v0.0.2 // indirect
228228
github.com/ipld/go-codec-dagpb v1.6.0
229229
github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect

Diff for: ‎go.sum

+12-11
Original file line numberDiff line numberDiff line change
@@ -442,7 +442,7 @@ github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM
442442
github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
443443
github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
444444
github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
445-
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
445+
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
446446
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
447447
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
448448
github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
@@ -861,8 +861,9 @@ github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA
861861
github.com/ipfs/go-ipld-cbor v0.0.4/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4=
862862
github.com/ipfs/go-ipld-cbor v0.0.5/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4=
863863
github.com/ipfs/go-ipld-cbor v0.0.6-0.20211211231443-5d9b9e1f6fa8/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA=
864-
github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eBQ0=
865864
github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA=
865+
github.com/ipfs/go-ipld-cbor v0.1.0 h1:dx0nS0kILVivGhfWuB6dUpMa/LAwElHPw1yOGYopoYs=
866+
github.com/ipfs/go-ipld-cbor v0.1.0/go.mod h1:U2aYlmVrJr2wsUBU67K4KgepApSZddGRDWBYR0H4sCk=
866867
github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms=
867868
github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k=
868869
github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs=
@@ -915,8 +916,8 @@ github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/
915916
github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU=
916917
github.com/ipfs/go-unixfs v0.4.5/go.mod h1:BIznJNvt/gEx/ooRMI4Us9K8+qeGO7vx1ohnbk8gjFg=
917918
github.com/ipfs/go-unixfsnode v1.4.0/go.mod h1:qc7YFFZ8tABc58p62HnIYbUMwj9chhUuFWmxSokfePo=
918-
github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s=
919-
github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
919+
github.com/ipfs/go-unixfsnode v1.7.4 h1:iLvKyAVKUYOIAW2t4kDYqsT7VLGj31eXJE2aeqGfbwA=
920+
github.com/ipfs/go-unixfsnode v1.7.4/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
920921
github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
921922
github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
922923
github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU=
@@ -927,8 +928,8 @@ github.com/ipld/go-car v0.6.1 h1:blWbEHf1j62JMWFIqWE//YR0m7k5ZMw0AuUOU5hjrH8=
927928
github.com/ipld/go-car v0.6.1/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8=
928929
github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI=
929930
github.com/ipld/go-car/v2 v2.4.1/go.mod h1:zjpRf0Jew9gHqSvjsKVyoq9OY9SWoEKdYCQUKVaaPT0=
930-
github.com/ipld/go-car/v2 v2.10.1 h1:MRDqkONNW9WRhB79u+Z3U5b+NoN7lYA5B8n8qI3+BoI=
931-
github.com/ipld/go-car/v2 v2.10.1/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg=
931+
github.com/ipld/go-car/v2 v2.11.1-0.20230901052914-ceb1d88d7b17 h1:KNTUCmJ6Gz5a2Ixzg2PAgKdFabQG2dT9VNAtJ4k7ebU=
932+
github.com/ipld/go-car/v2 v2.11.1-0.20230901052914-ceb1d88d7b17/go.mod h1:ff9dD1uh3glsvnjKyEdtvctNEMmRyReBLiP5LHoHuBE=
932933
github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s=
933934
github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA=
934935
github.com/ipld/go-codec-dagpb v1.3.1/go.mod h1:ErNNglIi5KMur/MfFE/svtgQthzVvf+43MrzLbpcIZY=
@@ -944,8 +945,8 @@ github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHt
944945
github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM=
945946
github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA=
946947
github.com/ipld/go-ipld-prime v0.19.0/go.mod h1:Q9j3BaVXwaA3o5JUDNvptDDr/x8+F7FG6XJ8WI3ILg4=
947-
github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g=
948-
github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M=
948+
github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E=
949+
github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ=
949950
github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU=
950951
github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY=
951952
github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
@@ -1871,7 +1872,7 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp
18711872
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
18721873
github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0=
18731874
github.com/warpfork/go-testmark v0.10.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0=
1874-
github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U=
1875+
github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s=
18751876
github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
18761877
github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
18771878
github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
@@ -1899,8 +1900,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163/go.mod h1:f
18991900
github.com/whyrusleeping/cbor-gen v0.0.0-20210118024343-169e9d70c0c2/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
19001901
github.com/whyrusleeping/cbor-gen v0.0.0-20210303213153-67a261a1d291/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
19011902
github.com/whyrusleeping/cbor-gen v0.0.0-20220323183124-98fa8256a799/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
1902-
github.com/whyrusleeping/cbor-gen v0.0.0-20230418232409-daab9ece03a0 h1:XYEgH2nJgsrcrj32p+SAbx6T3s/6QknOXezXtz7kzbg=
1903-
github.com/whyrusleeping/cbor-gen v0.0.0-20230418232409-daab9ece03a0/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
1903+
github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 h1:yVYDLoN2gmB3OdBXFW8e1UwgVbmCvNlnAKhvHPaNARI=
1904+
github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
19041905
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
19051906
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
19061907
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k=

Diff for: ‎piecedirectory/piecedirectory.go

+4-40
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ import (
2525
format "github.com/ipfs/go-ipld-format"
2626
logging "github.com/ipfs/go-log/v2"
2727
"github.com/ipld/go-car"
28-
"github.com/ipld/go-car/util"
2928
carv2 "github.com/ipld/go-car/v2"
3029
"github.com/ipld/go-car/v2/blockstore"
3130
carindex "github.com/ipld/go-car/v2/index"
@@ -250,19 +249,11 @@ func (ps *PieceDirectory) addIndexForPiece(ctx context.Context, pieceCid cid.Cid
250249
}
251250

252251
blockMetadata, err := blockReader.SkipNext()
253-
if blockReader.Version == 2 {
254-
log.Warnf("add index: carv2 deal encountered", "offset adjustment for index", carv2.HeaderSize)
255-
}
256252
for err == nil {
257-
offset := blockMetadata.Offset
258-
if blockReader.Version == 2 {
259-
offset += carv2.HeaderSize
260-
}
261-
262253
recs = append(recs, model.Record{
263254
Cid: blockMetadata.Cid,
264255
OffsetSize: model.OffsetSize{
265-
Offset: offset,
256+
Offset: blockMetadata.SourceOffset,
266257
Size: blockMetadata.Size,
267258
},
268259
})
@@ -537,36 +528,9 @@ func (ps *PieceDirectory) BlockstoreGet(ctx context.Context, c cid.Cid) ([]byte,
537528

538529
// Seek to the block offset
539530
readerAt := readerutil.NewReadSeekerFromReaderAt(reader, int64(offsetSize.Offset))
540-
541-
// Read the block data
542-
_, data, err := util.ReadNode(bufio.NewReader(readerAt))
543-
if err != nil {
544-
_, _ = reader.Seek(0, io.SeekStart)
545-
cv, _ := carv2.ReadVersion(reader)
546-
_, _ = reader.Seek(0, io.SeekStart)
547-
548-
if cv == 2 {
549-
// what should the offset be.
550-
br, _ := carv2.NewBlockReader(reader)
551-
n, err := br.SkipNext()
552-
for err == nil {
553-
if n.Cid.Equals(c) {
554-
log.Warnf("blockstore get: was told by index that cid was at %d, but found it at %d", offsetSize.Offset, n.Offset)
555-
_, _ = reader.Seek(int64(n.Offset), io.SeekStart)
556-
_, data, err = util.ReadNode(bufio.NewReader(reader))
557-
if err != nil {
558-
_, _ = reader.Seek(int64(n.Offset)-40, io.SeekStart)
559-
ob := make([]byte, 1024)
560-
io.ReadFull(reader, ob)
561-
562-
return nil, fmt.Errorf("recovery error reading data for block %s from reader for piece %s (at: %d): %w - buffer was: %x", c, pieceCid, n.Offset, err, ob)
563-
}
564-
return data, nil
565-
}
566-
n, err = br.SkipNext()
567-
}
568-
}
569-
return nil, fmt.Errorf("reading data for block %s from reader for piece %s (cv: %d): %w", c, pieceCid, cv, err)
531+
data := make([]byte, offsetSize.Size)
532+
if _, err = io.ReadFull(readerAt, data); err != nil {
533+
return nil, fmt.Errorf("reading data for block %s from reader for piece %s: %w", c, pieceCid, err)
570534
}
571535
return data, nil
572536
}()

0 commit comments

Comments
 (0)