Skip to content

p2p: HTTP catchup over p2p network #5898

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
algorandskiy merged 16 commits into from
Jan 29, 2024
Merged

p2p: HTTP catchup over p2p network #5898

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
16d75b8
tests: fixes to node_test
algorandskiy Jan 9, 2024
f39f3d2
add TestNodeHybridTopology test
algorandskiy Jan 12, 2024
211b0a5
Add capabilities lookup for p2pNetwork.GetPeers
algorandskiy Jan 16, 2024
a24d650
add net.GetPeers test
algorandskiy Jan 16, 2024
473fa3e
TestNodeHybridTopology debugging
algorandskiy Jan 17, 2024
f5de472
update go-libp2p to v0.32.2 to get libp2phttp
algorandskiy Jan 18, 2024
8696708
add p2p http for blocks/catchpoint fetching
algorandskiy Jan 18, 2024
b61e4e4
fix connection direction, fix deadlock
algorandskiy Jan 18, 2024
8f49ad0
fix http router
algorandskiy Jan 25, 2024
86a6e7b
fix linter
algorandskiy Jan 26, 2024
8582570
remove RegisterHTTPHandlerWithPrefix
algorandskiy Jan 26, 2024
0ff70a1
abstract dnsaddr for testing
algorandskiy Jan 26, 2024
7f48af0
fix linter
algorandskiy Jan 26, 2024
b125e67
fix capabilities tests caused by libp2p upgrade
algorandskiy Jan 26, 2024
390da1e
self-review fixes
algorandskiy Jan 26, 2024
17a8fe8
minor logging fixes to streams.go
algorandskiy Jan 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions catchup/universalFetcher.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,13 +219,19 @@ type HTTPFetcher struct {
// getBlockBytes gets a block.
// Core piece of FetcherClient interface
func (hf *HTTPFetcher) getBlockBytes(ctx context.Context, r basics.Round) (data []byte, err error) {
parsedURL, err := network.ParseHostOrURL(hf.rootURL)
if err != nil {
return nil, err
var blockURL string

if network.IsMultiaddr(hf.rootURL) {
blockURL = rpcs.FormatBlockQuery(uint64(r), "", hf.net)
} else {
if parsedURL, err0 := network.ParseHostOrURL(hf.rootURL); err0 == nil {
parsedURL.Path = rpcs.FormatBlockQuery(uint64(r), parsedURL.Path, hf.net)
blockURL = parsedURL.String()
} else {
return nil, err0
}
}

parsedURL.Path = rpcs.FormatBlockQuery(uint64(r), parsedURL.Path, hf.net)
blockURL := parsedURL.String()
hf.log.Debugf("block GET %#v peer %#v %T", blockURL, hf.peer, hf.peer)
request, err := http.NewRequest("GET", blockURL, nil)
if err != nil {
Expand Down
9 changes: 7 additions & 2 deletions data/transactions/logic/opcodes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -834,8 +834,13 @@ func OpcodesByVersion(version uint64) []OpSpec {
}
}
result := maps.Values(subv)
slices.SortFunc(result, func(a, b OpSpec) bool {
return a.Opcode < b.Opcode
slices.SortFunc(result, func(a, b OpSpec) int {
if a.Opcode == b.Opcode {
return 0
} else if a.Opcode > b.Opcode {
return 1
}
return -1
})
return result
}
Expand Down
50 changes: 24 additions & 26 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,21 +30,21 @@ require (
github.com/jmoiron/sqlx v1.2.0
github.com/karalabe/usb v0.0.2
github.com/labstack/echo/v4 v4.9.1
github.com/libp2p/go-libp2p v0.29.1
github.com/libp2p/go-libp2p v0.32.2
github.com/libp2p/go-libp2p-kad-dht v0.24.3
github.com/libp2p/go-libp2p-pubsub v0.9.3
github.com/libp2p/go-libp2p-pubsub v0.10.0
github.com/libp2p/go-yamux/v4 v4.0.1
github.com/mattn/go-sqlite3 v1.14.16
github.com/miekg/dns v1.1.55
github.com/multiformats/go-multiaddr v0.10.1
github.com/miekg/dns v1.1.56
github.com/multiformats/go-multiaddr v0.12.0
github.com/multiformats/go-multiaddr-dns v0.3.1
github.com/olivere/elastic v6.2.14+incompatible
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.5.0
github.com/stretchr/testify v1.8.4
golang.org/x/crypto v0.14.0
golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1
golang.org/x/sync v0.3.0
golang.org/x/exp v0.0.0-20231006140011-7918f672742d
golang.org/x/sync v0.4.0
golang.org/x/sys v0.13.0
golang.org/x/text v0.13.0
gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009
Expand Down Expand Up @@ -81,17 +81,16 @@ require (
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect
github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect
github.com/huin/goupnp v1.2.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect
github.com/huin/goupnp v1.3.0 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/invopop/yaml v0.1.0 // indirect
github.com/ipfs/boxo v0.10.0 // indirect
Expand All @@ -104,7 +103,7 @@ require (
github.com/jbenet/goprocess v0.1.4 // indirect
github.com/jmespath/go-jmespath v0.3.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/klauspost/compress v1.17.2 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/koron/go-ssdp v0.0.4 // indirect
github.com/kr/pretty v0.3.1 // indirect
Expand All @@ -119,11 +118,11 @@ require (
github.com/libp2p/go-msgio v0.3.0 // indirect
github.com/libp2p/go-nat v0.2.0 // indirect
github.com/libp2p/go-netroute v0.2.1 // indirect
github.com/libp2p/go-reuseport v0.3.0 // indirect
github.com/libp2p/go-reuseport v0.4.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
Expand All @@ -137,10 +136,10 @@ require (
github.com/multiformats/go-multibase v0.2.0 // indirect
github.com/multiformats/go-multicodec v0.9.0 // indirect
github.com/multiformats/go-multihash v0.2.3 // indirect
github.com/multiformats/go-multistream v0.4.1 // indirect
github.com/multiformats/go-multistream v0.5.0 // indirect
github.com/multiformats/go-varint v0.0.7 // indirect
github.com/onsi/ginkgo/v2 v2.11.0 // indirect
github.com/opencontainers/runtime-spec v1.0.2 // indirect
github.com/onsi/ginkgo/v2 v2.13.0 // indirect
github.com/opencontainers/runtime-spec v1.1.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
Expand All @@ -152,10 +151,9 @@ require (
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/quic-go/qpack v0.4.0 // indirect
github.com/quic-go/qtls-go1-19 v0.3.3 // indirect
github.com/quic-go/qtls-go1-20 v0.2.3 // indirect
github.com/quic-go/quic-go v0.36.3 // indirect
github.com/quic-go/webtransport-go v0.5.3 // indirect
github.com/quic-go/qtls-go1-20 v0.3.4 // indirect
github.com/quic-go/quic-go v0.39.4 // indirect
github.com/quic-go/webtransport-go v0.6.0 // indirect
github.com/raulk/go-watchdog v1.3.0 // indirect
github.com/rogpeppe/go-internal v1.9.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
Expand All @@ -169,16 +167,16 @@ require (
go.opentelemetry.io/otel v1.16.0 // indirect
go.opentelemetry.io/otel/metric v1.16.0 // indirect
go.opentelemetry.io/otel/trace v1.16.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/dig v1.17.0 // indirect
go.uber.org/fx v1.20.0 // indirect
go.uber.org/dig v1.17.1 // indirect
go.uber.org/fx v1.20.1 // indirect
go.uber.org/mock v0.3.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/mod v0.12.0 // indirect
go.uber.org/zap v1.26.0 // indirect
golang.org/x/mod v0.13.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
golang.org/x/tools v0.11.0 // indirect
golang.org/x/tools v0.14.0 // indirect
gonum.org/v1/gonum v0.13.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
Expand Down
Loading