Merge branch 'UlyanaAndrukhiv/6497-refactor-executionNodesForBlockID'…

… of github.com:The-K-R-O-K/flow-go into UlyanaAndrukhiv/6413-backfill-tx-error-messages

cmd/bootstrap/utils/key_generation.go

@@ -13,6 +13,7 @@ import (
 
 	sdk "github.com/onflow/flow-go-sdk"
 	sdkcrypto "github.com/onflow/flow-go-sdk/crypto"
+
 	"github.com/onflow/flow-go/fvm/systemcontracts"
 	"github.com/onflow/flow-go/model/bootstrap"
 	model "github.com/onflow/flow-go/model/bootstrap"

cmd/collection/main.go

@@ -9,6 +9,7 @@ import (
 
 	client "github.com/onflow/flow-go-sdk/access/grpc"
 	sdkcrypto "github.com/onflow/flow-go-sdk/crypto"
+
 	"github.com/onflow/flow-go/admin/commands"
 	collectionCommands "github.com/onflow/flow-go/admin/commands/collection"
 	storageCommands "github.com/onflow/flow-go/admin/commands/storage"

cmd/consensus/main.go

@@ -12,6 +12,7 @@ import (
 
 	client "github.com/onflow/flow-go-sdk/access/grpc"
 	"github.com/onflow/flow-go-sdk/crypto"
+
 	"github.com/onflow/flow-go/cmd"
 	"github.com/onflow/flow-go/cmd/util/cmd/common"
 	"github.com/onflow/flow-go/consensus"

cmd/observer/node_builder/observer_builder.go

@@ -19,10 +19,7 @@ import (
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
 	badgerds "github.com/ipfs/go-ds-badger2"
-	dht "github.com/libp2p/go-libp2p-kad-dht"
-	"github.com/libp2p/go-libp2p/core/host"
 	"github.com/libp2p/go-libp2p/core/peer"
-	"github.com/libp2p/go-libp2p/core/routing"
 	"github.com/onflow/crypto"
 	"github.com/rs/zerolog"
 	"github.com/spf13/pflag"
@@ -93,17 +90,12 @@ import (
 	"github.com/onflow/flow-go/network/converter"
 	"github.com/onflow/flow-go/network/p2p"
 	"github.com/onflow/flow-go/network/p2p/blob"
-	p2pbuilder "github.com/onflow/flow-go/network/p2p/builder"
-	p2pbuilderconfig "github.com/onflow/flow-go/network/p2p/builder/config"
 	"github.com/onflow/flow-go/network/p2p/cache"
 	"github.com/onflow/flow-go/network/p2p/conduit"
-	p2pdht "github.com/onflow/flow-go/network/p2p/dht"
 	"github.com/onflow/flow-go/network/p2p/keyutils"
 	p2plogging "github.com/onflow/flow-go/network/p2p/logging"
-	networkingsubscription "github.com/onflow/flow-go/network/p2p/subscription"
 	"github.com/onflow/flow-go/network/p2p/translator"
 	"github.com/onflow/flow-go/network/p2p/unicast/protocols"
-	"github.com/onflow/flow-go/network/p2p/utils"
 	"github.com/onflow/flow-go/network/slashing"
 	"github.com/onflow/flow-go/network/underlay"
 	"github.com/onflow/flow-go/network/validator"
@@ -138,8 +130,6 @@ import (
 // For a node running as a standalone process, the config fields will be populated from the command line params,
 // while for a node running as a library, the config fields are expected to be initialized by the caller.
 type ObserverServiceConfig struct {
-	bootstrapNodeAddresses               []string
-	bootstrapNodePublicKeys              []string
 	observerNetworkingKeyPath            string
 	bootstrapIdentities                  flow.IdentitySkeletonList // the identity list of bootstrap peers the node uses to discover other nodes
 	apiRatelimits                        map[string]int
@@ -223,8 +213,6 @@ func DefaultObserverServiceConfig() *ObserverServiceConfig {
 		rpcMetricsEnabled:                    false,
 		apiRatelimits:                        nil,
 		apiBurstlimits:                       nil,
-		bootstrapNodeAddresses:               []string{},
-		bootstrapNodePublicKeys:              []string{},
 		observerNetworkingKeyPath:            cmd.NotSet,
 		apiTimeout:                           3 * time.Second,
 		upstreamNodeAddresses:                []string{},
@@ -334,7 +322,7 @@ func (builder *ObserverServiceBuilder) deriveBootstrapPeerIdentities() error {
 		return nil
 	}
 
-	ids, err := cmd.BootstrapIdentities(builder.bootstrapNodeAddresses, builder.bootstrapNodePublicKeys)
+	ids, err := builder.DeriveBootstrapPeerIdentities()
 	if err != nil {
 		return fmt.Errorf("failed to derive bootstrap peer identities: %w", err)
 	}
@@ -655,14 +643,6 @@ func (builder *ObserverServiceBuilder) extraFlags() {
 			"observer-networking-key-path",
 			defaultConfig.observerNetworkingKeyPath,
 			"path to the networking key for observer")
-		flags.StringSliceVar(&builder.bootstrapNodeAddresses,
-			"bootstrap-node-addresses",
-			defaultConfig.bootstrapNodeAddresses,
-			"the network addresses of the bootstrap access node if this is an observer e.g. access-001.mainnet.flow.org:9653,access-002.mainnet.flow.org:9653")
-		flags.StringSliceVar(&builder.bootstrapNodePublicKeys,
-			"bootstrap-node-public-keys",
-			defaultConfig.bootstrapNodePublicKeys,
-			"the networking public key of the bootstrap access node if this is an observer (in the same order as the bootstrap node addresses) e.g. \"d57a5e9c5.....\",\"44ded42d....\"")
 		flags.DurationVar(&builder.apiTimeout, "upstream-api-timeout", defaultConfig.apiTimeout, "tcp timeout for Flow API gRPC sockets to upstrem nodes")
 		flags.StringSliceVar(&builder.upstreamNodeAddresses,
 			"upstream-node-addresses",
@@ -1002,10 +982,10 @@ func (builder *ObserverServiceBuilder) validateParams() error {
 	if len(builder.bootstrapIdentities) > 0 {
 		return nil
 	}
-	if len(builder.bootstrapNodeAddresses) == 0 {
+	if len(builder.BootstrapNodeAddresses) == 0 {
 		return errors.New("no bootstrap node address provided")
 	}
-	if len(builder.bootstrapNodeAddresses) != len(builder.bootstrapNodePublicKeys) {
+	if len(builder.BootstrapNodeAddresses) != len(builder.BootstrapNodePublicKeys) {
 		return errors.New("number of bootstrap node addresses and public keys should match")
 	}
 	if len(builder.upstreamNodePublicKeys) > 0 && len(builder.upstreamNodeAddresses) != len(builder.upstreamNodePublicKeys) {
@@ -1014,77 +994,6 @@ func (builder *ObserverServiceBuilder) validateParams() error {
 	return nil
 }
 
-// initPublicLibp2pNode creates a libp2p node for the observer service in the public (unstaked) network.
-// The factory function is later passed into the initMiddleware function to eventually instantiate the p2p.LibP2PNode instance
-// The LibP2P host is created with the following options:
-// * DHT as client and seeded with the given bootstrap peers
-// * The specified bind address as the listen address
-// * The passed in private key as the libp2p key
-// * No connection gater
-// * No connection manager
-// * No peer manager
-// * Default libp2p pubsub options.
-// Args:
-// - networkKey: the private key to use for the libp2p node
-// Returns:
-// - p2p.LibP2PNode: the libp2p node
-// - error: if any error occurs. Any error returned is considered irrecoverable.
-func (builder *ObserverServiceBuilder) initPublicLibp2pNode(networkKey crypto.PrivateKey) (p2p.LibP2PNode, error) {
-	var pis []peer.AddrInfo
-
-	for _, b := range builder.bootstrapIdentities {
-		pi, err := utils.PeerAddressInfo(*b)
-		if err != nil {
-			return nil, fmt.Errorf("could not extract peer address info from bootstrap identity %v: %w", b, err)
-		}
-
-		pis = append(pis, pi)
-	}
-
-	node, err := p2pbuilder.NewNodeBuilder(
-		builder.Logger,
-		&builder.FlowConfig.NetworkConfig.GossipSub,
-		&p2pbuilderconfig.MetricsConfig{
-			HeroCacheFactory: builder.HeroCacheMetricsFactory(),
-			Metrics:          builder.Metrics.Network,
-		},
-		network.PublicNetwork,
-		builder.BaseConfig.BindAddr,
-		networkKey,
-		builder.SporkID,
-		builder.IdentityProvider,
-		&builder.FlowConfig.NetworkConfig.ResourceManager,
-		p2pbuilderconfig.PeerManagerDisableConfig(), // disable peer manager for observer node.
-		&p2p.DisallowListCacheConfig{
-			MaxSize: builder.FlowConfig.NetworkConfig.DisallowListNotificationCacheSize,
-			Metrics: metrics.DisallowListCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork),
-		},
-		&p2pbuilderconfig.UnicastConfig{
-			Unicast: builder.FlowConfig.NetworkConfig.Unicast,
-		}).
-		SetSubscriptionFilter(
-			networkingsubscription.NewRoleBasedFilter(
-				networkingsubscription.UnstakedRole, builder.IdentityProvider,
-			),
-		).
-		SetRoutingSystem(func(ctx context.Context, h host.Host) (routing.Routing, error) {
-			return p2pdht.NewDHT(ctx, h, protocols.FlowPublicDHTProtocolID(builder.SporkID),
-				builder.Logger,
-				builder.Metrics.Network,
-				p2pdht.AsClient(),
-				dht.BootstrapPeers(pis...),
-			)
-		}).
-		Build()
-	if err != nil {
-		return nil, fmt.Errorf("could not initialize libp2p node for observer: %w", err)
-	}
-
-	builder.LibP2PNode = node
-
-	return builder.LibP2PNode, nil
-}
-
 // initObserverLocal initializes the observer's ID, network key and network address
 // Currently, it reads a node-info.priv.json like any other node.
 // TODO: read the node ID from the special bootstrap files
@@ -1673,11 +1582,13 @@ func (builder *ObserverServiceBuilder) enqueuePublicNetworkInit() {
 	builder.
 		Component("public libp2p node", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
 			var err error
-			publicLibp2pNode, err = builder.initPublicLibp2pNode(node.NetworkKey)
+			publicLibp2pNode, err = builder.BuildPublicLibp2pNode(builder.BaseConfig.BindAddr, builder.bootstrapIdentities)
 			if err != nil {
-				return nil, fmt.Errorf("could not create public libp2p node: %w", err)
+				return nil, fmt.Errorf("could not build public libp2p node: %w", err)
 			}
 
+			builder.LibP2PNode = publicLibp2pNode
+
 			return publicLibp2pNode, nil
 		}).
 		Component("public network", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) {

cmd/scaffold.go

@@ -138,8 +138,8 @@ type FlowNodeBuilder struct {
 	adminCommandBootstrapper *admin.CommandRunnerBootstrapper
 	adminCommands            map[string]func(config *NodeConfig) commands.AdminCommand
 	componentBuilder         component.ComponentManagerBuilder
-	bootstrapNodeAddresses   []string
-	bootstrapNodePublicKeys  []string
+	BootstrapNodeAddresses   []string
+	BootstrapNodePublicKeys  []string
 }
 
 var _ NodeBuilder = (*FlowNodeBuilder)(nil)
@@ -254,13 +254,13 @@ func (fnb *FlowNodeBuilder) BaseFlags() {
 
 	// observer mode allows a unstaked execution node to fetch blocks from a public staked access node, and being able to execute blocks
 	fnb.flags.BoolVar(&fnb.BaseConfig.ObserverMode, "observer-mode", defaultConfig.ObserverMode, "whether the node is running in observer mode")
-	fnb.flags.StringSliceVar(&fnb.bootstrapNodePublicKeys,
+	fnb.flags.StringSliceVar(&fnb.BootstrapNodePublicKeys,
 		"observer-mode-bootstrap-node-public-keys",
-		nil,
+		[]string{},
 		"the networking public key of the bootstrap access node if this is an observer (in the same order as the bootstrap node addresses) e.g. \"d57a5e9c5.....\",\"44ded42d....\"")
-	fnb.flags.StringSliceVar(&fnb.bootstrapNodeAddresses,
+	fnb.flags.StringSliceVar(&fnb.BootstrapNodeAddresses,
 		"observer-mode-bootstrap-node-addresses",
-		nil,
+		[]string{},
 		"the network addresses of the bootstrap access node if this is an observer e.g. access-001.mainnet.flow.org:9653,access-002.mainnet.flow.org:9653")
 }
 
@@ -413,8 +413,13 @@ func (fnb *FlowNodeBuilder) EnqueueNetworkInit() {
 		}
 
 		if fnb.ObserverMode {
-			// observer mode only init pulbic libp2p node
-			publicLibp2pNode, err := fnb.BuildPublicLibp2pNode(myAddr)
+			// observer mode only init public libp2p node
+			ids, err := fnb.DeriveBootstrapPeerIdentities()
+			if err != nil {
+				return nil, fmt.Errorf("failed to derive bootstrap peer identities: %w", err)
+			}
+
+			publicLibp2pNode, err := fnb.BuildPublicLibp2pNode(myAddr, ids)
 			if err != nil {
 				return nil, fmt.Errorf("could not build public libp2p node: %w", err)
 			}
@@ -500,7 +505,18 @@ func (fnb *FlowNodeBuilder) HeroCacheMetricsFactory() metrics.HeroCacheMetricsFa
 	return metrics.NewNoopHeroCacheMetricsFactory()
 }
 
-// initPublicLibp2pNode creates a libp2p node for the observer service in the public (unstaked) network.
+// DeriveBootstrapPeerIdentities derives the Flow Identity of the bootstrap peers from the parameters.
+// These are the identities of the observers also acting as the DHT bootstrap server
+func (fnb *FlowNodeBuilder) DeriveBootstrapPeerIdentities() (flow.IdentitySkeletonList, error) {
+	ids, err := BootstrapIdentities(fnb.BootstrapNodeAddresses, fnb.BootstrapNodePublicKeys)
+	if err != nil {
+		return nil, fmt.Errorf("failed to derive bootstrap peer identities: %w", err)
+	}
+
+	return ids, nil
+}
+
+// BuildPublicLibp2pNode creates a libp2p node for the observer service in the public (unstaked) network.
 // The factory function is later passed into the initMiddleware function to eventually instantiate the p2p.LibP2PNode instance
 // The LibP2P host is created with the following options:
 // * DHT as client and seeded with the given bootstrap peers
@@ -515,24 +531,10 @@ func (fnb *FlowNodeBuilder) HeroCacheMetricsFactory() metrics.HeroCacheMetricsFa
 // Returns:
 // - p2p.LibP2PNode: the libp2p node
 // - error: if any error occurs. Any error returned is considered irrecoverable.
-func (fnb *FlowNodeBuilder) BuildPublicLibp2pNode(address string) (p2p.LibP2PNode, error) {
+func (fnb *FlowNodeBuilder) BuildPublicLibp2pNode(address string, bootstrapIdentities flow.IdentitySkeletonList) (p2p.LibP2PNode, error) {
 	var pis []peer.AddrInfo
 
-	ids, err := BootstrapIdentities(fnb.bootstrapNodeAddresses, fnb.bootstrapNodePublicKeys)
-	if err != nil {
-		return nil, fmt.Errorf("could not create bootstrap identities: %w", err)
-	}
-
-	for _, b := range ids {
-		pi, err := utils.PeerAddressInfo(*b)
-		if err != nil {
-			return nil, fmt.Errorf("could not extract peer address info from bootstrap identity %v: %w", b, err)
-		}
-
-		pis = append(pis, pi)
-	}
-
-	for _, b := range ids {
+	for _, b := range bootstrapIdentities {
 		pi, err := utils.PeerAddressInfo(*b)
 		if err != nil {
 			return nil, fmt.Errorf("could not extract peer address info from bootstrap identity %v: %w", b, err)

cmd/util/cmd/common/snapshot.go

@@ -9,9 +9,10 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/sethvargo/go-retry"
 
+	"github.com/onflow/flow-go-sdk/access/grpc"
+
 	"github.com/onflow/flow-go/utils/logging"
 
-	"github.com/onflow/flow-go-sdk/access/grpc"
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/state/protocol"
 	"github.com/onflow/flow-go/state/protocol/inmem"

cmd/util/cmd/read-badger/cmd/find-block-by-commits/main.go

@@ -80,7 +80,7 @@ func FindBlockIDByCommits(
 
 func toStateCommitments(commitsStr string) ([]flow.StateCommitment, error) {
 	commitSlice := strings.Split(commitsStr, ",")
-	commits := make([]flow.StateCommitment, len(commitSlice))
+	commits := make([]flow.StateCommitment, 0, len(commitSlice))
 	for _, c := range commitSlice {
 		commit, err := toStateCommitment(c)
 		if err != nil {

cmd/util/ledger/migrations/account_based_migration_test.go

@@ -3,7 +3,6 @@ package migrations
 import (
 	"context"
 	"fmt"
-
 	"testing"
 
 	"github.com/onflow/cadence/common"

cmd/util/ledger/migrations/migrator_runtime.go

@@ -16,6 +16,7 @@ import (
 	"github.com/onflow/flow-go/fvm/evm"
 	evmStdlib "github.com/onflow/flow-go/fvm/evm/stdlib"
 	"github.com/onflow/flow-go/fvm/storage/state"
+	"github.com/onflow/flow-go/fvm/systemcontracts"
 	"github.com/onflow/flow-go/model/flow"
 )
 
@@ -111,8 +112,11 @@ func (c InterpreterMigrationRuntimeConfig) NewRuntimeInterface(
 		}
 	}
 
+	sc := systemcontracts.SystemContractsForChain(chainID)
+
 	return util.NewMigrationRuntimeInterface(
 		chainID,
+		common.Address(sc.Crypto.Address),
 		getCodeFunc,
 		getContractNames,
 		getOrLoadProgram,