Skip to content

Commit

Permalink
Merge branch 'master' into UlyanaAndrukhiv/6413-backfill-tx-error-mes…
Browse files Browse the repository at this point in the history
…sages
  • Loading branch information
UlyanaAndrukhiv authored Oct 30, 2024
2 parents a239bb3 + 3496c0f commit 09a19ea
Show file tree
Hide file tree
Showing 188 changed files with 12,619 additions and 2,757 deletions.
27 changes: 13 additions & 14 deletions CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
/engine/consensus/** @AlexHentschel @durkmurder @jordanschalm

# Execution Stream
/cmd/execution/** @ramtinms
/engine/execution/** @ramtinms
/cmd/execution/** @zhangchiqing
/engine/execution/** @zhangchiqing

# Access Stream
/access/** @peterargue
Expand All @@ -18,19 +18,18 @@
/engine/access/** @peterargue

# Verification Stream
/cmd/verification/** @ramtinms @yhassanzadeh13
/engine/verification/** @ramtinms @yhassanzadeh13
/module/chunking/** @ramtinms
/integration/tests/verification @ramtinms @yhassanzadeh13
/cmd/verification/** @zhangchiqing
/engine/verification/** @zhangchiqing
/integration/tests/verification @zhangchiqing

# Ledger Stream
/ledger/** @ramtinms @AlexHentschel
/ledger/** @AlexHentschel

# FVM Stream
/fvm/** @ramtinms @janezpodhostnik
/fvm/** @janezpodhostnik

# Networking Stream
/network/** @yhassanzadeh13
/network/** @Kay-Zee

# Cryptography Stream
/crypto/** @tarakby
Expand All @@ -39,13 +38,13 @@
/cmd/bootstrap/** @zhangchiqing

# Dev Tools Stream
.github/workflows/** @gomisha
/insecure/** @gomisha @yhassanzadeh13
/integration/benchnet2/** @gomisha
/tools/test_monitor/** @gomisha
.github/workflows/** @Kay-Zee
/insecure/** @Kay-Zee
/integration/benchnet2/** @Kay-Zee
/tools/test_monitor/** @Kay-Zee

# Performance Stream
/integration/benchmark/** @gomisha
/integration/benchmark/** @Kay-Zee

# Execution Sync
/module/executiondatasync/** @peterargue
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ go-math-rand-check:
# - "onflow/crypto/random" for deterministic randomness
grep --include=\*.go \
--exclude=*test* --exclude=*helper* --exclude=*example* --exclude=*fixture* --exclude=*benchmark* --exclude=*profiler* \
--exclude-dir=*test* --exclude-dir=*helper* --exclude-dir=*example* --exclude-dir=*fixture* --exclude-dir=*benchmark* --exclude-dir=*profiler* -rnw '"math/rand"'; \
--exclude-dir=*test* --exclude-dir=*helper* --exclude-dir=*example* --exclude-dir=*fixture* --exclude-dir=*benchmark* --exclude-dir=*profiler* --exclude-dir=*emulator* -rnw '"math/rand"'; \
if [ $$? -ne 1 ]; then \
echo "[Error] Go production code should not use math/rand package"; exit 1; \
fi
Expand Down
14 changes: 14 additions & 0 deletions cmd/access/node_builder/access_node_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ type AccessNodeConfig struct {
logTxTimeToFinalized bool
logTxTimeToExecuted bool
logTxTimeToFinalizedExecuted bool
logTxTimeToSealed bool
retryEnabled bool
rpcMetricsEnabled bool
executionDataSyncEnabled bool
Expand Down Expand Up @@ -243,6 +244,7 @@ func DefaultAccessNodeConfig() *AccessNodeConfig {
logTxTimeToFinalized: false,
logTxTimeToExecuted: false,
logTxTimeToFinalizedExecuted: false,
logTxTimeToSealed: false,
pingEnabled: false,
retryEnabled: false,
rpcMetricsEnabled: false,
Expand Down Expand Up @@ -304,6 +306,7 @@ type FlowAccessNodeBuilder struct {
CollectionsToMarkFinalized *stdmap.Times
CollectionsToMarkExecuted *stdmap.Times
BlocksToMarkExecuted *stdmap.Times
BlockTransactions *stdmap.IdentifierMap
TransactionMetrics *metrics.TransactionCollector
TransactionValidationMetrics *metrics.TransactionValidationCollector
RestMetrics *metrics.RestCollector
Expand Down Expand Up @@ -1240,6 +1243,10 @@ func (builder *FlowAccessNodeBuilder) extraFlags() {
"log-tx-time-to-finalized-executed",
defaultConfig.logTxTimeToFinalizedExecuted,
"log transaction time to finalized and executed")
flags.BoolVar(&builder.logTxTimeToSealed,
"log-tx-time-to-sealed",
defaultConfig.logTxTimeToSealed,
"log transaction time to sealed")
flags.BoolVar(&builder.pingEnabled,
"ping-enabled",
defaultConfig.pingEnabled,
Expand Down Expand Up @@ -1683,6 +1690,11 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) {
return err
}

builder.BlockTransactions, err = stdmap.NewIdentifierMap(10000)
if err != nil {
return err
}

builder.BlocksToMarkExecuted, err = stdmap.NewTimes(1 * 300) // assume 1 block per second * 300 seconds

return err
Expand All @@ -1694,6 +1706,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) {
builder.logTxTimeToFinalized,
builder.logTxTimeToExecuted,
builder.logTxTimeToFinalizedExecuted,
builder.logTxTimeToSealed,
)
return nil
}).
Expand Down Expand Up @@ -1728,6 +1741,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) {
builder.BlocksToMarkExecuted,
builder.Storage.Collections,
builder.Storage.Blocks,
builder.BlockTransactions,
)
if err != nil {
return err
Expand Down
1 change: 0 additions & 1 deletion cmd/bootstrap/utils/md5.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package utils

// The google storage API only provides md5 and crc32 hence overriding the linter flag for md5
// #nosec
import (
"crypto/md5" //nolint:gosec
"io"
Expand Down
52 changes: 7 additions & 45 deletions cmd/execution_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@ import (
txmetrics "github.com/onflow/flow-go/engine/execution/computation/metrics"
"github.com/onflow/flow-go/engine/execution/ingestion"
"github.com/onflow/flow-go/engine/execution/ingestion/fetcher"
"github.com/onflow/flow-go/engine/execution/ingestion/loader"
"github.com/onflow/flow-go/engine/execution/ingestion/stop"
"github.com/onflow/flow-go/engine/execution/ingestion/uploader"
exeprovider "github.com/onflow/flow-go/engine/execution/provider"
Expand Down Expand Up @@ -1081,61 +1080,24 @@ func (exeNode *ExecutionNode) LoadIngestionEngine(
exeNode.collectionRequester = reqEng
}

if exeNode.exeConf.enableNewIngestionEngine {
_, core, err := ingestion.NewMachine(
node.Logger,
node.ProtocolEvents,
exeNode.collectionRequester,
colFetcher,
node.Storage.Headers,
node.Storage.Blocks,
node.Storage.Collections,
exeNode.executionState,
node.State,
exeNode.collector,
exeNode.computationManager,
exeNode.providerEngine,
exeNode.blockDataUploader,
exeNode.stopControl,
)

return core, err
}

var blockLoader ingestion.BlockLoader
if exeNode.exeConf.enableStorehouse {
blockLoader = loader.NewUnfinalizedLoader(node.Logger, node.State, node.Storage.Headers, exeNode.executionState)
} else {
blockLoader = loader.NewUnexecutedLoader(node.Logger, node.State, node.Storage.Headers, exeNode.executionState)
}

ingestionEng, err := ingestion.New(
exeNode.ingestionUnit,
_, core, err := ingestion.NewMachine(
node.Logger,
node.EngineRegistry,
node.ProtocolEvents,
exeNode.collectionRequester,
colFetcher,
node.Storage.Headers,
node.Storage.Blocks,
node.Storage.Collections,
exeNode.computationManager,
exeNode.providerEngine,
exeNode.executionState,
node.State,
exeNode.collector,
node.Tracer,
exeNode.exeConf.extensiveLog,
exeNode.executionDataPruner,
exeNode.computationManager,
exeNode.providerEngine,
exeNode.blockDataUploader,
exeNode.stopControl,
blockLoader,
)

// TODO: we should solve these mutual dependencies better
// => https://github.com/dapperlabs/flow-go/issues/4360
exeNode.collectionRequester.WithHandle(ingestionEng.OnCollection)

node.ProtocolEvents.AddConsumer(ingestionEng)

return ingestionEng, err
return core, err
}

// create scripts engine for handling script execution
Expand Down
13 changes: 7 additions & 6 deletions cmd/execution_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,11 +69,10 @@ type ExecutionConfig struct {
// It works around an issue where some collection nodes are not configured with enough
// this works around an issue where some collection nodes are not configured with enough
// file descriptors causing connection failures.
onflowOnlyLNs bool
enableStorehouse bool
enableChecker bool
enableNewIngestionEngine bool
publicAccessID string
onflowOnlyLNs bool
enableStorehouse bool
enableChecker bool
publicAccessID string
}

func (exeConf *ExecutionConfig) SetupFlags(flags *pflag.FlagSet) {
Expand Down Expand Up @@ -132,7 +131,9 @@ func (exeConf *ExecutionConfig) SetupFlags(flags *pflag.FlagSet) {
flags.BoolVar(&exeConf.onflowOnlyLNs, "temp-onflow-only-lns", false, "do not use unless required. forces node to only request collections from onflow collection nodes")
flags.BoolVar(&exeConf.enableStorehouse, "enable-storehouse", false, "enable storehouse to store registers on disk, default is false")
flags.BoolVar(&exeConf.enableChecker, "enable-checker", true, "enable checker to check the correctness of the execution result, default is true")
flags.BoolVar(&exeConf.enableNewIngestionEngine, "enable-new-ingestion-engine", true, "enable new ingestion engine, default is true")
// deprecated. Retain it to prevent nodes that previously had this configuration from crashing.
var deprecatedEnableNewIngestionEngine bool
flags.BoolVar(&deprecatedEnableNewIngestionEngine, "enable-new-ingestion-engine", true, "enable new ingestion engine, default is true")
flags.StringVar(&exeConf.publicAccessID, "public-access-id", "", "public access ID for the node")

}
Expand Down
7 changes: 7 additions & 0 deletions cmd/observer/node_builder/observer_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@ type ObserverServiceConfig struct {
logTxTimeToFinalized bool
logTxTimeToExecuted bool
logTxTimeToFinalizedExecuted bool
logTxTimeToSealed bool
executionDataSyncEnabled bool
executionDataIndexingEnabled bool
executionDataDBMode string
Expand Down Expand Up @@ -223,6 +224,7 @@ func DefaultObserverServiceConfig() *ObserverServiceConfig {
logTxTimeToFinalized: false,
logTxTimeToExecuted: false,
logTxTimeToFinalizedExecuted: false,
logTxTimeToSealed: false,
executionDataSyncEnabled: false,
executionDataIndexingEnabled: false,
executionDataDBMode: execution_data.ExecutionDataDBModeBadger.String(),
Expand Down Expand Up @@ -659,6 +661,10 @@ func (builder *ObserverServiceBuilder) extraFlags() {
"log-tx-time-to-finalized-executed",
defaultConfig.logTxTimeToFinalizedExecuted,
"log transaction time to finalized and executed")
flags.BoolVar(&builder.logTxTimeToSealed,
"log-tx-time-to-sealed",
defaultConfig.logTxTimeToSealed,
"log transaction time to sealed")
flags.BoolVar(&builder.rpcMetricsEnabled, "rpc-metrics-enabled", defaultConfig.rpcMetricsEnabled, "whether to enable the rpc metrics")
flags.BoolVar(&builder.executionDataIndexingEnabled,
"execution-data-indexing-enabled",
Expand Down Expand Up @@ -1671,6 +1677,7 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() {
builder.logTxTimeToFinalized,
builder.logTxTimeToExecuted,
builder.logTxTimeToFinalizedExecuted,
builder.logTxTimeToSealed,
)
return nil
})
Expand Down
109 changes: 109 additions & 0 deletions cmd/util/cmd/export-evm-state/cmd.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
package evm_exporter

import (
"fmt"
"os"

"github.com/rs/zerolog/log"
"github.com/spf13/cobra"

"github.com/onflow/flow-go/cmd/util/ledger/util"
"github.com/onflow/flow-go/fvm/evm"
"github.com/onflow/flow-go/fvm/evm/emulator/state"
"github.com/onflow/flow-go/ledger"
"github.com/onflow/flow-go/ledger/common/convert"
"github.com/onflow/flow-go/model/flow"
)

var (
flagChain string
flagExecutionStateDir string
flagOutputDir string
flagStateCommitment string
)

var Cmd = &cobra.Command{
Use: "export-evm-state",
Short: "exports evm state into a several binary files",
Run: run,
}

func init() {
Cmd.Flags().StringVar(&flagChain, "chain", "", "Chain name")
_ = Cmd.MarkFlagRequired("chain")

Cmd.Flags().StringVar(&flagExecutionStateDir, "execution-state-dir", "",
"Execution Node state dir (where WAL logs are written")
_ = Cmd.MarkFlagRequired("execution-state-dir")

Cmd.Flags().StringVar(&flagOutputDir, "output-dir", "",
"Directory to write new Execution State to")
_ = Cmd.MarkFlagRequired("output-dir")

Cmd.Flags().StringVar(&flagStateCommitment, "state-commitment", "",
"State commitment (hex-encoded, 64 characters)")
}

func run(*cobra.Command, []string) {
log.Info().Msg("start exporting evm state")
err := ExportEVMState(flagChain, flagExecutionStateDir, flagStateCommitment, flagOutputDir)
if err != nil {
log.Fatal().Err(err).Msg("cannot get export evm state")
}
}

// ExportEVMState evm state
func ExportEVMState(
chainName string,
ledgerPath string,
targetState string,
outputPath string) error {

chainID := flow.ChainID(chainName)

storageRoot := evm.StorageAccountAddress(chainID)
rootOwner := string(storageRoot.Bytes())

payloads, err := util.ReadTrie(ledgerPath, util.ParseStateCommitment(targetState))
if err != nil {
return err
}

// filter payloads of evm storage
filteredPayloads := make(map[flow.RegisterID]*ledger.Payload, 0)
for _, payload := range payloads {
registerID, _, err := convert.PayloadToRegister(payload)
if err != nil {
return fmt.Errorf("failed to convert payload to register: %w", err)
}
if registerID.Owner == rootOwner {
filteredPayloads[registerID] = payload
}
}

payloadsLedger := util.NewPayloadsLedger(filteredPayloads)

exporter, err := state.NewExporter(payloadsLedger, storageRoot)
if err != nil {
return fmt.Errorf("failed to create exporter: %w", err)
}

if _, err := os.Stat(outputPath); os.IsNotExist(err) {
err := os.Mkdir(outputPath, os.ModePerm)
if err != nil {
return fmt.Errorf("failed to create path: %w", err)
}
}

fi, err := os.Create(outputPath)
if err != nil {
return err
}
defer fi.Close()

err = exporter.Export(outputPath)
if err != nil {
return fmt.Errorf("failed to export: %w", err)
}
return nil
}
Loading

0 comments on commit 09a19ea

Please sign in to comment.