Merge pull request #2433 from onflow/m4ksio/secure-cadence-master-update

Secure cadence master update

.github/workflows/bench.yml

@@ -77,8 +77,13 @@ jobs:
 
             Bench tests were run a total of ${{ steps.settings.outputs.benchmark_repetitions }} times on each branch.
 
-            ## Results
+            <details>
+            <summary>Collapsed results for better readability</summary>
+            <p>
 
             ${{ env.BENCHSTAT }}
             
+            </p>
+            </details>  
+            
           edit-mode: replace

.github/workflows/ci.yml

@@ -65,7 +65,7 @@ jobs:
         max_attempts: 3
         command: make ci
     - name: Upload coverage report
-      uses: codecov/codecov-action@v1
+      uses: codecov/codecov-action@v2
       with:
         file: ./coverage.txt
         flags: unittests

Makefile

@@ -332,7 +332,7 @@ tool-transit: docker-build-bootstrap-transit
 
 .PHONY: docker-build-loader
 docker-build-loader:
-	docker build -f ./integration/loader/Dockerfile --build-arg TARGET=./cmd/loader --target production \
+	docker build -f ./integration/loader/Dockerfile --build-arg TARGET=./loader --target production \
 		--label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \
 		-t "$(CONTAINER_REGISTRY)/loader:latest" -t "$(CONTAINER_REGISTRY)/loader:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/loader:$(IMAGE_TAG)" .
 

cmd/bootstrap/cmd/finalize.go

@@ -168,6 +168,8 @@ func finalize(cmd *cobra.Command, args []string) {
 	votes := readRootBlockVotes()
 	log.Info().Msg("")
 
+	log.Info().Msgf("received votes total: %v", len(votes))
+
 	log.Info().Msg("reading dkg data")
 	dkgData := readDKGData()
 	log.Info().Msg("")

cmd/bootstrap/run/qc.go

@@ -73,6 +73,11 @@ func GenerateRootQC(block *flow.Block, votes []*model.Vote, participantData *Par
 		}
 	}
 
+	if createdQC == nil {
+		return nil, fmt.Errorf("QC is not created, total number of votes %v, expect to have 2/3 votes of %v participants",
+			len(votes), len(identities))
+	}
+
 	// STEP 3: validate constructed QC
 	val, err := createValidator(committee)
 	if err != nil {

cmd/util/cmd/export-json-execution-state/cmd.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"compress/gzip"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -13,6 +14,9 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 
+	"github.com/onflow/flow-go/ledger/complete/mtrie"
+	"github.com/onflow/flow-go/ledger/complete/mtrie/trie"
+
 	"github.com/onflow/flow-go/ledger"
 	"github.com/onflow/flow-go/ledger/common/pathfinder"
 	"github.com/onflow/flow-go/ledger/complete"
@@ -25,6 +29,7 @@ var (
 	flagOutputDir         string
 	flagStateCommitment   string
 	flagGzip              bool
+	flagFullSearch        bool
 )
 
 var Cmd = &cobra.Command{
@@ -47,23 +52,28 @@ func init() {
 
 	Cmd.Flags().BoolVar(&flagGzip, "gzip", true,
 		"Write GZip-encoded")
+
+	Cmd.Flags().BoolVar(&flagFullSearch, "full-search", false,
+		"Use full search (WARNING - traverse all WALs, extremely slow)")
 }
 
 func run(*cobra.Command, []string) {
 	log.Info().Msg("start exporting ledger")
-	err := ExportLedger(flagExecutionStateDir, flagStateCommitment, flagOutputDir)
+	err := ExportLedger(flagExecutionStateDir, flagStateCommitment, flagOutputDir, flagFullSearch)
 	if err != nil {
 		log.Fatal().Err(err).Msg("cannot get export ledger")
 	}
 }
 
 // ExportLedger exports ledger key value pairs at the given blockID
-func ExportLedger(ledgerPath string, targetstate string, outputPath string) error {
+func ExportLedger(ledgerPath string, targetstate string, outputPath string, fullSearch bool) error {
+
+	noopMetrics := &metrics.NoopCollector{}
 
 	diskWal, err := wal.NewDiskWAL(
 		zerolog.Nop(),
 		nil,
-		&metrics.NoopCollector{},
+		noopMetrics,
 		ledgerPath,
 		complete.DefaultCacheSize,
 		pathfinder.PathByteSize,
@@ -76,17 +86,11 @@ func ExportLedger(ledgerPath string, targetstate string, outputPath string) erro
 		<-diskWal.Done()
 	}()
 
-	led, err := complete.NewLedger(diskWal, complete.DefaultCacheSize, &metrics.NoopCollector{}, log.Logger, 0)
-	if err != nil {
-		return fmt.Errorf("cannot create ledger from write-a-head logs and checkpoints: %w", err)
-	}
-
 	var state ledger.State
-	// if no target state provided export the most recent state
+
 	if len(targetstate) == 0 {
-		state, err = led.MostRecentTouchedState()
-		if err != nil {
-			return fmt.Errorf("failed to load most recently used state: %w", err)
+		if fullSearch {
+			return fmt.Errorf("target state must be provided when using full search")
 		}
 	} else {
 		st, err := hex.DecodeString(targetstate)
@@ -98,6 +102,86 @@ func ExportLedger(ledgerPath string, targetstate string, outputPath string) erro
 			return fmt.Errorf("failed to convert bytes to state: %w", err)
 		}
 	}
+
+	var write func(writer io.Writer) error
+
+	if fullSearch {
+		forest, err := mtrie.NewForest(complete.DefaultCacheSize, noopMetrics, func(evictedTrie *trie.MTrie) {})
+		if err != nil {
+			return fmt.Errorf("cannot create forest: %w", err)
+		}
+
+		diskWal.PauseRecord()
+
+		sentinel := fmt.Errorf("we_got_the_trie_error")
+
+		rootState := ledger.RootHash(state)
+
+		err = diskWal.ReplayLogsOnly(
+			func(tries []*trie.MTrie) error {
+				err = forest.AddTries(tries)
+				if err != nil {
+					return fmt.Errorf("adding rebuilt tries to forest failed: %w", err)
+				}
+
+				for _, trie := range tries {
+					rootHash := trie.RootHash()
+					if rootState.Equals(rootHash) {
+						return sentinel
+					}
+				}
+				return nil
+			},
+			func(update *ledger.TrieUpdate) error {
+				rootHash, err := forest.Update(update)
+				if rootState.Equals(rootHash) {
+					return sentinel
+				}
+				return err
+			},
+			func(rootHash ledger.RootHash) error {
+				forest.RemoveTrie(rootHash)
+				return nil
+			},
+		)
+
+		if err != nil {
+			if !errors.Is(err, sentinel) {
+				return fmt.Errorf("cannot restore LedgerWAL: %w", err)
+			}
+		}
+
+		write = func(writer io.Writer) error {
+			mTrie, err := forest.GetTrie(rootState)
+			if err != nil {
+				return fmt.Errorf("cannot get trie")
+			}
+			return mTrie.DumpAsJSON(writer)
+		}
+
+	} else {
+		led, err := complete.NewLedger(diskWal, complete.DefaultCacheSize, noopMetrics, log.Logger, 0)
+		if err != nil {
+			return fmt.Errorf("cannot create ledger from write-a-head logs and checkpoints: %w", err)
+		}
+
+		// if no target state provided export the most recent state
+		if len(targetstate) == 0 {
+			state, err = led.MostRecentTouchedState()
+			if err != nil {
+				return fmt.Errorf("failed to load most recently used state: %w", err)
+			}
+		}
+
+		write = func(writer io.Writer) error {
+			err = led.DumpTrieAsJSON(state, writer)
+			if err != nil {
+				return fmt.Errorf("cannot dump trie as json: %w", err)
+			}
+			return nil
+		}
+	}
+
 	filename := state.String() + ".trie.jsonl"
 	if flagGzip {
 		filename += ".gz"
@@ -120,9 +204,5 @@ func ExportLedger(ledgerPath string, targetstate string, outputPath string) erro
 		writer = gzipWriter
 	}
 
-	err = led.DumpTrieAsJSON(state, writer)
-	if err != nil {
-		return fmt.Errorf("cannot dump trie as json: %w", err)
-	}
-	return nil
+	return write(writer)
 }

consensus/hotstuff/forks/finalizer/finalizer.go

@@ -258,9 +258,6 @@ func (r *Finalizer) getNextAncestryLevel(block *model.Block) (*forks.BlockQC, er
 		return nil, ErrPrunedAncestry
 	}
 
-	if block.QC.View < r.lastFinalized.Block.View {
-		return nil, ErrPrunedAncestry
-	}
 	parentVertex, parentBlockKnown := r.forest.GetVertex(block.QC.BlockID)
 	if !parentBlockKnown {
 		return nil, model.MissingBlockError{View: block.QC.View, BlockID: block.QC.BlockID}

crypto/dkg_test.go

@@ -1,3 +1,4 @@
+//go:build relic
 // +build relic
 
 package crypto
@@ -285,7 +286,7 @@ func dkgCommonTest(t *testing.T, dkg int, n int, threshold int, test testCase) {
 	}
 
 	phase := 0
-	if dkg == feldmanVSS {
+	if dkg == feldmanVSS { // jump to the last phase since there is only one phase for feldmanVSS
 		phase = 2
 	}
 
@@ -392,6 +393,7 @@ func dkgRunChan(proc *testDKGProcessor,
 			}
 		// if no message is received by the channel, call the DKG timeout
 		case <-time.After(phaseSwitchTimeout):
+			proc.startSync.Wait() // avoids racing when starting isn't over yet
 			switch phase {
 			case 0:
 				log.Infof("%d shares phase ended\n", proc.current)

engine/access/rest/test_helpers.go

@@ -28,7 +28,7 @@ const (
 func executeRequest(req *http.Request, backend *mock.API) (*httptest.ResponseRecorder, error) {
 	var b bytes.Buffer
 	logger := zerolog.New(&b)
-	router, err := newRouter(backend, logger, flow.Canary.Chain())
+	router, err := newRouter(backend, logger, flow.Testnet.Chain())
 	if err != nil {
 		return nil, err
 	}

engine/collection/epochmgr/factories/builder.go

@@ -1,6 +1,8 @@
 package factories
 
 import (
+	"fmt"
+
 	"github.com/dgraph-io/badger/v2"
 
 	"github.com/onflow/flow-go/module"
@@ -46,7 +48,7 @@ func (f *BuilderFactory) Create(
 	pool mempool.Transactions,
 ) (module.Builder, *finalizer.Finalizer, error) {
 
-	build := builder.NewBuilder(
+	build, err := builder.NewBuilder(
 		f.db,
 		f.trace,
 		f.mainChainHeaders,
@@ -55,6 +57,9 @@ func (f *BuilderFactory) Create(
 		pool,
 		f.opts...,
 	)
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not create builder: %w", err)
+	}
 
 	final := finalizer.NewFinalizer(
 		f.db,

engine/execution/computation/execution_verification_test.go

@@ -34,7 +34,10 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-var chain = flow.Mainnet.Chain()
+var chain = flow.Emulator.Chain()
+
+// In the following tests the system transaction is expected to fail, as the epoch related things are not set up properly.
+// This is not relevant to the test, as only the non-system transactions are tested.
 
 func Test_ExecutionMatchesVerification(t *testing.T) {
 	t.Run("empty block", func(t *testing.T) {
@@ -624,6 +627,7 @@ func executeBlockAndVerifyWithParameters(t *testing.T,
 	logger := zerolog.Nop()
 
 	opts = append(opts, fvm.WithChain(chain))
+	opts = append(opts, fvm.WithBlocks(&fvm.NoopBlockFinder{}))
 
 	fvmContext :=
 		fvm.NewContext(
@@ -657,7 +661,7 @@ func executeBlockAndVerifyWithParameters(t *testing.T,
 
 	view := delta.NewView(state.LedgerGetRegister(ledger, initialCommit))
 
-	executableBlock := unittest.ExecutableBlockFromTransactions(txs)
+	executableBlock := unittest.ExecutableBlockFromTransactions(chain.ChainID(), txs)
 	executableBlock.StartState = &initialCommit
 
 	computationResult, err := blockComputer.ExecuteBlock(context.Background(), executableBlock, view, programs.NewEmptyPrograms())

fvm/fvm_bench_test.go

@@ -198,7 +198,7 @@ func (b *BasicBlockExecutor) ServiceAccount(_ testing.TB) *TestBenchAccount {
 }
 
 func (b *BasicBlockExecutor) ExecuteCollections(tb testing.TB, collections [][]*flow.TransactionBody) *execution.ComputationResult {
-	executableBlock := unittest.ExecutableBlockFromTransactions(collections)
+	executableBlock := unittest.ExecutableBlockFromTransactions(b.chain.ChainID(), collections)
 	executableBlock.StartState = &b.activeStateCommitment
 
 	computationResult, err := b.blockComputer.ExecuteBlock(context.Background(), executableBlock, b.activeView, b.programCache)

fvm/fvm_test.go

@@ -2993,6 +2993,53 @@ func TestBlockContext_ExecuteTransaction_FailingTransactions(t *testing.T) {
 		}),
 	)
 
+	t.Run("Transaction fails because of recipient account not existing", newVMTest().withBootstrapProcedureOptions(
+		fvm.WithMinimumStorageReservation(fvm.DefaultMinimumStorageReservation),
+		fvm.WithAccountCreationFee(fvm.DefaultAccountCreationFee),
+		fvm.WithStorageMBPerFLOW(fvm.DefaultStorageMBPerFLOW),
+	).run(
+		func(t *testing.T, vm *fvm.VirtualMachine, chain flow.Chain, ctx fvm.Context, view state.View, programs *programs.Programs) {
+			ctx.LimitAccountStorage = true // this test requires storage limits to be enforced
+
+			// Create an account private key.
+			privateKeys, err := testutil.GenerateAccountPrivateKeys(1)
+			require.NoError(t, err)
+
+			// Bootstrap a ledger, creating accounts with the provided private keys and the root account.
+			accounts, err := testutil.CreateAccounts(vm, view, programs, privateKeys, chain)
+			require.NoError(t, err)
+
+			// non-existent account
+			lastAddress, err := chain.AddressAtIndex((1 << 45) - 1)
+			require.NoError(t, err)
+
+			balanceBefore := getBalance(vm, chain, ctx, view, accounts[0])
+
+			// transfer tokens to non-existent account
+			txBody := transferTokensTx(chain).
+				AddAuthorizer(accounts[0]).
+				AddArgument(jsoncdc.MustEncode(cadence.UFix64(1))).
+				AddArgument(jsoncdc.MustEncode(cadence.NewAddress(lastAddress)))
+
+			txBody.SetProposalKey(accounts[0], 0, 0)
+			txBody.SetPayer(accounts[0])
+
+			err = testutil.SignEnvelope(txBody, accounts[0], privateKeys[0])
+			require.NoError(t, err)
+
+			tx := fvm.Transaction(txBody, 0)
+
+			err = vm.Run(ctx, tx, view, programs)
+			require.NoError(t, err)
+
+			require.Equal(t, (&errors.CadenceRuntimeError{}).Code(), tx.Err.Code())
+
+			balanceAfter := getBalance(vm, chain, ctx, view, accounts[0])
+
+			require.Equal(t, balanceAfter, balanceBefore)
+		}),
+	)
+
 	t.Run("Transaction sequence number check fails and sequence number is not incremented", newVMTest().withBootstrapProcedureOptions(
 		fvm.WithMinimumStorageReservation(fvm.DefaultMinimumStorageReservation),
 		fvm.WithAccountCreationFee(fvm.DefaultAccountCreationFee),