Merge pull request #2337 from onflow/feature/secure-cadence

Secure Cadence

access/validator.go

@@ -229,7 +229,7 @@ func (v *TransactionValidator) checkExpiry(tx *flow.TransactionBody) error {
 
 func (v *TransactionValidator) checkCanBeParsed(tx *flow.TransactionBody) error {
 	if v.options.CheckScriptsParse {
-		_, err := parser2.ParseProgram(string(tx.Script))
+		_, err := parser2.ParseProgram(string(tx.Script), nil)
 		if err != nil {
 			return InvalidScriptError{ParserErr: err}
 		}

admin/tools.go

@@ -1,3 +1,4 @@
+//go:build tools
 // +build tools
 
 package tools

cmd/util/cmd/epochs/cmd/reset_test.go

@@ -186,7 +186,7 @@ func verifyArguments(t *testing.T, expected []cadence.Value, actual []interface{
 		require.NoError(t, err)
 
 		// parse cadence value
-		decoded, err := jsoncdc.Decode(bz)
+		decoded, err := jsoncdc.Decode(nil, bz)
 		require.NoError(t, err)
 
 		assert.Equal(t, expected[index], decoded)

cmd/util/cmd/epochs/cmd/templates.go

@@ -49,8 +49,8 @@ transaction(name: String,
             randomSource: randomSource,
 
             // the below arguments are unused and are safe to be left empty
-            collectorClusters: [],
-            clusterQCs: [],
-            dkgPubKeys: [])
+            collectorClusters: [] as [FlowClusterQC.Cluster],
+            clusterQCs: [] as [FlowClusterQC.ClusterQC],
+            dkgPubKeys: [] as [String])
   }
 }`

cmd/util/ledger/migrations/ordered_map_migration.go

@@ -72,6 +72,7 @@ func (m *OrderedMapMigration) initPersistentSlabStorage(v *view) {
 
 	m.NewStorage = runtime.NewStorage(
 		NewAccountsAtreeLedger(accounts),
+		nil,
 	)
 }
 
@@ -97,6 +98,8 @@ type Pair = struct {
 	Value []byte
 }
 
+var _ interpreter.Value = RawStorable{}
+
 type RawStorable []byte
 
 func (RawStorable) IsValue() {}
@@ -105,32 +108,31 @@ func (v RawStorable) Accept(interpreter *interpreter.Interpreter, visitor interp
 	panic("unreachable")
 }
 
-func (RawStorable) Walk(_ func(interpreter.Value)) {
+func (RawStorable) Walk(*interpreter.Interpreter, func(interpreter.Value)) {
 	// NO-OP
 }
 
-func (RawStorable) DynamicType(_ *interpreter.Interpreter, _ interpreter.SeenReferences) interpreter.DynamicType {
+func (RawStorable) StaticType(_ *interpreter.Interpreter) interpreter.StaticType {
 	panic("unreachable")
 }
 
-func (RawStorable) StaticType() interpreter.StaticType {
+func (RawStorable) IsImportable(_ *interpreter.Interpreter) bool {
 	panic("unreachable")
 }
 
-func (RawStorable) String() string {
+func (RawStorable) ConformsToStaticType(_ *interpreter.Interpreter,
+	_ func() interpreter.LocationRange,
+	_ interpreter.StaticType,
+	_ interpreter.TypeConformanceResults,
+) bool {
 	panic("unreachable")
 }
 
-func (v RawStorable) RecursiveString(_ interpreter.SeenReferences) string {
+func (RawStorable) String() string {
 	panic("unreachable")
 }
 
-func (v RawStorable) ConformsToDynamicType(
-	_ *interpreter.Interpreter,
-	_ func() interpreter.LocationRange,
-	dynamicType interpreter.DynamicType,
-	_ interpreter.TypeConformanceResults,
-) bool {
+func (v RawStorable) RecursiveString(_ interpreter.SeenReferences) string {
 	panic("unreachable")
 }
 
@@ -231,7 +233,7 @@ func (m *OrderedMapMigration) migrate(storagePayloads []ledger.Payload) ([]ledge
 			if err != nil {
 				panic(err)
 			}
-			storageMap := m.NewStorage.GetStorageMap(address, domain)
+			storageMap := m.NewStorage.GetStorageMap(address, domain, true)
 			for _, pair := range keyValuePairs {
 				storageMap.SetValue(
 					m.Interpreter,

cmd/util/ledger/migrations/ordered_map_migration_test.go

@@ -53,9 +53,9 @@ func TestOrderedMapMigration(t *testing.T) {
 
 	t.Run("sort values", func(t *testing.T) {
 
-		one := interpreter.NewIntValueFromInt64(1)
+		one := interpreter.NewUnmeteredIntValueFromInt64(1)
 
-		str := interpreter.NewStringValue("test")
+		str := interpreter.NewUnmeteredStringValue("test")
 
 		array := interpreter.NewArrayValue(
 			mig.Interpreter,
@@ -143,12 +143,12 @@ func TestMultipleAccounts(t *testing.T) {
 
 	t.Run("sort values", func(t *testing.T) {
 
-		one := interpreter.NewIntValueFromInt64(1)
-		two := interpreter.NewIntValueFromInt64(1)
-		three := interpreter.NewIntValueFromInt64(1)
-		four := interpreter.NewIntValueFromInt64(1)
-		five := interpreter.NewIntValueFromInt64(1)
-		six := interpreter.NewIntValueFromInt64(1)
+		one := interpreter.NewUnmeteredIntValueFromInt64(1)
+		two := interpreter.NewUnmeteredIntValueFromInt64(1)
+		three := interpreter.NewUnmeteredIntValueFromInt64(1)
+		four := interpreter.NewUnmeteredIntValueFromInt64(1)
+		five := interpreter.NewUnmeteredIntValueFromInt64(1)
+		six := interpreter.NewUnmeteredIntValueFromInt64(1)
 
 		payload := []ledger.Payload{
 			{Key: createAccountPayloadKey(address1, "storage\x1fFoo"), Value: encodeValue(one, address1)},

cmd/util/ledger/reporters/account_reporter_test.go

@@ -48,7 +48,7 @@ func TestLookupValues(t *testing.T) {
 		return encodedInt
 	}
 
-	one := interpreter.NewIntValueFromInt64(1)
+	one := interpreter.NewUnmeteredIntValueFromInt64(1)
 
 	payload := []ledger.Payload{
 		{Key: createAccountPayloadKey(address1, state2.KeyExists), Value: []byte{1}},

cmd/util/ledger/reporters/fungible_token_tracker.go

@@ -142,16 +142,18 @@ func (r *FungibleTokenTracker) worker(
 		accounts := state.NewAccounts(sth)
 		storage := cadenceRuntime.NewStorage(
 			&migrations.AccountsAtreeLedger{Accounts: accounts},
+			nil,
 		)
 
 		owner, err := common.BytesToAddress(j.owner[:])
 		if err != nil {
 			panic(err)
 		}
 
+		inter := &interpreter.Interpreter{}
 		for _, domain := range domains {
-			storageMap := storage.GetStorageMap(owner, domain)
-			itr := storageMap.Iterator()
+			storageMap := storage.GetStorageMap(owner, domain, true)
+			itr := storageMap.Iterator(inter)
 			key, value := itr.Next()
 			for value != nil {
 				r.iterateChildren(append([]string{domain}, key), j.owner, value)
@@ -177,10 +179,11 @@ func (r *FungibleTokenTracker) iterateChildren(tr trace, addr flow.Address, valu
 
 	// because compValue.Kind == common.CompositeKindResource
 	// we could pass nil to the IsResourceKinded method
+	inter := &interpreter.Interpreter{}
 	if compValue.IsResourceKinded(nil) {
 		typeIDStr := string(compValue.TypeID())
 		if _, ok := r.vaultTypeIDs[typeIDStr]; ok {
-			b := uint64(compValue.GetField(nil, nil, "balance").(interpreter.UFix64Value))
+			b := uint64(compValue.GetField(inter, nil, "balance").(interpreter.UFix64Value))
 			if b > 0 {
 				r.rw.Write(TokenDataPoint{
 					Path:    tr.String(),
@@ -192,8 +195,9 @@ func (r *FungibleTokenTracker) iterateChildren(tr trace, addr flow.Address, valu
 		}
 
 		// iterate over fields of the composite value (skip the ones that are not resource typed)
-		compValue.ForEachField(func(key string, value interpreter.Value) {
-			r.iterateChildren(append(tr, key), addr, value)
-		})
+		compValue.ForEachField(inter,
+			func(key string, value interpreter.Value) {
+				r.iterateChildren(append(tr, key), addr, value)
+			})
 	}
 }

consensus/hotstuff/integration/slow_test.go

@@ -1,4 +1,6 @@
+//go:build timesensitivetest
 // +build timesensitivetest
+
 // This file includes a few time sensitive tests. They might pass on your powerful local machine
 // but fail on slow CI machine.
 // For now, these tests are only for local. Run it with the build tag on:

consensus/integration/blockordelay_test.go

@@ -1,3 +1,4 @@
+//go:build timesensitivetest
 // +build timesensitivetest
 
 package integration_test

consensus/integration/slow_test.go

@@ -1,4 +1,6 @@
+//go:build timesensitivetest
 // +build timesensitivetest
+
 // This file includes a few time sensitive tests. They might pass on your powerful local machine
 // but fail on slow CI machine.
 // For now, these tests are only for local. Run it with the build tag on:

engine/access/rpc/backend/backend.go

@@ -5,6 +5,7 @@ import (
 	"crypto/md5" //nolint:gosec
 	"errors"
 	"fmt"
+	"sync"
 	"time"
 
 	accessproto "github.com/onflow/flow/protobuf/go/flow/access"
@@ -105,7 +106,10 @@ func New(
 			connFactory:       connFactory,
 			state:             state,
 			log:               log,
-			seenScripts:       make(map[[md5.Size]byte]time.Time),
+			seenScripts: &scriptMap{
+				scripts: make(map[[md5.Size]byte]time.Time),
+				lock:    sync.RWMutex{},
+			},
 		},
 		backendTransactions: backendTransactions{
 			staticCollectionRPC:  collectionRPC,

engine/access/rpc/backend/backend_scripts.go

@@ -3,6 +3,7 @@ package backend
 import (
 	"context"
 	"crypto/md5" //nolint:gosec
+	"sync"
 	"time"
 
 	"github.com/hashicorp/go-multierror"
@@ -25,7 +26,25 @@ type backendScripts struct {
 	state             protocol.State
 	connFactory       ConnectionFactory
 	log               zerolog.Logger
-	seenScripts       map[[md5.Size]byte]time.Time // to keep track of unique scripts sent by clients. bounded to 1MB (2^16*2*8) due to fixed key size
+	seenScripts       *scriptMap
+}
+
+type scriptMap struct {
+	scripts map[[md5.Size]byte]time.Time // to keep track of unique scripts sent by clients. bounded to 1MB (2^16*2*8) due to fixed key size
+	lock    sync.RWMutex
+}
+
+func (s *scriptMap) getLastSeen(scriptId [md5.Size]byte) (time.Time, bool) {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+	timestamp, seen := s.scripts[scriptId]
+	return timestamp, seen
+}
+
+func (s *scriptMap) setLastSeenToTime(scriptId [md5.Size]byte, execTime time.Time) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	s.scripts[scriptId] = execTime
 }
 
 func (b *backendScripts) ExecuteScriptAtLatestBlock(
@@ -107,7 +126,7 @@ func (b *backendScripts) executeScriptOnExecutionNode(
 		if err == nil {
 			if b.log.GetLevel() == zerolog.DebugLevel {
 				executionTime := time.Now()
-				timestamp, seen := b.seenScripts[encodedScript]
+				timestamp, seen := b.seenScripts.getLastSeen(encodedScript)
 				// log if the script is unique in the time window
 				if !seen || executionTime.Sub(timestamp) >= uniqueScriptLoggingTimeWindow {
 					b.log.Debug().
@@ -116,7 +135,7 @@ func (b *backendScripts) executeScriptOnExecutionNode(
 						Hex("script_hash", encodedScript[:]).
 						Str("script", string(script)).
 						Msg("Successfully executed script")
-					b.seenScripts[encodedScript] = executionTime
+					b.seenScripts.setLastSeenToTime(encodedScript, executionTime)
 				}
 			}
 			return result, nil

engine/execution/computation/computer/computer.go

@@ -3,6 +3,7 @@ package computer
 import (
 	"context"
 	"fmt"
+	"runtime"
 	"sync"
 	"time"
 
@@ -331,6 +332,12 @@ func (e *blockComputer) executeTransaction(
 	isSystemChunk bool,
 ) error {
 	startedAt := time.Now()
+
+	var memAllocBefore uint64
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+	memAllocBefore = m.TotalAlloc
+
 	txID := txBody.ID()
 
 	// we capture two spans one for tx-based view and one for the current context (block-based) view
@@ -398,12 +405,19 @@ func (e *blockComputer) executeTransaction(
 	res.AddTransactionResult(&txResult)
 	res.AddComputationUsed(tx.ComputationUsed)
 
-	lg := e.log.With().
+	runtime.ReadMemStats(&m)
+	memAllocAfter := m.TotalAlloc
+
+	evt := e.log.With().
 		Hex("tx_id", txResult.TransactionID[:]).
 		Str("block_id", res.ExecutableBlock.ID().String()).
 		Str("traceID", traceID).
 		Uint64("computation_used", txResult.ComputationUsed).
-		Int64("timeSpentInMS", time.Since(startedAt).Milliseconds()).
+		Uint64("memory_used", tx.MemoryUsed).
+		Uint64("memAlloc", memAllocAfter-memAllocBefore).
+		Int64("timeSpentInMS", time.Since(startedAt).Milliseconds())
+
+	lg := evt.
 		Logger()
 
 	if tx.Err != nil {

engine/execution/computation/computer/computer_test.go

@@ -377,7 +377,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) {
 		rt := &testRuntime{
 			executeTransaction: func(script runtime.Script, r runtime.Context) error {
 
-				program, err := r.Interface.GetProgram(contractLocation)
+				program, err := r.Interface.GetProgram(contractLocation) //nolint:staticcheck
 				require.NoError(t, err)
 				require.Nil(t, program)
 
@@ -442,7 +442,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) {
 
 				// NOTE: set a program and revert all transactions but the system chunk transaction
 
-				program, err := r.Interface.GetProgram(contractLocation)
+				program, err := r.Interface.GetProgram(contractLocation) //nolint:staticcheck
 				require.NoError(t, err)
 
 				if executionCalls > collectionCount*transactionCount {
@@ -504,6 +504,10 @@ type testRuntime struct {
 
 var _ runtime.Runtime = &testRuntime{}
 
+func (e *testRuntime) SetInvalidatedResourceValidationEnabled(_ bool) {
+	panic("SetInvalidatedResourceValidationEnabled not expected")
+}
+
 func (e *testRuntime) SetTracingEnabled(_ bool) {
 	panic("SetTracingEnabled not expected")
 }

engine/execution/computation/execution_verification_test.go

@@ -218,12 +218,24 @@ func Test_ExecutionMatchesVerification(t *testing.T) {
 
 		require.NoError(t, err)
 
-		cr := executeBlockAndVerify(t, [][]*flow.TransactionBody{
+		cr := executeBlockAndVerifyWithParameters(t, [][]*flow.TransactionBody{
 			{
 				createAccountTx,
 				spamTx,
 			},
-		}, fvm.DefaultTransactionFees, fvm.DefaultMinimumStorageReservation)
+		},
+			[]fvm.Option{
+				fvm.WithTransactionFeesEnabled(true),
+				fvm.WithAccountStorageLimit(true),
+				// make sure we don't run out of memory first.
+				fvm.WithMemoryLimit(20_000_000_000),
+			}, []fvm.BootstrapProcedureOption{
+				fvm.WithInitialTokenSupply(unittest.GenesisTokenSupply),
+				fvm.WithAccountCreationFee(fvm.DefaultAccountCreationFee),
+				fvm.WithMinimumStorageReservation(fvm.DefaultMinimumStorageReservation),
+				fvm.WithTransactionFee(fvm.DefaultTransactionFees),
+				fvm.WithStorageMBPerFLOW(fvm.DefaultStorageMBPerFLOW),
+			})
 
 		// no error
 		assert.Equal(t, cr.TransactionResults[0].ErrorMessage, "")
@@ -261,8 +273,8 @@ func TestTransactionFeeDeduction(t *testing.T) {
 		checkResult   func(t *testing.T, cr *execution.ComputationResult)
 	}
 
-	txFees := uint64(1_0000)
-	fundingAmount := uint64(1_0000_0000)
+	txFees := uint64(1_000)
+	fundingAmount := uint64(100_000_000)
 	transferAmount := uint64(123_456)
 
 	testCases := []testCase{