WIP: Experimental valuegeneration techniques

.github/workflows/ci.yml

@@ -87,7 +87,6 @@ jobs:
           inputs: ./medusa-*.tar.gz
 
       - name: Upload artifact
-        if: github.ref == 'refs/heads/master' || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/'))
         uses: actions/upload-artifact@v4
         with:
           name: medusa-${{ runner.os }}-${{ runner.arch }}

DEV.md

@@ -0,0 +1,104 @@
+# Debugging and Development
+
+## Debugging
+
+The following scripts are available for Medusa developers for debugging changes to the fuzzer.
+
+### Corpus diff
+
+The corpus diff script is used to compare two corpora and identify the methods that are present in one but not the other. This is useful for identifying methods that are missing from a corpus that should be present.
+
+```shell
+python3 scripts/corpus_diff.py corpus1 corpus2
+```
+
+```shell
+Methods only in ~/corpus1:
+-  clampSplitWeight(uint32,uint32)
+
+Methods only in ~/corpus2:
+  <None>
+```
+
+### Corpus stats
+
+The corpus stats script is used to generate statistics about a corpus. This includes the number of sequences, the average length of sequences, and the frequency of methods called.
+
+```shell
+python3 scripts/corpus_stats.py corpus
+```
+
+```shell
+Number of Sequences in ~/corpus: 130
+
+Average Length of Transactions List: 43
+
+Frequency of Methods Called:
+-  testReceiversReceivedSplit(uint8): 280
+-  setMaxEndHints(uint32,uint32): 174
+-  setStreamBalanceWithdrawAll(uint8): 139
+-  giveClampedAmount(uint8,uint8,uint128): 136
+-  receiveStreamsSplitAndCollectToSelf(uint8): 133
+-  testSqueezeViewVsActual(uint8,uint8): 128
+-  testSqueeze(uint8,uint8): 128
+-  testSetStreamBalance(uint8,int128): 128
+-  addStreamWithClamping(uint8,uint8,uint160,uint32,uint32,int128): 125
+-  removeAllSplits(uint8): 118
+-  testSplittableAfterSplit(uint8): 113
+-  testSqueezableVsReceived(uint8): 111
+-  testBalanceAtInFuture(uint8,uint8,uint160): 108
+-  testRemoveStreamShouldNotRevert(uint8,uint256): 103
+-  invariantWithdrawAllTokensShouldNotRevert(): 103
+-  collect(uint8,uint8): 101
+-  invariantAmtPerSecVsMinAmtPerSec(uint8,uint256): 98
+-  testSqueezableAmountCantBeWithdrawn(uint8,uint8): 97
+-  split(uint8): 97
+-  invariantWithdrawAllTokens(): 95
+-  testReceiveStreams(uint8,uint32): 93
+-  invariantAccountingVsTokenBalance(): 92
+-  testSqueezeWithFuzzedHistoryShouldNotRevert(uint8,uint8,uint256,bytes32): 91
+-  testSqueezableAmountCantBeUndone(uint8,uint8,uint160,uint32,uint32,int128): 87
+-  testCollect(uint8,uint8): 86
+-  testSetStreamBalanceWithdrawAllShouldNotRevert(uint8): 86
+-  testAddStreamShouldNotRevert(uint8,uint8,uint160,uint32,uint32,int128): 85
+-  testReceiveStreamsShouldNotRevert(uint8): 84
+-  addSplitsReceiver(uint8,uint8,uint32): 84
+-  setStreamBalanceWithClamping(uint8,int128): 82
+-  addSplitsReceiverWithClamping(uint8,uint8,uint32): 80
+-  testSetStreamBalanceShouldNotRevert(uint8,int128): 80
+-  testSplitShouldNotRevert(uint8): 80
+-  squeezeAllAndReceiveAndSplitAndCollectToSelf(uint8): 79
+-  addStreamImmediatelySqueezable(uint8,uint8,uint160): 79
+-  testSetSplitsShouldNotRevert(uint8,uint8,uint32): 78
+-  invariantSumAmtDeltaIsZero(uint8): 78
+-  testReceiveStreamsViewConsistency(uint8,uint32): 76
+-  squeezeToSelf(uint8): 74
+-  collectToSelf(uint8): 72
+-  setStreams(uint8,uint8,uint160,uint32,uint32,int128): 70
+-  receiveStreamsAllCycles(uint8): 69
+-  invariantWithdrawShouldAlwaysFail(uint256): 68
+-  addStream(uint8,uint8,uint160,uint32,uint32,int128): 68
+-  squeezeWithFuzzedHistory(uint8,uint8,uint256,bytes32): 67
+-  setStreamsWithClamping(uint8,uint8,uint160,uint32,uint32,int128): 67
+-  splitAndCollectToSelf(uint8): 67
+-  testSqueezeWithFullyHashedHistory(uint8,uint8): 65
+-  give(uint8,uint8,uint128): 65
+-  setSplits(uint8,uint8,uint32): 65
+-  testSqueezeTwice(uint8,uint8,uint256,bytes32): 65
+-  testSetStreamsShouldNotRevert(uint8,uint8,uint160,uint32,uint32,int128): 64
+-  squeezeAllSenders(uint8): 63
+-  removeStream(uint8,uint256): 62
+-  testCollectableAfterSplit(uint8): 58
+-  testCollectShouldNotRevert(uint8,uint8): 56
+-  testReceiveStreamsViewVsActual(uint8,uint32): 55
+-  receiveStreams(uint8,uint32): 55
+-  setSplitsWithClamping(uint8,uint8,uint32): 55
+-  testGiveShouldNotRevert(uint8,uint8,uint128): 47
+-  setStreamBalance(uint8,int128): 47
+-  squeezeWithDefaultHistory(uint8,uint8): 45
+-  testSplitViewVsActual(uint8): 45
+-  testAddSplitsShouldNotRevert(uint8,uint8,uint32): 30
+-  testSqueezeWithDefaultHistoryShouldNotRevert(uint8,uint8): 23
+
+Number of Unique Methods: 65
+```

docs/src/static/medusa.json

@@ -7,6 +7,7 @@
     "callSequenceLength": 100,
     "corpusDirectory": "",
     "coverageEnabled": true,
+    "experimentalValueGenerationEnabled": true,
     "targetContracts": [],
     "targetContractsBalances": [],
     "constructorArgs": {},

fuzzing/config/config.go

@@ -60,6 +60,14 @@ type FuzzingConfig struct {
 	// CoverageEnabled describes whether to use coverage-guided fuzzing
 	CoverageEnabled bool `json:"coverageEnabled"`
 
+	// HtmlReportFile describes the name for the html coverage file. If empty,
+	// the html coverage file will not be saved
+	HtmlReportFile string `json:"htmlReportPath"`
+
+	// JsonReportFile describes the name for the html coverage file. If empty,
+	// the json coverage file will not be saved
+	JsonReportFile string `json:"jsonReportPath"`
+
 	// TargetContracts are the target contracts for fuzz testing
 	TargetContracts []string `json:"targetContracts"`
 
@@ -142,6 +150,10 @@ type TestingConfig struct {
 	// OptimizationTesting describes the configuration used for optimization testing.
 	OptimizationTesting OptimizationTestingConfig `json:"optimizationTesting"`
 
+	// ExperimentalValueGenerationEnabled describes the configuration used for testing of collection
+	// and addition of interesting values found during EVM execution to base value set
+	ExperimentalValueGenerationEnabled bool `json:"experimentalValueGenerationEnabled"`
+
 	// TargetFunctionSignatures is a list function signatures call the fuzzer should exclusively target by omitting calls to other signatures.
 	// The signatures should specify the contract name and signature in the ABI format like `Contract.func(uint256,bytes32)`.
 	TargetFunctionSignatures []string `json:"targetFunctionSignatures"`

fuzzing/config/config_defaults.go

@@ -46,6 +46,8 @@ func GetDefaultProjectConfig(platform string) (*ProjectConfig, error) {
 			ConstructorArgs:         map[string]map[string]any{},
 			CorpusDirectory:         "",
 			CoverageEnabled:         true,
+			HtmlReportFile:          "coverage_report.html",
+			JsonReportFile:          "coverage_report.json",
 			SenderAddresses: []string{
 				"0x10000",
 				"0x20000",
@@ -64,6 +66,7 @@ func GetDefaultProjectConfig(platform string) (*ProjectConfig, error) {
 				TraceAll:                     false,
 				TargetFunctionSignatures:     []string{},
 				ExcludeFunctionSignatures:    []string{},
+				ExperimentalValueGenerationEnabled: false,
 				AssertionTesting: AssertionTestingConfig{
 					Enabled:         true,
 					TestViewMethods: false,

fuzzing/coverage/report_generation.go

@@ -3,41 +3,122 @@ package coverage
 import (
 	_ "embed"
 	"fmt"
-	"github.com/crytic/medusa/compilation/types"
-	"github.com/crytic/medusa/utils"
 	"html/template"
 	"math"
 	"os"
 	"path/filepath"
 	"strconv"
 	"time"
+
+	"github.com/crytic/medusa/compilation/types"
+	"github.com/crytic/medusa/utils"
 )
 
 var (
 	//go:embed report_template.gohtml
 	htmlReportTemplate []byte
+	//go:embed report_template.gojson
+	jsonReportTemplate []byte
 )
 
 // GenerateReport takes a set of CoverageMaps and compilations, and produces a coverage report using them, detailing
 // all source mapped ranges of the source files which were covered or not.
 // Returns an error if one occurred.
-func GenerateReport(compilations []types.Compilation, coverageMaps *CoverageMaps, htmlReportPath string) error {
+func GenerateReport(compilations []types.Compilation, coverageMaps *CoverageMaps, corpusPath string, htmlReportPath string, jsonReportPath string) error {
 	// Perform source analysis.
 	sourceAnalysis, err := AnalyzeSourceCoverage(compilations, coverageMaps)
 	if err != nil {
 		return err
 	}
 
-	// Finally, export the report data we analyzed.
+	// Stores the output path of the report
+	var outputPath string
+
+	// Export the html report of the data we analyzed.
 	if htmlReportPath != "" {
-		err = exportCoverageReport(sourceAnalysis, htmlReportPath)
+		outputPath = filepath.Join(corpusPath, htmlReportPath)
+		err = exportHtmlCoverageReport(sourceAnalysis, outputPath)
 	}
+	// Export the json report of the data we analyzed.
+	if jsonReportPath != "" {
+		outputPath = filepath.Join(corpusPath, jsonReportPath)
+		err2 := exportJsonCoverageReport(sourceAnalysis, outputPath)
+		if err == nil && err2 != nil {
+			err = err2
+		}
+	}
+	return err
+}
+
+// exportCoverageReportJSON takes a previously performed source analysis and generates a JSON coverage report from it.
+// Returns an error if one occurs.
+func exportJsonCoverageReport(sourceAnalysis *SourceAnalysis, outputPath string) error {
+	functionMap := template.FuncMap{
+		"add": func(x int, y int) int {
+			return x + y
+		},
+		"sub": func(x int, y int) int {
+			return x - y
+		},
+		"relativePath": func(path string) string {
+			// Obtain a path relative to our current working directory.
+			// If we encounter an error, return the original path.
+			cwd, err := os.Getwd()
+			if err != nil {
+				return path
+			}
+			relativePath, err := filepath.Rel(cwd, path)
+			if err != nil {
+				return path
+			}
+
+			return relativePath
+		},
+		"lastActiveIndex": func(sourceFileAnalysis *SourceFileAnalysis) int {
+			// Determine the last active line index and return it
+			lastIndex := 0
+			for lineIndex, line := range sourceFileAnalysis.Lines {
+				if line.IsActive {
+					lastIndex = lineIndex
+				}
+			}
+			return lastIndex
+		},
+	}
+
+	// Parse our JSON template
+	tmpl, err := template.New("coverage_report.json").Funcs(functionMap).Parse(string(jsonReportTemplate))
+	if err != nil {
+		return fmt.Errorf("could not export report, failed to parse report template: %v", err)
+	}
+
+	// If the parent directory doesn't exist, create it.
+	parentDirectory := filepath.Dir(outputPath)
+	err = utils.MakeDirectory(parentDirectory)
+	if err != nil {
+		return err
+	}
+
+	// Create our report file
+	file, err := os.Create(outputPath)
+	if err != nil {
+		_ = file.Close()
+		return fmt.Errorf("could not export report, failed to open file for writing: %v", err)
+	}
+
+	// Execute the template and write it back to file.
+	err = tmpl.Execute(file, sourceAnalysis)
+	fileCloseErr := file.Close()
+	if err == nil {
+		err = fileCloseErr
+	}
+
 	return err
 }
 
 // exportCoverageReport takes a previously performed source analysis and generates an HTML coverage report from it.
 // Returns an error if one occurs.
-func exportCoverageReport(sourceAnalysis *SourceAnalysis, outputPath string) error {
+func exportHtmlCoverageReport(sourceAnalysis *SourceAnalysis, outputPath string) error {
 	// Define mappings onto some useful variables/functions.
 	functionMap := template.FuncMap{
 		"timeNow": time.Now,
@@ -62,9 +143,9 @@ func exportCoverageReport(sourceAnalysis *SourceAnalysis, outputPath string) err
 			// Determine our precision string
 			formatStr := "%." + strconv.Itoa(decimals) + "f"
 
-			// If no lines are active and none are covered, show 0% coverage
+			// If no lines are active and none are covered, show 100% coverage
 			if x == 0 && y == 0 {
-				return fmt.Sprintf(formatStr, float64(0))
+				return fmt.Sprintf(formatStr, float64(100))
 			}
 			return fmt.Sprintf(formatStr, (float64(x)/float64(y))*100)
 		},

fuzzing/coverage/report_template.gojson

@@ -0,0 +1,15 @@
+[{{range $index, $sourceFile := .SortedFiles}}{{if $index}},{{end}}
+{
+"lines": {
+"found": {{$sourceFile.ActiveLineCount}},
+"hit": {{$sourceFile.CoveredLineCount}},
+"details": [{{$lastActive := lastActiveIndex $sourceFile}}{{range $lineIndex, $line := $sourceFile.Lines}}{{if $line.IsActive}}
+{
+"line": {{add $lineIndex 1}},
+"hit": {{if or $line.IsCovered $line.IsCoveredReverted}} 1 {{else}} 0 {{end}}
+}{{if ne $lineIndex $lastActive}},{{end}}{{end}}{{end}}
+]
+},
+"file": "{{relativePath $sourceFile.Path}}"
+}{{end}}
+]

fuzzing/executiontracer/execution_tracer.go

@@ -1,11 +1,11 @@
 package executiontracer
 
 import (
+	"github.com/crytic/medusa/utils"
 	"math/big"
 
 	"github.com/crytic/medusa/chain"
 	"github.com/crytic/medusa/fuzzing/contracts"
-	"github.com/crytic/medusa/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"