Merge pull request #508 from hashicorp/go118fuzz

Port fuzz testing to Go 1.18 native fuzzing

.github/workflows/push.yml

@@ -29,6 +29,10 @@ jobs:
           git config --global core.autocrlf false
       - name: "Fetch source code"
         uses: actions/checkout@v2
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18
       - name: Go test
         run: |
           go test ./...
@@ -40,6 +44,10 @@ jobs:
     steps:
       - name: "Fetch source code"
         uses: actions/checkout@v2
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18
       - name: "Check vet"
         run: |
           go vet ./...

hclsyntax/fuzz/README.md

@@ -1,85 +1,62 @@
 # hclsyntax fuzzing utilities
 
-This directory contains helper functions and corpuses that can be used to
-fuzz-test the `hclsyntax` parsers using [go-fuzz](https://github.com/dvyukov/go-fuzz).
+This directory contains helper functions and corpora that can be used to
+fuzz-test the `hclsyntax` parsers using Go's native fuzz testing capabilities.
 
-## Work directory
+Please see https://go.dev/doc/fuzz/ for more information on fuzzing.
 
-`go-fuzz` needs a working directory where it can keep state as it works.  This
-should ideally be in a ramdisk for efficiency, and should probably _not_ be on
-an SSD to avoid thrashing it. Here's how to create a ramdisk:
+## Prerequisites
+* Go 1.18
 
-### macOS
-
-```
-$ SIZE_IN_MB=1024
-$ DEVICE=`hdiutil attach -nobrowse -nomount ram://$(($SIZE_IN_MB*2048))`
-$ diskutil erasevolume HFS+ RamDisk $DEVICE
-$ export RAMDISK=/Volumes/RamDisk
-```
+## Running the fuzzer
 
-### Linux
+Each exported function in the `hclsyntax` package has a corresponding fuzz test.
+These can be run one at a time via `go test`:
 
 ```
-$ mkdir /mnt/ramdisk
-$ mount -t tmpfs -o size=1024M tmpfs /mnt/ramdisk
-$ export RAMDISK=/mnt/ramdisk
+$ cd fuzz
+$ go test -fuzz FuzzParseTemplate
+$ go test -fuzz FuzzParseTraversalAbs
+$ go test -fuzz FuzzParseExpression
+$ go test -fuzz FuzzParseConfig
 ```
 
-## Running the fuzzer
+This command will exit only when a crasher is found (see "Understanding the 
+result" below.)
+
+## Seed corpus
 
-Next, install `go-fuzz` and its build tool in your `GOPATH`:
+The seed corpus for each fuzz test function is stored in the corresponding
+directory under `hclsyntax/fuzz/testdata/fuzz/`. For example:
 
 ```
-$ make tools FUZZ_WORK_DIR=$RAMDISK
+$ ls hclsyntax/fuzz/testdata/fuzz/FuzzParseTemplate
+empty.tmpl
+escape-dollar.tmpl
+escape-newline-tmpl
+...
 ```
 
-Now you can fuzz one or all of the parsers:
+Additional seed inputs can be added to this corpus. Each file must be in the Go 1.18 corpus file format. Files can be converted to this format using the `file2fuzz` tool. To install it:
 
 ```
-$ make fuzz-config FUZZ_WORK_DIR=$RAMDISK/hclsyntax-fuzz-config
-$ make fuzz-expr FUZZ_WORK_DIR=$RAMDISK/hclsyntax-fuzz-expr
-$ make fuzz-template FUZZ_WORK_DIR=$RAMDISK/hclsyntax-fuzz-template
-$ make fuzz-traversal FUZZ_WORK_DIR=$RAMDISK/hclsyntax-fuzz-traversal
+$ go install golang.org/x/tools/cmd/file2fuzz@latest
+$ file2fuzz -help
 ```
 
-~> Note: `go-fuzz` does not interact well with `goenv`. If you encounter build
-errors where the package `go.fuzz.main` could not be found, you may need to use
-a machine with a direct installation of Go.
-
 ## Understanding the result
 
 A small number of subdirectories will be created in the work directory.
 
 If you let `go-fuzz` run for a few minutes (the more minutes the better) it
-may detect "crashers", which are inputs that caused the parser to panic. Details
-about these are written to `$FUZZ_WORK_DIR/crashers`:
+may detect "crashers", which are inputs that caused the parser to panic.
+These are written to `hclsyntax/fuzz/testdata/fuzz/<fuzz test name>/`:
 
 ```
-$ ls /tmp/hcl2-fuzz-config/crashers
-7f5e9ec80c89da14b8b0b238ec88969f658f5a2d
-7f5e9ec80c89da14b8b0b238ec88969f658f5a2d.output
-7f5e9ec80c89da14b8b0b238ec88969f658f5a2d.quoted
+$ ls hclsyntax/fuzz/testdata/fuzz/FuzzParseTemplate
+582528ddfad69eb57775199a43e0f9fd5c94bba343ce7bb6724d4ebafe311ed4
 ```
 
-The base file above (with no extension) is the input that caused a crash. The
-`.output` file contains the panic stack trace, which you can use as a clue to
-figure out what caused the crash.
-
 A good first step to fixing a detected crasher is to copy the failing input
 into one of the unit tests in the `hclsyntax` package and see it crash there
-too. After that, it's easy to re-run the test as you try to fix it. The
-file with the `.quoted` extension contains a form of the input that is quoted
-in Go syntax for easy copy-paste into a test case, even if the input contains
-non-printable characters or other inconvenient symbols.
-
-## Rebuilding for new Upstream Code
-
-An archive file is created for `go-fuzz` to use on the first run of each
-of the above, as a `.zip` file created in this directory. If upstream code
-is changed these will need to be deleted to cause them to be rebuilt with
-the latest code:
-
-```
-$ make clean
-```
+too. After that, it's easy to re-run the test as you try to fix it. 

hclsyntax/fuzz/fuzz_test.go

@@ -0,0 +1,60 @@
+package fuzzhclsyntax
+
+import (
+	"testing"
+
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/hclsyntax"
+)
+
+func FuzzParseTemplate(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		_, diags := hclsyntax.ParseTemplate(data, "<fuzz-tmpl>", hcl.Pos{Line: 1, Column: 1})
+
+		if diags.HasErrors() {
+			t.Logf("Error when parsing template %v", data)
+			for _, diag := range diags {
+				t.Logf("- %s", diag.Error())
+			}
+		}
+	})
+}
+
+func FuzzParseTraversalAbs(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		_, diags := hclsyntax.ParseTraversalAbs(data, "<fuzz-trav>", hcl.Pos{Line: 1, Column: 1})
+
+		if diags.HasErrors() {
+			t.Logf("Error when parsing traversal %v", data)
+			for _, diag := range diags {
+				t.Logf("- %s", diag.Error())
+			}
+		}
+	})
+}
+
+func FuzzParseExpression(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		_, diags := hclsyntax.ParseExpression(data, "<fuzz-expr>", hcl.Pos{Line: 1, Column: 1})
+
+		if diags.HasErrors() {
+			t.Logf("Error when parsing expression %v", data)
+			for _, diag := range diags {
+				t.Logf("- %s", diag.Error())
+			}
+		}
+	})
+}
+
+func FuzzParseConfig(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		_, diags := hclsyntax.ParseConfig(data, "<fuzz-conf>", hcl.Pos{Line: 1, Column: 1})
+
+		if diags.HasErrors() {
+			t.Logf("Error when parsing config %v", data)
+			for _, diag := range diags {
+				t.Logf("- %s", diag.Error())
+			}
+		}
+	})
+}

hclsyntax/fuzz/testdata/fuzz/FuzzParseConfig/attr-expr.hcl

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("foo = upper(bar + baz[1])\n")

hclsyntax/fuzz/testdata/fuzz/FuzzParseConfig/attr-literal.hcl

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("foo = \"bar\"\n")

hclsyntax/fuzz/testdata/fuzz/FuzzParseConfig/block-attrs.hcl

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("block {\n  foo = true\n}\n")

hclsyntax/fuzz/testdata/fuzz/FuzzParseConfig/block-empty.hcl

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("block {\n}\n")

hclsyntax/fuzz/testdata/fuzz/FuzzParseConfig/block-nested.hcl

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("block {\n  another_block {\n    foo = bar\n  }\n}\n")

hclsyntax/fuzz/testdata/fuzz/FuzzParseConfig/empty.hcl

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("")

hclsyntax/fuzz/testdata/fuzz/FuzzParseConfig/utf8.hcl

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("foo = \"föo ${föo(\"föo\")}\"\n")

hclsyntax/fuzz/testdata/fuzz/FuzzParseExpression/empty.hcle

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("\"\"")

hclsyntax/fuzz/testdata/fuzz/FuzzParseExpression/escape-dollar.hcle

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("\"hi $${var.foo}\"")

hclsyntax/fuzz/testdata/fuzz/FuzzParseExpression/escape-newline.hcle

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("\"bar\\nbaz\"\n")

hclsyntax/fuzz/testdata/fuzz/FuzzParseExpression/function-call.hcle

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("title(var.name)")

hclsyntax/fuzz/testdata/fuzz/FuzzParseExpression/int.hcle

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("42")

hclsyntax/fuzz/testdata/fuzz/FuzzParseExpression/literal.hcle

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("foo")

hclsyntax/fuzz/testdata/fuzz/FuzzParseExpression/splat-attr.hcle

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("foo.bar.*.baz\n")

hclsyntax/fuzz/testdata/fuzz/FuzzParseExpression/splat-full.hcle

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("foo.bar[*].baz\n")

hclsyntax/fuzz/testdata/fuzz/FuzzParseExpression/utf8.hcle

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("föo(\"föo\") + föo")

hclsyntax/fuzz/testdata/fuzz/FuzzParseExpression/var.hcle

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("var.bar")

hclsyntax/fuzz/testdata/fuzz/FuzzParseTemplate/empty.tmpl

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("")

hclsyntax/fuzz/testdata/fuzz/FuzzParseTemplate/escape-dollar.tmpl

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("hi $${var.foo}")

hclsyntax/fuzz/testdata/fuzz/FuzzParseTemplate/escape-newline.tmpl

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("foo ${\"bar\\nbaz\"}")