From 10954df1df3c4e737b33dcee7142c149c08bfdb0 Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Fri, 9 Jun 2023 09:48:01 +0200 Subject: [PATCH] Honor explicitly stated corpus directory The JUnit integration always created corpus files in .cifuzz-corpus subdirectories, even if an explicit corpus directory was stated via command line. Now explicit corpus parameters are honored. --- .../com/example/CorpusDirectoryFuzzTest.java | 45 +++++ .../jazzer/junit/FuzzTestExecutor.java | 13 +- .../jazzer/junit/BUILD.bazel | 34 ++++ .../jazzer/junit/CorpusDirectoryTest.java | 183 ++++++++++++++++++ .../corpusDirectoryFuzz/seed | 1 + 5 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 examples/junit/src/test/java/com/example/CorpusDirectoryFuzzTest.java create mode 100644 src/test/java/com/code_intelligence/jazzer/junit/CorpusDirectoryTest.java create mode 100644 src/test/java/com/code_intelligence/jazzer/junit/test_resources_root/com/example/CorpusDirectoryFuzzTestInputs/corpusDirectoryFuzz/seed diff --git a/examples/junit/src/test/java/com/example/CorpusDirectoryFuzzTest.java b/examples/junit/src/test/java/com/example/CorpusDirectoryFuzzTest.java new file mode 100644 index 000000000..465c94cf4 --- /dev/null +++ b/examples/junit/src/test/java/com/example/CorpusDirectoryFuzzTest.java @@ -0,0 +1,45 @@ +// Copyright 2023 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.example; + +import com.code_intelligence.jazzer.api.FuzzedDataProvider; +import com.code_intelligence.jazzer.api.FuzzerSecurityIssueMedium; +import com.code_intelligence.jazzer.junit.FuzzTest; + +public class CorpusDirectoryFuzzTest { + private static int invocations = 0; + + @FuzzTest(maxDuration = "5s") + public void corpusDirectoryFuzz(FuzzedDataProvider data) { + // Throw on the third invocation to generate corpus entries. + if (data.remainingBytes() == 0) { + return; + } + // Add a few branch statements to generate different coverage. + switch (invocations) { + case 0: + invocations++; + break; + case 1: + invocations++; + break; + case 2: + invocations++; + break; + case 3: + throw new FuzzerSecurityIssueMedium(); + } + } +} diff --git a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java index e20f511ab..49252e84a 100644 --- a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java +++ b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java @@ -41,6 +41,7 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import java.util.stream.Stream; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.ExtensionContext.Namespace; @@ -94,7 +95,17 @@ public static FuzzTestExecutor prepare(ExtensionContext context, String maxDurat ArrayList libFuzzerArgs = new ArrayList<>(); libFuzzerArgs.add(argv0); - // Store the generated corpus in a per-class directory under the project root, just like cifuzz: + // Add passed in corpus directories (and files) at the beginning of the arguments list. + // libFuzzer uses the first directory to store discovered inputs, whereas all others are + // only used to provide additional seeds and aren't written into. + List corpusDirs = originalLibFuzzerArgs.stream() + .filter(arg -> !arg.startsWith("-")) + .collect(Collectors.toList()); + originalLibFuzzerArgs.removeAll(corpusDirs); + libFuzzerArgs.addAll(corpusDirs); + + // Use the specified corpus dir, if given, otherwise store the generated corpus in a per-class + // directory under the project root, just like cifuzz: // https://github.com/CodeIntelligenceTesting/cifuzz/blob/bf410dcfbafbae2a73cf6c5fbed031cdfe234f2f/internal/cmd/run/run.go#L381 // The path is specified relative to the current working directory, which with JUnit is the // project directory. diff --git a/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel index dfcb2ec64..a9a5e2ea0 100644 --- a/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel +++ b/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel @@ -169,6 +169,40 @@ java_test( ] ] +[ + java_test( + name = "CorpusDirectoryTest" + JAZZER_FUZZ, + srcs = ["CorpusDirectoryTest.java"], + args = [ + # Add a test resource root containing the seed corpus directory in a Maven layout to + # the classpath rather than seeds in a resource directory packaged in a JAR, as + # would happen if we added the directory to java_test's resources. + "--main_advice_classpath=$(rootpath test_resources_root)", + ], + data = ["test_resources_root"], + env = { + "JAZZER_FUZZ": JAZZER_FUZZ, + }, + test_class = "com.code_intelligence.jazzer.junit.CorpusDirectoryTest", + runtime_deps = [ + "//examples/junit/src/test/java/com/example:ExampleFuzzTests_deploy.jar", + "@maven//:org_junit_jupiter_junit_jupiter_engine", + ], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/api:hooks", + "@maven//:com_google_truth_extensions_truth_java8_extension", + "@maven//:com_google_truth_truth", + "@maven//:junit_junit", + "@maven//:org_junit_platform_junit_platform_engine", + "@maven//:org_junit_platform_junit_platform_testkit", + ], + ) + for JAZZER_FUZZ in [ + "", + "_fuzzing", + ] +] + [ java_test( name = "AutofuzzTest" + JAZZER_FUZZ, diff --git a/src/test/java/com/code_intelligence/jazzer/junit/CorpusDirectoryTest.java b/src/test/java/com/code_intelligence/jazzer/junit/CorpusDirectoryTest.java new file mode 100644 index 000000000..372718efb --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/CorpusDirectoryTest.java @@ -0,0 +1,183 @@ +// Copyright 2023 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; +import static org.junit.platform.testkit.engine.EventConditions.container; +import static org.junit.platform.testkit.engine.EventConditions.displayName; +import static org.junit.platform.testkit.engine.EventConditions.event; +import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; +import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; +import static org.junit.platform.testkit.engine.EventConditions.test; +import static org.junit.platform.testkit.engine.EventConditions.type; +import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings; +import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED; +import static org.junit.platform.testkit.engine.EventType.FINISHED; +import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED; +import static org.junit.platform.testkit.engine.EventType.STARTED; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; + +import com.code_intelligence.jazzer.api.FuzzerSecurityIssueMedium; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.Stream; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.platform.testkit.engine.EngineExecutionResults; +import org.junit.platform.testkit.engine.EngineTestKit; +import org.junit.rules.TemporaryFolder; + +public class CorpusDirectoryTest { + private static final String ENGINE = "engine:junit-jupiter"; + private static final String CLAZZ = "class:com.example.CorpusDirectoryFuzzTest"; + private static final String INPUTS_FUZZ = + "test-template:corpusDirectoryFuzz(com.code_intelligence.jazzer.api.FuzzedDataProvider)"; + private static final String INVOCATION = "test-template-invocation:#"; + + @Rule public TemporaryFolder temp = new TemporaryFolder(); + Path baseDir; + + @Before + public void setup() { + baseDir = temp.getRoot().toPath(); + } + + @Test + public void fuzzingEnabled() throws IOException { + assumeFalse(System.getenv("JAZZER_FUZZ").isEmpty()); + + // Create a fake test resource directory structure with an inputs directory to verify that + // Jazzer uses it and emits a crash file into it. + Path artifactsDirectory = baseDir.resolve(Paths.get("src", "test", "resources", "com", + "example", "CorpusDirectoryFuzzTestInputs", "corpusDirectoryFuzz")); + Files.createDirectories(artifactsDirectory); + + // An explicitly stated corpus directory should be used to save new corpus entries. + Path explicitGeneratedCorpus = baseDir.resolve(Paths.get("corpus")); + Files.createDirectories(explicitGeneratedCorpus); + + // The default generated corpus directory should only be used if no explicit corpus directory + // is given. + Path defaultGeneratedCorpus = baseDir.resolve( + Paths.get(".cifuzz-corpus", "com.example.CorpusDirectoryFuzzTest", "corpusDirectoryFuzz")); + + EngineExecutionResults results = + EngineTestKit.engine("junit-jupiter") + .selectors(selectClass("com.example.CorpusDirectoryFuzzTest")) + .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) + // Add corpus directory as initial libFuzzer parameter. + .configurationParameter("jazzer.internal.arg.0", "fake_test_argv0") + .configurationParameter( + "jazzer.internal.arg.1", explicitGeneratedCorpus.toAbsolutePath().toString()) + .execute(); + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ)), + finishedSuccessfully()), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + results.testEvents().assertEventsMatchExactly( + event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1))), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1)), + displayName(""), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2))), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2)), + displayName("seed"), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 3)), + displayName("Fuzzing...")), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 3)), + displayName("Fuzzing..."), + finishedWithFailure(instanceOf(FuzzerSecurityIssueMedium.class)))); + + // Crash file should be emitted into the artifacts directory and not into corpus directory. + assertCrashFileExistsIn(artifactsDirectory); + assertNoCrashFileExistsIn(baseDir); + assertNoCrashFileExistsIn(explicitGeneratedCorpus); + assertNoCrashFileExistsIn(defaultGeneratedCorpus); + + // Verify that corpus files are written to given corpus directory and not generated one. + assertThat(Files.list(explicitGeneratedCorpus)).isNotEmpty(); + assertThat(Files.list(defaultGeneratedCorpus)).isEmpty(); + } + + @Test + public void fuzzingDisabled() throws IOException { + assumeTrue(System.getenv("JAZZER_FUZZ").isEmpty()); + + Path corpusDirectory = baseDir.resolve(Paths.get("corpus")); + Files.createDirectories(corpusDirectory); + Files.createFile(corpusDirectory.resolve("corpus_entry")); + + EngineExecutionResults results = + EngineTestKit.engine("junit-jupiter") + .selectors(selectClass("com.example.CorpusDirectoryFuzzTest")) + .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) + // Add corpus directory as initial libFuzzer parameter. + .configurationParameter("jazzer.internal.arg.0", "fake_test_argv0") + .configurationParameter( + "jazzer.internal.arg.1", corpusDirectory.toAbsolutePath().toString()) + .execute(); + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + // Verify that corpus_entry is not picked up and corpus directory is ignored in regression mode. + results.testEvents().assertEventsMatchExactly( + event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1))), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1)), + displayName(""), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2))), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2)), + displayName("seed"), finishedSuccessfully())); + } + + private static void assertCrashFileExistsIn(Path artifactsDirectory) throws IOException { + try (Stream crashFiles = + Files.list(artifactsDirectory) + .filter(path -> path.getFileName().toString().startsWith("crash-"))) { + assertThat(crashFiles).isNotEmpty(); + } + } + + private static void assertNoCrashFileExistsIn(Path generatedCorpus) throws IOException { + try (Stream crashFiles = + Files.list(generatedCorpus) + .filter(path -> path.getFileName().toString().startsWith("crash-"))) { + assertThat(crashFiles).isEmpty(); + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/junit/test_resources_root/com/example/CorpusDirectoryFuzzTestInputs/corpusDirectoryFuzz/seed b/src/test/java/com/code_intelligence/jazzer/junit/test_resources_root/com/example/CorpusDirectoryFuzzTestInputs/corpusDirectoryFuzz/seed new file mode 100644 index 000000000..e31de1f3a --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/test_resources_root/com/example/CorpusDirectoryFuzzTestInputs/corpusDirectoryFuzz/seed @@ -0,0 +1 @@ +seed