diff --git a/libfuzzer/CMakeLists.txt b/libfuzzer/CMakeLists.txt index 679318e..852019d 100644 --- a/libfuzzer/CMakeLists.txt +++ b/libfuzzer/CMakeLists.txt @@ -3,9 +3,10 @@ set(LIBFUZZER_SOURCES FuzzerDataFlowTrace.cpp FuzzerDriver.cpp FuzzerExtFunctionsDlsym.cpp - FuzzerExtFunctionsDlsymWin.cpp FuzzerExtFunctionsWeak.cpp + FuzzerExtFunctionsWindows.cpp FuzzerExtraCounters.cpp + FuzzerFork.cpp FuzzerIO.cpp FuzzerIOPosix.cpp FuzzerIOWindows.cpp @@ -13,9 +14,6 @@ set(LIBFUZZER_SOURCES FuzzerMerge.cpp FuzzerMutate.cpp FuzzerSHA1.cpp - FuzzerShmemFuchsia.cpp - FuzzerShmemPosix.cpp - FuzzerShmemWindows.cpp FuzzerTracePC.cpp FuzzerUtil.cpp FuzzerUtilDarwin.cpp @@ -25,6 +23,8 @@ set(LIBFUZZER_SOURCES FuzzerUtilWindows.cpp) set(LIBFUZZER_HEADERS + FuzzerBuiltins.h + FuzzerBuiltinsMsvc.h FuzzerCommand.h FuzzerCorpus.h FuzzerDataFlowTrace.h @@ -33,6 +33,7 @@ set(LIBFUZZER_HEADERS FuzzerExtFunctions.def FuzzerExtFunctions.h FuzzerFlags.def + FuzzerFork.h FuzzerIO.h FuzzerInterface.h FuzzerInternal.h @@ -41,7 +42,6 @@ set(LIBFUZZER_HEADERS FuzzerOptions.h FuzzerRandom.h FuzzerSHA1.h - FuzzerShmem.h FuzzerTracePC.h FuzzerUtil.h FuzzerValueBitMap.h) @@ -55,7 +55,9 @@ CHECK_CXX_SOURCE_COMPILES(" set(LIBFUZZER_CFLAGS ${SANITIZER_COMMON_CFLAGS}) -if(OS_NAME MATCHES "Linux|Fuchsia" AND COMPILER_RT_LIBCXX_PATH) +if(OS_NAME MATCHES "Linux|Fuchsia" AND + COMPILER_RT_LIBCXX_PATH AND + COMPILER_RT_LIBCXXABI_PATH) list(APPEND LIBFUZZER_CFLAGS -nostdinc++ -D_LIBCPP_ABI_VERSION=Fuzzer) # Remove -stdlib= which is unused when passing -nostdinc++. string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) @@ -69,12 +71,21 @@ if (CMAKE_CXX_FLAGS MATCHES "fsanitize-coverage") list(APPEND LIBFUZZER_CFLAGS -fno-sanitize-coverage=trace-pc-guard,edge,trace-cmp,indirect-calls,8bit-counters) endif() -if(NOT HAS_THREAD_LOCAL) - list(APPEND LIBFUZZER_CFLAGS -Dthread_local=__thread) +if(MSVC) + # Silence warnings by turning off exceptions in MSVC headers and avoid an + # error by unecessarily defining thread_local when it isn't even used on + # Windows. + list(APPEND LIBFUZZER_CFLAGS -D_HAS_EXCEPTIONS=0) +else() + if(NOT HAS_THREAD_LOCAL) + list(APPEND LIBFUZZER_CFLAGS -Dthread_local=__thread) + endif() endif() set(FUZZER_SUPPORTED_OS ${SANITIZER_COMMON_SUPPORTED_OS}) +add_compiler_rt_component(fuzzer) + add_compiler_rt_object_libraries(RTfuzzer OS ${FUZZER_SUPPORTED_OS} ARCHS ${FUZZER_SUPPORTED_ARCH} @@ -106,12 +117,19 @@ add_compiler_rt_runtime(clang_rt.fuzzer_no_main CFLAGS ${LIBFUZZER_CFLAGS} PARENT_TARGET fuzzer) -if(OS_NAME MATCHES "Linux|Fuchsia" AND COMPILER_RT_LIBCXX_PATH) +if(OS_NAME MATCHES "Linux|Fuchsia" AND + COMPILER_RT_LIBCXX_PATH AND + COMPILER_RT_LIBCXXABI_PATH) macro(partially_link_libcxx name dir arch) + if(${arch} MATCHES "i386") + set(EMULATION_ARGUMENT "-m" "elf_i386") + else() + set(EMULATION_ARGUMENT "") + endif() set(cxx_${arch}_merge_dir "${CMAKE_CURRENT_BINARY_DIR}/cxx_${arch}_merge.dir") file(MAKE_DIRECTORY ${cxx_${arch}_merge_dir}) add_custom_command(TARGET clang_rt.${name}-${arch} POST_BUILD - COMMAND ${CMAKE_LINKER} --whole-archive "$" --no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o + COMMAND ${CMAKE_LINKER} ${EMULATION_ARGUMENT} --whole-archive "$" --no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o COMMAND ${CMAKE_OBJCOPY} --localize-hidden ${name}.o COMMAND ${CMAKE_COMMAND} -E remove "$" COMMAND ${CMAKE_AR} qcs "$" ${name}.o @@ -124,13 +142,8 @@ if(OS_NAME MATCHES "Linux|Fuchsia" AND COMPILER_RT_LIBCXX_PATH) set(LIBCXX_${arch}_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libcxx_fuzzer_${arch}) add_custom_libcxx(libcxx_fuzzer_${arch} ${LIBCXX_${arch}_PREFIX} CFLAGS ${TARGET_CFLAGS} - -D_LIBCPP_ABI_VERSION=Fuzzer - -D_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS=1 - -fvisibility=hidden CMAKE_ARGS -DCMAKE_CXX_COMPILER_WORKS=ON - -DLIBCXX_ENABLE_EXCEPTIONS=OFF - -DLIBCXX_ENABLE_SHARED=OFF - -DLIBCXX_CXX_ABI=none) + -DLIBCXX_ABI_NAMESPACE=Fuzzer) target_compile_options(RTfuzzer.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1) add_dependencies(RTfuzzer.${arch} libcxx_fuzzer_${arch}-build) target_compile_options(RTfuzzer_main.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1) diff --git a/libfuzzer/FuzzerBuiltins.h b/libfuzzer/FuzzerBuiltins.h new file mode 100644 index 0000000..5f1ccef --- /dev/null +++ b/libfuzzer/FuzzerBuiltins.h @@ -0,0 +1,35 @@ +//===- FuzzerBuiltins.h - Internal header for builtins ----------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Wrapper functions and marcos around builtin functions. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_BUILTINS_H +#define LLVM_FUZZER_BUILTINS_H + +#include "FuzzerDefs.h" + +#if !LIBFUZZER_MSVC +#include + +#define GET_CALLER_PC() __builtin_return_address(0) + +namespace fuzzer { + +inline uint8_t Bswap(uint8_t x) { return x; } +inline uint16_t Bswap(uint16_t x) { return __builtin_bswap16(x); } +inline uint32_t Bswap(uint32_t x) { return __builtin_bswap32(x); } +inline uint64_t Bswap(uint64_t x) { return __builtin_bswap64(x); } + +inline uint32_t Clzll(unsigned long long X) { return __builtin_clzll(X); } +inline uint32_t Clz(unsigned long long X) { return __builtin_clz(X); } +inline int Popcountll(unsigned long long X) { return __builtin_popcountll(X); } + +} // namespace fuzzer + +#endif // !LIBFUZZER_MSVC +#endif // LLVM_FUZZER_BUILTINS_H diff --git a/libfuzzer/FuzzerBuiltinsMsvc.h b/libfuzzer/FuzzerBuiltinsMsvc.h new file mode 100644 index 0000000..82709cf --- /dev/null +++ b/libfuzzer/FuzzerBuiltinsMsvc.h @@ -0,0 +1,58 @@ +//===- FuzzerBuiltinsMSVC.h - Internal header for builtins ------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Wrapper functions and marcos that use intrinsics instead of builtin functions +// which cannot be compiled by MSVC. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_BUILTINS_MSVC_H +#define LLVM_FUZZER_BUILTINS_MSVC_H + +#include "FuzzerDefs.h" + +#if LIBFUZZER_MSVC +#if !defined(_M_ARM) && !defined(_M_X64) +#error "_BitScanReverse64 unavailable on this platform so MSVC is unsupported." +#endif +#include +#include +#include + +// __builtin_return_address() cannot be compiled with MSVC. Use the equivalent +// from +#define GET_CALLER_PC() _ReturnAddress() + +namespace fuzzer { + +inline uint8_t Bswap(uint8_t x) { return x; } +// Use alternatives to __builtin functions from and on +// Windows since the builtins are not supported by MSVC. +inline uint16_t Bswap(uint16_t x) { return _byteswap_ushort(x); } +inline uint32_t Bswap(uint32_t x) { return _byteswap_ulong(x); } +inline uint64_t Bswap(uint64_t x) { return _byteswap_uint64(x); } + +// The functions below were mostly copied from +// compiler-rt/lib/builtins/int_lib.h which defines the __builtin functions used +// outside of Windows. +inline uint32_t Clzll(uint64_t X) { + unsigned long LeadZeroIdx = 0; + if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx; + return 64; +} + +inline uint32_t Clz(uint32_t X) { + unsigned long LeadZeroIdx = 0; + if (_BitScanReverse(&LeadZeroIdx, X)) return 31 - LeadZeroIdx; + return 32; +} + +inline int Popcountll(unsigned long long X) { return __popcnt64(X); } + +} // namespace fuzzer + +#endif // LIBFUZER_MSVC +#endif // LLVM_FUZZER_BUILTINS_MSVC_H diff --git a/libfuzzer/FuzzerCommand.h b/libfuzzer/FuzzerCommand.h index 255f571..8730886 100644 --- a/libfuzzer/FuzzerCommand.h +++ b/libfuzzer/FuzzerCommand.h @@ -1,9 +1,8 @@ //===- FuzzerCommand.h - Interface representing a process -------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // FuzzerCommand represents a command to run in a subprocess. It allows callers @@ -81,7 +80,7 @@ class Command final { } // Like hasArgument, but checks for "-[Flag]=...". - bool hasFlag(const std::string &Flag) { + bool hasFlag(const std::string &Flag) const { std::string Arg("-" + Flag + "="); auto IsMatch = [&](const std::string &Other) { return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0; @@ -92,7 +91,7 @@ class Command final { // Returns the value of the first instance of a given flag, or an empty string // if the flag isn't present. Ignores any occurrences after // "-ignore_remaining_args=1", if present. - std::string getFlagValue(const std::string &Flag) { + std::string getFlagValue(const std::string &Flag) const { std::string Arg("-" + Flag + "="); auto IsMatch = [&](const std::string &Other) { return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0; diff --git a/libfuzzer/FuzzerCorpus.h b/libfuzzer/FuzzerCorpus.h index 8ad1465..6a95ef3 100644 --- a/libfuzzer/FuzzerCorpus.h +++ b/libfuzzer/FuzzerCorpus.h @@ -1,9 +1,8 @@ //===- FuzzerCorpus.h - Internal header for the Fuzzer ----------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::InputCorpus @@ -86,9 +85,10 @@ class InputCorpus { bool empty() const { return Inputs.empty(); } const Unit &operator[] (size_t Idx) const { return Inputs[Idx]->U; } - void AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile, - bool HasFocusFunction, const Vector &FeatureSet, - const DataFlowTrace &DFT, const InputInfo *BaseII) { + InputInfo *AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile, + bool HasFocusFunction, + const Vector &FeatureSet, + const DataFlowTrace &DFT, const InputInfo *BaseII) { assert(!U.empty()); if (FeatureDebug) Printf("ADD_TO_CORPUS %zd NF %zd\n", Inputs.size(), NumFeatures); @@ -114,6 +114,7 @@ class InputCorpus { UpdateCorpusDistribution(); PrintCorpus(); // ValidateFeatureSet(); + return &II; } // Debug-only @@ -170,7 +171,7 @@ class InputCorpus { InputInfo &II = *Inputs[ChooseUnitIdxToMutate(Rand)]; assert(!II.U.empty()); return II; - }; + } // Returns an index of random unit from the corpus to mutate. size_t ChooseUnitIdxToMutate(Random &Rand) { @@ -238,12 +239,6 @@ class InputCorpus { return false; } - bool IsFeatureNew(size_t Idx, uint32_t NewSize, bool Shrink) { - assert(NewSize); - uint32_t OldSize = GetFeature(Idx % kFeatureSetSize); - return OldSize == 0 || (Shrink && OldSize > NewSize); - } - size_t NumFeatures() const { return NumAddedFeatures; } size_t NumFeatureUpdates() const { return NumUpdatedFeatures; } diff --git a/libfuzzer/FuzzerCrossOver.cpp b/libfuzzer/FuzzerCrossOver.cpp index 8b0fd7d..83d9f8d 100644 --- a/libfuzzer/FuzzerCrossOver.cpp +++ b/libfuzzer/FuzzerCrossOver.cpp @@ -1,9 +1,8 @@ //===- FuzzerCrossOver.cpp - Cross over two test inputs -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Cross over test inputs. diff --git a/libfuzzer/FuzzerDataFlowTrace.cpp b/libfuzzer/FuzzerDataFlowTrace.cpp index 764f3e4..99ff918 100644 --- a/libfuzzer/FuzzerDataFlowTrace.cpp +++ b/libfuzzer/FuzzerDataFlowTrace.cpp @@ -1,91 +1,281 @@ //===- FuzzerDataFlowTrace.cpp - DataFlowTrace ---*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::DataFlowTrace //===----------------------------------------------------------------------===// #include "FuzzerDataFlowTrace.h" + +#include "FuzzerCommand.h" #include "FuzzerIO.h" +#include "FuzzerRandom.h" +#include "FuzzerSHA1.h" +#include "FuzzerUtil.h" #include #include +#include +#include +#include #include +#include +#include #include namespace fuzzer { +static const char *kFunctionsTxt = "functions.txt"; + +bool BlockCoverage::AppendCoverage(const std::string &S) { + std::stringstream SS(S); + return AppendCoverage(SS); +} + +// Coverage lines have this form: +// CN X Y Z T +// where N is the number of the function, T is the total number of instrumented +// BBs, and X,Y,Z, if present, are the indecies of covered BB. +// BB #0, which is the entry block, is not explicitly listed. +bool BlockCoverage::AppendCoverage(std::istream &IN) { + std::string L; + while (std::getline(IN, L, '\n')) { + if (L.empty()) + continue; + std::stringstream SS(L.c_str() + 1); + size_t FunctionId = 0; + SS >> FunctionId; + if (L[0] == 'F') { + FunctionsWithDFT.insert(FunctionId); + continue; + } + if (L[0] != 'C') continue; + Vector CoveredBlocks; + while (true) { + uint32_t BB = 0; + SS >> BB; + if (!SS) break; + CoveredBlocks.push_back(BB); + } + if (CoveredBlocks.empty()) return false; + uint32_t NumBlocks = CoveredBlocks.back(); + CoveredBlocks.pop_back(); + for (auto BB : CoveredBlocks) + if (BB >= NumBlocks) return false; + auto It = Functions.find(FunctionId); + auto &Counters = + It == Functions.end() + ? Functions.insert({FunctionId, Vector(NumBlocks)}) + .first->second + : It->second; + + if (Counters.size() != NumBlocks) return false; // wrong number of blocks. + + Counters[0]++; + for (auto BB : CoveredBlocks) + Counters[BB]++; + } + return true; +} + +// Assign weights to each function. +// General principles: +// * any uncovered function gets weight 0. +// * a function with lots of uncovered blocks gets bigger weight. +// * a function with a less frequently executed code gets bigger weight. +Vector BlockCoverage::FunctionWeights(size_t NumFunctions) const { + Vector Res(NumFunctions); + for (auto It : Functions) { + auto FunctionID = It.first; + auto Counters = It.second; + assert(FunctionID < NumFunctions); + auto &Weight = Res[FunctionID]; + // Give higher weight if the function has a DFT. + Weight = FunctionsWithDFT.count(FunctionID) ? 1000. : 1; + // Give higher weight to functions with less frequently seen basic blocks. + Weight /= SmallestNonZeroCounter(Counters); + // Give higher weight to functions with the most uncovered basic blocks. + Weight *= NumberOfUncoveredBlocks(Counters) + 1; + } + return Res; +} + +void DataFlowTrace::ReadCoverage(const std::string &DirPath) { + Vector Files; + GetSizedFilesFromDir(DirPath, &Files); + for (auto &SF : Files) { + auto Name = Basename(SF.File); + if (Name == kFunctionsTxt) continue; + if (!CorporaHashes.count(Name)) continue; + std::ifstream IF(SF.File); + Coverage.AppendCoverage(IF); + } +} -void DataFlowTrace::Init(const std::string &DirPath, - const std::string &FocusFunction) { - if (DirPath.empty()) return; - const char *kFunctionsTxt = "functions.txt"; +static void DFTStringAppendToVector(Vector *DFT, + const std::string &DFTString) { + assert(DFT->size() == DFTString.size()); + for (size_t I = 0, Len = DFT->size(); I < Len; I++) + (*DFT)[I] = DFTString[I] == '1'; +} + +// converts a string of '0' and '1' into a Vector +static Vector DFTStringToVector(const std::string &DFTString) { + Vector DFT(DFTString.size()); + DFTStringAppendToVector(&DFT, DFTString); + return DFT; +} + +static bool ParseError(const char *Err, const std::string &Line) { + Printf("DataFlowTrace: parse error: %s: Line: %s\n", Err, Line.c_str()); + return false; +} + +// TODO(metzman): replace std::string with std::string_view for +// better performance. Need to figure our how to use string_view on Windows. +static bool ParseDFTLine(const std::string &Line, size_t *FunctionNum, + std::string *DFTString) { + if (!Line.empty() && Line[0] != 'F') + return false; // Ignore coverage. + size_t SpacePos = Line.find(' '); + if (SpacePos == std::string::npos) + return ParseError("no space in the trace line", Line); + if (Line.empty() || Line[0] != 'F') + return ParseError("the trace line doesn't start with 'F'", Line); + *FunctionNum = std::atol(Line.c_str() + 1); + const char *Beg = Line.c_str() + SpacePos + 1; + const char *End = Line.c_str() + Line.size(); + assert(Beg < End); + size_t Len = End - Beg; + for (size_t I = 0; I < Len; I++) { + if (Beg[I] != '0' && Beg[I] != '1') + return ParseError("the trace should contain only 0 or 1", Line); + } + *DFTString = Beg; + return true; +} + +bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, + Vector &CorporaFiles, Random &Rand) { + if (DirPath.empty()) return false; Printf("INFO: DataFlowTrace: reading from '%s'\n", DirPath.c_str()); Vector Files; GetSizedFilesFromDir(DirPath, &Files); std::string L; + size_t FocusFuncIdx = SIZE_MAX; + Vector FunctionNames; + + // Collect the hashes of the corpus files. + for (auto &SF : CorporaFiles) + CorporaHashes.insert(Hash(FileToVector(SF.File))); // Read functions.txt std::ifstream IF(DirPlusFile(DirPath, kFunctionsTxt)); - size_t FocusFuncIdx = SIZE_MAX; size_t NumFunctions = 0; while (std::getline(IF, L, '\n')) { + FunctionNames.push_back(L); NumFunctions++; - if (FocusFunction == L) + if (*FocusFunction == L) FocusFuncIdx = NumFunctions - 1; } + if (!NumFunctions) + return false; + + if (*FocusFunction == "auto") { + // AUTOFOCUS works like this: + // * reads the coverage data from the DFT files. + // * assigns weights to functions based on coverage. + // * chooses a random function according to the weights. + ReadCoverage(DirPath); + auto Weights = Coverage.FunctionWeights(NumFunctions); + Vector Intervals(NumFunctions + 1); + std::iota(Intervals.begin(), Intervals.end(), 0); + auto Distribution = std::piecewise_constant_distribution( + Intervals.begin(), Intervals.end(), Weights.begin()); + FocusFuncIdx = static_cast(Distribution(Rand)); + *FocusFunction = FunctionNames[FocusFuncIdx]; + assert(FocusFuncIdx < NumFunctions); + Printf("INFO: AUTOFOCUS: %zd %s\n", FocusFuncIdx, + FunctionNames[FocusFuncIdx].c_str()); + for (size_t i = 0; i < NumFunctions; i++) { + if (!Weights[i]) continue; + Printf(" [%zd] W %g\tBB-tot %u\tBB-cov %u\tEntryFreq %u:\t%s\n", i, + Weights[i], Coverage.GetNumberOfBlocks(i), + Coverage.GetNumberOfCoveredBlocks(i), Coverage.GetCounter(i, 0), + FunctionNames[i].c_str()); + } + } + if (!NumFunctions || FocusFuncIdx == SIZE_MAX || Files.size() <= 1) - return; + return false; + // Read traces. size_t NumTraceFiles = 0; size_t NumTracesWithFocusFunction = 0; for (auto &SF : Files) { auto Name = Basename(SF.File); if (Name == kFunctionsTxt) continue; - auto ParseError = [&](const char *Err) { - Printf("DataFlowTrace: parse error: %s\n File: %s\n Line: %s\n", Err, - Name.c_str(), L.c_str()); - }; + if (!CorporaHashes.count(Name)) continue; // not in the corpus. NumTraceFiles++; // Printf("=== %s\n", Name.c_str()); std::ifstream IF(SF.File); while (std::getline(IF, L, '\n')) { - size_t SpacePos = L.find(' '); - if (SpacePos == std::string::npos) - return ParseError("no space in the trace line"); - if (L.empty() || L[0] != 'F') - return ParseError("the trace line doesn't start with 'F'"); - size_t N = std::atol(L.c_str() + 1); - if (N >= NumFunctions) - return ParseError("N is greater than the number of functions"); - if (N == FocusFuncIdx) { + size_t FunctionNum = 0; + std::string DFTString; + if (ParseDFTLine(L, &FunctionNum, &DFTString) && + FunctionNum == FocusFuncIdx) { NumTracesWithFocusFunction++; - const char *Beg = L.c_str() + SpacePos + 1; - const char *End = L.c_str() + L.size(); - assert(Beg < End); - size_t Len = End - Beg; - Vector V(Len); - for (size_t I = 0; I < Len; I++) { - if (Beg[I] != '0' && Beg[I] != '1') - ParseError("the trace should contain only 0 or 1"); - V[I] = Beg[I] == '1'; - } - Traces[Name] = V; + + if (FunctionNum >= NumFunctions) + return ParseError("N is greater than the number of functions", L); + Traces[Name] = DFTStringToVector(DFTString); // Print just a few small traces. - if (NumTracesWithFocusFunction <= 3 && Len <= 16) - Printf("%s => |%s|\n", Name.c_str(), L.c_str() + SpacePos + 1); - break; // No need to parse the following lines. + if (NumTracesWithFocusFunction <= 3 && DFTString.size() <= 16) + Printf("%s => |%s|\n", Name.c_str(), std::string(DFTString).c_str()); + break; // No need to parse the following lines. } } } - assert(NumTraceFiles == Files.size() - 1); Printf("INFO: DataFlowTrace: %zd trace files, %zd functions, " "%zd traces with focus function\n", NumTraceFiles, NumFunctions, NumTracesWithFocusFunction); + return NumTraceFiles > 0; } -} // namespace fuzzer +int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, + const Vector &CorporaFiles) { + Printf("INFO: collecting data flow: bin: %s dir: %s files: %zd\n", + DFTBinary.c_str(), DirPath.c_str(), CorporaFiles.size()); + static char DFSanEnv[] = "DFSAN_OPTIONS=fast16labels=1:warn_unimplemented=0"; + putenv(DFSanEnv); + MkDir(DirPath); + for (auto &F : CorporaFiles) { + // For every input F we need to collect the data flow and the coverage. + // Data flow collection may fail if we request too many DFSan tags at once. + // So, we start from requesting all tags in range [0,Size) and if that fails + // we then request tags in [0,Size/2) and [Size/2, Size), and so on. + // Function number => DFT. + auto OutPath = DirPlusFile(DirPath, Hash(FileToVector(F.File))); + std::unordered_map> DFTMap; + std::unordered_set Cov; + Command Cmd; + Cmd.addArgument(DFTBinary); + Cmd.addArgument(F.File); + Cmd.addArgument(OutPath); + Printf("CMD: %s\n", Cmd.toString().c_str()); + ExecuteCommand(Cmd); + } + // Write functions.txt if it's currently empty or doesn't exist. + auto FunctionsTxtPath = DirPlusFile(DirPath, kFunctionsTxt); + if (FileToString(FunctionsTxtPath).empty()) { + Command Cmd; + Cmd.addArgument(DFTBinary); + Cmd.setOutputFile(FunctionsTxtPath); + ExecuteCommand(Cmd); + } + return 0; +} +} // namespace fuzzer diff --git a/libfuzzer/FuzzerDataFlowTrace.h b/libfuzzer/FuzzerDataFlowTrace.h index ad4faea..d6e3de3 100644 --- a/libfuzzer/FuzzerDataFlowTrace.h +++ b/libfuzzer/FuzzerDataFlowTrace.h @@ -1,9 +1,8 @@ //===- FuzzerDataFlowTrace.h - Internal header for the Fuzzer ---*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::DataFlowTrace; reads and handles a data-flow trace. @@ -30,15 +29,93 @@ #define LLVM_FUZZER_DATA_FLOW_TRACE #include "FuzzerDefs.h" +#include "FuzzerIO.h" #include +#include #include #include namespace fuzzer { + +int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, + const Vector &CorporaFiles); + +class BlockCoverage { + public: + bool AppendCoverage(std::istream &IN); + bool AppendCoverage(const std::string &S); + + size_t NumCoveredFunctions() const { return Functions.size(); } + + uint32_t GetCounter(size_t FunctionId, size_t BasicBlockId) { + auto It = Functions.find(FunctionId); + if (It == Functions.end()) return 0; + const auto &Counters = It->second; + if (BasicBlockId < Counters.size()) + return Counters[BasicBlockId]; + return 0; + } + + uint32_t GetNumberOfBlocks(size_t FunctionId) { + auto It = Functions.find(FunctionId); + if (It == Functions.end()) return 0; + const auto &Counters = It->second; + return Counters.size(); + } + + uint32_t GetNumberOfCoveredBlocks(size_t FunctionId) { + auto It = Functions.find(FunctionId); + if (It == Functions.end()) return 0; + const auto &Counters = It->second; + uint32_t Result = 0; + for (auto Cnt: Counters) + if (Cnt) + Result++; + return Result; + } + + Vector FunctionWeights(size_t NumFunctions) const; + void clear() { Functions.clear(); } + + private: + + typedef Vector CoverageVector; + + uint32_t NumberOfCoveredBlocks(const CoverageVector &Counters) const { + uint32_t Res = 0; + for (auto Cnt : Counters) + if (Cnt) + Res++; + return Res; + } + + uint32_t NumberOfUncoveredBlocks(const CoverageVector &Counters) const { + return Counters.size() - NumberOfCoveredBlocks(Counters); + } + + uint32_t SmallestNonZeroCounter(const CoverageVector &Counters) const { + assert(!Counters.empty()); + uint32_t Res = Counters[0]; + for (auto Cnt : Counters) + if (Cnt) + Res = Min(Res, Cnt); + assert(Res); + return Res; + } + + // Function ID => vector of counters. + // Each counter represents how many input files trigger the given basic block. + std::unordered_map Functions; + // Functions that have DFT entry. + std::unordered_set FunctionsWithDFT; +}; + class DataFlowTrace { public: - void Init(const std::string &DirPath, const std::string &FocusFunction); + void ReadCoverage(const std::string &DirPath); + bool Init(const std::string &DirPath, std::string *FocusFunction, + Vector &CorporaFiles, Random &Rand); void Clear() { Traces.clear(); } const Vector *Get(const std::string &InputSha1) const { auto It = Traces.find(InputSha1); @@ -50,6 +127,8 @@ class DataFlowTrace { private: // Input's sha1 => DFT for the FocusFunction. std::unordered_map > Traces; + BlockCoverage Coverage; + std::unordered_set CorporaHashes; }; } // namespace fuzzer diff --git a/libfuzzer/FuzzerDefs.h b/libfuzzer/FuzzerDefs.h index a35c7a1..320b37d 100644 --- a/libfuzzer/FuzzerDefs.h +++ b/libfuzzer/FuzzerDefs.h @@ -1,9 +1,8 @@ //===- FuzzerDefs.h - Internal header for the Fuzzer ------------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Basic definitions. @@ -82,6 +81,13 @@ #error "Support for your platform has not been implemented" #endif +#if defined(_MSC_VER) && !defined(__clang__) +// MSVC compiler is being used. +#define LIBFUZZER_MSVC 1 +#else +#define LIBFUZZER_MSVC 0 +#endif + #ifndef __has_attribute # define __has_attribute(x) 0 #endif @@ -113,7 +119,28 @@ # define ALWAYS_INLINE #endif // __clang__ +#if LIBFUZZER_WINDOWS +#define ATTRIBUTE_NO_SANITIZE_ADDRESS +#else #define ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) +#endif + +#if LIBFUZZER_WINDOWS +#define ATTRIBUTE_ALIGNED(X) __declspec(align(X)) +#define ATTRIBUTE_INTERFACE __declspec(dllexport) +// This is used for __sancov_lowest_stack which is needed for +// -fsanitize-coverage=stack-depth. That feature is not yet available on +// Windows, so make the symbol static to avoid linking errors. +#define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC static +#define ATTRIBUTE_NOINLINE __declspec(noinline) +#else +#define ATTRIBUTE_ALIGNED(X) __attribute__((aligned(X))) +#define ATTRIBUTE_INTERFACE __attribute__((visibility("default"))) +#define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC \ + ATTRIBUTE_INTERFACE __attribute__((tls_model("initial-exec"))) thread_local + +#define ATTRIBUTE_NOINLINE __attribute__((noinline)) +#endif #if defined(__has_feature) # if __has_feature(address_sanitizer) @@ -127,12 +154,6 @@ # define ATTRIBUTE_NO_SANITIZE_ALL #endif -#if LIBFUZZER_WINDOWS -#define ATTRIBUTE_INTERFACE __declspec(dllexport) -#else -#define ATTRIBUTE_INTERFACE __attribute__((visibility("default"))) -#endif - namespace fuzzer { template T Min(T a, T b) { return a < b ? a : b; } @@ -176,11 +197,6 @@ typedef int (*UserCallback)(const uint8_t *Data, size_t Size); int FuzzerDriver(int *argc, char ***argv, UserCallback Callback); -inline uint8_t Bswap(uint8_t x) { return x; } -inline uint16_t Bswap(uint16_t x) { return __builtin_bswap16(x); } -inline uint32_t Bswap(uint32_t x) { return __builtin_bswap32(x); } -inline uint64_t Bswap(uint64_t x) { return __builtin_bswap64(x); } - uint8_t *ExtraCountersBegin(); uint8_t *ExtraCountersEnd(); void ClearExtraCounters(); diff --git a/libfuzzer/FuzzerDictionary.h b/libfuzzer/FuzzerDictionary.h index 0d9d91b..301c5d9 100644 --- a/libfuzzer/FuzzerDictionary.h +++ b/libfuzzer/FuzzerDictionary.h @@ -1,9 +1,8 @@ //===- FuzzerDictionary.h - Internal header for the Fuzzer ------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::Dictionary diff --git a/libfuzzer/FuzzerDriver.cpp b/libfuzzer/FuzzerDriver.cpp index 783474a..54c7ff0 100644 --- a/libfuzzer/FuzzerDriver.cpp +++ b/libfuzzer/FuzzerDriver.cpp @@ -1,9 +1,8 @@ //===- FuzzerDriver.cpp - FuzzerDriver function and flags -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // FuzzerDriver and flag parsing. @@ -11,12 +10,13 @@ #include "FuzzerCommand.h" #include "FuzzerCorpus.h" +#include "FuzzerFork.h" #include "FuzzerIO.h" #include "FuzzerInterface.h" #include "FuzzerInternal.h" +#include "FuzzerMerge.h" #include "FuzzerMutate.h" #include "FuzzerRandom.h" -#include "FuzzerShmem.h" #include "FuzzerTracePC.h" #include #include @@ -26,10 +26,16 @@ #include #include #include +#include // This function should be present in the libFuzzer so that the client // binary can test for its existence. +#if LIBFUZZER_MSVC +extern "C" void __libfuzzer_is_present() {} +#pragma comment(linker, "/include:__libfuzzer_is_present") +#else extern "C" __attribute__((used)) void __libfuzzer_is_present() {} +#endif // LIBFUZZER_MSVC namespace fuzzer { @@ -176,7 +182,8 @@ static bool ParseOneFlag(const char *Param) { } // We don't use any library to minimize dependencies. -static void ParseFlags(const Vector &Args) { +static void ParseFlags(const Vector &Args, + const ExternalFunctions *EF) { for (size_t F = 0; F < kNumFlags; F++) { if (FlagDescriptions[F].IntFlag) *FlagDescriptions[F].IntFlag = FlagDescriptions[F].Default; @@ -186,6 +193,11 @@ static void ParseFlags(const Vector &Args) { if (FlagDescriptions[F].StrFlag) *FlagDescriptions[F].StrFlag = nullptr; } + + // Disable len_control by default, if LLVMFuzzerCustomMutator is used. + if (EF->LLVMFuzzerCustomMutator) + Flags.len_control = 0; + Inputs = new Vector; for (size_t A = 1; A < Args.size(); A++) { if (ParseOneFlag(Args[A].c_str())) { @@ -316,10 +328,8 @@ int CleanseCrashInput(const Vector &Args, assert(Cmd.hasArgument(InputFilePath)); Cmd.removeArgument(InputFilePath); - auto LogFilePath = DirPlusFile( - TmpDir(), "libFuzzerTemp." + std::to_string(GetPid()) + ".txt"); - auto TmpFilePath = DirPlusFile( - TmpDir(), "libFuzzerTemp." + std::to_string(GetPid()) + ".repro"); + auto LogFilePath = TempPath(".txt"); + auto TmpFilePath = TempPath(".repro"); Cmd.addArgument(TmpFilePath); Cmd.setOutputFile(LogFilePath); Cmd.combineOutAndErr(); @@ -379,8 +389,7 @@ int MinimizeCrashInput(const Vector &Args, BaseCmd.addFlag("max_total_time", "600"); } - auto LogFilePath = DirPlusFile( - TmpDir(), "libFuzzerTemp." + std::to_string(GetPid()) + ".txt"); + auto LogFilePath = TempPath(".txt"); BaseCmd.setOutputFile(LogFilePath); BaseCmd.combineOutAndErr(); @@ -464,6 +473,34 @@ int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) { return 0; } +void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector &Args, + const Vector &Corpora, const char *CFPathOrNull) { + if (Corpora.size() < 2) { + Printf("INFO: Merge requires two or more corpus dirs\n"); + exit(0); + } + + Vector OldCorpus, NewCorpus; + GetSizedFilesFromDir(Corpora[0], &OldCorpus); + for (size_t i = 1; i < Corpora.size(); i++) + GetSizedFilesFromDir(Corpora[i], &NewCorpus); + std::sort(OldCorpus.begin(), OldCorpus.end()); + std::sort(NewCorpus.begin(), NewCorpus.end()); + + std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath(".txt"); + Vector NewFiles; + Set NewFeatures, NewCov; + CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures, + {}, &NewCov, CFPath, true); + for (auto &Path : NewFiles) + F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen)); + // We are done, delete the control file if it was a temporary one. + if (!Flags.merge_control_file) + RemoveFile(CFPath); + + exit(0); +} + int AnalyzeDictionary(Fuzzer *F, const Vector& Dict, UnitVector& Corpus) { Printf("Started dictionary minimization (up to %d tests)\n", @@ -530,6 +567,45 @@ int AnalyzeDictionary(Fuzzer *F, const Vector& Dict, return 0; } +Vector ParseSeedInuts(const char *seed_inputs) { + // Parse -seed_inputs=file1,file2,... or -seed_inputs=@seed_inputs_file + Vector Files; + if (!seed_inputs) return Files; + std::string SeedInputs; + if (Flags.seed_inputs[0] == '@') + SeedInputs = FileToString(Flags.seed_inputs + 1); // File contains list. + else + SeedInputs = Flags.seed_inputs; // seed_inputs contains the list. + if (SeedInputs.empty()) { + Printf("seed_inputs is empty or @file does not exist.\n"); + exit(1); + } + // Parse SeedInputs. + size_t comma_pos = 0; + while ((comma_pos = SeedInputs.find_last_of(',')) != std::string::npos) { + Files.push_back(SeedInputs.substr(comma_pos + 1)); + SeedInputs = SeedInputs.substr(0, comma_pos); + } + Files.push_back(SeedInputs); + return Files; +} + +static Vector ReadCorpora(const Vector &CorpusDirs, + const Vector &ExtraSeedFiles) { + Vector SizedFiles; + size_t LastNumFiles = 0; + for (auto &Dir : CorpusDirs) { + GetSizedFilesFromDir(Dir, &SizedFiles); + Printf("INFO: % 8zd files found in %s\n", SizedFiles.size() - LastNumFiles, + Dir.c_str()); + LastNumFiles = SizedFiles.size(); + } + for (auto &File : ExtraSeedFiles) + if (auto Size = FileSize(File)) + SizedFiles.push_back({File, Size}); + return SizedFiles; +} + int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { using namespace fuzzer; assert(argc && argv && "Argument pointers cannot be nullptr"); @@ -546,7 +622,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Printf("ERROR: argv[0] has been modified in LLVMFuzzerInitialize\n"); exit(1); } - ParseFlags(Args); + ParseFlags(Args, EF); if (Flags.help) { PrintHelp(); return 0; @@ -573,6 +649,9 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.UnitTimeoutSec = Flags.timeout; Options.ErrorExitCode = Flags.error_exitcode; Options.TimeoutExitCode = Flags.timeout_exitcode; + Options.IgnoreTimeouts = Flags.ignore_timeouts; + Options.IgnoreOOMs = Flags.ignore_ooms; + Options.IgnoreCrashes = Flags.ignore_crashes; Options.MaxTotalTimeSec = Flags.max_total_time; Options.DoCrossOver = Flags.cross_over; Options.MutateDepth = Flags.mutate_depth; @@ -609,20 +688,14 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { return 1; if (Flags.verbosity > 0 && !Dictionary.empty()) Printf("Dictionary: %zd entries\n", Dictionary.size()); - bool DoPlainRun = AllInputsAreFiles(); + bool RunIndividualFiles = AllInputsAreFiles(); Options.SaveArtifacts = - !DoPlainRun || Flags.minimize_crash_internal_step; + !RunIndividualFiles || Flags.minimize_crash_internal_step; Options.PrintNewCovPcs = Flags.print_pcs; Options.PrintNewCovFuncs = Flags.print_funcs; Options.PrintFinalStats = Flags.print_final_stats; - Options.PrintMutationStats = Flags.print_mutation_stats; Options.PrintCorpusStats = Flags.print_corpus_stats; Options.PrintCoverage = Flags.print_coverage; - Options.PrintUnstableStats = Flags.print_unstable_stats; - if (Flags.handle_unstable == TracePC::MinUnstable || - Flags.handle_unstable == TracePC::ZeroUnstable) - Options.HandleUnstable = Flags.handle_unstable; - Options.DumpCoverage = Flags.dump_coverage; if (Flags.exit_on_src_pos) Options.ExitOnSrcPos = Flags.exit_on_src_pos; if (Flags.exit_on_item) @@ -631,6 +704,13 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.FocusFunction = Flags.focus_function; if (Flags.data_flow_trace) Options.DataFlowTrace = Flags.data_flow_trace; + if (Flags.features_dir) + Options.FeaturesDir = Flags.features_dir; + if (Flags.collect_data_flow) + Options.CollectDataFlow = Flags.collect_data_flow; + Options.LazyCounters = Flags.lazy_counters; + if (Flags.stop_file) + Options.StopFile = Flags.stop_file; unsigned Seed = Flags.seed; // Initialize Seed. @@ -640,6 +720,15 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { if (Flags.verbosity) Printf("INFO: Seed: %u\n", Seed); + if (Flags.collect_data_flow && !Flags.fork && !Flags.merge) { + if (RunIndividualFiles) + return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace, + ReadCorpora({}, *Inputs)); + else + return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace, + ReadCorpora(*Inputs, {})); + } + Random Rand(Seed); auto *MD = new MutationDispatcher(Rand, Options); auto *Corpus = new InputCorpus(Options.OutputCorpus); @@ -674,35 +763,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { if (Flags.cleanse_crash) return CleanseCrashInput(Args, Options); -#if 0 // deprecated, to be removed. - if (auto Name = Flags.run_equivalence_server) { - SMR.Destroy(Name); - if (!SMR.Create(Name)) { - Printf("ERROR: can't create shared memory region\n"); - return 1; - } - Printf("INFO: EQUIVALENCE SERVER UP\n"); - while (true) { - SMR.WaitClient(); - size_t Size = SMR.ReadByteArraySize(); - SMR.WriteByteArray(nullptr, 0); - const Unit tmp(SMR.GetByteArray(), SMR.GetByteArray() + Size); - F->ExecuteCallback(tmp.data(), tmp.size()); - SMR.PostServer(); - } - return 0; - } - - if (auto Name = Flags.use_equivalence_server) { - if (!SMR.Open(Name)) { - Printf("ERROR: can't open shared memory region\n"); - return 1; - } - Printf("INFO: EQUIVALENCE CLIENT UP\n"); - } -#endif - - if (DoPlainRun) { + if (RunIndividualFiles) { Options.SaveArtifacts = false; int Runs = std::max(1, Flags.runs); Printf("%s: Running %zd inputs %d time(s) each.\n", ProgName->c_str(), @@ -724,13 +785,11 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { exit(0); } - if (Flags.merge) { - F->CrashResistantMerge(Args, *Inputs, - Flags.load_coverage_summary, - Flags.save_coverage_summary, - Flags.merge_control_file); - exit(0); - } + if (Flags.fork) + FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); + + if (Flags.merge) + Merge(F, Options, Args, *Inputs, Flags.merge_control_file); if (Flags.merge_inner) { const size_t kDefaultMaxMergeLen = 1 << 20; @@ -762,7 +821,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { exit(0); } - F->Loop(*Inputs); + auto CorporaFiles = ReadCorpora(*Inputs, ParseSeedInuts(Flags.seed_inputs)); + F->Loop(CorporaFiles); if (Flags.verbosity) Printf("Done %zd runs in %zd second(s)\n", F->getTotalNumberOfRuns(), diff --git a/libfuzzer/FuzzerExtFunctions.def b/libfuzzer/FuzzerExtFunctions.def index 8bfffdd..41fa0fd 100644 --- a/libfuzzer/FuzzerExtFunctions.def +++ b/libfuzzer/FuzzerExtFunctions.def @@ -1,9 +1,8 @@ //===- FuzzerExtFunctions.def - External functions --------------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This defines the external function pointers that @@ -29,7 +28,7 @@ EXT_FUNC(LLVMFuzzerCustomCrossOver, size_t, EXT_FUNC(__lsan_enable, void, (), false); EXT_FUNC(__lsan_disable, void, (), false); EXT_FUNC(__lsan_do_recoverable_leak_check, int, (), false); -EXT_FUNC(__sanitizer_acquire_crash_state, bool, (), true); +EXT_FUNC(__sanitizer_acquire_crash_state, int, (), true); EXT_FUNC(__sanitizer_install_malloc_and_free_hooks, int, (void (*malloc_hook)(const volatile void *, size_t), void (*free_hook)(const volatile void *)), @@ -44,8 +43,7 @@ EXT_FUNC(__sanitizer_get_module_and_offset_for_pc, int, size_t module_path_len,void **pc_offset), false); EXT_FUNC(__sanitizer_set_death_callback, void, (void (*)(void)), true); EXT_FUNC(__sanitizer_set_report_fd, void, (void*), false); -EXT_FUNC(__sanitizer_dump_coverage, void, (const uintptr_t *, uintptr_t), - false); EXT_FUNC(__msan_scoped_disable_interceptor_checks, void, (), false); EXT_FUNC(__msan_scoped_enable_interceptor_checks, void, (), false); EXT_FUNC(__msan_unpoison, void, (const volatile void *, size_t size), false); +EXT_FUNC(__msan_unpoison_param, void, (size_t n), false); diff --git a/libfuzzer/FuzzerExtFunctions.h b/libfuzzer/FuzzerExtFunctions.h index 2672a38..c88aac4 100644 --- a/libfuzzer/FuzzerExtFunctions.h +++ b/libfuzzer/FuzzerExtFunctions.h @@ -1,9 +1,8 @@ //===- FuzzerExtFunctions.h - Interface to external functions ---*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Defines an interface to (possibly optional) functions. diff --git a/libfuzzer/FuzzerExtFunctionsDlsym.cpp b/libfuzzer/FuzzerExtFunctionsDlsym.cpp index 06bddd5..dcd7134 100644 --- a/libfuzzer/FuzzerExtFunctionsDlsym.cpp +++ b/libfuzzer/FuzzerExtFunctionsDlsym.cpp @@ -1,9 +1,8 @@ //===- FuzzerExtFunctionsDlsym.cpp - Interface to external functions ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Implementation for operating systems that support dlsym(). We only use it on diff --git a/libfuzzer/FuzzerExtFunctionsDlsymWin.cpp b/libfuzzer/FuzzerExtFunctionsDlsymWin.cpp deleted file mode 100644 index 321b3ec..0000000 --- a/libfuzzer/FuzzerExtFunctionsDlsymWin.cpp +++ /dev/null @@ -1,62 +0,0 @@ -//===- FuzzerExtFunctionsDlsymWin.cpp - Interface to external functions ---===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// Implementation using dynamic loading for Windows. -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" -#if LIBFUZZER_WINDOWS - -#include "FuzzerExtFunctions.h" -#include "FuzzerIO.h" -#include "Windows.h" - -// This must be included after Windows.h. -#include "Psapi.h" - -namespace fuzzer { - -ExternalFunctions::ExternalFunctions() { - HMODULE Modules[1024]; - DWORD BytesNeeded; - HANDLE CurrentProcess = GetCurrentProcess(); - - if (!EnumProcessModules(CurrentProcess, Modules, sizeof(Modules), - &BytesNeeded)) { - Printf("EnumProcessModules failed (error: %d).\n", GetLastError()); - exit(1); - } - - if (sizeof(Modules) < BytesNeeded) { - Printf("Error: the array is not big enough to hold all loaded modules.\n"); - exit(1); - } - - for (size_t i = 0; i < (BytesNeeded / sizeof(HMODULE)); i++) - { - FARPROC Fn; -#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - if (this->NAME == nullptr) { \ - Fn = GetProcAddress(Modules[i], #NAME); \ - if (Fn == nullptr) \ - Fn = GetProcAddress(Modules[i], #NAME "__dll"); \ - this->NAME = (decltype(ExternalFunctions::NAME)) Fn; \ - } -#include "FuzzerExtFunctions.def" -#undef EXT_FUNC - } - -#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - if (this->NAME == nullptr && WARN) \ - Printf("WARNING: Failed to find function \"%s\".\n", #NAME); -#include "FuzzerExtFunctions.def" -#undef EXT_FUNC -} - -} // namespace fuzzer - -#endif // LIBFUZZER_WINDOWS diff --git a/libfuzzer/FuzzerExtFunctionsWeak.cpp b/libfuzzer/FuzzerExtFunctionsWeak.cpp index a4e56fc..ea5b87b 100644 --- a/libfuzzer/FuzzerExtFunctionsWeak.cpp +++ b/libfuzzer/FuzzerExtFunctionsWeak.cpp @@ -1,9 +1,8 @@ //===- FuzzerExtFunctionsWeak.cpp - Interface to external functions -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Implementation for Linux. This relies on the linker's support for weak @@ -22,7 +21,7 @@ extern "C" { // Declare these symbols as weak to allow them to be optionally defined. #define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - __attribute__((weak)) RETURN_TYPE NAME FUNC_SIG + __attribute__((weak, visibility("default"))) RETURN_TYPE NAME FUNC_SIG #include "FuzzerExtFunctions.def" diff --git a/libfuzzer/FuzzerExtFunctionsWeakAlias.cpp b/libfuzzer/FuzzerExtFunctionsWeakAlias.cpp deleted file mode 100644 index e10f7b4..0000000 --- a/libfuzzer/FuzzerExtFunctionsWeakAlias.cpp +++ /dev/null @@ -1,56 +0,0 @@ -//===- FuzzerExtFunctionsWeakAlias.cpp - Interface to external functions --===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// Implementation using weak aliases. Works for Windows. -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" -#if LIBFUZZER_WINDOWS - -#include "FuzzerExtFunctions.h" -#include "FuzzerIO.h" - -using namespace fuzzer; - -extern "C" { -// Declare these symbols as weak to allow them to be optionally defined. -#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - RETURN_TYPE NAME##Def FUNC_SIG { \ - Printf("ERROR: Function \"%s\" not defined.\n", #NAME); \ - exit(1); \ - } \ - RETURN_TYPE NAME FUNC_SIG __attribute__((weak, alias(#NAME "Def"))); - -#include "FuzzerExtFunctions.def" - -#undef EXT_FUNC -} - -template -static T *GetFnPtr(T *Fun, T *FunDef, const char *FnName, bool WarnIfMissing) { - if (Fun == FunDef) { - if (WarnIfMissing) - Printf("WARNING: Failed to find function \"%s\".\n", FnName); - return nullptr; - } - return Fun; -} - -namespace fuzzer { - -ExternalFunctions::ExternalFunctions() { -#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - this->NAME = GetFnPtr(::NAME, ::NAME##Def, #NAME, WARN); - -#include "FuzzerExtFunctions.def" - -#undef EXT_FUNC -} - -} // namespace fuzzer - -#endif // LIBFUZZER_WINDOWS diff --git a/libfuzzer/FuzzerExtFunctionsWindows.cpp b/libfuzzer/FuzzerExtFunctionsWindows.cpp new file mode 100644 index 0000000..55efe8f --- /dev/null +++ b/libfuzzer/FuzzerExtFunctionsWindows.cpp @@ -0,0 +1,82 @@ +//=== FuzzerExtWindows.cpp - Interface to external functions --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Implementation of FuzzerExtFunctions for Windows. Uses alternatename when +// compiled with MSVC. Uses weak aliases when compiled with clang. Unfortunately +// the method each compiler supports is not supported by the other. +//===----------------------------------------------------------------------===// +#include "FuzzerDefs.h" +#if LIBFUZZER_WINDOWS + +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" + +using namespace fuzzer; + +// Intermediate macro to ensure the parameter is expanded before stringified. +#define STRINGIFY_(A) #A +#define STRINGIFY(A) STRINGIFY_(A) + +#if LIBFUZZER_MSVC +// Copied from compiler-rt/lib/sanitizer_common/sanitizer_win_defs.h +#if defined(_M_IX86) || defined(__i386__) +#define WIN_SYM_PREFIX "_" +#else +#define WIN_SYM_PREFIX +#endif + +// Declare external functions as having alternativenames, so that we can +// determine if they are not defined. +#define EXTERNAL_FUNC(Name, Default) \ + __pragma(comment(linker, "/alternatename:" WIN_SYM_PREFIX STRINGIFY( \ + Name) "=" WIN_SYM_PREFIX STRINGIFY(Default))) +#else +// Declare external functions as weak to allow them to default to a specified +// function if not defined explicitly. We must use weak symbols because clang's +// support for alternatename is not 100%, see +// https://bugs.llvm.org/show_bug.cgi?id=40218 for more details. +#define EXTERNAL_FUNC(Name, Default) \ + __attribute__((weak, alias(STRINGIFY(Default)))) +#endif // LIBFUZZER_MSVC + +extern "C" { +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + RETURN_TYPE NAME##Def FUNC_SIG { \ + Printf("ERROR: Function \"%s\" not defined.\n", #NAME); \ + exit(1); \ + } \ + EXTERNAL_FUNC(NAME, NAME##Def) RETURN_TYPE NAME FUNC_SIG + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +} + +template +static T *GetFnPtr(T *Fun, T *FunDef, const char *FnName, bool WarnIfMissing) { + if (Fun == FunDef) { + if (WarnIfMissing) + Printf("WARNING: Failed to find function \"%s\".\n", FnName); + return nullptr; + } + return Fun; +} + +namespace fuzzer { + +ExternalFunctions::ExternalFunctions() { +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + this->NAME = GetFnPtr(::NAME, ::NAME##Def, #NAME, WARN); + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +} + +} // namespace fuzzer + +#endif // LIBFUZZER_WINDOWS diff --git a/libfuzzer/FuzzerExtraCounters.cpp b/libfuzzer/FuzzerExtraCounters.cpp index c99cd89..3f38f4f 100644 --- a/libfuzzer/FuzzerExtraCounters.cpp +++ b/libfuzzer/FuzzerExtraCounters.cpp @@ -1,9 +1,8 @@ //===- FuzzerExtraCounters.cpp - Extra coverage counters ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Extra coverage counters defined by user code. diff --git a/libfuzzer/FuzzerFlags.def b/libfuzzer/FuzzerFlags.def index ba04bc2..a11cfe4 100644 --- a/libfuzzer/FuzzerFlags.def +++ b/libfuzzer/FuzzerFlags.def @@ -1,9 +1,8 @@ //===- FuzzerFlags.def - Run-time flags -------------------------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Flags. FUZZER_FLAG_INT/FUZZER_FLAG_STRING macros should be defined at the @@ -17,10 +16,13 @@ FUZZER_FLAG_INT(runs, -1, FUZZER_FLAG_INT(max_len, 0, "Maximum length of the test input. " "If 0, libFuzzer tries to guess a good value based on the corpus " "and reports it. ") -FUZZER_FLAG_INT(len_control, 1000, "Try generating small inputs first, " +FUZZER_FLAG_INT(len_control, 100, "Try generating small inputs first, " "then try larger inputs over time. Specifies the rate at which the length " "limit is increased (smaller == faster). If 0, immediately try inputs with " - "size up to max_len.") + "size up to max_len. Default value is 0, if LLVMFuzzerCustomMutator is used.") +FUZZER_FLAG_STRING(seed_inputs, "A comma-separated list of input files " + "to use as an additional seed corpus. Alternatively, an \"@\" followed by " + "the name of a file containing the comma-seperated list.") FUZZER_FLAG_INT(cross_over, 1, "If 1, cross over inputs.") FUZZER_FLAG_INT(mutate_depth, 5, "Apply this number of consecutive mutations to each input.") @@ -35,27 +37,26 @@ FUZZER_FLAG_INT( "If one unit runs more than this number of seconds the process will abort.") FUZZER_FLAG_INT(error_exitcode, 77, "When libFuzzer itself reports a bug " "this exit code will be used.") -FUZZER_FLAG_INT(timeout_exitcode, 77, "When libFuzzer reports a timeout " +FUZZER_FLAG_INT(timeout_exitcode, 70, "When libFuzzer reports a timeout " "this exit code will be used.") FUZZER_FLAG_INT(max_total_time, 0, "If positive, indicates the maximal total " "time in seconds to run the fuzzer.") FUZZER_FLAG_INT(help, 0, "Print help.") +FUZZER_FLAG_INT(fork, 0, "Experimental mode where fuzzing happens " + "in a subprocess") +FUZZER_FLAG_INT(ignore_timeouts, 1, "Ignore timeouts in fork mode") +FUZZER_FLAG_INT(ignore_ooms, 1, "Ignore OOMs in fork mode") +FUZZER_FLAG_INT(ignore_crashes, 0, "Ignore crashes in fork mode") FUZZER_FLAG_INT(merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be " "merged into the 1-st corpus. Only interesting units will be taken. " "This flag can be used to minimize a corpus.") +FUZZER_FLAG_STRING(stop_file, "Stop fuzzing ASAP if this file exists") FUZZER_FLAG_STRING(merge_inner, "internal flag") FUZZER_FLAG_STRING(merge_control_file, "Specify a control file used for the merge process. " "If a merge process gets killed it tries to leave this file " "in a state suitable for resuming the merge. " "By default a temporary file will be used.") -FUZZER_FLAG_STRING(save_coverage_summary, "Experimental:" - " save coverage summary to a given file." - " Used with -merge=1") -FUZZER_FLAG_STRING(load_coverage_summary, "Experimental:" - " load coverage summary from a given file." - " Treat this coverage as belonging to the first corpus. " - " Used with -merge=1") FUZZER_FLAG_INT(minimize_crash, 0, "If 1, minimizes the provided" " crash input. Use with -runs=N or -max_total_time=N to limit " "the number attempts." @@ -68,6 +69,10 @@ FUZZER_FLAG_INT(cleanse_crash, 0, "If 1, tries to cleanse the provided" " Use with -exact_artifact_path to specify the output." ) FUZZER_FLAG_INT(minimize_crash_internal_step, 0, "internal flag") +FUZZER_FLAG_STRING(features_dir, "internal flag. Used to dump feature sets on disk." + "Every time a new input is added to the corpus, a corresponding file in the features_dir" + " is created containing the unique features of that input." + " Features are stored in binary format.") FUZZER_FLAG_INT(use_counters, 1, "Use coverage counters") FUZZER_FLAG_INT(use_memmem, 1, "Use hints from intercepting memmem, strstr, etc") @@ -107,18 +112,7 @@ FUZZER_FLAG_INT(print_corpus_stats, 0, "If 1, print statistics on corpus elements at exit.") FUZZER_FLAG_INT(print_coverage, 0, "If 1, print coverage information as text" " at exit.") -FUZZER_FLAG_INT(dump_coverage, 0, "Deprecated." - " If 1, dump coverage information as a" - " .sancov file at exit.") -FUZZER_FLAG_INT(handle_unstable, 0, "Experimental." - " Executes every input 3 times in total if a unique feature" - " is found during the first execution." - " If 1, we only use the minimum hit count from the 3 runs" - " to determine whether an input is interesting." - " If 2, we disregard edges that are found unstable for" - " feature collection.") -FUZZER_FLAG_INT(print_unstable_stats, 0, "Experimental." - " If 1, print unstable statistics at exit.") +FUZZER_FLAG_INT(dump_coverage, 0, "Deprecated.") FUZZER_FLAG_INT(handle_segv, 1, "If 1, try to intercept SIGSEGV.") FUZZER_FLAG_INT(handle_bus, 1, "If 1, try to intercept SIGBUS.") FUZZER_FLAG_INT(handle_abrt, 1, "If 1, try to intercept SIGABRT.") @@ -129,6 +123,9 @@ FUZZER_FLAG_INT(handle_term, 1, "If 1, try to intercept SIGTERM.") FUZZER_FLAG_INT(handle_xfsz, 1, "If 1, try to intercept SIGXFSZ.") FUZZER_FLAG_INT(handle_usr1, 1, "If 1, try to intercept SIGUSR1.") FUZZER_FLAG_INT(handle_usr2, 1, "If 1, try to intercept SIGUSR2.") +FUZZER_FLAG_INT(lazy_counters, 0, "If 1, a performance optimization is" + "enabled for the 8bit inline counters. " + "Requires that libFuzzer successfully installs its SEGV handler") FUZZER_FLAG_INT(close_fd_mask, 0, "If 1, close stdout at startup; " "if 2, close stderr; if 3, close both. " "Be careful, this will also close e.g. stderr of asan.") @@ -155,11 +152,12 @@ FUZZER_FLAG_INT(ignore_remaining_args, 0, "If 1, ignore all arguments passed " "after this one. Useful for fuzzers that need to do their own " "argument parsing.") FUZZER_FLAG_STRING(focus_function, "Experimental. " - "Fuzzing will focus on inputs that trigger calls to this function") + "Fuzzing will focus on inputs that trigger calls to this function. " + "If -focus_function=auto and -data_flow_trace is used, libFuzzer " + "will choose the focus functions automatically.") -FUZZER_DEPRECATED_FLAG(run_equivalence_server) -FUZZER_DEPRECATED_FLAG(use_equivalence_server) FUZZER_FLAG_INT(analyze_dict, 0, "Experimental") FUZZER_DEPRECATED_FLAG(use_clang_coverage) FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace") -FUZZER_FLAG_INT(print_mutation_stats, 0, "Experimental") +FUZZER_FLAG_STRING(collect_data_flow, + "Experimental: collect the data flow trace") diff --git a/libfuzzer/FuzzerFork.cpp b/libfuzzer/FuzzerFork.cpp new file mode 100644 index 0000000..95ed365 --- /dev/null +++ b/libfuzzer/FuzzerFork.cpp @@ -0,0 +1,409 @@ +//===- FuzzerFork.cpp - run fuzzing in separate subprocesses --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Spawn and orchestrate separate fuzzing processes. +//===----------------------------------------------------------------------===// + +#include "FuzzerCommand.h" +#include "FuzzerFork.h" +#include "FuzzerIO.h" +#include "FuzzerInternal.h" +#include "FuzzerMerge.h" +#include "FuzzerSHA1.h" +#include "FuzzerTracePC.h" +#include "FuzzerUtil.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fuzzer { + +struct Stats { + size_t number_of_executed_units = 0; + size_t peak_rss_mb = 0; + size_t average_exec_per_sec = 0; +}; + +static Stats ParseFinalStatsFromLog(const std::string &LogPath) { + std::ifstream In(LogPath); + std::string Line; + Stats Res; + struct { + const char *Name; + size_t *Var; + } NameVarPairs[] = { + {"stat::number_of_executed_units:", &Res.number_of_executed_units}, + {"stat::peak_rss_mb:", &Res.peak_rss_mb}, + {"stat::average_exec_per_sec:", &Res.average_exec_per_sec}, + {nullptr, nullptr}, + }; + while (std::getline(In, Line, '\n')) { + if (Line.find("stat::") != 0) continue; + std::istringstream ISS(Line); + std::string Name; + size_t Val; + ISS >> Name >> Val; + for (size_t i = 0; NameVarPairs[i].Name; i++) + if (Name == NameVarPairs[i].Name) + *NameVarPairs[i].Var = Val; + } + return Res; +} + +struct FuzzJob { + // Inputs. + Command Cmd; + std::string CorpusDir; + std::string FeaturesDir; + std::string LogPath; + std::string SeedListPath; + std::string CFPath; + size_t JobId; + + int DftTimeInSeconds = 0; + + // Fuzzing Outputs. + int ExitCode; + + ~FuzzJob() { + RemoveFile(CFPath); + RemoveFile(LogPath); + RemoveFile(SeedListPath); + RmDirRecursive(CorpusDir); + RmDirRecursive(FeaturesDir); + } +}; + +struct GlobalEnv { + Vector Args; + Vector CorpusDirs; + std::string MainCorpusDir; + std::string TempDir; + std::string DFTDir; + std::string DataFlowBinary; + Set Features, Cov; + Set FilesWithDFT; + Vector Files; + Random *Rand; + std::chrono::system_clock::time_point ProcessStartTime; + int Verbosity = 0; + + size_t NumTimeouts = 0; + size_t NumOOMs = 0; + size_t NumCrashes = 0; + + + size_t NumRuns = 0; + + std::string StopFile() { return DirPlusFile(TempDir, "STOP"); } + + size_t secondsSinceProcessStartUp() const { + return std::chrono::duration_cast( + std::chrono::system_clock::now() - ProcessStartTime) + .count(); + } + + FuzzJob *CreateNewJob(size_t JobId) { + Command Cmd(Args); + Cmd.removeFlag("fork"); + Cmd.removeFlag("runs"); + Cmd.removeFlag("collect_data_flow"); + for (auto &C : CorpusDirs) // Remove all corpora from the args. + Cmd.removeArgument(C); + Cmd.addFlag("reload", "0"); // working in an isolated dir, no reload. + Cmd.addFlag("print_final_stats", "1"); + Cmd.addFlag("print_funcs", "0"); // no need to spend time symbolizing. + Cmd.addFlag("max_total_time", std::to_string(std::min((size_t)300, JobId))); + Cmd.addFlag("stop_file", StopFile()); + if (!DataFlowBinary.empty()) { + Cmd.addFlag("data_flow_trace", DFTDir); + if (!Cmd.hasFlag("focus_function")) + Cmd.addFlag("focus_function", "auto"); + } + auto Job = new FuzzJob; + std::string Seeds; + if (size_t CorpusSubsetSize = + std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) { + auto Time1 = std::chrono::system_clock::now(); + for (size_t i = 0; i < CorpusSubsetSize; i++) { + auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } + auto Time2 = std::chrono::system_clock::now(); + Job->DftTimeInSeconds = duration_cast(Time2 - Time1).count(); + } + if (!Seeds.empty()) { + Job->SeedListPath = + DirPlusFile(TempDir, std::to_string(JobId) + ".seeds"); + WriteToFile(Seeds, Job->SeedListPath); + Cmd.addFlag("seed_inputs", "@" + Job->SeedListPath); + } + Job->LogPath = DirPlusFile(TempDir, std::to_string(JobId) + ".log"); + Job->CorpusDir = DirPlusFile(TempDir, "C" + std::to_string(JobId)); + Job->FeaturesDir = DirPlusFile(TempDir, "F" + std::to_string(JobId)); + Job->CFPath = DirPlusFile(TempDir, std::to_string(JobId) + ".merge"); + Job->JobId = JobId; + + + Cmd.addArgument(Job->CorpusDir); + Cmd.addFlag("features_dir", Job->FeaturesDir); + + for (auto &D : {Job->CorpusDir, Job->FeaturesDir}) { + RmDirRecursive(D); + MkDir(D); + } + + Cmd.setOutputFile(Job->LogPath); + Cmd.combineOutAndErr(); + + Job->Cmd = Cmd; + + if (Verbosity >= 2) + Printf("Job %zd/%p Created: %s\n", JobId, Job, + Job->Cmd.toString().c_str()); + // Start from very short runs and gradually increase them. + return Job; + } + + void RunOneMergeJob(FuzzJob *Job) { + auto Stats = ParseFinalStatsFromLog(Job->LogPath); + NumRuns += Stats.number_of_executed_units; + + Vector TempFiles, MergeCandidates; + // Read all newly created inputs and their feature sets. + // Choose only those inputs that have new features. + GetSizedFilesFromDir(Job->CorpusDir, &TempFiles); + std::sort(TempFiles.begin(), TempFiles.end()); + for (auto &F : TempFiles) { + auto FeatureFile = F.File; + FeatureFile.replace(0, Job->CorpusDir.size(), Job->FeaturesDir); + auto FeatureBytes = FileToVector(FeatureFile, 0, false); + assert((FeatureBytes.size() % sizeof(uint32_t)) == 0); + Vector NewFeatures(FeatureBytes.size() / sizeof(uint32_t)); + memcpy(NewFeatures.data(), FeatureBytes.data(), FeatureBytes.size()); + for (auto Ft : NewFeatures) { + if (!Features.count(Ft)) { + MergeCandidates.push_back(F); + break; + } + } + } + // if (!FilesToAdd.empty() || Job->ExitCode != 0) + Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s %zd " + "oom/timeout/crash: %zd/%zd/%zd time: %zds job: %zd dft_time: %d\n", + NumRuns, Cov.size(), Features.size(), Files.size(), + Stats.average_exec_per_sec, NumOOMs, NumTimeouts, NumCrashes, + secondsSinceProcessStartUp(), Job->JobId, Job->DftTimeInSeconds); + + if (MergeCandidates.empty()) return; + + Vector FilesToAdd; + Set NewFeatures, NewCov; + CrashResistantMerge(Args, {}, MergeCandidates, &FilesToAdd, Features, + &NewFeatures, Cov, &NewCov, Job->CFPath, false); + for (auto &Path : FilesToAdd) { + auto U = FileToVector(Path); + auto NewPath = DirPlusFile(MainCorpusDir, Hash(U)); + WriteToFile(U, NewPath); + Files.push_back(NewPath); + } + Features.insert(NewFeatures.begin(), NewFeatures.end()); + Cov.insert(NewCov.begin(), NewCov.end()); + for (auto Idx : NewCov) + if (auto *TE = TPC.PCTableEntryByIdx(Idx)) + if (TPC.PcIsFuncEntry(TE)) + PrintPC(" NEW_FUNC: %p %F %L\n", "", + TPC.GetNextInstructionPc(TE->PC)); + + } + + + void CollectDFT(const std::string &InputPath) { + if (DataFlowBinary.empty()) return; + if (!FilesWithDFT.insert(InputPath).second) return; + Command Cmd(Args); + Cmd.removeFlag("fork"); + Cmd.removeFlag("runs"); + Cmd.addFlag("data_flow_trace", DFTDir); + Cmd.addArgument(InputPath); + for (auto &C : CorpusDirs) // Remove all corpora from the args. + Cmd.removeArgument(C); + Cmd.setOutputFile(DirPlusFile(TempDir, "dft.log")); + Cmd.combineOutAndErr(); + // Printf("CollectDFT: %s\n", Cmd.toString().c_str()); + ExecuteCommand(Cmd); + } + +}; + +struct JobQueue { + std::queue Qu; + std::mutex Mu; + std::condition_variable Cv; + + void Push(FuzzJob *Job) { + { + std::lock_guard Lock(Mu); + Qu.push(Job); + } + Cv.notify_one(); + } + FuzzJob *Pop() { + std::unique_lock Lk(Mu); + // std::lock_guard Lock(Mu); + Cv.wait(Lk, [&]{return !Qu.empty();}); + assert(!Qu.empty()); + auto Job = Qu.front(); + Qu.pop(); + return Job; + } +}; + +void WorkerThread(JobQueue *FuzzQ, JobQueue *MergeQ) { + while (auto Job = FuzzQ->Pop()) { + // Printf("WorkerThread: job %p\n", Job); + Job->ExitCode = ExecuteCommand(Job->Cmd); + MergeQ->Push(Job); + } +} + +// This is just a skeleton of an experimental -fork=1 feature. +void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + const Vector &Args, + const Vector &CorpusDirs, int NumJobs) { + Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs); + + GlobalEnv Env; + Env.Args = Args; + Env.CorpusDirs = CorpusDirs; + Env.Rand = &Rand; + Env.Verbosity = Options.Verbosity; + Env.ProcessStartTime = std::chrono::system_clock::now(); + Env.DataFlowBinary = Options.CollectDataFlow; + + Vector SeedFiles; + for (auto &Dir : CorpusDirs) + GetSizedFilesFromDir(Dir, &SeedFiles); + std::sort(SeedFiles.begin(), SeedFiles.end()); + Env.TempDir = TempPath(".dir"); + Env.DFTDir = DirPlusFile(Env.TempDir, "DFT"); + RmDirRecursive(Env.TempDir); // in case there is a leftover from old runs. + MkDir(Env.TempDir); + MkDir(Env.DFTDir); + + + if (CorpusDirs.empty()) + MkDir(Env.MainCorpusDir = DirPlusFile(Env.TempDir, "C")); + else + Env.MainCorpusDir = CorpusDirs[0]; + + auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); + CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features, + {}, &Env.Cov, + CFPath, false); + RemoveFile(CFPath); + Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs, + Env.Files.size(), Env.TempDir.c_str()); + + int ExitCode = 0; + + JobQueue FuzzQ, MergeQ; + + auto StopJobs = [&]() { + for (int i = 0; i < NumJobs; i++) + FuzzQ.Push(nullptr); + MergeQ.Push(nullptr); + WriteToFile(Unit({1}), Env.StopFile()); + }; + + size_t JobId = 1; + Vector Threads; + for (int t = 0; t < NumJobs; t++) { + Threads.push_back(std::thread(WorkerThread, &FuzzQ, &MergeQ)); + FuzzQ.Push(Env.CreateNewJob(JobId++)); + } + + while (true) { + std::unique_ptr Job(MergeQ.Pop()); + if (!Job) + break; + ExitCode = Job->ExitCode; + if (ExitCode == Options.InterruptExitCode) { + Printf("==%lu== libFuzzer: a child was interrupted; exiting\n", GetPid()); + StopJobs(); + break; + } + Fuzzer::MaybeExitGracefully(); + + Env.RunOneMergeJob(Job.get()); + + // Continue if our crash is one of the ignorred ones. + if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode) + Env.NumTimeouts++; + else if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode) + Env.NumOOMs++; + else if (ExitCode != 0) { + Env.NumCrashes++; + if (Options.IgnoreCrashes) { + std::ifstream In(Job->LogPath); + std::string Line; + while (std::getline(In, Line, '\n')) + if (Line.find("ERROR:") != Line.npos || + Line.find("runtime error:") != Line.npos) + Printf("%s\n", Line.c_str()); + } else { + // And exit if we don't ignore this crash. + Printf("INFO: log from the inner process:\n%s", + FileToString(Job->LogPath).c_str()); + StopJobs(); + break; + } + } + + // Stop if we are over the time budget. + // This is not precise, since other threads are still running + // and we will wait while joining them. + // We also don't stop instantly: other jobs need to finish. + if (Options.MaxTotalTimeSec > 0 && + Env.secondsSinceProcessStartUp() >= (size_t)Options.MaxTotalTimeSec) { + Printf("INFO: fuzzed for %zd seconds, wrapping up soon\n", + Env.secondsSinceProcessStartUp()); + StopJobs(); + break; + } + if (Env.NumRuns >= Options.MaxNumberOfRuns) { + Printf("INFO: fuzzed for %zd iterations, wrapping up soon\n", + Env.NumRuns); + StopJobs(); + break; + } + + FuzzQ.Push(Env.CreateNewJob(JobId++)); + } + + for (auto &T : Threads) + T.join(); + + // The workers have terminated. Don't try to remove the directory before they + // terminate to avoid a race condition preventing cleanup on Windows. + RmDirRecursive(Env.TempDir); + + // Use the exit code from the last child process. + Printf("INFO: exiting: %d time: %zds\n", ExitCode, + Env.secondsSinceProcessStartUp()); + exit(ExitCode); +} + +} // namespace fuzzer diff --git a/libfuzzer/FuzzerFork.h b/libfuzzer/FuzzerFork.h new file mode 100644 index 0000000..b29a43e --- /dev/null +++ b/libfuzzer/FuzzerFork.h @@ -0,0 +1,24 @@ +//===- FuzzerFork.h - run fuzzing in sub-processes --------------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_FORK_H +#define LLVM_FUZZER_FORK_H + +#include "FuzzerDefs.h" +#include "FuzzerOptions.h" +#include "FuzzerRandom.h" + +#include + +namespace fuzzer { +void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + const Vector &Args, + const Vector &CorpusDirs, int NumJobs); +} // namespace fuzzer + +#endif // LLVM_FUZZER_FORK_H diff --git a/libfuzzer/FuzzerIO.cpp b/libfuzzer/FuzzerIO.cpp index f3ead0e..7e5ba30 100644 --- a/libfuzzer/FuzzerIO.cpp +++ b/libfuzzer/FuzzerIO.cpp @@ -1,17 +1,17 @@ //===- FuzzerIO.cpp - IO utils. -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // IO functions. //===----------------------------------------------------------------------===// -#include "FuzzerIO.h" #include "FuzzerDefs.h" #include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" +#include "FuzzerUtil.h" #include #include #include @@ -31,7 +31,7 @@ long GetEpoch(const std::string &Path) { } Unit FileToVector(const std::string &Path, size_t MaxSize, bool ExitOnError) { - std::ifstream T(Path); + std::ifstream T(Path, std::ios::binary); if (ExitOnError && !T) { Printf("No such directory: %s; exiting\n", Path.c_str()); exit(1); @@ -51,7 +51,7 @@ Unit FileToVector(const std::string &Path, size_t MaxSize, bool ExitOnError) { } std::string FileToString(const std::string &Path) { - std::ifstream T(Path); + std::ifstream T(Path, std::ios::binary); return std::string((std::istreambuf_iterator(T)), std::istreambuf_iterator()); } @@ -61,10 +61,19 @@ void CopyFileToErr(const std::string &Path) { } void WriteToFile(const Unit &U, const std::string &Path) { + WriteToFile(U.data(), U.size(), Path); +} + +void WriteToFile(const std::string &Data, const std::string &Path) { + WriteToFile(reinterpret_cast(Data.c_str()), Data.size(), + Path); +} + +void WriteToFile(const uint8_t *Data, size_t Size, const std::string &Path) { // Use raw C interface because this function may be called from a sig handler. - FILE *Out = fopen(Path.c_str(), "w"); + FILE *Out = fopen(Path.c_str(), "wb"); if (!Out) return; - fwrite(U.data(), sizeof(U[0]), U.size(), Out); + fwrite(Data, sizeof(Data[0]), Size, Out); fclose(Out); } @@ -100,14 +109,6 @@ std::string DirPlusFile(const std::string &DirPath, return DirPath + GetSeparator() + FileName; } -std::string Basename(const std::string &Path, char Separator) { - size_t Pos = Path.rfind(Separator); - if (Pos == std::string::npos) - return Path; - assert(Pos < Path.size()); - return Path.substr(Pos + 1); -} - void DupAndCloseStderr() { int OutputFd = DuplicateFile(2); if (OutputFd > 0) { @@ -134,4 +135,25 @@ void Printf(const char *Fmt, ...) { fflush(OutputFile); } +void VPrintf(bool Verbose, const char *Fmt, ...) { + if (!Verbose) return; + va_list ap; + va_start(ap, Fmt); + vfprintf(OutputFile, Fmt, ap); + va_end(ap); + fflush(OutputFile); +} + +void RmDirRecursive(const std::string &Dir) { + IterateDirRecursive( + Dir, [](const std::string &Path) {}, + [](const std::string &Path) { RmDir(Path); }, + [](const std::string &Path) { RemoveFile(Path); }); +} + +std::string TempPath(const char *Extension) { + return DirPlusFile(TmpDir(), + "libFuzzerTemp." + std::to_string(GetPid()) + Extension); +} + } // namespace fuzzer diff --git a/libfuzzer/FuzzerIO.h b/libfuzzer/FuzzerIO.h index 6d77574..fe0d7b4 100644 --- a/libfuzzer/FuzzerIO.h +++ b/libfuzzer/FuzzerIO.h @@ -1,9 +1,8 @@ //===- FuzzerIO.h - Internal header for IO utils ----------------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // IO interface. @@ -25,6 +24,9 @@ std::string FileToString(const std::string &Path); void CopyFileToErr(const std::string &Path); +void WriteToFile(const uint8_t *Data, size_t Size, const std::string &Path); +// Write Data.c_str() to the file without terminating null character. +void WriteToFile(const std::string &Data, const std::string &Path); void WriteToFile(const Unit &U, const std::string &Path); void ReadDirToVectorOfUnits(const char *Path, Vector *V, @@ -40,6 +42,8 @@ std::string DirName(const std::string &FileName); // Returns path to a TmpDir. std::string TmpDir(); +std::string TempPath(const char *Extension); + bool IsInterestingCoverageFile(const std::string &FileName); void DupAndCloseStderr(); @@ -47,6 +51,7 @@ void DupAndCloseStderr(); void CloseStdout(); void Printf(const char *Fmt, ...); +void VPrintf(bool Verbose, const char *Fmt, ...); // Print using raw syscalls, useful when printing at early init stages. void RawPrint(const char *Str); @@ -58,6 +63,16 @@ size_t FileSize(const std::string &Path); void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, Vector *V, bool TopDir); +void RmDirRecursive(const std::string &Dir); + +// Iterate files and dirs inside Dir, recursively. +// Call DirPreCallback/DirPostCallback on dirs before/after +// calling FileCallback on files. +void IterateDirRecursive(const std::string &Dir, + void (*DirPreCallback)(const std::string &Dir), + void (*DirPostCallback)(const std::string &Dir), + void (*FileCallback)(const std::string &Dir)); + struct SizedFile { std::string File; size_t Size; @@ -68,7 +83,7 @@ void GetSizedFilesFromDir(const std::string &Dir, Vector *V); char GetSeparator(); // Similar to the basename utility: returns the file name w/o the dir prefix. -std::string Basename(const std::string &Path, char Separator = GetSeparator()); +std::string Basename(const std::string &Path); FILE* OpenFile(int Fd, const char *Mode); @@ -77,11 +92,17 @@ int CloseFile(int Fd); int DuplicateFile(int Fd); void RemoveFile(const std::string &Path); +void RenameFile(const std::string &OldPath, const std::string &NewPath); void DiscardOutput(int Fd); intptr_t GetHandleFromFd(int fd); +void MkDir(const std::string &Path); +void RmDir(const std::string &Path); + +const std::string &getDevNull(); + } // namespace fuzzer #endif // LLVM_FUZZER_IO_H diff --git a/libfuzzer/FuzzerIOPosix.cpp b/libfuzzer/FuzzerIOPosix.cpp index 17e884d..cfd69bb 100644 --- a/libfuzzer/FuzzerIOPosix.cpp +++ b/libfuzzer/FuzzerIOPosix.cpp @@ -1,9 +1,8 @@ //===- FuzzerIOPosix.cpp - IO utils for Posix. ----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // IO functions implementation using Posix API. @@ -46,6 +45,13 @@ size_t FileSize(const std::string &Path) { return St.st_size; } +std::string Basename(const std::string &Path) { + size_t Pos = Path.rfind(GetSeparator()); + if (Pos == std::string::npos) return Path; + assert(Pos < Path.size()); + return Path.substr(Pos + 1); +} + void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, Vector *V, bool TopDir) { auto E = GetEpoch(Dir); @@ -72,6 +78,28 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, *Epoch = E; } + +void IterateDirRecursive(const std::string &Dir, + void (*DirPreCallback)(const std::string &Dir), + void (*DirPostCallback)(const std::string &Dir), + void (*FileCallback)(const std::string &Dir)) { + DirPreCallback(Dir); + DIR *D = opendir(Dir.c_str()); + if (!D) return; + while (auto E = readdir(D)) { + std::string Path = DirPlusFile(Dir, E->d_name); + if (E->d_type == DT_REG || E->d_type == DT_LNK || + (E->d_type == DT_UNKNOWN && IsFile(Path))) + FileCallback(Path); + else if ((E->d_type == DT_DIR || + (E->d_type == DT_UNKNOWN && IsDirectory(Path))) && + *E->d_name != '.') + IterateDirRecursive(Path, DirPreCallback, DirPostCallback, FileCallback); + } + closedir(D); + DirPostCallback(Dir); +} + char GetSeparator() { return '/'; } @@ -92,6 +120,10 @@ void RemoveFile(const std::string &Path) { unlink(Path.c_str()); } +void RenameFile(const std::string &OldPath, const std::string &NewPath) { + rename(OldPath.c_str(), NewPath.c_str()); +} + void DiscardOutput(int Fd) { FILE* Temp = fopen("/dev/null", "w"); if (!Temp) @@ -130,11 +162,23 @@ bool IsInterestingCoverageFile(const std::string &FileName) { return true; } - void RawPrint(const char *Str) { write(2, Str, strlen(Str)); } +void MkDir(const std::string &Path) { + mkdir(Path.c_str(), 0700); +} + +void RmDir(const std::string &Path) { + rmdir(Path.c_str()); +} + +const std::string &getDevNull() { + static const std::string devNull = "/dev/null"; + return devNull; +} + } // namespace fuzzer #endif // LIBFUZZER_POSIX diff --git a/libfuzzer/FuzzerIOWindows.cpp b/libfuzzer/FuzzerIOWindows.cpp index 7485364..510afeb 100644 --- a/libfuzzer/FuzzerIOWindows.cpp +++ b/libfuzzer/FuzzerIOWindows.cpp @@ -1,9 +1,8 @@ //===- FuzzerIOWindows.cpp - IO utils for Windows. ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // IO functions implementation for Windows. @@ -72,6 +71,33 @@ bool IsFile(const std::string &Path) { return IsFile(Path, Att); } +static bool IsDir(DWORD FileAttrs) { + if (FileAttrs == INVALID_FILE_ATTRIBUTES) return false; + return FileAttrs & FILE_ATTRIBUTE_DIRECTORY; +} + +std::string Basename(const std::string &Path) { + size_t Pos = Path.find_last_of("/\\"); + if (Pos == std::string::npos) return Path; + assert(Pos < Path.size()); + return Path.substr(Pos + 1); +} + +size_t FileSize(const std::string &Path) { + WIN32_FILE_ATTRIBUTE_DATA attr; + if (!GetFileAttributesExA(Path.c_str(), GetFileExInfoStandard, &attr)) { + DWORD LastError = GetLastError(); + if (LastError != ERROR_FILE_NOT_FOUND) + Printf("GetFileAttributesExA() failed for \"%s\" (Error code: %lu).\n", + Path.c_str(), LastError); + return 0; + } + ULARGE_INTEGER size; + size.HighPart = attr.nFileSizeHigh; + size.LowPart = attr.nFileSizeLow; + return size.QuadPart; +} + void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, Vector *V, bool TopDir) { auto E = GetEpoch(Dir); @@ -91,7 +117,7 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, { if (GetLastError() == ERROR_FILE_NOT_FOUND) return; - Printf("No such directory: %s; exiting\n", Dir.c_str()); + Printf("No such file or directory: %s; exiting\n", Dir.c_str()); exit(1); } @@ -121,6 +147,58 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, *Epoch = E; } + +void IterateDirRecursive(const std::string &Dir, + void (*DirPreCallback)(const std::string &Dir), + void (*DirPostCallback)(const std::string &Dir), + void (*FileCallback)(const std::string &Dir)) { + // TODO(metzman): Implement ListFilesInDirRecursive via this function. + DirPreCallback(Dir); + + DWORD DirAttrs = GetFileAttributesA(Dir.c_str()); + if (!IsDir(DirAttrs)) return; + + std::string TargetDir(Dir); + assert(!TargetDir.empty()); + if (TargetDir.back() != '\\') TargetDir.push_back('\\'); + TargetDir.push_back('*'); + + WIN32_FIND_DATAA FindInfo; + // Find the directory's first file. + HANDLE FindHandle = FindFirstFileA(TargetDir.c_str(), &FindInfo); + if (FindHandle == INVALID_HANDLE_VALUE) { + DWORD LastError = GetLastError(); + if (LastError != ERROR_FILE_NOT_FOUND) { + // If the directory isn't empty, then something abnormal is going on. + Printf("FindFirstFileA failed for %s (Error code: %lu).\n", Dir.c_str(), + LastError); + } + return; + } + + do { + std::string Path = DirPlusFile(Dir, FindInfo.cFileName); + DWORD PathAttrs = FindInfo.dwFileAttributes; + if (IsDir(PathAttrs)) { + // Is Path the current directory (".") or the parent ("..")? + if (strcmp(FindInfo.cFileName, ".") == 0 || + strcmp(FindInfo.cFileName, "..") == 0) + continue; + IterateDirRecursive(Path, DirPreCallback, DirPostCallback, FileCallback); + } else if (PathAttrs != INVALID_FILE_ATTRIBUTES) { + FileCallback(Path); + } + } while (FindNextFileA(FindHandle, &FindInfo)); + + DWORD LastError = GetLastError(); + if (LastError != ERROR_NO_MORE_FILES) + Printf("FindNextFileA failed for %s (Error code: %lu).\n", Dir.c_str(), + LastError); + + FindClose(FindHandle); + DirPostCallback(Dir); +} + char GetSeparator() { return '\\'; } @@ -141,6 +219,10 @@ void RemoveFile(const std::string &Path) { _unlink(Path.c_str()); } +void RenameFile(const std::string &OldPath, const std::string &NewPath) { + rename(OldPath.c_str(), NewPath.c_str()); +} + void DiscardOutput(int Fd) { FILE* Temp = fopen("nul", "w"); if (!Temp) @@ -314,8 +396,24 @@ bool IsInterestingCoverageFile(const std::string &FileName) { } void RawPrint(const char *Str) { - // Not tested, may or may not work. Fix if needed. - Printf("%s", Str); + _write(2, Str, strlen(Str)); +} + +void MkDir(const std::string &Path) { + if (CreateDirectoryA(Path.c_str(), nullptr)) return; + Printf("CreateDirectoryA failed for %s (Error code: %lu).\n", Path.c_str(), + GetLastError()); +} + +void RmDir(const std::string &Path) { + if (RemoveDirectoryA(Path.c_str())) return; + Printf("RemoveDirectoryA failed for %s (Error code: %lu).\n", Path.c_str(), + GetLastError()); +} + +const std::string &getDevNull() { + static const std::string devNull = "NUL"; + return devNull; } } // namespace fuzzer diff --git a/libfuzzer/FuzzerInterface.h b/libfuzzer/FuzzerInterface.h index 0f7effb..4f62822 100644 --- a/libfuzzer/FuzzerInterface.h +++ b/libfuzzer/FuzzerInterface.h @@ -1,9 +1,8 @@ //===- FuzzerInterface.h - Interface header for the Fuzzer ------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Define the interface between libFuzzer and the library being tested. @@ -26,25 +25,32 @@ extern "C" { #endif // __cplusplus +// Define FUZZER_INTERFACE_VISIBILITY to set default visibility in a way that +// doesn't break MSVC. +#if defined(_WIN32) +#define FUZZER_INTERFACE_VISIBILITY __declspec(dllexport) +#else +#define FUZZER_INTERFACE_VISIBILITY __attribute__((visibility("default"))) +#endif + // Mandatory user-provided target function. // Executes the code under test with [Data, Data+Size) as the input. // libFuzzer will invoke this function *many* times with different inputs. // Must return 0. -__attribute__((visibility("default"))) int +FUZZER_INTERFACE_VISIBILITY int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); // Optional user-provided initialization function. // If provided, this function will be called by libFuzzer once at startup. // It may read and modify argc/argv. // Must return 0. -__attribute__((visibility("default"))) int LLVMFuzzerInitialize(int *argc, - char ***argv); +FUZZER_INTERFACE_VISIBILITY int LLVMFuzzerInitialize(int *argc, char ***argv); // Optional user-provided custom mutator. // Mutates raw data in [Data, Data+Size) inplace. // Returns the new size, which is not greater than MaxSize. // Given the same Seed produces the same mutation. -__attribute__((visibility("default"))) size_t +FUZZER_INTERFACE_VISIBILITY size_t LLVMFuzzerCustomMutator(uint8_t *Data, size_t Size, size_t MaxSize, unsigned int Seed); @@ -52,7 +58,7 @@ LLVMFuzzerCustomMutator(uint8_t *Data, size_t Size, size_t MaxSize, // Combines pieces of Data1 & Data2 together into Out. // Returns the new size, which is not greater than MaxOutSize. // Should produce the same mutation given the same Seed. -__attribute__((visibility("default"))) size_t +FUZZER_INTERFACE_VISIBILITY size_t LLVMFuzzerCustomCrossOver(const uint8_t *Data1, size_t Size1, const uint8_t *Data2, size_t Size2, uint8_t *Out, size_t MaxOutSize, unsigned int Seed); @@ -61,9 +67,11 @@ LLVMFuzzerCustomCrossOver(const uint8_t *Data1, size_t Size1, // libFuzzer-provided function to be used inside LLVMFuzzerCustomMutator. // Mutates raw data in [Data, Data+Size) inplace. // Returns the new size, which is not greater than MaxSize. -__attribute__((visibility("default"))) size_t +FUZZER_INTERFACE_VISIBILITY size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize); +#undef FUZZER_INTERFACE_VISIBILITY + #ifdef __cplusplus } // extern "C" #endif // __cplusplus diff --git a/libfuzzer/FuzzerInternal.h b/libfuzzer/FuzzerInternal.h index bfc8982..f2a4c43 100644 --- a/libfuzzer/FuzzerInternal.h +++ b/libfuzzer/FuzzerInternal.h @@ -1,9 +1,8 @@ //===- FuzzerInternal.h - Internal header for the Fuzzer --------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Define the main class fuzzer::Fuzzer and most functions. @@ -36,8 +35,8 @@ class Fuzzer { Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, FuzzingOptions Options); ~Fuzzer(); - void Loop(const Vector &CorpusDirs); - void ReadAndExecuteSeedCorpora(const Vector &CorpusDirs); + void Loop(Vector &CorporaFiles); + void ReadAndExecuteSeedCorpora(Vector &CorporaFiles); void MinimizeCrashLoop(const Unit &U); void RereadOutputCorpus(size_t MaxSize); @@ -67,17 +66,11 @@ class Fuzzer { static void StaticGracefulExitCallback(); void ExecuteCallback(const uint8_t *Data, size_t Size); - void CheckForUnstableCounters(const uint8_t *Data, size_t Size); bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false, InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr); // Merge Corpora[1:] into Corpora[0]. void Merge(const Vector &Corpora); - void CrashResistantMerge(const Vector &Args, - const Vector &Corpora, - const char *CoverageSummaryInputPathOrNull, - const char *CoverageSummaryOutputPathOrNull, - const char *MergeControlFilePathOrNull); void CrashResistantMergeInternalStep(const std::string &ControlFilePath); MutationDispatcher &GetMD() { return MD; } void PrintFinalStats(); @@ -91,20 +84,19 @@ class Fuzzer { bool DuringInitialCorpusExecution); void HandleMalloc(size_t Size); - void AnnounceOutput(const uint8_t *Data, size_t Size); + static void MaybeExitGracefully(); + std::string WriteToOutputCorpus(const Unit &U); private: void AlarmCallback(); void CrashCallback(); void ExitCallback(); - void MaybeExitGracefully(); void CrashOnOverwrittenData(); void InterruptCallback(); void MutateAndTestOne(); void PurgeAllocator(); void ReportNewCoverage(InputInfo *II, const Unit &U); void PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size); - void WriteToOutputCorpus(const Unit &U); void WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix); void PrintStats(const char *Where, const char *End = "\n", size_t Units = 0); void PrintStatusForNewUnit(const Unit &U, const char *Text); diff --git a/libfuzzer/FuzzerLoop.cpp b/libfuzzer/FuzzerLoop.cpp index 4bc8836..f773f9a 100644 --- a/libfuzzer/FuzzerLoop.cpp +++ b/libfuzzer/FuzzerLoop.cpp @@ -1,9 +1,8 @@ //===- FuzzerLoop.cpp - Fuzzer's main loop --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Fuzzer's main loop. @@ -14,7 +13,6 @@ #include "FuzzerInternal.h" #include "FuzzerMutate.h" #include "FuzzerRandom.h" -#include "FuzzerShmem.h" #include "FuzzerTracePC.h" #include #include @@ -41,8 +39,6 @@ static const size_t kMaxUnitSizeToPrint = 256; thread_local bool Fuzzer::IsMyThread; -SharedMemoryRegion SMR; - bool RunningUserCallback = false; // Only one Fuzzer per process. @@ -135,7 +131,7 @@ void Fuzzer::HandleMalloc(size_t Size) { DumpCurrentUnit("oom-"); Printf("SUMMARY: libFuzzer: out-of-memory\n"); PrintFinalStats(); - _Exit(Options.ErrorExitCode); // Stop right now. + _Exit(Options.OOMExitCode); // Stop right now. } Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, @@ -157,12 +153,10 @@ Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, if (!Options.OutputCorpus.empty() && Options.ReloadIntervalSec) EpochOfLastReadOfOutputCorpus = GetEpoch(Options.OutputCorpus); MaxInputLen = MaxMutationLen = Options.MaxLen; - TmpMaxMutationLen = Max(size_t(4), Corpus.MaxInputSize()); + TmpMaxMutationLen = 0; // Will be set once we load the corpus. AllocateCurrentUnitData(); CurrentUnitSize = 0; memset(BaseSha1, 0, sizeof(BaseSha1)); - TPC.SetFocusFunction(Options.FocusFunction); - DFT.Init(Options.DataFlowTrace, Options.FocusFunction); } Fuzzer::~Fuzzer() {} @@ -231,8 +225,9 @@ void Fuzzer::StaticFileSizeExceedCallback() { } void Fuzzer::CrashCallback() { - if (EF->__sanitizer_acquire_crash_state) - EF->__sanitizer_acquire_crash_state(); + if (EF->__sanitizer_acquire_crash_state && + !EF->__sanitizer_acquire_crash_state()) + return; Printf("==%lu== ERROR: libFuzzer: deadly signal\n", GetPid()); PrintStackTrace(); Printf("NOTE: libFuzzer has rudimentary signal handlers.\n" @@ -259,23 +254,28 @@ void Fuzzer::ExitCallback() { } void Fuzzer::MaybeExitGracefully() { - if (!GracefulExitRequested) return; + if (!F->GracefulExitRequested) return; Printf("==%lu== INFO: libFuzzer: exiting as requested\n", GetPid()); - PrintFinalStats(); + RmDirRecursive(TempPath(".dir")); + F->PrintFinalStats(); _Exit(0); } void Fuzzer::InterruptCallback() { Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid()); PrintFinalStats(); - _Exit(0); // Stop right now, don't perform any at-exit actions. + ScopedDisableMsanInterceptorChecks S; // RmDirRecursive may call opendir(). + RmDirRecursive(TempPath(".dir")); + // Stop right now, don't perform any at-exit actions. + _Exit(Options.InterruptExitCode); } NO_SANITIZE_MEMORY void Fuzzer::AlarmCallback() { assert(Options.UnitTimeoutSec > 0); // In Windows Alarm callback is executed by a different thread. -#if !LIBFUZZER_WINDOWS + // NetBSD's current behavior needs this change too. +#if !LIBFUZZER_WINDOWS && !LIBFUZZER_NETBSD if (!InFuzzingThread()) return; #endif @@ -316,7 +316,7 @@ void Fuzzer::RssLimitCallback() { DumpCurrentUnit("oom-"); Printf("SUMMARY: libFuzzer: out-of-memory\n"); PrintFinalStats(); - _Exit(Options.ErrorExitCode); // Stop right now. + _Exit(Options.OOMExitCode); // Stop right now. } void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) { @@ -354,13 +354,8 @@ void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) { void Fuzzer::PrintFinalStats() { if (Options.PrintCoverage) TPC.PrintCoverage(); - if (Options.PrintUnstableStats) - TPC.PrintUnstableStats(); - if (Options.DumpCoverage) - TPC.DumpCoverage(); if (Options.PrintCorpusStats) Corpus.PrintStats(); - if (Options.PrintMutationStats) MD.PrintMutationStats(); if (!Options.PrintFinalStats) return; size_t ExecPerSec = execPerSec(); @@ -390,10 +385,10 @@ void Fuzzer::SetMaxMutationLen(size_t MaxMutationLen) { void Fuzzer::CheckExitOnSrcPosOrItem() { if (!Options.ExitOnSrcPos.empty()) { static auto *PCsSet = new Set; - auto HandlePC = [&](uintptr_t PC) { - if (!PCsSet->insert(PC).second) + auto HandlePC = [&](const TracePC::PCTableEntry *TE) { + if (!PCsSet->insert(TE->PC).second) return; - std::string Descr = DescribePC("%F %L", PC + 1); + std::string Descr = DescribePC("%F %L", TE->PC + 1); if (Descr.find(Options.ExitOnSrcPos) != std::string::npos) { Printf("INFO: found line matching '%s', exiting.\n", Options.ExitOnSrcPos.c_str()); @@ -449,32 +444,21 @@ void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) { } } -void Fuzzer::CheckForUnstableCounters(const uint8_t *Data, size_t Size) { - auto CBSetupAndRun = [&]() { - ScopedEnableMsanInterceptorChecks S; - UnitStartTime = system_clock::now(); - TPC.ResetMaps(); - RunningUserCallback = true; - CB(Data, Size); - RunningUserCallback = false; - UnitStopTime = system_clock::now(); - }; - - // Copy original run counters into our unstable counters - TPC.InitializeUnstableCounters(); - - // First Rerun - CBSetupAndRun(); - TPC.UpdateUnstableCounters(Options.HandleUnstable); - - // Second Rerun - CBSetupAndRun(); - TPC.UpdateUnstableCounters(Options.HandleUnstable); +static void WriteFeatureSetToFile(const std::string &FeaturesDir, + const std::string &FileName, + const Vector &FeatureSet) { + if (FeaturesDir.empty() || FeatureSet.empty()) return; + WriteToFile(reinterpret_cast(FeatureSet.data()), + FeatureSet.size() * sizeof(FeatureSet[0]), + DirPlusFile(FeaturesDir, FileName)); +} - // Move minimum hit counts back to ModuleInline8bitCounters - if (Options.HandleUnstable == TracePC::MinUnstable || - Options.HandleUnstable == TracePC::ZeroUnstable) - TPC.ApplyUnstableCounters(); +static void RenameFeatureSetFile(const std::string &FeaturesDir, + const std::string &OldFile, + const std::string &NewFile) { + if (FeaturesDir.empty()) return; + RenameFile(DirPlusFile(FeaturesDir, OldFile), + DirPlusFile(FeaturesDir, NewFile)); } bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, @@ -487,17 +471,6 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, UniqFeatureSetTmp.clear(); size_t FoundUniqFeaturesOfII = 0; size_t NumUpdatesBefore = Corpus.NumFeatureUpdates(); - bool NewFeaturesUnstable = false; - - if (Options.HandleUnstable || Options.PrintUnstableStats) { - TPC.CollectFeatures([&](size_t Feature) { - if (Corpus.IsFeatureNew(Feature, Size, Options.Shrink)) - NewFeaturesUnstable = true; - }); - if (NewFeaturesUnstable) - CheckForUnstableCounters(Data, Size); - } - TPC.CollectFeatures([&](size_t Feature) { if (Corpus.AddFeature(Feature, Size, Options.Shrink)) UniqFeatureSetTmp.push_back(Feature); @@ -506,23 +479,27 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, II->UniqFeatureSet.end(), Feature)) FoundUniqFeaturesOfII++; }); - if (FoundUniqFeatures) *FoundUniqFeatures = FoundUniqFeaturesOfII; PrintPulseAndReportSlowInput(Data, Size); size_t NumNewFeatures = Corpus.NumFeatureUpdates() - NumUpdatesBefore; - if (NumNewFeatures) { TPC.UpdateObservedPCs(); - Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, MayDeleteFile, - TPC.ObservedFocusFunction(), UniqFeatureSetTmp, DFT, II); + auto NewII = Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, + MayDeleteFile, TPC.ObservedFocusFunction(), + UniqFeatureSetTmp, DFT, II); + WriteFeatureSetToFile(Options.FeaturesDir, Sha1ToString(NewII->Sha1), + NewII->UniqFeatureSet); return true; } if (II && FoundUniqFeaturesOfII && II->DataFlowTraceForFocusFunction.empty() && FoundUniqFeaturesOfII == II->UniqFeatureSet.size() && II->U.size() > Size) { + auto OldFeaturesFile = Sha1ToString(II->Sha1); Corpus.Replace(II, {Data, Data + Size}); + RenameFeatureSetFile(Options.FeaturesDir, OldFeaturesFile, + Sha1ToString(II->Sha1)); return true; } return false; @@ -556,14 +533,14 @@ void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { TPC.RecordInitialStack(); TotalNumberOfRuns++; assert(InFuzzingThread()); - if (SMR.IsClient()) - SMR.WriteByteArray(Data, Size); // We copy the contents of Unit into a separate heap buffer // so that we reliably find buffer overflows in it. uint8_t *DataCopy = new uint8_t[Size]; memcpy(DataCopy, Data, Size); if (EF->__msan_unpoison) EF->__msan_unpoison(DataCopy, Size); + if (EF->__msan_unpoison_param) + EF->__msan_unpoison_param(2); if (CurrentUnitData && CurrentUnitData != Data) memcpy(CurrentUnitData, Data, Size); CurrentUnitSize = Size; @@ -586,15 +563,16 @@ void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { delete[] DataCopy; } -void Fuzzer::WriteToOutputCorpus(const Unit &U) { +std::string Fuzzer::WriteToOutputCorpus(const Unit &U) { if (Options.OnlyASCII) assert(IsASCII(U)); if (Options.OutputCorpus.empty()) - return; + return ""; std::string Path = DirPlusFile(Options.OutputCorpus, Hash(U)); WriteToFile(U, Path); if (Options.Verbosity >= 2) Printf("Written %zd bytes to %s\n", U.size(), Path.c_str()); + return Path; } void Fuzzer::WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix) { @@ -680,6 +658,8 @@ void Fuzzer::MutateAndTestOne() { MD.StartMutationSequence(); auto &II = Corpus.ChooseUnitToMutate(MD.GetRand()); + if (Options.DoCrossOver) + MD.SetCrossOverWith(&Corpus.ChooseUnitToMutate(MD.GetRand()).U); const auto &U = II.U; memcpy(BaseSha1, II.Sha1, sizeof(BaseSha1)); assert(CurrentUnitData); @@ -702,7 +682,9 @@ void Fuzzer::MutateAndTestOne() { Size <= CurrentMaxMutationLen) NewSize = MD.MutateWithMask(CurrentUnitData, Size, Size, II.DataFlowTraceForFocusFunction); - else + + // If MutateWithMask either failed or wasn't called, call default Mutate. + if (!NewSize) NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen); assert(NewSize > 0 && "Mutator returned empty unit"); assert(NewSize <= CurrentMaxMutationLen && "Mutator return oversized unit"); @@ -719,7 +701,7 @@ void Fuzzer::MutateAndTestOne() { break; // We will mutate this input more in the next rounds. } if (Options.ReduceDepth && !FoundUniqFeatures) - break; + break; } } @@ -738,21 +720,13 @@ void Fuzzer::PurgeAllocator() { LastAllocatorPurgeAttemptTime = system_clock::now(); } -void Fuzzer::ReadAndExecuteSeedCorpora(const Vector &CorpusDirs) { +void Fuzzer::ReadAndExecuteSeedCorpora(Vector &CorporaFiles) { const size_t kMaxSaneLen = 1 << 20; const size_t kMinDefaultLen = 4096; - Vector SizedFiles; size_t MaxSize = 0; size_t MinSize = -1; size_t TotalSize = 0; - size_t LastNumFiles = 0; - for (auto &Dir : CorpusDirs) { - GetSizedFilesFromDir(Dir, &SizedFiles); - Printf("INFO: % 8zd files found in %s\n", SizedFiles.size() - LastNumFiles, - Dir.c_str()); - LastNumFiles = SizedFiles.size(); - } - for (auto &File : SizedFiles) { + for (auto &File : CorporaFiles) { MaxSize = Max(File.Size, MaxSize); MinSize = Min(File.Size, MinSize); TotalSize += File.Size; @@ -765,24 +739,28 @@ void Fuzzer::ReadAndExecuteSeedCorpora(const Vector &CorpusDirs) { uint8_t dummy = 0; ExecuteCallback(&dummy, 0); - if (SizedFiles.empty()) { + // Protect lazy counters here, after the once-init code has been executed. + if (Options.LazyCounters) + TPC.ProtectLazyCounters(); + + if (CorporaFiles.empty()) { Printf("INFO: A corpus is not provided, starting from an empty corpus\n"); Unit U({'\n'}); // Valid ASCII input. RunOne(U.data(), U.size()); } else { Printf("INFO: seed corpus: files: %zd min: %zdb max: %zdb total: %zdb" " rss: %zdMb\n", - SizedFiles.size(), MinSize, MaxSize, TotalSize, GetPeakRSSMb()); + CorporaFiles.size(), MinSize, MaxSize, TotalSize, GetPeakRSSMb()); if (Options.ShuffleAtStartUp) - std::shuffle(SizedFiles.begin(), SizedFiles.end(), MD.GetRand()); + std::shuffle(CorporaFiles.begin(), CorporaFiles.end(), MD.GetRand()); if (Options.PreferSmall) { - std::stable_sort(SizedFiles.begin(), SizedFiles.end()); - assert(SizedFiles.front().Size <= SizedFiles.back().Size); + std::stable_sort(CorporaFiles.begin(), CorporaFiles.end()); + assert(CorporaFiles.front().Size <= CorporaFiles.back().Size); } // Load and execute inputs one by one. - for (auto &SF : SizedFiles) { + for (auto &SF : CorporaFiles) { auto U = FileToVector(SF.File, MaxInputLen, /*ExitOnError=*/false); assert(U.size() <= MaxInputLen); RunOne(U.data(), U.size()); @@ -807,16 +785,25 @@ void Fuzzer::ReadAndExecuteSeedCorpora(const Vector &CorpusDirs) { } } -void Fuzzer::Loop(const Vector &CorpusDirs) { - ReadAndExecuteSeedCorpora(CorpusDirs); +void Fuzzer::Loop(Vector &CorporaFiles) { + auto FocusFunctionOrAuto = Options.FocusFunction; + DFT.Init(Options.DataFlowTrace, &FocusFunctionOrAuto, CorporaFiles, + MD.GetRand()); + TPC.SetFocusFunction(FocusFunctionOrAuto); + ReadAndExecuteSeedCorpora(CorporaFiles); DFT.Clear(); // No need for DFT any more. TPC.SetPrintNewPCs(Options.PrintNewCovPcs); TPC.SetPrintNewFuncs(Options.PrintNewCovFuncs); system_clock::time_point LastCorpusReload = system_clock::now(); - if (Options.DoCrossOver) - MD.SetCorpus(&Corpus); + + TmpMaxMutationLen = + Min(MaxMutationLen, Max(size_t(4), Corpus.MaxInputSize())); + while (true) { auto Now = system_clock::now(); + if (!Options.StopFile.empty() && + !FileToVector(Options.StopFile, 1, false).empty()) + break; if (duration_cast(Now - LastCorpusReload).count() >= Options.ReloadIntervalSec) { RereadOutputCorpus(MaxInputLen); @@ -867,44 +854,14 @@ void Fuzzer::MinimizeCrashLoop(const Unit &U) { } } -void Fuzzer::AnnounceOutput(const uint8_t *Data, size_t Size) { - if (SMR.IsServer()) { - SMR.WriteByteArray(Data, Size); - } else if (SMR.IsClient()) { - SMR.PostClient(); - SMR.WaitServer(); - size_t OtherSize = SMR.ReadByteArraySize(); - uint8_t *OtherData = SMR.GetByteArray(); - if (Size != OtherSize || memcmp(Data, OtherData, Size) != 0) { - size_t i = 0; - for (i = 0; i < Min(Size, OtherSize); i++) - if (Data[i] != OtherData[i]) - break; - Printf("==%lu== ERROR: libFuzzer: equivalence-mismatch. Sizes: %zd %zd; " - "offset %zd\n", - GetPid(), Size, OtherSize, i); - DumpCurrentUnit("mismatch-"); - Printf("SUMMARY: libFuzzer: equivalence-mismatch\n"); - PrintFinalStats(); - _Exit(Options.ErrorExitCode); - } - } -} - } // namespace fuzzer extern "C" { -__attribute__((visibility("default"))) size_t +ATTRIBUTE_INTERFACE size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) { assert(fuzzer::F); return fuzzer::F->GetMD().DefaultMutate(Data, Size, MaxSize); } -// Experimental -__attribute__((visibility("default"))) void -LLVMFuzzerAnnounceOutput(const uint8_t *Data, size_t Size) { - assert(fuzzer::F); - fuzzer::F->AnnounceOutput(Data, Size); -} } // extern "C" diff --git a/libfuzzer/FuzzerMain.cpp b/libfuzzer/FuzzerMain.cpp index f2c8e9c..771a34a 100644 --- a/libfuzzer/FuzzerMain.cpp +++ b/libfuzzer/FuzzerMain.cpp @@ -1,9 +1,8 @@ //===- FuzzerMain.cpp - main() function and flags -------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // main() and flags. @@ -16,6 +15,6 @@ extern "C" { int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); } // extern "C" -__attribute__((visibility("default"))) int main(int argc, char **argv) { +ATTRIBUTE_INTERFACE int main(int argc, char **argv) { return fuzzer::FuzzerDriver(&argc, &argv, LLVMFuzzerTestOneInput); } diff --git a/libfuzzer/FuzzerMerge.cpp b/libfuzzer/FuzzerMerge.cpp index 5f3052a..75b2b5d 100644 --- a/libfuzzer/FuzzerMerge.cpp +++ b/libfuzzer/FuzzerMerge.cpp @@ -1,9 +1,8 @@ //===- FuzzerMerge.cpp - merging corpora ----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Merging corpora. @@ -43,10 +42,12 @@ void Merger::ParseOrExit(std::istream &IS, bool ParseCoverage) { // file1 // file2 # One file name per line. // STARTED 0 123 # FileID, file size -// DONE 0 1 4 6 8 # FileID COV1 COV2 ... -// STARTED 1 456 # If DONE is missing, the input crashed while processing. +// FT 0 1 4 6 8 # FileID COV1 COV2 ... +// COV 0 7 8 9 # FileID COV1 COV1 +// STARTED 1 456 # If FT is missing, the input crashed while processing. // STARTED 2 567 -// DONE 2 8 9 +// FT 2 8 9 +// COV 2 11 12 bool Merger::Parse(std::istream &IS, bool ParseCoverage) { LastFailure.clear(); std::string Line; @@ -71,11 +72,12 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { if (!std::getline(IS, Files[i].Name, '\n')) return false; - // Parse STARTED and DONE lines. + // Parse STARTED, FT, and COV lines. size_t ExpectedStartMarker = 0; const size_t kInvalidStartMarker = -1; size_t LastSeenStartMarker = kInvalidStartMarker; Vector TmpFeatures; + Set PCs; while (std::getline(IS, Line, '\n')) { std::istringstream ISS1(Line); std::string Marker; @@ -90,19 +92,25 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { LastSeenStartMarker = ExpectedStartMarker; assert(ExpectedStartMarker < Files.size()); ExpectedStartMarker++; - } else if (Marker == "DONE") { - // DONE FILE_ID COV1 COV2 COV3 ... + } else if (Marker == "FT") { + // FT FILE_ID COV1 COV2 COV3 ... size_t CurrentFileIdx = N; if (CurrentFileIdx != LastSeenStartMarker) return false; LastSeenStartMarker = kInvalidStartMarker; if (ParseCoverage) { TmpFeatures.clear(); // use a vector from outer scope to avoid resizes. - while (ISS1 >> std::hex >> N) + while (ISS1 >> N) TmpFeatures.push_back(N); std::sort(TmpFeatures.begin(), TmpFeatures.end()); Files[CurrentFileIdx].Features = TmpFeatures; } + } else if (Marker == "COV") { + size_t CurrentFileIdx = N; + if (ParseCoverage) + while (ISS1 >> N) + if (PCs.insert(N).second) + Files[CurrentFileIdx].Cov.push_back(N); } else { return false; } @@ -121,21 +129,21 @@ size_t Merger::ApproximateMemoryConsumption() const { return Res; } -// Decides which files need to be merged (add thost to NewFiles). +// Decides which files need to be merged (add those to NewFiles). // Returns the number of new features added. size_t Merger::Merge(const Set &InitialFeatures, + Set *NewFeatures, + const Set &InitialCov, Set *NewCov, Vector *NewFiles) { NewFiles->clear(); assert(NumFilesInFirstCorpus <= Files.size()); - Set AllFeatures(InitialFeatures); + Set AllFeatures = InitialFeatures; // What features are in the initial corpus? for (size_t i = 0; i < NumFilesInFirstCorpus; i++) { auto &Cur = Files[i].Features; AllFeatures.insert(Cur.begin(), Cur.end()); } - size_t InitialNumFeatures = AllFeatures.size(); - // Remove all features that we already know from all other inputs. for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) { auto &Cur = Files[i].Features; @@ -161,22 +169,20 @@ size_t Merger::Merge(const Set &InitialFeatures, auto &Cur = Files[i].Features; // Printf("%s -> sz %zd ft %zd\n", Files[i].Name.c_str(), // Files[i].Size, Cur.size()); - size_t OldSize = AllFeatures.size(); - AllFeatures.insert(Cur.begin(), Cur.end()); - if (AllFeatures.size() > OldSize) + bool FoundNewFeatures = false; + for (auto Fe: Cur) { + if (AllFeatures.insert(Fe).second) { + FoundNewFeatures = true; + NewFeatures->insert(Fe); + } + } + if (FoundNewFeatures) NewFiles->push_back(Files[i].Name); + for (auto Cov : Files[i].Cov) + if (InitialCov.find(Cov) == InitialCov.end()) + NewCov->insert(Cov); } - return AllFeatures.size() - InitialNumFeatures; -} - -void Merger::PrintSummary(std::ostream &OS) { - for (auto &File : Files) { - OS << std::hex; - OS << File.Name << " size: " << File.Size << " features: "; - for (auto Feature : File.Features) - OS << " " << Feature; - OS << "\n"; - } + return NewFeatures->size(); } Set Merger::AllFeatures() const { @@ -186,25 +192,6 @@ Set Merger::AllFeatures() const { return S; } -Set Merger::ParseSummary(std::istream &IS) { - std::string Line, Tmp; - Set Res; - while (std::getline(IS, Line, '\n')) { - size_t N; - std::istringstream ISS1(Line); - ISS1 >> Tmp; // Name - ISS1 >> Tmp; // size: - assert(Tmp == "size:" && "Corrupt summary file"); - ISS1 >> std::hex; - ISS1 >> N; // File Size - ISS1 >> Tmp; // features: - assert(Tmp == "features:" && "Corrupt summary file"); - while (ISS1 >> std::hex >> N) - Res.insert(N); - } - return Res; -} - // Inner process. May crash if the target crashes. void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { Printf("MERGE-INNER: using the control file '%s'\n", CFPath.c_str()); @@ -223,8 +210,9 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { std::ofstream OF(CFPath, std::ofstream::out | std::ofstream::app); Set AllFeatures; + Set AllPCs; for (size_t i = M.FirstNotProcessedFile; i < M.Files.size(); i++) { - MaybeExitGracefully(); + Fuzzer::MaybeExitGracefully(); auto U = FileToVector(M.Files[i].Name); if (U.size() > MaxInputLen) { U.resize(MaxInputLen); @@ -232,7 +220,7 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { } std::ostringstream StartedLine; // Write the pre-run marker. - OF << "STARTED " << std::dec << i << " " << U.size() << "\n"; + OF << "STARTED " << i << " " << U.size() << "\n"; OF.flush(); // Flush is important since Command::Execute may crash. // Run. TPC.ResetMaps(); @@ -247,26 +235,36 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { if (AllFeatures.insert(Feature).second) UniqFeatures.insert(Feature); }); + TPC.UpdateObservedPCs(); // Show stats. if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1))) PrintStats("pulse "); // Write the post-run marker and the coverage. - OF << "DONE " << i; + OF << "FT " << i; for (size_t F : UniqFeatures) - OF << " " << std::hex << F; + OF << " " << F; + OF << "\n"; + OF << "COV " << i; + TPC.ForEachObservedPC([&](const TracePC::PCTableEntry *TE) { + if (AllPCs.insert(TE).second) + OF << " " << TPC.PCTableEntryIdx(TE); + }); OF << "\n"; OF.flush(); } + PrintStats("DONE "); } static void WriteNewControlFile(const std::string &CFPath, - const Vector &AllFiles, - size_t NumFilesInFirstCorpus) { + const Vector &OldCorpus, + const Vector &NewCorpus) { RemoveFile(CFPath); std::ofstream ControlFile(CFPath); - ControlFile << AllFiles.size() << "\n"; - ControlFile << NumFilesInFirstCorpus << "\n"; - for (auto &SF: AllFiles) + ControlFile << (OldCorpus.size() + NewCorpus.size()) << "\n"; + ControlFile << OldCorpus.size() << "\n"; + for (auto &SF: OldCorpus) + ControlFile << SF.File << "\n"; + for (auto &SF: NewCorpus) ControlFile << SF.File << "\n"; if (!ControlFile) { Printf("MERGE-OUTER: failed to write to the control file: %s\n", @@ -275,116 +273,90 @@ static void WriteNewControlFile(const std::string &CFPath, } } -// Outer process. Does not call the target code and thus sohuld not fail. -void Fuzzer::CrashResistantMerge(const Vector &Args, - const Vector &Corpora, - const char *CoverageSummaryInputPathOrNull, - const char *CoverageSummaryOutputPathOrNull, - const char *MergeControlFilePathOrNull) { - if (Corpora.size() <= 1) { - Printf("Merge requires two or more corpus dirs\n"); - return; - } - auto CFPath = - MergeControlFilePathOrNull - ? MergeControlFilePathOrNull - : DirPlusFile(TmpDir(), - "libFuzzerTemp." + std::to_string(GetPid()) + ".txt"); - +// Outer process. Does not call the target code and thus should not fail. +void CrashResistantMerge(const Vector &Args, + const Vector &OldCorpus, + const Vector &NewCorpus, + Vector *NewFiles, + const Set &InitialFeatures, + Set *NewFeatures, + const Set &InitialCov, + Set *NewCov, + const std::string &CFPath, + bool V /*Verbose*/) { + if (NewCorpus.empty() && OldCorpus.empty()) return; // Nothing to merge. size_t NumAttempts = 0; - if (MergeControlFilePathOrNull && FileSize(MergeControlFilePathOrNull)) { - Printf("MERGE-OUTER: non-empty control file provided: '%s'\n", - MergeControlFilePathOrNull); + if (FileSize(CFPath)) { + VPrintf(V, "MERGE-OUTER: non-empty control file provided: '%s'\n", + CFPath.c_str()); Merger M; - std::ifstream IF(MergeControlFilePathOrNull); + std::ifstream IF(CFPath); if (M.Parse(IF, /*ParseCoverage=*/false)) { - Printf("MERGE-OUTER: control file ok, %zd files total," + VPrintf(V, "MERGE-OUTER: control file ok, %zd files total," " first not processed file %zd\n", M.Files.size(), M.FirstNotProcessedFile); if (!M.LastFailure.empty()) - Printf("MERGE-OUTER: '%s' will be skipped as unlucky " + VPrintf(V, "MERGE-OUTER: '%s' will be skipped as unlucky " "(merge has stumbled on it the last time)\n", M.LastFailure.c_str()); if (M.FirstNotProcessedFile >= M.Files.size()) { - Printf("MERGE-OUTER: nothing to do, merge has been completed before\n"); + VPrintf( + V, "MERGE-OUTER: nothing to do, merge has been completed before\n"); exit(0); } NumAttempts = M.Files.size() - M.FirstNotProcessedFile; } else { - Printf("MERGE-OUTER: bad control file, will overwrite it\n"); + VPrintf(V, "MERGE-OUTER: bad control file, will overwrite it\n"); } } if (!NumAttempts) { // The supplied control file is empty or bad, create a fresh one. - Vector AllFiles; - GetSizedFilesFromDir(Corpora[0], &AllFiles); - size_t NumFilesInFirstCorpus = AllFiles.size(); - std::sort(AllFiles.begin(), AllFiles.end()); - for (size_t i = 1; i < Corpora.size(); i++) - GetSizedFilesFromDir(Corpora[i], &AllFiles); - std::sort(AllFiles.begin() + NumFilesInFirstCorpus, AllFiles.end()); - Printf("MERGE-OUTER: %zd files, %zd in the initial corpus\n", - AllFiles.size(), NumFilesInFirstCorpus); - WriteNewControlFile(CFPath, AllFiles, NumFilesInFirstCorpus); - NumAttempts = AllFiles.size(); + NumAttempts = OldCorpus.size() + NewCorpus.size(); + VPrintf(V, "MERGE-OUTER: %zd files, %zd in the initial corpus\n", + NumAttempts, OldCorpus.size()); + WriteNewControlFile(CFPath, OldCorpus, NewCorpus); } // Execute the inner process until it passes. // Every inner process should execute at least one input. Command BaseCmd(Args); BaseCmd.removeFlag("merge"); - bool Success = false; + BaseCmd.removeFlag("fork"); + BaseCmd.removeFlag("collect_data_flow"); for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) { - MaybeExitGracefully(); - Printf("MERGE-OUTER: attempt %zd\n", Attempt); + Fuzzer::MaybeExitGracefully(); + VPrintf(V, "MERGE-OUTER: attempt %zd\n", Attempt); Command Cmd(BaseCmd); Cmd.addFlag("merge_control_file", CFPath); Cmd.addFlag("merge_inner", "1"); + if (!V) { + Cmd.setOutputFile(getDevNull()); + Cmd.combineOutAndErr(); + } auto ExitCode = ExecuteCommand(Cmd); if (!ExitCode) { - Printf("MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt); - Success = true; + VPrintf(V, "MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt); break; } } - if (!Success) { - Printf("MERGE-OUTER: zero succesfull attempts, exiting\n"); - exit(1); - } // Read the control file and do the merge. Merger M; std::ifstream IF(CFPath); IF.seekg(0, IF.end); - Printf("MERGE-OUTER: the control file has %zd bytes\n", (size_t)IF.tellg()); + VPrintf(V, "MERGE-OUTER: the control file has %zd bytes\n", + (size_t)IF.tellg()); IF.seekg(0, IF.beg); M.ParseOrExit(IF, true); IF.close(); - Printf("MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n", - M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb()); - if (CoverageSummaryOutputPathOrNull) { - Printf("MERGE-OUTER: writing coverage summary for %zd files to %s\n", - M.Files.size(), CoverageSummaryOutputPathOrNull); - std::ofstream SummaryOut(CoverageSummaryOutputPathOrNull); - M.PrintSummary(SummaryOut); - } - Vector NewFiles; - Set InitialFeatures; - if (CoverageSummaryInputPathOrNull) { - std::ifstream SummaryIn(CoverageSummaryInputPathOrNull); - InitialFeatures = M.ParseSummary(SummaryIn); - Printf("MERGE-OUTER: coverage summary loaded from %s, %zd features found\n", - CoverageSummaryInputPathOrNull, InitialFeatures.size()); - } - size_t NumNewFeatures = M.Merge(InitialFeatures, &NewFiles); - Printf("MERGE-OUTER: %zd new files with %zd new features added\n", - NewFiles.size(), NumNewFeatures); - for (auto &F: NewFiles) - WriteToOutputCorpus(FileToVector(F, MaxInputLen)); - // We are done, delete the control file if it was a temporary one. - if (!MergeControlFilePathOrNull) - RemoveFile(CFPath); + VPrintf(V, + "MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n", + M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb()); + M.Merge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles); + VPrintf(V, "MERGE-OUTER: %zd new files with %zd new features added; " + "%zd new coverage edges\n", + NewFiles->size(), NewFeatures->size(), NewCov->size()); } } // namespace fuzzer diff --git a/libfuzzer/FuzzerMerge.h b/libfuzzer/FuzzerMerge.h index e54885a..c14dd58 100644 --- a/libfuzzer/FuzzerMerge.h +++ b/libfuzzer/FuzzerMerge.h @@ -1,9 +1,8 @@ //===- FuzzerMerge.h - merging corpa ----------------------------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Merging Corpora. @@ -52,7 +51,7 @@ namespace fuzzer { struct MergeFileInfo { std::string Name; size_t Size = 0; - Vector Features; + Vector Features, Cov; }; struct Merger { @@ -64,17 +63,24 @@ struct Merger { bool Parse(std::istream &IS, bool ParseCoverage); bool Parse(const std::string &Str, bool ParseCoverage); void ParseOrExit(std::istream &IS, bool ParseCoverage); - void PrintSummary(std::ostream &OS); - Set ParseSummary(std::istream &IS); - size_t Merge(const Set &InitialFeatures, + size_t Merge(const Set &InitialFeatures, Set *NewFeatures, + const Set &InitialCov, Set *NewCov, Vector *NewFiles); - size_t Merge(Vector *NewFiles) { - return Merge(Set{}, NewFiles); - } size_t ApproximateMemoryConsumption() const; Set AllFeatures() const; }; +void CrashResistantMerge(const Vector &Args, + const Vector &OldCorpus, + const Vector &NewCorpus, + Vector *NewFiles, + const Set &InitialFeatures, + Set *NewFeatures, + const Set &InitialCov, + Set *NewCov, + const std::string &CFPath, + bool Verbose); + } // namespace fuzzer #endif // LLVM_FUZZER_MERGE_H diff --git a/libfuzzer/FuzzerMutate.cpp b/libfuzzer/FuzzerMutate.cpp index ff076cc..29541ea 100644 --- a/libfuzzer/FuzzerMutate.cpp +++ b/libfuzzer/FuzzerMutate.cpp @@ -1,20 +1,19 @@ //===- FuzzerMutate.cpp - Mutate a test input -----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Mutate a test input. //===----------------------------------------------------------------------===// -#include "FuzzerMutate.h" -#include "FuzzerCorpus.h" #include "FuzzerDefs.h" #include "FuzzerExtFunctions.h" #include "FuzzerIO.h" +#include "FuzzerMutate.h" #include "FuzzerOptions.h" +#include "FuzzerTracePC.h" namespace fuzzer { @@ -30,36 +29,34 @@ MutationDispatcher::MutationDispatcher(Random &Rand, DefaultMutators.insert( DefaultMutators.begin(), { - {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes", 0, 0}, - {&MutationDispatcher::Mutate_InsertByte, "InsertByte", 0, 0}, + {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes"}, + {&MutationDispatcher::Mutate_InsertByte, "InsertByte"}, {&MutationDispatcher::Mutate_InsertRepeatedBytes, - "InsertRepeatedBytes", 0, 0}, - {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte", 0, 0}, - {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit", 0, 0}, - {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes", 0, 0}, - {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt", 0, - 0}, - {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt", 0, - 0}, - {&MutationDispatcher::Mutate_CopyPart, "CopyPart", 0, 0}, - {&MutationDispatcher::Mutate_CrossOver, "CrossOver", 0, 0}, + "InsertRepeatedBytes"}, + {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte"}, + {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit"}, + {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes"}, + {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt"}, + {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt"}, + {&MutationDispatcher::Mutate_CopyPart, "CopyPart"}, + {&MutationDispatcher::Mutate_CrossOver, "CrossOver"}, {&MutationDispatcher::Mutate_AddWordFromManualDictionary, - "ManualDict", 0, 0}, + "ManualDict"}, {&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary, - "PersAutoDict", 0, 0}, + "PersAutoDict"}, }); if(Options.UseCmp) DefaultMutators.push_back( - {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP", 0, 0}); + {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"}); if (EF->LLVMFuzzerCustomMutator) - Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom", 0, 0}); + Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"}); else Mutators = DefaultMutators; if (EF->LLVMFuzzerCustomCrossOver) Mutators.push_back( - {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver", 0, 0}); + {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"}); } static char RandCh(Random &Rand) { @@ -75,10 +72,10 @@ size_t MutationDispatcher::Mutate_Custom(uint8_t *Data, size_t Size, size_t MutationDispatcher::Mutate_CustomCrossOver(uint8_t *Data, size_t Size, size_t MaxSize) { - if (!Corpus || Corpus->size() < 2 || Size == 0) + if (Size == 0) return 0; - size_t Idx = Rand(Corpus->size()); - const Unit &Other = (*Corpus)[Idx]; + if (!CrossOverWith) return 0; + const Unit &Other = *CrossOverWith; if (Other.empty()) return 0; CustomCrossOverInPlaceHere.resize(MaxSize); @@ -424,9 +421,9 @@ size_t MutationDispatcher::Mutate_ChangeBinaryInteger(uint8_t *Data, size_t MutationDispatcher::Mutate_CrossOver(uint8_t *Data, size_t Size, size_t MaxSize) { if (Size > MaxSize) return 0; - if (!Corpus || Corpus->size() < 2 || Size == 0) return 0; - size_t Idx = Rand(Corpus->size()); - const Unit &O = (*Corpus)[Idx]; + if (Size == 0) return 0; + if (!CrossOverWith) return 0; + const Unit &O = *CrossOverWith; if (O.empty()) return 0; MutateInPlaceHere.resize(MaxSize); auto &U = MutateInPlaceHere; @@ -466,7 +463,6 @@ void MutationDispatcher::RecordSuccessfulMutationSequence() { if (!PersistentAutoDictionary.ContainsWord(DE->GetW())) PersistentAutoDictionary.push_back({DE->GetW(), 1}); } - RecordUsefulMutations(); } void MutationDispatcher::PrintRecommendedDictionary() { @@ -487,7 +483,8 @@ void MutationDispatcher::PrintRecommendedDictionary() { void MutationDispatcher::PrintMutationSequence() { Printf("MS: %zd ", CurrentMutatorSequence.size()); - for (auto M : CurrentMutatorSequence) Printf("%s-", M->Name); + for (auto M : CurrentMutatorSequence) + Printf("%s-", M.Name); if (!CurrentDictionaryEntrySequence.empty()) { Printf(" DE: "); for (auto DE : CurrentDictionaryEntrySequence) { @@ -515,13 +512,12 @@ size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size, // in which case they will return 0. // Try several times before returning un-mutated data. for (int Iter = 0; Iter < 100; Iter++) { - auto M = &Mutators[Rand(Mutators.size())]; - size_t NewSize = (this->*(M->Fn))(Data, Size, MaxSize); + auto M = Mutators[Rand(Mutators.size())]; + size_t NewSize = (this->*(M.Fn))(Data, Size, MaxSize); if (NewSize && NewSize <= MaxSize) { if (Options.OnlyASCII) ToASCII(Data, NewSize); CurrentMutatorSequence.push_back(M); - M->TotalCount++; return NewSize; } } @@ -533,7 +529,7 @@ size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size, size_t MutationDispatcher::MutateWithMask(uint8_t *Data, size_t Size, size_t MaxSize, const Vector &Mask) { - assert(Size <= Mask.size()); + size_t MaskedSize = std::min(Size, Mask.size()); // * Copy the worthy bytes into a temporary array T // * Mutate T // * Copy T back. @@ -542,16 +538,17 @@ size_t MutationDispatcher::MutateWithMask(uint8_t *Data, size_t Size, if (T.size() < Size) T.resize(Size); size_t OneBits = 0; - for (size_t I = 0; I < Size; I++) + for (size_t I = 0; I < MaskedSize; I++) if (Mask[I]) T[OneBits++] = Data[I]; + if (!OneBits) return 0; assert(!T.empty()); size_t NewSize = Mutate(T.data(), OneBits, OneBits); assert(NewSize <= OneBits); (void)NewSize; // Even if NewSize < OneBits we still use all OneBits bytes. - for (size_t I = 0, J = 0; I < Size; I++) + for (size_t I = 0, J = 0; I < MaskedSize; I++) if (Mask[I]) Data[I] = T[J++]; return Size; @@ -562,21 +559,4 @@ void MutationDispatcher::AddWordToManualDictionary(const Word &W) { {W, std::numeric_limits::max()}); } -void MutationDispatcher::RecordUsefulMutations() { - for (auto M : CurrentMutatorSequence) M->UsefulCount++; -} - -void MutationDispatcher::PrintMutationStats() { - Printf("\nstat::mutation_usefulness: "); - for (size_t i = 0; i < Mutators.size(); i++) { - double UsefulPercentage = - Mutators[i].TotalCount - ? (100.0 * Mutators[i].UsefulCount) / Mutators[i].TotalCount - : 0; - Printf("%.3f", UsefulPercentage); - if (i < Mutators.size() - 1) Printf(","); - } - Printf("\n"); -} - } // namespace fuzzer diff --git a/libfuzzer/FuzzerMutate.h b/libfuzzer/FuzzerMutate.h index 828ecc1..6cbce80 100644 --- a/libfuzzer/FuzzerMutate.h +++ b/libfuzzer/FuzzerMutate.h @@ -1,9 +1,8 @@ //===- FuzzerMutate.h - Internal header for the Fuzzer ----------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::MutationDispatcher @@ -64,7 +63,7 @@ class MutationDispatcher { /// Change a 1-, 2-, 4-, or 8-byte integer in interesting ways. size_t Mutate_ChangeBinaryInteger(uint8_t *Data, size_t Size, size_t MaxSize); - /// CrossOver Data with some other element of the corpus. + /// CrossOver Data with CrossOverWith. size_t Mutate_CrossOver(uint8_t *Data, size_t Size, size_t MaxSize); /// Applies one of the configured mutations. @@ -89,20 +88,14 @@ class MutationDispatcher { void PrintRecommendedDictionary(); - void SetCorpus(const InputCorpus *Corpus) { this->Corpus = Corpus; } + void SetCrossOverWith(const Unit *U) { CrossOverWith = U; } Random &GetRand() { return Rand; } - void PrintMutationStats(); - - void RecordUsefulMutations(); - private: struct Mutator { size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max); const char *Name; - uint64_t UsefulCount; - uint64_t TotalCount; }; size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size, @@ -141,13 +134,12 @@ class MutationDispatcher { Dictionary PersistentAutoDictionary; Vector CurrentDictionaryEntrySequence; - Vector CurrentMutatorSequence; static const size_t kCmpDictionaryEntriesDequeSize = 16; DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize]; size_t CmpDictionaryEntriesDequeIdx = 0; - const InputCorpus *Corpus = nullptr; + const Unit *CrossOverWith = nullptr; Vector MutateInPlaceHere; Vector MutateWithMaskTemp; // CustomCrossOver needs its own buffer as a custom implementation may call @@ -156,6 +148,7 @@ class MutationDispatcher { Vector Mutators; Vector DefaultMutators; + Vector CurrentMutatorSequence; }; } // namespace fuzzer diff --git a/libfuzzer/FuzzerOptions.h b/libfuzzer/FuzzerOptions.h index ce39c08..ad3df01 100644 --- a/libfuzzer/FuzzerOptions.h +++ b/libfuzzer/FuzzerOptions.h @@ -1,8 +1,7 @@ // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::FuzzingOptions @@ -20,8 +19,13 @@ struct FuzzingOptions { size_t MaxLen = 0; size_t LenControl = 1000; int UnitTimeoutSec = 300; - int TimeoutExitCode = 77; + int TimeoutExitCode = 70; + int OOMExitCode = 71; + int InterruptExitCode = 72; int ErrorExitCode = 77; + bool IgnoreTimeouts = true; + bool IgnoreOOMs = true; + bool IgnoreCrashes = false; int MaxTotalTimeSec = 0; int RssLimitMb = 0; int MallocLimitMb = 0; @@ -47,16 +51,16 @@ struct FuzzingOptions { std::string ExitOnItem; std::string FocusFunction; std::string DataFlowTrace; + std::string CollectDataFlow; + std::string FeaturesDir; + std::string StopFile; bool SaveArtifacts = true; bool PrintNEW = true; // Print a status line when new units are found; bool PrintNewCovPcs = false; int PrintNewCovFuncs = 0; bool PrintFinalStats = false; - bool PrintMutationStats = false; bool PrintCorpusStats = false; bool PrintCoverage = false; - bool PrintUnstableStats = false; - int HandleUnstable = 0; bool DumpCoverage = false; bool DetectLeaks = true; int PurgeAllocatorIntervalSec = 1; @@ -71,6 +75,7 @@ struct FuzzingOptions { bool HandleXfsz = false; bool HandleUsr1 = false; bool HandleUsr2 = false; + bool LazyCounters = false; }; } // namespace fuzzer diff --git a/libfuzzer/FuzzerRandom.h b/libfuzzer/FuzzerRandom.h index 8a1aa3e..659283e 100644 --- a/libfuzzer/FuzzerRandom.h +++ b/libfuzzer/FuzzerRandom.h @@ -1,9 +1,8 @@ //===- FuzzerRandom.h - Internal header for the Fuzzer ----------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::Random @@ -15,12 +14,17 @@ #include namespace fuzzer { -class Random : public std::mt19937 { +class Random : public std::minstd_rand { public: - Random(unsigned int seed) : std::mt19937(seed) {} - result_type operator()() { return this->std::mt19937::operator()(); } + Random(unsigned int seed) : std::minstd_rand(seed) {} + result_type operator()() { return this->std::minstd_rand::operator()(); } size_t Rand() { return this->operator()(); } size_t RandBool() { return Rand() % 2; } + size_t SkewTowardsLast(size_t n) { + size_t T = this->operator()(n * n); + size_t Res = sqrt(T); + return Res; + } size_t operator()(size_t n) { return n ? Rand() % n : 0; } intptr_t operator()(intptr_t From, intptr_t To) { assert(From < To); diff --git a/libfuzzer/FuzzerSHA1.cpp b/libfuzzer/FuzzerSHA1.cpp index d2f8e81..43e5e78 100644 --- a/libfuzzer/FuzzerSHA1.cpp +++ b/libfuzzer/FuzzerSHA1.cpp @@ -1,9 +1,8 @@ //===- FuzzerSHA1.h - Private copy of the SHA1 implementation ---*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This code is taken from public domain @@ -32,7 +31,8 @@ namespace { // Added for LibFuzzer #ifdef __BIG_ENDIAN__ # define SHA_BIG_ENDIAN -#elif defined __LITTLE_ENDIAN__ +// Windows is always little endian and MSVC doesn't have +#elif defined __LITTLE_ENDIAN__ || LIBFUZZER_WINDOWS /* override */ #elif defined __BYTE_ORDER # if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ diff --git a/libfuzzer/FuzzerSHA1.h b/libfuzzer/FuzzerSHA1.h index 3b5e6e8..05cbacd 100644 --- a/libfuzzer/FuzzerSHA1.h +++ b/libfuzzer/FuzzerSHA1.h @@ -1,9 +1,8 @@ //===- FuzzerSHA1.h - Internal header for the SHA1 utils --------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // SHA1 utils. diff --git a/libfuzzer/FuzzerShmem.h b/libfuzzer/FuzzerShmem.h deleted file mode 100644 index 53568e0..0000000 --- a/libfuzzer/FuzzerShmem.h +++ /dev/null @@ -1,69 +0,0 @@ -//===- FuzzerShmem.h - shared memory interface ------------------*- C++ -* ===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// SharedMemoryRegion -//===----------------------------------------------------------------------===// - -#ifndef LLVM_FUZZER_SHMEM_H -#define LLVM_FUZZER_SHMEM_H - -#include -#include -#include - -#include "FuzzerDefs.h" - -namespace fuzzer { - -class SharedMemoryRegion { - public: - bool Create(const char *Name); - bool Open(const char *Name); - bool Destroy(const char *Name); - uint8_t *GetData() { return Data; } - void PostServer() {Post(0);} - void WaitServer() {Wait(0);} - void PostClient() {Post(1);} - void WaitClient() {Wait(1);} - - size_t WriteByteArray(const uint8_t *Bytes, size_t N) { - assert(N <= kShmemSize - sizeof(N)); - memcpy(GetData(), &N, sizeof(N)); - memcpy(GetData() + sizeof(N), Bytes, N); - assert(N == ReadByteArraySize()); - return N; - } - size_t ReadByteArraySize() { - size_t Res; - memcpy(&Res, GetData(), sizeof(Res)); - return Res; - } - uint8_t *GetByteArray() { return GetData() + sizeof(size_t); } - - bool IsServer() const { return Data && IAmServer; } - bool IsClient() const { return Data && !IAmServer; } - -private: - - static const size_t kShmemSize = 1 << 22; - bool IAmServer; - std::string Path(const char *Name); - std::string SemName(const char *Name, int Idx); - void Post(int Idx); - void Wait(int Idx); - - bool Map(int fd); - uint8_t *Data = nullptr; - void *Semaphore[2]; -}; - -extern SharedMemoryRegion SMR; - -} // namespace fuzzer - -#endif // LLVM_FUZZER_SHMEM_H diff --git a/libfuzzer/FuzzerShmemFuchsia.cpp b/libfuzzer/FuzzerShmemFuchsia.cpp deleted file mode 100644 index e9ce50c..0000000 --- a/libfuzzer/FuzzerShmemFuchsia.cpp +++ /dev/null @@ -1,38 +0,0 @@ -//===- FuzzerShmemPosix.cpp - Posix shared memory ---------------*- C++ -* ===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// SharedMemoryRegion. For Fuchsia, this is just stubs as equivalence servers -// are not currently supported. -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" - -#if LIBFUZZER_FUCHSIA - -#include "FuzzerShmem.h" - -namespace fuzzer { - -bool SharedMemoryRegion::Create(const char *Name) { - return false; -} - -bool SharedMemoryRegion::Open(const char *Name) { - return false; -} - -bool SharedMemoryRegion::Destroy(const char *Name) { - return false; -} - -void SharedMemoryRegion::Post(int Idx) {} - -void SharedMemoryRegion::Wait(int Idx) {} - -} // namespace fuzzer - -#endif // LIBFUZZER_FUCHSIA diff --git a/libfuzzer/FuzzerShmemPosix.cpp b/libfuzzer/FuzzerShmemPosix.cpp deleted file mode 100644 index 41a93f6..0000000 --- a/libfuzzer/FuzzerShmemPosix.cpp +++ /dev/null @@ -1,108 +0,0 @@ -//===- FuzzerShmemPosix.cpp - Posix shared memory ---------------*- C++ -* ===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// SharedMemoryRegion -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" -#if LIBFUZZER_POSIX - -#include "FuzzerIO.h" -#include "FuzzerShmem.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace fuzzer { - -std::string SharedMemoryRegion::Path(const char *Name) { - return DirPlusFile(TmpDir(), Name); -} - -std::string SharedMemoryRegion::SemName(const char *Name, int Idx) { - std::string Res(Name); - // When passing a name without a leading character to - // sem_open, the behaviour is unspecified in POSIX. Add a leading - // character for the name if there is no such one. - if (!Res.empty() && Res[0] != '/') - Res.insert(Res.begin(), '/'); - return Res + (char)('0' + Idx); -} - -bool SharedMemoryRegion::Map(int fd) { - Data = - (uint8_t *)mmap(0, kShmemSize, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0); - if (Data == (uint8_t*)-1) - return false; - return true; -} - -bool SharedMemoryRegion::Create(const char *Name) { - int fd = open(Path(Name).c_str(), O_CREAT | O_RDWR, 0777); - if (fd < 0) return false; - if (ftruncate(fd, kShmemSize) < 0) return false; - if (!Map(fd)) - return false; - for (int i = 0; i < 2; i++) { - sem_unlink(SemName(Name, i).c_str()); - Semaphore[i] = sem_open(SemName(Name, i).c_str(), O_CREAT, 0644, 0); - if (Semaphore[i] == SEM_FAILED) - return false; - } - IAmServer = true; - return true; -} - -bool SharedMemoryRegion::Open(const char *Name) { - int fd = open(Path(Name).c_str(), O_RDWR); - if (fd < 0) return false; - struct stat stat_res; - if (0 != fstat(fd, &stat_res)) - return false; - assert(stat_res.st_size == kShmemSize); - if (!Map(fd)) - return false; - for (int i = 0; i < 2; i++) { - Semaphore[i] = sem_open(SemName(Name, i).c_str(), 0); - if (Semaphore[i] == SEM_FAILED) - return false; - } - IAmServer = false; - return true; -} - -bool SharedMemoryRegion::Destroy(const char *Name) { - return 0 == unlink(Path(Name).c_str()); -} - -void SharedMemoryRegion::Post(int Idx) { - assert(Idx == 0 || Idx == 1); - sem_post((sem_t*)Semaphore[Idx]); -} - -void SharedMemoryRegion::Wait(int Idx) { - assert(Idx == 0 || Idx == 1); - for (int i = 0; i < 10 && sem_wait((sem_t*)Semaphore[Idx]); i++) { - // sem_wait may fail if interrupted by a signal. - sleep(i); - if (i) - Printf("%s: sem_wait[%d] failed %s\n", i < 9 ? "WARNING" : "ERROR", i, - strerror(errno)); - if (i == 9) abort(); - } -} - -} // namespace fuzzer - -#endif // LIBFUZZER_POSIX diff --git a/libfuzzer/FuzzerShmemWindows.cpp b/libfuzzer/FuzzerShmemWindows.cpp deleted file mode 100644 index d330ebf..0000000 --- a/libfuzzer/FuzzerShmemWindows.cpp +++ /dev/null @@ -1,64 +0,0 @@ -//===- FuzzerShmemWindows.cpp - Posix shared memory -------------*- C++ -* ===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// SharedMemoryRegion -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" -#if LIBFUZZER_WINDOWS - -#include "FuzzerIO.h" -#include "FuzzerShmem.h" - -#include -#include -#include -#include - -namespace fuzzer { - -std::string SharedMemoryRegion::Path(const char *Name) { - return DirPlusFile(TmpDir(), Name); -} - -std::string SharedMemoryRegion::SemName(const char *Name, int Idx) { - std::string Res(Name); - return Res + (char)('0' + Idx); -} - -bool SharedMemoryRegion::Map(int fd) { - assert(0 && "UNIMPLEMENTED"); - return false; -} - -bool SharedMemoryRegion::Create(const char *Name) { - assert(0 && "UNIMPLEMENTED"); - return false; -} - -bool SharedMemoryRegion::Open(const char *Name) { - assert(0 && "UNIMPLEMENTED"); - return false; -} - -bool SharedMemoryRegion::Destroy(const char *Name) { - assert(0 && "UNIMPLEMENTED"); - return false; -} - -void SharedMemoryRegion::Post(int Idx) { - assert(0 && "UNIMPLEMENTED"); -} - -void SharedMemoryRegion::Wait(int Idx) { - Semaphore[1] = nullptr; - assert(0 && "UNIMPLEMENTED"); -} - -} // namespace fuzzer - -#endif // LIBFUZZER_WINDOWS diff --git a/libfuzzer/FuzzerTracePC.cpp b/libfuzzer/FuzzerTracePC.cpp index 29ffc8e..4a1308d 100644 --- a/libfuzzer/FuzzerTracePC.cpp +++ b/libfuzzer/FuzzerTracePC.cpp @@ -1,9 +1,8 @@ //===- FuzzerTracePC.cpp - PC tracing--------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Trace PCs. @@ -13,6 +12,8 @@ //===----------------------------------------------------------------------===// #include "FuzzerTracePC.h" +#include "FuzzerBuiltins.h" +#include "FuzzerBuiltinsMsvc.h" #include "FuzzerCorpus.h" #include "FuzzerDefs.h" #include "FuzzerDictionary.h" @@ -22,93 +23,87 @@ #include "FuzzerValueBitMap.h" #include -// The coverage counters and PCs. -// These are declared as global variables named "__sancov_*" to simplify -// experiments with inlined instrumentation. -alignas(64) ATTRIBUTE_INTERFACE -uint8_t __sancov_trace_pc_guard_8bit_counters[fuzzer::TracePC::kNumPCs]; - -ATTRIBUTE_INTERFACE -uintptr_t __sancov_trace_pc_pcs[fuzzer::TracePC::kNumPCs]; - // Used by -fsanitize-coverage=stack-depth to track stack depth -ATTRIBUTE_INTERFACE __attribute__((tls_model("initial-exec"))) -thread_local uintptr_t __sancov_lowest_stack; +ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC uintptr_t __sancov_lowest_stack; namespace fuzzer { TracePC TPC; -uint8_t *TracePC::Counters() const { - return __sancov_trace_pc_guard_8bit_counters; -} - -uintptr_t *TracePC::PCs() const { - return __sancov_trace_pc_pcs; -} - size_t TracePC::GetTotalPCCoverage() { - if (ObservedPCs.size()) - return ObservedPCs.size(); - size_t Res = 0; - for (size_t i = 1, N = GetNumPCs(); i < N; i++) - if (PCs()[i]) - Res++; - return Res; + return ObservedPCs.size(); } -template -void TracePC::IterateInline8bitCounters(CallBack CB) const { - if (NumInline8bitCounters && NumInline8bitCounters == NumPCsInPCTables) { - size_t CounterIdx = 0; - for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) { - uint8_t *Beg = ModuleCounters[i].Start; - size_t Size = ModuleCounters[i].Stop - Beg; - assert(Size == (size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start)); - for (size_t j = 0; j < Size; j++, CounterIdx++) - CB(i, j, CounterIdx); - } - } -} -// Initializes unstable counters by copying Inline8bitCounters to unstable -// counters. -void TracePC::InitializeUnstableCounters() { - IterateInline8bitCounters([&](int i, int j, int UnstableIdx) { - UnstableCounters[UnstableIdx].Counter = ModuleCounters[i].Start[j]; - }); -} - -// Compares the current counters with counters from previous runs -// and records differences as unstable edges. -void TracePC::UpdateUnstableCounters(int UnstableMode) { - IterateInline8bitCounters([&](int i, int j, int UnstableIdx) { - if (ModuleCounters[i].Start[j] != UnstableCounters[UnstableIdx].Counter) { - UnstableCounters[UnstableIdx].IsUnstable = true; - if (UnstableMode == ZeroUnstable) - UnstableCounters[UnstableIdx].Counter = 0; - else if (UnstableMode == MinUnstable) - UnstableCounters[UnstableIdx].Counter = std::min( - ModuleCounters[i].Start[j], UnstableCounters[UnstableIdx].Counter); +void TracePC::HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop) { + if (Start == Stop) return; + if (NumModules && + Modules[NumModules - 1].Start() == Start) + return; + assert(NumModules < + sizeof(Modules) / sizeof(Modules[0])); + auto &M = Modules[NumModules++]; + uint8_t *AlignedStart = RoundUpByPage(Start); + uint8_t *AlignedStop = RoundDownByPage(Stop); + size_t NumFullPages = AlignedStop > AlignedStart ? + (AlignedStop - AlignedStart) / PageSize() : 0; + bool NeedFirst = Start < AlignedStart || !NumFullPages; + bool NeedLast = Stop > AlignedStop && AlignedStop >= AlignedStart; + M.NumRegions = NumFullPages + NeedFirst + NeedLast;; + assert(M.NumRegions > 0); + M.Regions = new Module::Region[M.NumRegions]; + assert(M.Regions); + size_t R = 0; + if (NeedFirst) + M.Regions[R++] = {Start, std::min(Stop, AlignedStart), true, false}; + for (uint8_t *P = AlignedStart; P < AlignedStop; P += PageSize()) + M.Regions[R++] = {P, P + PageSize(), true, true}; + if (NeedLast) + M.Regions[R++] = {AlignedStop, Stop, true, false}; + assert(R == M.NumRegions); + assert(M.Size() == (size_t)(Stop - Start)); + assert(M.Stop() == Stop); + assert(M.Start() == Start); + NumInline8bitCounters += M.Size(); +} + +// Mark all full page counter regions as PROT_NONE and set Enabled=false. +// The first time the instrumented code hits such a protected/disabled +// counter region we should catch a SEGV and call UnprotectLazyCounters, +// which will mark the page as PROT_READ|PROT_WRITE and set Enabled=true. +// +// Whenever other functions iterate over the counters they should ignore +// regions with Enabled=false. +void TracePC::ProtectLazyCounters() { + size_t NumPagesProtected = 0; + IterateCounterRegions([&](Module::Region &R) { + if (!R.OneFullPage) return; + if (Mprotect(R.Start, R.Stop - R.Start, false)) { + R.Enabled = false; + NumPagesProtected++; } }); + if (NumPagesProtected) + Printf("INFO: %zd pages of counters where protected;" + " libFuzzer's SEGV handler must be installed\n", + NumPagesProtected); } -// Moves the minimum hit counts to ModuleCounters. -void TracePC::ApplyUnstableCounters() { - IterateInline8bitCounters([&](int i, int j, int UnstableIdx) { - ModuleCounters[i].Start[j] = UnstableCounters[UnstableIdx].Counter; +bool TracePC::UnprotectLazyCounters(void *CounterPtr) { + // Printf("UnprotectLazyCounters: %p\n", CounterPtr); + if (!CounterPtr) + return false; + bool Done = false; + uint8_t *Addr = reinterpret_cast(CounterPtr); + IterateCounterRegions([&](Module::Region &R) { + if (!R.OneFullPage || R.Enabled || Done) return; + if (Addr >= R.Start && Addr < R.Stop) + if (Mprotect(R.Start, R.Stop - R.Start, true)) { + R.Enabled = true; + Done = true; + } }); -} - -void TracePC::HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop) { - if (Start == Stop) return; - if (NumModulesWithInline8bitCounters && - ModuleCounters[NumModulesWithInline8bitCounters-1].Start == Start) return; - assert(NumModulesWithInline8bitCounters < - sizeof(ModuleCounters) / sizeof(ModuleCounters[0])); - ModuleCounters[NumModulesWithInline8bitCounters++] = {Start, Stop}; - NumInline8bitCounters += Stop - Start; + return Done; } void TracePC::HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop) { @@ -120,38 +115,13 @@ void TracePC::HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop) { NumPCsInPCTables += E - B; } -void TracePC::HandleInit(uint32_t *Start, uint32_t *Stop) { - if (Start == Stop || *Start) return; - assert(NumModules < sizeof(Modules) / sizeof(Modules[0])); - for (uint32_t *P = Start; P < Stop; P++) { - NumGuards++; - if (NumGuards == kNumPCs) { - RawPrint( - "WARNING: The binary has too many instrumented PCs.\n" - " You may want to reduce the size of the binary\n" - " for more efficient fuzzing and precise coverage data\n"); - } - *P = NumGuards % kNumPCs; - } - Modules[NumModules].Start = Start; - Modules[NumModules].Stop = Stop; - NumModules++; -} - void TracePC::PrintModuleInfo() { - if (NumGuards) { - Printf("INFO: Loaded %zd modules (%zd guards): ", NumModules, NumGuards); - for (size_t i = 0; i < NumModules; i++) - Printf("%zd [%p, %p), ", Modules[i].Stop - Modules[i].Start, - Modules[i].Start, Modules[i].Stop); - Printf("\n"); - } - if (NumModulesWithInline8bitCounters) { + if (NumModules) { Printf("INFO: Loaded %zd modules (%zd inline 8-bit counters): ", - NumModulesWithInline8bitCounters, NumInline8bitCounters); - for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) - Printf("%zd [%p, %p), ", ModuleCounters[i].Stop - ModuleCounters[i].Start, - ModuleCounters[i].Start, ModuleCounters[i].Stop); + NumModules, NumInline8bitCounters); + for (size_t i = 0; i < NumModules; i++) + Printf("%zd [%p, %p), ", Modules[i].Size(), Modules[i].Start(), + Modules[i].Stop()); Printf("\n"); } if (NumPCTables) { @@ -163,8 +133,7 @@ void TracePC::PrintModuleInfo() { } Printf("\n"); - if ((NumGuards && NumGuards != NumPCsInPCTables) || - (NumInline8bitCounters && NumInline8bitCounters != NumPCsInPCTables)) { + if (NumInline8bitCounters && NumInline8bitCounters != NumPCsInPCTables) { Printf("ERROR: The size of coverage PC tables does not match the\n" "number of instrumented PCs. This might be a compiler bug,\n" "please contact the libFuzzer developers.\n" @@ -185,38 +154,67 @@ void TracePC::HandleCallerCallee(uintptr_t Caller, uintptr_t Callee) { ValueProfileMap.AddValueModPrime(Idx); } +/// \return the address of the previous instruction. +/// Note: the logic is copied from `sanitizer_common/sanitizer_stacktrace.h` +inline ALWAYS_INLINE uintptr_t GetPreviousInstructionPc(uintptr_t PC) { +#if defined(__arm__) + // T32 (Thumb) branch instructions might be 16 or 32 bit long, + // so we return (pc-2) in that case in order to be safe. + // For A32 mode we return (pc-4) because all instructions are 32 bit long. + return (PC - 3) & (~1); +#elif defined(__powerpc__) || defined(__powerpc64__) || defined(__aarch64__) + // PCs are always 4 byte aligned. + return PC - 4; +#elif defined(__sparc__) || defined(__mips__) + return PC - 8; +#else + return PC - 1; +#endif +} + +/// \return the address of the next instruction. +/// Note: the logic is copied from `sanitizer_common/sanitizer_stacktrace.cc` +ALWAYS_INLINE uintptr_t TracePC::GetNextInstructionPc(uintptr_t PC) { +#if defined(__mips__) + return PC + 8; +#elif defined(__powerpc__) || defined(__sparc__) || defined(__arm__) || \ + defined(__aarch64__) + return PC + 4; +#else + return PC + 1; +#endif +} + void TracePC::UpdateObservedPCs() { Vector CoveredFuncs; - auto ObservePC = [&](uintptr_t PC) { - if (ObservedPCs.insert(PC).second && DoPrintNewPCs) { - PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p", PC + 1); + auto ObservePC = [&](const PCTableEntry *TE) { + if (ObservedPCs.insert(TE).second && DoPrintNewPCs) { + PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p", + GetNextInstructionPc(TE->PC)); Printf("\n"); } }; - auto Observe = [&](const PCTableEntry &TE) { - if (TE.PCFlags & 1) - if (++ObservedFuncs[TE.PC] == 1 && NumPrintNewFuncs) - CoveredFuncs.push_back(TE.PC); - ObservePC(TE.PC); + auto Observe = [&](const PCTableEntry *TE) { + if (PcIsFuncEntry(TE)) + if (++ObservedFuncs[TE->PC] == 1 && NumPrintNewFuncs) + CoveredFuncs.push_back(TE->PC); + ObservePC(TE); }; if (NumPCsInPCTables) { if (NumInline8bitCounters == NumPCsInPCTables) { - IterateInline8bitCounters([&](int i, int j, int CounterIdx) { - if (ModuleCounters[i].Start[j]) - Observe(ModulePCTable[i].Start[j]); - }); - } else if (NumGuards == NumPCsInPCTables) { - size_t GuardIdx = 1; for (size_t i = 0; i < NumModules; i++) { - uint32_t *Beg = Modules[i].Start; - size_t Size = Modules[i].Stop - Beg; - assert(Size == + auto &M = Modules[i]; + assert(M.Size() == (size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start)); - for (size_t j = 0; j < Size; j++, GuardIdx++) - if (Counters()[GuardIdx]) - Observe(ModulePCTable[i].Start[j]); + for (size_t r = 0; r < M.NumRegions; r++) { + auto &R = M.Regions[r]; + if (!R.Enabled) continue; + for (uint8_t *P = R.Start; P < R.Stop; P++) + if (*P) + Observe(&ModulePCTable[i].Start[M.Idx(P)]); + } } } } @@ -224,21 +222,31 @@ void TracePC::UpdateObservedPCs() { for (size_t i = 0, N = Min(CoveredFuncs.size(), NumPrintNewFuncs); i < N; i++) { Printf("\tNEW_FUNC[%zd/%zd]: ", i + 1, CoveredFuncs.size()); - PrintPC("%p %F %L", "%p", CoveredFuncs[i] + 1); + PrintPC("%p %F %L", "%p", GetNextInstructionPc(CoveredFuncs[i])); Printf("\n"); } } -inline ALWAYS_INLINE uintptr_t GetPreviousInstructionPc(uintptr_t PC) { - // TODO: this implementation is x86 only. - // see sanitizer_common GetPreviousInstructionPc for full implementation. - return PC - 1; +uintptr_t TracePC::PCTableEntryIdx(const PCTableEntry *TE) { + size_t TotalTEs = 0; + for (size_t i = 0; i < NumPCTables; i++) { + auto &M = ModulePCTable[i]; + if (TE >= M.Start && TE < M.Stop) + return TotalTEs + TE - M.Start; + TotalTEs += M.Stop - M.Start; + } + assert(0); + return 0; } -inline ALWAYS_INLINE uintptr_t GetNextInstructionPc(uintptr_t PC) { - // TODO: this implementation is x86 only. - // see sanitizer_common GetPreviousInstructionPc for full implementation. - return PC + 1; +const TracePC::PCTableEntry *TracePC::PCTableEntryByIdx(uintptr_t Idx) { + for (size_t i = 0; i < NumPCTables; i++) { + auto &M = ModulePCTable[i]; + size_t Size = M.Stop - M.Start; + if (Idx < Size) return &M.Start[Idx]; + Idx -= Size; + } + return nullptr; } static std::string GetModuleName(uintptr_t PC) { @@ -259,47 +267,38 @@ void TracePC::IterateCoveredFunctions(CallBack CB) { auto ModuleName = GetModuleName(M.Start->PC); for (auto NextFE = M.Start; NextFE < M.Stop; ) { auto FE = NextFE; - assert((FE->PCFlags & 1) && "Not a function entry point"); + assert(PcIsFuncEntry(FE) && "Not a function entry point"); do { NextFE++; - } while (NextFE < M.Stop && !(NextFE->PCFlags & 1)); - if (ObservedFuncs.count(FE->PC)) - CB(FE, NextFE, ObservedFuncs[FE->PC]); + } while (NextFE < M.Stop && !(PcIsFuncEntry(NextFE))); + CB(FE, NextFE, ObservedFuncs[FE->PC]); } } } void TracePC::SetFocusFunction(const std::string &FuncName) { // This function should be called once. - assert(FocusFunction.first > NumModulesWithInline8bitCounters); + assert(!FocusFunctionCounterPtr); if (FuncName.empty()) return; - for (size_t M = 0; M < NumModulesWithInline8bitCounters; M++) { + for (size_t M = 0; M < NumModules; M++) { auto &PCTE = ModulePCTable[M]; size_t N = PCTE.Stop - PCTE.Start; for (size_t I = 0; I < N; I++) { - if (!(PCTE.Start[I].PCFlags & 1)) continue; // not a function entry. + if (!(PcIsFuncEntry(&PCTE.Start[I]))) continue; // not a function entry. auto Name = DescribePC("%F", GetNextInstructionPc(PCTE.Start[I].PC)); if (Name[0] == 'i' && Name[1] == 'n' && Name[2] == ' ') Name = Name.substr(3, std::string::npos); if (FuncName != Name) continue; Printf("INFO: Focus function is set to '%s'\n", Name.c_str()); - FocusFunction = {M, I}; + FocusFunctionCounterPtr = Modules[M].Start() + I; return; } } } bool TracePC::ObservedFocusFunction() { - size_t I = FocusFunction.first; - size_t J = FocusFunction.second; - if (I >= NumModulesWithInline8bitCounters) - return false; - auto &MC = ModuleCounters[I]; - size_t Size = MC.Stop - MC.Start; - if (J >= Size) - return false; - return MC.Start[J] != 0; + return FocusFunctionCounterPtr && *FocusFunctionCounterPtr; } void TracePC::PrintCoverage() { @@ -323,41 +322,24 @@ void TracePC::PrintCoverage() { if (FunctionStr.find("in ") == 0) FunctionStr = FunctionStr.substr(3); std::string LineStr = DescribePC("%l", VisualizePC); - size_t Line = std::stoul(LineStr); size_t NumEdges = Last - First; Vector UncoveredPCs; for (auto TE = First; TE < Last; TE++) - if (!ObservedPCs.count(TE->PC)) + if (!ObservedPCs.count(TE)) UncoveredPCs.push_back(TE->PC); - Printf("COVERED_FUNC: hits: %zd", Counter); + Printf("%sCOVERED_FUNC: hits: %zd", Counter ? "" : "UN", Counter); Printf(" edges: %zd/%zd", NumEdges - UncoveredPCs.size(), NumEdges); - Printf(" %s %s:%zd\n", FunctionStr.c_str(), FileStr.c_str(), Line); - for (auto PC: UncoveredPCs) - Printf(" UNCOVERED_PC: %s\n", - DescribePC("%s:%l", GetNextInstructionPc(PC)).c_str()); + Printf(" %s %s:%s\n", FunctionStr.c_str(), FileStr.c_str(), + LineStr.c_str()); + if (Counter) + for (auto PC : UncoveredPCs) + Printf(" UNCOVERED_PC: %s\n", + DescribePC("%s:%l", GetNextInstructionPc(PC)).c_str()); }; IterateCoveredFunctions(CoveredFunctionCallback); } -void TracePC::DumpCoverage() { - if (EF->__sanitizer_dump_coverage) { - Vector PCsCopy(GetNumPCs()); - for (size_t i = 0; i < GetNumPCs(); i++) - PCsCopy[i] = PCs()[i] ? GetPreviousInstructionPc(PCs()[i]) : 0; - EF->__sanitizer_dump_coverage(PCsCopy.data(), PCsCopy.size()); - } -} - -void TracePC::PrintUnstableStats() { - size_t count = 0; - for (size_t i = 0; i < NumInline8bitCounters; i++) - if (UnstableCounters[i].IsUnstable) - count++; - Printf("stat::stability_rate: %.2f\n", - 100 - static_cast(count * 100) / NumInline8bitCounters); -} - // Value profile. // We keep track of various values that affect control flow. // These values are inserted into a bit-set-based hash map. @@ -387,11 +369,16 @@ void TracePC::AddValueForMemcmp(void *caller_pc, const void *s1, const void *s2, Hash ^= (T << 8) | B2[i]; } size_t I = 0; - for (; I < Len; I++) - if (B1[I] != B2[I] || (StopAtZero && B1[I] == 0)) + uint8_t HammingDistance = 0; + for (; I < Len; I++) { + if (B1[I] != B2[I] || (StopAtZero && B1[I] == 0)) { + HammingDistance = Popcountll(B1[I] ^ B2[I]); break; + } + } size_t PC = reinterpret_cast(caller_pc); size_t Idx = (PC & 4095) | (I << 12); + Idx += HammingDistance; ValueProfileMap.AddValue(Idx); TORCW.Insert(Idx ^ Hash, Word(B1, Len), Word(B2, Len)); } @@ -401,20 +388,14 @@ ATTRIBUTE_TARGET_POPCNT ALWAYS_INLINE ATTRIBUTE_NO_SANITIZE_ALL void TracePC::HandleCmp(uintptr_t PC, T Arg1, T Arg2) { uint64_t ArgXor = Arg1 ^ Arg2; - uint64_t ArgDistance = __builtin_popcountll(ArgXor) + 1; // [1,65] - uintptr_t Idx = ((PC & 4095) + 1) * ArgDistance; if (sizeof(T) == 4) TORC4.Insert(ArgXor, Arg1, Arg2); else if (sizeof(T) == 8) TORC8.Insert(ArgXor, Arg1, Arg2); - // TODO: remove these flags and instead use all metrics at once. - if (UseValueProfileMask & 1) - ValueProfileMap.AddValue(Idx); - if (UseValueProfileMask & 2) - ValueProfileMap.AddValue( - PC * 64 + (Arg1 == Arg2 ? 0 : __builtin_clzll(Arg1 - Arg2) + 1)); - if (UseValueProfileMask & 4) // alternative way to use the hamming distance - ValueProfileMap.AddValue(PC * 64 + ArgDistance); + uint64_t HammingDistance = Popcountll(ArgXor); // [0,64] + uint64_t AbsoluteDistance = (Arg1 == Arg2 ? 0 : Clzll(Arg1 - Arg2) + 1); + ValueProfileMap.AddValue(PC * 128 + HammingDistance); + ValueProfileMap.AddValue(PC * 128 + 64 + AbsoluteDistance); } static size_t InternalStrnlen(const char *S, size_t MaxLen) { @@ -432,11 +413,10 @@ static size_t InternalStrnlen2(const char *S1, const char *S2) { } void TracePC::ClearInlineCounters() { - for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) { - uint8_t *Beg = ModuleCounters[i].Start; - size_t Size = ModuleCounters[i].Stop - Beg; - memset(Beg, 0, Size); - } + IterateCounterRegions([](const Module::Region &R){ + if (R.Enabled) + memset(R.Start, 0, R.Stop - R.Start); + }); } ATTRIBUTE_NO_SANITIZE_ALL @@ -449,16 +429,25 @@ uintptr_t TracePC::GetMaxStackOffset() const { return InitialStack - __sancov_lowest_stack; // Stack grows down } +void WarnAboutDeprecatedInstrumentation(const char *flag) { + // Use RawPrint because Printf cannot be used on Windows before OutputFile is + // initialized. + RawPrint(flag); + RawPrint( + " is no longer supported by libFuzzer.\n" + "Please either migrate to a compiler that supports -fsanitize=fuzzer\n" + "or use an older version of libFuzzer\n"); + exit(1); +} + } // namespace fuzzer extern "C" { ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL void __sanitizer_cov_trace_pc_guard(uint32_t *Guard) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); - uint32_t Idx = *Guard; - __sancov_trace_pc_pcs[Idx] = PC; - __sancov_trace_pc_guard_8bit_counters[Idx]++; + fuzzer::WarnAboutDeprecatedInstrumentation( + "-fsanitize-coverage=trace-pc-guard"); } // Best-effort support for -fsanitize-coverage=trace-pc, which is available @@ -466,15 +455,13 @@ void __sanitizer_cov_trace_pc_guard(uint32_t *Guard) { ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL void __sanitizer_cov_trace_pc() { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); - uintptr_t Idx = PC & (((uintptr_t)1 << fuzzer::TracePC::kTracePcBits) - 1); - __sancov_trace_pc_pcs[Idx] = PC; - __sancov_trace_pc_guard_8bit_counters[Idx]++; + fuzzer::WarnAboutDeprecatedInstrumentation("-fsanitize-coverage=trace-pc"); } ATTRIBUTE_INTERFACE void __sanitizer_cov_trace_pc_guard_init(uint32_t *Start, uint32_t *Stop) { - fuzzer::TPC.HandleInit(Start, Stop); + fuzzer::WarnAboutDeprecatedInstrumentation( + "-fsanitize-coverage=trace-pc-guard"); } ATTRIBUTE_INTERFACE @@ -491,7 +478,7 @@ void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL void __sanitizer_cov_trace_pc_indir(uintptr_t Callee) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCallerCallee(PC, Callee); } @@ -499,7 +486,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_cmp8(uint64_t Arg1, uint64_t Arg2) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -510,7 +497,7 @@ ATTRIBUTE_TARGET_POPCNT // the behaviour of __sanitizer_cov_trace_cmp[1248] ones. This, however, // should be changed later to make full use of instrumentation. void __sanitizer_cov_trace_const_cmp8(uint64_t Arg1, uint64_t Arg2) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -518,7 +505,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_cmp4(uint32_t Arg1, uint32_t Arg2) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -526,7 +513,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_const_cmp4(uint32_t Arg1, uint32_t Arg2) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -534,7 +521,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_cmp2(uint16_t Arg1, uint16_t Arg2) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -542,7 +529,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_const_cmp2(uint16_t Arg1, uint16_t Arg2) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -550,7 +537,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_cmp1(uint8_t Arg1, uint8_t Arg2) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -558,7 +545,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_const_cmp1(uint8_t Arg1, uint8_t Arg2) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -569,31 +556,51 @@ void __sanitizer_cov_trace_switch(uint64_t Val, uint64_t *Cases) { uint64_t N = Cases[0]; uint64_t ValSizeInBits = Cases[1]; uint64_t *Vals = Cases + 2; - // Skip the most common and the most boring case. - if (Vals[N - 1] < 256 && Val < 256) + // Skip the most common and the most boring case: all switch values are small. + // We may want to skip this at compile-time, but it will make the + // instrumentation less general. + if (Vals[N - 1] < 256) return; - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + // Also skip small inputs values, they won't give good signal. + if (Val < 256) + return; + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); size_t i; - uint64_t Token = 0; + uint64_t Smaller = 0; + uint64_t Larger = ~(uint64_t)0; + // Find two switch values such that Smaller < Val < Larger. + // Use 0 and 0xfff..f as the defaults. for (i = 0; i < N; i++) { - Token = Val ^ Vals[i]; - if (Val < Vals[i]) + if (Val < Vals[i]) { + Larger = Vals[i]; break; + } + if (Val > Vals[i]) Smaller = Vals[i]; } - if (ValSizeInBits == 16) - fuzzer::TPC.HandleCmp(PC + i, static_cast(Token), (uint16_t)(0)); - else if (ValSizeInBits == 32) - fuzzer::TPC.HandleCmp(PC + i, static_cast(Token), (uint32_t)(0)); - else - fuzzer::TPC.HandleCmp(PC + i, Token, (uint64_t)(0)); + // Apply HandleCmp to {Val,Smaller} and {Val, Larger}, + // use i as the PC modifier for HandleCmp. + if (ValSizeInBits == 16) { + fuzzer::TPC.HandleCmp(PC + 2 * i, static_cast(Val), + (uint16_t)(Smaller)); + fuzzer::TPC.HandleCmp(PC + 2 * i + 1, static_cast(Val), + (uint16_t)(Larger)); + } else if (ValSizeInBits == 32) { + fuzzer::TPC.HandleCmp(PC + 2 * i, static_cast(Val), + (uint32_t)(Smaller)); + fuzzer::TPC.HandleCmp(PC + 2 * i + 1, static_cast(Val), + (uint32_t)(Larger)); + } else { + fuzzer::TPC.HandleCmp(PC + 2*i, Val, Smaller); + fuzzer::TPC.HandleCmp(PC + 2*i + 1, Val, Larger); + } } ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_div4(uint32_t Val) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Val, (uint32_t)0); } @@ -601,7 +608,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_div8(uint64_t Val) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Val, (uint64_t)0); } @@ -609,7 +616,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_gep(uintptr_t Idx) { - uintptr_t PC = reinterpret_cast(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Idx, (uintptr_t)0); } diff --git a/libfuzzer/FuzzerTracePC.h b/libfuzzer/FuzzerTracePC.h index 097ba69..4f5ebeb 100644 --- a/libfuzzer/FuzzerTracePC.h +++ b/libfuzzer/FuzzerTracePC.h @@ -1,9 +1,8 @@ //===- FuzzerTracePC.h - Internal header for the Fuzzer ---------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // fuzzer::TracePC @@ -70,16 +69,6 @@ struct MemMemTable { class TracePC { public: - static const size_t kNumPCs = 1 << 21; - // How many bits of PC are used from __sanitizer_cov_trace_pc. - static const size_t kTracePcBits = 18; - - enum HandleUnstableOptions { - MinUnstable = 1, - ZeroUnstable = 2, - }; - - void HandleInit(uint32_t *Start, uint32_t *Stop); void HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop); void HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop); void HandleCallerCallee(uintptr_t Caller, uintptr_t Callee); @@ -94,8 +83,6 @@ class TracePC { void ResetMaps() { ValueProfileMap.Reset(); - if (NumModules) - memset(Counters(), 0, GetNumPCs()); ClearExtraCounters(); ClearInlineCounters(); } @@ -108,8 +95,6 @@ class TracePC { void PrintModuleInfo(); void PrintCoverage(); - void DumpCoverage(); - void PrintUnstableStats(); template void IterateCoveredFunctions(CallBack CB); @@ -122,14 +107,6 @@ class TracePC { TableOfRecentCompares TORCW; MemMemTable<1024> MMT; - size_t GetNumPCs() const { - return NumGuards == 0 ? (1 << kTracePcBits) : Min(kNumPCs, NumGuards + 1); - } - uintptr_t GetPC(size_t Idx) { - assert(Idx < GetNumPCs()); - return PCs()[Idx]; - } - void RecordInitialStack(); uintptr_t GetMaxStackOffset() const; @@ -142,53 +119,63 @@ class TracePC { void SetFocusFunction(const std::string &FuncName); bool ObservedFocusFunction(); - void InitializeUnstableCounters(); - void UpdateUnstableCounters(int UnstableMode); - void ApplyUnstableCounters(); + void ProtectLazyCounters(); + bool UnprotectLazyCounters(void *CounterPtr); -private: - struct UnstableEdge { - uint8_t Counter; - bool IsUnstable; + struct PCTableEntry { + uintptr_t PC, PCFlags; }; - UnstableEdge UnstableCounters[kNumPCs]; + uintptr_t PCTableEntryIdx(const PCTableEntry *TE); + const PCTableEntry *PCTableEntryByIdx(uintptr_t Idx); + static uintptr_t GetNextInstructionPc(uintptr_t PC); + bool PcIsFuncEntry(const PCTableEntry *TE) { return TE->PCFlags & 1; } +private: bool UseCounters = false; uint32_t UseValueProfileMask = false; bool DoPrintNewPCs = false; size_t NumPrintNewFuncs = 0; + // Module represents the array of 8-bit counters split into regions + // such that every region, except maybe the first and the last one, is one + // full page. struct Module { - uint32_t *Start, *Stop; + struct Region { + uint8_t *Start, *Stop; + bool Enabled; + bool OneFullPage; + }; + Region *Regions; + size_t NumRegions; + uint8_t *Start() { return Regions[0].Start; } + uint8_t *Stop() { return Regions[NumRegions - 1].Stop; } + size_t Size() { return Stop() - Start(); } + size_t Idx(uint8_t *P) { + assert(P >= Start() && P < Stop()); + return P - Start(); + } }; Module Modules[4096]; size_t NumModules; // linker-initialized. - size_t NumGuards; // linker-initialized. - - struct { uint8_t *Start, *Stop; } ModuleCounters[4096]; - size_t NumModulesWithInline8bitCounters; // linker-initialized. size_t NumInline8bitCounters; - struct PCTableEntry { - uintptr_t PC, PCFlags; - }; + template + void IterateCounterRegions(Callback CB) { + for (size_t m = 0; m < NumModules; m++) + for (size_t r = 0; r < Modules[m].NumRegions; r++) + CB(Modules[m].Regions[r]); + } struct { const PCTableEntry *Start, *Stop; } ModulePCTable[4096]; size_t NumPCTables; size_t NumPCsInPCTables; - uint8_t *Counters() const; - uintptr_t *PCs() const; - - Set ObservedPCs; + Set ObservedPCs; std::unordered_map ObservedFuncs; // PC => Counter. - template - void IterateInline8bitCounters(Callback CB) const; - - std::pair FocusFunction = {-1, -1}; // Module and PC IDs. + uint8_t *FocusFunctionCounterPtr = nullptr; ValueBitMap ValueProfileMap; uintptr_t InitialStack; @@ -197,7 +184,7 @@ class TracePC { template // void Callback(size_t FirstFeature, size_t Idx, uint8_t Value); ATTRIBUTE_NO_SANITIZE_ALL -void ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End, +size_t ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End, size_t FirstFeature, Callback Handle8bitCounter) { typedef uintptr_t LargeType; const size_t Step = sizeof(LargeType) / sizeof(uint8_t); @@ -219,6 +206,7 @@ void ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End, for (; P < End; P++) if (uint8_t V = *P) Handle8bitCounter(FirstFeature, P - Begin, V); + return End - Begin; } // Given a non-zero Counter returns a number in the range [0,7]. @@ -249,10 +237,8 @@ unsigned CounterToFeature(T Counter) { template // void Callback(size_t Feature) ATTRIBUTE_NO_SANITIZE_ADDRESS -__attribute__((noinline)) +ATTRIBUTE_NOINLINE void TracePC::CollectFeatures(Callback HandleFeature) const { - uint8_t *Counters = this->Counters(); - size_t N = GetNumPCs(); auto Handle8bitCounter = [&](size_t FirstFeature, size_t Idx, uint8_t Counter) { if (UseCounters) @@ -263,22 +249,18 @@ void TracePC::CollectFeatures(Callback HandleFeature) const { size_t FirstFeature = 0; - if (!NumInline8bitCounters) { - ForEachNonZeroByte(Counters, Counters + N, FirstFeature, Handle8bitCounter); - FirstFeature += N * 8; - } - - if (NumInline8bitCounters) { - for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) { - ForEachNonZeroByte(ModuleCounters[i].Start, ModuleCounters[i].Stop, - FirstFeature, Handle8bitCounter); - FirstFeature += 8 * (ModuleCounters[i].Stop - ModuleCounters[i].Start); + for (size_t i = 0; i < NumModules; i++) { + for (size_t r = 0; r < Modules[i].NumRegions; r++) { + if (!Modules[i].Regions[r].Enabled) continue; + FirstFeature += 8 * ForEachNonZeroByte(Modules[i].Regions[r].Start, + Modules[i].Regions[r].Stop, + FirstFeature, Handle8bitCounter); } } - ForEachNonZeroByte(ExtraCountersBegin(), ExtraCountersEnd(), FirstFeature, - Handle8bitCounter); - FirstFeature += (ExtraCountersEnd() - ExtraCountersBegin()) * 8; + FirstFeature += + 8 * ForEachNonZeroByte(ExtraCountersBegin(), ExtraCountersEnd(), + FirstFeature, Handle8bitCounter); if (UseValueProfileMask) { ValueProfileMap.ForEach([&](size_t Idx) { diff --git a/libfuzzer/FuzzerUtil.cpp b/libfuzzer/FuzzerUtil.cpp index 6286f9a..7aa84a1 100644 --- a/libfuzzer/FuzzerUtil.cpp +++ b/libfuzzer/FuzzerUtil.cpp @@ -1,9 +1,8 @@ //===- FuzzerUtil.cpp - Misc utils ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Misc utils. diff --git a/libfuzzer/FuzzerUtil.h b/libfuzzer/FuzzerUtil.h index 8c5c57c..0a12791 100644 --- a/libfuzzer/FuzzerUtil.h +++ b/libfuzzer/FuzzerUtil.h @@ -1,9 +1,8 @@ //===- FuzzerUtil.h - Internal header for the Fuzzer Utils ------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Util functions. @@ -12,8 +11,10 @@ #ifndef LLVM_FUZZER_UTIL_H #define LLVM_FUZZER_UTIL_H -#include "FuzzerDefs.h" +#include "FuzzerBuiltins.h" +#include "FuzzerBuiltinsMsvc.h" #include "FuzzerCommand.h" +#include "FuzzerDefs.h" namespace fuzzer { @@ -51,6 +52,8 @@ void SetSignalHandler(const FuzzingOptions& Options); void SleepSeconds(int Seconds); +bool Mprotect(void *Ptr, size_t Size, bool AllowReadWrite); + unsigned long GetPid(); size_t GetPeakRSSMb(); @@ -84,7 +87,21 @@ std::string SearchRegexCmd(const std::string &Regex); size_t SimpleFastHash(const uint8_t *Data, size_t Size); -inline uint32_t Log(uint32_t X) { return 32 - __builtin_clz(X) - 1; } +inline uint32_t Log(uint32_t X) { return 32 - Clz(X) - 1; } + +inline size_t PageSize() { return 4096; } +inline uint8_t *RoundUpByPage(uint8_t *P) { + uintptr_t X = reinterpret_cast(P); + size_t Mask = PageSize() - 1; + X = (X + Mask) & ~Mask; + return reinterpret_cast(X); +} +inline uint8_t *RoundDownByPage(uint8_t *P) { + uintptr_t X = reinterpret_cast(P); + size_t Mask = PageSize() - 1; + X = X & ~Mask; + return reinterpret_cast(X); +} } // namespace fuzzer diff --git a/libfuzzer/FuzzerUtilDarwin.cpp b/libfuzzer/FuzzerUtilDarwin.cpp index 4bfbc11..171db23 100644 --- a/libfuzzer/FuzzerUtilDarwin.cpp +++ b/libfuzzer/FuzzerUtilDarwin.cpp @@ -1,9 +1,8 @@ //===- FuzzerUtilDarwin.cpp - Misc utils ----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Misc utils for Darwin. diff --git a/libfuzzer/FuzzerUtilFuchsia.cpp b/libfuzzer/FuzzerUtilFuchsia.cpp index cd2bb74..1f04b33 100644 --- a/libfuzzer/FuzzerUtilFuchsia.cpp +++ b/libfuzzer/FuzzerUtilFuchsia.cpp @@ -1,9 +1,8 @@ //===- FuzzerUtilFuchsia.cpp - Misc utils for Fuchsia. --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Misc utils implementation using Fuchsia/Zircon APIs. @@ -14,6 +13,7 @@ #include "FuzzerInternal.h" #include "FuzzerUtil.h" +#include #include #include #include @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include namespace fuzzer { @@ -49,13 +49,6 @@ void CrashTrampolineAsm() __asm__("CrashTrampolineAsm"); namespace { -// TODO(phosek): remove this and replace it with ZX_TIME_INFINITE -#define ZX_TIME_INFINITE_OLD INT64_MAX - -// A magic value for the Zircon exception port, chosen to spell 'FUZZING' -// when interpreted as a byte sequence on little-endian platforms. -const uint64_t kFuzzingCrash = 0x474e495a5a5546; - // Helper function to handle Zircon syscall failures. void ExitOnErr(zx_status_t Status, const char *Syscall) { if (Status != ZX_OK) { @@ -221,76 +214,101 @@ void CrashHandler(zx_handle_t *Event) { zx_handle_t Handle = ZX_HANDLE_INVALID; }; - // Create and bind the exception port. We need to claim to be a "debugger" so - // the kernel will allow us to modify and resume dying threads (see below). - // Once the port is set, we can signal the main thread to continue and wait + // Create the exception channel. We need to claim to be a "debugger" so the + // kernel will allow us to modify and resume dying threads (see below). Once + // the channel is set, we can signal the main thread to continue and wait // for the exception to arrive. - ScopedHandle Port; - ExitOnErr(_zx_port_create(0, &Port.Handle), "_zx_port_create"); + ScopedHandle Channel; zx_handle_t Self = _zx_process_self(); - - ExitOnErr(_zx_task_bind_exception_port(Self, Port.Handle, kFuzzingCrash, - ZX_EXCEPTION_PORT_DEBUGGER), - "_zx_task_bind_exception_port"); + ExitOnErr(_zx_task_create_exception_channel( + Self, ZX_EXCEPTION_CHANNEL_DEBUGGER, &Channel.Handle), + "_zx_task_create_exception_channel"); ExitOnErr(_zx_object_signal(*Event, 0, ZX_USER_SIGNAL_0), "_zx_object_signal"); - zx_port_packet_t Packet; - ExitOnErr(_zx_port_wait(Port.Handle, ZX_TIME_INFINITE_OLD, &Packet), - "_zx_port_wait"); - - // At this point, we want to get the state of the crashing thread, but - // libFuzzer and the sanitizers assume this will happen from that same thread - // via a POSIX signal handler. "Resurrecting" the thread in the middle of the - // appropriate callback is as simple as forcibly setting the instruction - // pointer/program counter, provided we NEVER EVER return from that function - // (since otherwise our stack will not be valid). - ScopedHandle Thread; - ExitOnErr(_zx_object_get_child(Self, Packet.exception.tid, - ZX_RIGHT_SAME_RIGHTS, &Thread.Handle), - "_zx_object_get_child"); - - zx_thread_state_general_regs_t GeneralRegisters; - ExitOnErr(_zx_thread_read_state(Thread.Handle, ZX_THREAD_STATE_GENERAL_REGS, - &GeneralRegisters, sizeof(GeneralRegisters)), - "_zx_thread_read_state"); - - // To unwind properly, we need to push the crashing thread's register state - // onto the stack and jump into a trampoline with CFI instructions on how - // to restore it. + // This thread lives as long as the process in order to keep handling + // crashes. In practice, the first crashed thread to reach the end of the + // StaticCrashHandler will end the process. + while (true) { + ExitOnErr(_zx_object_wait_one(Channel.Handle, ZX_CHANNEL_READABLE, + ZX_TIME_INFINITE, nullptr), + "_zx_object_wait_one"); + + zx_exception_info_t ExceptionInfo; + ScopedHandle Exception; + ExitOnErr(_zx_channel_read(Channel.Handle, 0, &ExceptionInfo, + &Exception.Handle, sizeof(ExceptionInfo), 1, + nullptr, nullptr), + "_zx_channel_read"); + + // Ignore informational synthetic exceptions. + if (ZX_EXCP_THREAD_STARTING == ExceptionInfo.type || + ZX_EXCP_THREAD_EXITING == ExceptionInfo.type || + ZX_EXCP_PROCESS_STARTING == ExceptionInfo.type) { + continue; + } + + // At this point, we want to get the state of the crashing thread, but + // libFuzzer and the sanitizers assume this will happen from that same + // thread via a POSIX signal handler. "Resurrecting" the thread in the + // middle of the appropriate callback is as simple as forcibly setting the + // instruction pointer/program counter, provided we NEVER EVER return from + // that function (since otherwise our stack will not be valid). + ScopedHandle Thread; + ExitOnErr(_zx_exception_get_thread(Exception.Handle, &Thread.Handle), + "_zx_exception_get_thread"); + + zx_thread_state_general_regs_t GeneralRegisters; + ExitOnErr(_zx_thread_read_state(Thread.Handle, ZX_THREAD_STATE_GENERAL_REGS, + &GeneralRegisters, + sizeof(GeneralRegisters)), + "_zx_thread_read_state"); + + // To unwind properly, we need to push the crashing thread's register state + // onto the stack and jump into a trampoline with CFI instructions on how + // to restore it. #if defined(__x86_64__) - uintptr_t StackPtr = - (GeneralRegisters.rsp - (128 + sizeof(GeneralRegisters))) & - -(uintptr_t)16; - __unsanitized_memcpy(reinterpret_cast(StackPtr), &GeneralRegisters, - sizeof(GeneralRegisters)); - GeneralRegisters.rsp = StackPtr; - GeneralRegisters.rip = reinterpret_cast(CrashTrampolineAsm); + uintptr_t StackPtr = + (GeneralRegisters.rsp - (128 + sizeof(GeneralRegisters))) & + -(uintptr_t)16; + __unsanitized_memcpy(reinterpret_cast(StackPtr), &GeneralRegisters, + sizeof(GeneralRegisters)); + GeneralRegisters.rsp = StackPtr; + GeneralRegisters.rip = reinterpret_cast(CrashTrampolineAsm); #elif defined(__aarch64__) - uintptr_t StackPtr = - (GeneralRegisters.sp - sizeof(GeneralRegisters)) & -(uintptr_t)16; - __unsanitized_memcpy(reinterpret_cast(StackPtr), &GeneralRegisters, - sizeof(GeneralRegisters)); - GeneralRegisters.sp = StackPtr; - GeneralRegisters.pc = reinterpret_cast(CrashTrampolineAsm); + uintptr_t StackPtr = + (GeneralRegisters.sp - sizeof(GeneralRegisters)) & -(uintptr_t)16; + __unsanitized_memcpy(reinterpret_cast(StackPtr), &GeneralRegisters, + sizeof(GeneralRegisters)); + GeneralRegisters.sp = StackPtr; + GeneralRegisters.pc = reinterpret_cast(CrashTrampolineAsm); #else #error "Unsupported architecture for fuzzing on Fuchsia" #endif - // Now force the crashing thread's state. - ExitOnErr(_zx_thread_write_state(Thread.Handle, ZX_THREAD_STATE_GENERAL_REGS, - &GeneralRegisters, sizeof(GeneralRegisters)), - "_zx_thread_write_state"); - - ExitOnErr(_zx_task_resume_from_exception(Thread.Handle, Port.Handle, 0), - "_zx_task_resume_from_exception"); + // Now force the crashing thread's state. + ExitOnErr( + _zx_thread_write_state(Thread.Handle, ZX_THREAD_STATE_GENERAL_REGS, + &GeneralRegisters, sizeof(GeneralRegisters)), + "_zx_thread_write_state"); + + // Set the exception to HANDLED so it resumes the thread on close. + uint32_t ExceptionState = ZX_EXCEPTION_STATE_HANDLED; + ExitOnErr(_zx_object_set_property(Exception.Handle, ZX_PROP_EXCEPTION_STATE, + &ExceptionState, sizeof(ExceptionState)), + "zx_object_set_property"); + } } } // namespace +bool Mprotect(void *Ptr, size_t Size, bool AllowReadWrite) { + return false; // UNIMPLEMENTED +} + // Platform specific functions. void SetSignalHandler(const FuzzingOptions &Options) { // Set up alarm handler if needed. @@ -315,8 +333,8 @@ void SetSignalHandler(const FuzzingOptions &Options) { ExitOnErr(_zx_event_create(0, &Event), "_zx_event_create"); std::thread T(CrashHandler, &Event); - zx_status_t Status = _zx_object_wait_one(Event, ZX_USER_SIGNAL_0, - ZX_TIME_INFINITE_OLD, nullptr); + zx_status_t Status = + _zx_object_wait_one(Event, ZX_USER_SIGNAL_0, ZX_TIME_INFINITE, nullptr); _zx_handle_close(Event); ExitOnErr(Status, "_zx_object_wait_one"); @@ -378,19 +396,28 @@ int ExecuteCommand(const Command &Cmd) { Argv[i] = Args[i].c_str(); Argv[Argc] = nullptr; - // Determine stdout + // Determine output. On Fuchsia, the fuzzer is typically run as a component + // that lacks a mutable working directory. Fortunately, when this is the case + // a mutable output directory must be specified using "-artifact_prefix=...", + // so write the log file(s) there. int FdOut = STDOUT_FILENO; - if (Cmd.hasOutputFile()) { - auto Filename = Cmd.getOutputFile(); - FdOut = open(Filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0); + std::string Path; + if (Cmd.hasFlag("artifact_prefix")) + Path = Cmd.getFlagValue("artifact_prefix") + "/" + Cmd.getOutputFile(); + else + Path = Cmd.getOutputFile(); + FdOut = open(Path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0); if (FdOut == -1) { - Printf("libFuzzer: failed to open %s: %s\n", Filename.c_str(), + Printf("libFuzzer: failed to open %s: %s\n", Path.c_str(), strerror(errno)); return ZX_ERR_IO; } } - auto CloseFdOut = at_scope_exit([&]() { close(FdOut); } ); + auto CloseFdOut = at_scope_exit([FdOut]() { + if (FdOut != STDOUT_FILENO) + close(FdOut); + }); // Determine stderr int FdErr = STDERR_FILENO; @@ -440,7 +467,7 @@ int ExecuteCommand(const Command &Cmd) { // Now join the process and return the exit status. if ((rc = _zx_object_wait_one(ProcessHandle, ZX_PROCESS_TERMINATED, - ZX_TIME_INFINITE_OLD, nullptr)) != ZX_OK) { + ZX_TIME_INFINITE, nullptr)) != ZX_OK) { Printf("libFuzzer: failed to join '%s': %s\n", Argv[0], _zx_status_get_string(rc)); return rc; diff --git a/libfuzzer/FuzzerUtilLinux.cpp b/libfuzzer/FuzzerUtilLinux.cpp index c103fd2..d5a15d1 100644 --- a/libfuzzer/FuzzerUtilLinux.cpp +++ b/libfuzzer/FuzzerUtilLinux.cpp @@ -1,9 +1,8 @@ //===- FuzzerUtilLinux.cpp - Misc utils for Linux. ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Misc utils for Linux. @@ -14,12 +13,18 @@ #include "FuzzerCommand.h" #include +#include +#include + namespace fuzzer { int ExecuteCommand(const Command &Cmd) { std::string CmdLine = Cmd.toString(); - return system(CmdLine.c_str()); + int exit_code = system(CmdLine.c_str()); + if (WIFEXITED(exit_code)) + return WEXITSTATUS(exit_code); + return exit_code; } } // namespace fuzzer diff --git a/libfuzzer/FuzzerUtilPosix.cpp b/libfuzzer/FuzzerUtilPosix.cpp index bc64d32..110785d 100644 --- a/libfuzzer/FuzzerUtilPosix.cpp +++ b/libfuzzer/FuzzerUtilPosix.cpp @@ -1,9 +1,8 @@ //===- FuzzerUtilPosix.cpp - Misc utils for Posix. ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Misc utils implementation using Posix API. @@ -12,6 +11,7 @@ #if LIBFUZZER_POSIX #include "FuzzerIO.h" #include "FuzzerInternal.h" +#include "FuzzerTracePC.h" #include #include #include @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +33,16 @@ static void AlarmHandler(int, siginfo_t *, void *) { Fuzzer::StaticAlarmCallback(); } +static void (*upstream_segv_handler)(int, siginfo_t *, void *); + +static void SegvHandler(int sig, siginfo_t *si, void *ucontext) { + assert(si->si_signo == SIGSEGV); + if (TPC.UnprotectLazyCounters(si->si_addr)) return; + if (upstream_segv_handler) + return upstream_segv_handler(sig, si, ucontext); + Fuzzer::StaticCrashSignalCallback(); +} + static void CrashHandler(int, siginfo_t *, void *) { Fuzzer::StaticCrashSignalCallback(); } @@ -56,8 +67,11 @@ static void SetSigaction(int signum, exit(1); } if (sigact.sa_flags & SA_SIGINFO) { - if (sigact.sa_sigaction) - return; + if (sigact.sa_sigaction) { + if (signum != SIGSEGV) + return; + upstream_segv_handler = sigact.sa_sigaction; + } } else { if (sigact.sa_handler != SIG_DFL && sigact.sa_handler != SIG_IGN && sigact.sa_handler != SIG_ERR) @@ -65,6 +79,7 @@ static void SetSigaction(int signum, } sigact = {}; + sigact.sa_flags = SA_SIGINFO; sigact.sa_sigaction = callback; if (sigaction(signum, &sigact, 0)) { Printf("libFuzzer: sigaction failed with %d\n", errno); @@ -83,6 +98,11 @@ void SetTimer(int Seconds) { SetSigaction(SIGALRM, AlarmHandler); } +bool Mprotect(void *Ptr, size_t Size, bool AllowReadWrite) { + return 0 == mprotect(Ptr, Size, + AllowReadWrite ? (PROT_READ | PROT_WRITE) : PROT_NONE); +} + void SetSignalHandler(const FuzzingOptions& Options) { if (Options.UnitTimeoutSec > 0) SetTimer(Options.UnitTimeoutSec / 2 + 1); @@ -91,7 +111,7 @@ void SetSignalHandler(const FuzzingOptions& Options) { if (Options.HandleTerm) SetSigaction(SIGTERM, InterruptHandler); if (Options.HandleSegv) - SetSigaction(SIGSEGV, CrashHandler); + SetSigaction(SIGSEGV, SegvHandler); if (Options.HandleBus) SetSigaction(SIGBUS, CrashHandler); if (Options.HandleAbrt) diff --git a/libfuzzer/FuzzerUtilWindows.cpp b/libfuzzer/FuzzerUtilWindows.cpp index 8227e77..074e1eb 100644 --- a/libfuzzer/FuzzerUtilWindows.cpp +++ b/libfuzzer/FuzzerUtilWindows.cpp @@ -1,9 +1,8 @@ //===- FuzzerUtilWindows.cpp - Misc utils for Windows. --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Misc utils implementation for Windows. @@ -24,7 +23,7 @@ #include // This must be included after windows.h. -#include +#include namespace fuzzer { @@ -86,11 +85,11 @@ void CALLBACK AlarmHandler(PVOID, BOOLEAN) { class TimerQ { HANDLE TimerQueue; public: - TimerQ() : TimerQueue(NULL) {}; + TimerQ() : TimerQueue(NULL) {} ~TimerQ() { if (TimerQueue) DeleteTimerQueueEx(TimerQueue, NULL); - }; + } void SetTimer(int Seconds) { if (!TimerQueue) { TimerQueue = CreateTimerQueue(); @@ -105,13 +104,17 @@ class TimerQ { Printf("libFuzzer: CreateTimerQueueTimer failed.\n"); exit(1); } - }; + } }; static TimerQ Timer; static void CrashHandler(int) { Fuzzer::StaticCrashSignalCallback(); } +bool Mprotect(void *Ptr, size_t Size, bool AllowReadWrite) { + return false; // UNIMPLEMENTED +} + void SetSignalHandler(const FuzzingOptions& Options) { HandlerOpt = &Options; @@ -179,7 +182,9 @@ const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt, } std::string DisassembleCmd(const std::string &FileName) { - if (ExecuteCommand("dumpbin /summary > nul") == 0) + Vector command_vector; + command_vector.push_back("dumpbin /summary > nul"); + if (ExecuteCommand(Command(command_vector)) == 0) return "dumpbin /disasm " + FileName; Printf("libFuzzer: couldn't find tool to disassemble (dumpbin)\n"); exit(1); diff --git a/libfuzzer/FuzzerValueBitMap.h b/libfuzzer/FuzzerValueBitMap.h index 13d7cbd..bc039f1 100644 --- a/libfuzzer/FuzzerValueBitMap.h +++ b/libfuzzer/FuzzerValueBitMap.h @@ -1,9 +1,8 @@ //===- FuzzerValueBitMap.h - INTERNAL - Bit map -----------------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // ValueBitMap. @@ -35,7 +34,7 @@ struct ValueBitMap { uintptr_t WordIdx = Idx / kBitsInWord; uintptr_t BitIdx = Idx % kBitsInWord; uintptr_t Old = Map[WordIdx]; - uintptr_t New = Old | (1UL << BitIdx); + uintptr_t New = Old | (1ULL << BitIdx); Map[WordIdx] = New; return New != Old; } @@ -49,7 +48,7 @@ struct ValueBitMap { assert(Idx < kMapSizeInBits); uintptr_t WordIdx = Idx / kBitsInWord; uintptr_t BitIdx = Idx % kBitsInWord; - return Map[WordIdx] & (1UL << BitIdx); + return Map[WordIdx] & (1ULL << BitIdx); } size_t SizeInBits() const { return kMapSizeInBits; } @@ -65,7 +64,7 @@ struct ValueBitMap { } private: - uintptr_t Map[kMapSizeInWords] __attribute__((aligned(512))); + ATTRIBUTE_ALIGNED(512) uintptr_t Map[kMapSizeInWords]; }; } // namespace fuzzer diff --git a/libfuzzer/afl/afl_driver.cpp b/libfuzzer/afl/afl_driver.cpp index fa494c0..f21dfc5 100644 --- a/libfuzzer/afl/afl_driver.cpp +++ b/libfuzzer/afl/afl_driver.cpp @@ -1,9 +1,8 @@ //===- afl_driver.cpp - a glue between AFL and libFuzzer --------*- C++ -* ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception //===----------------------------------------------------------------------===// /* This file allows to fuzz libFuzzer-style target functions @@ -32,32 +31,22 @@ clang++ afl_driver.cpp test_fuzzer.o afl-llvm-rt.o.o rm -rf IN OUT; mkdir IN OUT; echo z > IN/z; $AFL_HOME/afl-fuzz -i IN -o OUT ./a.out ################################################################################ -Environment Variables: -There are a few environment variables that can be set to use features that -afl-fuzz doesn't have. - AFL_DRIVER_STDERR_DUPLICATE_FILENAME: Setting this *appends* stderr to the file specified. If the file does not exist, it is created. This is useful for getting -stack traces (when using ASAN for example) or original error messages on hard to -reproduce bugs. +stack traces (when using ASAN for example) or original error messages on hard +to reproduce bugs. Note that any content written to stderr will be written to +this file instead of stderr's usual location. -AFL_DRIVER_EXTRA_STATS_FILENAME: Setting this causes afl_driver to write extra -statistics to the file specified. Currently these are peak_rss_mb -(the peak amount of virtual memory used in MB) and slowest_unit_time_secs. If -the file does not exist it is created. If the file does exist then -afl_driver assumes it was restarted by afl-fuzz and will try to read old -statistics from the file. If that fails then the process will quit. +AFL_DRIVER_CLOSE_FD_MASK: Similar to libFuzzer's -close_fd_mask behavior option. +If 1, close stdout at startup. If 2 close stderr; if 3 close both. */ #include #include -#include #include #include #include #include -#include -#include #include #include @@ -99,17 +88,6 @@ statistics from the file. If that fails then the process will quit. #error "Support for your platform has not been implemented" #endif -// Used to avoid repeating error checking boilerplate. If cond is false, a -// fatal error has occurred in the program. In this event print error_message -// to stderr and abort(). Otherwise do nothing. Note that setting -// AFL_DRIVER_STDERR_DUPLICATE_FILENAME may cause error_message to be appended -// to the file as well, if the error occurs after the duplication is performed. -#define CHECK_ERROR(cond, error_message) \ - if (!(cond)) { \ - fprintf(stderr, "%s\n", (error_message)); \ - abort(); \ - } - // libFuzzer interface is thin, so we don't include any libFuzzer headers. extern "C" { int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); @@ -123,24 +101,24 @@ static volatile char suppress_warning2 = AFL_PERSISTENT[0]; // Notify AFL about deferred forkserver. static volatile char AFL_DEFER_FORKSVR[] = "##SIG_AFL_DEFER_FORKSRV##"; -extern "C" void __afl_manual_init(); +extern "C" void __afl_manual_init(); static volatile char suppress_warning1 = AFL_DEFER_FORKSVR[0]; // Input buffer. static const size_t kMaxAflInputSize = 1 << 20; static uint8_t AflInputBuf[kMaxAflInputSize]; -// Variables we need for writing to the extra stats file. -static FILE *extra_stats_file = NULL; -static uint32_t previous_peak_rss = 0; -static time_t slowest_unit_time_secs = 0; -static const int kNumExtraStats = 2; -static const char *kExtraStatsFormatString = "peak_rss_mb : %u\n" - "slowest_unit_time_sec : %u\n"; +// Use this optionally defined function to output sanitizer messages even if +// user asks to close stderr. +__attribute__((weak)) extern "C" void __sanitizer_set_report_fd(void *); + +// Keep track of where stderr content is being written to, so that +// dup_and_close_stderr can use the correct one. +static FILE *output_file = stderr; // Experimental feature to use afl_driver without AFL's deferred mode. // Needs to run before __afl_auto_init. -__attribute__((constructor(0))) void __decide_deferred_forkserver(void) { +__attribute__((constructor(0))) static void __decide_deferred_forkserver(void) { if (getenv("AFL_DRIVER_DONT_DEFER")) { if (unsetenv("__AFL_DEFER_FORKSRV")) { perror("Failed to unset __AFL_DEFER_FORKSRV"); @@ -149,126 +127,15 @@ __attribute__((constructor(0))) void __decide_deferred_forkserver(void) { } } -// Copied from FuzzerUtil.cpp. -size_t GetPeakRSSMb() { - struct rusage usage; - if (getrusage(RUSAGE_SELF, &usage)) - return 0; - if (LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD || - LIBFUZZER_OPENBSD) { - // ru_maxrss is in KiB - return usage.ru_maxrss >> 10; - } else if (LIBFUZZER_APPLE) { - // ru_maxrss is in bytes - return usage.ru_maxrss >> 20; - } - assert(0 && "GetPeakRSSMb() is not implemented for your platform"); - return 0; -} - -// Based on SetSigaction in FuzzerUtil.cpp -static void SetSigaction(int signum, - void (*callback)(int, siginfo_t *, void *)) { - struct sigaction sigact; - memset(&sigact, 0, sizeof(sigact)); - sigact.sa_sigaction = callback; - if (sigaction(signum, &sigact, 0)) { - fprintf(stderr, "libFuzzer: sigaction failed with %d\n", errno); - exit(1); - } -} - -// Write extra stats to the file specified by the user. If none is specified -// this function will never be called. -static void write_extra_stats() { - uint32_t peak_rss = GetPeakRSSMb(); - - if (peak_rss < previous_peak_rss) - peak_rss = previous_peak_rss; - - int chars_printed = fprintf(extra_stats_file, kExtraStatsFormatString, - peak_rss, slowest_unit_time_secs); - - CHECK_ERROR(chars_printed != 0, "Failed to write extra_stats_file"); - - CHECK_ERROR(fclose(extra_stats_file) == 0, - "Failed to close extra_stats_file"); -} - -// Call write_extra_stats before we exit. -static void crash_handler(int, siginfo_t *, void *) { - // Make sure we don't try calling write_extra_stats again if we crashed while - // trying to call it. - static bool first_crash = true; - CHECK_ERROR(first_crash, - "Crashed in crash signal handler. This is a bug in the fuzzer."); - - first_crash = false; - write_extra_stats(); -} - -// If the user has specified an extra_stats_file through the environment -// variable AFL_DRIVER_EXTRA_STATS_FILENAME, then perform necessary set up -// to write stats to it on exit. If no file is specified, do nothing. Otherwise -// install signal and exit handlers to write to the file when the process exits. -// Then if the file doesn't exist create it and set extra stats to 0. But if it -// does exist then read the initial values of the extra stats from the file -// and check that the file is writable. -static void maybe_initialize_extra_stats() { - // If AFL_DRIVER_EXTRA_STATS_FILENAME isn't set then we have nothing to do. - char *extra_stats_filename = getenv("AFL_DRIVER_EXTRA_STATS_FILENAME"); - if (!extra_stats_filename) - return; - - // Open the file and find the previous peak_rss_mb value. - // This is necessary because the fuzzing process is restarted after N - // iterations are completed. So we may need to get this value from a previous - // process to be accurate. - extra_stats_file = fopen(extra_stats_filename, "r"); - - // If extra_stats_file already exists: read old stats from it. - if (extra_stats_file) { - int matches = fscanf(extra_stats_file, kExtraStatsFormatString, - &previous_peak_rss, &slowest_unit_time_secs); - - // Make sure we have read a real extra stats file and that we have used it - // to set slowest_unit_time_secs and previous_peak_rss. - CHECK_ERROR(matches == kNumExtraStats, "Extra stats file is corrupt"); - - CHECK_ERROR(fclose(extra_stats_file) == 0, "Failed to close file"); - - // Now open the file for writing. - extra_stats_file = fopen(extra_stats_filename, "w"); - CHECK_ERROR(extra_stats_file, - "Failed to open extra stats file for writing"); - } else { - // Looks like this is the first time in a fuzzing job this is being called. - extra_stats_file = fopen(extra_stats_filename, "w+"); - CHECK_ERROR(extra_stats_file, "failed to create extra stats file"); - } - - // Make sure that crash_handler gets called on any kind of fatal error. - int crash_signals[] = {SIGSEGV, SIGBUS, SIGABRT, SIGILL, SIGFPE, SIGINT, - SIGTERM}; - - const size_t num_signals = sizeof(crash_signals) / sizeof(crash_signals[0]); - - for (size_t idx = 0; idx < num_signals; idx++) - SetSigaction(crash_signals[idx], crash_handler); - - // Make sure it gets called on other kinds of exits. - atexit(write_extra_stats); -} - // If the user asks us to duplicate stderr, then do it. static void maybe_duplicate_stderr() { - char* stderr_duplicate_filename = + char *stderr_duplicate_filename = getenv("AFL_DRIVER_STDERR_DUPLICATE_FILENAME"); if (!stderr_duplicate_filename) return; - FILE* stderr_duplicate_stream = + FILE *stderr_duplicate_stream = freopen(stderr_duplicate_filename, "a+", stderr); if (!stderr_duplicate_stream) { @@ -277,6 +144,54 @@ static void maybe_duplicate_stderr() { "Failed to duplicate stderr to AFL_DRIVER_STDERR_DUPLICATE_FILENAME"); abort(); } + output_file = stderr_duplicate_stream; +} + +// Most of these I/O functions were inspired by/copied from libFuzzer's code. +static void discard_output(int fd) { + FILE *temp = fopen("/dev/null", "w"); + if (!temp) + abort(); + dup2(fileno(temp), fd); + fclose(temp); +} + +static void close_stdout() { discard_output(STDOUT_FILENO); } + +// Prevent the targeted code from writing to "stderr" but allow sanitizers and +// this driver to do so. +static void dup_and_close_stderr() { + int output_fileno = fileno(output_file); + int output_fd = dup(output_fileno); + if (output_fd <= 0) + abort(); + FILE *new_output_file = fdopen(output_fd, "w"); + if (!new_output_file) + abort(); + if (!__sanitizer_set_report_fd) + return; + __sanitizer_set_report_fd(reinterpret_cast(output_fd)); + discard_output(output_fileno); +} + +static void Printf(const char *Fmt, ...) { + va_list ap; + va_start(ap, Fmt); + vfprintf(output_file, Fmt, ap); + va_end(ap); + fflush(output_file); +} + +// Close stdout and/or stderr if user asks for it. +static void maybe_close_fd_mask() { + char *fd_mask_str = getenv("AFL_DRIVER_CLOSE_FD_MASK"); + if (!fd_mask_str) + return; + int fd_mask = atoi(fd_mask_str); + if (fd_mask & 2) + dup_and_close_stderr(); + if (fd_mask & 1) + close_stdout(); } // Define LLVMFuzzerMutate to avoid link failures for targets that use it @@ -287,9 +202,9 @@ extern "C" size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) { } // Execute any files provided as parameters. -int ExecuteFilesOnyByOne(int argc, char **argv) { +static int ExecuteFilesOnyByOne(int argc, char **argv) { for (int i = 1; i < argc; i++) { - std::ifstream in(argv[i]); + std::ifstream in(argv[i], std::ios::binary); in.seekg(0, in.end); size_t length = in.tellg(); in.seekg (0, in.beg); @@ -306,7 +221,7 @@ int ExecuteFilesOnyByOne(int argc, char **argv) { } int main(int argc, char **argv) { - fprintf(stderr, + Printf( "======================= INFO =========================\n" "This binary is built for AFL-fuzz.\n" "To run the target function on individual input(s) execute this:\n" @@ -319,13 +234,13 @@ int main(int argc, char **argv) { "re-spawning the process (default: 1000)\n" "======================================================\n", argv[0], argv[0], argv[0]); + + maybe_duplicate_stderr(); + maybe_close_fd_mask(); if (LLVMFuzzerInitialize) LLVMFuzzerInitialize(&argc, &argv); // Do any other expensive one-time initialization here. - maybe_duplicate_stderr(); - maybe_initialize_extra_stats(); - if (!getenv("AFL_DRIVER_DONT_DEFER")) __afl_manual_init(); @@ -333,8 +248,7 @@ int main(int argc, char **argv) { if (argc == 2 && argv[1][0] == '-') N = atoi(argv[1] + 1); else if(argc == 2 && (N = atoi(argv[1])) > 0) - fprintf(stderr, "WARNING: using the deprecated call style `%s %d`\n", - argv[0], N); + Printf("WARNING: using the deprecated call style `%s %d`\n", argv[0], N); else if (argc > 1) return ExecuteFilesOnyByOne(argc, argv); @@ -345,7 +259,6 @@ int main(int argc, char **argv) { uint8_t dummy_input[1] = {0}; LLVMFuzzerTestOneInput(dummy_input, 1); - time_t unit_time_secs; int num_runs = 0; while (__afl_persistent_loop(N)) { ssize_t n_read = read(0, AflInputBuf, kMaxAflInputSize); @@ -354,25 +267,10 @@ int main(int argc, char **argv) { // overflows. Don't use unique_ptr/etc to avoid extra dependencies. uint8_t *copy = new uint8_t[n_read]; memcpy(copy, AflInputBuf, n_read); - - struct timeval unit_start_time; - CHECK_ERROR(gettimeofday(&unit_start_time, NULL) == 0, - "Calling gettimeofday failed"); - num_runs++; LLVMFuzzerTestOneInput(copy, n_read); - - struct timeval unit_stop_time; - CHECK_ERROR(gettimeofday(&unit_stop_time, NULL) == 0, - "Calling gettimeofday failed"); - - // Update slowest_unit_time_secs if we see a new max. - unit_time_secs = unit_stop_time.tv_sec - unit_start_time.tv_sec; - if (slowest_unit_time_secs < unit_time_secs) - slowest_unit_time_secs = unit_time_secs; - delete[] copy; } } - fprintf(stderr, "%s: successfully executed %d input(s)\n", argv[0], num_runs); + Printf("%s: successfully executed %d input(s)\n", argv[0], num_runs); } diff --git a/libfuzzer/dataflow/DataFlow.cpp b/libfuzzer/dataflow/DataFlow.cpp index a79c796..8bf4e25 100644 --- a/libfuzzer/dataflow/DataFlow.cpp +++ b/libfuzzer/dataflow/DataFlow.cpp @@ -1,9 +1,8 @@ /*===- DataFlow.cpp - a standalone DataFlow tracer -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // An experimental data-flow tracer for fuzz targets. @@ -14,25 +13,30 @@ // It executes the fuzz target on the given input while monitoring the // data flow for every instrumented comparison instruction. // -// The output shows which functions depend on which bytes of the input. +// The output shows which functions depend on which bytes of the input, +// and also provides basic-block coverage for every input. // // Build: -// 1. Compile this file with -fsanitize=dataflow -// 2. Build the fuzz target with -g -fsanitize=dataflow -// -fsanitize-coverage=trace-pc-guard,pc-table,func,trace-cmp -// 3. Link those together with -fsanitize=dataflow +// 1. Compile this file (DataFlow.cpp) with -fsanitize=dataflow and -O2. +// 2. Compile DataFlowCallbacks.cpp with -O2 -fPIC. +// 3. Build the fuzz target with -g -fsanitize=dataflow +// -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp +// 4. Link those together with -fsanitize=dataflow // // -fsanitize-coverage=trace-cmp inserts callbacks around every comparison // instruction, DFSan modifies the calls to pass the data flow labels. // The callbacks update the data flow label for the current function. // See e.g. __dfsw___sanitizer_cov_trace_cmp1 below. // -// -fsanitize-coverage=trace-pc-guard,pc-table,func instruments function +// -fsanitize-coverage=trace-pc-guard,pc-table,bb instruments function // entries so that the comparison callback knows that current function. +// -fsanitize-coverage=...,bb also allows to collect basic block coverage. // // // Run: -// # Collect data flow for INPUT_FILE, write to OUTPUT_FILE (default: stdout) +// # Collect data flow and coverage for INPUT_FILE +// # write to OUTPUT_FILE (default: stdout) +// export DFSAN_OPTIONS=fast16labels=1:warn_unimplemented=0 // ./a.out INPUT_FILE [OUTPUT_FILE] // // # Print all instrumented functions. llvm-symbolizer must be present in PATH @@ -42,10 +46,14 @@ // =============== // F0 11111111111111 // F1 10000000000000 +// C0 1 2 3 4 5 +// C1 8 // =============== // "FN xxxxxxxxxx": tells what bytes of the input does the function N depend on. -// The byte string is LEN+1 bytes. The last byte is set if the function -// depends on the input length. +// "CN X Y Z T": tells that a function N has basic blocks X, Y, and Z covered +// in addition to the function's entry block, out of T total instrumented +// blocks. +// //===----------------------------------------------------------------------===*/ #include @@ -56,20 +64,23 @@ #include // backtrace_symbols_fd -#include +#include "DataFlow.h" extern "C" { extern int LLVMFuzzerTestOneInput(const unsigned char *Data, size_t Size); __attribute__((weak)) extern int LLVMFuzzerInitialize(int *argc, char ***argv); } // extern "C" +CallbackData __dft; static size_t InputLen; -static size_t NumFuncs; -static const uintptr_t *FuncsBeg; -static __thread size_t CurrentFunc; -static dfsan_label *FuncLabels; // Array of NumFuncs elements. -static char *PrintableStringForLabel; // InputLen + 2 bytes. -static bool LabelSeen[1 << 8 * sizeof(dfsan_label)]; +static size_t NumIterations; +static dfsan_label **FuncLabelsPerIter; // NumIterations x NumFuncs; + +static inline bool BlockIsEntry(size_t BlockIdx) { + return __dft.PCsBeg[BlockIdx * 2 + 1] & PCFLAG_FUNC_ENTRY; +} + +const int kNumLabels = 16; // Prints all instrumented functions. static int PrintFunctions() { @@ -80,9 +91,12 @@ static int PrintFunctions() { FILE *Pipe = popen("sed 's/(+/ /g; s/).*//g' " "| llvm-symbolizer " "| grep 'dfs\\$' " - "| sed 's/dfs\\$//g'", "w"); - for (size_t I = 0; I < NumFuncs; I++) { - uintptr_t PC = FuncsBeg[I * 2]; + "| sed 's/dfs\\$//g' " + "| c++filt", + "w"); + for (size_t I = 0; I < __dft.NumGuards; I++) { + uintptr_t PC = __dft.PCsBeg[I * 2]; + if (!BlockIsEntry(I)) continue; void *const Buf[1] = {(void*)PC}; backtrace_symbols_fd(Buf, 1, fileno(Pipe)); } @@ -90,33 +104,53 @@ static int PrintFunctions() { return 0; } -extern "C" -void SetBytesForLabel(dfsan_label L, char *Bytes) { - if (LabelSeen[L]) - return; - LabelSeen[L] = true; - assert(L); - if (L <= InputLen + 1) { - Bytes[L - 1] = '1'; - } else { - auto *DLI = dfsan_get_label_info(L); - SetBytesForLabel(DLI->l1, Bytes); - SetBytesForLabel(DLI->l2, Bytes); - } +static void PrintBinary(FILE *Out, dfsan_label L, size_t Len) { + char buf[kNumLabels + 1]; + assert(Len <= kNumLabels); + for (int i = 0; i < kNumLabels; i++) + buf[i] = (L & (1 << i)) ? '1' : '0'; + buf[Len] = 0; + fprintf(Out, "%s", buf); } -static char *GetPrintableStringForLabel(dfsan_label L) { - memset(PrintableStringForLabel, '0', InputLen + 1); - PrintableStringForLabel[InputLen + 1] = 0; - memset(LabelSeen, 0, sizeof(LabelSeen)); - SetBytesForLabel(L, PrintableStringForLabel); - return PrintableStringForLabel; +static void PrintDataFlow(FILE *Out) { + for (size_t Func = 0; Func < __dft.NumFuncs; Func++) { + bool HasAny = false; + for (size_t Iter = 0; Iter < NumIterations; Iter++) + if (FuncLabelsPerIter[Iter][Func]) + HasAny = true; + if (!HasAny) + continue; + fprintf(Out, "F%zd ", Func); + size_t LenOfLastIteration = kNumLabels; + if (auto Tail = InputLen % kNumLabels) + LenOfLastIteration = Tail; + for (size_t Iter = 0; Iter < NumIterations; Iter++) + PrintBinary(Out, FuncLabelsPerIter[Iter][Func], + Iter == NumIterations - 1 ? LenOfLastIteration : kNumLabels); + fprintf(Out, "\n"); + } } -static void PrintDataFlow(FILE *Out) { - for (size_t I = 0; I < NumFuncs; I++) - if (FuncLabels[I]) - fprintf(Out, "F%zd %s\n", I, GetPrintableStringForLabel(FuncLabels[I])); +static void PrintCoverage(FILE *Out) { + ssize_t CurrentFuncGuard = -1; + ssize_t CurrentFuncNum = -1; + ssize_t NumBlocksInCurrentFunc = -1; + for (size_t FuncBeg = 0; FuncBeg < __dft.NumGuards;) { + CurrentFuncNum++; + assert(BlockIsEntry(FuncBeg)); + size_t FuncEnd = FuncBeg + 1; + for (; FuncEnd < __dft.NumGuards && !BlockIsEntry(FuncEnd); FuncEnd++) + ; + if (__dft.BBExecuted[FuncBeg]) { + fprintf(Out, "C%zd", CurrentFuncNum); + for (size_t I = FuncBeg + 1; I < FuncEnd; I++) + if (__dft.BBExecuted[I]) + fprintf(Out, " %zd", I - FuncBeg); + fprintf(Out, " %zd\n", FuncEnd - FuncBeg); + } + FuncBeg = FuncEnd; + } } int main(int argc, char **argv) { @@ -124,12 +158,9 @@ int main(int argc, char **argv) { LLVMFuzzerInitialize(&argc, &argv); if (argc == 1) return PrintFunctions(); - assert(argc == 4 || argc == 5); - size_t Beg = atoi(argv[1]); - size_t End = atoi(argv[2]); - assert(Beg < End); + assert(argc == 2 || argc == 3); - const char *Input = argv[3]; + const char *Input = argv[1]; fprintf(stderr, "INFO: reading '%s'\n", Input); FILE *In = fopen(Input, "r"); assert(In); @@ -139,79 +170,36 @@ int main(int argc, char **argv) { unsigned char *Buf = (unsigned char*)malloc(InputLen); size_t NumBytesRead = fread(Buf, 1, InputLen, In); assert(NumBytesRead == InputLen); - PrintableStringForLabel = (char*)malloc(InputLen + 2); fclose(In); - fprintf(stderr, "INFO: running '%s'\n", Input); - for (size_t I = 1; I <= InputLen; I++) { - dfsan_label L = dfsan_create_label("", nullptr); - assert(L == I); - size_t Idx = I - 1; - if (Idx >= Beg && Idx < End) - dfsan_set_label(L, Buf + Idx, 1); + NumIterations = (NumBytesRead + kNumLabels - 1) / kNumLabels; + FuncLabelsPerIter = + (dfsan_label **)calloc(NumIterations, sizeof(dfsan_label *)); + for (size_t Iter = 0; Iter < NumIterations; Iter++) + FuncLabelsPerIter[Iter] = + (dfsan_label *)calloc(__dft.NumFuncs, sizeof(dfsan_label)); + + for (size_t Iter = 0; Iter < NumIterations; Iter++) { + fprintf(stderr, "INFO: running '%s' %zd/%zd\n", Input, Iter, NumIterations); + dfsan_flush(); + dfsan_set_label(0, Buf, InputLen); + __dft.FuncLabels = FuncLabelsPerIter[Iter]; + + size_t BaseIdx = Iter * kNumLabels; + size_t LastIdx = BaseIdx + kNumLabels < NumBytesRead ? BaseIdx + kNumLabels + : NumBytesRead; + assert(BaseIdx < LastIdx); + for (size_t Idx = BaseIdx; Idx < LastIdx; Idx++) + dfsan_set_label(1 << (Idx - BaseIdx), Buf + Idx, 1); + LLVMFuzzerTestOneInput(Buf, InputLen); } - dfsan_label SizeL = dfsan_create_label("", nullptr); - assert(SizeL == InputLen + 1); - dfsan_set_label(SizeL, &InputLen, sizeof(InputLen)); - - LLVMFuzzerTestOneInput(Buf, InputLen); free(Buf); - bool OutIsStdout = argc == 4; + bool OutIsStdout = argc == 2; fprintf(stderr, "INFO: writing dataflow to %s\n", - OutIsStdout ? "" : argv[4]); - FILE *Out = OutIsStdout ? stdout : fopen(argv[4], "w"); + OutIsStdout ? "" : argv[2]); + FILE *Out = OutIsStdout ? stdout : fopen(argv[2], "w"); PrintDataFlow(Out); + PrintCoverage(Out); if (!OutIsStdout) fclose(Out); } - -extern "C" { - -void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, - uint32_t *stop) { - assert(NumFuncs == 0 && "This tool does not support DSOs"); - assert(start < stop && "The code is not instrumented for coverage"); - if (start == stop || *start) return; // Initialize only once. - for (uint32_t *x = start; x < stop; x++) - *x = ++NumFuncs; // The first index is 1. - FuncLabels = (dfsan_label*)calloc(NumFuncs, sizeof(dfsan_label)); - fprintf(stderr, "INFO: %zd instrumented function(s) observed\n", NumFuncs); -} - -void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, - const uintptr_t *pcs_end) { - assert(NumFuncs == (pcs_end - pcs_beg) / 2); - FuncsBeg = pcs_beg; -} - -void __sanitizer_cov_trace_pc_indir(uint64_t x){} // unused. - -void __sanitizer_cov_trace_pc_guard(uint32_t *guard){ - uint32_t FuncNum = *guard - 1; // Guards start from 1. - assert(FuncNum < NumFuncs); - CurrentFunc = FuncNum; -} - -void __dfsw___sanitizer_cov_trace_switch(uint64_t Val, uint64_t *Cases, - dfsan_label L1, dfsan_label UnusedL) { - assert(CurrentFunc < NumFuncs); - FuncLabels[CurrentFunc] = dfsan_union(FuncLabels[CurrentFunc], L1); -} - -#define HOOK(Name, Type) \ - void Name(Type Arg1, Type Arg2, dfsan_label L1, dfsan_label L2) { \ - assert(CurrentFunc < NumFuncs); \ - FuncLabels[CurrentFunc] = \ - dfsan_union(FuncLabels[CurrentFunc], dfsan_union(L1, L2)); \ - } - -HOOK(__dfsw___sanitizer_cov_trace_const_cmp1, uint8_t) -HOOK(__dfsw___sanitizer_cov_trace_const_cmp2, uint16_t) -HOOK(__dfsw___sanitizer_cov_trace_const_cmp4, uint32_t) -HOOK(__dfsw___sanitizer_cov_trace_const_cmp8, uint64_t) -HOOK(__dfsw___sanitizer_cov_trace_cmp1, uint8_t) -HOOK(__dfsw___sanitizer_cov_trace_cmp2, uint16_t) -HOOK(__dfsw___sanitizer_cov_trace_cmp4, uint32_t) -HOOK(__dfsw___sanitizer_cov_trace_cmp8, uint64_t) - -} // extern "C" diff --git a/libfuzzer/dataflow/DataFlow.h b/libfuzzer/dataflow/DataFlow.h new file mode 100644 index 0000000..35849fd --- /dev/null +++ b/libfuzzer/dataflow/DataFlow.h @@ -0,0 +1,32 @@ +/*===- DataFlow.h - a standalone DataFlow trace -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Internal header file to connect DataFlow.cpp and DataFlowCallbacks.cpp. +//===----------------------------------------------------------------------===*/ + +#ifndef __LIBFUZZER_DATAFLOW_H +#define __LIBFUZZER_DATAFLOW_H + +#include +#include +#include + +// This data is shared between DataFlowCallbacks.cpp and DataFlow.cpp. +struct CallbackData { + size_t NumFuncs, NumGuards; + const uintptr_t *PCsBeg, *PCsEnd; + dfsan_label *FuncLabels; // Array of NumFuncs elements. + bool *BBExecuted; // Array of NumGuards elements. +}; + +extern CallbackData __dft; + +enum { + PCFLAG_FUNC_ENTRY = 1, +}; + +#endif // __LIBFUZZER_DATAFLOW_H diff --git a/libfuzzer/dataflow/DataFlowCallbacks.cpp b/libfuzzer/dataflow/DataFlowCallbacks.cpp new file mode 100644 index 0000000..170b4d8 --- /dev/null +++ b/libfuzzer/dataflow/DataFlowCallbacks.cpp @@ -0,0 +1,86 @@ +/*===- DataFlowCallbacks.cpp - a standalone DataFlow trace -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Instrumentation callbacks for DataFlow.cpp. +// These functions should not be instrumented by DFSan, so we +// keep them in a separate file and compile it w/o DFSan. +//===----------------------------------------------------------------------===*/ +#include "DataFlow.h" + +#include +#include +#include + +static __thread size_t CurrentFunc; +static uint32_t *GuardsBeg, *GuardsEnd; +static inline bool BlockIsEntry(size_t BlockIdx) { + return __dft.PCsBeg[BlockIdx * 2 + 1] & PCFLAG_FUNC_ENTRY; +} + +extern "C" { + +void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, + uint32_t *stop) { + assert(__dft.NumFuncs == 0 && "This tool does not support DSOs"); + assert(start < stop && "The code is not instrumented for coverage"); + if (start == stop || *start) return; // Initialize only once. + GuardsBeg = start; + GuardsEnd = stop; +} + +void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, + const uintptr_t *pcs_end) { + if (__dft.NumGuards) return; // Initialize only once. + __dft.NumGuards = GuardsEnd - GuardsBeg; + __dft.PCsBeg = pcs_beg; + __dft.PCsEnd = pcs_end; + assert(__dft.NumGuards == (__dft.PCsEnd - __dft.PCsBeg) / 2); + for (size_t i = 0; i < __dft.NumGuards; i++) { + if (BlockIsEntry(i)) { + __dft.NumFuncs++; + GuardsBeg[i] = __dft.NumFuncs; + } + } + __dft.BBExecuted = (bool*)calloc(__dft.NumGuards, sizeof(bool)); + fprintf(stderr, "INFO: %zd instrumented function(s) observed " + "and %zd basic blocks\n", __dft.NumFuncs, __dft.NumGuards); +} + +void __sanitizer_cov_trace_pc_indir(uint64_t x){} // unused. + +void __sanitizer_cov_trace_pc_guard(uint32_t *guard) { + size_t GuardIdx = guard - GuardsBeg; + // assert(GuardIdx < __dft.NumGuards); + __dft.BBExecuted[GuardIdx] = true; + if (!*guard) return; // not a function entry. + uint32_t FuncNum = *guard - 1; // Guards start from 1. + // assert(FuncNum < __dft.NumFuncs); + CurrentFunc = FuncNum; +} + +void __dfsw___sanitizer_cov_trace_switch(uint64_t Val, uint64_t *Cases, + dfsan_label L1, dfsan_label UnusedL) { + assert(CurrentFunc < __dft.NumFuncs); + __dft.FuncLabels[CurrentFunc] |= L1; +} + +#define HOOK(Name, Type) \ + void Name(Type Arg1, Type Arg2, dfsan_label L1, dfsan_label L2) { \ + __dft.FuncLabels[CurrentFunc] |= L1 | L2; \ + } + //assert(CurrentFunc < __dft.NumFuncs); + +HOOK(__dfsw___sanitizer_cov_trace_const_cmp1, uint8_t) +HOOK(__dfsw___sanitizer_cov_trace_const_cmp2, uint16_t) +HOOK(__dfsw___sanitizer_cov_trace_const_cmp4, uint32_t) +HOOK(__dfsw___sanitizer_cov_trace_const_cmp8, uint64_t) +HOOK(__dfsw___sanitizer_cov_trace_cmp1, uint8_t) +HOOK(__dfsw___sanitizer_cov_trace_cmp2, uint16_t) +HOOK(__dfsw___sanitizer_cov_trace_cmp4, uint32_t) +HOOK(__dfsw___sanitizer_cov_trace_cmp8, uint64_t) + +} // extern "C" diff --git a/libfuzzer/scripts/collect_data_flow.py b/libfuzzer/scripts/collect_data_flow.py deleted file mode 100755 index 3edff66..0000000 --- a/libfuzzer/scripts/collect_data_flow.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python -#===- lib/fuzzer/scripts/collect_data_flow.py ------------------------------===# -# -# The LLVM Compiler Infrastructure -# -# This file is distributed under the University of Illinois Open Source -# License. See LICENSE.TXT for details. -# -#===------------------------------------------------------------------------===# -# Runs the data-flow tracer several times on the same input in order to collect -# the complete trace for all input bytes (running it on all bytes at once -# may fail if DFSan runs out of labels). -# Usage: -# -# # Collect dataflow for one input, store it in OUTPUT (default is stdout) -# collect_data_flow.py BINARY INPUT [OUTPUT] -# -# # Collect dataflow for all inputs in CORPUS_DIR, store them in OUTPUT_DIR -# collect_data_flow.py BINARY CORPUS_DIR OUTPUT_DIR -#===------------------------------------------------------------------------===# -import atexit -import hashlib -import sys -import os -import subprocess -import tempfile -import shutil - -tmpdir = "" - -def cleanup(d): - print("removing: %s" % d) - shutil.rmtree(d) - -def collect_dataflow_for_corpus(self, exe, corpus_dir, output_dir): - print("Collecting dataflow for corpus: %s output_dir: %s" % (corpus_dir, - output_dir)) - assert not os.path.exists(output_dir) - os.mkdir(output_dir) - for root, dirs, files in os.walk(corpus_dir): - for f in files: - path = os.path.join(root, f) - sha1 = hashlib.sha1(open(path).read()).hexdigest() - output = os.path.join(output_dir, sha1) - subprocess.call([self, exe, path, output]) - functions_txt = open(os.path.join(output_dir, "functions.txt"), "w") - subprocess.call([exe], stdout=functions_txt) - - -def main(argv): - exe = argv[1] - inp = argv[2] - if os.path.isdir(inp): - return collect_dataflow_for_corpus(argv[0], exe, inp, argv[3]) - size = os.path.getsize(inp) - q = [[0, size]] - tmpdir = tempfile.mkdtemp(prefix="libfuzzer-tmp-") - atexit.register(cleanup, tmpdir) - print "tmpdir: ", tmpdir - outputs = [] - while len(q): - r = q.pop() - print "******* Trying: ", r - tmpfile = os.path.join(tmpdir, str(r[0]) + "-" + str(r[1])) - ret = subprocess.call([exe, str(r[0]), str(r[1]), inp, tmpfile]) - if ret and r[1] - r[0] >= 2: - q.append([r[0], (r[1] + r[0]) / 2]) - q.append([(r[1] + r[0]) / 2, r[1]]) - else: - outputs.append(tmpfile) - print "******* Success: ", r - f = sys.stdout - if len(argv) >= 4: - f = open(argv[3], "w") - merge = os.path.join(os.path.dirname(argv[0]), "merge_data_flow.py") - subprocess.call([merge] + outputs, stdout=f) - -if __name__ == '__main__': - main(sys.argv) diff --git a/libfuzzer/scripts/merge_data_flow.py b/libfuzzer/scripts/merge_data_flow.py deleted file mode 100755 index d2f5081..0000000 --- a/libfuzzer/scripts/merge_data_flow.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -#===- lib/fuzzer/scripts/merge_data_flow.py ------------------------------===# -# -# The LLVM Compiler Infrastructure -# -# This file is distributed under the University of Illinois Open Source -# License. See LICENSE.TXT for details. -# -#===------------------------------------------------------------------------===# -# Merge several data flow traces into one. -# Usage: -# merge_data_flow.py trace1 trace2 ... > result -#===------------------------------------------------------------------------===# -import sys -import fileinput -from array import array - -def Merge(a, b): - res = array('b') - for i in range(0, len(a)): - res.append(ord('1' if a[i] == '1' or b[i] == '1' else '0')) - return res.tostring() - -def main(argv): - D = {} - for line in fileinput.input(): - [F,BV] = line.strip().split(' ') - if F in D: - D[F] = Merge(D[F], BV) - else: - D[F] = BV; - for F in D.keys(): - print("%s %s" % (F, D[F])) - -if __name__ == '__main__': - main(sys.argv) diff --git a/libfuzzer/scripts/unbalanced_allocs.py b/libfuzzer/scripts/unbalanced_allocs.py index 74478ad..579e481 100755 --- a/libfuzzer/scripts/unbalanced_allocs.py +++ b/libfuzzer/scripts/unbalanced_allocs.py @@ -1,10 +1,9 @@ #!/usr/bin/env python #===- lib/fuzzer/scripts/unbalanced_allocs.py ------------------------------===# # -# The LLVM Compiler Infrastructure -# -# This file is distributed under the University of Illinois Open Source -# License. See LICENSE.TXT for details. +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # #===------------------------------------------------------------------------===# # diff --git a/libfuzzer/standalone/StandaloneFuzzTargetMain.c b/libfuzzer/standalone/StandaloneFuzzTargetMain.c index 0d76ea4..efe512c 100644 --- a/libfuzzer/standalone/StandaloneFuzzTargetMain.c +++ b/libfuzzer/standalone/StandaloneFuzzTargetMain.c @@ -1,9 +1,8 @@ /*===- StandaloneFuzzTargetMain.c - standalone main() for fuzz targets. ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This main() function can be linked to a fuzz target (i.e. a library @@ -33,6 +32,7 @@ int main(int argc, char **argv) { fseek(f, 0, SEEK_SET); unsigned char *buf = (unsigned char*)malloc(len); size_t n_read = fread(buf, 1, len, f); + fclose(f); assert(n_read == len); LLVMFuzzerTestOneInput(buf, len); free(buf); diff --git a/libfuzzer/tests/CMakeLists.txt b/libfuzzer/tests/CMakeLists.txt index ed58071..6f66510 100644 --- a/libfuzzer/tests/CMakeLists.txt +++ b/libfuzzer/tests/CMakeLists.txt @@ -1,3 +1,5 @@ +include(CompilerRTCompile) + set(LIBFUZZER_UNITTEST_CFLAGS ${COMPILER_RT_UNITTEST_CFLAGS} ${COMPILER_RT_GTEST_CFLAGS} @@ -12,17 +14,27 @@ endif() add_custom_target(FuzzerUnitTests) set_target_properties(FuzzerUnitTests PROPERTIES FOLDER "Compiler-RT Tests") +add_custom_target(FuzzedDataProviderUnitTests) +set_target_properties(FuzzedDataProviderUnitTests PROPERTIES FOLDER "Compiler-RT Tests") + set(LIBFUZZER_UNITTEST_LINK_FLAGS ${COMPILER_RT_UNITTEST_LINK_FLAGS}) list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS --driver-mode=g++) -if(APPLE OR CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") - list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -lc++ -lpthread) +if(WIN32) + list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -Wl,-defaultlib:libcmt,-defaultlib:oldnames) else() - list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -lstdc++ -lpthread) + list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -lpthread) +endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND + COMPILER_RT_LIBCXX_PATH AND + COMPILER_RT_LIBCXXABI_PATH) + list(APPEND LIBFUZZER_UNITTEST_CFLAGS -nostdinc++) endif() -if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND COMPILER_RT_LIBCXX_PATH) - list(APPEND LIBFUZZER_UNITTEST_CFLAGS -nostdinc++ -D_LIBCPP_ABI_VERSION=Fuzzer) +if ("-fvisibility=hidden" IN_LIST LIBFUZZER_CFLAGS) + # Match visibility settings. + list(APPEND LIBFUZZER_UNITTEST_CFLAGS "-fvisibility=hidden") endif() if(COMPILER_RT_DEFAULT_TARGET_ARCH IN_LIST FUZZER_SUPPORTED_ARCH) @@ -43,9 +55,12 @@ if(COMPILER_RT_DEFAULT_TARGET_ARCH IN_LIST FUZZER_SUPPORTED_ARCH) ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} FOLDER "Compiler-RT Runtime tests") - if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND COMPILER_RT_LIBCXX_PATH) - set(LIBFUZZER_TEST_RUNTIME_DEPS libcxx_fuzzer_${arch}-build) - set(LIBFUZZER_TEST_RUNTIME_CFLAGS -isystem ${COMPILER_RT_LIBCXX_PATH}/include) + if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND + COMPILER_RT_LIBCXX_PATH AND + COMPILER_RT_LIBCXXABI_PATH) + file(GLOB libfuzzer_headers ../*.h) + set(LIBFUZZER_TEST_RUNTIME_DEPS libcxx_fuzzer_${arch}-build ${libfuzzer_headers}) + set(LIBFUZZER_TEST_RUNTIME_CFLAGS -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1) set(LIBFUZZER_TEST_RUNTIME_LINK_FLAGS ${LIBCXX_${arch}_PREFIX}/lib/libc++.a) endif() @@ -54,9 +69,21 @@ if(COMPILER_RT_DEFAULT_TARGET_ARCH IN_LIST FUZZER_SUPPORTED_ARCH) FuzzerUnitTests "Fuzzer-${arch}-Test" ${arch} SOURCES FuzzerUnittest.cpp ${COMPILER_RT_GTEST_SOURCE} RUNTIME ${LIBFUZZER_TEST_RUNTIME} - DEPS gtest ${LIBFUZZER_TEST_RUNTIME_DEPS} + DEPS gtest ${LIBFUZZER_TEST_RUNTIME_DEPS} CFLAGS ${LIBFUZZER_UNITTEST_CFLAGS} ${LIBFUZZER_TEST_RUNTIME_CFLAGS} LINK_FLAGS ${LIBFUZZER_UNITTEST_LINK_FLAGS} ${LIBFUZZER_TEST_RUNTIME_LINK_FLAGS}) set_target_properties(FuzzerUnitTests PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + + list(APPEND LIBFUZZER_UNITTEST_CFLAGS -I${COMPILER_RT_SOURCE_DIR}/lib/fuzzer/utils) + + set(FuzzedDataProviderTestObjects) + generate_compiler_rt_tests(FuzzedDataProviderTestObjects + FuzzedDataProviderUnitTests "FuzzerUtils-${arch}-Test" ${arch} + SOURCES FuzzedDataProviderUnittest.cpp ${COMPILER_RT_GTEST_SOURCE} + DEPS gtest ${LIBFUZZER_TEST_RUNTIME_DEPS} + CFLAGS ${LIBFUZZER_UNITTEST_CFLAGS} ${LIBFUZZER_TEST_RUNTIME_CFLAGS} + LINK_FLAGS ${LIBFUZZER_UNITTEST_LINK_FLAGS} ${LIBFUZZER_TEST_RUNTIME_LINK_FLAGS}) + set_target_properties(FuzzedDataProviderUnitTests PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) endif() diff --git a/libfuzzer/tests/FuzzedDataProviderUnittest.cpp b/libfuzzer/tests/FuzzedDataProviderUnittest.cpp new file mode 100644 index 0000000..e1b2722 --- /dev/null +++ b/libfuzzer/tests/FuzzedDataProviderUnittest.cpp @@ -0,0 +1,354 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "gtest/gtest.h" +#include +#include + +#include "FuzzedDataProvider.h" + +// The test is intentionally extensive, as behavior of |FuzzedDataProvider| must +// not be broken, given than many fuzz targets depend on it. Changing the +// behavior might invalidate existing corpora and make the fuzz targets using +// |FuzzedDataProvider| to lose code coverage accumulated over time. + +/* A random 1KB buffer generated by: +$ python -c "import os; print ',\n'.join([', '.join(['0x%02X' % ord(i) for i \ + in list(os.urandom(8))]) for _ in xrange(128)])" +*/ +const uint8_t Data[] = { + 0x8A, 0x19, 0x0D, 0x44, 0x37, 0x0D, 0x38, 0x5E, 0x9B, 0xAA, 0xF3, 0xDA, + 0xAA, 0x88, 0xF2, 0x9B, 0x6C, 0xBA, 0xBE, 0xB1, 0xF2, 0xCF, 0x13, 0xB8, + 0xAC, 0x1A, 0x7F, 0x1C, 0xC9, 0x90, 0xD0, 0xD9, 0x5C, 0x42, 0xB3, 0xFD, + 0xE3, 0x05, 0xA4, 0x03, 0x37, 0x49, 0x50, 0x4B, 0xBC, 0x39, 0xA2, 0x09, + 0x6C, 0x2F, 0xAF, 0xD1, 0xB5, 0x47, 0xBF, 0x92, 0xBD, 0x79, 0xE5, 0xC5, + 0x6E, 0x51, 0xA4, 0xED, 0xE9, 0xBD, 0x40, 0x4A, 0xFC, 0x25, 0x7A, 0x27, + 0xC8, 0x92, 0xF7, 0x30, 0xDE, 0x40, 0x66, 0x66, 0xE8, 0x5F, 0x65, 0x39, + 0x7E, 0x9E, 0x80, 0x2B, 0x01, 0x71, 0x2A, 0xFF, 0xD3, 0x0A, 0xAC, 0x6E, + 0x49, 0x32, 0x79, 0x10, 0x6A, 0x6F, 0x97, 0x96, 0x70, 0x7E, 0x50, 0x65, + 0xC9, 0x1D, 0xBD, 0x4E, 0x17, 0x04, 0x1E, 0xBA, 0x26, 0xAC, 0x1F, 0xE3, + 0x37, 0x1C, 0x15, 0x43, 0x60, 0x41, 0x2A, 0x7C, 0xCA, 0x70, 0xCE, 0xAB, + 0x20, 0x24, 0xF8, 0xD9, 0x1F, 0x14, 0x7C, 0x5C, 0xDD, 0x6F, 0xB3, 0xD7, + 0x8B, 0x63, 0x10, 0xB7, 0xDA, 0x99, 0xAF, 0x99, 0x01, 0x21, 0xE6, 0xE1, + 0x86, 0x27, 0xBE, 0x8D, 0xDF, 0x1E, 0xEA, 0x80, 0x0B, 0x8A, 0x60, 0xC3, + 0x3A, 0x85, 0x33, 0x53, 0x59, 0xE1, 0xB5, 0xF1, 0x62, 0xA6, 0x7B, 0x24, + 0x94, 0xE3, 0x8C, 0x10, 0x93, 0xF8, 0x6E, 0xC2, 0x00, 0x91, 0x90, 0x0B, + 0x5D, 0x52, 0x4F, 0x21, 0xE3, 0x40, 0x3A, 0x6E, 0xB6, 0x32, 0x15, 0xDB, + 0x5D, 0x01, 0x86, 0x63, 0x83, 0x24, 0xC5, 0xDE, 0xAB, 0x31, 0x84, 0xAA, + 0xE5, 0x64, 0x02, 0x8D, 0x23, 0x82, 0x86, 0x14, 0x16, 0x18, 0x9F, 0x3D, + 0x31, 0xBE, 0x3B, 0xF0, 0x6C, 0x26, 0x42, 0x9A, 0x67, 0xFE, 0x28, 0xEC, + 0x28, 0xDB, 0x01, 0xB4, 0x52, 0x41, 0x81, 0x7C, 0x54, 0xD3, 0xC8, 0x00, + 0x01, 0x66, 0xB0, 0x2C, 0x3F, 0xBC, 0xAF, 0xAC, 0x87, 0xCD, 0x83, 0xCF, + 0x23, 0xFC, 0xC8, 0x97, 0x8C, 0x71, 0x32, 0x8B, 0xBF, 0x70, 0xC0, 0x48, + 0x31, 0x92, 0x18, 0xFE, 0xE5, 0x33, 0x48, 0x82, 0x98, 0x1E, 0x30, 0xCC, + 0xAD, 0x5D, 0x97, 0xC4, 0xB4, 0x39, 0x7C, 0xCD, 0x39, 0x44, 0xF1, 0xA9, + 0xD0, 0xF4, 0x27, 0xB7, 0x78, 0x85, 0x9E, 0x72, 0xFC, 0xCC, 0xEE, 0x98, + 0x25, 0x3B, 0x69, 0x6B, 0x0C, 0x11, 0xEA, 0x22, 0xB6, 0xD0, 0xCD, 0xBF, + 0x6D, 0xBE, 0x12, 0xDE, 0xFE, 0x78, 0x2E, 0x54, 0xCB, 0xBA, 0xD7, 0x2E, + 0x54, 0x25, 0x14, 0x84, 0xFE, 0x1A, 0x10, 0xCE, 0xCC, 0x20, 0xE6, 0xE2, + 0x7F, 0xE0, 0x5F, 0xDB, 0xA7, 0xF3, 0xE2, 0x4C, 0x52, 0x82, 0xFC, 0x0B, + 0xA0, 0xBD, 0x34, 0x21, 0xF7, 0xEB, 0x1C, 0x5B, 0x67, 0xD0, 0xAF, 0x22, + 0x15, 0xA1, 0xFF, 0xC2, 0x68, 0x25, 0x5B, 0xB2, 0x13, 0x3F, 0xFF, 0x98, + 0x53, 0x25, 0xC5, 0x58, 0x39, 0xD0, 0x43, 0x86, 0x6C, 0x5B, 0x57, 0x8E, + 0x83, 0xBA, 0xB9, 0x09, 0x09, 0x14, 0x0C, 0x9E, 0x99, 0x83, 0x88, 0x53, + 0x79, 0xFD, 0xF7, 0x49, 0xE9, 0x2C, 0xCE, 0xE6, 0x7B, 0xF5, 0xC2, 0x27, + 0x5E, 0x56, 0xB5, 0xB4, 0x46, 0x90, 0x91, 0x7F, 0x99, 0x88, 0xA7, 0x23, + 0xC1, 0x80, 0xB8, 0x2D, 0xCD, 0xF7, 0x6F, 0x9A, 0xEC, 0xBD, 0x16, 0x9F, + 0x7D, 0x87, 0x1E, 0x15, 0x51, 0xC4, 0x96, 0xE2, 0xBF, 0x61, 0x66, 0xB5, + 0xFD, 0x01, 0x67, 0xD6, 0xFF, 0xD2, 0x14, 0x20, 0x98, 0x8E, 0xEF, 0xF3, + 0x22, 0xDB, 0x7E, 0xCE, 0x70, 0x2D, 0x4C, 0x06, 0x5A, 0xA0, 0x4F, 0xC8, + 0xB0, 0x4D, 0xA6, 0x52, 0xB2, 0xD6, 0x2F, 0xD8, 0x57, 0xE5, 0xEF, 0xF9, + 0xEE, 0x52, 0x0F, 0xEC, 0xC4, 0x90, 0x33, 0xAD, 0x25, 0xDA, 0xCD, 0x12, + 0x44, 0x5F, 0x32, 0xF6, 0x6F, 0xEF, 0x85, 0xB8, 0xDC, 0x3C, 0x01, 0x48, + 0x28, 0x5D, 0x2D, 0x9C, 0x9B, 0xC0, 0x49, 0x36, 0x1E, 0x6A, 0x0A, 0x0C, + 0xB0, 0x6E, 0x81, 0x89, 0xCB, 0x0A, 0x89, 0xCF, 0x73, 0xC6, 0x63, 0x3D, + 0x8E, 0x13, 0x57, 0x91, 0x4E, 0xA3, 0x93, 0x8C, 0x61, 0x67, 0xFD, 0x13, + 0xE0, 0x14, 0x72, 0xB3, 0xE4, 0x23, 0x45, 0x08, 0x4E, 0x4E, 0xF5, 0xA7, + 0xA8, 0xEE, 0x30, 0xFD, 0x81, 0x80, 0x1F, 0xF3, 0x4F, 0xD7, 0xE7, 0xF2, + 0x16, 0xC0, 0xD6, 0x15, 0x6A, 0x0F, 0x89, 0x15, 0xA9, 0xCF, 0x35, 0x50, + 0x6B, 0x49, 0x3E, 0x12, 0x4A, 0x72, 0xE4, 0x59, 0x9D, 0xD7, 0xDB, 0xD2, + 0xD1, 0x61, 0x7D, 0x52, 0x4A, 0x36, 0xF6, 0xBA, 0x0E, 0xFA, 0x88, 0x6F, + 0x3C, 0x82, 0x16, 0xF0, 0xD5, 0xED, 0x4D, 0x78, 0xEF, 0x38, 0x17, 0x90, + 0xEA, 0x28, 0x32, 0xA9, 0x79, 0x40, 0xFF, 0xAA, 0xE6, 0xF5, 0xC7, 0x96, + 0x56, 0x65, 0x61, 0x83, 0x3D, 0xBD, 0xD7, 0xED, 0xD6, 0xB6, 0xC0, 0xED, + 0x34, 0xAA, 0x60, 0xA9, 0xE8, 0x82, 0x78, 0xEA, 0x69, 0xF6, 0x47, 0xAF, + 0x39, 0xAB, 0x11, 0xDB, 0xE9, 0xFB, 0x68, 0x0C, 0xFE, 0xDF, 0x97, 0x9F, + 0x3A, 0xF4, 0xF3, 0x32, 0x27, 0x30, 0x57, 0x0E, 0xF7, 0xB2, 0xEE, 0xFB, + 0x1E, 0x98, 0xA8, 0xA3, 0x25, 0x45, 0xE4, 0x6D, 0x2D, 0xAE, 0xFE, 0xDA, + 0xB3, 0x32, 0x9B, 0x5D, 0xF5, 0x32, 0x74, 0xEA, 0xE5, 0x02, 0x30, 0x53, + 0x95, 0x13, 0x7A, 0x23, 0x1F, 0x10, 0x30, 0xEA, 0x78, 0xE4, 0x36, 0x1D, + 0x92, 0x96, 0xB9, 0x91, 0x2D, 0xFA, 0x43, 0xAB, 0xE6, 0xEF, 0x14, 0x14, + 0xC9, 0xBC, 0x46, 0xC6, 0x05, 0x7C, 0xC6, 0x11, 0x23, 0xCF, 0x3D, 0xC8, + 0xBE, 0xEC, 0xA3, 0x58, 0x31, 0x55, 0x65, 0x14, 0xA7, 0x94, 0x93, 0xDD, + 0x2D, 0x76, 0xC9, 0x66, 0x06, 0xBD, 0xF5, 0xE7, 0x30, 0x65, 0x42, 0x52, + 0xA2, 0x50, 0x9B, 0xE6, 0x40, 0xA2, 0x4B, 0xEC, 0xA6, 0xB7, 0x39, 0xAA, + 0xD7, 0x61, 0x2C, 0xBF, 0x37, 0x5A, 0xDA, 0xB3, 0x5D, 0x2F, 0x5D, 0x11, + 0x82, 0x97, 0x32, 0x8A, 0xC1, 0xA1, 0x13, 0x20, 0x17, 0xBD, 0xA2, 0x91, + 0x94, 0x2A, 0x4E, 0xBE, 0x3E, 0x77, 0x63, 0x67, 0x5C, 0x0A, 0xE1, 0x22, + 0x0A, 0x4F, 0x63, 0xE2, 0x84, 0xE9, 0x9F, 0x14, 0x86, 0xE2, 0x4B, 0x20, + 0x9F, 0x50, 0xB3, 0x56, 0xED, 0xDE, 0x39, 0xD8, 0x75, 0x64, 0x45, 0x54, + 0xE5, 0x34, 0x57, 0x8C, 0x3B, 0xF2, 0x0E, 0x94, 0x1B, 0x10, 0xA2, 0xA2, + 0x38, 0x76, 0x21, 0x8E, 0x2A, 0x57, 0x64, 0x58, 0x0A, 0x27, 0x6D, 0x4C, + 0xD0, 0xB5, 0xC1, 0xFC, 0x75, 0xD0, 0x01, 0x86, 0x66, 0xA8, 0xF1, 0x98, + 0x58, 0xFB, 0xFC, 0x64, 0xD2, 0x31, 0x77, 0xAD, 0x0E, 0x46, 0x87, 0xCC, + 0x9B, 0x86, 0x90, 0xFF, 0xB6, 0x64, 0x35, 0xA5, 0x5D, 0x9E, 0x44, 0x51, + 0x87, 0x9E, 0x1E, 0xEE, 0xF3, 0x3B, 0x5C, 0xDD, 0x94, 0x03, 0xAA, 0x18, + 0x2C, 0xB7, 0xC4, 0x37, 0xD5, 0x53, 0x28, 0x60, 0xEF, 0x77, 0xEF, 0x3B, + 0x9E, 0xD2, 0xCE, 0xE9, 0x53, 0x2D, 0xF5, 0x19, 0x7E, 0xBB, 0xB5, 0x46, + 0xE2, 0xF7, 0xD6, 0x4D, 0x6D, 0x5B, 0x81, 0x56, 0x6B, 0x12, 0x55, 0x63, + 0xC3, 0xAB, 0x08, 0xBB, 0x2E, 0xD5, 0x11, 0xBC, 0x18, 0xCB, 0x8B, 0x12, + 0x2E, 0x3E, 0x75, 0x32, 0x98, 0x8A, 0xDE, 0x3C, 0xEA, 0x33, 0x46, 0xE7, + 0x7A, 0xA5, 0x12, 0x09, 0x26, 0x7E, 0x7E, 0x03, 0x4F, 0xFD, 0xC0, 0xFD, + 0xEA, 0x4F, 0x83, 0x85, 0x39, 0x62, 0xFB, 0xA2, 0x33, 0xD9, 0x2D, 0xB1, + 0x30, 0x6F, 0x88, 0xAB, 0x61, 0xCB, 0x32, 0xEB, 0x30, 0xF9, 0x51, 0xF6, + 0x1F, 0x3A, 0x11, 0x4D, 0xFD, 0x54, 0xD6, 0x3D, 0x43, 0x73, 0x39, 0x16, + 0xCF, 0x3D, 0x29, 0x4A}; + +TEST(FuzzedDataProvider, ConsumeBytes) { + FuzzedDataProvider DataProv(Data, sizeof(Data)); + EXPECT_EQ(std::vector(1, 0x8A), + DataProv.ConsumeBytes(1)); + EXPECT_EQ(std::vector( + {0x19, 0x0D, 0x44, 0x37, 0x0D, 0x38, 0x5E, 0x9B, 0xAA, 0xF3}), + DataProv.ConsumeBytes(10)); + + std::vector UChars = DataProv.ConsumeBytes(24); + EXPECT_EQ(std::vector({0xDA, 0xAA, 0x88, 0xF2, 0x9B, 0x6C, + 0xBA, 0xBE, 0xB1, 0xF2, 0xCF, 0x13, + 0xB8, 0xAC, 0x1A, 0x7F, 0x1C, 0xC9, + 0x90, 0xD0, 0xD9, 0x5C, 0x42, 0xB3}), + UChars); + + EXPECT_EQ(std::vector(Data + 1 + 10 + 24, Data + sizeof(Data)), + DataProv.ConsumeBytes(31337)); +} + +TEST(FuzzedDataProvider, ConsumeBytesWithTerminator) { + FuzzedDataProvider DataProv(Data, sizeof(Data)); + EXPECT_EQ(std::vector({0x8A, 0x00}), + DataProv.ConsumeBytesWithTerminator(1)); + EXPECT_EQ(std::vector({0x19, 0x0D, 0x44, 0x37, 0x0D, 0x38, 0x5E, + 0x9B, 0xAA, 0xF3, 111}), + DataProv.ConsumeBytesWithTerminator(10, 111)); + + std::vector UChars = + DataProv.ConsumeBytesWithTerminator(24); + EXPECT_EQ(std::vector( + {0xDA, 0xAA, 0x88, 0xF2, 0x9B, 0x6C, 0xBA, 0xBE, 0xB1, + 0xF2, 0xCF, 0x13, 0xB8, 0xAC, 0x1A, 0x7F, 0x1C, 0xC9, + 0x90, 0xD0, 0xD9, 0x5C, 0x42, 0xB3, 0x00}), + UChars); + + std::vector Expected(Data + 1 + 10 + 24, Data + sizeof(Data)); + Expected.push_back(65); + EXPECT_EQ(Expected, + DataProv.ConsumeBytesWithTerminator(31337, 65)); +} + +TEST(FuzzedDataProvider, ConsumeBytesAsString) { + FuzzedDataProvider DataProv(Data, sizeof(Data)); + EXPECT_EQ(std::string("\x8A\x19\x0D\x44\x37\x0D\x38\x5E\x9B\xAA\xF3\xDA"), + DataProv.ConsumeBytesAsString(12)); + EXPECT_EQ(std::string(Data + 12, Data + sizeof(Data)), + DataProv.ConsumeBytesAsString(31337)); +} + +TEST(FuzzedDataProvider, ConsumeIntegralInRange) { + FuzzedDataProvider DataProv(Data, sizeof(Data)); + EXPECT_EQ(int32_t(21), DataProv.ConsumeIntegralInRange(10, 30)); + EXPECT_EQ(int32_t(1337), + DataProv.ConsumeIntegralInRange(1337, 1337)); + EXPECT_EQ(int8_t(-59), DataProv.ConsumeIntegralInRange(-100, 100)); + EXPECT_EQ(uint16_t(15823), + DataProv.ConsumeIntegralInRange(0, 65535)); + EXPECT_EQ((signed char)(-101), + DataProv.ConsumeIntegralInRange(-123, 123)); + EXPECT_EQ(int64_t(-53253077544), DataProv.ConsumeIntegralInRange( + -99999999999, 99999999999)); + + // Exhaust the buffer. + auto String = DataProv.ConsumeBytesAsString(31337); + EXPECT_EQ(size_t(1014), String.length()); + EXPECT_EQ(uint64_t(123456789), + DataProv.ConsumeIntegralInRange(123456789, 987654321)); +} + +TEST(FuzzedDataProvider, ConsumeRandomLengthString) { + FuzzedDataProvider DataProv(Data, sizeof(Data)); + EXPECT_EQ( + std::string( + "\x8A\x19\x0D\x44\x37\x0D\x38\x5E\x9B\xAA\xF3\xDA\xAA\x88\xF2\x9B\x6C" + "\xBA\xBE\xB1\xF2\xCF\x13\xB8\xAC\x1A\x7F\x1C\xC9\x90\xD0\xD9"), + DataProv.ConsumeRandomLengthString(1337)); + EXPECT_EQ(std::string( + "\xB3\xFD\xE3\x05\xA4\x03\x37\x49\x50\x4B\xBC\x39\xA2\x09\x6C" + "\x2F\xAF\xD1\xB5\x47\xBF\x92\xBD\x79\xE5\xC5\x6E\x51\xA4\xED" + "\xE9\xBD\x40\x4A\xFC\x25\x7A\x27\xC8\x92\xF7\x30\xDE\x40\x66" + "\x66\xE8\x5F\x65\x39\x7E\x9E\x80\x2B\x01\x71\x2A\xFF\xD3\x0A" + "\xAC\x6E\x49\x32\x79\x10\x6A\x6F\x97\x96\x70\x7E\x50\x65\xC9" + "\x1D\xBD\x4E\x17\x04\x1E\xBA\x26\xAC\x1F\xE3\x37\x1C\x15\x43" + "\x60\x41\x2A\x7C\xCA\x70\xCE\xAB\x20\x24\xF8\xD9\x1F\x14\x7C"), + DataProv.ConsumeRandomLengthString(31337)); + EXPECT_EQ(std::string(Data + 141, Data + 141 + 5), + DataProv.ConsumeRandomLengthString(5)); + EXPECT_EQ(std::string(Data + 141 + 5, Data + 141 + 5 + 2), + DataProv.ConsumeRandomLengthString(2)); + + // Exhaust the buffer. + auto String = DataProv.ConsumeBytesAsString(31337); + EXPECT_EQ(size_t(876), String.length()); + EXPECT_EQ(std::string(), DataProv.ConsumeRandomLengthString(1)); +} + +TEST(FuzzedDataProvider, ConsumeRemainingBytes) { + { + FuzzedDataProvider DataProv(Data, sizeof(Data)); + EXPECT_EQ(std::vector(Data, Data + sizeof(Data)), + DataProv.ConsumeRemainingBytes()); + EXPECT_EQ(std::vector(), + DataProv.ConsumeRemainingBytes()); + } + + { + FuzzedDataProvider DataProv(Data, sizeof(Data)); + EXPECT_EQ(std::vector(Data, Data + 123), + DataProv.ConsumeBytes(123)); + EXPECT_EQ(std::vector(Data + 123, Data + sizeof(Data)), + DataProv.ConsumeRemainingBytes()); + } +} + +TEST(FuzzedDataProvider, ConsumeRemainingBytesAsString) { + { + FuzzedDataProvider DataProv(Data, sizeof(Data)); + EXPECT_EQ(std::string(Data, Data + sizeof(Data)), + DataProv.ConsumeRemainingBytesAsString()); + EXPECT_EQ(std::string(""), DataProv.ConsumeRemainingBytesAsString()); + } + + { + FuzzedDataProvider DataProv(Data, sizeof(Data)); + EXPECT_EQ(std::vector(Data, Data + 123), + DataProv.ConsumeBytes(123)); + EXPECT_EQ(std::string(Data + 123, Data + sizeof(Data)), + DataProv.ConsumeRemainingBytesAsString()); + } +} + +TEST(FuzzedDataProvider, ConsumeIntegral) { + FuzzedDataProvider DataProv(Data, sizeof(Data)); + EXPECT_EQ(int32_t(-903266865), DataProv.ConsumeIntegral()); + EXPECT_EQ(uint32_t(372863811), DataProv.ConsumeIntegral()); + EXPECT_EQ(uint8_t(61), DataProv.ConsumeIntegral()); + EXPECT_EQ(int16_t(22100), DataProv.ConsumeIntegral()); + EXPECT_EQ(uint64_t(18252263806144500217u), + DataProv.ConsumeIntegral()); + + // Exhaust the buffer. + auto String = DataProv.ConsumeBytesAsString(31337); + EXPECT_EQ(size_t(1005), String.length()); + EXPECT_EQ(std::numeric_limits::min(), + DataProv.ConsumeIntegral()); +} + +TEST(FuzzedDataProvider, ConsumeBool) { + FuzzedDataProvider DataProv(Data, sizeof(Data)); + EXPECT_EQ(false, DataProv.ConsumeBool()); + EXPECT_EQ(true, DataProv.ConsumeBool()); + EXPECT_EQ(true, DataProv.ConsumeBool()); + EXPECT_EQ(true, DataProv.ConsumeBool()); + EXPECT_EQ(false, DataProv.ConsumeBool()); + EXPECT_EQ(true, DataProv.ConsumeBool()); + EXPECT_EQ(true, DataProv.ConsumeBool()); + EXPECT_EQ(true, DataProv.ConsumeBool()); + EXPECT_EQ(true, DataProv.ConsumeBool()); + EXPECT_EQ(false, DataProv.ConsumeBool()); + + // Exhaust the buffer. + auto String = DataProv.ConsumeBytesAsString(31337); + EXPECT_EQ(size_t(1014), String.length()); + EXPECT_EQ(false, DataProv.ConsumeBool()); +} + +TEST(FuzzedDataProvider, PickValueInArray) { + FuzzedDataProvider DataProv(Data, sizeof(Data)); + const int Array[] = {1, 2, 3, 4, 5}; + EXPECT_EQ(5, DataProv.PickValueInArray(Array)); + EXPECT_EQ(2, DataProv.PickValueInArray(Array)); + EXPECT_EQ(2, DataProv.PickValueInArray(Array)); + EXPECT_EQ(3, DataProv.PickValueInArray(Array)); + EXPECT_EQ(3, DataProv.PickValueInArray(Array)); + EXPECT_EQ(3, DataProv.PickValueInArray(Array)); + EXPECT_EQ(1, DataProv.PickValueInArray(Array)); + EXPECT_EQ(3, DataProv.PickValueInArray(Array)); + EXPECT_EQ(2, DataProv.PickValueInArray(Array)); + + EXPECT_EQ(uint8_t(0x9D), DataProv.PickValueInArray(Data)); + EXPECT_EQ(uint8_t(0xBA), DataProv.PickValueInArray(Data)); + EXPECT_EQ(uint8_t(0x69), DataProv.PickValueInArray(Data)); + EXPECT_EQ(uint8_t(0xD6), DataProv.PickValueInArray(Data)); + + EXPECT_EQ(uint32_t(777), DataProv.PickValueInArray({1337, 777})); + EXPECT_EQ(uint32_t(777), DataProv.PickValueInArray({1337, 777})); + EXPECT_EQ(uint64_t(1337), DataProv.PickValueInArray({1337, 777})); + EXPECT_EQ(size_t(777), DataProv.PickValueInArray({1337, 777})); + EXPECT_EQ(int16_t(1337), DataProv.PickValueInArray({1337, 777})); + EXPECT_EQ(int32_t(777), DataProv.PickValueInArray({1337, 777})); + EXPECT_EQ(int64_t(777), DataProv.PickValueInArray({1337, 777})); + + // Exhaust the buffer. + auto String = DataProv.ConsumeBytesAsString(31337); + EXPECT_EQ(size_t(1000), String.length()); + EXPECT_EQ(uint8_t(0x8A), DataProv.PickValueInArray(Data)); +} + +TEST(FuzzedDataProvider, ConsumeEnum) { + FuzzedDataProvider DataProv(Data, sizeof(Data)); + enum class Enum { + Zero, + One, + Two, + Three, + Four, + Five, + Six, + Seven, + kMaxValue = Seven + }; + EXPECT_EQ(Enum::Two, DataProv.ConsumeEnum()); + EXPECT_EQ(Enum::One, DataProv.ConsumeEnum()); + EXPECT_EQ(Enum::Five, DataProv.ConsumeEnum()); + EXPECT_EQ(Enum::Seven, DataProv.ConsumeEnum()); + EXPECT_EQ(Enum::Six, DataProv.ConsumeEnum()); + EXPECT_EQ(Enum::One, DataProv.ConsumeEnum()); + EXPECT_EQ(Enum::Three, DataProv.ConsumeEnum()); + EXPECT_EQ(Enum::Three, DataProv.ConsumeEnum()); + EXPECT_EQ(Enum::Five, DataProv.ConsumeEnum()); + EXPECT_EQ(Enum::Six, DataProv.ConsumeEnum()); + + // Exhaust the buffer. + auto String = DataProv.ConsumeBytesAsString(31337); + EXPECT_EQ(size_t(1014), String.length()); + EXPECT_EQ(Enum::Zero, DataProv.ConsumeEnum()); +} + +TEST(FuzzedDataProvider, remaining_bytes) { + FuzzedDataProvider DataProv(Data, sizeof(Data)); + EXPECT_EQ(size_t(1024), DataProv.remaining_bytes()); + EXPECT_EQ(false, DataProv.ConsumeBool()); + EXPECT_EQ(size_t(1024 - 1), DataProv.remaining_bytes()); + EXPECT_EQ(std::vector(Data, Data + 8), + DataProv.ConsumeBytes(8)); + EXPECT_EQ(size_t(1024 - 1 - 8), DataProv.remaining_bytes()); + + // Exhaust the buffer. + EXPECT_EQ(std::vector(Data + 8, Data + sizeof(Data) - 1), + DataProv.ConsumeRemainingBytes()); + EXPECT_EQ(size_t(0), DataProv.remaining_bytes()); +} + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/libfuzzer/tests/FuzzerUnittest.cpp b/libfuzzer/tests/FuzzerUnittest.cpp index e3b0670..abb203d 100644 --- a/libfuzzer/tests/FuzzerUnittest.cpp +++ b/libfuzzer/tests/FuzzerUnittest.cpp @@ -1,5 +1,6 @@ -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // Avoid ODR violations (LibFuzzer is built without ASan and this test is built // with ASan) involving C++ standard library types when using libcxx. @@ -34,6 +35,13 @@ TEST(Fuzzer, Basename) { EXPECT_EQ(Basename("/bar"), "bar"); EXPECT_EQ(Basename("foo/x"), "x"); EXPECT_EQ(Basename("foo/"), ""); +#if LIBFUZZER_WINDOWS + EXPECT_EQ(Basename("foo\\bar"), "bar"); + EXPECT_EQ(Basename("foo\\bar/baz"), "baz"); + EXPECT_EQ(Basename("\\bar"), "bar"); + EXPECT_EQ(Basename("foo\\x"), "x"); + EXPECT_EQ(Basename("foo\\"), ""); +#endif } TEST(Fuzzer, CrossOver) { @@ -611,7 +619,7 @@ TEST(Merge, Bad) { "2\n2\nA\n", "2\n2\nA\nB\nC\n", "0\n0\n", - "1\n1\nA\nDONE 0", + "1\n1\nA\nFT 0", "1\n1\nA\nSTARTED 1", }; Merger M; @@ -636,11 +644,9 @@ static void Merge(const std::string &Input, size_t NumNewFeatures) { Merger M; Vector NewFiles; + Set NewFeatures, NewCov; EXPECT_TRUE(M.Parse(Input, true)); - std::stringstream SS; - M.PrintSummary(SS); - EXPECT_EQ(NumNewFeatures, M.Merge(&NewFiles)); - EXPECT_EQ(M.AllFeatures(), M.ParseSummary(SS)); + EXPECT_EQ(NumNewFeatures, M.Merge({}, &NewFeatures, {}, &NewCov, &NewFiles)); EQ(NewFiles, Result); } @@ -664,9 +670,9 @@ TEST(Merge, Good) { EXPECT_TRUE(M.Parse("3\n1\nAA\nBB\nC\n" "STARTED 0 1000\n" - "DONE 0 1 2 3\n" + "FT 0 1 2 3\n" "STARTED 1 1001\n" - "DONE 1 4 5 6 \n" + "FT 1 4 5 6 \n" "STARTED 2 1002\n" "", true)); EXPECT_EQ(M.Files.size(), 3U); @@ -684,11 +690,12 @@ TEST(Merge, Good) { Vector NewFiles; + Set NewFeatures, NewCov; EXPECT_TRUE(M.Parse("3\n2\nAA\nBB\nC\n" - "STARTED 0 1000\nDONE 0 1 2 3\n" - "STARTED 1 1001\nDONE 1 4 5 6 \n" - "STARTED 2 1002\nDONE 2 6 1 3 \n" + "STARTED 0 1000\nFT 0 1 2 3\n" + "STARTED 1 1001\nFT 1 4 5 6 \n" + "STARTED 2 1002\nFT 2 6 1 3 \n" "", true)); EXPECT_EQ(M.Files.size(), 3U); EXPECT_EQ(M.NumFilesInFirstCorpus, 2U); @@ -697,24 +704,24 @@ TEST(Merge, Good) { EQ(M.Files[0].Features, {1, 2, 3}); EQ(M.Files[1].Features, {4, 5, 6}); EQ(M.Files[2].Features, {1, 3, 6}); - EXPECT_EQ(0U, M.Merge(&NewFiles)); + EXPECT_EQ(0U, M.Merge({}, &NewFeatures, {}, &NewCov, &NewFiles)); EQ(NewFiles, {}); EXPECT_TRUE(M.Parse("3\n1\nA\nB\nC\n" - "STARTED 0 1000\nDONE 0 1 2 3\n" - "STARTED 1 1001\nDONE 1 4 5 6 \n" - "STARTED 2 1002\nDONE 2 6 1 3\n" + "STARTED 0 1000\nFT 0 1 2 3\n" + "STARTED 1 1001\nFT 1 4 5 6 \n" + "STARTED 2 1002\nFT 2 6 1 3\n" "", true)); EQ(M.Files[0].Features, {1, 2, 3}); EQ(M.Files[1].Features, {4, 5, 6}); EQ(M.Files[2].Features, {1, 3, 6}); - EXPECT_EQ(3U, M.Merge(&NewFiles)); + EXPECT_EQ(3U, M.Merge({}, &NewFeatures, {}, &NewCov, &NewFiles)); EQ(NewFiles, {"B"}); // Same as the above, but with InitialFeatures. EXPECT_TRUE(M.Parse("2\n0\nB\nC\n" - "STARTED 0 1001\nDONE 0 4 5 6 \n" - "STARTED 1 1002\nDONE 1 6 1 3\n" + "STARTED 0 1001\nFT 0 4 5 6 \n" + "STARTED 1 1002\nFT 1 6 1 3\n" "", true)); EQ(M.Files[0].Features, {4, 5, 6}); EQ(M.Files[1].Features, {1, 3, 6}); @@ -722,39 +729,131 @@ TEST(Merge, Good) { InitialFeatures.insert(1); InitialFeatures.insert(2); InitialFeatures.insert(3); - EXPECT_EQ(3U, M.Merge(InitialFeatures, &NewFiles)); + EXPECT_EQ(3U, M.Merge(InitialFeatures, &NewFeatures, {}, &NewCov, &NewFiles)); EQ(NewFiles, {"B"}); } TEST(Merge, Merge) { Merge("3\n1\nA\nB\nC\n" - "STARTED 0 1000\nDONE 0 1 2 3\n" - "STARTED 1 1001\nDONE 1 4 5 6 \n" - "STARTED 2 1002\nDONE 2 6 1 3 \n", + "STARTED 0 1000\nFT 0 1 2 3\n" + "STARTED 1 1001\nFT 1 4 5 6 \n" + "STARTED 2 1002\nFT 2 6 1 3 \n", {"B"}, 3); Merge("3\n0\nA\nB\nC\n" - "STARTED 0 2000\nDONE 0 1 2 3\n" - "STARTED 1 1001\nDONE 1 4 5 6 \n" - "STARTED 2 1002\nDONE 2 6 1 3 \n", + "STARTED 0 2000\nFT 0 1 2 3\n" + "STARTED 1 1001\nFT 1 4 5 6 \n" + "STARTED 2 1002\nFT 2 6 1 3 \n", {"A", "B", "C"}, 6); Merge("4\n0\nA\nB\nC\nD\n" - "STARTED 0 2000\nDONE 0 1 2 3\n" - "STARTED 1 1101\nDONE 1 4 5 6 \n" - "STARTED 2 1102\nDONE 2 6 1 3 100 \n" - "STARTED 3 1000\nDONE 3 1 \n", + "STARTED 0 2000\nFT 0 1 2 3\n" + "STARTED 1 1101\nFT 1 4 5 6 \n" + "STARTED 2 1102\nFT 2 6 1 3 100 \n" + "STARTED 3 1000\nFT 3 1 \n", {"A", "B", "C", "D"}, 7); Merge("4\n1\nA\nB\nC\nD\n" - "STARTED 0 2000\nDONE 0 4 5 6 7 8\n" - "STARTED 1 1100\nDONE 1 1 2 3 \n" - "STARTED 2 1100\nDONE 2 2 3 \n" - "STARTED 3 1000\nDONE 3 1 \n", + "STARTED 0 2000\nFT 0 4 5 6 7 8\n" + "STARTED 1 1100\nFT 1 1 2 3 \n" + "STARTED 2 1100\nFT 2 2 3 \n" + "STARTED 3 1000\nFT 3 1 \n", {"B", "D"}, 3); } +TEST(DFT, BlockCoverage) { + BlockCoverage Cov; + // Assuming C0 has 5 instrumented blocks, + // C1: 7 blocks, C2: 4, C3: 9, C4 never covered, C5: 15, + + // Add C0 + EXPECT_TRUE(Cov.AppendCoverage("C0 5\n")); + EXPECT_EQ(Cov.GetCounter(0, 0), 1U); + EXPECT_EQ(Cov.GetCounter(0, 1), 0U); // not seen this BB yet. + EXPECT_EQ(Cov.GetCounter(0, 5), 0U); // BB ID out of bounds. + EXPECT_EQ(Cov.GetCounter(1, 0), 0U); // not seen this function yet. + + EXPECT_EQ(Cov.GetNumberOfBlocks(0), 5U); + EXPECT_EQ(Cov.GetNumberOfCoveredBlocks(0), 1U); + EXPECT_EQ(Cov.GetNumberOfBlocks(1), 0U); + + // Various errors. + EXPECT_FALSE(Cov.AppendCoverage("C0\n")); // No total number. + EXPECT_FALSE(Cov.AppendCoverage("C0 7\n")); // No total number. + EXPECT_FALSE(Cov.AppendCoverage("CZ\n")); // Wrong function number. + EXPECT_FALSE(Cov.AppendCoverage("C1 7 7")); // BB ID is too big. + EXPECT_FALSE(Cov.AppendCoverage("C1 100 7")); // BB ID is too big. + + // Add C0 more times. + EXPECT_TRUE(Cov.AppendCoverage("C0 5\n")); + EXPECT_EQ(Cov.GetCounter(0, 0), 2U); + EXPECT_TRUE(Cov.AppendCoverage("C0 1 2 5\n")); + EXPECT_EQ(Cov.GetCounter(0, 0), 3U); + EXPECT_EQ(Cov.GetCounter(0, 1), 1U); + EXPECT_EQ(Cov.GetCounter(0, 2), 1U); + EXPECT_EQ(Cov.GetCounter(0, 3), 0U); + EXPECT_EQ(Cov.GetCounter(0, 4), 0U); + EXPECT_EQ(Cov.GetNumberOfCoveredBlocks(0), 3U); + EXPECT_TRUE(Cov.AppendCoverage("C0 1 3 4 5\n")); + EXPECT_EQ(Cov.GetCounter(0, 0), 4U); + EXPECT_EQ(Cov.GetCounter(0, 1), 2U); + EXPECT_EQ(Cov.GetCounter(0, 2), 1U); + EXPECT_EQ(Cov.GetCounter(0, 3), 1U); + EXPECT_EQ(Cov.GetCounter(0, 4), 1U); + EXPECT_EQ(Cov.GetNumberOfCoveredBlocks(0), 5U); + + EXPECT_TRUE(Cov.AppendCoverage("C1 7\nC2 4\nC3 9\nC5 15\nC0 5\n")); + EXPECT_EQ(Cov.GetCounter(0, 0), 5U); + EXPECT_EQ(Cov.GetCounter(1, 0), 1U); + EXPECT_EQ(Cov.GetCounter(2, 0), 1U); + EXPECT_EQ(Cov.GetCounter(3, 0), 1U); + EXPECT_EQ(Cov.GetCounter(4, 0), 0U); + EXPECT_EQ(Cov.GetCounter(5, 0), 1U); + + EXPECT_TRUE(Cov.AppendCoverage("C3 4 5 9\nC5 11 12 15")); + EXPECT_EQ(Cov.GetCounter(0, 0), 5U); + EXPECT_EQ(Cov.GetCounter(1, 0), 1U); + EXPECT_EQ(Cov.GetCounter(2, 0), 1U); + EXPECT_EQ(Cov.GetCounter(3, 0), 2U); + EXPECT_EQ(Cov.GetCounter(3, 4), 1U); + EXPECT_EQ(Cov.GetCounter(3, 5), 1U); + EXPECT_EQ(Cov.GetCounter(3, 6), 0U); + EXPECT_EQ(Cov.GetCounter(4, 0), 0U); + EXPECT_EQ(Cov.GetCounter(5, 0), 2U); + EXPECT_EQ(Cov.GetCounter(5, 10), 0U); + EXPECT_EQ(Cov.GetCounter(5, 11), 1U); + EXPECT_EQ(Cov.GetCounter(5, 12), 1U); +} + +TEST(DFT, FunctionWeights) { + BlockCoverage Cov; + // unused function gets zero weight. + EXPECT_TRUE(Cov.AppendCoverage("C0 5\n")); + auto Weights = Cov.FunctionWeights(2); + EXPECT_GT(Weights[0], 0.); + EXPECT_EQ(Weights[1], 0.); + + // Less frequently used function gets less weight. + Cov.clear(); + EXPECT_TRUE(Cov.AppendCoverage("C0 5\nC1 5\nC1 5\n")); + Weights = Cov.FunctionWeights(2); + EXPECT_GT(Weights[0], Weights[1]); + + // A function with more uncovered blocks gets more weight. + Cov.clear(); + EXPECT_TRUE(Cov.AppendCoverage("C0 1 2 3 5\nC1 2 4\n")); + Weights = Cov.FunctionWeights(2); + EXPECT_GT(Weights[1], Weights[0]); + + // A function with DFT gets more weight than the function w/o DFT. + Cov.clear(); + EXPECT_TRUE(Cov.AppendCoverage("F1 111\nC0 3\nC1 1 2 3\n")); + Weights = Cov.FunctionWeights(2); + EXPECT_GT(Weights[1], Weights[0]); +} + + TEST(Fuzzer, ForEachNonZeroByte) { const size_t N = 64; alignas(64) uint8_t Ar[N + 8] = { diff --git a/libfuzzer/utils/FuzzedDataProvider.h b/libfuzzer/utils/FuzzedDataProvider.h new file mode 100644 index 0000000..1b5b4bb --- /dev/null +++ b/libfuzzer/utils/FuzzedDataProvider.h @@ -0,0 +1,245 @@ +//===- FuzzedDataProvider.h - Utility header for fuzz targets ---*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// A single header library providing an utility class to break up an array of +// bytes. Whenever run on the same input, provides the same output, as long as +// its methods are called in the same order, with the same arguments. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_ +#define LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +class FuzzedDataProvider { +public: + // |data| is an array of length |size| that the FuzzedDataProvider wraps to + // provide more granular access. |data| must outlive the FuzzedDataProvider. + FuzzedDataProvider(const uint8_t *data, size_t size) + : data_ptr_(data), remaining_bytes_(size) {} + ~FuzzedDataProvider() = default; + + // Returns a std::vector containing |num_bytes| of input data. If fewer than + // |num_bytes| of data remain, returns a shorter std::vector containing all + // of the data that's left. Can be used with any byte sized type, such as + // char, unsigned char, uint8_t, etc. + template std::vector ConsumeBytes(size_t num_bytes) { + num_bytes = std::min(num_bytes, remaining_bytes_); + return ConsumeBytes(num_bytes, num_bytes); + } + + // Similar to |ConsumeBytes|, but also appends the terminator value at the end + // of the resulting vector. Useful, when a mutable null-terminated C-string is + // needed, for example. But that is a rare case. Better avoid it, if possible, + // and prefer using |ConsumeBytes| or |ConsumeBytesAsString| methods. + template + std::vector ConsumeBytesWithTerminator(size_t num_bytes, + T terminator = 0) { + num_bytes = std::min(num_bytes, remaining_bytes_); + std::vector result = ConsumeBytes(num_bytes + 1, num_bytes); + result.back() = terminator; + return result; + } + + // Returns a std::string containing |num_bytes| of input data. Using this and + // |.c_str()| on the resulting string is the best way to get an immutable + // null-terminated C string. If fewer than |num_bytes| of data remain, returns + // a shorter std::string containing all of the data that's left. + std::string ConsumeBytesAsString(size_t num_bytes) { + static_assert(sizeof(std::string::value_type) == sizeof(uint8_t), + "ConsumeBytesAsString cannot convert the data to a string."); + + num_bytes = std::min(num_bytes, remaining_bytes_); + std::string result( + reinterpret_cast(data_ptr_), + num_bytes); + Advance(num_bytes); + return result; + } + + // Returns a number in the range [min, max] by consuming bytes from the + // input data. The value might not be uniformly distributed in the given + // range. If there's no input data left, always returns |min|. |min| must + // be less than or equal to |max|. + template T ConsumeIntegralInRange(T min, T max) { + static_assert(std::is_integral::value, "An integral type is required."); + static_assert(sizeof(T) <= sizeof(uint64_t), "Unsupported integral type."); + + if (min > max) + abort(); + + // Use the biggest type possible to hold the range and the result. + uint64_t range = static_cast(max) - min; + uint64_t result = 0; + size_t offset = 0; + + while (offset < sizeof(T) * CHAR_BIT && (range >> offset) > 0 && + remaining_bytes_ != 0) { + // Pull bytes off the end of the seed data. Experimentally, this seems to + // allow the fuzzer to more easily explore the input space. This makes + // sense, since it works by modifying inputs that caused new code to run, + // and this data is often used to encode length of data read by + // |ConsumeBytes|. Separating out read lengths makes it easier modify the + // contents of the data that is actually read. + --remaining_bytes_; + result = (result << CHAR_BIT) | data_ptr_[remaining_bytes_]; + offset += CHAR_BIT; + } + + // Avoid division by 0, in case |range + 1| results in overflow. + if (range != std::numeric_limits::max()) + result = result % (range + 1); + + return static_cast(min + result); + } + + // Returns a std::string of length from 0 to |max_length|. When it runs out of + // input data, returns what remains of the input. Designed to be more stable + // with respect to a fuzzer inserting characters than just picking a random + // length and then consuming that many bytes with |ConsumeBytes|. + std::string ConsumeRandomLengthString(size_t max_length) { + // Reads bytes from the start of |data_ptr_|. Maps "\\" to "\", and maps "\" + // followed by anything else to the end of the string. As a result of this + // logic, a fuzzer can insert characters into the string, and the string + // will be lengthened to include those new characters, resulting in a more + // stable fuzzer than picking the length of a string independently from + // picking its contents. + std::string result; + + // Reserve the anticipated capaticity to prevent several reallocations. + result.reserve(std::min(max_length, remaining_bytes_)); + for (size_t i = 0; i < max_length && remaining_bytes_ != 0; ++i) { + char next = ConvertUnsignedToSigned(data_ptr_[0]); + Advance(1); + if (next == '\\' && remaining_bytes_ != 0) { + next = ConvertUnsignedToSigned(data_ptr_[0]); + Advance(1); + if (next != '\\') + break; + } + result += next; + } + + result.shrink_to_fit(); + return result; + } + + // Returns a std::vector containing all remaining bytes of the input data. + template std::vector ConsumeRemainingBytes() { + return ConsumeBytes(remaining_bytes_); + } + + // Prefer using |ConsumeRemainingBytes| unless you actually need a std::string + // object. + // Returns a std::vector containing all remaining bytes of the input data. + std::string ConsumeRemainingBytesAsString() { + return ConsumeBytesAsString(remaining_bytes_); + } + + // Returns a number in the range [Type's min, Type's max]. The value might + // not be uniformly distributed in the given range. If there's no input data + // left, always returns |min|. + template T ConsumeIntegral() { + return ConsumeIntegralInRange(std::numeric_limits::min(), + std::numeric_limits::max()); + } + + // Reads one byte and returns a bool, or false when no data remains. + bool ConsumeBool() { return 1 & ConsumeIntegral(); } + + // Returns a copy of a value selected from a fixed-size |array|. + template + T PickValueInArray(const T (&array)[size]) { + static_assert(size > 0, "The array must be non empty."); + return array[ConsumeIntegralInRange(0, size - 1)]; + } + + template + T PickValueInArray(std::initializer_list list) { + // static_assert(list.size() > 0, "The array must be non empty."); + return *(list.begin() + ConsumeIntegralInRange(0, list.size() - 1)); + } + + // Return an enum value. The enum must start at 0 and be contiguous. It must + // also contain |kMaxValue| aliased to its largest (inclusive) value. Such as: + // enum class Foo { SomeValue, OtherValue, kMaxValue = OtherValue }; + template T ConsumeEnum() { + static_assert(std::is_enum::value, "|T| must be an enum type."); + return static_cast(ConsumeIntegralInRange( + 0, static_cast(T::kMaxValue))); + } + + // Reports the remaining bytes available for fuzzed input. + size_t remaining_bytes() { return remaining_bytes_; } + +private: + FuzzedDataProvider(const FuzzedDataProvider &) = delete; + FuzzedDataProvider &operator=(const FuzzedDataProvider &) = delete; + + void Advance(size_t num_bytes) { + if (num_bytes > remaining_bytes_) + abort(); + + data_ptr_ += num_bytes; + remaining_bytes_ -= num_bytes; + } + + template + std::vector ConsumeBytes(size_t size, size_t num_bytes_to_consume) { + static_assert(sizeof(T) == sizeof(uint8_t), "Incompatible data type."); + + // The point of using the size-based constructor below is to increase the + // odds of having a vector object with capacity being equal to the length. + // That part is always implementation specific, but at least both libc++ and + // libstdc++ allocate the requested number of bytes in that constructor, + // which seems to be a natural choice for other implementations as well. + // To increase the odds even more, we also call |shrink_to_fit| below. + std::vector result(size); + std::memcpy(result.data(), data_ptr_, num_bytes_to_consume); + Advance(num_bytes_to_consume); + + // Even though |shrink_to_fit| is also implementation specific, we expect it + // to provide an additional assurance in case vector's constructor allocated + // a buffer which is larger than the actual amount of data we put inside it. + result.shrink_to_fit(); + return result; + } + + template TS ConvertUnsignedToSigned(TU value) { + static_assert(sizeof(TS) == sizeof(TU), "Incompatible data types."); + static_assert(!std::numeric_limits::is_signed, + "Source type must be unsigned."); + + // TODO(Dor1s): change to `if constexpr` once C++17 becomes mainstream. + if (std::numeric_limits::is_modulo) + return static_cast(value); + + // Avoid using implementation-defined unsigned to signer conversions. + // To learn more, see https://stackoverflow.com/questions/13150449. + if (value <= std::numeric_limits::max()) + return static_cast(value); + else { + constexpr auto TS_min = std::numeric_limits::min(); + return TS_min + static_cast(value - TS_min); + } + } + + const uint8_t *data_ptr_; + size_t remaining_bytes_; +}; + +#endif // LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_