Merge pull request #35 from rust-fuzz/frewsxcv-llvm-7

Update libfuzzer files from LLVM 7.0 release.

libfuzzer/CMakeLists.txt

@@ -1,6 +1,6 @@
 set(LIBFUZZER_SOURCES
-  FuzzerClangCounters.cpp
   FuzzerCrossOver.cpp
+  FuzzerDataFlowTrace.cpp
   FuzzerDriver.cpp
   FuzzerExtFunctionsDlsym.cpp
   FuzzerExtFunctionsDlsymWin.cpp
@@ -13,6 +13,7 @@ set(LIBFUZZER_SOURCES
   FuzzerMerge.cpp
   FuzzerMutate.cpp
   FuzzerSHA1.cpp
+  FuzzerShmemFuchsia.cpp
   FuzzerShmemPosix.cpp
   FuzzerShmemWindows.cpp
   FuzzerTracePC.cpp
@@ -21,8 +22,29 @@ set(LIBFUZZER_SOURCES
   FuzzerUtilFuchsia.cpp
   FuzzerUtilLinux.cpp
   FuzzerUtilPosix.cpp
-  FuzzerUtilWindows.cpp
-  )
+  FuzzerUtilWindows.cpp)
+
+set(LIBFUZZER_HEADERS
+  FuzzerCommand.h
+  FuzzerCorpus.h
+  FuzzerDataFlowTrace.h
+  FuzzerDefs.h
+  FuzzerDictionary.h
+  FuzzerExtFunctions.def
+  FuzzerExtFunctions.h
+  FuzzerFlags.def
+  FuzzerIO.h
+  FuzzerInterface.h
+  FuzzerInternal.h
+  FuzzerMerge.h
+  FuzzerMutate.h
+  FuzzerOptions.h
+  FuzzerRandom.h
+  FuzzerSHA1.h
+  FuzzerShmem.h
+  FuzzerTracePC.h
+  FuzzerUtil.h
+  FuzzerValueBitMap.h)
 
 CHECK_CXX_SOURCE_COMPILES("
   static thread_local int blah;
@@ -33,10 +55,12 @@ CHECK_CXX_SOURCE_COMPILES("
 
 set(LIBFUZZER_CFLAGS ${SANITIZER_COMMON_CFLAGS})
 
-if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND COMPILER_RT_LIBCXX_PATH)
+if(OS_NAME MATCHES "Linux|Fuchsia" AND COMPILER_RT_LIBCXX_PATH)
   list(APPEND LIBFUZZER_CFLAGS -nostdinc++ -D_LIBCPP_ABI_VERSION=Fuzzer)
   # Remove -stdlib= which is unused when passing -nostdinc++.
   string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+elseif(TARGET cxx-headers OR HAVE_LIBCXX)
+  set(LIBFUZZER_DEPS cxx-headers)
 endif()
 
 append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG -fno-omit-frame-pointer LIBFUZZER_CFLAGS)
@@ -49,21 +73,22 @@ if(NOT HAS_THREAD_LOCAL)
   list(APPEND LIBFUZZER_CFLAGS -Dthread_local=__thread)
 endif()
 
-if(APPLE)
-  set(FUZZER_SUPPORTED_OS osx)
-endif()
+set(FUZZER_SUPPORTED_OS ${SANITIZER_COMMON_SUPPORTED_OS})
 
 add_compiler_rt_object_libraries(RTfuzzer
   OS ${FUZZER_SUPPORTED_OS}
   ARCHS ${FUZZER_SUPPORTED_ARCH}
   SOURCES ${LIBFUZZER_SOURCES}
-  CFLAGS ${LIBFUZZER_CFLAGS})
+  ADDITIONAL_HEADERS ${LIBFUZZER_HEADERS}
+  CFLAGS ${LIBFUZZER_CFLAGS}
+  DEPS ${LIBFUZZER_DEPS})
 
 add_compiler_rt_object_libraries(RTfuzzer_main
   OS ${FUZZER_SUPPORTED_OS}
   ARCHS ${FUZZER_SUPPORTED_ARCH}
   SOURCES FuzzerMain.cpp
-  CFLAGS ${LIBFUZZER_CFLAGS})
+  CFLAGS ${LIBFUZZER_CFLAGS}
+  DEPS ${LIBFUZZER_DEPS})
 
 add_compiler_rt_runtime(clang_rt.fuzzer
   STATIC
@@ -81,12 +106,12 @@ add_compiler_rt_runtime(clang_rt.fuzzer_no_main
   CFLAGS ${LIBFUZZER_CFLAGS}
   PARENT_TARGET fuzzer)
 
-if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND COMPILER_RT_LIBCXX_PATH)
+if(OS_NAME MATCHES "Linux|Fuchsia" AND COMPILER_RT_LIBCXX_PATH)
   macro(partially_link_libcxx name dir arch)
     set(cxx_${arch}_merge_dir "${CMAKE_CURRENT_BINARY_DIR}/cxx_${arch}_merge.dir")
     file(MAKE_DIRECTORY ${cxx_${arch}_merge_dir})
     add_custom_command(TARGET clang_rt.${name}-${arch} POST_BUILD
-      COMMAND ${CMAKE_LINKER} --whole-archive "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" --no-whole-archive ${dir}/src/libcxx_fuzzer_${arch}-build/lib/libc++.a -r -o ${name}.o
+      COMMAND ${CMAKE_LINKER} --whole-archive "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" --no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o
       COMMAND ${CMAKE_OBJCOPY} --localize-hidden ${name}.o
       COMMAND ${CMAKE_COMMAND} -E remove "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>"
       COMMAND ${CMAKE_AR} qcs "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" ${name}.o
@@ -102,11 +127,13 @@ if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND COMPILER_RT_LIBCXX_PATH)
              -D_LIBCPP_ABI_VERSION=Fuzzer
              -D_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS=1
              -fvisibility=hidden
-      CMAKE_ARGS -DLIBCXX_ENABLE_EXCEPTIONS=OFF
+      CMAKE_ARGS -DCMAKE_CXX_COMPILER_WORKS=ON
+                 -DLIBCXX_ENABLE_EXCEPTIONS=OFF
+                 -DLIBCXX_ENABLE_SHARED=OFF
                  -DLIBCXX_CXX_ABI=none)
-    target_compile_options(RTfuzzer.${arch} PRIVATE -isystem ${COMPILER_RT_LIBCXX_PATH}/include)
+    target_compile_options(RTfuzzer.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)
     add_dependencies(RTfuzzer.${arch} libcxx_fuzzer_${arch}-build)
-    target_compile_options(RTfuzzer_main.${arch} PRIVATE -isystem ${COMPILER_RT_LIBCXX_PATH}/include)
+    target_compile_options(RTfuzzer_main.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)
     add_dependencies(RTfuzzer_main.${arch} libcxx_fuzzer_${arch}-build)
     partially_link_libcxx(fuzzer_no_main ${LIBCXX_${arch}_PREFIX} ${arch})
     partially_link_libcxx(fuzzer ${LIBCXX_${arch}_PREFIX} ${arch})

libfuzzer/FuzzerCommand.h

@@ -29,8 +29,7 @@ class Command final {
   // is immutable, meaning this flag effectively marks the end of the mutable
   // argument list.
   static inline const char *ignoreRemainingArgs() {
-    static const char *kIgnoreRemaining = "-ignore_remaining_args=1";
-    return kIgnoreRemaining;
+    return "-ignore_remaining_args=1";
   }
 
   Command() : CombinedOutAndErr(false) {}

libfuzzer/FuzzerCorpus.h

@@ -12,6 +12,7 @@
 #ifndef LLVM_FUZZER_CORPUS
 #define LLVM_FUZZER_CORPUS
 
+#include "FuzzerDataFlowTrace.h"
 #include "FuzzerDefs.h"
 #include "FuzzerIO.h"
 #include "FuzzerRandom.h"
@@ -35,8 +36,9 @@ struct InputInfo {
   size_t NumSuccessfullMutations = 0;
   bool MayDeleteFile = false;
   bool Reduced = false;
+  bool HasFocusFunction = false;
   Vector<uint32_t> UniqFeatureSet;
-  float FeatureFrequencyScore = 1.0;
+  Vector<uint8_t> DataFlowTraceForFocusFunction;
 };
 
 class InputCorpus {
@@ -45,7 +47,6 @@ class InputCorpus {
   InputCorpus(const std::string &OutputCorpus) : OutputCorpus(OutputCorpus) {
     memset(InputSizesPerFeature, 0, sizeof(InputSizesPerFeature));
     memset(SmallestElementPerFeature, 0, sizeof(SmallestElementPerFeature));
-    memset(FeatureFrequency, 0, sizeof(FeatureFrequency));
   }
   ~InputCorpus() {
     for (auto II : Inputs)
@@ -70,10 +71,24 @@ class InputCorpus {
         Res = std::max(Res, II->U.size());
     return Res;
   }
+
+  size_t NumInputsThatTouchFocusFunction() {
+    return std::count_if(Inputs.begin(), Inputs.end(), [](const InputInfo *II) {
+      return II->HasFocusFunction;
+    });
+  }
+
+  size_t NumInputsWithDataFlowTrace() {
+    return std::count_if(Inputs.begin(), Inputs.end(), [](const InputInfo *II) {
+      return !II->DataFlowTraceForFocusFunction.empty();
+    });
+  }
+
   bool empty() const { return Inputs.empty(); }
   const Unit &operator[] (size_t Idx) const { return Inputs[Idx]->U; }
   void AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile,
-                   const Vector<uint32_t> &FeatureSet) {
+                   bool HasFocusFunction, const Vector<uint32_t> &FeatureSet,
+                   const DataFlowTrace &DFT, const InputInfo *BaseII) {
     assert(!U.empty());
     if (FeatureDebug)
       Printf("ADD_TO_CORPUS %zd NF %zd\n", Inputs.size(), NumFeatures);
@@ -83,9 +98,19 @@ class InputCorpus {
     II.NumFeatures = NumFeatures;
     II.MayDeleteFile = MayDeleteFile;
     II.UniqFeatureSet = FeatureSet;
+    II.HasFocusFunction = HasFocusFunction;
     std::sort(II.UniqFeatureSet.begin(), II.UniqFeatureSet.end());
     ComputeSHA1(U.data(), U.size(), II.Sha1);
-    Hashes.insert(Sha1ToString(II.Sha1));
+    auto Sha1Str = Sha1ToString(II.Sha1);
+    Hashes.insert(Sha1Str);
+    if (HasFocusFunction)
+      if (auto V = DFT.Get(Sha1Str))
+        II.DataFlowTraceForFocusFunction = *V;
+    // This is a gross heuristic.
+    // Ideally, when we add an element to a corpus we need to know its DFT.
+    // But if we don't, we'll use the DFT of its base input.
+    if (II.DataFlowTraceForFocusFunction.empty() && BaseII)
+      II.DataFlowTraceForFocusFunction = BaseII->DataFlowTraceForFocusFunction;
     UpdateCorpusDistribution();
     PrintCorpus();
     // ValidateFeatureSet();
@@ -157,9 +182,9 @@ class InputCorpus {
   void PrintStats() {
     for (size_t i = 0; i < Inputs.size(); i++) {
       const auto &II = *Inputs[i];
-      Printf("  [%zd %s]\tsz: %zd\truns: %zd\tsucc: %zd\n", i,
+      Printf("  [% 3zd %s] sz: % 5zd runs: % 5zd succ: % 5zd focus: %d\n", i,
              Sha1ToString(II.Sha1).c_str(), II.U.size(),
-             II.NumExecutedMutations, II.NumSuccessfullMutations);
+             II.NumExecutedMutations, II.NumSuccessfullMutations, II.HasFocusFunction);
     }
   }
 
@@ -213,18 +238,10 @@ class InputCorpus {
     return false;
   }
 
-  void UpdateFeatureFrequency(size_t Idx) {
-    FeatureFrequency[Idx % kFeatureSetSize]++;
-  }
-  float GetFeatureFrequency(size_t Idx) const {
-    return FeatureFrequency[Idx % kFeatureSetSize];
-  }
-  void UpdateFeatureFrequencyScore(InputInfo *II) {
-    const float kMin = 0.01, kMax = 100.;
-    II->FeatureFrequencyScore = kMin;
-    for (auto Idx : II->UniqFeatureSet)
-      II->FeatureFrequencyScore += 1. / (GetFeatureFrequency(Idx) + 1.);
-    II->FeatureFrequencyScore = Min(II->FeatureFrequencyScore, kMax);
+  bool IsFeatureNew(size_t Idx, uint32_t NewSize, bool Shrink) {
+    assert(NewSize);
+    uint32_t OldSize = GetFeature(Idx % kFeatureSetSize);
+    return OldSize == 0 || (Shrink && OldSize > NewSize);
   }
 
   size_t NumFeatures() const { return NumAddedFeatures; }
@@ -264,14 +281,11 @@ class InputCorpus {
     std::iota(Intervals.begin(), Intervals.end(), 0);
     for (size_t i = 0; i < N; i++)
       Weights[i] = Inputs[i]->NumFeatures
-                       ? (i + 1) * Inputs[i]->FeatureFrequencyScore
+                       ? (i + 1) * (Inputs[i]->HasFocusFunction ? 1000 : 1)
                        : 0.;
     if (FeatureDebug) {
       for (size_t i = 0; i < N; i++)
         Printf("%zd ", Inputs[i]->NumFeatures);
-      Printf("NUM\n");
-      for (size_t i = 0; i < N; i++)
-        Printf("%f ", Inputs[i]->FeatureFrequencyScore);
       Printf("SCORE\n");
       for (size_t i = 0; i < N; i++)
         Printf("%f ", Weights[i]);
@@ -292,7 +306,6 @@ class InputCorpus {
   size_t NumUpdatedFeatures = 0;
   uint32_t InputSizesPerFeature[kFeatureSetSize];
   uint32_t SmallestElementPerFeature[kFeatureSetSize];
-  float FeatureFrequency[kFeatureSetSize];
 
   std::string OutputCorpus;
 };