Use the stop time to cap command deadlines when running batches for fuzzing.

centipede/BUILD

@@ -665,6 +665,7 @@ cc_library(
         ":runner_request",
         ":runner_result",
         ":shared_memory_blob_sequence",
+        ":stop",
         ":util",
         ":workdir",
         "@abseil-cpp//absl/base:nullability",

centipede/centipede.cc

@@ -362,9 +362,17 @@ bool Centipede::InputPassesFilter(const ByteArray &input) {
 bool Centipede::ExecuteAndReportCrash(std::string_view binary,
                                       const std::vector<ByteArray> &input_vec,
                                       BatchResult &batch_result) {
-  bool success = user_callbacks_.Execute(binary, input_vec, batch_result);
-  if (!success) ReportCrash(binary, input_vec, batch_result);
-  return success || batch_result.IsIgnoredFailure();
+  bool success =
+      user_callbacks_.Execute(binary, input_vec, batch_result, GetStopTime());
+  if (success) return true;
+  if (ShouldStop()) {
+    FUZZTEST_LOG_FIRST_N(WARNING, 1)
+        << "Stop condition met - not reporting further crashes possibly "
+           "related to the stop condition.";
+    return true;
+  }
+  ReportCrash(binary, input_vec, batch_result);
+  return batch_result.IsIgnoredFailure();
 }
 
 // *** Highly experimental and risky. May not scale well for large targets. ***
@@ -976,7 +984,8 @@ void Centipede::ReportCrash(std::string_view binary,
     if (ShouldStop()) return;
     const auto &one_input = input_vec[input_idx];
     BatchResult one_input_batch_result;
-    if (!user_callbacks_.Execute(binary, {one_input}, one_input_batch_result)) {
+    if (!user_callbacks_.Execute(binary, {one_input}, one_input_batch_result,
+                                 absl::InfiniteFuture())) {
       auto hash = Hash(one_input);
       auto crash_dir = wd_.CrashReproducerDirPaths().MyShard();
       FUZZTEST_CHECK_OK(RemoteMkdir(crash_dir));

centipede/centipede_callbacks.cc

@@ -50,6 +50,7 @@
 #include "./centipede/mutation_input.h"
 #include "./centipede/runner_request.h"
 #include "./centipede/runner_result.h"
+#include "./centipede/stop.h"
 #include "./centipede/util.h"
 #include "./centipede/workdir.h"
 #include "./common/blob_file.h"
@@ -472,14 +473,15 @@ void CentipedeCallbacks::CleanUpPersistentMode() {
       command_contexts_.end());
 }
 
-int CentipedeCallbacks::RunBatchForBinary(std::string_view binary) {
+int CentipedeCallbacks::RunBatchForBinary(std::string_view binary,
+                                          absl::Time deadline) {
   auto& command_context = GetOrCreateCommandContextForBinary(binary);
   auto& cmd = command_context.cmd;
   const absl::Duration amortized_timeout =
       env_.timeout_per_batch == 0
           ? absl::InfiniteDuration()
           : absl::Seconds(env_.timeout_per_batch) + absl::Seconds(5);
-  const auto deadline = absl::Now() + amortized_timeout;
+  deadline = std::min(absl::Now() + amortized_timeout, deadline);
   int exit_code = EXIT_SUCCESS;
   const bool should_clean_up = [&] {
     if (!cmd.is_executing() && !cmd.ExecuteAsync()) {
@@ -497,11 +499,12 @@ int CentipedeCallbacks::RunBatchForBinary(std::string_view binary) {
   if (should_clean_up) {
     exit_code = [&] {
       if (!cmd.is_executing()) return EXIT_FAILURE;
-      FUZZTEST_LOG(ERROR) << "Cleaning up the batch execution.";
+      FUZZTEST_LOG(ERROR) << "Cleaning up the batch execution with timeout "
+                          << kCommandCleanupTimeout;
       cmd.RequestStop();
       const auto ret = cmd.Wait(absl::Now() + kCommandCleanupTimeout);
       if (ret.has_value()) return *ret;
-      FUZZTEST_LOG(ERROR) << "Batch execution cleanup failed to end in 60s.";
+      FUZZTEST_LOG(ERROR) << "Failed to wait for the batch execution cleanup.";
       return EXIT_FAILURE;
     }();
     command_contexts_.erase(
@@ -515,7 +518,7 @@ int CentipedeCallbacks::RunBatchForBinary(std::string_view binary) {
 
 int CentipedeCallbacks::ExecuteCentipedeSancovBinaryWithShmem(
     std::string_view binary, const std::vector<ByteArray>& inputs,
-    BatchResult& batch_result) {
+    BatchResult& batch_result, absl::Time deadline) {
   auto start_time = absl::Now();
   batch_result.ClearAndResize(inputs.size());
 
@@ -541,7 +544,7 @@ int CentipedeCallbacks::ExecuteCentipedeSancovBinaryWithShmem(
   }
 
   // Run.
-  const int exit_code = RunBatchForBinary(binary);
+  const int exit_code = RunBatchForBinary(binary, deadline);
   inputs_blobseq_.ReleaseSharedMemory();  // Inputs are already consumed.
 
   // Get results.
@@ -699,7 +702,8 @@ MutationResult CentipedeCallbacks::MutateViaExternalBinary(
       << VV(num_inputs_written) << VV(inputs.size());
 
   // Execute.
-  const int exit_code = RunBatchForBinary(binary);
+  const int exit_code =
+      RunBatchForBinary(binary, /*deadline=*/absl::InfiniteFuture());
   inputs_blobseq_.ReleaseSharedMemory();  // Inputs are already consumed.
 
   if (exit_code != EXIT_SUCCESS) {

centipede/centipede_callbacks.h

@@ -65,8 +65,8 @@ class CentipedeCallbacks {
   // Post-condition:
   // `batch_result` has results for every `input`, even on failure.
   virtual bool Execute(std::string_view binary,
-                       const std::vector<ByteArray> &inputs,
-                       BatchResult &batch_result) = 0;
+                       const std::vector<ByteArray>& inputs,
+                       BatchResult& batch_result, absl::Time deadline) = 0;
 
   // Takes non-empty `inputs` and returns at most `num_mutants` mutated inputs.
   virtual std::vector<ByteArray> Mutate(
@@ -103,8 +103,8 @@ class CentipedeCallbacks {
   // Same as ExecuteCentipedeSancovBinary, but uses shared memory.
   // Much faster for fast targets since it uses fewer system calls.
   int ExecuteCentipedeSancovBinaryWithShmem(
-      std::string_view binary, const std::vector<ByteArray> &inputs,
-      BatchResult &batch_result);
+      std::string_view binary, const std::vector<ByteArray>& inputs,
+      BatchResult& batch_result, absl::Time deadline);
 
   // Constructs a string CENTIPEDE_RUNNER_FLAGS=":flag1:flag2:...",
   // where the flags are determined by `env` and also include `extra_flags`.
@@ -173,7 +173,7 @@ class CentipedeCallbacks {
   // Returns a CommandContext with matching `binary`. Creates one if needed.
   CommandContext& GetOrCreateCommandContextForBinary(std::string_view binary);
   // Runs a batch with the command `binary` and returns the exit code.
-  int RunBatchForBinary(std::string_view binary);
+  int RunBatchForBinary(std::string_view binary, absl::Time deadline);
 
   // Prints the execution log from the last executed binary.
   void PrintExecutionLog() const;

centipede/centipede_default_callbacks.cc

@@ -46,10 +46,11 @@ CentipedeDefaultCallbacks::CentipedeDefaultCallbacks(const Environment &env)
 }
 
 bool CentipedeDefaultCallbacks::Execute(std::string_view binary,
-                                        const std::vector<ByteArray> &inputs,
-                                        BatchResult &batch_result) {
-  return ExecuteCentipedeSancovBinaryWithShmem(binary, inputs, batch_result) ==
-         0;
+                                        const std::vector<ByteArray>& inputs,
+                                        BatchResult& batch_result,
+                                        absl::Time deadline) {
+  return ExecuteCentipedeSancovBinaryWithShmem(binary, inputs, batch_result,
+                                               deadline) == 0;
 }
 
 size_t CentipedeDefaultCallbacks::GetSeeds(size_t num_seeds,

centipede/centipede_default_callbacks.h

@@ -40,8 +40,8 @@ class CentipedeDefaultCallbacks : public CentipedeCallbacks {
   explicit CentipedeDefaultCallbacks(const Environment &env);
   size_t GetSeeds(size_t num_seeds, std::vector<ByteArray> &seeds) override;
   absl::StatusOr<std::string> GetSerializedTargetConfig() override;
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override;
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override;
   std::vector<ByteArray> Mutate(const std::vector<MutationInputRef> &inputs,
                                 size_t num_mutants) override;
 

centipede/centipede_interface.cc

@@ -305,7 +305,7 @@ absl::flat_hash_set<std::string> PruneOldCrashesAndGetRemainingCrashSignatures(
     FUZZTEST_CHECK_OK(
         RemoteFileGetContents(crashing_input_file, crashing_input));
     const bool is_reproducible = !scoped_callbacks.callbacks()->Execute(
-        env.binary, {crashing_input}, batch_result);
+        env.binary, {crashing_input}, batch_result, absl::InfiniteFuture());
     const bool is_duplicate =
         is_reproducible && !batch_result.IsSetupFailure() &&
         !remaining_crash_signatures.insert(batch_result.failure_signature())
@@ -686,6 +686,8 @@ int UpdateCorpusDatabaseForFuzzTests(
           << "Skip updating corpus database due to early stop requested.";
       continue;
     }
+    // The test time limit does not apply for the rest of the steps.
+    ClearEarlyStopRequestAndSetStopTime(/*stop_time=*/absl::InfiniteFuture());
 
     // TODO(xinhaoyuan): Have a separate flag to skip corpus updating instead
     // of checking whether workdir is specified or not.

centipede/centipede_test.cc

@@ -68,8 +68,8 @@ class CentipedeMock : public CentipedeCallbacks {
   // Doesn't execute anything
   // Sets `batch_result.results()` based on the values of `inputs`:
   // Collects various stats about the inputs, to be checked in tests.
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override {
     batch_result.results().clear();
     // For every input, we create a 256-element array `counters`, where
     // i-th element is the number of bytes with the value 'i' in the input.
@@ -356,8 +356,8 @@ class MutateCallbacks : public CentipedeCallbacks {
  public:
   explicit MutateCallbacks(const Environment &env) : CentipedeCallbacks(env) {}
   // Will not be called.
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override {
     FUZZTEST_LOG(FATAL);
     return false;
   }
@@ -499,8 +499,8 @@ class MergeMock : public CentipedeCallbacks {
   // Doesn't execute anything.
   // All inputs are 1-byte long.
   // For an input {X}, the feature output is {X}.
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override {
     batch_result.results().resize(inputs.size());
     for (size_t i = 0, n = inputs.size(); i < n; ++i) {
       FUZZTEST_CHECK_EQ(inputs[i].size(), 1);
@@ -588,10 +588,10 @@ class FunctionFilterMock : public CentipedeCallbacks {
   }
 
   // Executes the target in the normal way.
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
-    return ExecuteCentipedeSancovBinaryWithShmem(env_.binary, inputs,
-                                                 batch_result) == EXIT_SUCCESS;
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override {
+    return ExecuteCentipedeSancovBinaryWithShmem(
+               env_.binary, inputs, batch_result, deadline) == EXIT_SUCCESS;
   }
 
   // Sets the inputs to one of 3 pre-defined values.
@@ -696,8 +696,8 @@ class ExtraBinariesMock : public CentipedeCallbacks {
 
   // Doesn't execute anything.
   // On certain combinations of {binary,input} returns false.
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override {
     bool res = true;
     for (const auto &input : inputs) {
       if (input.size() != 1) continue;
@@ -814,8 +814,8 @@ class UndetectedCrashingInputMock : public CentipedeCallbacks {
   // Doesn't execute anything.
   // Crash when 0th char of input to binary b1 equals `crashing_input_idx_`, but
   // only on 1st exec.
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override {
     batch_result.ClearAndResize(inputs.size());
     bool res = true;
     if (!first_pass_) {
@@ -982,7 +982,8 @@ TEST_F(CentipedeWithTemporaryLocalDir, CleansUpMetadataAfterStartup) {
 
   BatchResult batch_result;
   const std::vector<ByteArray> inputs = {{0}};
-  ASSERT_TRUE(callbacks.Execute(env.binary, inputs, batch_result));
+  ASSERT_TRUE(callbacks.Execute(env.binary, inputs, batch_result,
+                                /*deadline=*/absl::InfiniteFuture()));
   ASSERT_EQ(batch_result.results().size(), 1);
   bool found_startup_cmp_entry = false;
   batch_result.results()[0].metadata().ForEachCmpEntry(
@@ -999,8 +1000,8 @@ class FakeCentipedeCallbacksForThreadChecking : public CentipedeCallbacks {
                                           std::thread::id execute_thread_id)
       : CentipedeCallbacks(env), execute_thread_id_(execute_thread_id) {}
 
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override {
     batch_result.ClearAndResize(inputs.size());
     thread_check_passed_ = thread_check_passed_ &&
                            std::this_thread::get_id() == execute_thread_id_;
@@ -1044,7 +1045,8 @@ TEST_F(CentipedeWithTemporaryLocalDir, DetectsStackOverflow) {
   BatchResult batch_result;
   const std::vector<ByteArray> inputs = {ByteArray{'s', 't', 'k'}};
 
-  ASSERT_FALSE(callbacks.Execute(env.binary, inputs, batch_result));
+  ASSERT_FALSE(callbacks.Execute(env.binary, inputs, batch_result,
+                                 /*deadline=*/absl::InfiniteFuture()));
   EXPECT_THAT(batch_result.log(), HasSubstr("Stack limit exceeded"));
   EXPECT_EQ(batch_result.failure_description(), "stack-limit-exceeded");
 }
@@ -1053,8 +1055,8 @@ class SetupFailureCallbacks : public CentipedeCallbacks {
  public:
   using CentipedeCallbacks::CentipedeCallbacks;
 
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override {
     ++execute_count_;
     batch_result.ClearAndResize(inputs.size());
     batch_result.exit_code() = EXIT_FAILURE;
@@ -1090,8 +1092,8 @@ class SkippedTestCallbacks : public CentipedeCallbacks {
  public:
   using CentipedeCallbacks::CentipedeCallbacks;
 
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override {
     ++execute_count_;
     batch_result.ClearAndResize(inputs.size());
     batch_result.exit_code() = EXIT_FAILURE;
@@ -1128,8 +1130,8 @@ class IgnoredFailureCallbacks : public CentipedeCallbacks {
  public:
   using CentipedeCallbacks::CentipedeCallbacks;
 
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time daedline) override {
     ++execute_count_;
     batch_result.ClearAndResize(inputs.size());
     batch_result.exit_code() = EXIT_FAILURE;
@@ -1224,7 +1226,8 @@ TEST_F(CentipedeWithTemporaryLocalDir, HangingFuzzTargetExitsAfterTimeout) {
   env.fork_server = false;
 
   // Test that the process does not get stuck and exits promptly.
-  EXPECT_FALSE(callbacks.Execute(env.binary, {{0}}, batch_result));
+  EXPECT_FALSE(callbacks.Execute(env.binary, {{0}}, batch_result,
+                                 /*deadline=*/absl::InfiniteFuture()));
 }
 
 }  // namespace

centipede/environment.cc

@@ -285,17 +285,6 @@ void Environment::UpdateWithTargetConfig(
   timeout_per_input = time_limit_per_input_sec;
   UpdateTimeoutPerBatchIfEqualTo(autocomputed_timeout_per_batch);
 
-  // Adjust `timeout_per_batch` to never exceed the test time limit.
-  if (const auto test_time_limit = config.GetTimeLimitPerTest();
-      test_time_limit < absl::InfiniteDuration()) {
-    const size_t test_time_limit_seconds =
-        convert_to_seconds(test_time_limit, "Test time limit");
-    timeout_per_batch =
-        timeout_per_batch == 0
-            ? test_time_limit_seconds
-            : std::min(timeout_per_batch, test_time_limit_seconds);
-  }
-
   // Convert bytes to MB by rounding up.
   constexpr auto bytes_to_mb = [](size_t bytes) {
     return bytes == 0 ? 0 : (bytes - 1) / 1024 / 1024 + 1;

centipede/environment_test.cc

@@ -147,30 +147,6 @@ TEST(Environment,
   EXPECT_EQ(env.timeout_per_batch, 0);
 }
 
-TEST(Environment, UpdatesTimeoutPerBatchFromTargetConfigTimeLimit) {
-  Environment env;
-  fuzztest::internal::Configuration config;
-  config.time_limit = absl::Seconds(123);
-  config.time_budget_type = fuzztest::internal::TimeBudgetType::kPerTest;
-  FUZZTEST_CHECK(config.GetTimeLimitPerTest() == absl::Seconds(123));
-  env.UpdateWithTargetConfig(config);
-  EXPECT_EQ(env.timeout_per_batch, 123)
-      << "`timeout_per_batch` should be set to the test time limit when it was "
-         "previously unset";
-
-  env.timeout_per_batch = 456;
-  env.UpdateWithTargetConfig(config);
-  EXPECT_EQ(env.timeout_per_batch, 123)
-      << "`timeout_per_batch` should be set to test time limit when it is "
-         "shorter than the previous value";
-
-  env.timeout_per_batch = 56;
-  env.UpdateWithTargetConfig(config);
-  EXPECT_EQ(env.timeout_per_batch, 56)
-      << "`timeout_per_batch` should not be updated with the test time limit "
-         "when it is longer than the previous value";
-}
-
 TEST(Environment, UpdatesRssLimitMbFromTargetConfigRssLimit) {
   Environment env;
   env.rss_limit_mb = Environment::Default().rss_limit_mb;

centipede/minimize_crash.cc

@@ -122,7 +122,8 @@ static void MinimizeCrash(const Environment &env,
     }
 
     // Execute all mutants. If a new crasher is found, add it to `queue`.
-    if (!callbacks->Execute(env.binary, smaller_mutants, batch_result)) {
+    if (!callbacks->Execute(env.binary, smaller_mutants, batch_result,
+                            absl::InfiniteFuture())) {
       size_t crash_inputs_idx = batch_result.num_outputs_read();
       FUZZTEST_CHECK_LT(crash_inputs_idx, smaller_mutants.size());
       const auto &new_crasher = smaller_mutants[crash_inputs_idx];
@@ -142,7 +143,8 @@ int MinimizeCrash(ByteSpan crashy_input, const Environment &env,
 
   BatchResult batch_result;
   ByteArray original_crashy_input(crashy_input.begin(), crashy_input.end());
-  if (callbacks->Execute(env.binary, {original_crashy_input}, batch_result)) {
+  if (callbacks->Execute(env.binary, {original_crashy_input}, batch_result,
+                         absl::InfiniteFuture())) {
     FUZZTEST_LOG(INFO) << "The original crashy input did not crash; exiting";
     return EXIT_FAILURE;
   }