fix: Detect ABI change of fuzzing harnesses

tooling/greybox_fuzzer/src/lib.rs

@@ -24,7 +24,7 @@
 mod types;
 
 use corpus::{Corpus, DEFAULT_CORPUS_FOLDER, TestCase, TestCaseId};
-use mutation::InputMutator;
+use mutation::{InputMutator, add_elements_from_input_map_to_vector_without_abi};
 use rayon::iter::ParallelIterator;
 use termcolor::{ColorChoice, StandardStream};
 pub use types::FuzzTestResult;
@@ -35,8 +35,8 @@
 
 use noirc_artifacts::program::ProgramArtifact;
 use rand::prelude::*;
 use rand::{Rng, SeedableRng};
 use rand_xorshift::XorShiftRng;
 
 use rayon::prelude::IntoParallelIterator;
 use std::io::Write;
@@ -44,14 +44,14 @@
 
 const FOREIGN_CALL_FAILURE_SUBSTRING: &str = "Failed calling external resolver.";
 
 /// We aim the number of testcases per round so one round takes these many microseconds
 const SINGLE_FUZZING_ROUND_TARGET_TIME: u128 = 100_000u128;
 
 /// Minimum pulse interval in milliseconds for printing metrics
 const MINIMUM_PULSE_INTERVAL_MILLIS: u64 = 1000u64;
 
 /// A seed for the XorShift RNG for use during mutation
 type SimpleXorShiftRNGSeed = <XorShiftRng as SeedableRng>::Seed;
 
 /// Information collected from testcase execution on success
 pub type WitnessAndCoverage = (WitnessStack<FieldElement>, Option<Vec<u32>>);
@@ -68,7 +68,7 @@
     main_testcase_id: TestCaseId,
     /// An optional id of a second testcase that will be used for splicing
     additional_testcase_id: Option<TestCaseId>,
     /// A seed for the PRNG that will be used for mutating/splicing
     seed: SimpleXorShiftRNGSeed,
 }
 
@@ -83,7 +83,7 @@
     }
 
     /// Create a task for executing a testcase without mutation
     pub fn mutationless(main_testcase_id: TestCaseId) -> Self {
         Self {
             main_testcase_id,
             additional_testcase_id: None,
@@ -91,7 +91,7 @@
         }
     }
 
     pub fn prng_seed(&self) -> SimpleXorShiftRNGSeed {
         self.seed
     }
     pub fn main_id(&self) -> TestCaseId {
@@ -102,7 +102,7 @@
     }
 }
 
 /// Contains information from parallel execution of testcases for quick single-threaded processing
 /// If no new coverage is detected, the fuzzer can simply quickly update the timing metrics without parsing the outcome
 #[derive(Debug)]
 struct FastParallelFuzzResult {
@@ -174,9 +174,9 @@
     total_acir_execution_time: u128,
     /// Total time spent executing Brillig programs in microseconds
     total_brillig_execution_time: u128,
     /// Total time spent mutating testcases
     total_mutation_time: u128,
     /// The number of unique testcases run
     processed_testcase_count: usize,
     /// Number of testcases removed from the corpus
     removed_testcase_count: usize,
@@ -432,6 +432,39 @@
         accumulated_coverage.detect_new_coverage(&new_coverage)
     }
 
+    /// Filter the starting corpus and add elements to the dictionary
+    /// Removes testcases that can't be encoded by the new ABI and adds interesting values from them to the dictionary
+    fn filter_starting_corpus(
+        &self,
+        corpus: &Corpus,
+        starting_corpus_ids: &mut Vec<TestCaseId>,
+    ) -> Option<Vec<FieldElement>> {
+        let mut elements_for_dictionary = Vec::new();
+        let mut ids_to_remove = Vec::new();
+        for (index, id) in starting_corpus_ids.iter().enumerate() {
+            let testcase = corpus.get_testcase_by_id(*id);
+            match self.acir_program.abi.encode(testcase, None) {
+                Ok(_) => (),
+                Err(_) => {
+                    add_elements_from_input_map_to_vector_without_abi(
+                        testcase,
+                        &mut elements_for_dictionary,
+                    );
+                    // Remove the testcase from the corpus
+                    ids_to_remove.push(index);
+                }
+            }
+        }
+        ids_to_remove.sort();
+        for index in ids_to_remove.iter().rev() {
+            starting_corpus_ids.remove(*index);
+        }
+        if elements_for_dictionary.is_empty() {
+            return None;
+        }
+        Some(elements_for_dictionary)
+    }
+
     /// Start the fuzzing campaign
     pub fn fuzz(&mut self) -> FuzzTestResult {
         self.metrics.set_num_threads(self.num_threads);
@@ -465,11 +498,21 @@
         let mut starting_corpus_ids: Vec<_> =
             corpus.get_full_stored_corpus().iter().map(|x| x.id()).collect();
 
+        let elements_for_dictionary =
+            self.filter_starting_corpus(&corpus, &mut starting_corpus_ids);
+
         // Can't minimize if there is no corpus
         if self.minimize_corpus && starting_corpus_ids.is_empty() {
-            return FuzzTestResult::MinimizationFailure(
-                "No initial corpus found to minimize".to_string(),
-            );
+            let minimization_failure_message = if elements_for_dictionary.is_some() {
+                "The corpus has only elements from a previous ABI version of the fuzzing harness. Nothing to minimize"
+            } else {
+                "No initial corpus found to minimize"
+            };
+            return FuzzTestResult::MinimizationFailure(minimization_failure_message.to_string());
+        }
+        let abi_change_detected = elements_for_dictionary.is_some();
+        if let Some(elements_for_dictionary) = elements_for_dictionary {
+            self.mutator.update_dictionary_from_vector(&elements_for_dictionary);
         }
 
         let minimized_corpus = if self.minimize_corpus {
@@ -496,6 +539,7 @@
             &self.function_name,
             corpus.get_corpus_storage_path(),
             &minimized_corpus_path,
+            abi_change_detected,
         );
 
         // Generate the default input (it is needed if the corpus is empty)
@@ -1225,6 +1269,7 @@
     fuzzing_harness_name: &str,
     corpus_path: &Path,
     minimized_corpus_path: &Path,
+    abi_change_detected: bool,
 ) -> Result<(), std::io::Error> {
     let writer = StandardStream::stderr(ColorChoice::Always);
     let mut writer = writer.lock();
@@ -1249,6 +1294,7 @@
             minimized_corpus_path.to_str().unwrap_or("No minimized corpus path provided")
         )?;
         writer.reset()?;
+
         writeln!(writer, "\"")?;
     } else {
         write!(writer, "Starting fuzzing with harness ")?;
@@ -1279,6 +1325,16 @@
     writer.set_color(ColorSpec::new().set_fg(Some(Color::Blue)))?;
     writeln!(writer, "{}", num_threads)?;
     writer.reset()?;
+
+    if abi_change_detected {
+        writer.set_color(ColorSpec::new().set_fg(Some(Color::Yellow)))?;
+        write!(writer, "ABI change detected. ")?;
+        writeln!(
+            writer,
+            "Some testcases will be skipped due to ABI change and elements from them will be added to the dictionary."
+        )?;
+        writer.reset()?;
+    }
     writer.flush()?;
     Ok(())
 }

tooling/greybox_fuzzer/src/mutation/dictionary.rs

@@ -66,6 +66,43 @@ pub struct FullDictionary {
     field_dictionary: Vec<FieldElement>,
     int_dictionary: IntDictionary,
 }
+/// Parse input value and add elements to the dictionary
+/// We use this when the ABI for a harness has changed and we can no longer directly decode previous testcases,
+/// but want to use interesting values from previous testcases in the fuzzing campaign
+fn add_elements_from_input_value_to_vector(
+    elements_for_dictionary: &mut Vec<FieldElement>,
+    input_value: &InputValue,
+) {
+    match input_value {
+        InputValue::Field(field_element) => {
+            elements_for_dictionary.push(*field_element);
+        }
+        // String bytes are easy to brute force, so we don't add them to the dictionary
+        InputValue::String(_) => (),
+        InputValue::Struct(input_value_map) => {
+            for (_, value) in input_value_map.iter() {
+                add_elements_from_input_value_to_vector(elements_for_dictionary, value);
+            }
+        }
+        InputValue::Vec(input_value_vec) => {
+            for value in input_value_vec.iter() {
+                add_elements_from_input_value_to_vector(elements_for_dictionary, value);
+            }
+        }
+    }
+}
+
+/// Parse input map and add elements to the dictionary
+/// We use this when the ABI for a harness has changed and we can no longer directly decode previous testcases,
+/// but want to use interesting values from previous testcases in the fuzzing campaign
+pub fn add_elements_from_input_map_to_vector_without_abi(
+    input_map: &InputMap,
+    elements_for_dictionary: &mut Vec<FieldElement>,
+) {
+    for (_, value) in input_map.iter() {
+        add_elements_from_input_value_to_vector(elements_for_dictionary, value);
+    }
+}
 
 impl FullDictionary {
     /// Parse input value and collect value(s) for the dictionary from it
@@ -165,6 +202,15 @@ impl FullDictionary {
         self.int_dictionary = IntDictionary::new(&self.field_dictionary);
     }
 
+    /// Update the dictionary with values from a vector of field elements
+    pub fn update_from_vector(&mut self, elements: &[FieldElement]) {
+        let mut testcase_full_dictionary: HashSet<_> =
+            self.field_dictionary.iter().copied().collect();
+        testcase_full_dictionary.extend(elements.iter().copied());
+        self.field_dictionary = testcase_full_dictionary.iter().copied().collect();
+        self.int_dictionary = IntDictionary::new(&self.field_dictionary);
+    }
+
     /// Get a reference to the int dictionary
     pub fn get_int_dictionary(&self) -> &IntDictionary {
         &self.int_dictionary

tooling/greybox_fuzzer/src/mutation/mod.rs

@@ -31,6 +31,7 @@ pub struct InputMutator {
     full_dictionary: FullDictionary,
 }
 
+pub use dictionary::add_elements_from_input_map_to_vector_without_abi;
 const MUTATION_LOG_MIN: u32 = 0;
 const MUTATION_LOG_MAX: u32 = 5;
 /// NodeWeight determines the probability of mutating a particular object
@@ -78,6 +79,11 @@ impl InputMutator {
         self.full_dictionary.update(&self.abi, testcase);
     }
 
+    /// Update the dictionary with values from a vector of field elements
+    pub fn update_dictionary_from_vector(&mut self, elements: &[FieldElement]) {
+        self.full_dictionary.update_from_vector(elements);
+    }
+
     /// Count weights of each element recursively (complex structures return a vector of weights of their most basic elements)
     fn count_single_input_weight(abi_type: &AbiType) -> NodeWeight {
         match abi_type {