feat(greybox_fuzzer): Should_fail and should_fail_with

compiler/noirc_frontend/src/hir/def_map/mod.rs

@@ -438,6 +438,17 @@ impl FuzzingHarness {
         match self.scope {
             FuzzingScope::OnlyFailWith { .. } => true,
             FuzzingScope::None => false,
+            FuzzingScope::ShouldFailWith { .. } => false,
+        }
+    }
+    /// Returns true if the fuzzing harness has been specified to fail
+    /// This is done by annotating the function with `#[fuzz(should_fail)]`
+    /// or `#[fuzz(should_fail_with = "reason")]`
+    pub fn should_fail_enabled(&self) -> bool {
+        match self.scope {
+            FuzzingScope::OnlyFailWith { .. } => false,
+            FuzzingScope::None => false,
+            FuzzingScope::ShouldFailWith { .. } => true,
         }
     }
 
@@ -447,6 +458,7 @@ impl FuzzingHarness {
         match &self.scope {
             FuzzingScope::None => None,
             FuzzingScope::OnlyFailWith { reason } => Some(reason.clone()),
+            FuzzingScope::ShouldFailWith { reason } => reason.clone(),
         }
     }
 }

compiler/noirc_frontend/src/lexer/errors.rs

@@ -125,7 +125,7 @@ impl LexerErrorKind {
             ),
             LexerErrorKind::MalformedFuzzAttribute { location } => (
                 "Malformed fuzz attribute".to_string(),
-                "The fuzz attribute can be written in one of these forms: `#[fuzz]` or `#[fuzz(only_fail_with = \"message\")]`".to_string(),
+                "The fuzz attribute can be written in one of these forms: `#[fuzz]`, `#[fuzz(should_fail)]`, `#[fuzz(should_fail_with = \"message\")]` or `#[fuzz(only_fail_with = \"message\")]`".to_string(),
                 *location,
             ),
             LexerErrorKind::InvalidInnerAttribute { location, found } => (

compiler/noirc_frontend/src/lexer/token.rs

@@ -751,6 +751,12 @@ impl fmt::Display for TestScope {
 /// FuzzingScopr is used to specify additional annotations for fuzzing harnesses
 #[derive(PartialEq, Eq, Hash, Debug, Clone, PartialOrd, Ord)]
 pub enum FuzzingScope {
+    /// If the fuzzing harness has a scope of ShouldFailWith, then it should only pass
+    /// if it fails with the specified reason. If the reason is None, then
+    /// the harness must unconditionally fail
+    ShouldFailWith {
+        reason: Option<String>,
+    },
     /// If a fuzzing harness has a scope of OnlyFailWith, then it will only detect an assert
     /// if it fails with the specified reason.
     OnlyFailWith {
@@ -764,6 +770,10 @@ impl fmt::Display for FuzzingScope {
         match self {
             FuzzingScope::None => write!(f, ""),
             FuzzingScope::OnlyFailWith { reason } => write!(f, "(only_fail_with = {reason:?})"),
+            FuzzingScope::ShouldFailWith { reason } => match reason {
+                Some(failure_reason) => write!(f, "(should_fail_with = {failure_reason:?})"),
+                None => write!(f, "(should_fail)"),
+            },
         }
     }
 }

compiler/noirc_frontend/src/parser/parser/attributes.rs

@@ -58,6 +58,8 @@ impl Parser<'_> {
     ///     | 'test' '(' 'should_fail_with' '=' string ')'
     ///     | 'fuzz'
     ///     | 'fuzz' '(' 'only_fail_with' '=' string ')'
+    ///     | 'fuzz' '(' 'should_fail' ')'
+    ///     | 'fuzz' '(' 'should_fail_with' '=' string ')'
     ///
     /// SecondaryAttribute
     ///     = 'abi' '(' AttributeValue ')'
@@ -147,7 +149,7 @@ impl Parser<'_> {
                     self.parse_test_attribute(start_location)
                 } else if ident.as_str() == "fuzz" {
                     // The fuzz attribute is a secondary attribute that has `a = b` in its syntax
-                    // (`only_fail_with = "..."``) so we parse it differently.
+                    // (`only_fail_with = "..."``) or (`should_fail_with = "..."``) so we parse it differently.
                     self.parse_fuzz_attribute(start_location)
                 } else {
                     // Every other attribute has the form `name(arg1, arg2, .., argN)`
@@ -366,6 +368,15 @@ impl Parser<'_> {
         let scope = if self.eat_left_paren() {
             let scope = if let Some(ident) = self.eat_ident() {
                 match ident.as_str() {
+                    "should_fail" => Some(FuzzingScope::ShouldFailWith { reason: None }),
+                    "should_fail_with" => {
+                        self.eat_or_error(Token::Assign);
+                        if let Some(reason) = self.eat_str() {
+                            Some(FuzzingScope::ShouldFailWith { reason: Some(reason) })
+                        } else {
+                            Some(FuzzingScope::ShouldFailWith { reason: None })
+                        }
+                    }
                     "only_fail_with" => {
                         self.eat_or_error(Token::Assign);
                         self.eat_str().map(|reason| FuzzingScope::OnlyFailWith { reason })

docs/docs/tooling/fuzzing.md

@@ -84,6 +84,34 @@ It is recommended to use `--skip-underconstrained-check` to increase compilation
 
 
 ## Fuzzing more complex programs
+### Using `should_fail` and `should_fail_with`
+
+The fuzzer can be used to fuzz programs that are expected to fail. To do this, you can use the `should_fail` and `should_fail_with` attributes.
+
+The following example will fuzz the program with the `should_fail` attribute, and will only consider a test case as a failure if the program passes:
+```rust
+#[fuzz(should_fail)]
+fn fuzz_should_fail(a: [bool; 32]) {
+    let mut or_sum= false;
+    for i in 0..32 {
+        or_sum=or_sum|(a[i]==((i&1)as bool));
+    }
+    assert(!or_sum);
+}
+```
+
+The `should_fail_with` expects that the program will fail with a specific error message. The following example will fuzz the program with the `should_fail_with` attribute, and will only consider a test case as a failure if the program passes or fails with the message different from "This is the message that will be checked for":
+```rust
+#[fuzz(should_fail_with = "This is the message that will be checked for")]
+fn fuzz_should_fail_with(a: [bool; 32]) {
+    let mut or_sum= false;
+    for i in 0..32 {
+        or_sum=or_sum|(a[i]==((i&1)as bool));
+    }
+    assert(or_sum);
+    assert(false, "This is the message that will be checked for");
+}
+```
 
 ### Using `only_fail_with`
 A lot of the time, the program will already have many expected assertions that would lead to a failing test case, for example:
@@ -94,7 +122,7 @@ fn fuzz_add(a: u64, b: u64) {
     assert((a+b-15)!=(a-b+30));
 }
 ```
-Using integer arithmetic will often automatically lead to overflows and underflows, which will lead to a failing test case. To avoid this we can specify and "only_fail_with" attribute to the fuzzing harness, which will only mark a testcase as failing if the assertion contains a specific message:
+Using integer arithmetic will often automatically lead to overflows and underflows, which will lead to a failing test case. If we want to check that a specific property is broken, rather than detect all failures, we can specify an "only_fail_with" attribute to the fuzzing harness, which will only mark a testcase as failing if the assertion contains a specific message:
 
 ```rust
 #[fuzz(only_fail_with = "This is the message that will be checked for")]

test_programs/fuzzing_failure/partial_witness_should_fail/Nargo.toml

@@ -0,0 +1,3 @@
+[package]
+name = "partial_witness_should_fail"
+type = "bin"

test_programs/fuzzing_failure/partial_witness_should_fail/src/main.nr

@@ -0,0 +1,107 @@
+#[fuzz(should_fail)]
+// This program is expected to fail, unless one of two combinations of inputs is provided (all true or all but 0th input is true)
+// The fuzzer should find the case where the program does not fail. It is important that the program has 2 conditions, otherwise the compiler will solve the single condition for brillig
+// and the fuzzer without partial witness parsing for ACIR will find the solution trivially.
+fn fuzz_partial_witness_fail(
+    a0: bool,
+    a1: bool,
+    a2: bool,
+    a3: bool,
+    a4: bool,
+    a5: bool,
+    a6: bool,
+    a7: bool,
+    a8: bool,
+    a9: bool,
+    a10: bool,
+    a11: bool,
+    a12: bool,
+    a13: bool,
+    a14: bool,
+    a15: bool,
+    a16: bool,
+    a17: bool,
+    a18: bool,
+    a19: bool,
+    a20: bool,
+    a21: bool,
+    a22: bool,
+    a23: bool,
+    a24: bool,
+    a25: bool,
+    a26: bool,
+    a27: bool,
+    a28: bool,
+    a29: bool,
+    a30: bool,
+    a31: bool,
+) {
+    assert(
+        a0
+            & a1
+            & a2
+            & a3
+            & a4
+            & a5
+            & a6
+            & a7
+            & a8
+            & a9
+            & a10
+            & a11
+            & a12
+            & a13
+            & a14
+            & a15
+            & a16
+            & a17
+            & a18
+            & a19
+            & a20
+            & a21
+            & a22
+            & a23
+            & a24
+            & a25
+            & a26
+            & a27
+            & a28
+            & a29
+            & a30
+            & a31
+            | (
+                !a0
+                    & a1
+                    & a2
+                    & a3
+                    & a4
+                    & a5
+                    & a6
+                    & a7
+                    & a8
+                    & a9
+                    & a10
+                    & a11
+                    & a12
+                    & a13
+                    & a14
+                    & a15
+                    & a16
+                    & a17
+                    & a18
+                    & a19
+                    & a20
+                    & a21
+                    & a22
+                    & a23
+                    & a24
+                    & a25
+                    & a26
+                    & a27
+                    & a28
+                    & a29
+                    & a30
+                    & a31
+            ),
+    );
+}

test_programs/fuzzing_failure/should_fail/Nargo.toml

@@ -0,0 +1,3 @@
+[package]
+name = "should_fail"
+type = "bin"

test_programs/fuzzing_failure/should_fail/src/main.nr

@@ -0,0 +1,59 @@
+// This program is expected to fail unless all the inputs are true, which is the cast that the fuzzer should find
+#[fuzz(should_fail)]
+fn fuzz(
+    a_00: bool,
+    a_01: bool,
+    a_02: bool,
+    a_03: bool,
+    a_04: bool,
+    a_05: bool,
+    a_06: bool,
+    a_07: bool,
+    a_08: bool,
+    a_09: bool,
+    a_10: bool,
+    a_11: bool,
+    a_12: bool,
+    a_13: bool,
+    a_14: bool,
+    a_15: bool,
+    a_16: bool,
+    a_17: bool,
+    a_18: bool,
+    a_19: bool,
+    a_20: bool,
+    a_21: bool,
+    a_22: bool,
+    a_23: bool,
+    a_24: bool,
+) {
+    assert(
+        (
+            a_00
+                & a_01
+                & a_02
+                & a_03
+                & a_04
+                & a_05
+                & a_06
+                & a_07
+                & a_08
+                & a_09
+                & a_10
+                & a_11
+                & a_12
+                & a_13
+                & a_14
+                & a_15
+                & a_16
+                & a_17
+                & a_18
+                & a_19
+                & a_20
+                & a_21
+                & a_22
+                & a_23
+                & a_24
+        ),
+    );
+}

test_programs/fuzzing_failure/should_fail_with/Nargo.toml

@@ -0,0 +1,3 @@
+[package]
+name = "should_fail"
+type = "bin"

test_programs/fuzzing_failure/should_fail_with/src/main.nr

@@ -0,0 +1,6 @@
+// This program expects the non-deadbeef failure message, so the deadbeef case is treated as an actual failure
+#[fuzz(should_fail_with = "input is not 0xdeadbeef")]
+fn fuzz(input: u64) {
+    assert(input == 0xdeadbeef, "input is not 0xdeadbeef");
+    assert(input != 0xdeadbeef, "input is 0xdeadbeef");
+}