fix: Fix barrett reduction bug

src/fns/unconstrained_helpers.nr

@@ -214,7 +214,7 @@ pub(crate) unconstrained fn __barrett_reduction<let N: u32>(
             mulout[i + j] += x[i] * redc_param[j];
         }
     }
-    mulout = split_bits::__normalize_limbs(mulout, 3 * N - 2);
+    mulout = split_bits::__normalize_limbs(mulout, 3 * N - 1);
     let mulout_u60: U60Repr<N, 6> = U60Repr::new(mulout);
 
     // When we apply the barrett reduction, the maximum value of the output will be

src/tests/runtime_bignum_test.nr

@@ -1,3 +1,4 @@
+use crate::fns::unconstrained_helpers::__barrett_reduction;
 use crate::params::{BigNumParams, BigNumParamsGetter};
 use crate::runtime_bignum::RuntimeBigNum;
 use crate::utils::u60_representation::U60Repr;
@@ -682,3 +683,65 @@ fn test_sqrt_BN() {
 
     assert(sqrt_x * sqrt_x == x);
 }
+
+// N.B. this test is only here to check that the barrett reduction is working correctly
+//      a bug was introduced in 0.3.7 that caused issues with the RSA library
+//      this test is to check that the bug is fixed
+#[test]
+fn test_barrett_reduction_fix() {
+    // the parameters and input that the barrett reduction fails on in the RSA library
+    let params: BigNumParams<9, 1024> = BigNumParams::new(
+        false,
+        [
+            0xab238ad9cb37979a43aefbf10be8fb,
+            0x31347febe45fe8c2dac1dd30900704,
+            0xa5a9a6b9cd0cc2b9d13bbd4e068263,
+            0x5eac6390f7873fe97ff9bb14a173ea,
+            0xbc41f700c91fd733a2c63177bbdbd4,
+            0x41442bd58769a3595b659a2ec9c6be,
+            0x4ddc91395f330382aa2e2d3fbe147,
+            0x3d008ff255a0bc71c7887f5728ba1,
+            0xb640c3a8f511c64e,
+        ],
+        [
+            0x5d53d2634c6a0918266043968ce263,
+            0x5dd4be3dce0323a492ee9340aec4db,
+            0xf82d0e2e5c8319f01a460c72c01854,
+            0x236e6fc6e62e8a1d522acda5fb3892,
+            0xdaf755619d66e580901aa224d03174,
+            0x8366291616480e7e1f202dbcedda87,
+            0x40ba1202537d1e94561ccc05265586,
+            0x69b993d857ba89ea5de9822aeb4b93,
+            0x167968c0000761a273,
+        ],
+    );
+    let to_reduce: [Field; 18] = [
+        0xc4b8aacb43f12eb7f3f356ca26fb91,
+        0xfc57d87e81f39810ffb63957395499,
+        0x8308e90fc38cfe95d2a0f7ab577c17,
+        0xa08ec48b11c8f8106bf59274163f0e,
+        0x31d13967c472c9a02fc8099199432c,
+        0x8b9ed833878dcb05c9dd19b917fb56,
+        0xadd411af43c4dc57d5e1e50800a644,
+        0x9075bad2ae6053c55d96ae819c6dfe,
+        0xebe4650d4f5449bbdd48b322ec0c65,
+        0xf17bc78e881061f738f27e058be410,
+        0x1a6d1af41bdb753888b2757986a131,
+        0x06dc4cf83a7ceebe8e996e4329c63a,
+        0xf64759a546537a899c8e2e7ea745fb,
+        0x253f1a623444aa404f989ca5843cf7,
+        0x65728e80aa429c676336b3a0613188,
+        0x189d0a1000d5b3755d91e0455137fe,
+        0x17f512a1cf7cc3470fc054743ceafe,
+        0x05,
+    ];
+
+    let (quotient, remainder) = __barrett_reduction(
+        to_reduce,
+        params.redc_param,
+        1024,
+        params.modulus,
+        params.modulus_u60_x4,
+    );
+    assert(remainder[8].lt(params.modulus[8]));
+}