diff --git a/compiler/noirc_evaluator/src/ssa/ssa_gen/context.rs b/compiler/noirc_evaluator/src/ssa/ssa_gen/context.rs
index 0c6041029da..ddc3365b551 100644
--- a/compiler/noirc_evaluator/src/ssa/ssa_gen/context.rs
+++ b/compiler/noirc_evaluator/src/ssa/ssa_gen/context.rs
@@ -172,6 +172,7 @@ impl<'a> FunctionContext<'a> {
     /// Always returns a Value::Mutable wrapping the allocate instruction.
     pub(super) fn new_mutable_variable(&mut self, value_to_store: ValueId) -> Value {
         let element_type = self.builder.current_function.dfg.type_of_value(value_to_store);
+        self.builder.increment_array_reference_count(value_to_store);
         let alloc = self.builder.insert_allocate(element_type);
         self.builder.insert_store(alloc, value_to_store);
         let typ = self.builder.type_of_value(value_to_store);
@@ -735,7 +736,6 @@ impl<'a> FunctionContext<'a> {
             // Reference counting in brillig relies on us incrementing reference
             // counts when arrays/slices are constructed or indexed.
             // Thus, if we dereference an lvalue which happens to be array/slice we should increment its reference counter.
-            self.builder.increment_array_reference_count(reference);
             self.builder.insert_load(reference, element_type).into()
         })
     }
@@ -916,7 +916,10 @@ impl<'a> FunctionContext<'a> {
         let parameters = self.builder.current_function.dfg.block_parameters(entry).to_vec();
 
         for parameter in parameters {
-            self.builder.increment_array_reference_count(parameter);
+            // Avoid reference counts for immutable arrays that aren't behind references.
+            if self.builder.current_function.dfg.value_is_reference(parameter) {
+                self.builder.increment_array_reference_count(parameter);
+            }
         }
 
         entry
@@ -933,7 +936,9 @@ impl<'a> FunctionContext<'a> {
         dropped_parameters.retain(|parameter| !terminator_args.contains(parameter));
 
         for parameter in dropped_parameters {
-            self.builder.decrement_array_reference_count(parameter);
+            if self.builder.current_function.dfg.value_is_reference(parameter) {
+                self.builder.decrement_array_reference_count(parameter);
+            }
         }
     }
 
diff --git a/compiler/noirc_evaluator/src/ssa/ssa_gen/mod.rs b/compiler/noirc_evaluator/src/ssa/ssa_gen/mod.rs
index c50f0a7f45c..d28236bd360 100644
--- a/compiler/noirc_evaluator/src/ssa/ssa_gen/mod.rs
+++ b/compiler/noirc_evaluator/src/ssa/ssa_gen/mod.rs
@@ -665,12 +665,11 @@ impl<'a> FunctionContext<'a> {
         values = values.map(|value| {
             let value = value.eval(self);
 
-            // Make sure to increment array reference counts on each let binding
-            self.builder.increment_array_reference_count(value);
-
             Tree::Leaf(if let_expr.mutable {
                 self.new_mutable_variable(value)
             } else {
+                // `new_mutable_variable` already increments rcs internally
+                self.builder.increment_array_reference_count(value);
                 value::Value::Normal(value)
             })
         });