diff --git a/crates/oxc_semantic/src/builder.rs b/crates/oxc_semantic/src/builder.rs
index 13e2137a140d5..3c4a01de765b0 100644
--- a/crates/oxc_semantic/src/builder.rs
+++ b/crates/oxc_semantic/src/builder.rs
@@ -9,6 +9,7 @@ use oxc_cfg::{
     IterationInstructionKind, ReturnInstructionKind,
 };
 use oxc_diagnostics::OxcDiagnostic;
+use oxc_index::{index_vec, IndexVec};
 use oxc_span::{CompactStr, SourceType, Span};
 use oxc_syntax::{module_record::ModuleRecord, operator::AssignmentOperator};
 
@@ -22,7 +23,7 @@ use crate::{
     module_record::ModuleRecordBuilder,
     node::{AstNode, AstNodeId, AstNodes, NodeFlags},
     reference::{Reference, ReferenceFlag, ReferenceId},
-    scope::{ScopeFlags, ScopeId, ScopeTree},
+    scope::{ScopeFlags, ScopeId, ScopeTree, UnresolvedReferences},
     symbol::{SymbolFlags, SymbolId, SymbolTable},
     JSDocFinder, Semantic,
 };
@@ -65,6 +66,9 @@ pub struct SemanticBuilder<'a> {
     pub nodes: AstNodes<'a>,
     pub scope: ScopeTree,
     pub symbols: SymbolTable,
+    /// NOTE(lucab): lazy vector, always access this through the
+    /// `unresolved_references_by_scope()` helper.
+    unresolved_references: IndexVec<ScopeId, UnresolvedReferences>,
 
     pub(crate) module_record: Arc<ModuleRecord>,
 
@@ -108,6 +112,7 @@ impl<'a> SemanticBuilder<'a> {
             nodes: AstNodes::default(),
             scope,
             symbols: SymbolTable::default(),
+            unresolved_references: index_vec![UnresolvedReferences::default()],
             module_record: Arc::new(ModuleRecord::default()),
             label_builder: LabelBuilder::default(),
             build_jsdoc: false,
@@ -174,6 +179,8 @@ impl<'a> SemanticBuilder<'a> {
                 checker::check_module_record(&self);
             }
         }
+        self.scope.root_unresolved_references =
+            self.unresolved_references.swap_remove(self.scope.root_scope_id());
 
         let jsdoc = if self.build_jsdoc { self.jsdoc.build() } else { JSDocFinder::default() };
 
@@ -317,7 +324,11 @@ impl<'a> SemanticBuilder<'a> {
     pub fn declare_reference(&mut self, reference: Reference) -> ReferenceId {
         let reference_name = reference.name().clone();
         let reference_id = self.symbols.create_reference(reference);
-        self.scope.add_unresolved_reference(self.current_scope_id, reference_name, reference_id);
+
+        self.unresolved_references_by_scope(self.current_scope_id)
+            .entry(reference_name)
+            .or_default()
+            .push(reference_id);
         reference_id
     }
 
@@ -340,8 +351,7 @@ impl<'a> SemanticBuilder<'a> {
 
     fn resolve_references_for_current_scope(&mut self) {
         let all_references = self
-            .scope
-            .unresolved_references_mut(self.current_scope_id)
+            .unresolved_references_by_scope(self.current_scope_id)
             .drain()
             .collect::<Vec<(_, Vec<_>)>>();
 
@@ -360,7 +370,10 @@ impl<'a> SemanticBuilder<'a> {
             }
             self.symbols.resolved_references[symbol_id].extend(reference_ids);
         } else {
-            self.scope.extend_unresolved_reference(parent_scope_id, name, reference_ids);
+            self.unresolved_references_by_scope(parent_scope_id)
+                .entry(name)
+                .or_default()
+                .extend(reference_ids);
         }
     }
 
@@ -392,6 +405,14 @@ impl<'a> SemanticBuilder<'a> {
             self.symbols.union_flag(symbol_id, SymbolFlags::Export);
         }
     }
+
+    fn unresolved_references_by_scope(&mut self, scope_id: ScopeId) -> &mut UnresolvedReferences {
+        let min_new_len = scope_id + 1;
+        if self.unresolved_references.len() < min_new_len {
+            self.unresolved_references.resize_with(min_new_len.into(), Default::default);
+        }
+        &mut self.unresolved_references[scope_id]
+    }
 }
 
 impl<'a> Visit<'a> for SemanticBuilder<'a> {
diff --git a/crates/oxc_semantic/src/scope.rs b/crates/oxc_semantic/src/scope.rs
index 2698f90bef93d..c671b5ab4e895 100644
--- a/crates/oxc_semantic/src/scope.rs
+++ b/crates/oxc_semantic/src/scope.rs
@@ -11,7 +11,7 @@ use crate::{reference::ReferenceId, symbol::SymbolId, AstNodeId};
 type FxIndexMap<K, V> = IndexMap<K, V, BuildHasherDefault<FxHasher>>;
 
 type Bindings = FxIndexMap<CompactStr, SymbolId>;
-type UnresolvedReferences = FxHashMap<CompactStr, Vec<ReferenceId>>;
+pub(crate) type UnresolvedReferences = FxHashMap<CompactStr, Vec<ReferenceId>>;
 
 /// Scope Tree
 ///
@@ -27,7 +27,7 @@ pub struct ScopeTree {
     node_ids: FxHashMap<ScopeId, AstNodeId>,
     flags: IndexVec<ScopeId, ScopeFlags>,
     bindings: IndexVec<ScopeId, Bindings>,
-    unresolved_references: IndexVec<ScopeId, UnresolvedReferences>,
+    pub(crate) root_unresolved_references: UnresolvedReferences,
 }
 
 impl ScopeTree {
@@ -88,7 +88,7 @@ impl ScopeTree {
     }
 
     pub fn root_unresolved_references(&self) -> &UnresolvedReferences {
-        &self.unresolved_references[self.root_scope_id()]
+        &self.root_unresolved_references
     }
 
     pub fn get_flags(&self, scope_id: ScopeId) -> ScopeFlags {
@@ -142,7 +142,7 @@ impl ScopeTree {
     }
 
     pub fn add_root_unresolved_reference(&mut self, name: CompactStr, reference_id: ReferenceId) {
-        self.add_unresolved_reference(self.root_scope_id(), name, reference_id);
+        self.root_unresolved_references.entry(name).or_default().push(reference_id);
     }
 
     pub fn has_binding(&self, scope_id: ScopeId, name: &str) -> bool {
@@ -184,7 +184,6 @@ impl ScopeTree {
         let scope_id = self.parent_ids.push(parent_id);
         _ = self.flags.push(flags);
         _ = self.bindings.push(Bindings::default());
-        _ = self.unresolved_references.push(UnresolvedReferences::default());
 
         if let Some(parent_id) = parent_id {
             self.child_ids.entry(parent_id).or_default().push(scope_id);
@@ -204,29 +203,4 @@ impl ScopeTree {
     pub fn remove_binding(&mut self, scope_id: ScopeId, name: &CompactStr) {
         self.bindings[scope_id].shift_remove(name);
     }
-
-    pub(crate) fn add_unresolved_reference(
-        &mut self,
-        scope_id: ScopeId,
-        name: CompactStr,
-        reference_id: ReferenceId,
-    ) {
-        self.unresolved_references[scope_id].entry(name).or_default().push(reference_id);
-    }
-
-    pub(crate) fn extend_unresolved_reference(
-        &mut self,
-        scope_id: ScopeId,
-        name: CompactStr,
-        reference_ids: Vec<ReferenceId>,
-    ) {
-        self.unresolved_references[scope_id].entry(name).or_default().extend(reference_ids);
-    }
-
-    pub(crate) fn unresolved_references_mut(
-        &mut self,
-        scope_id: ScopeId,
-    ) -> &mut UnresolvedReferences {
-        &mut self.unresolved_references[scope_id]
-    }
 }