diff --git a/crates/oxc_semantic/src/builder.rs b/crates/oxc_semantic/src/builder.rs
index 3dff6832897a9..4d9d119b96484 100644
--- a/crates/oxc_semantic/src/builder.rs
+++ b/crates/oxc_semantic/src/builder.rs
@@ -256,7 +256,7 @@ impl<'a> SemanticBuilder<'a> {
             .unresolved_references
             .into_root()
             .into_iter()
-            .map(|(k, v)| (k.into(), v.into_iter().map(|(reference_id, _)| reference_id).collect()))
+            .map(|(k, v)| (k.into(), v))
             .collect();
 
         let jsdoc = if self.build_jsdoc { self.jsdoc.build() } else { JSDocFinder::default() };
@@ -425,14 +425,9 @@ impl<'a> SemanticBuilder<'a> {
         name: Atom<'a>,
         reference: Reference,
     ) -> ReferenceId {
-        let reference_flag = *reference.flag();
         let reference_id = self.symbols.create_reference(reference);
 
-        self.unresolved_references
-            .current_mut()
-            .entry(name)
-            .or_default()
-            .push((reference_id, reference_flag));
+        self.unresolved_references.current_mut().entry(name).or_default().push(reference_id);
         reference_id
     }
 
@@ -476,12 +471,13 @@ impl<'a> SemanticBuilder<'a> {
                 // Reserve space for all references to avoid reallocations.
                 resolved_references.reserve(references.len());
 
-                references.retain(|(id, flag)| {
+                references.retain(|&reference_id| {
+                    let reference = &mut self.symbols.references[reference_id];
+                    let flag = *reference.flag();
                     if flag.is_type() && symbol_flag.can_be_referenced_by_type()
                         || flag.is_value() && symbol_flag.can_be_referenced_by_value()
                         || flag.is_ts_type_query() && symbol_flag.is_import()
                     {
-                        let reference = &mut self.symbols.references[*id];
                         // The non type-only ExportSpecifier can reference a type/value symbol,
                         // If the symbol is a value symbol and reference flag is not type-only, remove the type flag.
                         if symbol_flag.is_value() && !flag.is_type_only() {
@@ -500,7 +496,7 @@ impl<'a> SemanticBuilder<'a> {
                         }
 
                         reference.set_symbol_id(symbol_id);
-                        resolved_references.push(*id);
+                        resolved_references.push(reference_id);
                         false
                     } else {
                         true
diff --git a/crates/oxc_semantic/src/unresolved_stack.rs b/crates/oxc_semantic/src/unresolved_stack.rs
index 4e1ddf5f83167..d9500ce92770b 100644
--- a/crates/oxc_semantic/src/unresolved_stack.rs
+++ b/crates/oxc_semantic/src/unresolved_stack.rs
@@ -2,10 +2,10 @@ use assert_unchecked::assert_unchecked;
 use oxc_span::Atom;
 use rustc_hash::FxHashMap;
 
-use oxc_syntax::reference::{ReferenceFlag, ReferenceId};
+use oxc_syntax::reference::ReferenceId;
 
 /// The difference with Scope's `UnresolvedReferences` is that this type uses Atom as the key. its clone is very cheap!
-type TempUnresolvedReferences<'a> = FxHashMap<Atom<'a>, Vec<(ReferenceId, ReferenceFlag)>>;
+type TempUnresolvedReferences<'a> = FxHashMap<Atom<'a>, Vec<ReferenceId>>;
 
 // Stack used to accumulate unresolved refs while traversing scopes.
 // Indexed by scope depth. We recycle `UnresolvedReferences` instances during traversal