diff --git a/compiler/rustc_typeck/src/check/fn_ctxt/checks.rs b/compiler/rustc_typeck/src/check/fn_ctxt/checks.rs
index 7e225bf43b900..75976ebdf2822 100644
--- a/compiler/rustc_typeck/src/check/fn_ctxt/checks.rs
+++ b/compiler/rustc_typeck/src/check/fn_ctxt/checks.rs
@@ -429,9 +429,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             errors.drain_filter(|error| {
                 let Error::Invalid(input_idx, Compatibility::Incompatible(error)) = error else { return false };
                 let expected_ty = expected_input_tys[*input_idx];
-                let provided_ty = final_arg_types[*input_idx].map(|ty| ty.0).unwrap();
+                let Some(Some((provided_ty, _))) = final_arg_types.get(*input_idx) else { return false };
                 let cause = &self.misc(provided_args[*input_idx].span);
-                let trace = TypeTrace::types(cause, true, expected_ty, provided_ty);
+                let trace = TypeTrace::types(cause, true, expected_ty, *provided_ty);
                 if let Some(e) = error {
                     if !matches!(trace.cause.as_failure_code(e), FailureCode::Error0308(_)) {
                         self.report_and_explain_type_error(trace, e).emit();
@@ -679,8 +679,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     Error::Invalid(input_idx, compatibility) => {
                         let expected_ty = expected_input_tys[input_idx];
                         if let Compatibility::Incompatible(error) = &compatibility {
-                            let provided_ty = final_arg_types[input_idx].map(|ty| ty.0).unwrap();
-                            let cause = &self.misc(provided_args[input_idx].span);
+                            let provided_ty = final_arg_types
+                                .get(input_idx)
+                                .and_then(|x| x.as_ref())
+                                .map(|ty| ty.0)
+                                .unwrap_or(tcx.ty_error());
+                            let cause = &self.misc(
+                                provided_args.get(input_idx).map(|i| i.span).unwrap_or(call_span),
+                            );
                             let trace = TypeTrace::types(cause, true, expected_ty, provided_ty);
                             if let Some(e) = error {
                                 self.note_type_err(
@@ -695,14 +701,16 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             }
                         }
 
-                        self.emit_coerce_suggestions(
-                            &mut err,
-                            &provided_args[input_idx],
-                            final_arg_types[input_idx].map(|ty| ty.0).unwrap(),
-                            final_arg_types[input_idx].map(|ty| ty.1).unwrap(),
-                            None,
-                            None,
-                        );
+                        if let Some(expr) = provided_args.get(input_idx) {
+                            self.emit_coerce_suggestions(
+                                &mut err,
+                                &expr,
+                                final_arg_types[input_idx].map(|ty| ty.0).unwrap(),
+                                final_arg_types[input_idx].map(|ty| ty.1).unwrap(),
+                                None,
+                                None,
+                            );
+                        }
                     }
                     Error::Extra(arg_idx) => {
                         let arg_type = if let Some((_, ty)) = final_arg_types[arg_idx] {
diff --git a/src/test/ui/argument-suggestions/issue-96638.rs b/src/test/ui/argument-suggestions/issue-96638.rs
new file mode 100644
index 0000000000000..9c6e81ab8cc75
--- /dev/null
+++ b/src/test/ui/argument-suggestions/issue-96638.rs
@@ -0,0 +1,9 @@
+fn f(_: usize, _: &usize, _: usize) {}
+
+fn arg<T>() -> T { todo!() }
+
+fn main() {
+    let x = arg(); // `x` must be inferred
+    // The reference on `&x` is important to reproduce the ICE
+    f(&x, ""); //~ ERROR this function takes 3 arguments but 2 arguments were supplied
+}
diff --git a/src/test/ui/argument-suggestions/issue-96638.stderr b/src/test/ui/argument-suggestions/issue-96638.stderr
new file mode 100644
index 0000000000000..35190e2ca0d82
--- /dev/null
+++ b/src/test/ui/argument-suggestions/issue-96638.stderr
@@ -0,0 +1,19 @@
+error[E0061]: this function takes 3 arguments but 2 arguments were supplied
+  --> $DIR/issue-96638.rs:8:5
+   |
+LL |     f(&x, "");
+   |     ^ -- an argument of type `usize` is missing
+   |
+note: function defined here
+  --> $DIR/issue-96638.rs:1:4
+   |
+LL | fn f(_: usize, _: &usize, _: usize) {}
+   |    ^ --------  ---------  --------
+help: provide the argument
+   |
+LL |     f({usize}, &x, {usize});
+   |     ~~~~~~~~~~~~~~~~~~~~~~~
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0061`.