diff --git a/src/accelerate/utils/modeling.py b/src/accelerate/utils/modeling.py
index c1483355716..e471731a4e9 100644
--- a/src/accelerate/utils/modeling.py
+++ b/src/accelerate/utils/modeling.py
@@ -476,7 +476,11 @@ def infer_auto_device_map(
     current_memory_used = 0
 
     # Direct submodules and parameters
-    modules_to_treat = list(model.named_parameters(recurse=False)) + list(model.named_children())
+    modules_to_treat = (
+        list(model.named_parameters(recurse=False))
+        + list(model.named_children())
+        + list(model.named_buffers(recurse=False))
+    )
     # Initialize maximum largest layer, to know which space to keep in memory
     max_layer_size, max_layer_names = get_max_layer_size(modules_to_treat, module_sizes, no_split_module_classes)