diff --git a/src/transformers/models/auto/image_processing_auto.py b/src/transformers/models/auto/image_processing_auto.py
index 2a9e2720106e..1e868161c9f2 100644
--- a/src/transformers/models/auto/image_processing_auto.py
+++ b/src/transformers/models/auto/image_processing_auto.py
@@ -725,7 +725,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
         has_remote_code = image_processor_auto_map is not None
         has_local_code = image_processor_class is not None or type(config) in IMAGE_PROCESSOR_MAPPING
         explicit_local_code = has_local_code and not (
-            image_processor_class or IMAGE_PROCESSOR_MAPPING[type(config)]
+            image_processor_class or _load_class_with_fallback(IMAGE_PROCESSOR_MAPPING[type(config)], backend)
         ).__module__.startswith("transformers.")
         if has_remote_code:
             class_ref = _resolve_auto_map_class_ref(image_processor_auto_map, backend)
diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py
index 0e50f2a0041d..2ccae521e558 100644
--- a/src/transformers/models/auto/tokenization_auto.py
+++ b/src/transformers/models/auto/tokenization_auto.py
@@ -738,11 +738,15 @@ def from_pretrained(
                 or tokenizer_class_from_name(tokenizer_config_class + "Fast") is not None
             )
         )
-        explicit_local_code = has_local_code and (
-            tokenizer_config_class is not None
-            and not (
-                tokenizer_class_from_name(tokenizer_config_class).__module__.startswith("transformers.")
-                and tokenizer_class_from_name(tokenizer_config_class + "Fast").__module__.startswith("transformers.")
+        explicit_local_code = (
+            has_local_code
+            and type(config) not in TOKENIZER_MAPPING
+            and (
+                tokenizer_config_class is not None
+                and not (
+                    tokenizer_class_from_name(tokenizer_config_class)
+                    or tokenizer_class_from_name(tokenizer_config_class + "Fast")
+                ).__module__.startswith("transformers.")
             )
         )
         # V5: Skip remote tokenizer for custom models with incorrect hub tokenizer class