fix: Prevent AutoTokenizer type mismatch from directory name substrin…

src/transformers/models/auto/configuration_auto.py

@@ -1421,17 +1421,10 @@ def from_pretrained(cls, pretrained_model_name_or_path: str | os.PathLike[str],
                     "`pip install git+https://github.com/huggingface/transformers.git`"
                 )
             return config_class.from_dict(config_dict, **unused_kwargs)
-        else:
-            # Fallback: use pattern matching on the string.
-            # We go from longer names to shorter names to catch roberta before bert (for instance)
-            for pattern in sorted(CONFIG_MAPPING.keys(), key=len, reverse=True):
-                if pattern in str(pretrained_model_name_or_path):
-                    return CONFIG_MAPPING[pattern].from_dict(config_dict, **unused_kwargs)
 
         raise ValueError(
             f"Unrecognized model in {pretrained_model_name_or_path}. "
-            f"Should have a `model_type` key in its {CONFIG_NAME}, or contain one of the following strings "
-            f"in its name: {', '.join(CONFIG_MAPPING.keys())}"
+            f"Should have a `model_type` key in its {CONFIG_NAME}."
         )
 
     @staticmethod

src/transformers/models/auto/tokenization_auto.py

@@ -645,12 +645,15 @@ def from_pretrained(
             != tokenizer_config_class.replace("Fast", "")
         ):
             # new model, but we ignore it unless the model type is the same
-            try:
-                return TokenizersBackend.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-            except Exception:
-                return tokenizer_class_from_name(tokenizer_config_class).from_pretrained(
-                    pretrained_model_name_or_path, *inputs, **kwargs
-                )
+            if TokenizersBackend is not None:
+                try:
+                    return TokenizersBackend.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+                except Exception as e:
+                    logger.debug(f"Failed to use TokenizersBackend: {e}")
+
+            return tokenizer_class_from_name(tokenizer_config_class).from_pretrained(
+                pretrained_model_name_or_path, *inputs, **kwargs
+            )
 
         if "_commit_hash" in tokenizer_config:
             kwargs["_commit_hash"] = tokenizer_config["_commit_hash"]

tests/models/auto/test_configuration_auto.py

@@ -60,16 +60,6 @@ def test_config_for_model_str(self):
         config = AutoConfig.for_model("roberta")
         self.assertIsInstance(config, RobertaConfig)
 
-    def test_pattern_matching_fallback(self):
-        with tempfile.TemporaryDirectory() as tmp_dir:
-            # This model name contains bert and roberta, but roberta ends up being picked.
-            folder = os.path.join(tmp_dir, "fake-roberta")
-            os.makedirs(folder, exist_ok=True)
-            with open(os.path.join(folder, "config.json"), "w") as f:
-                f.write(json.dumps({}))
-            config = AutoConfig.from_pretrained(folder)
-            self.assertEqual(type(config), RobertaConfig)
-
     def test_new_config_registration(self):
         try:
             AutoConfig.register("custom", CustomConfig)
@@ -148,3 +138,18 @@ class NewModelConfigLocal(BertConfig):
         finally:
             if "new-model" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["new-model"]
+
+    def test_config_missing_model_type(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            config_dict = {
+                "hidden_size": 768,
+                "num_attention_heads": 12,
+                "num_hidden_layers": 12,
+            }
+            config_path = os.path.join(tmp_dir, "config.json")
+
+            with open(config_path, "w") as f:
+                json.dump(config_dict, f)
+
+            with self.assertRaisesRegex(ValueError, "Should have a `model_type` key"):
+                AutoConfig.from_pretrained(tmp_dir)