diff --git a/Makefile b/Makefile index c10e05fd9939..9c27af9a31c3 100644 --- a/Makefile +++ b/Makefile @@ -20,12 +20,12 @@ REPO_CONSISTENCY_CHECKERS := \ import_complexity, \ copies, \ modular_conversion, \ + inits, \ doc_toc, \ modeling_rules_doc, \ docstrings, \ dummies, \ repo, \ - inits, \ pipeline_typing, \ config_docstrings, \ config_attributes, \ diff --git a/docs/source/en/pr_checks.md b/docs/source/en/pr_checks.md index 1aef4db1c0e1..981bfff8fdcc 100644 --- a/docs/source/en/pr_checks.md +++ b/docs/source/en/pr_checks.md @@ -72,7 +72,7 @@ The repository consistency check is similar to `make check-repo`, except it stop | Category | What it validates | Auto-fixed? | |---|---|---| -| Init files | Every new public object must appear in both `_import_structure` (lazy loading) and the `if TYPE_CHECKING` block (type checker imports) in `__init__.py` | Manual | +| Init files | `src/transformers/models/__init__.py` matches the import structure on disk, so the `if TYPE_CHECKING` block (type checker imports) exposes the same models as the lazy runtime half | `make fix-repo` | | Copies and modular | `# Copied from` blocks match their source and modular-generated files are up to date | `make fix-repo` | | Docstrings and docs | Argument docstrings match function signatures and documentation table of contents | `make fix-repo` | | Auto-generated files | Dummies, pipeline typing, doctest list, metadata, dependency table | `make fix-repo` | diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index 073d59ef3471..3f97cd24e765 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -1,3 +1,9 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from the model directories in `src/transformers/models`. +# Do NOT edit this file manually as any edits will be overwritten by auto-generation of the file. +# A model is picked up once one of its modules contains an `__all__`. +# Regenerate the file with: `python utils/check_inits.py --fix_and_overwrite` +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # Copyright 2020 The HuggingFace Team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -81,7 +87,7 @@ from .convnext import * from .convnextv2 import * from .cosmos3_edge import * - from .cosmos3_reasoner import * + from .cosmos3_omni import * from .cpm import * from .cpmant import * from .csm import * @@ -106,12 +112,10 @@ from .deformable_detr import * from .deimv2 import * from .deit import * - from .deprecated import * from .depth_anything import * from .depth_pro import * from .detr import * from .dia import * - from .dialogpt import * from .diffllama import * from .diffusion_gemma import * from .dinat import * @@ -120,7 +124,6 @@ from .dinov3_convnext import * from .dinov3_vit import * from .distilbert import * - from .dit import * from .doge import * from .donut import * from .dots1 import * @@ -141,6 +144,7 @@ from .ernie4_5_moe import * from .ernie4_5_vl_moe import * from .esm import * + from .eurobert import * from .evolla import * from .exaone4 import * from .exaone4_5 import * @@ -207,9 +211,11 @@ from .hiera import * from .higgs_audio_v2 import * from .higgs_audio_v2_tokenizer import * + from .hrm_text import * from .hubert import * from .hunyuan_v1_dense import * from .hunyuan_v1_moe import * + from .hunyuan_vl import * from .hy_v3 import * from .hyperclovax import * from .ibert import * @@ -244,6 +250,7 @@ from .lfm2_moe import * from .lfm2_vl import * from .lightglue import * + from .lighton_ocr import * from .lilt import * from .llama import * from .llama4 import * @@ -267,7 +274,6 @@ from .mbart import * from .mbart50 import * from .megatron_bert import * - from .megatron_gpt2 import * from .mellum import * from .metaclip_2 import * from .mgp_str import * @@ -310,6 +316,8 @@ from .myt5 import * from .nanochat import * from .nemotron import * + from .nemotron3_5_asr import * + from .nemotron_asr_streaming import * from .nemotron_h import * from .nllb import * from .nllb_moe import * @@ -348,7 +356,6 @@ from .phimoe import * from .phobert import * from .pi0 import * - from .pi0_fast import * from .pix2struct import * from .pixio import * from .pixtral import * @@ -358,11 +365,14 @@ from .pp_chart2table import * from .pp_doclayout_v2 import * from .pp_doclayout_v3 import * + from .pp_formulanet import * from .pp_lcnet import * from .pp_lcnet_v3 import * from .pp_lcnet_v4 import * from .pp_ocrv5_mobile_det import * + from .pp_ocrv5_mobile_rec import * from .pp_ocrv5_server_det import * + from .pp_ocrv5_server_rec import * from .pp_ocrv6_medium_det import * from .pp_ocrv6_small_det import * from .pp_ocrv6_small_rec import * @@ -381,6 +391,7 @@ from .qwen3 import * from .qwen3_5 import * from .qwen3_5_moe import * + from .qwen3_asr import * from .qwen3_moe import * from .qwen3_next import * from .qwen3_omni_moe import * @@ -452,6 +463,8 @@ from .timesformer import * from .timm_backbone import * from .timm_wrapper import * + from .tipsv2 import * + from .tipsv2_dpt import * from .trocr import * from .tvp import * from .udop import * @@ -462,6 +475,7 @@ from .upernet import * from .uvdoc import * from .vaultgemma import * + from .vibevoice_acoustic_tokenizer import * from .vibevoice_asr import * from .video_llama_3 import * from .video_llava import * diff --git a/utils/check_inits.py b/utils/check_inits.py index 0d6092416c2c..5272b962d9b3 100644 --- a/utils/check_inits.py +++ b/utils/check_inits.py @@ -1,4 +1,4 @@ -# Copyright 2020 The HuggingFace Inc. team. +# Copyright 2026 The HuggingFace Inc. team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,348 +12,131 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -Utility that checks the custom inits of Transformers are well-defined: Transformers uses init files that delay the -import of an object to when it's actually needed. This is to avoid the main init importing all models, which would -make the line `import transformers` very slow when the user has all optional dependencies installed. The inits with -delayed imports have two halves: one defining a dictionary `_import_structure` which maps modules to the name of the -objects in each module, and one in `TYPE_CHECKING` which looks like a normal init for type-checkers. The goal of this -script is to check the objects defined in both halves are the same. +Utility that regenerates `src/transformers/models/__init__.py` from the import structure on disk. -This also checks the main init properly references all submodules, even if it doesn't import anything from them: every -submodule should be defined as a key of `_import_structure`, with an empty list as value potentially, or the submodule -won't be importable. +The `TYPE_CHECKING` half of that init only exists to give type checkers a static view of what the +`_LazyModule` half exposes at runtime, so both halves are derived from the same `define_import_structure` +call. Hand-writing the `TYPE_CHECKING` half lets new models go missing and removed ones linger; runtime +is unaffected either way, so the `imports` checker cannot catch it. -Use from the root of the repo with: +Usage (from the root of the repo): + +Check that the init is up to date (used in `make check-repo`): ```bash python utils/check_inits.py ``` -for a check that will error in case of inconsistencies (used by `make check-repo`). +Regenerate it if needed (used in `make fix-repo`): -There is no auto-fix possible here sadly :-( +```bash +python utils/check_inits.py --fix_and_overwrite +``` """ -import collections -import os +import argparse +import difflib import re from pathlib import Path +from transformers.utils.import_utils import define_import_structure + CHECKER_CONFIG = { "name": "inits", - "label": "Init files", - "cache_globs": ["src/transformers/**/__init__.py"], + "label": "Model init files", + "cache_globs": ["src/transformers/models/**/*.py"], "check_args": [], - "fix_args": None, + "fix_args": ["--fix_and_overwrite"], } -# Path is set with the intent you should run this script from the root of the repo. -PATH_TO_TRANSFORMERS = "src/transformers" - - -# Matches is_xxx_available() -_re_backend = re.compile(r"is\_([a-z_]*)_available()") -# Catches a one-line _import_struct = {xxx} -_re_one_line_import_struct = re.compile(r"^_import_structure\s+=\s+\{([^\}]+)\}") -# Catches a line with a key-values pattern: "bla": ["foo", "bar"] -_re_import_struct_key_value = re.compile(r'\s+"\S*":\s+\[([^\]]*)\]') -# Catches a line if not is_foo_available -_re_test_backend = re.compile(r"^\s*if\s+not\s+is\_[a-z_]*\_available\(\)") -# Catches a line _import_struct["bla"].append("foo") -_re_import_struct_add_one = re.compile(r'^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)') -# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"] -_re_import_struct_add_many = re.compile(r"^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]") -# Catches a line with an object between quotes and a comma: "MyModel", -_re_quote_object = re.compile(r'^\s+"([^"]+)",') -# Catches a line with objects between brackets only: ["foo", "bar"], -_re_between_brackets = re.compile(r"^\s+\[([^\]]+)\]") -# Catches a line with from foo import bar, bla, boo -_re_import = re.compile(r"\s+from\s+\S*\s+import\s+([^\(\s].*)\n") -# Catches a line with try: -_re_try = re.compile(r"^\s*try:") -# Catches a line with else: -_re_else = re.compile(r"^\s*else:") - - -def find_backend(line: str) -> str | None: - """ - Find one (or multiple) backend in a code line of the init. - - Args: - line (`str`): A code line of the main init. - - Returns: - Optional[`str`]: If one (or several) backend is found, returns it. In the case of multiple backends (the line - contains `if is_xxx_available() and `is_yyy_available()`) returns all backends joined on `_and_` (so - `xxx_and_yyy` for instance). - """ - if _re_test_backend.search(line) is None: - return None - backends = [b[0] for b in _re_backend.findall(line)] - backends.sort() - return "_and_".join(backends) - - -def parse_init(init_file) -> tuple[dict[str, list[str]], dict[str, list[str]]] | None: - """ - Read an init_file and parse (per backend) the `_import_structure` objects defined and the `TYPE_CHECKING` objects - defined. - - Args: - init_file (`str`): Path to the init file to inspect. - - Returns: - `Optional[Tuple[Dict[str, List[str]], Dict[str, List[str]]]]`: A tuple of two dictionaries mapping backends to list of - imported objects, one for the `_import_structure` part of the init and one for the `TYPE_CHECKING` part of the - init. Returns `None` if the init is not a custom init. - """ - with open(init_file, "r", encoding="utf-8", newline="\n") as f: - lines = f.readlines() - - # Get the to `_import_structure` definition. - line_index = 0 - while line_index < len(lines) and not lines[line_index].startswith("_import_structure = {"): - line_index += 1 - - # If this is a traditional init, just return. - if line_index >= len(lines): - return None - - # First grab the objects without a specific backend in _import_structure - objects = [] - while not lines[line_index].startswith("if TYPE_CHECKING") and find_backend(lines[line_index]) is None: - line = lines[line_index] - # If we have everything on a single line, let's deal with it. - if _re_one_line_import_struct.search(line): - content = _re_one_line_import_struct.search(line).groups()[0] - imports = re.findall(r"\[([^\]]+)\]", content) - for imp in imports: - objects.extend([obj[1:-1] for obj in imp.split(", ")]) - line_index += 1 - continue - single_line_import_search = _re_import_struct_key_value.search(line) - if single_line_import_search is not None: - imports = [obj[1:-1] for obj in single_line_import_search.groups()[0].split(", ") if len(obj) > 0] - objects.extend(imports) - elif line.startswith(" " * 8 + '"'): - objects.append(line[9:-3]) - line_index += 1 - - # Those are stored with the key "none". - import_dict_objects = {"none": objects} - - # Let's continue with backend-specific objects in _import_structure - while not lines[line_index].startswith("if TYPE_CHECKING"): - # If the line is an if not is_backend_available, we grab all objects associated. - backend = find_backend(lines[line_index]) - # Check if the backend declaration is inside a try block: - if _re_try.search(lines[line_index - 1]) is None: - backend = None - - if backend is not None: - line_index += 1 - - # Scroll until we hit the else block of try-except-else - while _re_else.search(lines[line_index]) is None: - line_index += 1 - - line_index += 1 - - objects = [] - # Until we unindent, add backend objects to the list - while len(lines[line_index]) <= 1 or lines[line_index].startswith(" " * 4): - line = lines[line_index] - if _re_import_struct_add_one.search(line) is not None: - objects.append(_re_import_struct_add_one.search(line).groups()[0]) - elif _re_import_struct_add_many.search(line) is not None: - imports = _re_import_struct_add_many.search(line).groups()[0].split(", ") - imports = [obj[1:-1] for obj in imports if len(obj) > 0] - objects.extend(imports) - elif _re_between_brackets.search(line) is not None: - imports = _re_between_brackets.search(line).groups()[0].split(", ") - imports = [obj[1:-1] for obj in imports if len(obj) > 0] - objects.extend(imports) - elif _re_quote_object.search(line) is not None: - objects.append(_re_quote_object.search(line).groups()[0]) - elif line.startswith(" " * 8 + '"'): - objects.append(line[9:-3]) - elif line.startswith(" " * 12 + '"'): - objects.append(line[13:-3]) - line_index += 1 - - import_dict_objects[backend] = objects - else: - line_index += 1 - - # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend - objects = [] - while ( - line_index < len(lines) - and find_backend(lines[line_index]) is None - and not lines[line_index].startswith("else") - ): - line = lines[line_index] - single_line_import_search = _re_import.search(line) - if single_line_import_search is not None: - objects.extend(single_line_import_search.groups()[0].split(", ")) - elif line.startswith(" " * 8): - objects.append(line[8:-2]) - line_index += 1 - - type_hint_objects = {"none": objects} - - # Let's continue with backend-specific objects - while line_index < len(lines): - # If the line is an if is_backend_available, we grab all objects associated. - backend = find_backend(lines[line_index]) - # Check if the backend declaration is inside a try block: - if _re_try.search(lines[line_index - 1]) is None: - backend = None - - if backend is not None: - line_index += 1 - - # Scroll until we hit the else block of try-except-else - while _re_else.search(lines[line_index]) is None: - line_index += 1 - - line_index += 1 - - objects = [] - # Until we unindent, add backend objects to the list - while len(lines[line_index]) <= 1 or lines[line_index].startswith(" " * 8): - line = lines[line_index] - single_line_import_search = _re_import.search(line) - if single_line_import_search is not None: - objects.extend(single_line_import_search.groups()[0].split(", ")) - elif line.startswith(" " * 12): - objects.append(line[12:-2]) - line_index += 1 - - type_hint_objects[backend] = objects - else: - line_index += 1 - - return import_dict_objects, type_hint_objects - - -def analyze_results(import_dict_objects: dict[str, list[str]], type_hint_objects: dict[str, list[str]]) -> list[str]: - """ - Analyze the differences between _import_structure objects and TYPE_CHECKING objects found in an init. +REPO_ROOT = Path(__file__).parent.parent +MODELS_PATH = REPO_ROOT / "src" / "transformers" / "models" +MODELS_INIT_PATH = MODELS_PATH / "__init__.py" + +AUTO_GENERATED_HEADER = """# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from the model directories in `src/transformers/models`. +# Do NOT edit this file manually as any edits will be overwritten by auto-generation of the file. +# A model is picked up once one of its modules contains an `__all__`. +# Regenerate the file with: `python utils/check_inits.py --fix_and_overwrite` +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" - Args: - import_dict_objects (`Dict[str, List[str]]`): - A dictionary mapping backend names (`"none"` for the objects independent of any specific backend) to - list of imported objects. - type_hint_objects (`Dict[str, List[str]]`): - A dictionary mapping backend names (`"none"` for the objects independent of any specific backend) to - list of imported objects. +MODELS_INIT_TEMPLATE = """from typing import TYPE_CHECKING - Returns: - `List[str]`: The list of errors corresponding to mismatches. - """ +from ..utils import _LazyModule +from ..utils.import_utils import define_import_structure - def find_duplicates(seq): - return [k for k, v in collections.Counter(seq).items() if v > 1] - # If one backend is missing from the other part of the init, error early. - if list(import_dict_objects.keys()) != list(type_hint_objects.keys()): - return ["Both sides of the init do not have the same backends!"] +if TYPE_CHECKING: +{imports}else: + import sys - errors = [] - # Find all errors. - for key in import_dict_objects: - # Duplicate imports in any half. - duplicate_imports = find_duplicates(import_dict_objects[key]) - if duplicate_imports: - errors.append(f"Duplicate _import_structure definitions for: {duplicate_imports}") - duplicate_type_hints = find_duplicates(type_hint_objects[key]) - if duplicate_type_hints: - errors.append(f"Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}") + _file = globals()["__file__"] + sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) +""" - # Missing imports in either part of the init. - if sorted(set(import_dict_objects[key])) != sorted(set(type_hint_objects[key])): - name = "base imports" if key == "none" else f"{key} backend" - errors.append(f"Differences for {name}:") - for a in type_hint_objects[key]: - if a not in import_dict_objects[key]: - errors.append(f" {a} in TYPE_HINT but not in _import_structure.") - for a in import_dict_objects[key]: - if a not in type_hint_objects[key]: - errors.append(f" {a} in _import_structure but not in TYPE_HINT.") - return errors +def natural_sort_key(name: str) -> tuple[str | int, ...]: + """Sort key matching `ruff`'s import ordering, which compares digit runs numerically.""" + return tuple(int(part) if part.isdigit() else part for part in re.split(r"(\d+)", name)) -def get_transformers_submodules() -> list[str]: - """ - Returns the list of Transformers submodules. - """ - submodules = [] - for path, directories, files in os.walk(PATH_TO_TRANSFORMERS): - for folder in directories: - # Ignore private modules - if folder.startswith("_"): - directories.remove(folder) - continue - # Ignore leftovers from branches (empty folders apart from pycache) - if len(list((Path(path) / folder).glob("*.py"))) == 0: - continue - short_path = str((Path(path) / folder).relative_to(PATH_TO_TRANSFORMERS)) - submodule = short_path.replace(os.path.sep, ".") - submodules.append(submodule) - for fname in files: - if fname == "__init__.py": - continue - short_path = str((Path(path) / fname).relative_to(PATH_TO_TRANSFORMERS)) - submodule = short_path.replace(".py", "").replace(os.path.sep, ".") - if len(submodule.split(".")) == 1: - submodules.append(submodule) - return submodules +def get_model_names() -> list[str]: + """Return the models exposed by `models/__init__.py`, sorted.""" + import_structure = define_import_structure(str(MODELS_INIT_PATH)) + model_names = {module.split(".")[0] for modules in import_structure.values() for module in modules} + return sorted(model_names, key=natural_sort_key) -IGNORE_SUBMODULES = [ - "convert_pytorch_checkpoint_to_tf2", - "models.esm.openfold_utils", - "safetensors_conversion", - "modeling_gguf_pytorch_utils", - "kernels.falcon_mamba", - "kernels", -] +def generate_models_init() -> str: + """Render the full expected content of `models/__init__.py`.""" + imports = "".join(f" from .{model_name} import *\n" for model_name in get_model_names()) + return AUTO_GENERATED_HEADER + MODELS_INIT_TEMPLATE.format(imports=imports) -def check_submodules(): - """ - Check all submodules of Transformers are properly registered in the main init. Error otherwise. - """ - # This is to make sure the transformers module imported is the one in the repo. - from transformers.utils import direct_transformers_import - transformers = direct_transformers_import(PATH_TO_TRANSFORMERS) +def main(overwrite: bool): + old_content = MODELS_INIT_PATH.read_text(encoding="utf-8") + new_content = generate_models_init() - import_structure_keys = set(transformers._import_structure.keys()) - # This contains all the base keys of the _import_structure object defined in the init, but if the user is missing - # some optional dependencies, they may not have all of them. Thus we read the init to read all additions and - # (potentiall re-) add them. - with open(os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"), "r") as f: - init_content = f.read() - import_structure_keys.update(set(re.findall(r"import_structure\[\"([^\"]*)\"\]", init_content))) + if old_content == new_content: + return - module_not_registered = [ - module - for module in get_transformers_submodules() - if module not in IGNORE_SUBMODULES and module not in import_structure_keys - ] + if overwrite: + MODELS_INIT_PATH.write_text(new_content, encoding="utf-8") + return - if len(module_not_registered) > 0: - list_of_modules = "\n".join(f"- {module}" for module in module_not_registered) - raise ValueError( - "The following submodules are not properly registered in the main init of Transformers:\n" - f"{list_of_modules}\n" - "Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value." + relative_path = MODELS_INIT_PATH.relative_to(REPO_ROOT) + diff = "".join( + difflib.unified_diff( + old_content.splitlines(keepends=True), + new_content.splitlines(keepends=True), + fromfile=f"{relative_path} (on disk)", + tofile=f"{relative_path} (regenerated)", ) + ) + raise Exception( + f"`{relative_path}` is not consistent with the import structure on disk.\n" + "Run `make fix-repo` or `python utils/check_inits.py --fix_and_overwrite` to fix it.\n\n" + f"Diff (on disk → regenerated):\n{diff}" + ) if __name__ == "__main__": - # This entire files needs an overhaul - pass + parser = argparse.ArgumentParser() + parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.") + args = parser.parse_args() + main(overwrite=args.fix_and_overwrite)