Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions src/transformers/dynamic_module_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -319,20 +319,20 @@ def _compute_local_source_files_hash(
Computes a stable hash from the bytes of the local source file and its relative-import source files.
"""
model_path = Path(pretrained_model_name_or_path).resolve()
resolved_module_file = Path(resolved_module_file).resolve()
resolved_module_file = Path(resolved_module_file)

def _resolve_relative_source_path(source_file_path: Path) -> str:
canonical_path = source_file_path.parent.resolve() / source_file_path.name
try:
return source_file_path.relative_to(model_path).as_posix()
return canonical_path.relative_to(model_path).as_posix()
except ValueError:
# Fallback for edge cases where the source file is not under the local model directory.
return source_file_path.as_posix()
return canonical_path.as_posix()

files_to_hash = [
(_resolve_relative_source_path(resolved_module_file), resolved_module_file),
]
for source_file in get_relative_import_files(resolved_module_file):
source_file_path = Path(source_file).resolve()
source_file_path = Path(source_file)
files_to_hash.append((_resolve_relative_source_path(source_file_path), source_file_path))

source_files_hash = hashlib.sha256()
Expand Down
40 changes: 40 additions & 0 deletions tests/utils/test_dynamic_module_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import hashlib
import os
from pathlib import Path

Expand Down Expand Up @@ -221,6 +222,45 @@ def test_get_cached_module_file_local_cache_key_includes_transitive_import_sourc
assert cached_a != cached_b


def _build_symlinked_hub_cache(repo_root: Path, files: dict[str, str], revision: str = "abc123") -> Path:
blobs = repo_root / "blobs"
snapshot = repo_root / "snapshots" / revision
blobs.mkdir(parents=True)
snapshot.mkdir(parents=True)
for name, content in files.items():
sha = hashlib.sha256(content.encode("utf-8")).hexdigest()
(blobs / sha).write_text(content, encoding="utf-8")
(snapshot / name).symlink_to(Path("..") / ".." / "blobs" / sha)

return snapshot


def test_get_cached_module_file_local_handles_symlinked_hub_cache(monkeypatch, tmp_path):
# In a real hub cache the snapshot files are symlinks into a content-addressed ``blobs/`` dir,
# so relative-import discovery must follow the named ``*.py`` symlinks in the snapshot dir rather
# than their opaque blob targets
modules_cache = tmp_path / "hf_modules_cache"
monkeypatch.setattr(dynamic_module_utils, "HF_MODULES_CACHE", str(modules_cache))

snapshot = _build_symlinked_hub_cache(
tmp_path / "models--org--repo",
{
# A → B → C: only A is the entry point; C is a transitive dep reached via B
"custom_model.py": "from .helper import VALUE\n",
"helper.py": "from .base import BASE\nVALUE = BASE\n",
"base.py": 'BASE = "transitive"\n',
},
)

cached_module = get_cached_module_file(str(snapshot), "custom_model.py")
cache_dir = modules_cache / Path(cached_module).parent

assert (cache_dir / "custom_model.py").read_text(encoding="utf-8") == "from .helper import VALUE\n"
assert (cache_dir / "helper.py").exists(), "direct import must be copied"
assert (cache_dir / "base.py").exists(), "transitive import must be copied"
assert (cache_dir / "base.py").read_text(encoding="utf-8") == 'BASE = "transitive"\n'


def test_get_cached_module_file_local_cache_key_keeps_hash_stable_with_different_basenames(monkeypatch, tmp_path):
modules_cache = tmp_path / "hf_modules_cache"
monkeypatch.setattr(dynamic_module_utils, "HF_MODULES_CACHE", str(modules_cache))
Expand Down
Loading