Skip to content
Merged
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
d144b48
[None][test] Add support for nemotron_3_ultra_550b_nvfp4 model in per…
yufeiwu-nv Jun 9, 2026
ea9ef23
Merge branch 'main' into bug
yufeiwu-nv Jun 9, 2026
f782448
[test] Update llm_perf_core.yml to include additional performance tes…
yufeiwu-nv Jun 9, 2026
3aa70ee
[test] Update llm_perf_core.yml to refine performance test cases
yufeiwu-nv Jun 9, 2026
b84cfcd
Merge branch 'main' into bug
yufeiwu-nv Jun 10, 2026
5c94d9b
[None][test] Remove unnecessary f-string prefix in build command
yufeiwu-nv Jun 10, 2026
0d3c81c
[None][fix] Enhance lock handling in config_file_lock
yufeiwu-nv Jun 10, 2026
2784114
Merge branch 'main' into bug
yufeiwu-nv Jun 10, 2026
5b1a3da
[None][fix] Refine lock error handling in config_file_lock
yufeiwu-nv Jun 10, 2026
9e65d1e
Merge branch 'main' into bug
yufeiwu-nv Jun 11, 2026
28c798e
Merge branch 'main' into bug
yufeiwu-nv Jun 12, 2026
6b0d18f
Merge branch 'main' into bug
yufeiwu-nv Jun 12, 2026
a16f6e0
Merge branch 'main' into bug
yufeiwu-nv Jun 12, 2026
e443439
Merge branch 'main' into bug
yufeiwu-nv Jun 15, 2026
c8eb175
Merge branch 'main' into bug
yufeiwu-nv Jun 15, 2026
7e04e21
Merge branch 'main' into bug
yufeiwu-nv Jun 16, 2026
02f7c9f
Merge branch 'main' into bug
yufeiwu-nv Jun 16, 2026
15f8af1
Merge branch 'main' into bug
yufeiwu-nv Jun 25, 2026
46f2396
Merge branch 'main' into bug
yufeiwu-nv Jun 25, 2026
c329db0
Merge branch 'main' into bug
yufeiwu-nv Jun 26, 2026
b15ff78
Merge branch 'main' into bug
yufeiwu-nv Jul 10, 2026
b442f45
Merge branch 'main' into bug
yufeiwu-nv Jul 13, 2026
7826b06
[None][fix] Update error handling in _is_lock_infra_error to clarify …
yufeiwu-nv Jul 13, 2026
0d1c2eb
[None][fix] Remove outdated comment in _is_lock_infra_error for clarity
yufeiwu-nv Jul 13, 2026
27b23dd
Merge branch 'main' into bug
yufeiwu-nv Jul 13, 2026
dfd3d31
Merge branch 'main' into bug
yufeiwu-nv Jul 13, 2026
583410a
Merge branch 'main' into bug
yufeiwu-nv Jul 13, 2026
50873ff
Merge branch 'main' into bug
yufeiwu-nv Jul 13, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 37 additions & 30 deletions tensorrt_llm/_torch/model_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,16 @@
}


def _is_lock_infra_error(exc: BaseException) -> bool:
"""Whether exc indicates broken lock infrastructure (not mere contention)."""
if isinstance(exc, PermissionError):
return True
if isinstance(exc, OSError):
return exc.errno in (errno.EACCES, errno.EPERM, errno.ENOLCK,
errno.ESTALE)
return False


@contextlib.contextmanager
def config_file_lock(timeout: int = 10):
"""
Expand All @@ -77,47 +87,44 @@ def config_file_lock(timeout: int = 10):
Args:
timeout: Maximum time to wait for lock acquisition in seconds
"""
# Use a single global lock file in HF cache directory
# This serializes all model loading operations to prevent race conditions
# Use a single global lock file in HF cache directory to serialize loads.
lock_path = Path(HF_MODULES_CACHE) / "_remote_code.lock"

# Create and acquire the lock
lock = filelock.FileLock(str(lock_path), timeout=timeout)

# Guard only acquisition so caller-body exceptions propagate (single-yield).
try:
with lock:
yield
except (PermissionError, OSError, filelock.Timeout) as e:
# Fallback to tempdir when primary lock path is unusable (e.g.,
# NFS locking failures like ENOLCK/ESTALE, permission issues,
# or lock acquisition timeouts)
if isinstance(e,
OSError) and e.errno not in (errno.EACCES, errno.EPERM,
errno.ENOLCK, errno.ESTALE):
lock.acquire(timeout=timeout)
except filelock.Timeout:
# Contention, not broken infra: a tempdir lock can't serialize against
# the holder, so degrade to no lock instead of crashing the process.
logger.warning(
f"could not acquire config lock within {timeout}s, proceeding without lock"
)
yield
except (PermissionError, OSError) as e:
# Broken lock infra (perms / NFS ENOLCK/ESTALE): retry on a tempdir lock.
if not _is_lock_infra_error(e):
raise
tmp_dir = Path(tempfile.gettempdir())
tmp_dir.mkdir(parents=True, exist_ok=True)
tmp_lock_path = tmp_dir / "_remote_code.lock"
tmp_lock = filelock.FileLock(str(tmp_lock_path), timeout=timeout)
tmp_lock = filelock.FileLock(str(tmp_dir / "_remote_code.lock"),
timeout=timeout)
try:
with tmp_lock:
yield
except filelock.Timeout:
tmp_lock.acquire(timeout=timeout)
except (PermissionError, OSError, filelock.Timeout):
logger.warning(
f"failed to acquire tempdir config lock within {timeout} seconds, proceeding without lock"
)
# proceed without lock
"tempdir config lock unavailable, proceeding without lock")
yield
except (PermissionError, OSError) as e:
if isinstance(
e, OSError) and e.errno not in (errno.EACCES, errno.EPERM,
errno.ENOLCK, errno.ESTALE):
raise
logger.warning(
f"tempdir config lock unavailable due to OS/permission issue: {e}, proceeding without lock"
)
# proceed without lock
else:
try:
yield
finally:
tmp_lock.release()
else:
try:
yield
finally:
lock.release()


@dataclass(kw_only=True)
Expand Down
Loading