diff --git a/isaaclab.sh b/isaaclab.sh index c5cf66b95af..ef23b4cd0bc 100755 --- a/isaaclab.sh +++ b/isaaclab.sh @@ -348,6 +348,8 @@ while [[ $# -gt 0 ]]; do # install the python packages in IsaacLab/source directory echo "[INFO] Installing extensions inside the Isaac Lab repository..." python_exe=$(extract_python_exe) + # install omni.client via packman helper + ${python_exe} "${ISAACLAB_PATH}/tools/installation/install_omni_client_packman.py" # check if pytorch is installed and its version # install pytorch with cuda 12.8 for blackwell support if ${python_exe} -m pip list 2>/dev/null | grep -q "torch"; then diff --git a/source/isaaclab/isaaclab/sim/spawners/from_files/from_files.py b/source/isaaclab/isaaclab/sim/spawners/from_files/from_files.py index 2a7cb14a0b2..599ea3768da 100644 --- a/source/isaaclab/isaaclab/sim/spawners/from_files/from_files.py +++ b/source/isaaclab/isaaclab/sim/spawners/from_files/from_files.py @@ -28,6 +28,7 @@ select_usd_variants, ) from isaaclab.sim.utils.stage import get_current_stage +from isaaclab.utils.assets import check_file_path, retrieve_file_path if TYPE_CHECKING: from . import from_files_cfg @@ -258,20 +259,16 @@ def _spawn_from_usd_file( Raises: FileNotFoundError: If the USD file does not exist at the given path. """ - # get stage handle - stage = get_current_stage() - - # check file path exists - if not stage.ResolveIdentifierToEditTarget(usd_path): - if "4.5" in usd_path: - usd_5_0_path = ( - usd_path.replace("http", "https").replace("-production.", "-staging.").replace("/4.5", "/5.0") - ) - if not stage.ResolveIdentifierToEditTarget(usd_5_0_path): - raise FileNotFoundError(f"USD file not found at path at either: '{usd_path}' or '{usd_5_0_path}'.") - usd_path = usd_5_0_path - else: - raise FileNotFoundError(f"USD file not found at path at: '{usd_path}'.") + # check file path exists (supports local paths, S3, HTTP/HTTPS URLs) + # check_file_path returns: 0 (not found), 1 (local), 2 (remote) + file_status = check_file_path(usd_path) + if file_status == 0: + raise FileNotFoundError(f"USD file not found at path: '{usd_path}'.") + + # Download remote files (S3, HTTP, HTTPS) to local cache + # This also downloads all USD dependencies to maintain references + if file_status == 2: + usd_path = retrieve_file_path(usd_path) # spawn asset if it doesn't exist. if not prim_utils.is_prim_path_valid(prim_path): # add prim as reference to stage diff --git a/source/isaaclab/isaaclab/utils/assets.py b/source/isaaclab/isaaclab/utils/assets.py index 2318a9be55c..8f9a43d702e 100644 --- a/source/isaaclab/isaaclab/utils/assets.py +++ b/source/isaaclab/isaaclab/utils/assets.py @@ -14,15 +14,21 @@ """ import io +import logging import os +import posixpath import tempfile +from pathlib import Path from typing import Literal +from urllib.parse import urlparse -import carb import omni.client -NUCLEUS_ASSET_ROOT_DIR = carb.settings.get_settings().get("/persistent/isaac/asset_root/cloud") -"""Path to the root directory on the Nucleus Server.""" +logger = logging.getLogger(__name__) +from pxr import Sdf + +NUCLEUS_ASSET_ROOT_DIR = "https://omniverse-content-production.s3-us-west-2.amazonaws.com/Assets/Isaac/5.0" +"""Path to the root directory on the cloud storage.""" NVIDIA_NUCLEUS_DIR = f"{NUCLEUS_ASSET_ROOT_DIR}/NVIDIA" """Path to the root directory on the NVIDIA Nucleus Server.""" @@ -33,6 +39,8 @@ ISAACLAB_NUCLEUS_DIR = f"{ISAAC_NUCLEUS_DIR}/IsaacLab" """Path to the ``Isaac/IsaacLab`` directory on the NVIDIA Nucleus Server.""" +USD_EXTENSIONS = {".usd", ".usda", ".usdz"} + def check_file_path(path: str) -> Literal[0, 1, 2]: """Checks if a file exists on the Nucleus Server or locally. @@ -91,16 +99,38 @@ def retrieve_file_path(path: str, download_dir: str | None = None, force_downloa # create download directory if it does not exist if not os.path.exists(download_dir): os.makedirs(download_dir) - # download file in temp directory using os - file_name = os.path.basename(omni.client.break_url(path.replace(os.sep, "/")).path) - target_path = os.path.join(download_dir, file_name) - # check if file already exists locally - if not os.path.isfile(target_path) or force_download: - # copy file to local machine - result = omni.client.copy(path.replace(os.sep, "/"), target_path, omni.client.CopyBehavior.OVERWRITE) - if result != omni.client.Result.OK and force_download: - raise RuntimeError(f"Unable to copy file: '{path}'. Is the Nucleus Server running?") - return os.path.abspath(target_path) + # recursive download: mirror remote tree under download_dir + remote_url = path.replace(os.sep, "/") + to_visit = [remote_url] + visited = set() + local_root = None + + while to_visit: + cur_url = to_visit.pop() + if cur_url in visited: + continue + visited.add(cur_url) + + cur_rel = urlparse(cur_url).path.lstrip("/") + target_path = os.path.join(download_dir, cur_rel) + os.makedirs(os.path.dirname(target_path), exist_ok=True) + + if not os.path.isfile(target_path) or force_download: + result = omni.client.copy(cur_url, target_path, omni.client.CopyBehavior.OVERWRITE) + if result != omni.client.Result.OK and force_download: + raise RuntimeError(f"Unable to copy file: '{cur_url}'. Is the Nucleus Server running?") + + if local_root is None: + local_root = target_path + + # recurse into USD dependencies and referenced assets + if Path(target_path).suffix.lower() in USD_EXTENSIONS: + for ref in _find_usd_references(target_path): + ref_url = _resolve_reference_url(cur_url, ref) + if ref_url and ref_url not in visited: + to_visit.append(ref_url) + + return os.path.abspath(local_root) else: raise FileNotFoundError(f"Unable to find the file: {path}") @@ -127,3 +157,102 @@ def read_file(path: str) -> io.BytesIO: return io.BytesIO(memoryview(file_content).tobytes()) else: raise FileNotFoundError(f"Unable to find the file: {path}") + + +def _is_downloadable_asset(path: str) -> bool: + """Return True for USD or other asset types we mirror locally (textures, etc.).""" + clean = path.split("?", 1)[0].split("#", 1)[0] + suffix = Path(clean).suffix.lower() + + if suffix == ".mdl": + # MDL modules (OmniPBR.mdl, OmniSurface.mdl, ...) come from MDL search paths + return False + if not suffix: + return False + if suffix not in {".usd", ".usda", ".usdz", ".png", ".jpg", ".jpeg", ".exr", ".hdr", ".tif", ".tiff"}: + return False + return True + + +def _find_usd_references(local_usd_path: str) -> set[str]: + """Use Sdf API to collect referenced assets from a USD layer.""" + try: + layer = Sdf.Layer.FindOrOpen(local_usd_path) + except Exception: + logger.warning("Failed to open USD layer: %s", local_usd_path, exc_info=True) + return set() + + if layer is None: + return set() + + refs: set[str] = set() + + # Sublayers + for sub_path in getattr(layer, "subLayerPaths", []) or []: + if sub_path and _is_downloadable_asset(sub_path): + refs.add(str(sub_path)) + + def _walk_prim(prim_spec: Sdf.PrimSpec) -> None: + # References + ref_list = prim_spec.referenceList + for field in ("addedItems", "prependedItems", "appendedItems", "explicitItems"): + items = getattr(ref_list, field, None) + if not items: + continue + for ref in items: + asset_path = getattr(ref, "assetPath", None) + if asset_path and _is_downloadable_asset(asset_path): + refs.add(str(asset_path)) + + # Payloads + payload_list = prim_spec.payloadList + for field in ("addedItems", "prependedItems", "appendedItems", "explicitItems"): + items = getattr(payload_list, field, None) + if not items: + continue + for payload in items: + asset_path = getattr(payload, "assetPath", None) + if asset_path and _is_downloadable_asset(asset_path): + refs.add(str(asset_path)) + + # AssetPath-valued attributes (this is where OmniPBR.mdl, textures, etc. show up) + for attr_spec in prim_spec.attributes.values(): + default = attr_spec.default + if isinstance(default, Sdf.AssetPath): + if default.path and _is_downloadable_asset(default.path): + refs.add(default.path) + elif isinstance(default, Sdf.AssetPathArray): + for ap in default: + if ap.path and _is_downloadable_asset(ap.path): + refs.add(ap.path) + + for child in prim_spec.nameChildren.values(): + _walk_prim(child) + + for root_prim in layer.rootPrims.values(): + _walk_prim(root_prim) + + return refs + + +def _resolve_reference_url(base_url: str, ref: str) -> str: + """Resolve a USD asset reference against a base URL (http/local).""" + ref = ref.strip() + if not ref: + return ref + + parsed_ref = urlparse(ref) + if parsed_ref.scheme: + return ref + + base = urlparse(base_url) + if base.scheme == "": + base_dir = os.path.dirname(base_url) + return os.path.normpath(os.path.join(base_dir, ref)) + + base_dir = posixpath.dirname(base.path) + if ref.startswith("/"): + new_path = posixpath.normpath(ref) + else: + new_path = posixpath.normpath(posixpath.join(base_dir, ref)) + return f"{base.scheme}://{base.netloc}{new_path}" diff --git a/tools/installation/install_omni_client_packman.py b/tools/installation/install_omni_client_packman.py new file mode 100755 index 00000000000..84beb2bedd9 --- /dev/null +++ b/tools/installation/install_omni_client_packman.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Install omni.client from a prebuilt 7z payload into the current Python environment. + +- Downloads https://d4i3qtqj3r0z5.cloudfront.net/omni_client_library.linux-x86_64@.7z +- Extracts to a cache dir (default: $TMPDIR/omni_client_cache; override with OMNI_CLIENT_CACHE) +- Copies the Python package (release/bindings-python) and native libs (release/*.so) into site-packages/_omni_client +- Drops a .pth for import visibility +- Creates a minimal dist-info so `pip uninstall omni-client-offline` works + +# TODO: Once pip has been shipped, remove this script and use pip install omniverseclient== instead. +""" + +import logging +import os +import pathlib +import shutil +import site +import subprocess +import sys +import tempfile +import urllib.request + +# Ensure py7zr is available +try: + import py7zr # type: ignore # noqa: F401 +except ImportError: + subprocess.check_call([sys.executable, "-m", "pip", "install", "py7zr"]) + import py7zr # type: ignore + + +logger = logging.getLogger(__name__) + +# Configuration +pkg_ver = os.environ.get("OMNI_CLIENT_VERSION", "2.68.1") +cache_root = pathlib.Path(os.environ.get("OMNI_CLIENT_CACHE", tempfile.gettempdir())) / "omni_client_cache" +payload_url = f"https://d4i3qtqj3r0z5.cloudfront.net/omni_client_library.linux-x86_64%40{pkg_ver}.7z" + +# Paths +cache_root.mkdir(parents=True, exist_ok=True) +payload = cache_root / f"omni_client.{pkg_ver}.7z" +extract_root = cache_root / f"omni_client.{pkg_ver}.extracted" + +# Download payload if missing +if not payload.exists(): + logger.info(f" Downloading omni.client payload from {payload_url} ...") + urllib.request.urlretrieve(payload_url, payload) + +# Extract payload only if not already present +extract_root.mkdir(parents=True, exist_ok=True) +already_extracted = (extract_root / "release" / "bindings-python" / "omni" / "client").exists() +if not already_extracted: + logger.info(f" Extracting omni.client payload into {extract_root} ...") + with py7zr.SevenZipFile(payload, mode="r") as z: + z.extractall(path=extract_root) +else: + logger.info(f" Reusing existing extraction at {extract_root}") + +# Locate python package and native libs +src_py = extract_root / "release" / "bindings-python" +if not (src_py / "omni" / "client").exists(): + raise RuntimeError(f"Could not locate omni.client python package at {src_py}") + +src_lib = extract_root / "release" +if not any(src_lib.glob("libomni*.so*")): + raise RuntimeError(f"Could not locate native libs under {src_lib}") + +# Install into site-packages +if hasattr(site, "getsitepackages"): + candidates = [pathlib.Path(p) for p in site.getsitepackages() if p.startswith(sys.prefix)] + site_pkgs = candidates[0] if candidates else pathlib.Path(site.getusersitepackages()) +else: + site_pkgs = pathlib.Path(site.getusersitepackages()) +dest = site_pkgs / "_omni_client" +dest.mkdir(parents=True, exist_ok=True) +shutil.copytree(src_py, dest, dirs_exist_ok=True) +shutil.copytree(src_lib, dest / "lib", dirs_exist_ok=True) + +# Ensure the extension can find its libs without env vars +client_dir = dest / "omni" / "client" +client_dir.mkdir(parents=True, exist_ok=True) +for libfile in (dest / "lib").glob("libomni*.so*"): + target = client_dir / libfile.name + if not target.exists(): + try: + target.symlink_to(libfile) + except Exception: + shutil.copy2(libfile, target) + +# Add .pth for import visibility +with open(site_pkgs / "omni_client.pth", "w", encoding="utf-8") as f: + f.write(str(dest) + "\n") + f.write(str(dest / "lib") + "\n") + +# Minimal dist-info so pip can uninstall (pip uninstall omni-client) +dist_name = "omni-client" +dist_info = site_pkgs / f"{dist_name.replace('-', '_')}-{pkg_ver}.dist-info" +dist_info.mkdir(parents=True, exist_ok=True) +(dist_info / "INSTALLER").write_text("manual\n", encoding="utf-8") +metadata = "\n".join([ + f"Name: {dist_name}", + f"Version: {pkg_ver}", + "Summary: Offline omni.client bundle", + "", +]) +(dist_info / "METADATA").write_text(metadata, encoding="utf-8") + +records = [] +for path in [ + site_pkgs / "omni_client.pth", + dist_info / "INSTALLER", + dist_info / "METADATA", +]: + records.append(f"{path.relative_to(site_pkgs)},,") +for path in dest.rglob("*"): + records.append(f"{path.relative_to(site_pkgs)},,") +for path in dist_info.rglob("*"): + if path.name != "RECORD": + records.append(f"{path.relative_to(site_pkgs)},,") +(dist_info / "RECORD").write_text("\n".join(records), encoding="utf-8") + +logger.info(f"Installed omni.client to {dest} (dist: {dist_info.name})") +logger.info("Uninstall with: pip uninstall omni-client")