Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions nemo_gym/sandbox/providers/apptainer/provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@ class _ApptainerInstance:
mount_point: str # where the folder shows up inside
image: str # what it was built from
env: dict[str, str] = field(default_factory=dict)
overlay_dir: Path | None = None # per-instance disk overlay (cleaned on close)


def _resource_flags(resources: SandboxResources) -> list[str]:
Expand Down Expand Up @@ -386,6 +387,14 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle:
for key, value in spec.env.items():
argv += ["--env", f"{key}={value}"]
start_args = list(self._create_config.extra_start_args)
# --writable-tmpfs caps the writable layer at apptainer's `sessiondir max size`
# (default 64 MiB), which ENOSPCs for repos that rebuild on apply/eval (e.g. astropy's
# C extensions). Swap it for a per-instance DISK-backed overlay (bounded by host disk).
overlay_dir: Path | None = None
if "--writable-tmpfs" in start_args:
start_args = [a for a in start_args if a != "--writable-tmpfs"]
overlay_dir = Path(tempfile.mkdtemp(prefix="nemo-gym-apptainer-ovl-"))
start_args += ["--overlay", str(overlay_dir)]
resource_limit_flags = _resource_limit_flags(spec.resources)
if resource_limit_flags and self._create_config.apply_resource_limits:
if "--fakeroot" in start_args:
Expand All @@ -403,9 +412,13 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle:
code, _out, err = await self._run(argv, timeout_s=self._create_config.start_timeout_s, daemonize=True)
except TimeoutError as e:
shutil.rmtree(staging_dir, ignore_errors=True)
if overlay_dir:
shutil.rmtree(overlay_dir, ignore_errors=True)
raise ApptainerCreateError(f"apptainer instance start timed out for image={image!r}: {e}") from e
if code != 0:
shutil.rmtree(staging_dir, ignore_errors=True)
if overlay_dir:
shutil.rmtree(overlay_dir, ignore_errors=True)
raise ApptainerCreateError(
f"apptainer instance start failed (code={code}) for image={image!r}: {err.strip()}"
)
Expand All @@ -420,6 +433,7 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle:
mount_point=mount_point,
image=image,
env=dict(spec.env),
overlay_dir=overlay_dir,
),
)

Expand Down Expand Up @@ -485,6 +499,8 @@ async def _cleanup_failed_create_handle(self, handle: SandboxHandle) -> None:
timeout_s=self._exec_config.default_timeout_s,
)
shutil.rmtree(inst.staging_dir, ignore_errors=True)
if inst.overlay_dir:
shutil.rmtree(inst.overlay_dir, ignore_errors=True)

async def exec(
self,
Expand Down Expand Up @@ -667,6 +683,8 @@ async def close(self, handle: SandboxHandle) -> None:
shutil.rmtree(inst.staging_dir, ignore_errors=False)
except OSError as e:
LOGGER.warning("failed to remove staging dir %s: %s", inst.staging_dir, e)
if inst.overlay_dir:
shutil.rmtree(inst.overlay_dir, ignore_errors=True)

if stop_error is not None:
raise stop_error
Expand Down
20 changes: 20 additions & 0 deletions nemo_gym/sandbox/providers/docker/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Docker sandbox provider package."""

from nemo_gym.sandbox.providers.docker.provider import DockerSandboxProvider


__all__ = ["DockerSandboxProvider"]
Loading
Loading