From 16cd6d6756dcf31edfe669e78d58a8e6c3a65ba5 Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Thu, 30 Jul 2026 12:49:56 -0300 Subject: [PATCH] test(deployments): keep the image pull out of the observe-wait timing window test_never_deployment_outlives_observe_wait_then_succeeds asserts that create_deployment returns within observe_timeout + 2.0s. It pre-pulls alpine:3.20 with a comment saying this keeps an uncached pull out of the timed window, but that is not what happens: the backend is built with pull_images=True, and create_deployment pulls unconditionally rather than only when the image is missing locally. The warm-up avoids re-downloading layers; it does nothing about the registry round-trip, which still lands inside the measurement. A fully cached alpine:3.20 pull measures ~1.7s locally, so the unfixed test runs at 2.85s against a 3.0s budget on a fast machine with a warm cache -- 0.15s of margin, all of it hostage to Docker Hub latency. CI has been tipping over it on main and on unrelated branches: run 30554298719 (main) 4.39s run 30550043391 (PR #987) 3.84s run 30549563915 (experimentalist-run-progress) 3.42s Build the backend with pull_images=False so the test's own pre-pull is what puts the image on the host, and the timed window covers container create, start, and the observe wait -- what the assertion is actually about. Measured 1.18s across three runs afterwards, stable, with 1.82s of headroom. This only changes the one helper used by that test; the other three tests in the file keep the default pull_images=True. Co-Authored-By: Claude Opus 5 Signed-off-by: Sandy Chapman --- .../backends/docker/test_docker_backend.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/plugins/nemo-deployments/tests/integration/backends/docker/test_docker_backend.py b/plugins/nemo-deployments/tests/integration/backends/docker/test_docker_backend.py index a2938160fa..c8f07ebe1d 100644 --- a/plugins/nemo-deployments/tests/integration/backends/docker/test_docker_backend.py +++ b/plugins/nemo-deployments/tests/integration/backends/docker/test_docker_backend.py @@ -93,7 +93,14 @@ def _docker_backend_with_observe_timeout( *, oneshot_observe_timeout_seconds: int, ) -> DockerDeploymentBackend: - return _build_docker_backend(oneshot_observe_timeout_seconds=oneshot_observe_timeout_seconds) + # `pull_images` is off so the caller can pre-pull and keep the registry + # round-trip out of any window it times. `create_deployment` pulls + # unconditionally, not just when the image is missing locally, so leaving + # this on would put ~2s of Docker Hub latency inside the measurement. + return _build_docker_backend( + oneshot_observe_timeout_seconds=oneshot_observe_timeout_seconds, + pull_images=False, + ) def _always_http_config() -> DeploymentConfig: @@ -169,8 +176,9 @@ async def test_never_deployment_outlives_observe_wait_then_succeeds() -> None: client = docker.from_env() try: - # Warm the image cache so the timed window below measures the observe wait - # rather than an uncached image pull. + # The backend is built with `pull_images=False`, so this pull is what puts + # the image on the host. Doing it here keeps it out of the timed window + # below, which is measuring the observe wait. await asyncio.to_thread(client.images.pull, ALPINE_IMAGE) started = time.monotonic()