From 5159e0ec115508856d69d4f8ef4a52a59d7cbb9c Mon Sep 17 00:00:00 2001 From: Anderson Leal Date: Mon, 6 Jul 2026 18:35:28 -0300 Subject: [PATCH] fix(scrapling): ship as python source bundle instead of deploy: image scrapling/v0.2.0 dropped its Dockerfile for runtime.base_image + scripts.install, but deploy: image still fired the container-build job, which died on `open Dockerfile: no such file or directory` (run 28822396424). Switch scrapling to deploy: bundle and teach the pipeline about python bundles: - _bundle.yml: python path stages the worker source (src/ + pyproject.toml + iii.worker.yaml); scripts.start self-bootstraps since the engine's bundle validator executes only scripts.start - _publish-registry.yml: boot bundles via the manifest's scripts.start instead of hardcoded `node ./index.mjs`; widen the interface-collection wait to 600s for self-bootstrapping bundles - validate_worker.py: mirror the engine's bundle-manifest rules at PR time (no scripts.install/setup, non-empty scripts.start, runtime.base_image must name an engine-preset ref) - scrapling/iii.worker.yaml: deploy: bundle, runtime.base_image pinned to the engine-preset docker.io/iiidev/python:latest (runtime.kind is deprecated), dependency bootstrap folded into scripts.start Requires the engine change accepting preset refs for bundle runtime.base_image (iii repo, bundle_download.rs). Fixes MOT-3889 --- .github/scripts/validate_worker.py | 27 +++++++++++++++++++ .github/workflows/_bundle.yml | 36 ++++++++++++++++--------- .github/workflows/_publish-registry.yml | 24 ++++++++++------- scrapling/iii.worker.yaml | 23 +++++++++------- 4 files changed, 79 insertions(+), 31 deletions(-) diff --git a/.github/scripts/validate_worker.py b/.github/scripts/validate_worker.py index b0c2edec7..69c64458d 100644 --- a/.github/scripts/validate_worker.py +++ b/.github/scripts/validate_worker.py @@ -52,6 +52,15 @@ "acp", # editors launch the `iii-acp` binary by name (ACP subprocess) }) +# The engine's bundle validator (iii-worker/src/cli/bundle_download.rs) only +# accepts runtime.base_image values that name a sandbox-catalog preset ref +# verbatim (sandbox_daemon/catalog.rs PRESETS); that's how a non-node bundle +# picks its rootfs now that runtime.kind is deprecated. +BUNDLE_PRESET_IMAGES = frozenset({ + "docker.io/iiidev/python:latest", + "docker.io/iiidev/node:latest", +}) + def main(argv: list[str] | None = None) -> int: p = argparse.ArgumentParser() @@ -121,6 +130,24 @@ def soft(msg: str) -> None: f"extracts the release archive by worker name and would fail " f"with \"Binary '{worker}' not found in archive\"" ) + # Mirror the engine's bundle-manifest validator + # (iii-worker/src/cli/bundle_download.rs): it executes only + # scripts.start and rejects install/setup/base_image at install + # time. Catch that at PR time instead of at the user's install. + if m.deploy == "bundle": + scripts = m.raw.get("scripts") or {} + if str(scripts.get("setup") or "").strip(): + hard(f"{worker}/iii.worker.yaml: bundle workers must not declare scripts.setup (engine rejects it)") + if str(scripts.get("install") or "").strip(): + hard(f"{worker}/iii.worker.yaml: bundle workers must not declare scripts.install (engine rejects it)") + if not str(scripts.get("start") or "").strip(): + hard(f"{worker}/iii.worker.yaml: bundle workers must declare a non-empty scripts.start") + base_image = (m.raw.get("runtime") or {}).get("base_image") + if base_image is not None and base_image not in BUNDLE_PRESET_IMAGES: + hard( + f"{worker}/iii.worker.yaml: bundle runtime.base_image must be one of " + f"{sorted(BUNDLE_PRESET_IMAGES)} (engine rejects anything else)" + ) # 3. Manifest version >= base if m is not None and m.manifest: diff --git a/.github/workflows/_bundle.yml b/.github/workflows/_bundle.yml index 2bbefaca3..9d5fb1a2a 100644 --- a/.github/workflows/_bundle.yml +++ b/.github/workflows/_bundle.yml @@ -114,16 +114,26 @@ jobs: cp "$WORKER/iii.worker.yaml" "$stage/iii.worker.yaml" echo "STAGE_DIR=$stage" >> "$GITHUB_ENV" - # ── Python bundle path (TODO) ──────────────────────────────────── - # When the first Python worker with `deploy: bundle` lands, choose - # a single-file packaging tool (PEP 723 / shiv / pex / PyOxidizer) - # and produce the same `index.mjs`-equivalent entrypoint + per-tool - # `iii.worker.yaml` in $STAGE_DIR. Until then, fail loudly. - - name: Python bundle (unsupported) + # ── Python bundle path ─────────────────────────────────────────── + # Python workers ship as a source bundle (src/ + pyproject.toml + + # iii.worker.yaml). The engine's bundle validator executes only + # `scripts.start` (scripts.install/setup are rejected, and + # runtime.base_image must name an engine-preset ref like + # docker.io/iiidev/python:latest), so the manifest's start command + # self-bootstraps (e.g. "pip install -e . && python -m src.main"). + # Native deps and browsers can't be vendored per-arch into one + # archive anyway — install-at-start sidesteps that. + - name: Stage Python artefact if: inputs.language == 'python' + env: + WORKER: ${{ inputs.worker }} run: | - echo "::error::Python bundle path is not implemented yet" - exit 1 + set -euo pipefail + stage="$RUNNER_TEMP/bundle-stage" + mkdir -p "$stage" + rsync -a --exclude tests --exclude '.git*' --exclude README.md \ + "$WORKER/" "$stage/" + echo "STAGE_DIR=$stage" >> "$GITHUB_ENV" - name: Unsupported language if: inputs.language != 'node' && inputs.language != 'javascript' && inputs.language != 'python' @@ -135,11 +145,13 @@ jobs: - name: Verify staged artefact env: - WORKER: ${{ inputs.worker }} + LANGUAGE: ${{ inputs.language }} run: | set -euo pipefail ls -la "$STAGE_DIR" - for required in index.mjs iii.worker.yaml; do + entry=index.mjs + if [[ "$LANGUAGE" == "python" ]]; then entry=pyproject.toml; fi + for required in "$entry" iii.worker.yaml; do if [[ ! -f "$STAGE_DIR/$required" ]]; then echo "::error::staged bundle missing $required" exit 1 @@ -159,8 +171,8 @@ jobs: # `tar -C` so the archive contents are rooted at the staged # files, not at `$STAGE_DIR`. The resulting archive expands - # straight to `index.mjs` + `iii.worker.yaml`. - tar -C "$STAGE_DIR" -czf "$archive" index.mjs iii.worker.yaml + # straight to the staged entrypoint + `iii.worker.yaml`. + tar -C "$STAGE_DIR" -czf "$archive" . digest=$(sha256sum "$archive" | awk '{print $1}') printf '%s %s\n' "$digest" "${WORKER}.tar.gz" > "$checksum" diff --git a/.github/workflows/_publish-registry.yml b/.github/workflows/_publish-registry.yml index 47ecc3d88..7ab38b0c1 100644 --- a/.github/workflows/_publish-registry.yml +++ b/.github/workflows/_publish-registry.yml @@ -173,12 +173,14 @@ jobs: fi ;; release-bundle) - # `_bundle.yml` uploaded `.tar.gz` (containing - # `index.mjs` + `iii.worker.yaml`) to the GitHub Release. - # Download, extract, and spawn `scripts.start` directly on - # the runner (same pattern as `release-binary`). Avoids - # `iii worker add`, which no longer accepts `--wait` and - # boots local dirs in a VM sandbox unsuitable for CI. + # `_bundle.yml` uploaded `.tar.gz` (staged worker + # files + `iii.worker.yaml`) to the GitHub Release. + # Download, extract, and spawn the manifest's + # `scripts.start` directly on the runner (same pattern as + # `release-binary`) — the same command the engine's bundle + # path executes. Avoids `iii worker add`, which no longer + # accepts `--wait` and boots local dirs in a VM sandbox + # unsuitable for CI. bundle_url="$REPO_URL/releases/download/$TAG/${WORKER}.tar.gz" echo "Fetching prebuilt bundle: $bundle_url" curl -fsSL "$bundle_url" -o /tmp/worker-bundle.tar.gz @@ -188,11 +190,13 @@ jobs: tar -xzf /tmp/worker-bundle.tar.gz -C "$stage_dir" ls -la "$stage_dir" + start_cmd=$(python3 -c 'import sys, yaml; print(yaml.safe_load(open(sys.argv[1]))["scripts"]["start"])' "$stage_dir/iii.worker.yaml") + worker_log="worker-$WORKER.log" workspace_root="$PWD" - echo "Starting local worker with: (cd $stage_dir && node ./index.mjs)" + echo "Starting local worker with: (cd $stage_dir && $start_cmd)" pushd "$stage_dir" >/dev/null - node ./index.mjs > "$workspace_root/$worker_log" 2>&1 & + sh -c "$start_cmd" > "$workspace_root/$worker_log" 2>&1 & echo "$!" > "$workspace_root/worker.pid" popd >/dev/null @@ -248,7 +252,9 @@ jobs: args=( "--worker" "$WORKER" "--out" "worker-interface.json" - "--wait-seconds" "120" + # Bundles may self-bootstrap in scripts.start (pip install, + # browser downloads), so give them longer to register. + "--wait-seconds" "${{ inputs.deploy == 'bundle' && '600' || '120' }}" "--trigger-types-baseline" "trigger-types-baseline.json" "--workers-baseline" "workers-baseline.json" ) diff --git a/scrapling/iii.worker.yaml b/scrapling/iii.worker.yaml index 50e9089e4..cf73232f7 100644 --- a/scrapling/iii.worker.yaml +++ b/scrapling/iii.worker.yaml @@ -1,12 +1,12 @@ # CI (.github/scripts/validate_worker.py) requires name/language/deploy/manifest -# and reads the release version from `manifest`. The engine accepts iii/deploy/ -# manifest as publish metadata and ignores them at add/start time, so one file -# satisfies both validators. The image is still built from runtime.base_image + -# scripts.install (no Dockerfile); CI does not validate the runtime/scripts block. +# and reads the release version from `manifest`. `deploy: bundle` ships a source +# tarball built by .github/workflows/_bundle.yml. The engine's bundle validator +# rejects scripts.install/setup and runtime.base_image and executes only +# scripts.start, so dependency bootstrap lives in scripts.start. iii: v1 name: scrapling language: python -deploy: image +deploy: bundle manifest: pyproject.toml description: >- Scrapling as an iii worker — scrapling::* functions run HTTP / anti-bot / @@ -16,9 +16,9 @@ description: >- env: III_URL: "ws://localhost:49134" -# base_image + scripts.install build the image (supersedes the deprecated -# runtime.language + Dockerfile). `scrapling install` downloads the -# Camoufox/Chromium browsers used by stealthy-fetch / dynamic-fetch / screenshot. +# Must name an engine-preset image ref verbatim: bundle workers may not pull +# arbitrary base images, and this is how a non-node bundle picks its rootfs +# (runtime.kind is deprecated). runtime: base_image: docker.io/iiidev/python:latest @@ -28,5 +28,8 @@ resources: cpus: 2 scripts: - install: "pip install -e . && scrapling install" - start: "python -m src.main" + # First boot installs deps + browsers (`scrapling install` downloads the + # Camoufox/Chromium used by stealthy-fetch / dynamic-fetch / screenshot); + # later boots are fast because pip and the browser cache persist in the + # sandbox. + start: "pip install -e . && scrapling install && python -m src.main"