(MOT-3889) fix(scrapling): ship as python source bundle instead of deploy: image

.github/scripts/validate_worker.py

@@ -52,6 +52,15 @@
     "acp",  # editors launch the `iii-acp` binary by name (ACP subprocess)
 })
 
+# The engine's bundle validator (iii-worker/src/cli/bundle_download.rs) only
+# accepts runtime.base_image values that name a sandbox-catalog preset ref
+# verbatim (sandbox_daemon/catalog.rs PRESETS); that's how a non-node bundle
+# picks its rootfs now that runtime.kind is deprecated.
+BUNDLE_PRESET_IMAGES = frozenset({
+    "docker.io/iiidev/python:latest",
+    "docker.io/iiidev/node:latest",
+})
+
 
 def main(argv: list[str] | None = None) -> int:
     p = argparse.ArgumentParser()
@@ -121,6 +130,24 @@ def soft(msg: str) -> None:
                     f"extracts the release archive by worker name and would fail "
                     f"with \"Binary '{worker}' not found in archive\""
                 )
+        # Mirror the engine's bundle-manifest validator
+        # (iii-worker/src/cli/bundle_download.rs): it executes only
+        # scripts.start and rejects install/setup/base_image at install
+        # time. Catch that at PR time instead of at the user's install.
+        if m.deploy == "bundle":
+            scripts = m.raw.get("scripts") or {}
+            if str(scripts.get("setup") or "").strip():
+                hard(f"{worker}/iii.worker.yaml: bundle workers must not declare scripts.setup (engine rejects it)")
+            if str(scripts.get("install") or "").strip():
+                hard(f"{worker}/iii.worker.yaml: bundle workers must not declare scripts.install (engine rejects it)")
+            if not str(scripts.get("start") or "").strip():
+                hard(f"{worker}/iii.worker.yaml: bundle workers must declare a non-empty scripts.start")
+            base_image = (m.raw.get("runtime") or {}).get("base_image")
+            if base_image is not None and base_image not in BUNDLE_PRESET_IMAGES:
+                hard(
+                    f"{worker}/iii.worker.yaml: bundle runtime.base_image must be one of "
+                    f"{sorted(BUNDLE_PRESET_IMAGES)} (engine rejects anything else)"
+                )
 
     # 3. Manifest version >= base
     if m is not None and m.manifest:

.github/workflows/_bundle.yml

@@ -114,16 +114,26 @@ jobs:
           cp "$WORKER/iii.worker.yaml"       "$stage/iii.worker.yaml"
           echo "STAGE_DIR=$stage" >> "$GITHUB_ENV"
 
-      # ── Python bundle path (TODO) ────────────────────────────────────
-      # When the first Python worker with `deploy: bundle` lands, choose
-      # a single-file packaging tool (PEP 723 / shiv / pex / PyOxidizer)
-      # and produce the same `index.mjs`-equivalent entrypoint + per-tool
-      # `iii.worker.yaml` in $STAGE_DIR. Until then, fail loudly.
-      - name: Python bundle (unsupported)
+      # ── Python bundle path ───────────────────────────────────────────
+      # Python workers ship as a source bundle (src/ + pyproject.toml +
+      # iii.worker.yaml). The engine's bundle validator executes only
+      # `scripts.start` (scripts.install/setup are rejected, and
+      # runtime.base_image must name an engine-preset ref like
+      # docker.io/iiidev/python:latest), so the manifest's start command
+      # self-bootstraps (e.g. "pip install -e . && python -m src.main").
+      # Native deps and browsers can't be vendored per-arch into one
+      # archive anyway — install-at-start sidesteps that.
+      - name: Stage Python artefact
         if: inputs.language == 'python'
+        env:
+          WORKER: ${{ inputs.worker }}
         run: |
-          echo "::error::Python bundle path is not implemented yet"
-          exit 1
+          set -euo pipefail
+          stage="$RUNNER_TEMP/bundle-stage"
+          mkdir -p "$stage"
+          rsync -a --exclude tests --exclude '.git*' --exclude README.md \
+            "$WORKER/" "$stage/"
+          echo "STAGE_DIR=$stage" >> "$GITHUB_ENV"
 
       - name: Unsupported language
         if: inputs.language != 'node' && inputs.language != 'javascript' && inputs.language != 'python'
@@ -135,11 +145,13 @@ jobs:
 
       - name: Verify staged artefact
         env:
-          WORKER: ${{ inputs.worker }}
+          LANGUAGE: ${{ inputs.language }}
         run: |
           set -euo pipefail
           ls -la "$STAGE_DIR"
-          for required in index.mjs iii.worker.yaml; do
+          entry=index.mjs
+          if [[ "$LANGUAGE" == "python" ]]; then entry=pyproject.toml; fi
+          for required in "$entry" iii.worker.yaml; do
             if [[ ! -f "$STAGE_DIR/$required" ]]; then
               echo "::error::staged bundle missing $required"
               exit 1
@@ -159,8 +171,8 @@ jobs:
 
           # `tar -C` so the archive contents are rooted at the staged
           # files, not at `$STAGE_DIR`. The resulting archive expands
-          # straight to `index.mjs` + `iii.worker.yaml`.
-          tar -C "$STAGE_DIR" -czf "$archive" index.mjs iii.worker.yaml
+          # straight to the staged entrypoint + `iii.worker.yaml`.
+          tar -C "$STAGE_DIR" -czf "$archive" .
 
           digest=$(sha256sum "$archive" | awk '{print $1}')
           printf '%s  %s\n' "$digest" "${WORKER}.tar.gz" > "$checksum"

.github/workflows/_publish-registry.yml

@@ -173,12 +173,14 @@ jobs:
               fi
               ;;
             release-bundle)
-              # `_bundle.yml` uploaded `<worker>.tar.gz` (containing
-              # `index.mjs` + `iii.worker.yaml`) to the GitHub Release.
-              # Download, extract, and spawn `scripts.start` directly on
-              # the runner (same pattern as `release-binary`). Avoids
-              # `iii worker add`, which no longer accepts `--wait` and
-              # boots local dirs in a VM sandbox unsuitable for CI.
+              # `_bundle.yml` uploaded `<worker>.tar.gz` (staged worker
+              # files + `iii.worker.yaml`) to the GitHub Release.
+              # Download, extract, and spawn the manifest's
+              # `scripts.start` directly on the runner (same pattern as
+              # `release-binary`) — the same command the engine's bundle
+              # path executes. Avoids `iii worker add`, which no longer
+              # accepts `--wait` and boots local dirs in a VM sandbox
+              # unsuitable for CI.
               bundle_url="$REPO_URL/releases/download/$TAG/${WORKER}.tar.gz"
               echo "Fetching prebuilt bundle: $bundle_url"
               curl -fsSL "$bundle_url" -o /tmp/worker-bundle.tar.gz
@@ -188,11 +190,13 @@ jobs:
               tar -xzf /tmp/worker-bundle.tar.gz -C "$stage_dir"
               ls -la "$stage_dir"
 
+              start_cmd=$(python3 -c 'import sys, yaml; print(yaml.safe_load(open(sys.argv[1]))["scripts"]["start"])' "$stage_dir/iii.worker.yaml")
+
               worker_log="worker-$WORKER.log"
               workspace_root="$PWD"
-              echo "Starting local worker with: (cd $stage_dir && node ./index.mjs)"
+              echo "Starting local worker with: (cd $stage_dir && $start_cmd)"
               pushd "$stage_dir" >/dev/null
-              node ./index.mjs > "$workspace_root/$worker_log" 2>&1 &
+              sh -c "$start_cmd" > "$workspace_root/$worker_log" 2>&1 &
               echo "$!" > "$workspace_root/worker.pid"
               popd >/dev/null
 
@@ -248,7 +252,9 @@ jobs:
           args=(
             "--worker" "$WORKER"
             "--out" "worker-interface.json"
-            "--wait-seconds" "120"
+            # Bundles may self-bootstrap in scripts.start (pip install,
+            # browser downloads), so give them longer to register.
+            "--wait-seconds" "${{ inputs.deploy == 'bundle' && '600' || '120' }}"
             "--trigger-types-baseline" "trigger-types-baseline.json"
             "--workers-baseline" "workers-baseline.json"
           )

scrapling/iii.worker.yaml

@@ -1,12 +1,12 @@
 # CI (.github/scripts/validate_worker.py) requires name/language/deploy/manifest
-# and reads the release version from `manifest`. The engine accepts iii/deploy/
-# manifest as publish metadata and ignores them at add/start time, so one file
-# satisfies both validators. The image is still built from runtime.base_image +
-# scripts.install (no Dockerfile); CI does not validate the runtime/scripts block.
+# and reads the release version from `manifest`. `deploy: bundle` ships a source
+# tarball built by .github/workflows/_bundle.yml. The engine's bundle validator
+# rejects scripts.install/setup and runtime.base_image and executes only
+# scripts.start, so dependency bootstrap lives in scripts.start.
 iii: v1
 name: scrapling
 language: python
-deploy: image
+deploy: bundle
 manifest: pyproject.toml
 description: >-
   Scrapling as an iii worker — scrapling::* functions run HTTP / anti-bot /
@@ -16,9 +16,9 @@ description: >-
 env:
   III_URL: "ws://localhost:49134"
 
-# base_image + scripts.install build the image (supersedes the deprecated
-# runtime.language + Dockerfile). `scrapling install` downloads the
-# Camoufox/Chromium browsers used by stealthy-fetch / dynamic-fetch / screenshot.
+# Must name an engine-preset image ref verbatim: bundle workers may not pull
+# arbitrary base images, and this is how a non-node bundle picks its rootfs
+# (runtime.kind is deprecated).
 runtime:
   base_image: docker.io/iiidev/python:latest
 
@@ -28,5 +28,8 @@ resources:
   cpus: 2
 
 scripts:
-  install: "pip install -e . && scrapling install"
-  start: "python -m src.main"
+  # First boot installs deps + browsers (`scrapling install` downloads the
+  # Camoufox/Chromium used by stealthy-fetch / dynamic-fetch / screenshot);
+  # later boots are fast because pip and the browser cache persist in the
+  # sandbox.
+  start: "pip install -e . && scrapling install && python -m src.main"