Make rerun-py use an embedded rerun-cli executable (#5996)

### What - Resolves: #5196 - Move `rerun_py` over to a dependency on `re_sdk` instead of `rerun` - Modify the pyproject / build script to look for a bin/rerun in the python project. - Update the launch/spawn scripts to use the embedded copy of the `rerun` executable. - Spawn now dispatches through the SDK the same way as cpp / rust. - When executed inside the editable dev environment, use the copy of `rerun` from `target/debug` - Build and upload a copy of rerun-cli to embed into the wheel during PR builds. - [x] Verify wheels build correctly on nightly ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested the web demo (if applicable): * Using examples from latest `main` build: [rerun.io/viewer](https://rerun.io/viewer/pr/5996?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) * Using full set of examples from `nightly` build: [rerun.io/viewer](https://rerun.io/viewer/pr/5996?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG * [x] If applicable, add a new check to the [release checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)! - [PR Build Summary](https://build.rerun.io/pr/5996) - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html) To run all checks from `main`, comment on the PR with `@rerun-bot full-check`. --------- Co-authored-by: Andreas Reich <[email protected]> Co-authored-by: Emil Ernerfeldt <[email protected]>
rerun-io · Apr 18, 2024 · cbe281b · cbe281b
1 parent 702c992
commit cbe281b
Show file tree

Hide file tree

Showing 23 changed files with 439 additions and 319 deletions.
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -74,11 +74,107 @@ jobs:
       NIGHTLY: true
     secrets: inherit
 
+  # -----------------------------------------------------------------------------------
+  # Build rerun_c library binaries:
+
+  build-rerun_c-and-upload-linux-arm64:
+    needs: [checks]
+    name: "Linux-Arm64: Build & Upload rerun_c"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml
+    with:
+      CONCURRENCY: nightly-linux-arm64
+      PLATFORM: linux-arm64
+    secrets: inherit
+
+  build-rerun_c-and-upload-linux-x64:
+    needs: [checks]
+    name: "Linux-x64: Build & Upload rerun_c"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml
+    with:
+      CONCURRENCY: nightly-linux-x64
+      PLATFORM: linux-x64
+    secrets: inherit
+
+  build-rerun_c-and-upload-macos-x64:
+    needs: [checks]
+    name: "Mac-Intel: Build & Upload rerun_c"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml
+    with:
+      CONCURRENCY: nightly-macos-x64
+      PLATFORM: macos-x64
+    secrets: inherit
+
+  build-rerun_c-and-upload-macos-arm64:
+    needs: [checks]
+    name: "Mac-Arm64: Build & Upload rerun_c"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml
+    with:
+      CONCURRENCY: nightly-macos-arm64
+      PLATFORM: macos-arm64
+    secrets: inherit
+
+  build-rerun_c-and-upload-windows-x64:
+    needs: [checks]
+    name: "Windows-x64: Build & Upload rerun_c"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml
+    with:
+      CONCURRENCY: nightly-windows-x64
+      PLATFORM: windows-x64
+    secrets: inherit
+
+  # -----------------------------------------------------------------------------------
+  # Build rerun-cli (rerun binaries):
+
+  build-rerun-cli-and-upload-linux-arm64:
+    needs: [checks]
+    name: "Linux-arm64: Build & Upload rerun-cli"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
+    with:
+      CONCURRENCY: nightly-linux-arm64
+      PLATFORM: linux-arm64
+    secrets: inherit
+
+  build-rerun-cli-and-upload-linux-x64:
+    needs: [checks]
+    name: "Linux-x64: Build & Upload rerun-cli"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
+    with:
+      CONCURRENCY: nightly-linux-x64
+      PLATFORM: linux-x64
+    secrets: inherit
+
+  build-rerun-cli-and-upload-macos-x64:
+    needs: [checks]
+    name: "Mac-x64: Build & Upload rerun-cli"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
+    with:
+      CONCURRENCY: nightly-macos-x64
+      PLATFORM: macos-x64
+    secrets: inherit
+
+  build-rerun-cli-and-upload-macos-arm64:
+    needs: [checks]
+    name: "Mac-arm64: Build & Upload rerun-cli"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
+    with:
+      CONCURRENCY: nightly-macos-arm64
+      PLATFORM: macos-arm64
+    secrets: inherit
+
+  build-rerun-cli-and-upload-windows-x64:
+    needs: [checks]
+    name: "Windows-x64: Build & Upload rerun-cli"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
+    with:
+      CONCURRENCY: nightly-windows-x64
+      PLATFORM: windows-x64
+    secrets: inherit
+
   # ---------------------------------------------------------------------------
   # Build wheels:
 
   build-wheel-linux-arm64:
-    needs: [checks]
+    needs: [checks, build-rerun-cli-and-upload-linux-arm64]
     name: "Linux-arm64: Build & Upload Wheels"
     uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
     with:
@@ -89,7 +185,7 @@ jobs:
     secrets: inherit
 
   build-wheel-linux-x64:
-    needs: [checks]
+    needs: [checks, build-rerun-cli-and-upload-linux-x64]
     name: "Linux-x64: Build & Upload Wheels"
     uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
     with:
@@ -100,7 +196,7 @@ jobs:
     secrets: inherit
 
   build-wheel-macos-arm64:
-    needs: [checks]
+    needs: [checks, build-rerun-cli-and-upload-macos-arm64]
     name: "Macos-arm64: Build & Upload Wheels"
     uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
     with:
@@ -111,7 +207,7 @@ jobs:
     secrets: inherit
 
   build-wheel-macos-x64:
-    needs: [checks]
+    needs: [checks, build-rerun-cli-and-upload-macos-x64]
     name: "Macos-x64: Build & Upload Wheels"
     uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
     with:
@@ -122,7 +218,7 @@ jobs:
     secrets: inherit
 
   build-wheel-windows-x64:
-    needs: [checks]
+    needs: [checks, build-rerun-cli-and-upload-windows-x64]
     name: "Windows-x64: Build & Upload Wheels"
     uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
     with:
@@ -185,103 +281,6 @@ jobs:
       WHEEL_ARTIFACT_NAME: windows-x64-wheel
     secrets: inherit
 
-  # -----------------------------------------------------------------------------------
-  # Build rerun_c library binaries:
-
-  build-rerun_c-and-upload-linux-arm64:
-    needs: [checks]
-    name: "Linux-Arm64: Build & Upload rerun_c"
-    uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml
-    with:
-      CONCURRENCY: nightly-linux-arm64
-      PLATFORM: linux-arm64
-    secrets: inherit
-
-  build-rerun_c-and-upload-linux-x64:
-    needs: [checks]
-    name: "Linux-x64: Build & Upload rerun_c"
-    uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml
-    with:
-      CONCURRENCY: nightly-linux-x64
-      PLATFORM: linux-x64
-    secrets: inherit
-
-  build-rerun_c-and-upload-macos-x64:
-    needs: [checks]
-    name: "Mac-Intel: Build & Upload rerun_c"
-    uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml
-    with:
-      CONCURRENCY: nightly-macos-x64
-      PLATFORM: macos-x64
-    secrets: inherit
-
-  build-rerun_c-and-upload-macos-arm64:
-    needs: [checks]
-    name: "Mac-Arm64: Build & Upload rerun_c"
-    uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml
-    with:
-      CONCURRENCY: nightly-macos-arm64
-      PLATFORM: macos-arm64
-    secrets: inherit
-
-  build-rerun_c-and-upload-windows-x64:
-    needs: [checks]
-    name: "Windows-x64: Build & Upload rerun_c"
-    uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml
-    with:
-      CONCURRENCY: nightly-windows-x64
-      PLATFORM: windows-x64
-    secrets: inherit
-
-  # -----------------------------------------------------------------------------------
-  # TODO(emilk): build and test one additional platorm, picked at random
-  # Build rerun-cli (rerun binaries):
-
-  build-rerun-cli-and-upload-linux-arm64:
-    needs: [checks]
-    name: "Linux-arm64: Build & Upload rerun-cli"
-    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
-    with:
-      CONCURRENCY: nightly-linux-arm64
-      PLATFORM: linux-arm64
-    secrets: inherit
-
-  build-rerun-cli-and-upload-linux-x64:
-    needs: [checks]
-    name: "Linux-x64: Build & Upload rerun-cli"
-    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
-    with:
-      CONCURRENCY: nightly-linux-x64
-      PLATFORM: linux-x64
-    secrets: inherit
-
-  build-rerun-cli-and-upload-macos-x64:
-    needs: [checks]
-    name: "Mac-x64: Build & Upload rerun-cli"
-    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
-    with:
-      CONCURRENCY: nightly-macos-x64
-      PLATFORM: macos-x64
-    secrets: inherit
-
-  build-rerun-cli-and-upload-macos-arm64:
-    needs: [checks]
-    name: "Mac-arm64: Build & Upload rerun-cli"
-    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
-    with:
-      CONCURRENCY: nightly-macos-arm64
-      PLATFORM: macos-arm64
-    secrets: inherit
-
-  build-rerun-cli-and-upload-windows-x64:
-    needs: [checks]
-    name: "Windows-x64: Build & Upload rerun-cli"
-    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
-    with:
-      CONCURRENCY: nightly-windows-x64
-      PLATFORM: windows-x64
-    secrets: inherit
-
   # ---------------------------------------------------------------------------
 
   run-notebook:

diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml
@@ -161,11 +161,19 @@ jobs:
       FULL: "true"
     secrets: inherit
 
+  min-cli-build:
+    name: "Minimum CLI Build"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
+    with:
+      CONCURRENCY: pr-${{ github.event.pull_request.number }}
+      PLATFORM: linux-x64
+    secrets: inherit
+
   # Build and test a single wheel to limit CI cost. We use linux-x64 because it's fast. linux-arm64 would also be a good
   # choice, but reusable_test_wheels.yml is broken for that target (https://github.com/rerun-io/rerun/issues/5525)
   min-wheel-build:
     name: "Minimum Wheel Build"
-    needs: [python-paths-filter, rust-paths-filter]
+    needs: [min-cli-build, python-paths-filter, rust-paths-filter]
     if: github.event.pull_request.head.repo.owner.login == 'rerun-io' && (needs.python-paths-filter.outputs.python_changes == 'true' || needs.rust-paths-filter.outputs.rust_changes == 'true')
     uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
     with:

diff --git a/.github/workflows/on_push_main.yml b/.github/workflows/on_push_main.yml
@@ -100,7 +100,7 @@ jobs:
     secrets: inherit
 
   # -----------------------------------------------------------------------------------
-  # TODO(emilk): build and test one additional platorm, picked at random
+  # TODO(emilk): build and test one additional platform, picked at random
 
   build-rerun_c-and-upload-linux-x64:
     needs: [checks]
@@ -112,7 +112,7 @@ jobs:
     secrets: inherit
 
   # -----------------------------------------------------------------------------------
-  # TODO(emilk): build and test one additional platorm, picked at random
+  # TODO(emilk): build and test one additional platform, picked at random
 
   build-rerun-cli-and-upload-linux-x64:
     needs: [checks]
@@ -124,10 +124,10 @@ jobs:
     secrets: inherit
 
   # -----------------------------------------------------------------------------------
-  # TODO(emilk): build and test one additional platorm, picked at random
+  # TODO(emilk): build and test one additional platform, picked at random
 
   build-wheel-linux-x64:
-    needs: [checks]
+    needs: [checks, build-rerun-cli-and-upload-linux-x64]
     name: "Linux-x64: Build & Upload Wheels"
     uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
     with:

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -223,46 +223,46 @@ jobs:
       RELEASE_COMMIT: ${{ needs.version.outputs.release-commit }}
     secrets: inherit
 
-  publish-wheels:
-    name: "Build and Publish Wheels"
+  publish-rerun_c:
+    name: "Build and Publish rerun_c"
     needs: [version]
-    uses: ./.github/workflows/reusable_publish_wheels.yml
+    uses: ./.github/workflows/reusable_publish_rerun_c.yml
     with:
       release-version: ${{ needs.version.outputs.current }}
-      concurrency: ${{ github.ref_name }}
       release-commit: ${{ needs.version.outputs.release-commit }}
+      concurrency: ${{ github.ref_name }}
     secrets: inherit
 
-  publish-web:
-    name: "Build and Publish Web"
-    needs: [version, publish-wheels]
-    uses: ./.github/workflows/reusable_publish_web.yml
+  publish-rerun-cli:
+    name: "Publish rerun-cli"
+    needs: [version]
+    uses: ./.github/workflows/reusable_publish_rerun_cli.yml
     with:
       release-version: ${{ needs.version.outputs.current }}
       release-commit: ${{ needs.version.outputs.release-commit }}
       concurrency: ${{ github.ref_name }}
-      wheel-artifact-name: linux-x64-wheel
-      update-latest: ${{ inputs.release-type == 'final' }}
     secrets: inherit
 
-  publish-rerun_c:
-    name: "Build and Publish rerun_c"
-    needs: [version]
-    uses: ./.github/workflows/reusable_publish_rerun_c.yml
+  publish-wheels:
+    name: "Build and Publish Wheels"
+    needs: [version, publish-rerun-cli]
+    uses: ./.github/workflows/reusable_publish_wheels.yml
     with:
       release-version: ${{ needs.version.outputs.current }}
-      release-commit: ${{ needs.version.outputs.release-commit }}
       concurrency: ${{ github.ref_name }}
+      release-commit: ${{ needs.version.outputs.release-commit }}
     secrets: inherit
 
-  publish-rerun-cli:
-    name: "Publish rerun-cli"
-    needs: [version]
-    uses: ./.github/workflows/reusable_publish_rerun_cli.yml
+  publish-web:
+    name: "Build and Publish Web"
+    needs: [version, publish-wheels]
+    uses: ./.github/workflows/reusable_publish_web.yml
     with:
       release-version: ${{ needs.version.outputs.current }}
       release-commit: ${{ needs.version.outputs.release-commit }}
       concurrency: ${{ github.ref_name }}
+      wheel-artifact-name: linux-x64-wheel
+      update-latest: ${{ inputs.release-type == 'final' }}
     secrets: inherit
 
   publish-js:

diff --git a/.github/workflows/reusable_build_and_upload_wheels.yml b/.github/workflows/reusable_build_and_upload_wheels.yml
@@ -195,9 +195,17 @@ jobs:
       - name: Get sha
         id: get-sha
         run: |
-          full_commit="${{ inputs.RELEASE_COMMIT || ((github.event_name == 'pull_request' && github.event.pull_request.head.ref) || github.sha) }}"
+          full_commit="${{ inputs.RELEASE_COMMIT || ((github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha) }}"
           echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT"
 
+      - name: "Download rerun-cli"
+        run: |
+          pixi run fetch-artifact \
+          --commit-sha ${{ steps.get-sha.outputs.sha }} \
+          --artifact rerun-cli \
+          --platform ${{ inputs.PLATFORM }} \
+          --dest rerun_py/rerun_sdk/bin
+
       - name: Build
         run: |
           pixi run python scripts/ci/build_and_upload_wheels.py \

diff --git a/Cargo.lock b/Cargo.lock