Automatically synchronize build.rerun.io & release assets (#3945)

This introduces new scripts and workflows to synchronize the contents of
our cloud storage (`build.rerun.io`) with our release assets.

This can either be run manually from a dev machine, e.g.:
```sh
python scripts/ci/sync_release_assets.py --github-release prerelease --github-token <GITHUB_TOKEN> --remove --update
```
or manually dispatched through the Actions interface, e.g.
[here](https://github.com/rerun-io/rerun/actions/runs/6613455304/job/17961328298#step:7:25).

It will run automatically as part of the pre-release process on
push-to-main.

---

Here's how that looks today:

![image](https://github.com/rerun-io/rerun/assets/2910679/9fdc4664-ade3-40f6-bcdb-bdef76e2effe)


---

- Part of #3942 
- Fixes #2107

.github/workflows/manual_dispatch.yml

@@ -59,6 +59,12 @@ on:
         required: false
         default: false
 
+      SYNC_RELEASE_ASSETS:
+        description: "Sync release assets and build.rerun.io"
+        type: boolean
+        required: false
+        default: false
+
 jobs:
   checks:
     name: Run All Checks
@@ -275,3 +281,11 @@ jobs:
       PR_NUMBER: ${{ needs.check-for-pr.outputs.PR_NUMBER }}
     secrets: inherit
 
+  sync-release-assets:
+    name: Sync release assets and build.rerun.io
+    if: ${{ github.event.inputs.SYNC_RELEASE_ASSETS == 'true' }}
+    uses: ./.github/workflows/reusable_sync_release_assets.yml
+    with:
+      CONCURRENCY: manual-dispatch-${{ github.run_id}}
+      RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VERSION }}
+    secrets: inherit

.github/workflows/on_push_main.yml

@@ -288,13 +288,23 @@ jobs:
             ```
             pip install --pre -f https://build.rerun.io/commit/${{ env.SHORT_SHA }}/wheels --upgrade rerun-sdk
             ```
+            or
+            ```
+            pip install --pre -f https://github.com/rerun-io/rerun/releases/download/prerelease --upgrade rerun-sdk
+            ```
 
             ## CMake fetch-content for C++ SDK
             ```
             include(FetchContent)
             FetchContent_Declare(rerun_sdk URL https://build.rerun.io/commit/${{ env.SHORT_SHA }}/rerun_cpp_sdk.zip)
             FetchContent_MakeAvailable(rerun_sdk)
             ```
+            or
+            ```
+            include(FetchContent)
+            FetchContent_Declare(rerun_sdk URL https://github.com/rerun-io/rerun/releases/download/prerelease/rerun_cpp_sdk.zip)
+            FetchContent_MakeAvailable(rerun_sdk)
+            ```
 
             ## Static libraries for rerun_c
             * [Windows x64](https://build.rerun.io/commit/${{ env.SHORT_SHA }}/rerun_c/windows/rerun_c.lib)
@@ -310,3 +320,11 @@ jobs:
           allowUpdates: true
           removeArtifacts: true
           replacesArtifacts: true
+
+  sync-release-assets:
+    name: "Sync pre-release assets & build.rerun.io"
+    uses: ./.github/workflows/reusable_sync_release_assets.yml
+    with:
+      CONCURRENCY: push-${{ github.ref_name }}
+      RELEASE_VERSION: prerelease
+    secrets: inherit

.github/workflows/reusable_sync_release_assets.yml

@@ -0,0 +1,58 @@
+name: Sync assets with release
+
+on:
+  workflow_call:
+    inputs:
+      CONCURRENCY:
+        required: true
+        type: string
+      RELEASE_VERSION:
+        required: true
+        type: string
+        default: ""
+
+concurrency:
+  group: ${{ inputs.CONCURRENCY }}-sync-assets
+  cancel-in-progress: true
+
+jobs:
+  sync-assets:
+    name: Upload assets from build.rerun.io
+
+    permissions:
+      contents: "write"
+      id-token: "write"
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Install Python dependencies
+        run: pip install google-cloud-storage "PyGithub==1.59.0" "requests>=2.31,<3"
+
+      - id: "auth"
+        uses: google-github-actions/auth@v1
+        with:
+          workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
+
+      - name: "Set up Cloud SDK"
+        uses: "google-github-actions/setup-gcloud@v1"
+        with:
+          version: ">= 363.0.0"
+
+      - name: Sync release assets & build.rerun.io
+        run: |
+          python ./scripts/ci/sync_release_assets.py \
+            --github-release ${{ inputs.RELEASE_VERSION }} \
+            --github-token ${{ secrets.GITHUB_TOKEN }} \
+            --remove --update

scripts/ci/sync_release_assets.py

@@ -0,0 +1,163 @@
+"""
+Script to update a Github release's assets.
+
+Given a Github release ID (e.g. `prerelease` or `0.9.0`), this script will fetch the associated
+binary assets from our cloud storage (`build.rerun.io`) and upload them to the release as
+native assets.
+
+This is expected to be run by the release & pre-release workflows.
+
+You can also run it manually if you want to update a specific release's assets:
+  python scripts/ci/sync_release_assets.py --github-release prerelease --github-token <token> --update
+
+Requires the following packages:
+  pip install google-cloud-storage PyGithub
+"""
+from __future__ import annotations
+
+import argparse
+from typing import Dict
+
+from github import Github
+from github.GitRelease import GitRelease
+from google.cloud import storage
+
+Assets = Dict[str, storage.Blob]
+
+
+def fetch_binary_assets(
+    commit: str, *, do_wheels: bool = True, do_rerun_c: bool = True, do_rerun_cpp_sdk: bool = True
+) -> Assets:
+    """Given a release ID, fetches all associated binary assets from our cloud storage (build.rerun.io)."""
+    assets = dict()
+
+    gcs = storage.Client()
+    bucket = gcs.bucket("rerun-builds")
+
+    commit_short = commit[:7]
+    print(f"Fetching binary assets for #{commit_short}…")
+    print(f"  - wheels: {do_wheels}")
+    print(f"  - C libs: {do_rerun_c}")
+    print(f"  - C++ uber SDK: {do_rerun_cpp_sdk}")
+
+    # Python wheels
+    if do_wheels:
+        wheel_blobs = list(bucket.list_blobs(prefix=f"commit/{commit_short}/wheels"))
+        for blob in [bucket.get_blob(blob.name) for blob in wheel_blobs if blob.name.endswith(".whl")]:
+            if blob is not None and blob.name is not None:
+                name = blob.name.split("/")[-1]
+                print(f"    Found Python wheel: {name} ")
+                assets[name] = blob
+
+    # rerun_c
+    if do_rerun_c:
+        rerun_c_blobs = [
+            (
+                "librerun_c.x86_64-pc-windows-msvc.lib",
+                bucket.get_blob(f"commit/{commit_short}/rerun_c/windows/rerun_c.lib"),
+            ),
+            (
+                "librerun_c.x86_64-unknown-linux-gnu.a",
+                bucket.get_blob(f"commit/{commit_short}/rerun_c/linux/librerun_c.a"),
+            ),
+            (
+                "librerun_c.aarch64-apple-darwin.a",
+                bucket.get_blob(f"commit/{commit_short}/rerun_c/macos-arm/librerun_c.a"),
+            ),
+            (
+                "librerun_c.x86_64-apple-darwin.a",
+                bucket.get_blob(f"commit/{commit_short}/rerun_c/macos-intel/librerun_c.a"),
+            ),
+        ]
+        for name, blob in rerun_c_blobs:
+            if blob is not None:
+                print(f"    Found Rerun C library: {name}")
+                assets[name] = blob
+
+    # rerun_cpp_sdk
+    if do_rerun_cpp_sdk:
+        rerun_cpp_sdk_blob = bucket.get_blob(f"commit/{commit_short}/rerun_cpp_sdk.zip")
+        for blob in [rerun_cpp_sdk_blob]:
+            if blob is not None and blob.name is not None:
+                name = blob.name.split("/")[-1]
+                print(f"    Found Rerun cross-platform bundle: {name}")
+                assets[name] = blob
+
+    # rerun_cpp_sdk
+    rerun_cpp_sdk_blob = bucket.get_blob(f"commit/{commit_short}/rerun_cpp_sdk.zip")
+    for blob in [rerun_cpp_sdk_blob]:
+        if blob is not None and blob.name is not None:
+            name = blob.name.split("/")[-1]
+            print(f"    Found Rerun cross-platform bundle: {name} ({blob.size} bytes)")
+            assets[name] = blob
+
+    return assets
+
+
+def remove_release_assets(release: GitRelease):
+    print("Removing pre-existing release assets…")
+
+    for asset in release.get_assets():
+        print(f"    Removing {asset.name}…")
+        asset.delete_asset()
+
+
+def update_release_assets(release: GitRelease, assets: Assets):
+    print("Updating release assets…")
+
+    for name, blob in assets.items():
+        blob_contents = blob.download_as_bytes()
+        # NOTE: Do _not_ ever use `blob.size`, it might or might not give you the size you expect
+        # depending on the versions of your gcloud dependencies, which in turn might or might not fail
+        # the upload in all kinds of unexpected ways (including SSL errors!) depending on the versions
+        # of your reqwest & pygithub dependencies.
+        blob_raw_size = len(blob_contents)
+        print(f"    Uploading {name} ({blob_raw_size} bytes)…")
+        release.upload_asset_from_memory(
+            blob_contents,
+            blob_raw_size,
+            name,
+            content_type="application/octet-stream",
+        )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--github-token", required=True, help="GitHub token")
+    parser.add_argument("--github-repository", default="rerun-io/rerun", help="GitHub repository")
+    parser.add_argument(
+        "--github-release", required=True, help="ID of the Github (pre)release (e.g. `prerelease` or `0.9.0`)"
+    )
+    parser.add_argument("--github-timeout", default=120, help="Timeout for Github related operations")
+    parser.add_argument("--remove", action="store_true", help="Remove existing assets from the specified release")
+    parser.add_argument("--update", action="store_true", help="Update new assets to the specified release")
+    parser.add_argument("--no-wheels", action="store_true", help="Don't upload Python wheels")
+    parser.add_argument("--no-rerun-c", action="store_true", help="Don't upload C libraries")
+    parser.add_argument("--no-rerun-cpp-sdk", action="store_true", help="Don't upload C++ uber SDK")
+    args = parser.parse_args()
+
+    gh = Github(args.github_token, timeout=args.github_timeout)
+    repo = gh.get_repo(args.github_repository)
+    release = repo.get_release(args.github_release)
+    commit = dict([(tag.name, tag.commit) for tag in repo.get_tags()])[args.github_release]
+
+    print(
+        f'Syncing binary assets for release `{release.tag_name}` ("{release.title}" @{release.published_at}) #{commit.sha[:7]}…'
+    )
+
+    assets = fetch_binary_assets(
+        commit.sha,
+        do_wheels=not args.no_wheels,
+        do_rerun_c=not args.no_rerun_c,
+        do_rerun_cpp_sdk=not args.no_rerun_cpp_sdk,
+    )
+
+    if args.remove:
+        remove_release_assets(release)
+
+    if args.update:
+        update_release_assets(release, assets)
+
+
+if __name__ == "__main__":
+    main()