Skip to content

Commit

Permalink
run sync-release-assets on PR for testing purposes
Browse files Browse the repository at this point in the history
  • Loading branch information
teh-cmc committed Oct 23, 2023
1 parent 7e5991c commit 4c39091
Show file tree
Hide file tree
Showing 3 changed files with 89 additions and 48 deletions.
9 changes: 9 additions & 0 deletions .github/workflows/on_pull_request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -133,3 +133,12 @@ jobs:
CONCURRENCY: pr-${{ github.event.pull_request.number }}
PR_NUMBER: ${{ github.event.pull_request.number }}
secrets: inherit

sync-release-assets:
name: "TEST: Sync pre-release assets with build.rerun.io"
if: github.event.pull_request.head.repo.owner.login == 'rerun-io'
uses: ./.github/workflows/reusable_sync_release_assets.yml
with:
CONCURRENCY: pr-${{ github.event.pull_request.number }}
RELEASE_VERSION: prerelease
secrets: inherit
21 changes: 13 additions & 8 deletions .github/workflows/reusable_sync_release_assets.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,24 @@ jobs:
name: Upload assets from build.rerun.io

permissions:
contents: "read"
contents: "write"
id-token: "write"

runs-on: ubuntu-latest

steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }}

- name: Set up Python
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: 3.x
python-version: "3.11"

- name: Install Python dependencies
run: pip install google-cloud-storage "PyGithub==1.59.0" "requests>=2.31,<3"

- id: "auth"
uses: google-github-actions/auth@v1
Expand All @@ -42,12 +50,9 @@ jobs:
with:
version: ">= 363.0.0"

- name: Install deps
run: pip install google-cloud-storage PyGithub

- name: Sync assets
- name: Sync release assets & build.rerun.io
run: |
python scripts/ci/sync_release_assets.py \
python ./scripts/ci/sync_release_assets.py \
--github-release ${{ inputs.RELEASE_VERSION }} \
--github-token ${{ secrets.GITHUB_TOKEN }} \
--remove --update
107 changes: 67 additions & 40 deletions scripts/ci/sync_release_assets.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@
Assets = Dict[str, storage.Blob]


def fetch_binary_assets(commit: str) -> Assets:
def fetch_binary_assets(
commit: str, *, do_wheels: bool = True, do_rerun_c: bool = True, do_rerun_cpp_sdk: bool = True
) -> Assets:
"""Given a release ID, fetches all associated binary assets from our cloud storage (build.rerun.io)."""
assets = dict()

Expand All @@ -34,46 +36,52 @@ def fetch_binary_assets(commit: str) -> Assets:

commit_short = commit[:7]
print(f"Fetching binary assets for #{commit_short}…")
print(f" - wheels: {do_wheels}")
print(f" - C libs: {do_rerun_c}")
print(f" - C++ uber SDK: {do_rerun_cpp_sdk}")

# Python wheels
wheel_blobs = list(bucket.list_blobs(prefix=f"commit/{commit_short}/wheels"))
for blob in [bucket.get_blob(blob.name) for blob in wheel_blobs if blob.name.endswith(".whl")]:
if blob is not None and blob.name is not None:
name = blob.name.split("/")[-1]
print(f" Found Python wheel: {name} ({blob.size} bytes)")
assets[name] = blob
if do_wheels:
wheel_blobs = list(bucket.list_blobs(prefix=f"commit/{commit_short}/wheels"))
for blob in [bucket.get_blob(blob.name) for blob in wheel_blobs if blob.name.endswith(".whl")]:
if blob is not None and blob.name is not None:
name = blob.name.split("/")[-1]
print(f" Found Python wheel: {name} ")
assets[name] = blob

# rerun_c
rerun_c_blobs = [
(
"librerun_c.x86_64-pc-windows-msvc.lib",
bucket.get_blob(f"commit/{commit_short}/rerun_c/windows/rerun_c.lib"),
),
(
"librerun_c.x86_64-unknown-linux-gnu.a",
bucket.get_blob(f"commit/{commit_short}/rerun_c/linux/librerun_c.a"),
),
(
"librerun_c.aarch64-apple-darwin.a",
bucket.get_blob(f"commit/{commit_short}/rerun_c/macos-arm/librerun_c.a"),
),
(
"librerun_c.x86_64-apple-darwin.a",
bucket.get_blob(f"commit/{commit_short}/rerun_c/macos-intel/librerun_c.a"),
),
]
for name, blob in rerun_c_blobs:
if blob is not None:
print(f" Found Rerun C library: {name} ({blob.size} bytes)")
assets[name] = blob
if do_rerun_c:
rerun_c_blobs = [
(
"librerun_c.x86_64-pc-windows-msvc.lib",
bucket.get_blob(f"commit/{commit_short}/rerun_c/windows/rerun_c.lib"),
),
(
"librerun_c.x86_64-unknown-linux-gnu.a",
bucket.get_blob(f"commit/{commit_short}/rerun_c/linux/librerun_c.a"),
),
(
"librerun_c.aarch64-apple-darwin.a",
bucket.get_blob(f"commit/{commit_short}/rerun_c/macos-arm/librerun_c.a"),
),
(
"librerun_c.x86_64-apple-darwin.a",
bucket.get_blob(f"commit/{commit_short}/rerun_c/macos-intel/librerun_c.a"),
),
]
for name, blob in rerun_c_blobs:
if blob is not None:
print(f" Found Rerun C library: {name} ({blob.size} bytes)")
assets[name] = blob

# rerun_cpp_sdk
rerun_cpp_sdk_blob = bucket.get_blob(f"commit/{commit_short}/rerun_cpp_sdk.zip")
for blob in [rerun_cpp_sdk_blob]:
if blob is not None and blob.name is not None:
name = blob.name.split("/")[-1]
print(f" Found Rerun cross-platform bundle: {name} ({blob.size} bytes)")
assets[name] = blob
if do_rerun_cpp_sdk:
rerun_cpp_sdk_blob = bucket.get_blob(f"commit/{commit_short}/rerun_cpp_sdk.zip")
for blob in [rerun_cpp_sdk_blob]:
if blob is not None and blob.name is not None:
name = blob.name.split("/")[-1]
print(f" Found Rerun cross-platform bundle: {name} ({blob.size} bytes)")
assets[name] = blob

return assets

Expand All @@ -90,9 +98,19 @@ def update_release_assets(release: GitRelease, assets: Assets):
print("Updating release assets…")

for name, blob in assets.items():
with blob.open("rb") as f:
print(f" Uploading {name}…")
release.upload_asset_from_memory(f, blob.size, name, content_type="application/octet-stream")
blob_contents = blob.download_as_bytes()
# NOTE: Do _not_ ever use `blob.size`, it might or might not give you the size you expect
# depending on the versions of your gcloud dependencies, which in turn might or might not fail
# the upload in all kinds of unexpected ways (including SSL errors!) depending on the versions
# of your reqwest & pygithub dependencies.
blob_raw_size = len(blob_contents)
print(f" Uploading {name} ({blob_raw_size} bytes)…")
release.upload_asset_from_memory(
blob_contents,
blob_raw_size,
name,
content_type="application/octet-stream",
)


def main() -> None:
Expand All @@ -102,11 +120,15 @@ def main() -> None:
parser.add_argument(
"--github-release", required=True, help="ID of the Github (pre)release (e.g. `prerelease` or `0.9.0`)"
)
parser.add_argument("--github-timeout", default=120, help="Timeout for Github related operations")
parser.add_argument("--remove", action="store_true", help="Remove existing assets from the specified release")
parser.add_argument("--update", action="store_true", help="Update new assets to the specified release")
parser.add_argument("--no-wheels", action="store_true", help="Don't upload Python wheels")
parser.add_argument("--no-rerun-c", action="store_true", help="Don't upload C libraries")
parser.add_argument("--no-rerun-cpp-sdk", action="store_true", help="Don't upload C++ uber SDK")
args = parser.parse_args()

gh = Github(args.github_token)
gh = Github(args.github_token, timeout=args.github_timeout)
repo = gh.get_repo(args.github_repository)
release = repo.get_release(args.github_release)
commit = dict([(tag.name, tag.commit) for tag in repo.get_tags()])[args.github_release]
Expand All @@ -115,7 +137,12 @@ def main() -> None:
f'Syncing binary assets for release `{release.tag_name}` ("{release.title}" @{release.published_at}) #{commit.sha[:7]}…'
)

assets = fetch_binary_assets(commit.sha)
assets = fetch_binary_assets(
commit.sha,
do_wheels=not args.no_wheels,
do_rerun_c=not args.no_rerun_c,
do_rerun_cpp_sdk=not args.no_rerun_cpp_sdk,
)

if args.remove:
remove_release_assets(release)
Expand Down

0 comments on commit 4c39091

Please sign in to comment.