Switch ci_docker to a container based on ubuntu 20.04 (#1334)

* Create a 20.04-based dockerfile with rust and all our required packages
* Use the new docker container from ci jobs, including the wheel builder
* Force manylinux_2_31
* Install newer binaryen

.github/workflows/python.yml

@@ -90,7 +90,7 @@ jobs:
             matrix+=('{"platform": "macos", "runs_on": "macos-latest"},')
           fi
           matrix+=('{"platform": "windows", "runs_on": "windows-latest-8-cores"},')
-          matrix+=('{"platform": "linux", "runs_on": "ubuntu-latest-16-cores"}')
+          matrix+=('{"platform": "linux", "runs_on": "ubuntu-latest-16-cores", container: {"image": "rerunio/ci_docker:0.5"}}')
 
           echo "Matrix values: ${matrix[@]}"
 
@@ -105,15 +105,20 @@ jobs:
 
     runs-on: ${{ matrix.runs_on }}
 
+    container: ${{ matrix.container }}
+
     steps:
       - uses: actions/checkout@v3
 
+      # These should already be in the docker container, but run for good measure. A no-op install
+      # should be fast, and this way things don't break if we add new packages without rebuilding
+      # docker
       - name: Cache APT Packages
         if: matrix.platform == 'linux'
         uses: awalsh128/[email protected]
         with:
           packages: ${{ env.UBUNTU_REQUIRED_PKGS }}
-          version: 1.0
+          version: 2.0 # Increment this to pull newer packages
           execute_install_scripts: true
 
       - name: Set up cargo cache
@@ -125,13 +130,19 @@ jobs:
           # the cache. Better cross-job sequencing would be nice here
           save-if: False
 
+      # The pip-cache setup logic doesn't work in the ubuntu docker container
+      # That's probably fine since we bake these deps into the container already
       - name: Setup python
+        if: matrix.platform != 'linux'
         uses: actions/setup-python@v4
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: "pip"
           cache-dependency-path: "rerun_py/requirements-build.txt"
 
+      # These should already be in the docker container, but run for good measure. A no-op install
+      # should be fast, and this way things don't break if we add new packages without rebuilding
+      # docker
       - run: pip install -r rerun_py/requirements-build.txt
 
       # ----------------------------------------------------------------------------------
@@ -140,11 +151,13 @@ jobs:
       # We have a nice script for that: ./scripts/setup_web.sh
       # Unfortunately, we can't run that on Windows, because Windows doesn't come with
       # a package manager like grown-up OSes do (at least not the CI version of Windows).
-      # So we only run the script on !Windows, and then on Windows we do the parts of the script manually.
-      # That way we still get to test the script on Linux and Mac.
+      # Also we can't run it on linux because the 20.04 Docker container will install
+      # an old version of binaryen/wasm-opt that barfs on the `--fast-math` flag
+      # So we only run the script on macos, and then on Windows we do the parts of the script manually.
+      # On ubuntu, the correct packages are pre-installed in our docker container.
 
       - name: Install prerequisites for building the web-viewer Wasm (non-Windows)
-        if: matrix.platform != 'windows'
+        if: matrix.platform == 'macos'
         shell: bash
         run: ./scripts/setup_web.sh
 
@@ -181,30 +194,11 @@ jobs:
         run: |
           python scripts/version_util.py --check_version
 
-      - name: Build and install rerun
-        run: pip install rerun_py/
-
-      - name: Cache RRD dataset
-        id: dataset
-        uses: actions/cache@v3
-        with:
-          path: examples/python/colmap/dataset/
-          # TODO(jleibs): Derive this key from the invocation below
-          key: colmap-dataset-colmap-fiat-v0
-
-      - name: Generate Embedded RRD file
-        shell: bash
-        # If you change the line below you should almost definitely change the `key:` line above by giving it a new, unique name
-        run: |
-          pip install -r examples/python/colmap/requirements.txt
-          python examples/python/colmap/main.py --dataset colmap_fiat --resize 800x600 --save colmap.rrd
-          cp colmap.rrd rerun_py/rerun_sdk/rerun_demo/colmap.rrd
-
       - name: Build Wheel
         uses: PyO3/maturin-action@v1
         with:
           maturin-version: "0.14.10"
-          manylinux: auto
+          manylinux: manylinux_2_31
           container: off
           command: build
           args: |
@@ -213,18 +207,47 @@ jobs:
             --no-default-features
             --features pypi
             --universal2
-            --out dist
+            --out pre-dist
 
       - name: Install built wheel
         run: |
-          pip install rerun-sdk --find-links dist --force-reinstall
-          pip install pytest #TODO(john) move to requirements-build.txt
-          cd rerun_py/tests && pytest
+          pip install rerun-sdk --find-links pre-dist --force-reinstall
+
+      - name: Run tests
+        run: cd rerun_py/tests && pytest
+
+      - name: Unpack the wheel
+        shell: bash
+        run: |
+          mkdir unpack-dist
+          wheel unpack pre-dist/*.whl --dest unpack-dist
 
-      - name: Run Car example
+      - name: Get the folder name
+        shell: bash
+        run: |
+          echo "pkg_folder=$(ls unpack-dist)" >> $GITHUB_ENV
+
+      - name: Cache RRD dataset
+        id: dataset
+        uses: actions/cache@v3
+        with:
+          path: examples/python/colmap/dataset/
+          # TODO(jleibs): Derive this key from the invocation below
+          key: colmap-dataset-colmap-fiat-v0
+
+      - name: Generate Embedded RRD file
+        shell: bash
+        # If you change the line below you should almost definitely change the `key:` line above by giving it a new, unique name
+        run: |
+          pip install -r examples/python/colmap/requirements.txt
+          python3 examples/python/colmap/main.py --dataset colmap_fiat --resize 800x600 --save colmap.rrd
+          cp colmap.rrd unpack-dist/${{ env.pkg_folder }}/rerun_sdk/rerun_demo/colmap.rrd
+
+      - name: Repack the wheel
+        shell: bash
         run: |
-          pip install -r examples/python/car/requirements.txt
-          python examples/python/car/main.py --headless
+          mkdir dist
+          wheel pack unpack-dist/${{ env.pkg_folder }} --dest dist/
 
       - name: Upload wheels
         uses: actions/upload-artifact@v3

.github/workflows/rust.yml

@@ -34,7 +34,7 @@ jobs:
 
     runs-on: ubuntu-latest-16-cores
     container:
-      image: rerunio/ci_docker:0.4
+      image: rerunio/ci_docker:0.5
       env:
         RUSTFLAGS: ${{env.RUSTFLAGS}}
         RUSTDOCFLAGS: ${{env.RUSTDOCFLAGS}}
@@ -89,7 +89,7 @@ jobs:
     name: Rust lints (fmt, check, cranky, tests, doc)
     runs-on: ubuntu-latest-16-cores
     container:
-      image: rerunio/ci_docker:0.4
+      image: rerunio/ci_docker:0.5
       env:
         RUSTFLAGS: ${{env.RUSTFLAGS}}
         RUSTDOCFLAGS: ${{env.RUSTDOCFLAGS}}
@@ -186,7 +186,7 @@ jobs:
     name: Check Rust web build (wasm32 + wasm-bindgen)
     runs-on: ubuntu-latest-16-cores
     container:
-      image: rerunio/ci_docker:0.4
+      image: rerunio/ci_docker:0.5
       env:
         RUSTFLAGS: ${{env.RUSTFLAGS}}
         RUSTDOCFLAGS: ${{env.RUSTDOCFLAGS}}
@@ -227,7 +227,7 @@ jobs:
     name: Check Rust dependencies (cargo-deny)
     runs-on: ubuntu-latest-16-cores
     container:
-      image: rerunio/ci_docker:0.4
+      image: rerunio/ci_docker:0.5
       env:
         RUSTFLAGS: ${{env.RUSTFLAGS}}
         RUSTDOCFLAGS: ${{env.RUSTDOCFLAGS}}

ci_docker/Dockerfile

@@ -1,19 +1,16 @@
-FROM rust:1.67
+FROM ubuntu:20.04
 LABEL maintainer="[email protected]"
 # Remember to update the version in publish.sh
 # TODO(jleibs) use this version in the publish.sh script and below in the CACHE_KEY
-LABEL version="0.4"
+LABEL version="0.5"
 LABEL description="Docker image used for the CI of https://github.com/rerun-io/rerun"
 
-# Install some cargo tools we know we'll always need
-# We do this first in its own layer because the layer is quite large (500MB+)
-# And updating the crates.io index is one of the slower steps
-RUN cargo install cargo-deny && \
-    cargo install cargo-cranky
-
 # Install the ubuntu package dependencies
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    curl \
+    git \
     libatk-bridge2.0 \
     libfontconfig1-dev \
     libfreetype6-dev \
@@ -24,19 +21,57 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libxcb-shape0-dev \
     libxcb-xfixes0-dev \
     libxkbcommon-dev \
-    patchelf \
-    python3-pip && \
+    lsb-release \
+    python3-pip \
+    sudo \
+    wget && \
     rm -rf /var/lib/apt/lists/*
 
+# Need a more recent pip for manylinux packages to work properly
+RUN python3 -m pip install -U pip
+
+# We need a more modern patchelf than ships on ubuntu-20.04
+RUN curl -L https://github.com/NixOS/patchelf/releases/download/0.17.2/patchelf-0.17.2-x86_64.tar.gz | tar -xz ./bin/patchelf
+
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH \
+    RUST_VERSION=1.67 \
+    RUSTUP_VERSION=1.25.2
+
+# Install Rust
+# Borrowed from: https://github.com/rust-lang/docker-rust/blob/master/1.67.1/bookworm/Dockerfile
+RUN set -eux; \
+    rustArch='x86_64-unknown-linux-gnu'; \
+    rustupSha256='bb31eaf643926b2ee9f4d8d6fc0e2835e03c0a60f34d324048aa194f0b29a71c'; \
+    url="https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/${rustArch}/rustup-init"; \
+    wget "$url"; \
+    echo "${rustupSha256} *rustup-init" | sha256sum -c -; \
+    chmod +x rustup-init; \
+    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch}; \
+    rm rustup-init; \
+    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
+    rustup --version; \
+    cargo --version; \
+    rustc --version;
+
+# Install some cargo tools we know we'll always need
+# We can't do this until after we've installed rust / cargo above
+RUN cargo install cargo-deny && \
+    cargo install cargo-cranky
+
 # Install the python build dependencies
 ADD rerun_py/requirements-build.txt requirements-build.txt
 RUN pip install -r requirements-build.txt
 
-# Install some additional versioned cargo tools
+# Install tools from setup_web.sh
 RUN cargo install [email protected]
+# Note: We need a more modern binaryen than ships on ubuntu-20.4, so we pull it from github
+RUN curl -L https://github.com/WebAssembly/binaryen/releases/download/version_112/binaryen-version_112-x86_64-linux.tar.gz | tar xzk --strip-components 1
 
 # Increment this to invalidate cache
-ENV CACHE_KEY=rerun_docker_v0.4
+ENV CACHE_KEY=rerun_docker_v0.5
 
 # See: https://github.com/actions/runner-images/issues/6775#issuecomment-1410270956
 RUN git config --system --add safe.directory '*'
+

ci_docker/publish.sh

@@ -1,4 +1,4 @@
-VERSION=0.4 # Bump on each new version. Remember to update the version in the Dockerfile too.
+VERSION=0.5 # Bump on each new version. Remember to update the version in the Dockerfile too.
 
 # The build needs to run from top of repo to access the requirments.txt
 cd `git rev-parse --show-toplevel`

rerun_py/requirements-build.txt

@@ -1,2 +1,4 @@
 maturin>=0.14,<0.15
 semver>=2.13,<2.14
+wheel>=0.38,<0.39
+pytest