Add linux aarch64 wheel build support

.github/workflows/test.yaml

@@ -287,6 +287,79 @@ jobs:
         python-version:
         - '3.7'
     timeout-minutes: 40
+  build_aarch64_wheels_linux:
+    env:
+      PANTS_REMOTE_CACHE_READ: 'false'
+      PANTS_REMOTE_CACHE_WRITE: 'false'
+    if: ${{ github.repository_owner == 'pantsbuild' }}
+    name: Build AArch64 wheels and fs_util (Linux)
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 10
+    - if: github.event_name == 'push'
+      name: Get commit message for branch builds
+      run: 'echo "COMMIT_MESSAGE<<EOF" >> $GITHUB_ENV
+
+        echo "$(git log --format=%B -n 1 HEAD)" >> $GITHUB_ENV
+
+        echo "EOF" >> $GITHUB_ENV
+
+        '
+    - if: github.event_name == 'pull_request'
+      name: Get commit message for PR builds
+      run: 'echo "COMMIT_MESSAGE<<EOF" >> $GITHUB_ENV
+
+        echo "$(git log --format=%B -n 1 HEAD^2)" >> $GITHUB_ENV
+
+        echo "EOF" >> $GITHUB_ENV
+
+        '
+    - name: Setup QEMU
+      uses: docker/setup-qemu-action@v1
+    - name: Expose Pythons
+      run: echo "PATH=${PATH}:/opt/python/cp37-cp37m/bin:/opt/python/cp38-cp38/bin:/opt/python/cp39-cp39/bin"
+        >> $GITHUB_ENV
+    - if: github.event_name != 'pull_request'
+      name: Setup toolchain auth
+      run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV
+
+        '
+    - env:
+        PANTS_CONFIG_FILES: +['pants.ci.toml']
+      if: github.event_name == 'push' || !contains(env.COMMIT_MESSAGE, '[ci skip-build-wheels]')
+      name: Build wheels
+      run: 'docker run --rm -v ${{ github.workspace }}:/ws:rw --workdir=/ws quay.io/pypa/manylinux2014_aarch64
+        bash -exc ''curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh
+        -s -- -v -y --default-toolchain none
+
+        export PATH=${PATH}:${HOME}/.cargo/bin
+
+        export MODE=debug
+
+        ./build-support/bin/release.sh build-wheels
+
+        USE_PY38=true ./build-support/bin/release.sh build-wheels
+
+        USE_PY39=true ./build-support/bin/release.sh build-wheels
+
+        ./build-support/bin/release.sh build-fs-util''
+
+        '
+    - if: always()
+      name: Upload pants.log
+      uses: actions/upload-artifact@v2
+      with:
+        name: pants-log-wheels-linux
+        path: .pants.d/pants.log
+    - env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      if: github.event_name == 'push'
+      name: Deploy to S3
+      run: ./build-support/bin/deploy_to_s3.py
   build_wheels_linux:
     container: quay.io/pypa/manylinux2014_x86_64:latest
     env:

build-support/bin/_release_helper.py

@@ -558,7 +558,7 @@ def build_pex(fetch: bool) -> None:
     if fetch:
         extra_pex_args = [
             f"--platform={plat}-{abi}"
-            for plat in ("linux_x86_64", "macosx_10.15_x86_64")
+            for plat in ("linux_x86_64", "linux_aarch64", "macosx_10.15_x86_64")
             for abi in ("cp-37-m", "cp-38-cp38", "cp-39-cp39")
         ]
         pex_name = f"pants.{CONSTANTS.pants_unstable_version}.pex"
@@ -918,10 +918,10 @@ def check_pants_wheels_present(check_dir: str | Path) -> None:
         if not local_files:
             missing_packages.append(package.name)
             continue
-        if is_cross_platform(local_files) and len(local_files) != 6:
+        if is_cross_platform(local_files) and len(local_files) != 9:
             formatted_local_files = ", ".join(f.name for f in local_files)
             missing_packages.append(
-                f"{package.name} (expected 6 wheels, {{macosx, linux}} x {{cp37m, cp38, cp39}}, "
+                f"{package.name} (expected 9 wheels, {{macosx, linux_x86_64, linux_arm64}} x {{cp37m, cp38, cp39}}, "
                 f"but found {formatted_local_files})"
             )
     if missing_packages:

build-support/bin/generate_github_workflows.py

@@ -107,6 +107,11 @@ def checkout() -> Sequence[Step]:
         },
     ]
 
+def setup_qemu() -> Step:
+    return {
+        "name": "Setup QEMU",
+        "uses": "docker/setup-qemu-action@v1",
+    }
 
 def setup_toolchain_auth() -> Step:
     return {
@@ -429,6 +434,28 @@ def build_steps(*, is_macos: bool) -> list[Step]:
                     "env": env,
                 },
             ]
+        def build_steps_aarch64(*, is_macos: bool) -> list[Step]:
+            env = {"PANTS_CONFIG_FILES": "+['pants.ci.toml']", **(MACOS_ENV if is_macos else {})}
+            return [
+                {
+                    "name": "Build wheels",
+                    "run": dedent(
+                        # We use MODE=debug on PR builds to speed things up, given that those are
+                        # only smoke tests of our release process.
+                        """\
+                        docker run --rm -v ${{ github.workspace }}:/ws:rw --workdir=/ws quay.io/pypa/manylinux2014_aarch64 bash -exc 'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -v -y --default-toolchain none
+                        export PATH=${PATH}:${HOME}/.cargo/bin
+                        export MODE=debug
+                        ./build-support/bin/release.sh build-wheels
+                        USE_PY38=true ./build-support/bin/release.sh build-wheels
+                        USE_PY39=true ./build-support/bin/release.sh build-wheels
+                        ./build-support/bin/release.sh build-fs-util'
+                        """
+                    ),
+                    "if": DONT_SKIP_WHEELS,
+                    "env": env,
+                },
+            ]
 
         deploy_to_s3_step = {
             "name": "Deploy to S3",
@@ -466,6 +493,29 @@ def build_steps(*, is_macos: bool) -> list[Step]:
                         deploy_to_s3_step,
                     ],
                 },
+                "build_aarch64_wheels_linux": {
+                    "name": "Build AArch64 wheels and fs_util (Linux)",
+                    "runs-on": LINUX_VERSION,
+                    "env": DISABLE_REMOTE_CACHE_ENV,
+                    "if": IS_PANTS_OWNER,
+                    "steps": [
+                        *checkout(),
+                        setup_qemu(),
+                        {
+                            "name": "Expose Pythons",
+                            "run": (
+                                'echo "PATH=${PATH}:'
+                                "/opt/python/cp37-cp37m/bin:"
+                                "/opt/python/cp38-cp38/bin:"
+                                '/opt/python/cp39-cp39/bin" >> $GITHUB_ENV'
+                            ),
+                        },
+                        setup_toolchain_auth(),
+                        *build_steps_aarch64(is_macos=False),
+                        upload_log_artifacts(name="wheels-linux"),
+                        deploy_to_s3_step,
+                    ],
+                },
                 "build_wheels_macos": {
                     "name": "Build wheels and fs_util (macOS)",
                     "runs-on": MACOS_VERSION,

src/python/pants/backend/python/util_rules/pex_cli.py

@@ -52,7 +52,7 @@ def default_known_versions(cls):
                     "3640830",
                 )
             )
-            for plat in ["macos_arm64", "macos_x86_64", "linux_x86_64"]
+            for plat in ["macos_arm64", "macos_x86_64", "linux_x86_64", "linux_arm64"]
         ]
 
 

src/python/pants/base/exception_sink_test.py

@@ -59,6 +59,10 @@ def test_set_invalid_log_location():
             "Error opening fatal error log streams for log location '/': [Errno 13] Permission "
             "denied: '/.pids'"
         ),
+        Platform.linux_arm64: (
+            "Error opening fatal error log streams for log location '/': [Errno 13] Permission "
+            "denied: '/.pids'"
+        ),
     }
     assert match(Platform.current, err_str) in str(exc.value)
 

src/python/pants/engine/platform.py

@@ -15,6 +15,7 @@ class Platform(Enum):
     linux_x86_64 = "linux_x86_64"
     macos_arm64 = "macos_arm64"
     macos_x86_64 = "macos_x86_64"
+    linux_arm64 = "linux_arm64"
 
     @property
     def is_macos(self) -> bool:

src/python/pants/option/global_options.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 import enum
+import platform
 import importlib
 import logging
 import os
@@ -391,7 +392,10 @@ class LocalStoreOptions:
 
     store_dir: str = os.path.join(get_pants_cachedir(), "lmdb_store")
     processes_max_size_bytes: int = 16 * GIGABYTES
-    files_max_size_bytes: int = 256 * GIGABYTES
+    if platform.machine() != "aarch64":
+        files_max_size_bytes: int = 256* GIGABYTES
+    else:
+        files_max_size_bytes: int = 128 * GIGABYTES
     directories_max_size_bytes: int = 16 * GIGABYTES
     shard_count: int = 16
 

src/rust/engine/process_execution/src/lib.rs

@@ -77,6 +77,7 @@ pub enum Platform {
   Macos_x86_64,
   Macos_arm64,
   Linux_x86_64,
+  Linux_arm64,
 }
 
 impl Platform {
@@ -91,6 +92,13 @@ impl Platform {
       } if sysname.to_lowercase() == "linux" && machine.to_lowercase() == "x86_64" => {
         Ok(Platform::Linux_x86_64)
       }
+      uname::Info {
+        ref sysname,
+        ref machine,
+        ..
+      } if sysname.to_lowercase() == "linux" && machine.to_lowercase() == "aarch64" => {
+        Ok(Platform::Linux_arm64)
+      }
       uname::Info {
         ref sysname,
         ref machine,
@@ -121,6 +129,7 @@ impl From<Platform> for String {
   fn from(platform: Platform) -> String {
     match platform {
       Platform::Linux_x86_64 => "linux_x86_64".to_string(),
+      Platform::Linux_arm64 => "linux_arm64".to_string(),
       Platform::Macos_arm64 => "macos_arm64".to_string(),
       Platform::Macos_x86_64 => "macos_x86_64".to_string(),
     }
@@ -134,6 +143,7 @@ impl TryFrom<String> for Platform {
       "macos_arm64" => Ok(Platform::Macos_arm64),
       "macos_x86_64" => Ok(Platform::Macos_x86_64),
       "linux_x86_64" => Ok(Platform::Linux_x86_64),
+      "linux_arm64" => Ok(Platform::Linux_arm64),
       other => Err(format!(
         "Unknown platform {:?} encountered in parsing",
         other