Build wheels for macOS m1 (#2701)

## Related issue number

Fixes #2083
Fixes #2668

Signed-off-by: Tao He <[email protected]>

.cirrus.yml

@@ -0,0 +1,137 @@
+# only run for releases
+# only_if: $CIRRUS_TAG != ''
+
+env:
+  MACOSX_DEPLOYMENT_TARGET: "11"
+
+macosx_arm64_wheel_task:
+  macos_instance:
+    image: ghcr.io/cirruslabs/macos-monterey-base:latest
+
+  timeout_in: 120m
+
+  # can be manually triggered when needed (nightly or release)
+  trigger_type: manual
+
+  checkout_script:
+    - git submodule init
+    - git submodule update
+    - |
+      pushd ./learning_engine/graph-learn
+      git submodule update --init third_party/pybind11
+      popd
+    - |
+      # change the version for nightly release
+      # e.g. 0.15.0 -> 0.15.0a20220808
+      time=$(date "+%Y%m%d")
+      version=$(cat ./VERSION)
+      if [[ "$CIRRUS_TAG" == "" ]];
+      then
+        echo "${version}a${time}" > ./VERSION;
+      fi
+
+  install_deps_script:
+    - brew update
+    - brew install coreutils pyenv python
+
+  install_graphscope_deps_script: |
+    brew update || true
+    brew install bash coreutils
+    alias bash=$(brew --prefix)/bin/bash
+    sudo mkdir /opt/graphscope
+    sudo chown -R $(id -u):$(id -g) /opt/graphscope
+
+    # Not sure when the first installation would fail, saying that the lock of python3.10 is taken
+    # However the second trial would success.
+    ./gs install-deps dev || true
+    ./gs install-deps dev
+
+    brew install openjdk@11
+    sudo ln -sfn /opt/homebrew/opt/openjdk@11/libexec/openjdk.jdk /Library/Java/JavaVirtualMachines/openjdk-11.jdk
+
+    python3 -m pip install -r ./python/requirements.txt
+    python3 -m pip install -r ./python/requirements-dev.txt
+
+    echo "export PATH=/opt/homebrew/opt/coreutils/libexec/gnubin:\$PATH" >> ~/.graphscope_env
+
+    # for debugging with "Rerun with Terminal Access" on cirrus CI
+    bash ./debugging.sh || true
+
+  build_graphscope_wheels_script: |
+    # source environment variable
+    . ~/.graphscope_env
+    python3 -m pip install libclang
+    echo ${CC}
+
+    # for grpc-java
+    /usr/sbin/softwareupdate --install-rosetta --agree-to-license
+
+    # for building GLE
+    git config --global --add safe.directory .
+
+    # prepare
+    pushd ./python
+    python3 setup.py build_proto
+    popd
+    pushd ./coordinator
+    python3 setup.py build_builtin
+    popd
+
+    # build graphscope server wheel
+    pushd ./k8s/internal
+    sudo -E env PATH=$PATH make graphscope-py3-package GRAPHSCOPE_HOME=/usr/local
+    popd
+
+  graphscope_artifacts:
+    path: coordinator/dist/wheelhouse/*.whl
+
+  install_various_python_script:
+    - pyenv install 3.8.16
+    - pyenv install 3.9.16
+    - pyenv install 3.10.10
+    - pyenv install 3.11.2
+
+  build_graphscope_client_script: |
+    . ~/.graphscope_env
+
+    for py in 3.8.16 3.9.16 3.10.10 3.11.2; do
+      current_python=$(pyenv root)/versions/$py/bin/python
+      echo "Python is: $current_python, $($current_python --version)"
+      $current_python -m pip install delocate numpy wheel
+
+      export PATH=$(dirname $current_python):$PATH
+      python3 -c "import sys; print(sys.version)"
+
+      # build graphscope client wheel
+      pushd ./k8s/internal
+      make graphscope-client-py3-package GRAPHSCOPE_HOME=/usr/local
+      popd
+    done
+
+  graphscope_client_artifacts:
+    path: python/dist/wheelhouse/*.whl
+
+  run_tests_script: |
+    pushd python/dist/wheelhouse
+    for f in * ; do
+      python3 -m pip install $f --user || true;
+    done
+    popd
+
+    sudo python3 -m pip install coordinator/dist/wheelhouse/*.whl
+
+    # run tests
+    export GS_TEST_DIR="$(pwd)/gstest"
+    git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR}
+
+    export JAVA_HOME=$(/usr/libexec/java_home -v11)
+    export PATH=$JAVA_HOME/bin:$HOME/.local/bin:$PATH
+
+    sudo mkdir -p /tmp || true
+    sudo chmod -R a+wrx /tmp || true
+
+    # for debugging with "Rerun with Terminal Access" on cirrus CI
+    bash ./debugging.sh || true
+
+    python3 -c "import sys; print(sys.version)"
+    python3 -m pytest -s -v $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/tests/minitest

.github/workflows/build-graphscope-wheels-linux.yml

@@ -38,14 +38,18 @@ jobs:
         # Due to an observation of changing hostname in github runners,
         # append 127.0.0.1 to etc/hosts to avoid DNS lookup.
         r=`cat /etc/hosts | grep $(hostname) || true`
-        if [ -z "${r}" ];then export hn=$(hostname); sudo -E bash -c 'echo "127.0.0.1 ${hn}" >> /etc/hosts'; fi
+        if [[ -z "${r}" ]]; then
+          export hn=$(hostname); sudo -E bash -c 'echo "127.0.0.1 ${hn}" >> /etc/hosts';
+        fi
         cat /etc/hosts
 
         # change the version for nightly release
         # 0.15.0 -> 0.15.0a20220808
         time=$(date "+%Y%m%d")
         version=$(cat ${GITHUB_WORKSPACE}/VERSION)
-        if [ "${{ GITHUB.REF }}" == "refs/heads/main" ];then echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; fi
+        if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]]; then
+          echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION;
+        fi
 
         cd ${GITHUB_WORKSPACE}/k8s/internal
         # build graphscope wheels

.github/workflows/build-graphscope-wheels-macos.yml

@@ -32,8 +32,6 @@ jobs:
         alias bash=$(brew --prefix)/bin/bash
         sudo mkdir /opt/graphscope
         sudo chown -R $(id -u):$(id -g) /opt/graphscope
-        # This will install vineyard by homebrew
-        # But we need to install vineyard to /opt/vineyard, so install it from source again.
 
         # Not sure when the first installation would fail, saying that the lock of python3.10 is taken
         # However the second trial would success.
@@ -89,7 +87,7 @@ jobs:
         # e.g. 0.15.0 -> 0.15.0a20220808
         time=$(date "+%Y%m%d")
         version=$(cat ${GITHUB_WORKSPACE}/VERSION)
-        if [ "${{ GITHUB.REF }}" == "refs/heads/main" ];
+        if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]];
         then
           echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION;
         fi
@@ -140,8 +138,6 @@ jobs:
         sudo mkdir /opt/graphscope
         sudo chown -R $(id -u):$(id -g) /opt/graphscope
 
-        # Not sure when the first installation would fail, saying that the lock of python3.10 is taken
-        # However the second trial would success.
         ./gs install-deps dev || true
         ./gs install-deps dev
 
@@ -180,7 +176,9 @@ jobs:
         # 0.15.0 -> 0.15.0a20220808
         time=$(date "+%Y%m%d")
         version=$(cat ${GITHUB_WORKSPACE}/VERSION)
-        if [ "${{ GITHUB.REF }}" == "refs/heads/main" ];then echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; fi
+        if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]]; then
+          echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION;
+        fi
 
         # build graphscope client wheel
         cd ${GITHUB_WORKSPACE}/k8s/internal

.github/workflows/gss.yml

@@ -113,7 +113,7 @@ jobs:
     runs-on: [self-hosted, ubuntu2004]
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     needs: [gremlin-test]
-    environment:
+    env:
       JAVA_HOME: /usr/lib/jvm/default-java
       GS_TEST_DIR: ${{ github.workspace }}/gstest
     steps:

Makefile

@@ -143,8 +143,8 @@ learning-install: learning
 learning: $(LEARNING_DIR)/graphlearn/built/lib/libgraphlearn_shared.$(SUFFIX)
 
 $(LEARNING_DIR)/graphlearn/built/lib/libgraphlearn_shared.$(SUFFIX):
-	git submodule update --init
-	cd $(LEARNING_DIR) && git submodule update --init third_party/pybind11
+	(git submodule update --init || true)
+	cd $(LEARNING_DIR) && (git submodule update --init third_party/pybind11 || true)
 	mkdir -p $(LEARNING_BUILD_DIR)
 	cd $(LEARNING_BUILD_DIR) && \
 	cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX) \

coordinator/gscoordinator/utils.py

@@ -64,6 +64,12 @@
 except KeyError:
     WORKSPACE = os.path.join(get_tempdir(), "gs")
 
+# make sure we have permission to create instance workspace
+try:
+    os.makedirs(os.path.join(WORKSPACE, ".ignore"), exist_ok=True)
+except:  # noqa: E722, pylint: disable=bare-except
+    WORKSPACE = os.path.expanduser("~/.graphscope/gs")
+
 # COORDINATOR_HOME
 #   1) get from gscoordinator python module, if failed,
 #   2) infer from current directory

coordinator/setup.py

@@ -61,7 +61,7 @@ def get_version(file):
 
 
 # copy any files contains in ${INSTALL_PREFIX} into site-packages/graphscope.runtime
-def _get_extra_data():
+def _get_extra_data():  # noqa: C901
     # Copy
     #   1) ${INSTALL_PREFIX}
     #   2) headers of arrow/glog/gflags/google/openmpi/vineyard
@@ -79,12 +79,11 @@ def _get_extra_data():
     def __unknown_platform(action, platform):
         raise RuntimeError(f"Unknown platform '{platform}' to {action}")
 
-    def __get_homebrew_prefix(package):
-        return (
-            subprocess.check_output([shutil.which("brew"), "--prefix", package])
-            .decode("utf-8", errors="ignore")
-            .strip("\n")
-        )
+    def __get_homebrew_prefix(package=None):
+        cmd = [shutil.which("brew"), "--prefix"]
+        if package is not None:
+            cmd.append(package)
+        return subprocess.check_output(cmd).decode("utf-8", errors="ignore").strip("\n")
 
     def __get_openmpi_prefix():
         if platform.system() == "Linux":
@@ -161,9 +160,14 @@ def __get_lib_suffix():
             data["/usr/include/msgpack"] = os.path.join(RUNTIME_ROOT, "include")
             data["/usr/include/msgpack.hpp"] = os.path.join(RUNTIME_ROOT, "include")
         elif platform.system() == "Darwin":
-            data["/usr/local/include/rapidjson"] = os.path.join(RUNTIME_ROOT, "include")
-            data["/usr/local/include/msgpack"] = os.path.join(RUNTIME_ROOT, "include")
-            data["/usr/local/include/msgpack.hpp"] = os.path.join(
+            homebrew_prefix = __get_homebrew_prefix()
+            data[f"{homebrew_prefix}/include/rapidjson"] = os.path.join(
+                RUNTIME_ROOT, "include"
+            )
+            data[f"{homebrew_prefix}/include/msgpack"] = os.path.join(
+                RUNTIME_ROOT, "include"
+            )
+            data[f"{homebrew_prefix}/include/msgpack.hpp"] = os.path.join(
                 RUNTIME_ROOT, "include"
             )
     elif name == "gs-apps":

k8s/internal/Makefile

@@ -19,6 +19,11 @@ endif
 
 # x86_64 or arm64
 ARCH := $(shell uname -m)
+ifeq ($(ARCH), arm64)
+	MACOS_WHEEL_VERSION := 12_0
+else
+	MACOS_WHEEL_VERSION := 11_0
+endif
 
 BUILD_PROGRESS    = auto
 
@@ -69,32 +74,32 @@ graphscope-darwin-py3:
 		export INSTALL_PREFIX=/opt/graphscope && \
 		for _ in {1..5}; do if make INSTALL_PREFIX=${INSTALL_PREFIX}; then break; fi; done && \
 		for _ in {1..5}; do if $(MAKE_INSTALL_COMMAND); then break; fi; done && \
-		sudo cp -rs ${INSTALL_PREFIX}/* /usr/local/ && \
+		(sudo cp -rs ${INSTALL_PREFIX}/* /usr/local/ || true) && \
 		python3 $(WORKING_DIR)/../utils/precompile.py --graph --app
 	# build and delocate wheel
 	cd $(WORKING_DIR)/../../coordinator && \
 		export WITH_EXTRA_DATA=ON && \
-		export GRAPHSCOPE_HOME=/usr/local && \
+		if [[ "${ARCH}" == "arm64" ]]; then export GRAPHSCOPE_HOME=/opt/homebrew; else export GRAPHSCOPE_HOME=/usr/local; fi && \
 		rm -rf build dist/*.whl || true && \
 		sudo strip -s $(WORKING_DIR)/../../analytical_engine/exported_symbols_osx.lds ${INSTALL_PREFIX}/bin/grape_engine || true && \
 		sudo strip ${INSTALL_PREFIX}/bin/gaia_executor && \
 		export DYLD_LIBRARY_PATH=/usr/local/lib:$$DYLD_LIBRARY_PATH && \
 		install_name_tool -add_rpath /usr/local/lib ${INSTALL_PREFIX}/bin/gaia_executor && \
-		package_name=gs-include python3 setup.py bdist_wheel --plat=macosx_10_9_${ARCH} && \
+		package_name=gs-include python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \
 		rm -rf build && \
-		package_name=gs-apps python3 setup.py bdist_wheel --plat=macosx_10_9_${ARCH} && \
+		package_name=gs-apps python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \
 		rm -rf build && \
-		package_name=gs-engine python3 setup.py bdist_wheel --plat=macosx_10_9_${ARCH} && \
+		package_name=gs-engine python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \
 		rm -rf build && \
-		package_name=gs-coordinator python3 setup.py bdist_wheel --plat=macosx_10_9_${ARCH} && \
+		package_name=gs-coordinator python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \
 		rm -rf build && \
-		package_name=graphscope python3 setup.py bdist_wheel --plat=macosx_10_9_${ARCH} && \
+		package_name=graphscope python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \
 		pip3 install delocate && \
 		for wheel in `ls dist/*.whl`; do \
 			delocate-listdeps -a -d $$wheel; \
 			delocate-wheel -w dist/wheelhouse -v $$wheel && rm $$wheel; \
 		done
-# TODO(siyuan): Remove the sed
+
 graphscope-manylinux2014-py3-nodocker:
 	cd $(WORKING_DIR)/../.. && \
 	for _ in {1..5}; do if make INSTALL_PREFIX=${INSTALL_PREFIX}; then break; fi; done && \
@@ -149,11 +154,11 @@ graphscope-client-manylinux2014-py3-nodocker:
 		cd $(WORKING_DIR)/../../python; \
 		export PATH=/opt/python/$$py/bin:$$PATH; \
 		python3 -m pip install -U pip; \
-		if [ "$$py" == "cp311-cp311" ]; then \
-			python3 -m pip install "numpy" "pandas" "grpcio" "grpcio-tools" wheel "auditwheel==5.0.0"; \
-		elif [ "$$py" == "cp310-cp310" ]; then \
-			 python3 -m pip install "numpy==1.21.6" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" wheel "auditwheel==5.0.0"; \
-		elif [ "$$py" == "cp39-cp39" ]; then \
+		if [[ "$$py" == "cp311-cp311" ]]; then \
+			python3 -m pip install "numpy==1.23.2" "pandas" "grpcio" "grpcio-tools" wheel "auditwheel==5.0.0"; \
+		elif [[ "$$py" == "cp310-cp310" ]]; then \
+			python3 -m pip install "numpy==1.21.2" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" wheel "auditwheel==5.0.0"; \
+		elif [[ "$$py" == "cp39-cp39" ]]; then \
 			python3 -m pip install "numpy==1.19.3" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" wheel "auditwheel==5.0.0"; \
 		else \
 			python3 -m pip install "numpy==1.18.5" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" wheel "auditwheel==5.0.0"; \
@@ -167,10 +172,11 @@ graphscope-client-manylinux2014-py3-nodocker:
 
 graphscope-client-darwin-py3:
 	cd $(WORKING_DIR)/../../learning_engine/graph-learn && \
-		git submodule update --init third_party/pybind11 && \
+		(git submodule update --init third_party/pybind11 || true) && \
 		export GRAPHSCOPE_HOME=${GRAPHSCOPE_HOME} && \
 		cd graphlearn && \
-		rm -rf cmake-build && \
+		sudo rm -rf cmake-build && \
+		sudo rm -rf src/generated && \
 		mkdir -p cmake-build && \
 		cd cmake-build && \
 		cmake -DKNN=OFF -DWITH_VINEYARD=ON -DTESTING=OFF .. && \
@@ -179,15 +185,25 @@ graphscope-client-darwin-py3:
 		cd $(WORKING_DIR)/../../python && \
 		py=$$(python3 -V 2>&1 | awk '{print $$2}' | awk -F '.' '{print $$1$$2}') && \
 		pip3 install -U pip && \
-		if [ "$$py" == "311" ]; then \
-			pip3 install "numpy" "pandas" "grpcio" "grpcio-tools" delocate wheel; \
-		elif [ "$$py" == "310" ]; then \
-			pip3 install "numpy==1.21.6" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \
+		if [[ "$$py" == "311" ]]; then \
+			pip3 install "numpy==1.23.2" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \
+		elif [[ "$$py" == "310" ]]; then \
+			pip3 install "numpy==1.21.2" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \
+		elif [[ "$$py" == "cp39-cp39" ]]; then \
+			if [[ "${ARCH}" == "arm64" ]]; then \
+				pip3 install "numpy==1.21.0" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \
+			else \
+				pip3 install "numpy==1.19.3" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \
+			fi; \
 		else \
-			pip3 install "numpy==1.18.5" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \
+			if [[ "${ARCH}" == "arm64" ]]; then \
+				pip3 install "numpy==1.21.0" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \
+			else \
+				pip3 install "numpy==1.18.5" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \
+			fi; \
 		fi; \
 		rm -rf build dist/*.whl || true && \
-		python3 setup.py bdist_wheel --plat=macosx_10_9_${ARCH} && \
+		python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \
 		for wheel in `ls dist/*.whl`; do \
 			delocate-wheel -w dist/wheelhouse -v $$wheel && rm $$wheel; \
 		done