From 3b11b77ff1da76d255b6cd0c9ade7546a610813f Mon Sep 17 00:00:00 2001 From: Tao He Date: Mon, 15 May 2023 17:55:22 +0800 Subject: [PATCH] Build wheels for macOS m1 (#2701) ## Related issue number Fixes #2083 Fixes #2668 Signed-off-by: Tao He --- .cirrus.yml | 137 ++++++++++++++++++ .../build-graphscope-wheels-linux.yml | 8 +- .../build-graphscope-wheels-macos.yml | 10 +- .github/workflows/gss.yml | 2 +- Makefile | 4 +- coordinator/gscoordinator/utils.py | 6 + coordinator/setup.py | 24 +-- k8s/internal/Makefile | 58 +++++--- python/graphscope/tests/minitest/test_min.py | 5 +- 9 files changed, 210 insertions(+), 44 deletions(-) create mode 100644 .cirrus.yml diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 000000000000..890c66b3c87f --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,137 @@ +# only run for releases +# only_if: $CIRRUS_TAG != '' + +env: + MACOSX_DEPLOYMENT_TARGET: "11" + +macosx_arm64_wheel_task: + macos_instance: + image: ghcr.io/cirruslabs/macos-monterey-base:latest + + timeout_in: 120m + + # can be manually triggered when needed (nightly or release) + trigger_type: manual + + checkout_script: + - git submodule init + - git submodule update + - | + pushd ./learning_engine/graph-learn + git submodule update --init third_party/pybind11 + popd + - | + # change the version for nightly release + # e.g. 0.15.0 -> 0.15.0a20220808 + time=$(date "+%Y%m%d") + version=$(cat ./VERSION) + if [[ "$CIRRUS_TAG" == "" ]]; + then + echo "${version}a${time}" > ./VERSION; + fi + + install_deps_script: + - brew update + - brew install coreutils pyenv python + + install_graphscope_deps_script: | + brew update || true + brew install bash coreutils + alias bash=$(brew --prefix)/bin/bash + sudo mkdir /opt/graphscope + sudo chown -R $(id -u):$(id -g) /opt/graphscope + + # Not sure when the first installation would fail, saying that the lock of python3.10 is taken + # However the second trial would success. + ./gs install-deps dev || true + ./gs install-deps dev + + brew install openjdk@11 + sudo ln -sfn /opt/homebrew/opt/openjdk@11/libexec/openjdk.jdk /Library/Java/JavaVirtualMachines/openjdk-11.jdk + + python3 -m pip install -r ./python/requirements.txt + python3 -m pip install -r ./python/requirements-dev.txt + + echo "export PATH=/opt/homebrew/opt/coreutils/libexec/gnubin:\$PATH" >> ~/.graphscope_env + + # for debugging with "Rerun with Terminal Access" on cirrus CI + bash ./debugging.sh || true + + build_graphscope_wheels_script: | + # source environment variable + . ~/.graphscope_env + python3 -m pip install libclang + echo ${CC} + + # for grpc-java + /usr/sbin/softwareupdate --install-rosetta --agree-to-license + + # for building GLE + git config --global --add safe.directory . + + # prepare + pushd ./python + python3 setup.py build_proto + popd + pushd ./coordinator + python3 setup.py build_builtin + popd + + # build graphscope server wheel + pushd ./k8s/internal + sudo -E env PATH=$PATH make graphscope-py3-package GRAPHSCOPE_HOME=/usr/local + popd + + graphscope_artifacts: + path: coordinator/dist/wheelhouse/*.whl + + install_various_python_script: + - pyenv install 3.8.16 + - pyenv install 3.9.16 + - pyenv install 3.10.10 + - pyenv install 3.11.2 + + build_graphscope_client_script: | + . ~/.graphscope_env + + for py in 3.8.16 3.9.16 3.10.10 3.11.2; do + current_python=$(pyenv root)/versions/$py/bin/python + echo "Python is: $current_python, $($current_python --version)" + $current_python -m pip install delocate numpy wheel + + export PATH=$(dirname $current_python):$PATH + python3 -c "import sys; print(sys.version)" + + # build graphscope client wheel + pushd ./k8s/internal + make graphscope-client-py3-package GRAPHSCOPE_HOME=/usr/local + popd + done + + graphscope_client_artifacts: + path: python/dist/wheelhouse/*.whl + + run_tests_script: | + pushd python/dist/wheelhouse + for f in * ; do + python3 -m pip install $f --user || true; + done + popd + + sudo python3 -m pip install coordinator/dist/wheelhouse/*.whl + + # run tests + export GS_TEST_DIR="$(pwd)/gstest" + git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} + + export JAVA_HOME=$(/usr/libexec/java_home -v11) + export PATH=$JAVA_HOME/bin:$HOME/.local/bin:$PATH + + sudo mkdir -p /tmp || true + sudo chmod -R a+wrx /tmp || true + + # for debugging with "Rerun with Terminal Access" on cirrus CI + bash ./debugging.sh || true + + python3 -c "import sys; print(sys.version)" + python3 -m pytest -s -v $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/tests/minitest diff --git a/.github/workflows/build-graphscope-wheels-linux.yml b/.github/workflows/build-graphscope-wheels-linux.yml index 3a4142b41a2c..0535ad1f3fc4 100644 --- a/.github/workflows/build-graphscope-wheels-linux.yml +++ b/.github/workflows/build-graphscope-wheels-linux.yml @@ -38,14 +38,18 @@ jobs: # Due to an observation of changing hostname in github runners, # append 127.0.0.1 to etc/hosts to avoid DNS lookup. r=`cat /etc/hosts | grep $(hostname) || true` - if [ -z "${r}" ];then export hn=$(hostname); sudo -E bash -c 'echo "127.0.0.1 ${hn}" >> /etc/hosts'; fi + if [[ -z "${r}" ]]; then + export hn=$(hostname); sudo -E bash -c 'echo "127.0.0.1 ${hn}" >> /etc/hosts'; + fi cat /etc/hosts # change the version for nightly release # 0.15.0 -> 0.15.0a20220808 time=$(date "+%Y%m%d") version=$(cat ${GITHUB_WORKSPACE}/VERSION) - if [ "${{ GITHUB.REF }}" == "refs/heads/main" ];then echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; fi + if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]]; then + echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; + fi cd ${GITHUB_WORKSPACE}/k8s/internal # build graphscope wheels diff --git a/.github/workflows/build-graphscope-wheels-macos.yml b/.github/workflows/build-graphscope-wheels-macos.yml index d39b0184066e..3f001536ebf0 100644 --- a/.github/workflows/build-graphscope-wheels-macos.yml +++ b/.github/workflows/build-graphscope-wheels-macos.yml @@ -32,8 +32,6 @@ jobs: alias bash=$(brew --prefix)/bin/bash sudo mkdir /opt/graphscope sudo chown -R $(id -u):$(id -g) /opt/graphscope - # This will install vineyard by homebrew - # But we need to install vineyard to /opt/vineyard, so install it from source again. # Not sure when the first installation would fail, saying that the lock of python3.10 is taken # However the second trial would success. @@ -89,7 +87,7 @@ jobs: # e.g. 0.15.0 -> 0.15.0a20220808 time=$(date "+%Y%m%d") version=$(cat ${GITHUB_WORKSPACE}/VERSION) - if [ "${{ GITHUB.REF }}" == "refs/heads/main" ]; + if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]]; then echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; fi @@ -140,8 +138,6 @@ jobs: sudo mkdir /opt/graphscope sudo chown -R $(id -u):$(id -g) /opt/graphscope - # Not sure when the first installation would fail, saying that the lock of python3.10 is taken - # However the second trial would success. ./gs install-deps dev || true ./gs install-deps dev @@ -180,7 +176,9 @@ jobs: # 0.15.0 -> 0.15.0a20220808 time=$(date "+%Y%m%d") version=$(cat ${GITHUB_WORKSPACE}/VERSION) - if [ "${{ GITHUB.REF }}" == "refs/heads/main" ];then echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; fi + if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]]; then + echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; + fi # build graphscope client wheel cd ${GITHUB_WORKSPACE}/k8s/internal diff --git a/.github/workflows/gss.yml b/.github/workflows/gss.yml index 154227199483..d7b0ace98288 100644 --- a/.github/workflows/gss.yml +++ b/.github/workflows/gss.yml @@ -113,7 +113,7 @@ jobs: runs-on: [self-hosted, ubuntu2004] if: ${{ github.repository == 'alibaba/GraphScope' }} needs: [gremlin-test] - environment: + env: JAVA_HOME: /usr/lib/jvm/default-java GS_TEST_DIR: ${{ github.workspace }}/gstest steps: diff --git a/Makefile b/Makefile index 1485cabd28a5..897ce87a360c 100644 --- a/Makefile +++ b/Makefile @@ -143,8 +143,8 @@ learning-install: learning learning: $(LEARNING_DIR)/graphlearn/built/lib/libgraphlearn_shared.$(SUFFIX) $(LEARNING_DIR)/graphlearn/built/lib/libgraphlearn_shared.$(SUFFIX): - git submodule update --init - cd $(LEARNING_DIR) && git submodule update --init third_party/pybind11 + (git submodule update --init || true) + cd $(LEARNING_DIR) && (git submodule update --init third_party/pybind11 || true) mkdir -p $(LEARNING_BUILD_DIR) cd $(LEARNING_BUILD_DIR) && \ cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX) \ diff --git a/coordinator/gscoordinator/utils.py b/coordinator/gscoordinator/utils.py index ce25358459f0..cf2b111c01a0 100644 --- a/coordinator/gscoordinator/utils.py +++ b/coordinator/gscoordinator/utils.py @@ -64,6 +64,12 @@ except KeyError: WORKSPACE = os.path.join(get_tempdir(), "gs") +# make sure we have permission to create instance workspace +try: + os.makedirs(os.path.join(WORKSPACE, ".ignore"), exist_ok=True) +except: # noqa: E722, pylint: disable=bare-except + WORKSPACE = os.path.expanduser("~/.graphscope/gs") + # COORDINATOR_HOME # 1) get from gscoordinator python module, if failed, # 2) infer from current directory diff --git a/coordinator/setup.py b/coordinator/setup.py index 77ff8b5d23df..b4dc523f44a5 100644 --- a/coordinator/setup.py +++ b/coordinator/setup.py @@ -61,7 +61,7 @@ def get_version(file): # copy any files contains in ${INSTALL_PREFIX} into site-packages/graphscope.runtime -def _get_extra_data(): +def _get_extra_data(): # noqa: C901 # Copy # 1) ${INSTALL_PREFIX} # 2) headers of arrow/glog/gflags/google/openmpi/vineyard @@ -79,12 +79,11 @@ def _get_extra_data(): def __unknown_platform(action, platform): raise RuntimeError(f"Unknown platform '{platform}' to {action}") - def __get_homebrew_prefix(package): - return ( - subprocess.check_output([shutil.which("brew"), "--prefix", package]) - .decode("utf-8", errors="ignore") - .strip("\n") - ) + def __get_homebrew_prefix(package=None): + cmd = [shutil.which("brew"), "--prefix"] + if package is not None: + cmd.append(package) + return subprocess.check_output(cmd).decode("utf-8", errors="ignore").strip("\n") def __get_openmpi_prefix(): if platform.system() == "Linux": @@ -161,9 +160,14 @@ def __get_lib_suffix(): data["/usr/include/msgpack"] = os.path.join(RUNTIME_ROOT, "include") data["/usr/include/msgpack.hpp"] = os.path.join(RUNTIME_ROOT, "include") elif platform.system() == "Darwin": - data["/usr/local/include/rapidjson"] = os.path.join(RUNTIME_ROOT, "include") - data["/usr/local/include/msgpack"] = os.path.join(RUNTIME_ROOT, "include") - data["/usr/local/include/msgpack.hpp"] = os.path.join( + homebrew_prefix = __get_homebrew_prefix() + data[f"{homebrew_prefix}/include/rapidjson"] = os.path.join( + RUNTIME_ROOT, "include" + ) + data[f"{homebrew_prefix}/include/msgpack"] = os.path.join( + RUNTIME_ROOT, "include" + ) + data[f"{homebrew_prefix}/include/msgpack.hpp"] = os.path.join( RUNTIME_ROOT, "include" ) elif name == "gs-apps": diff --git a/k8s/internal/Makefile b/k8s/internal/Makefile index dfaf1c23576f..db0edfb37070 100644 --- a/k8s/internal/Makefile +++ b/k8s/internal/Makefile @@ -19,6 +19,11 @@ endif # x86_64 or arm64 ARCH := $(shell uname -m) +ifeq ($(ARCH), arm64) + MACOS_WHEEL_VERSION := 12_0 +else + MACOS_WHEEL_VERSION := 11_0 +endif BUILD_PROGRESS = auto @@ -69,32 +74,32 @@ graphscope-darwin-py3: export INSTALL_PREFIX=/opt/graphscope && \ for _ in {1..5}; do if make INSTALL_PREFIX=${INSTALL_PREFIX}; then break; fi; done && \ for _ in {1..5}; do if $(MAKE_INSTALL_COMMAND); then break; fi; done && \ - sudo cp -rs ${INSTALL_PREFIX}/* /usr/local/ && \ + (sudo cp -rs ${INSTALL_PREFIX}/* /usr/local/ || true) && \ python3 $(WORKING_DIR)/../utils/precompile.py --graph --app # build and delocate wheel cd $(WORKING_DIR)/../../coordinator && \ export WITH_EXTRA_DATA=ON && \ - export GRAPHSCOPE_HOME=/usr/local && \ + if [[ "${ARCH}" == "arm64" ]]; then export GRAPHSCOPE_HOME=/opt/homebrew; else export GRAPHSCOPE_HOME=/usr/local; fi && \ rm -rf build dist/*.whl || true && \ sudo strip -s $(WORKING_DIR)/../../analytical_engine/exported_symbols_osx.lds ${INSTALL_PREFIX}/bin/grape_engine || true && \ sudo strip ${INSTALL_PREFIX}/bin/gaia_executor && \ export DYLD_LIBRARY_PATH=/usr/local/lib:$$DYLD_LIBRARY_PATH && \ install_name_tool -add_rpath /usr/local/lib ${INSTALL_PREFIX}/bin/gaia_executor && \ - package_name=gs-include python3 setup.py bdist_wheel --plat=macosx_10_9_${ARCH} && \ + package_name=gs-include python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ rm -rf build && \ - package_name=gs-apps python3 setup.py bdist_wheel --plat=macosx_10_9_${ARCH} && \ + package_name=gs-apps python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ rm -rf build && \ - package_name=gs-engine python3 setup.py bdist_wheel --plat=macosx_10_9_${ARCH} && \ + package_name=gs-engine python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ rm -rf build && \ - package_name=gs-coordinator python3 setup.py bdist_wheel --plat=macosx_10_9_${ARCH} && \ + package_name=gs-coordinator python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ rm -rf build && \ - package_name=graphscope python3 setup.py bdist_wheel --plat=macosx_10_9_${ARCH} && \ + package_name=graphscope python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ pip3 install delocate && \ for wheel in `ls dist/*.whl`; do \ delocate-listdeps -a -d $$wheel; \ delocate-wheel -w dist/wheelhouse -v $$wheel && rm $$wheel; \ done -# TODO(siyuan): Remove the sed + graphscope-manylinux2014-py3-nodocker: cd $(WORKING_DIR)/../.. && \ for _ in {1..5}; do if make INSTALL_PREFIX=${INSTALL_PREFIX}; then break; fi; done && \ @@ -149,11 +154,11 @@ graphscope-client-manylinux2014-py3-nodocker: cd $(WORKING_DIR)/../../python; \ export PATH=/opt/python/$$py/bin:$$PATH; \ python3 -m pip install -U pip; \ - if [ "$$py" == "cp311-cp311" ]; then \ - python3 -m pip install "numpy" "pandas" "grpcio" "grpcio-tools" wheel "auditwheel==5.0.0"; \ - elif [ "$$py" == "cp310-cp310" ]; then \ - python3 -m pip install "numpy==1.21.6" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" wheel "auditwheel==5.0.0"; \ - elif [ "$$py" == "cp39-cp39" ]; then \ + if [[ "$$py" == "cp311-cp311" ]]; then \ + python3 -m pip install "numpy==1.23.2" "pandas" "grpcio" "grpcio-tools" wheel "auditwheel==5.0.0"; \ + elif [[ "$$py" == "cp310-cp310" ]]; then \ + python3 -m pip install "numpy==1.21.2" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" wheel "auditwheel==5.0.0"; \ + elif [[ "$$py" == "cp39-cp39" ]]; then \ python3 -m pip install "numpy==1.19.3" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" wheel "auditwheel==5.0.0"; \ else \ python3 -m pip install "numpy==1.18.5" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" wheel "auditwheel==5.0.0"; \ @@ -167,10 +172,11 @@ graphscope-client-manylinux2014-py3-nodocker: graphscope-client-darwin-py3: cd $(WORKING_DIR)/../../learning_engine/graph-learn && \ - git submodule update --init third_party/pybind11 && \ + (git submodule update --init third_party/pybind11 || true) && \ export GRAPHSCOPE_HOME=${GRAPHSCOPE_HOME} && \ cd graphlearn && \ - rm -rf cmake-build && \ + sudo rm -rf cmake-build && \ + sudo rm -rf src/generated && \ mkdir -p cmake-build && \ cd cmake-build && \ cmake -DKNN=OFF -DWITH_VINEYARD=ON -DTESTING=OFF .. && \ @@ -179,15 +185,25 @@ graphscope-client-darwin-py3: cd $(WORKING_DIR)/../../python && \ py=$$(python3 -V 2>&1 | awk '{print $$2}' | awk -F '.' '{print $$1$$2}') && \ pip3 install -U pip && \ - if [ "$$py" == "311" ]; then \ - pip3 install "numpy" "pandas" "grpcio" "grpcio-tools" delocate wheel; \ - elif [ "$$py" == "310" ]; then \ - pip3 install "numpy==1.21.6" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \ + if [[ "$$py" == "311" ]]; then \ + pip3 install "numpy==1.23.2" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \ + elif [[ "$$py" == "310" ]]; then \ + pip3 install "numpy==1.21.2" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \ + elif [[ "$$py" == "cp39-cp39" ]]; then \ + if [[ "${ARCH}" == "arm64" ]]; then \ + pip3 install "numpy==1.21.0" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \ + else \ + pip3 install "numpy==1.19.3" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \ + fi; \ else \ - pip3 install "numpy==1.18.5" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \ + if [[ "${ARCH}" == "arm64" ]]; then \ + pip3 install "numpy==1.21.0" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \ + else \ + pip3 install "numpy==1.18.5" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \ + fi; \ fi; \ rm -rf build dist/*.whl || true && \ - python3 setup.py bdist_wheel --plat=macosx_10_9_${ARCH} && \ + python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ for wheel in `ls dist/*.whl`; do \ delocate-wheel -w dist/wheelhouse -v $$wheel && rm $$wheel; \ done diff --git a/python/graphscope/tests/minitest/test_min.py b/python/graphscope/tests/minitest/test_min.py index 44734fc32359..228d3d69c21f 100644 --- a/python/graphscope/tests/minitest/test_min.py +++ b/python/graphscope/tests/minitest/test_min.py @@ -219,8 +219,9 @@ def test_demo_with_default_session(ogbn_small_script): sub_graph = sub_graph.add_column(ret1, {"kcore": "r"}) sub_graph = sub_graph.add_column(ret2, {"tc": "r"}) - # waiting for graphlearn support python3.10 - if sys.version_info[1] >= 10: + try: + import tensorflow + except ImportError: return # GLE on ogbn_mag_small graph