From 8fd52b4e085e36039a9399d3b671b83ba14d5fb8 Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Tue, 22 Dec 2020 14:54:05 +0800 Subject: [PATCH] Migrate CI to GitHub runners. (#21) Migrate some CI steps to GitHub runners. While two steps that requires k8s cluster still remains on self-hosted, because resource requested is exceed the standard configuration of GitHub runner (2 core CPU, 8G memory). This PR also * Parallels the building process of three engines, * Reduce the size of artifacts, * Add code coverage report for graphscope. However, this PR also introduced several overhead, which is two additional Dockerfile for assemble artifact content into graphscope-runtime image, however, this is basically copied from the stage 2 of the normal Dockerfile. And the manipulation of upload and download artifact is more complicated than before (to preserve file permissions of artifact, one need to assemble all files into a tar ball) --- .github/codecov.yml | 20 + .github/workflows/ci.yml | 532 ++++++++++-------- .github/workflows/docs.yml | 1 - .github/workflows/graphscope.Dockerfile | 26 + .github/workflows/manager.Dockerfile | 22 + README.md | 1 + analytical_engine/core/grape_instance.cc | 3 + k8s/graphscope.Dockerfile | 2 +- k8s/gsruntime.Dockerfile | 2 +- python/.coveragerc | 4 + .../deploy/tests/test_demo_script.py | 139 ++--- python/graphscope/learning/graph.py | 2 +- 12 files changed, 436 insertions(+), 318 deletions(-) create mode 100644 .github/codecov.yml create mode 100644 .github/workflows/graphscope.Dockerfile create mode 100644 .github/workflows/manager.Dockerfile create mode 100644 python/.coveragerc diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 000000000000..108be8475267 --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,20 @@ +codecov: + require_ci_to_pass: yes + +coverage: + precision: 2 + round: down + range: "60...80" + +parsers: + gcov: + branch_detection: + conditional: yes + loop: yes + method: no + macro: no + +comment: + layout: "reach,diff,flags,files,footer" + behavior: default + require_changes: no diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5cc491a2c400..f752dc4b2477 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,47 +22,40 @@ on: - 'scripts/**' jobs: - build: - runs-on: self-hosted + gae: + runs-on: ubuntu-20.04 + defaults: + run: + shell: scl enable devtoolset-7 -- bash --noprofile --norc -eo pipefail {0} + container: + image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:latest + options: + --shm-size 4096m strategy: matrix: experimental: [ON, OFF] - profile: [release] + steps: - - name: Clean up - shell: bash + - name: Install Dependencies run: | - echo "CI is running on host $(curl -s 'https://api.ipify.org')" - sudo docker ps --no-trunc -aqf "status=exited" | xargs sudo docker rm || true - sudo docker images --no-trunc -aqf "dangling=true" | xargs sudo docker rmi -f || true - sudo rm -rf ./* || true - sudo rm -rf /tmp/graphscope/app_* || true - find ./ -name "*.egg-info" | xargs sudo rm -rf || true - find ./ -name "*.whl" | xargs sudo rm -rf || true - find ./ -name "*_pb2.py" | xargs sudo rm -rf || true - find ./ -name "*_pb2_grpc.py" | xargs sudo rm -rf || true + yum install -y http://opensource.wandisco.com/centos/7/git/x86_64/wandisco-git-release-7-2.noarch.rpm + yum install -y git - - name: Check out code - uses: actions/checkout@v2.3.2 - with: - submodules: true + - uses: actions/checkout@v2.3.2 - name: Cpp Format And Lint Check run: | # install clang-format if [ ! -f /usr/bin/clang-format ]; then - sudo curl -L https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-22538c65/clang-format-8_linux-amd64 --output /usr/bin/clang-format - sudo chmod +x /usr/bin/clang-format + curl -L https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-22538c65/clang-format-8_linux-amd64 --output /usr/bin/clang-format + chmod +x /usr/bin/clang-format fi - # run clang-format cd analytical_engine/ find ./apps ./benchmarks ./core ./frame ./misc ./test -name "*.h" | xargs clang-format -i --style=file find ./apps ./benchmarks ./core ./frame ./misc ./test -name "*.cc" | xargs clang-format -i --style=file - # validate format function prepend() { while read line; do echo "${1}${line}"; done; } - GIT_DIFF=$(git diff --ignore-submodules) if [[ -n $GIT_DIFF ]]; then echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" @@ -83,8 +76,6 @@ jobs: echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" exit -1 fi - - # run cpplint python3 ./misc/cpplint.py --recursive ./apps ./benchmarks ./core ./frame ./test - name: Python Format and Lint Check @@ -101,216 +92,242 @@ jobs: python3 -m black --check --diff . python3 -m flake8 . - - name: Get Test Data - shell: bash - run: | - export GS_TEST_DIR=${GITHUB_WORKSPACE}/gstest - git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} || true - - - name: Build image - shell: bash + - name: Build run: | - sudo docker build --build-arg EXPERIMENTAL_ON=${{ matrix.experimental }} \ - --build-arg profile=${{ matrix.profile }} \ - --build-arg ci=${{ matrix.ci }} \ - -t registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} \ - --network=host \ - -f ./k8s/graphscope.Dockerfile . + strip /opt/graphscope/lib/* || true + strip /opt/graphscope/lib64/* || true + + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/graphscope/lib:/opt/graphscope/lib64 + cd analytical_engine + mkdir build + cd build + cmake .. -DCMAKE_PREFIX_PATH=/opt/graphscope \ + -DCMAKE_INSTALL_PREFIX=/opt/graphscope \ + -DEXPERIMENTAL_ON=${{ matrix.experimental }} + make -j`nproc` + make install + cd ${GITHUB_WORKSPACE} + cp k8s/kube_ssh /opt/graphscope/bin/kube_ssh + cp k8s/pre_stop.py /opt/graphscope/bin/pre_stop.py + tar -cf gae.tar /opt/graphscope - name: Run GAE Tests - shell: bash run: | - sudo docker run --rm --shm-size=4096m \ - -v ${GS_TEST_DIR}:/root/gstest \ - -v `pwd`:/root/gs \ - registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} \ - sh -c "echo Container id $(hostname) && \ - set pipefail && \ - export GS_TEST_DIR='/root/gstest' && \ - cd /root/gs/analytical_engine && \ - mkdir build && \ - cd build && \ - cmake -DEXPERIMENTAL_ON=${{ matrix.experimental }} .. && \ - make run_app run_vy_app run_pregel_app -j`nproc` && \ - bash /root/gs/analytical_engine/test/app_tests.sh --test_dir /root/gstest" - - - name: Save Image - if: matrix.experimental == 'ON' - run: sudo docker save registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} | gzip > graphscope-${{ github.sha }}.tar.gz + export GS_TEST_DIR=${GITHUB_WORKSPACE}/gstest + export PATH=${PATH}:/opt/graphscope/bin/ + git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} + + cd analytical_engine/build + ../test/app_tests.sh --test_dir ${GITHUB_WORKSPACE}/gstest - - name: Archive image + - name: Upload Artifact if: matrix.experimental == 'ON' uses: actions/upload-artifact@v2 with: - name: graphscope - path: graphscope-${{ github.sha }}.tar.gz + name: gae-${{ github.sha }} + path: gae.tar retention-days: 5 - - name: Clean up - shell: bash + gie: + runs-on: ubuntu-20.04 + defaults: + run: + shell: scl enable devtoolset-7 -- bash --noprofile --norc -eo pipefail {0} + container: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:latest + steps: + - name: Install Dependencies run: | - sudo docker rmi registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} || true + yum install -y http://opensource.wandisco.com/centos/7/git/x86_64/wandisco-git-release-7-2.noarch.rpm + yum install -y git + wget --no-verbose https://golang.org/dl/go1.15.5.linux-amd64.tar.gz + tar -C /usr/local -xzf go1.15.5.linux-amd64.tar.gz + curl -sf -L https://static.rust-lang.org/rustup.sh | sh -s -- -y --profile minimal --default-toolchain 1.48.0 + echo "source ~/.cargo/env" >> ~/.bashrc - build-manager: - runs-on: self-hosted + - uses: actions/checkout@v2.3.2 + + - name: Build + run: | + export PATH=${PATH}:/usr/local/go/bin + export CMAKE_PREFIX_PATH=/opt/graphscope + export LIBRARY_PATH=$LIBRARY_PATH:/opt/graphscope/lib + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/graphscope/lib:/opt/graphscope/lib64 + source ~/.bashrc + + cd interactive_engine/src/executor + ./exec.sh cargo build --all --release + cd ${GITHUB_WORKSPACE} + tar -cf gie.tar interactive_engine/src/executor/target/release/executor \ + interactive_engine/src/executor/store/log4rs.yml \ + interactive_engine/deploy/docker/dockerfile + + - uses: actions/upload-artifact@v2 + with: + name: gie-${{ github.sha }} + path: gie.tar + retention-days: 5 + + gle: + runs-on: ubuntu-20.04 + defaults: + run: + shell: scl enable devtoolset-7 -- bash --noprofile --norc -eo pipefail {0} + container: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:latest steps: - - name: Clean up - shell: bash + - name: Install Dependencies run: | - echo "CI is running on host $(curl -s 'https://api.ipify.org')" - sudo docker ps --no-trunc -aqf "status=exited" | xargs sudo docker rm || true - sudo docker images --no-trunc -aqf "dangling=true" | xargs sudo docker rmi -f || true - sudo rm -rf ./* || true - sudo rm -rf /tmp/graphscope/app_* || true - find ./ -name "*.egg-info" | xargs sudo rm -rf || true - find ./ -name "*.whl" | xargs sudo rm -rf || true - find ./ -name "*_pb2.py" | xargs sudo rm -rf || true - find ./ -name "*_pb2_grpc.py" | xargs sudo rm -rf || true + yum install -y http://opensource.wandisco.com/centos/7/git/x86_64/wandisco-git-release-7-2.noarch.rpm + yum install -y git - - name: Check out code - uses: actions/checkout@v2.3.2 + - uses: actions/checkout@v2.3.2 with: submodules: true - - - name: Build manager - shell: bash + - name: Build run: | - cd ./k8s - sudo make manager REGISTRY='registry.cn-hongkong.aliyuncs.com' + strip /opt/graphscope/lib/* || true + strip /opt/graphscope/lib64/* || true + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/graphscope/lib:/opt/graphscope/lib64 + export PATH=${PATH}:/opt/rh/devtoolset-7/root/usr/bin + cd learning_engine/graph-learn + git submodule update --init third_party/pybind11 + mkdir cmake-build + cd cmake-build + cmake -DCMAKE_PREFIX_PATH=/opt/graphscope \ + -DCMAKE_INSTALL_PREFIX=/opt/graphscope \ + -DWITH_VINEYARD=ON \ + -DTESTING=OFF .. + make graphlearn_shared install -j + + - name: build wheel + run: | + sed -i 's/p.error/logger.warning/g' /usr/local/lib/python3.6/site-packages/auditwheel/main_repair.py - - name: Save Image - run: sudo docker save registry.cn-hongkong.aliyuncs.com/graphscope/maxgraph_standalone_manager:1.0 | gzip > maxgraph_standalone_manager.tar.gz + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/graphscope/lib:/opt/graphscope/lib64 + export WITH_LEARNING_ENGINE=ON + pushd python + pip3 install -U setuptools + pip3 install -r requirements.txt -r requirements-dev.txt + python3 setup.py bdist_wheel + pushd ./dist + auditwheel repair --plat=manylinux2014_x86_64 ./*.whl + cp ./wheelhouse/* /opt/graphscope/dist/ + popd + popd + pushd coordinator + pip3 install -r requirements.txt -r requirements-dev.txt + python3 setup.py bdist_wheel + cp ./dist/* /opt/graphscope/dist/ - - name: Archive image - uses: actions/upload-artifact@v2 + cd ${GITHUB_WORKSPACE} + tar -cf gle.tar /opt/graphscope + + - uses: actions/upload-artifact@v2 with: - name: maxgraph_standalone_manager - path: maxgraph_standalone_manager.tar.gz + name: gle-${{ github.sha }} + path: gle.tar retention-days: 5 - - name: Clean - shell: bash - run: | - sudo docker rmi registry.cn-hongkong.aliyuncs.com/graphscope/maxgraph_standalone_manager:1.0 \ - || true - - - gae-and-python-tests: - runs-on: self-hosted - needs: [build, build-manager] - strategy: - matrix: - experimental: [ON, OFF] + manager: + runs-on: ubuntu-20.04 + env: + profile: release + defaults: + run: + shell: scl enable devtoolset-7 -- bash --noprofile --norc -eo pipefail {0} + container: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-runtime:latest steps: - - name: Clean up - shell: bash + - name: Install Dependencies run: | - echo "CI is running on host $(curl -s 'https://api.ipify.org')" - sudo docker ps --no-trunc -aqf "status=exited" | xargs sudo docker rm || true - sudo docker images --no-trunc -aqf "dangling=true" | xargs sudo docker rmi -f || true - sudo rm -rf ./* || true - sudo rm -rf /tmp/graphscope/app_* || true - find ./ -name "*.egg-info" | xargs sudo rm -rf || true - find ./ -name "*.whl" | xargs sudo rm -rf || true - find ./ -name "*_pb2.py" | xargs sudo rm -rf || true - find ./ -name "*_pb2_grpc.py" | xargs sudo rm -rf || true + yum install -y http://opensource.wandisco.com/centos/7/git/x86_64/wandisco-git-release-7-2.noarch.rpm + yum install -y git - - name: Check out code - uses: actions/checkout@v2.3.2 - with: - submodules: true + - uses: actions/checkout@v2.3.2 - - name: Cache - uses: actions/cache@v2 + - name: Build + run: | + mkdir /root/.m2 + mv interactive_engine/deploy/docker/dockerfile/maven.settings.xml /root/.m2/settings.xml + cd interactive_engine + mvn clean package -DskipTests -Pjava-release --quiet + cd ${GITHUB_WORKSPACE} + tar -cf manager.tar interactive_engine/src/assembly/target/0.0.1-SNAPSHOT.tar.gz \ + interactive_engine/src/instance-manager/target/0.0.1-SNAPSHOT.tar.gz \ + interactive_engine/deploy/docker/dockerfile + + - uses: actions/upload-artifact@v2 with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- + name: manager-${{ github.sha }} + path: manager.tar + retention-days: 5 - - name: Get Test Data - shell: bash + python-test: + runs-on: ubuntu-20.04 + needs: [gae, gle] + defaults: + run: + shell: scl enable devtoolset-7 -- bash --noprofile --norc -eo pipefail {0} + container: + image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-runtime:latest + options: + --shm-size 4096m + strategy: + matrix: + experimental: [ON, OFF] + steps: + - name: Install Dependencies run: | - export GS_TEST_DIR=${GITHUB_WORKSPACE}/gstest - git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} || true + yum install -y http://opensource.wandisco.com/centos/7/git/x86_64/wandisco-git-release-7-2.noarch.rpm + yum install -y git - - name: Download Image - uses: actions/download-artifact@v2 - with: - name: graphscope + - uses: actions/checkout@v2.3.2 + - uses: actions/download-artifact@v2 - name: Prepare environment - env: - ALIYUN_TOKEN: ${{ secrets.ALIYUN_TOKEN }} - shell: bash run: | - sudo docker load < graphscope-${{ github.sha }}.tar.gz - sudo python3 -m pip install -r python/requirements.txt - sudo python3 -m pip install -r python/requirements-dev.txt - - pushd python && sudo -E python3 setup.py build_proto && popd - pushd coordinator && sudo -E python3 setup.py build_builtin && popd + export GS_TEST_DIR='/root/gstest' + git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} + python3 -m pip install pytest-cov + tar -xf ./gae-${{ github.sha }}/gae.tar + tar -xf ./gle-${{ github.sha }}/gle.tar + cp -R -n opt/graphscope/* /usr/local/ + ls -la /usr/local/bin/ + pushd /usr/local/dist/ && pip3 install ./*.whl && popd + python3 -m pip install -r python/requirements.txt + python3 -m pip install -r python/requirements-dev.txt + pushd python && python3 setup.py build_proto && popd + pushd coordinator && python3 setup.py build_builtin && popd - name: App Test with Expr Off - shell: bash if: matrix.experimental == 'OFF' run: | - sudo docker run --rm --shm-size=4096m \ - -v ${GITHUB_WORKSPACE}/gstest:/root/gstest \ - -v `pwd`:/root/gs \ - registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} \ - sh -c "echo Container id $(hostname) && \ - set pipefail && \ - export GS_TEST_DIR='/root/gstest' && \ - python3 -m pytest --exitfirst -s -v /root/gs/python/tests" + export GS_TEST_DIR='/root/gstest' + cd python + python3 -m pytest --cov=graphscope --cov-config=.coveragerc --cov-report=xml \ + --cov-report=term --exitfirst -s -v tests + + - name: Upload Coverage + if: matrix.experimental == 'OFF' + uses: codecov/codecov-action@v1 + with: + file: ./python/coverage.xml + fail_ci_if_error: true - name: App Test with Expr On - shell: bash - if: matrix.experimental == 'ON' - run: | - sudo docker run --rm --shm-size=4096m \ - -v ${GITHUB_WORKSPACE}/gstest:/root/gstest \ - -v `pwd`/python:/root/python \ - registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} \ - sh -c "echo Container id $(hostname) && \ - set pipefail && \ - cd /root/python && \ - export GS_TEST_DIR='/root/gstest' && \ - python3 -m pytest --exitfirst -s -v ./graphscope/experimental/nx/tests/classes && \ - python3 -m pytest --exitfirst -s -v ./graphscope/experimental/nx/tests/test_nx.py && \ - python3 -m pytest --exitfirst -s -v ./graphscope/experimental/nx/tests/algorithms/builtin \ - ./graphscope/experimental/nx/tests/test_utils.py" - - - name: NetworkX algo test - shell: bash if: matrix.experimental == 'ON' run: | - info=$(git log -1 --pretty=%B) - if echo ${info} | grep -iqFw ci-algo; then echo ''run nx-algo-ci''; else exit 0; fi - sudo docker run --rm --shm-size=4096m \ - -v ${GITHUB_WORKSPACE}/gstest:/root/gstest \ - -v `pwd`/python:/root/python \ - registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} \ - sh -c "set pipefail && \ - cd /root/python && \ - export GS_TEST_DIR='/root/gstest' && \ - python3 -m pytest --exitfirst -s -v graphscope/experimental/nx/tests/algorithms/forward" - - - name: Clean - shell: bash - run: | - sudo docker rmi registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} || true - + export GS_TEST_DIR='/root/gstest' + python3 -m pytest --exitfirst -s -v python/graphscope/experimental/nx/tests/classes + python3 -m pytest --exitfirst -s -v python/graphscope/experimental/nx/tests/test_nx.py + python3 -m pytest --exitfirst -s -v python/graphscope/experimental/nx/tests/algorithms/builtin \ + python/graphscope/experimental/nx/tests/test_utils.py k8s-test: runs-on: self-hosted - - needs: [build, build-manager] + needs: [gae, gie, gle, manager] steps: - name: Clean up - shell: bash run: | echo "CI is running on host $(curl -s 'https://api.ipify.org')" sudo docker ps --no-trunc -aqf "status=exited" | xargs sudo docker rm || true @@ -321,62 +338,67 @@ jobs: find ./ -name "*_pb2.py" | xargs sudo rm -rf || true find ./ -name "*_pb2_grpc.py" | xargs sudo rm -rf || true - - name: Check out code - uses: actions/checkout@v2.3.2 - with: - submodules: true + - uses: actions/checkout@v2.3.2 - - name: Cache - uses: actions/cache@v2 + - uses: actions/cache@v2 with: - path: ~/.cache/pip + path: | + ~/.cache/pip + /root/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- - - name: Get Test Data - shell: bash - run: | - export GS_TEST_DIR=${GITHUB_WORKSPACE}/gstest - git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} || true - - - name: Download Image - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v2 + with: + path: artifacts - - name: Prepare environment - shell: bash + - name: Prepare Image run: | - sudo docker load < graphscope/graphscope-${{ github.sha }}.tar.gz - sudo docker load < maxgraph_standalone_manager/maxgraph_standalone_manager.tar.gz + cp .github/workflows/*.Dockerfile artifacts/ + pushd artifacts + tar -xf ./gae-${{ github.sha }}/gae.tar + tar -xf ./gle-${{ github.sha }}/gle.tar + tar -xf ./gie-${{ github.sha }}/gie.tar + tar -xf ./manager-${{ github.sha }}/manager.tar + sudo docker build -t registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} \ + --network=host \ + -f ./graphscope.Dockerfile . + sudo docker build -t registry.cn-hongkong.aliyuncs.com/graphscope/maxgraph_standalone_manager:1.0 \ + --network=host \ + -f ./manager.Dockerfile . + popd sudo python3 -m pip install -r python/requirements.txt sudo python3 -m pip install -r python/requirements-dev.txt - + sudo python3 -m pip install pytest-cov pushd python && sudo -E python3 setup.py develop && popd - name: Kubernetes test env: CHANGE_MINIKUBE_NONE_USER: true GS_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} - shell: bash run: | export GS_TEST_DIR=${GITHUB_WORKSPACE}/gstest - export PYTHONPATH=${GITHUB_WORKSPACE}/python - python3 -m pytest --exitfirst -s -vvv --log-cli-level=INFO \ - ./python/graphscope/deploy/tests/ - + git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} + cd python + python3 -m pytest --cov=graphscope --cov-config=.coveragerc --cov-report=xml \ + --cov-report=term --exitfirst -s -vvv --log-cli-level=INFO \ + ./graphscope/deploy/tests/ + - name: Upload Coverage + uses: codecov/codecov-action@v1 + with: + file: ./python/coverage.xml + fail_ci_if_error: true - name: Clean - shell: bash run: | sudo docker rmi registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} \ registry.cn-hongkong.aliyuncs.com/graphscope/maxgraph_standalone_manager:1.0 || true - gie-test: runs-on: self-hosted - needs: [build, build-manager] + needs: [gae, gie, gle, manager] steps: - name: Clean up - shell: bash run: | echo "CI is running on host $(curl -s 'https://api.ipify.org')" sudo docker ps --no-trunc -aqf "status=exited" | xargs sudo docker rm || true @@ -388,68 +410,86 @@ jobs: find ./ -name "*_pb2.py" | xargs sudo rm -rf || true find ./ -name "*_pb2_grpc.py" | xargs sudo rm -rf || true - - name: Check out code - uses: actions/checkout@v2.3.2 - with: - submodules: true + - uses: actions/checkout@v2.3.2 - - name: Cache - uses: actions/cache@v2 + - uses: actions/cache@v2 with: - path: ~/.cache/pip + path: | + ~/.cache/pip + /root/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- - - name: Download Image - uses: actions/download-artifact@v2 - - - name: Prepare environment - shell: bash + - uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - uses: actions/download-artifact@v2 + with: + path: artifacts + - name: Prepare Image run: | - sudo docker load < graphscope/graphscope-${{ github.sha }}.tar.gz - sudo docker load < maxgraph_standalone_manager/maxgraph_standalone_manager.tar.gz - + cp .github/workflows/*.Dockerfile artifacts/ + pushd artifacts + tar -xf ./gae-${{ github.sha }}/gae.tar + tar -xf ./gle-${{ github.sha }}/gle.tar + tar -xf ./gie-${{ github.sha }}/gie.tar + tar -xf ./manager-${{ github.sha }}/manager.tar + sudo docker build -t registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} \ + --network=host \ + -f ./graphscope.Dockerfile . + sudo docker build -t registry.cn-hongkong.aliyuncs.com/graphscope/maxgraph_standalone_manager:1.0 \ + --network=host \ + -f ./manager.Dockerfile . + popd sudo python3 -m pip install -r python/requirements.txt sudo python3 -m pip install -r python/requirements-dev.txt + sudo python3 -m pip install pytest-cov pushd python && sudo -E python3 setup.py develop && popd - name: Run function test - shell: bash run: | export PYTHONPATH=${GITHUB_WORKSPACE}/python cd interactive_engine/tests ./function_test.sh 8111 1 registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} ./function_test.sh 8111 2 registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} - - name: Clean - shell: bash run: | sudo docker rmi registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} \ registry.cn-hongkong.aliyuncs.com/graphscope/maxgraph_standalone_manager:1.0 || true - release-image: - runs-on: self-hosted + runs-on: ubuntu-20.04 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - needs: [ gie-test, gae-and-python-tests, k8s-test ] + needs: [ gie-test, python-test, k8s-test ] steps: - - name: Download Image - uses: actions/download-artifact@v2 - - - name: Prepare environment - shell: bash + - uses: actions/download-artifact@v2 + with: + path: artifacts + - name: Prepare Image run: | - sudo docker load < graphscope/graphscope-${{ github.sha }}.tar.gz - sudo docker load < maxgraph_standalone_manager/maxgraph_standalone_manager.tar.gz + cp .github/workflows/*.Dockerfile artifacts/ + cd artifacts + tar -xf ./gae-${{ github.sha }}/gae.tar + tar -xf ./gle-${{ github.sha }}/gle.tar + tar -xf ./gie-${{ github.sha }}/gie.tar + tar -xf ./manager-${{ github.sha }}/manager.tar + sudo docker build -t registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} \ + --network=host \ + -f ./graphscope.Dockerfile . + sudo docker build -t registry.cn-hongkong.aliyuncs.com/graphscope/maxgraph_standalone_manager:1.0 \ + --network=host \ + -f ./manager.Dockerfile . - name: Release images - shell: bash run: | echo ${{ secrets.ALIYUN_TOKEN }} | sudo docker login --username=grape_dev registry.cn-hongkong.aliyuncs.com --password-stdin - sudo docker tag registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} \ registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:latest + sudo docker push registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} sudo docker push registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:latest + sudo docker push registry.cn-hongkong.aliyuncs.com/graphscope/maxgraph_standalone_manager:1.0 - diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d5cf95ea8b3c..29f74446e466 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -81,7 +81,6 @@ jobs: git fetch origin gh-pages --no-recurse-submodules git checkout gh-pages - cd docs/ rm -rf !(_build) mv _build/html/* ./ diff --git a/.github/workflows/graphscope.Dockerfile b/.github/workflows/graphscope.Dockerfile new file mode 100644 index 000000000000..7eb6c1c5626b --- /dev/null +++ b/.github/workflows/graphscope.Dockerfile @@ -0,0 +1,26 @@ +FROM registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-runtime:latest + +ARG profile=release + +COPY ./opt/graphscope/ /usr/local/ +RUN cd /usr/local/dist/ && pip3 install ./*.whl + +RUN mkdir -p /home/maxgraph +ENV VINEYARD_IPC_SOCKET /home/maxgraph/data/vineyard/vineyard.sock +COPY ./interactive_engine/src/executor/target/release/executor /home/maxgraph/executor + +COPY ./interactive_engine/src/executor/store/log4rs.yml /home/maxgraph/log4rs.yml +RUN mkdir -p /home/maxgraph/native +ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:/home/maxgraph/native + +# enable debugging +ENV RUST_BACKTRACE=1 + +# copy start script from builder +RUN mkdir -p /home/maxgraph/config +COPY ./interactive_engine/deploy/docker/dockerfile/executor-entrypoint.sh /home/maxgraph/executor-entrypoint.sh +COPY ./interactive_engine/deploy/docker/dockerfile/executor.vineyard.properties.bak /home/maxgraph/config/executor.application.properties + +RUN mkdir -p /root/maxgraph +COPY ./interactive_engine/deploy/docker/dockerfile/set_config.sh /root/maxgraph/set_config.sh +COPY ./interactive_engine/deploy/docker/dockerfile/kill_process.sh /root/maxgraph/kill_process.sh diff --git a/.github/workflows/manager.Dockerfile b/.github/workflows/manager.Dockerfile new file mode 100644 index 000000000000..94e76e37c092 --- /dev/null +++ b/.github/workflows/manager.Dockerfile @@ -0,0 +1,22 @@ +FROM registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-runtime:latest +RUN mkdir -p /home/maxgraph +# copy binary from builder +COPY ./interactive_engine/src/assembly/target/0.0.1-SNAPSHOT.tar.gz /home/maxgraph/ +COPY ./interactive_engine/src/instance-manager/target/0.0.1-SNAPSHOT.tar.gz /home/maxgraph/instance-0.0.1-SNAPSHOT.tar.gz +# copy start script from builder +COPY ./interactive_engine/deploy/docker/dockerfile/coordinator-entrypoint.sh /home/maxgraph/coordinator-entrypoint.sh +RUN mkdir -p /home/maxgraph/config +COPY ./interactive_engine/deploy/docker/dockerfile/coordinator.application.properties /home/maxgraph/config/coordinator.application.properties + +COPY ./interactive_engine/deploy/docker/dockerfile/frontend-entrypoint.sh /home/maxgraph/frontend-entrypoint.sh +COPY ./interactive_engine/deploy/docker/dockerfile/frontend.vineyard.properties /home/maxgraph/config/frontend.application.properties +COPY ./interactive_engine/deploy/docker/dockerfile/manager-entrypoint.sh /home/maxgraph/manager-entrypoint.sh + +COPY ./interactive_engine/deploy/docker/dockerfile/create_maxgraph_instance.sh /root/maxgraph/create_maxgraph_instance.sh +COPY ./interactive_engine/deploy/docker/dockerfile/delete_maxgraph_instance.sh /root/maxgraph/delete_maxgraph_instance.sh +COPY ./interactive_engine/deploy/docker/dockerfile/kill_process.sh /root/maxgraph/kill_process.sh +COPY ./interactive_engine/deploy/docker/dockerfile/set_config.sh /root/maxgraph/set_config.sh +COPY ./interactive_engine/deploy/docker/dockerfile/func.sh /root/maxgraph/func.sh +COPY ./interactive_engine/deploy/docker/dockerfile/pod.yaml /root/maxgraph/pod.yaml +COPY ./interactive_engine/deploy/docker/dockerfile/expose_gremlin_server.sh /root/maxgraph/expose_gremlin_server.sh + diff --git a/README.md b/README.md index 4135c43365df..6cc32de2b9e2 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ [![GraphScope CI](https://github.com/alibaba/GraphScope/workflows/GraphScope%20CI/badge.svg)](https://github.com/alibaba/GraphScope/actions?workflow=GraphScope+CI) [![Docs](https://github.com/alibaba/GraphScope/workflows/Docs/badge.svg)](https://graphscope.io/docs) +[![Coverage](https://codecov.io/gh/alibaba/GraphScope/branch/main/graph/badge.svg)](https://codecov.io/gh/alibaba/GraphScope) GraphScope is a unified distributed graph computing platform that provides a one-stop environment for performing diverse graph operations on a cluster of computers through a user-friendly Python interface. GraphScope makes multi-staged processing of large-scale graph data on compute clusters simple by combining several important pieces of Alibaba technology: including [GRAPE](https://github.com/alibaba/libgrape-lite), [MaxGraph](interactive_engine/), and [Graph-Learn](https://github.com/alibaba/graph-learn) (GL) for analytics, interactive, and graph neural networks (GNN) computation, respectively, and the [vineyard](https://github.com/alibaba/libvineyard) store that offers efficient in-memory data transfers. diff --git a/analytical_engine/core/grape_instance.cc b/analytical_engine/core/grape_instance.cc index 119de9caad73..b22d74982b0d 100644 --- a/analytical_engine/core/grape_instance.cc +++ b/analytical_engine/core/grape_instance.cc @@ -124,6 +124,9 @@ bl::result GrapeInstance::unloadGraph(const rpc::GSParams& params) { auto fid = comm_spec_.WorkerToFrag(comm_spec_.worker_id()); auto frag_id = fg->Fragments().at(fid); VY_OK_OR_RAISE(client_->DelData(frag_id, false, true)); + } + MPI_Barrier(comm_spec_.comm()); + if (comm_spec_.worker_id() == 0) { VINEYARD_SUPPRESS(client_->DelData(frag_group_id, false, true)); } } diff --git a/k8s/graphscope.Dockerfile b/k8s/graphscope.Dockerfile index 1d5ae15c8b03..a320c2f408bf 100644 --- a/k8s/graphscope.Dockerfile +++ b/k8s/graphscope.Dockerfile @@ -89,7 +89,7 @@ RUN source ~/.bashrc \ echo "nightly mode" && export RUSTFLAGS=-Zsanitizer=address && \ rustup toolchain install nightly && \ ./exec.sh cargo +nightly build --target x86_64-unknown-linux-gnu --all; \ - else \ + else \ echo "debug mode" && ./exec.sh cargo build --all; \ fi diff --git a/k8s/gsruntime.Dockerfile b/k8s/gsruntime.Dockerfile index fff18f0c2f2f..38c8d0e16d34 100644 --- a/k8s/gsruntime.Dockerfile +++ b/k8s/gsruntime.Dockerfile @@ -21,7 +21,7 @@ RUN yum install -y autoconf automake double-conversion-devel git \ yum clean all && \ rm -fr /var/cache/yum -# yum install cmake3 +# cmake3 RUN cd /tmp && \ wget https://github.com/Kitware/CMake/releases/download/v3.19.1/cmake-3.19.1-Linux-x86_64.sh && \ bash cmake-3.19.1-Linux-x86_64.sh --prefix=/usr --skip-license && \ diff --git a/python/.coveragerc b/python/.coveragerc new file mode 100644 index 000000000000..feb6b1fcf6fe --- /dev/null +++ b/python/.coveragerc @@ -0,0 +1,4 @@ +[run] +omit = + graphscope/experimental/* + /usr/* diff --git a/python/graphscope/deploy/tests/test_demo_script.py b/python/graphscope/deploy/tests/test_demo_script.py index 2e809456d755..11c60d193a5d 100644 --- a/python/graphscope/deploy/tests/test_demo_script.py +++ b/python/graphscope/deploy/tests/test_demo_script.py @@ -26,6 +26,7 @@ import pytest import graphscope +from graphscope.config import GSConfig as gs_config from graphscope.dataset.ldbc import load_ldbc from graphscope.dataset.modern_graph import load_modern_graph from graphscope.framework.loader import Loader @@ -44,25 +45,26 @@ def modern_graph_data_dir(): def get_gs_image_on_ci_env(): - if "CI" in os.environ and "GS_IMAGE" in os.environ: - return os.environ["GS_IMAGE"], True + if "GS_IMAGE" in os.environ: + return os.environ["GS_IMAGE"] else: - return "", False + return gs_config.GS_IMAGE def test_demo(data_dir): - image, ci = get_gs_image_on_ci_env() - if ci: - sess = graphscope.session( - show_log=True, - num_workers=1, - k8s_gs_image=image, - ) - else: - sess = graphscope.session( - show_log=True, - num_workers=1, - ) + image = get_gs_image_on_ci_env() + sess = graphscope.session( + show_log=True, + num_workers=1, + k8s_gs_image=image, + k8s_coordinator_cpu=0.5, + k8s_coordinator_mem="2500Mi", + k8s_vineyard_cpu=0.1, + k8s_vineyard_mem="512Mi", + k8s_engine_cpu=0.1, + k8s_engine_mem="1500Mi", + k8s_vineyard_shared_mem="2Gi", + ) graph = load_ldbc(sess, data_dir) # Interactive engine @@ -88,19 +90,19 @@ def test_demo(data_dir): def test_demo_distribute(data_dir, modern_graph_data_dir): - image, ci = get_gs_image_on_ci_env() - if ci: - sess = graphscope.session( - show_log=True, - num_workers=2, - k8s_gs_image=image, - ) - else: - sess = graphscope.session( - show_log=True, - num_workers=2, - ) - + image = get_gs_image_on_ci_env() + sess = graphscope.session( + show_log=True, + num_workers=1, + k8s_gs_image=image, + k8s_coordinator_cpu=0.5, + k8s_coordinator_mem="2500Mi", + k8s_vineyard_cpu=0.1, + k8s_vineyard_mem="512Mi", + k8s_engine_cpu=0.1, + k8s_engine_mem="1500Mi", + k8s_vineyard_shared_mem="2Gi", + ) graph = load_ldbc(sess, data_dir) # Interactive engine @@ -166,38 +168,38 @@ def test_multiple_session(data_dir): [random.choice(string.ascii_lowercase) for _ in range(6)] ) - image, ci = get_gs_image_on_ci_env() - if ci: - sess = graphscope.session( - show_log=True, - k8s_namespace=namespace, - num_workers=2, - k8s_gs_image=image, - ) - else: - sess = graphscope.session( - show_log=True, - k8s_namespace=namespace, - num_workers=2, - ) + image = get_gs_image_on_ci_env() + sess = graphscope.session( + show_log=True, + num_workers=1, + k8s_gs_image=image, + k8s_coordinator_cpu=0.5, + k8s_coordinator_mem="2500Mi", + k8s_vineyard_cpu=0.1, + k8s_vineyard_mem="512Mi", + k8s_engine_cpu=0.1, + k8s_engine_mem="1500Mi", + k8s_vineyard_shared_mem="2Gi", + ) info = sess.info assert info["status"] == "active" assert info["type"] == "k8s" - assert len(info["engine_hosts"].split(",")) == 2 + assert len(info["engine_hosts"].split(",")) == 1 + + sess2 = graphscope.session( + show_log=True, + k8s_namespace=namespace, + num_workers=2, + k8s_gs_image=image, + k8s_coordinator_cpu=0.5, + k8s_coordinator_mem="2500Mi", + k8s_vineyard_cpu=0.1, + k8s_vineyard_mem="512Mi", + k8s_engine_cpu=0.1, + k8s_engine_mem="1500Mi", + k8s_vineyard_shared_mem="2Gi", + ) - if ci: - sess2 = graphscope.session( - show_log=True, - k8s_namespace=namespace, - num_workers=2, - k8s_gs_image=image, - ) - else: - sess2 = graphscope.session( - show_log=True, - k8s_namespace=namespace, - num_workers=2, - ) info = sess2.info assert info["status"] == "active" assert info["type"] == "k8s" @@ -208,18 +210,19 @@ def test_multiple_session(data_dir): def test_load_modern_graph(modern_graph_data_dir): - image, ci = get_gs_image_on_ci_env() - if ci: - sess = graphscope.session( - show_log=True, - num_workers=1, - k8s_gs_image=image, - ) - else: - sess = graphscope.session( - show_log=True, - num_workers=1, - ) + image = get_gs_image_on_ci_env() + sess = graphscope.session( + show_log=True, + num_workers=1, + k8s_gs_image=image, + k8s_coordinator_cpu=0.5, + k8s_coordinator_mem="2500Mi", + k8s_vineyard_cpu=0.1, + k8s_vineyard_mem="512Mi", + k8s_engine_cpu=0.1, + k8s_engine_mem="1500Mi", + k8s_vineyard_shared_mem="2Gi", + ) graph = load_modern_graph(sess, modern_graph_data_dir) interactive = sess.gremlin(graph) queries = [ diff --git a/python/graphscope/learning/graph.py b/python/graphscope/learning/graph.py index cf1ee78cf2b2..19492008ba98 100644 --- a/python/graphscope/learning/graph.py +++ b/python/graphscope/learning/graph.py @@ -86,7 +86,7 @@ def close(self): self.graphscope_session._close_learning_instance(self) super(Graph, self).close() - @staticmethod + @staticmethod # noqa: C901 def preprocess_args(handle, nodes, edges, gen_labels): # noqa: C901 handle = json.loads(base64.b64decode(handle).decode("utf-8", errors="ignore")) node_names = []