Merge pull request #5169 from filecoin-project/ci/split_workflow

ci: 拆分git工作流/split workflow

.github/workflows/baisc_checks.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - master
-      - main
   pull_request:
     branches:
       - '**'

.github/workflows/build_upload.yml

@@ -0,0 +1,13 @@
+name: build and upload
+
+on:
+  push:
+    branches: ['prep/**', 'release/**', 'test/**', master]
+    tags: ['**']
+
+jobs:
+  build_upload:
+    uses: ./.github/workflows/common_build_upload.yml
+    with:
+      bin_name: 'venus'
+      has_ffi: true

.github/workflows/tag-workflow.yml → .github/workflows/common_build_upload.yml

@@ -1,11 +1,16 @@
-name: build for test/release
+name: reuse build and upload
 
 on:
-  push:
-    branches: ['prep/**', 'release/**', 'test/**', master, main]
-    tags: ['**']
-  workflow_dispatch:
+
+  workflow_call:
+    inputs:
+      bin_name:
+        type: string
+      has_ffi:
+        type: boolean
+
 jobs:
+
   build:
     runs-on: ubuntu-20.04
     steps:
@@ -66,6 +71,8 @@ jobs:
 
       - name: show environment
         run: |
+          echo bin_name = ${{inputs.bin_name}}
+          echo has_ffi = ${{inputs.has_ffi}}
           echo event = ${{github.event_name}}
           echo github_repository: $GITHUB_REPOSITORY
           echo vars.commit = ${{steps.vars.outputs.commit}}
@@ -85,19 +92,34 @@ jobs:
           echo github.ref_name = ${{github.ref_name}}
           echo vars.job_url = ${{steps.vars.outputs.job_url}}
           echo ftp_url = ftp://${{secrets.FTP_HOST}}/${{steps.vars.outputs.repo_name}}/${{steps.vars.outputs.artifact_name}}
+
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
           go-version: 1.17
+
       - name: install deps
+        if: ${{ !inputs.has_ffi }}
+        run: |
+          sudo apt-get update
+          sudo apt-get install ncftp
+
+      - name: install more deps
+        if: ${{ inputs.has_ffi }}
         run: |
           sudo apt-get update
           sudo apt-get -o Acquire::Retries=3 install make ncftp mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl clang build-essential hwloc libhwloc-dev wget -y && sudo apt upgrade -y
 
       - name: Build
         run: |
-          go clean --modcache && make deps && make
-          mkdir ./release && mv ./venus ./release
+          go clean --modcache && make
+          mkdir ./release
+          if [[ "${{steps.vars.outputs.repo_name}}" == "venus-market" ]]; then
+            mv ./market-client ./venus-market ./release
+          fi
+          if [[ "${{steps.vars.outputs.repo_name}}" != "venus-market" ]]; then
+            mv ./${{inputs.bin_name}} ./release
+          fi
 
       - name: Zip Release
         uses: TheDoctor0/[email protected]
@@ -106,6 +128,14 @@ jobs:
           path: ./release
           type: tar
 
+      - name: upload artifacts
+        uses: actions/upload-artifact@v2
+        if: ${{ steps.vars.outputs.pub_method == 'pushRelease' }}
+        with:
+          name: ${{steps.vars.outputs.artifact_name}}
+          path: ./${{steps.vars.outputs.artifact_name}}
+          if-no-files-found: error
+
       - name: release
         id: release
         uses: ncipollo/release-action@v1
@@ -120,15 +150,13 @@ jobs:
       - name: upload ftp
         id: uploadftp
         if: ${{ steps.vars.outputs.ftp_exists == '1' }}
-        continue-on-error: true
         run: |
           ncftpput -m -R -v -u ${{secrets.FTP_USER}} -p ${{secrets.FTP_PWD}} ${{secrets.FTP_HOST}} ./${{steps.vars.outputs.repo_name}} ./${{steps.vars.outputs.artifact_name}}
           echo "upload file: ${{steps.vars.outputs.artifact_name}} successfully!"
 
       - name: setup oss
         id: setuposs
         if: ${{ steps.vars.outputs.oss_exists == '1' && steps.uploadftp.outcome != 'success' && steps.vars.outputs.pub_method == 'pushTest' }}
-        continue-on-error: true
         uses: manyuanrong/setup-ossutil@master
         with:
           endpoint: ${{secrets.OSS_ENDPOINT}}
@@ -138,21 +166,11 @@ jobs:
       - name: cp files to aliyun
         id: cposs
         if: ${{ steps.setuposs.outcome == 'success' }}
-        continue-on-error: true
         run: |
           ossutil cp ./${{steps.vars.outputs.artifact_name}} ${{secrets.OSS_BUCKET}}
           export signed_url=`ossutil sign ${{secrets.OSS_BUCKET}}/${{steps.vars.outputs.artifact_name}} --timeout 31104000 | sed -n 1p`
           echo '::set-output name=oss_signed_url::$(signed_url)'
 
-      - name: upload artifacts
-        if: ${{ steps.vars.outputs.pub_method == 'pushRelease' }}
-        continue-on-error: true
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{steps.vars.outputs.artifact_name}}
-          path: ./release
-          if-no-files-found: error
-
       - name: push god-eye
         run: |
           export link=${{steps.vars.outputs.job_url}}
@@ -174,16 +192,3 @@ jobs:
                   --data-urlencode "description=message:${{steps.vars.outputs.git_message}}, branch:${{steps.vars.outputs.branch}}, commit:${{steps.vars.outputs.short}}, tag:${{steps.vars.outputs.github_tag}}" \
                   --data-urlencode "version=${{steps.vars.outputs.short}}"
           set -e
-
-      - name: Publish the Docker image 
-        if: ${{ steps.vars.outputs.is_tag_create == 'true' }}
-        run: |
-          make docker-buildenv
-          make docker-runtime
-          docker build . --file dockerfile --tag filvenus/venus:latest
-          docker tag filvenus/venus:latest filvenus/venus:${{steps.vars.outputs.github_tag}}
-          docker login --username=filvenus --password ${{ secrets.DOCKER_PASSWORD }} 
-          docker push filvenus/venus:${{steps.vars.outputs.github_tag}}
-          docker push filvenus/venus:latest
-          docker push filvenus/venus-runtime:latest
-          docker push filvenus/venus-buildenv:latest

.github/workflows/common_docker_image.yml

@@ -0,0 +1,54 @@
+name: Reuse Docker Image CI
+
+on:
+
+  workflow_call:
+
+jobs:
+
+  build_docker_image:
+
+    runs-on: ubuntu-20.04
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Create vars
+      id: vars
+      run: |
+        export github_tag=${{ github.ref_name }}
+        export repo_name=${GITHUB_REPOSITORY##*/}
+        export is_tag_create=false
+        export docker_user_name='filvenus'
+
+        export rx_tag='^refs\/tags\/.*'
+        export rx_version_tag='^v([0-9]+\.){0,2}(\*|[0-9]+)(-rc[0-9]*){0,1}$'
+        if [[ "${{github.ref}}" =~ $rx_tag ]]; then
+          export is_tag_create=true
+        fi
+
+        echo "::set-output name=github_tag::$github_tag"
+        echo "::set-output name=repo_name::$repo_name"
+        echo "::set-output name=is_tag_create::$is_tag_create"
+        echo "::set-output name=docker_user_name::$docker_user_name"
+
+    - name: Show environment
+      run: |
+        echo is_tag_create = ${{ steps.vars.outputs.is_tag_create }}
+        echo github_tag = ${{ steps.vars.outputs.github_tag }}
+        echo repo_name = ${{ steps.vars.outputs.repo_name }}
+        echo docker_user_name = ${{steps.vars.outputs.docker_user_name}}
+
+    - name: Build the Docker image
+      if: ${{ steps.vars.outputs.is_tag_create == 'true' }}
+      run: |
+        if [[ "${{steps.vars.outputs.repo_name}}" = "venus" ]]; then
+          make docker-buildenv
+          make docker-runtime
+        fi
+        docker build . --file dockerfile --tag ${{steps.vars.outputs.docker_user_name}}/${{steps.vars.outputs.repo_name}}:latest
+        docker tag ${{steps.vars.outputs.docker_user_name}}/${{steps.vars.outputs.repo_name}}:latest ${{steps.vars.outputs.docker_user_name}}/${{steps.vars.outputs.repo_name}}:${{ steps.vars.outputs.github_tag }}
+        docker login --username=${{steps.vars.outputs.docker_user_name}} --password ${{ secrets.DOCKER_PASSWORD }} 
+        docker push ${{steps.vars.outputs.docker_user_name}}/${{steps.vars.outputs.repo_name}}:${{ steps.vars.outputs.github_tag }}
+        docker push ${{steps.vars.outputs.docker_user_name}}/${{steps.vars.outputs.repo_name}}:latest
+        docker push ${{steps.vars.outputs.docker_user_name}}/${{steps.vars.outputs.repo_name}}-runtime:latest
+        docker push ${{steps.vars.outputs.docker_user_name}}/${{steps.vars.outputs.repo_name}}-buildenv:latest

.github/workflows/common_go.yml

@@ -0,0 +1,64 @@
+name: build and golangci-lint and test
+
+on:
+
+  workflow_call:
+    inputs:
+      has_ffi:
+        type: boolean
+
+jobs:
+
+  check:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+
+      - name: vars
+        id: vars
+        run: |
+          export repo_name=${GITHUB_REPOSITORY##*/}
+          echo "::set-output name=repo_name::$repo_name"
+
+      - name: show vars
+        run: |
+          echo vars.repo_name = ${{steps.vars.outputs.repo_name}}
+
+      - name: install deps
+        if: ${{ inputs.has_ffi }}
+        run: sudo apt-get -o Acquire::Retries=3 update && sudo apt-get -o Acquire::Retries=3 install make ftp git bzr curl hwloc libhwloc-dev mesa-opencl-icd ocl-icd-opencl-dev wget -y && sudo apt upgrade -y
+
+      - name: Build
+        env:
+          GOPROXY: "https://proxy.golang.org,direct"
+          GO111MODULE: "on"
+        run: |
+          make
+
+      - name: Lint
+        run: |
+          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.44.2
+          golangci-lint run --timeout 10m
+
+      - name: Detect changes
+        run: |
+          git status --porcelain
+          test -z "$(git status --porcelain)"
+
+      - name: Run coverage
+        run:  go test -coverpkg=./... -race -coverprofile=coverage.txt -covermode=atomic ./...
+
+      - name: Upload
+        uses: codecov/codecov-action@v2
+        with:
+          token:
+          files:  ./coverage.txt
+          flags: unittests
+          name: ${{steps.vars.outputs.repo_name}}
+          fail_ci_if_error: true
+          verbose: true

.github/workflows/docker_image.yml

@@ -0,0 +1,11 @@
+name: Docker Image CI
+
+on:
+  push:
+    branches: ['prep/**', 'release/**', 'test/**', master]
+    tags: ['**']
+
+jobs:
+
+  build_docker_image:
+    uses: ./.github/workflows/common_docker_image.yml

.github/workflows/test.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - master
-      - main
   pull_request:
     branches:
       - '**'
@@ -34,13 +33,6 @@ jobs:
           cd /tmp/statediff
           go install ./cmd/statediff || exit 0
 
-      - name: dep
-        env:
-          GOPROXY: "https://proxy.golang.org,direct"
-          GO111MODULE: "on"
-        run: |
-          make deps
-
       - name: Build
         env:
           GOPROXY: "https://proxy.golang.org,direct"