Export the k8s logs of graphscope helm test and upload to artifact when the helm test failed (#2683)

Signed-off-by: Ye Cao <[email protected]>

Author: dashanji

.github/workflows/gss.yml

@@ -113,6 +113,9 @@ jobs:
     runs-on: [self-hosted, ubuntu2004]
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     needs: [gremlin-test]
+    environment:
+      JAVA_HOME: /usr/lib/jvm/default-java
+      GS_TEST_DIR: ${{ github.workspace }}/gstest
     steps:
     - uses: actions/checkout@v3
       with:
@@ -149,9 +152,6 @@ jobs:
         sudo /etc/init.d/ssh start
 
     - name: Python Test with Helm Deployment
-      env:
-        JAVA_HOME: /usr/lib/jvm/default-java
-        GS_TEST_DIR: ${{ github.workspace }}/gstest
       run: |
         # groot.tar.gz is needed for offline_load.sh
         # see .github/workflows/hadoop_scripts/offline_load.sh.template
@@ -173,7 +173,26 @@ jobs:
         cd ${GITHUB_WORKSPACE}/charts
         helm install ci --set image.tag=${SHORT_SHA} ./graphscope-store
         helm test ci --timeout 5m0s
+        mkdir -p ${{ github.workspace }}/helm-deployment-logs
 
+    - name: Export kubernetes logs
+      uses: dashanji/kubernetes-log-export-action@v3
+      if: ${{ failure() }}
+      env:
+        SHOW_TIMESTAMPS: 'true'
+        OUTPUT_DIR: ${{ github.workspace }}/helm-deployment-logs
+        # use wildcard here to match the namespaces that you want to export logs
+        NAMESPACES: default
+
+    - name: upload the k8s logs to artifact
+      uses: actions/upload-artifact@v2
+      if: ${{ failure() }}
+      with:
+        name: helm-deployment-logs
+        path: ${{ github.workspace }}/helm-deployment-logs
+
+    - name: Test the helm deployment
+      run: |
         # 1. get gss service endpoint
         export GRPC_PORT=$(kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services ci-graphscope-store-frontend)
         export GREMLIN_PORT=$(kubectl get --namespace default -o jsonpath="{.spec.ports[1].nodePort}" services ci-graphscope-store-frontend)
@@ -207,6 +226,26 @@ jobs:
         sleep 30
         cd ${GITHUB_WORKSPACE}/charts
         helm install ci --set image.tag=${SHORT_SHA} ./graphscope-store
+        mkdir -p ${{ github.workspace }}/helm-deployment-with-pv-logs
+
+    - name: Export kubernetes logs
+      uses: dashanji/kubernetes-log-export-action@v3
+      if: ${{ failure() }}
+      env:
+        SHOW_TIMESTAMPS: 'true'
+        OUTPUT_DIR: ${{ github.workspace }}/helm-deployment-logs
+        # use wildcard here to match the namespaces that you want to export logs
+        NAMESPACES: default
+
+    - name: upload the k8s logs to artifact
+      uses: actions/upload-artifact@v2
+      if: ${{ failure() }}
+      with:
+        name: helm-deployment-logs
+        path: ${{ github.workspace }}/helm-deployment-logs
+
+    - name: Python Test with Helm Deployment and PersistentVolume
+      run: |
         # helm test and python test on the restarted store
         export GRPC_PORT=$(kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services ci-graphscope-store-frontend)
         export GREMLIN_PORT=$(kubectl get --namespace default -o jsonpath="{.spec.ports[1].nodePort}" services ci-graphscope-store-frontend)

.github/workflows/k8s-ci.yml

@@ -426,7 +426,7 @@ jobs:
           minikube image load graphscope/learning:${SHORT_SHA}
           echo "loaded learning"
 
-      - name: Helm Test
+      - name: Helm Test of installation
         env:
           GS_TEST_DIR: ${{ github.workspace }}/gstest
         run: |
@@ -436,12 +436,51 @@ jobs:
           helm install graphscope --set image.registry="",image.tag="${SHORT_SHA}",withJupyter=false,volumes.enabled=true,volumes.items.data.field.path=${GS_TEST_DIR},engines.log_level=DEBUG \
                                   ./graphscope
           helm test graphscope --timeout 5m0s
+          mkdir -p ${{ github.workspace }}/helm-installation-logs
+
+      - name: Export kubernetes logs
+        uses: dashanji/kubernetes-log-export-action@v3
+        if: ${{ failure() }}
+        env:
+          SHOW_TIMESTAMPS: 'true'
+          OUTPUT_DIR: ${{ github.workspace }}/helm-installation-logs
+          # use wildcard here to match the namespaces that you want to export logs
+          NAMESPACES: default
+
+      - name: upload the k8s logs to artifact
+        uses: actions/upload-artifact@v2
+        if: ${{ failure() }}
+        with:
+          name: helm-installation-logs
+          path: ${{ github.workspace }}/helm-installation-logs
+
+      - name: Helm Test with pytest
+        run: |
           export NODE_IP=$(kubectl get pod -lgraphscope.coordinator.name=coordinator-graphscope -ojsonpath="{.items[0].status.hostIP}")
           export NODE_PORT=$(kubectl get services coordinator-service-graphscope -ojsonpath="{.spec.ports[0].nodePort}")
           echo "GraphScope service listen on ${NODE_IP}:${NODE_PORT}"
           export GS_ADDR=${NODE_IP}:${NODE_PORT}
           cd ${GITHUB_WORKSPACE}/python
           python3 -m pytest -s -vvv ./graphscope/tests/kubernetes/test_demo_script.py -k test_helm_installation
+          mkdir -p ${{ github.workspace }}/helm-pytest-logs
+
+      - name: Export kubernetes logs
+        uses: dashanji/kubernetes-log-export-action@v3
+        if: ${{ failure() }}
+        env:
+          SHOW_TIMESTAMPS: 'true'
+          OUTPUT_DIR: ${{ github.workspace }}/helm-pytest-logs
+          NAMESPACES: default
+
+      - name: upload the k8s logs to artifact
+        uses: actions/upload-artifact@v2
+        if: ${{ failure() }}
+        with:
+          name: helm-pytest-logs
+          path: ${{ github.workspace }}/helm-pytest-logs
+
+      - name: Delete Helm Cluster
+        run: |
           helm delete graphscope
 
       - name: Kubernetes Test
@@ -664,3 +703,20 @@ jobs:
               --set frontend.service.type=NodePort
           # run failover test
           cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && ./ir_k8s_failover_ci.sh default test-gie-standalone 2 1
+          mkdir -p ${{ github.workspace }}/k8s-failover-logs
+
+      - name: Export kubernetes logs
+        uses: dashanji/kubernetes-log-export-action@v3
+        if: ${{ failure() }}
+        env:
+          SHOW_TIMESTAMPS: 'true'
+          OUTPUT_DIR: ${{ github.workspace }}/k8s-failover-logs
+          # use wildcard here to match the namespaces that you want to export logs
+          NAMESPACES: default
+
+      - name: upload the k8s logs to artifact
+        uses: actions/upload-artifact@v2
+        if: ${{ failure() }}
+        with:
+          name: k8s-failover-logs
+          path: ${{ github.workspace }}/k8s-failover-logs