Merge pull request apache#455 from liu-sheng/collect-log-spark-k8s

Add support for collecting logs of Spark+K8s jobs

Author: huangtianhua

playbooks/spark-integration-test-kubeadm-k8s/post.yaml

@@ -0,0 +1,16 @@
+- hosts: all
+  become: yes
+  tasks:
+    - name: Collect k8s cluster log
+      shell: |
+        set -ex
+        export KUBECONFIG=/etc/kubernetes/admin.conf
+        mkdir -p '{{ k8s_log_dir }}'
+        pods=$(kubectl -n kube-system get pods -o jsonpath='{.items[*].metadata.name}')
+        for pod in $pods; do
+            for co in $(kubectl -n kube-system get pods $pod -o jsonpath='{.spec.containers[*].name}'); do
+                kubectl -n kube-system logs $pod  -c $co > '{{ k8s_log_dir }}'/$pod-$co.log
+            done
+        done
+      args:
+        executable: /bin/bash

playbooks/spark-integration-test-minikube-k8s/post.yaml

@@ -0,0 +1,15 @@
+- hosts: all
+  become: yes
+  tasks:
+    - name: Collect k8s cluster log
+      shell: |
+        set -ex
+        mkdir -p '{{ k8s_log_dir }}'
+        pods=$(kubectl -n kube-system get pods -o jsonpath='{.items[*].metadata.name}')
+        for pod in $pods; do
+            for co in $(kubectl -n kube-system get pods $pod -o jsonpath='{.spec.containers[*].name}'); do
+                kubectl -n kube-system logs $pod  -c $co > '{{ k8s_log_dir }}'/$pod-$co.log
+            done
+        done
+      args:
+        executable: /bin/bash

roles/create-single-k8s-cluster-with-minikube/tasks/main.yml

@@ -7,8 +7,15 @@
     curl -Lo /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/"{{ kubectl_version }}"/bin/linux/amd64/kubectl && chmod +x /usr/local/bin/kubectl
     sed -i 's/nameserver 127.0.0.1/nameserver 8.8.8.8/' /etc/resolv.conf
     minikube start --cpus 4 --memory 6144 --vm-driver=none
-    # Sleep 4 mins, waiting for cluster ready
-    sleep 240
-    kubectl get pods --all-namespaces
+    # sleep for waiting cluster ready
+    sleep 180
+    timeout 300 bash -c '
+        while :
+        do
+            [ -z "$(kubectl -n kube-system get pods --field-selector=status.phase!=Running)" ] && break
+            sleep 5
+        done
+        '
+    kubectl -n kube-system get all
   args:
     executable: /bin/bash

zuul.d/jobs.yaml

@@ -1087,6 +1087,8 @@
     description: |
       Run integration tests of spark of master with k8s cluster deployed by kubeadm
     run: playbooks/spark-integration-test-kubeadm-k8s/run.yaml
+    vars:
+      k8s_log_dir: '{{ ansible_user_dir }}/workspace/logs/kubernetes'
 
 - job:
     name: spark-integration-test-minikube-k8s
@@ -1095,3 +1097,5 @@
       Run integration tests of spark of v2.4.0 with k8s cluster deployed by minikube
     run: playbooks/spark-integration-test-minikube-k8s/run.yaml
     override-checkout: branch-2.4
+    vars:
+      k8s_log_dir: '{{ ansible_user_dir }}/workspace/logs/kubernetes'