correct benchmark diag_tables for coupled model configurations; move bm_ic directory out of inputdata directory; use aws ec2 for CI test; auto-rt fixes

.github/workflows/main.yml → .github/workflows/build_test.yml

@@ -1,17 +1,16 @@
-name: Pull Request Tests
+name: Pull request tests
 
 on:
   push:
-    branches:
-    - develop
-  pull_request:
-    branches:
-    - develop
+    branches: ['develop']
+  pull_request_review:
+    types: [submitted]
+    branches: ['develop']
 
 jobs:
   setup:
     name: Set up
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
 
     outputs:
       tn: ${{ steps.parse.outputs.tn }}
@@ -46,7 +45,8 @@ jobs:
   build:
     name: Build (${{ matrix.bld_set }})
     needs: setup
-    runs-on: ubuntu-latest
+    if: github.event_name == 'push' || (github.event.review.state == 'approved' && toJson(github.event.pull_request.requested_reviewers) == '[]')
+    runs-on: ubuntu-20.04
 
     strategy:
       fail-fast: false
@@ -85,8 +85,7 @@ jobs:
   utest:
     name: Unit test (${{ needs.setup.outputs.tn }}, ${{ matrix.test_set }})
     needs: [setup,build]
-    runs-on: ubuntu-latest
-    #runs-on: self-hosted
+    runs-on: self-hosted
 
     strategy:
       fail-fast: false
@@ -101,7 +100,7 @@ jobs:
     - name: Prepare artifacts
       run: |
         tar xvjf artifact.tar.bz2 && rm -f artifact.tar.bz2
-        sudo docker load --input ${{ needs.setup.outputs.img }}.tar.gz && rm -f ${{ needs.setup.outputs.img }}.tar.gz
+        docker load --input ${{ needs.setup.outputs.img }}.tar.gz && rm -f ${{ needs.setup.outputs.img }}.tar.gz
 
     - name: Run utest
       run: ./ci.sh -n ${{ needs.setup.outputs.tn }} -r ${{ matrix.test_set }}
@@ -117,5 +116,7 @@ jobs:
       if: ${{ always() }}
       run: |
         rm -f ci.sh ci.test
-        sudo docker rm  my-container && sudo docker rmi ${{ needs.setup.outputs.img }}:latest
-        sudo docker volume rm DataVolume
+        docker stop my-container && docker rm  my-container && docker rmi ${{ needs.setup.outputs.img }}:latest
+        docker volume rm DataVolume
+        #docker rmi minsukjinoaa/fv3-input-data:input-data-20210115
+        rm -f memory_stat

.github/workflows/manage.yml → .github/workflows/manage_workflows.yml

@@ -2,21 +2,22 @@ name: Manage workflows
 
 on:
   workflow_run:
-    workflows: ["Pull Request Tests"]
+    workflows: ["Pull request tests"]
     types:
       - requested
 
 jobs:
   job1:
-    name: Job 1
-    runs-on: ubuntu-latest
+    name: Cancel workflows
+    runs-on: ubuntu-20.04
 
     steps:
       - name: Checkout codes
         uses: actions/checkout@v2
 
       - name: Check if skip-ci is requested
         run: |
+          sleep 40
           cd ${GITHUB_WORKSPACE}/tests/ci
           repo="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/runs"
           tr_id=$(cat ${GITHUB_EVENT_PATH} | ./json_helper.py get_trigger_id)

.github/workflows/start_runners.yml

@@ -0,0 +1,33 @@
+name: Start runners
+
+on:
+  workflow_run:
+    workflows: ["Pull request tests"]
+    types:
+      - requested
+
+jobs:
+  job1:
+    name: Start AWS runners
+    runs-on: ubuntu-20.04
+
+    steps:
+    - name: Check out codes
+      uses: actions/checkout@v2
+
+    - name: Configure AWS
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: us-east-1
+
+    - name: Check the status of build and start self-hosted runners
+      env:
+        aws_instance_id: ${{ secrets.AWS_INSTANCE_ID }}
+      run: |
+        cd ${GITHUB_WORKSPACE}/tests/ci
+        conclusion=$(cat ${GITHUB_EVENT_PATH} | ./build_status_check.py)
+        if [[ $conclusion == "success" ]]; then
+          aws ec2 start-instances --instance-ids $aws_instance_id
+        fi

.github/workflows/stop_runners.yml

@@ -0,0 +1,25 @@
+name: Stop runners
+
+on:
+  workflow_run:
+    workflows: ["Pull request tests"]
+    types:
+      - completed
+
+jobs:
+  job1:
+    name: Stop AWS runners
+    runs-on: ubuntu-20.04
+
+    steps:
+    - name: Configure AWS
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: us-east-1
+
+    - name: Stop self-hosted runners
+      env:
+        aws_instance_id: ${{ secrets.AWS_INSTANCE_ID }}
+      run: aws ec2 stop-instances --instance-ids $aws_instance_id