Use aws ec2 for CI test

.github/workflows/main.yml → .github/workflows/build_test.yml

@@ -1,17 +1,16 @@
-name: Pull Request Tests
+name: Pull request tests
 
 on:
   push:
-    branches:
-    - develop
-  pull_request:
-    branches:
-    - develop
+    branches: ['develop']
+  pull_request_review:
+    types: [submitted]
+    branches: ['develop']
 
 jobs:
   setup:
     name: Set up
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
 
     outputs:
       tn: ${{ steps.parse.outputs.tn }}
@@ -46,7 +45,8 @@ jobs:
   build:
     name: Build (${{ matrix.bld_set }})
     needs: setup
-    runs-on: ubuntu-latest
+    if: github.event_name == 'push' || (github.event.review.state == 'approved' && toJson(github.event.pull_request.requested_reviewers) == '[]')
+    runs-on: ubuntu-20.04
 
     strategy:
       fail-fast: false
@@ -85,8 +85,7 @@ jobs:
   utest:
     name: Unit test (${{ needs.setup.outputs.tn }}, ${{ matrix.test_set }})
     needs: [setup,build]
-    runs-on: ubuntu-latest
-    #runs-on: self-hosted
+    runs-on: self-hosted
 
     strategy:
       fail-fast: false
@@ -101,7 +100,7 @@ jobs:
     - name: Prepare artifacts
       run: |
         tar xvjf artifact.tar.bz2 && rm -f artifact.tar.bz2
-        sudo docker load --input ${{ needs.setup.outputs.img }}.tar.gz && rm -f ${{ needs.setup.outputs.img }}.tar.gz
+        docker load --input ${{ needs.setup.outputs.img }}.tar.gz && rm -f ${{ needs.setup.outputs.img }}.tar.gz
 
     - name: Run utest
       run: ./ci.sh -n ${{ needs.setup.outputs.tn }} -r ${{ matrix.test_set }}
@@ -117,5 +116,7 @@ jobs:
       if: ${{ always() }}
       run: |
         rm -f ci.sh ci.test
-        sudo docker rm  my-container && sudo docker rmi ${{ needs.setup.outputs.img }}:latest
-        sudo docker volume rm DataVolume
+        docker stop my-container && docker rm  my-container && docker rmi ${{ needs.setup.outputs.img }}:latest
+        docker volume rm DataVolume
+        #docker rmi minsukjinoaa/fv3-input-data:input-data-20210115
+        rm -f memory_stat

.github/workflows/manage.yml → .github/workflows/manage_workflows.yml

@@ -2,21 +2,22 @@ name: Manage workflows
 
 on:
   workflow_run:
-    workflows: ["Pull Request Tests"]
+    workflows: ["Pull request tests"]
     types:
       - requested
 
 jobs:
   job1:
-    name: Job 1
-    runs-on: ubuntu-latest
+    name: Cancel workflows
+    runs-on: ubuntu-20.04
 
     steps:
       - name: Checkout codes
         uses: actions/checkout@v2
 
       - name: Check if skip-ci is requested
         run: |
+          sleep 40
           cd ${GITHUB_WORKSPACE}/tests/ci
           repo="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/runs"
           tr_id=$(cat ${GITHUB_EVENT_PATH} | ./json_helper.py get_trigger_id)

.github/workflows/start_runners.yml

@@ -0,0 +1,33 @@
+name: Start runners
+
+on:
+  workflow_run:
+    workflows: ["Pull request tests"]
+    types:
+      - requested
+
+jobs:
+  job1:
+    name: Start AWS runners
+    runs-on: ubuntu-20.04
+
+    steps:
+    - name: Check out codes
+      uses: actions/checkout@v2
+
+    - name: Configure AWS
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: us-east-1
+
+    - name: Check the status of build and start self-hosted runners
+      env:
+        aws_instance_id: ${{ secrets.AWS_INSTANCE_ID }}
+      run: |
+        cd ${GITHUB_WORKSPACE}/tests/ci
+        conclusion=$(cat ${GITHUB_EVENT_PATH} | ./build_status_check.py)
+        if [[ $conclusion == "success" ]]; then
+          aws ec2 start-instances --instance-ids $aws_instance_id
+        fi

.github/workflows/stop_runners.yml

@@ -0,0 +1,25 @@
+name: Stop runners
+
+on:
+  workflow_run:
+    workflows: ["Pull request tests"]
+    types:
+      - completed
+
+jobs:
+  job1:
+    name: Stop AWS runners
+    runs-on: ubuntu-20.04
+
+    steps:
+    - name: Configure AWS
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: us-east-1
+
+    - name: Stop self-hosted runners
+      env:
+        aws_instance_id: ${{ secrets.AWS_INSTANCE_ID }}
+      run: aws ec2 stop-instances --instance-ids $aws_instance_id

tests/ci/build_status_check.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+
+import re
+import sys
+import json
+import time
+from urllib.request import urlopen
+
+def update_url_data(response):
+  data = json.loads(response.read().decode())
+  indices=[]
+  for n in range(data["total_count"]):
+    if re.search("Build", data["jobs"][n]["name"]):
+      indices.append(n)
+
+  if len(indices) == 0:
+    raise ValueError("No build job exists.")
+
+  return data, indices
+
+def main():
+
+  time.sleep(40)
+  url = json.load(sys.stdin)["workflow_run"]["jobs_url"]
+
+  status="not-completed"
+  no_completed_jobs = 0
+
+  while status != "completed":
+    response = urlopen(url)
+    data, indices = update_url_data(response)
+
+    for i in indices:
+      if data["jobs"][i]["status"] == "completed":
+        no_completed_jobs += 1
+
+    if no_completed_jobs == len(indices):
+      status = "completed"
+    else:
+      no_completed_jobs = 0
+      time.sleep(40)
+
+  time.sleep(40)
+  conclusion="failure"
+  no_successful_jobs = 0
+  for i in indices:
+    if data["jobs"][i]["conclusion"] == "success":
+      no_successful_jobs += 1
+
+  if no_successful_jobs == len(indices):
+    conclusion = "success"
+
+  print(conclusion)
+
+if __name__ == "__main__": main()

tests/ci/ci.sh

@@ -72,15 +72,15 @@ if [ $BUILD = "true" ]; then
 
 elif [ $RUN == "true" ]; then
 
-  sudo docker run -d --rm -v DataVolume:/tmp minsukjinoaa/fv3-input-data:input-data-20210115
-  sudo docker run -d -e test_case=${TEST_CASE} -v DataVolume:/home/builder/data/NEMSfv3gfs/input-data-20210115 --name my-container ${IMG_NAME}
+  docker run -d --rm -v DataVolume:/tmp minsukjinoaa/fv3-input-data:input-data-20210115
+  docker run -d -e test_case=${TEST_CASE} --shm-size=512m -v DataVolume:/home/builder/data/NEMSfv3gfs/input-data-20210115 --name my-container ${IMG_NAME}
 
   echo 'cache,rss,shmem' >memory_stat
   sleep 3
-  containerID=$(sudo docker ps -q --no-trunc)
+  containerID=$(docker ps -q --no-trunc)
   check_memory_usage $containerID >>memory_stat &
 
-  sudo docker logs -f $containerID
-  exit $(sudo docker inspect $containerID --format='{{.State.ExitCode}}')
+  docker logs -f $containerID
+  exit $(docker inspect $containerID --format='{{.State.ExitCode}}')
 
 fi

tests/ci/ci.test

@@ -1,3 +1,3 @@
 fv3_ccpp_control
-rst bit
+thr mpi dcp rst bit dbg
 ci-test-weather

tests/ci/json_helper.py

@@ -19,6 +19,7 @@ def cancel_workflow(data):
         x["id"]!=int(os.environ["GITHUB_RUN_ID"]) and
         x["id"]!=int(os.environ["TRIGGER_ID"]) and
         x["head_branch"]==os.environ["TRIGGER_BR"] and
+        x["event"]!="workflow_run" and
         (x["status"]=="queued" or x["status"]=="in_progress")]
 
   return wfs

tests/default_vars.sh

@@ -146,7 +146,7 @@ elif [[ $MACHINE_ID = hera.* ]]; then
 elif [[ $MACHINE_ID = linux.* ]]; then
 
   if [[ $CI_TEST = true ]]; then
-  TASKS_dflt=12 ; TPN_dflt=16 ; INPES_dflt=1 ; JNPES_dflt=1
+  TASKS_dflt=42 ; TPN_dflt=48 ; INPES_dflt=3 ; JNPES_dflt=2
   else
   TASKS_dflt=150 ; TPN_dflt=40 ; INPES_dflt=3 ; JNPES_dflt=8
   fi

tests/run_test.sh

@@ -150,7 +150,11 @@ atparse < ${PATHRT}/parm/${NEMS_CONFIGURE:-nems.configure} > nems.configure
 if [[ $SCHEDULER = 'none' ]]; then
 
   ulimit -s unlimited
-  mpiexec -n ${TASKS} ./fv3.exe >out 2> >(tee err >&3)
+  if [[ $CI_TEST = 'true' ]]; then
+    eval mpiexec -n ${TASKS} ${MPI_PROC_BIND} ./fv3.exe >out 2> >(tee err >&3)
+  else
+    mpiexec -n ${TASKS} ./fv3.exe >out 2> >(tee err >&3)
+  fi
 
 else