[v0.x] Backport CI from Master to v0.x

.github/workflows/buildwebsite.yml

@@ -0,0 +1,107 @@
+name: v0.x continuous build - docs
+
+on: [push, pull_request_target]
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  website-build:
+    runs-on: ubuntu-latest
+    strategy: 
+      fail-fast: false
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+          architecture: x64
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      - name: Install Other Dependencies
+        run: |
+          python -m pip install --quiet --upgrade pip
+          python -m pip install --quiet boto3
+      - name: Build Website (For push)
+        if: ${{ (failure() || success()) && github.event_name == 'push' }}
+        run: |
+          python ./ci/batch/submit-job.py --region us-east-1 \
+                                             --job-type g4dn.4x \
+                                             --name GluonNLP-Website-v0.x-${{ github.ref }} \
+                                             --remote https://github.com/${{ github.repository }} \
+                                             --source-ref ${{ github.ref }} \
+                                             --work-dir . \
+                                             --saved-output docs/tutorials \
+                                             --save-path tutorials \
+                                             --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+                                                        && chmod +x Miniconda3-latest-Linux-x86_64.sh \
+                                                        && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \
+                                                        && source ci/prepare_clean_env.sh gpu/cy3 \
+                                                        && python3 -m pip install mxnet-cu102==1.6.0.post0 flaky numpy==1.18.4 \
+                                                        && ./ci/batch/compile_notebooks.sh \
+                                                        'push' ${GITHUB_REF##*/}" \
+                                             --wait | tee build_website.log
+
+      - name: Build Website (For pull request)
+        if: ${{ (failure() || success()) && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') }}
+        run: |
+          python ./ci/batch/submit-job.py --region us-east-1 \
+                                             --job-type g4dn.4x \
+                                             --name GluonNLP-Website-v0.x-PR#${{ github.event.number }} \
+                                             --remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \
+                                             --source-ref ${{ github.event.pull_request.head.sha }} \
+                                             --work-dir . \
+                                             --saved-output docs/tutorials \
+                                             --save-path tutorials \
+                                             --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+                                                        && chmod +x Miniconda3-latest-Linux-x86_64.sh \
+                                                        && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \
+                                                        && source ci/prepare_clean_env.sh gpu/cy3 \
+                                                        && python3 -m pip install mxnet-cu102==1.6.0.post0 flaky numpy==1.18.4 \
+                                                        && ./ci/batch/compile_notebooks.sh \
+                                                        ${{ github.event.number }} ${{ github.event.pull_request.head.sha }}" \
+                                             --wait | tee build_website.log
+
+      - name: Wait for job and copy logs from AWS s3
+        if: ${{ failure() || success() }}
+        run: |
+          head -100 build_website.log | grep -oP -m 1 'jobId: \K(.*)' > jobid.log
+
+          echo "Job ID is"
+          cat jobid.log
+          cat jobid.log | xargs -i python ./tools/batch/wait-job.py --job-id {}
+          echo "Copy Log file"
+          cat jobid.log | xargs -i aws s3api wait object-exists --bucket gluon-nlp-dev --key batch/{}/tutorials/index.rst
+          cat jobid.log | xargs -i aws s3 cp s3://gluon-nlp-dev/batch/{}/tutorials tutorials --recursive
+
+      - name: Upload Notebook Compile Logs 
+        if: ${{ failure() || success() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: Notebook_Logs
+          path: ./tutorials/**/*.stdout.log
+
+      - name: Upload Website Build Log 
+        if: ${{ failure() || success() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: Website_Log
+          path: ./build_website.log
+
+      - name: Create comment
+        if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}
+        uses: peter-evans/create-or-update-comment@v1
+        with: 
+          issue-number: ${{ github.event.number }}
+          body: |
+            The documentation website for preview: http://gluon-nlp-staging.s3-accelerate.dualstack.amazonaws.com/PR${{ github.event.number }}/${{ github.event.pull_request.head.sha }}/index.html

.github/workflows/unittests-gpu.yml

@@ -0,0 +1,106 @@
+name: v0.x continuous build - gpu
+
+on: [push, pull_request_target]
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  unittest-gpu:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Install Linux dependencies
+        run: sudo apt-get install libopenblas-dev
+
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+          architecture: x64
+
+      - name: Install Other Dependencies
+        run: |
+          python -m pip install --quiet --upgrade pip
+          python -m pip install --quiet boto3
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      - name: Test project on AWS Batch(For push)
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          echo "Start submitting job"
+          python ./ci/batch/submit-job.py --region us-east-1 \
+                                             --job-type g4dn.4x \
+                                             --name GluonNLP-GPU-v0.x-Test-${{ github.ref }} \
+                                             --source-ref ${{ github.ref }} \
+                                             --work-dir . \
+                                             --saved-output coverage.xml \
+                                             --save-path coverage.xml \
+                                             --remote https://github.com/${{ github.repository }} \
+                                             --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+                                             && chmod +x Miniconda3-latest-Linux-x86_64.sh \
+                                             && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \
+                                             && source ci/prepare_clean_env.sh gpu/cy3 \
+                                             && python3 -m pip install mxnet-cu102==1.6.0.post0 flaky numpy==1.18.4 \
+                                             && pytest -m 'gpu and not serial' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \
+                                             --wait | tee batch_job.log
+
+      - name: Test project on AWS Batch(For pull request)
+        if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}
+        run: |
+          echo "Start submitting job"
+          python ./ci/batch/submit-job.py --region us-east-1 \
+                                             --job-type g4dn.4x \
+                                             --name GluonNLP-GPU-v0.x-Test-PR#${{ github.event.number }} \
+                                             --source-ref ${{ github.event.pull_request.head.sha }} \
+                                             --work-dir . \
+                                             --saved-output coverage.xml \
+                                             --save-path coverage.xml \
+                                             --remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \
+                                             --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+                                             && chmod +x Miniconda3-latest-Linux-x86_64.sh \
+                                             && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \
+                                             && source ci/prepare_clean_env.sh gpu/cy3 \
+                                             && python3 -m pip install mxnet-cu102==1.6.0.post0 flaky numpy==1.18.4 \
+                                             && pytest -m 'gpu and not serial' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \
+                                             --wait | tee batch_job.log
+
+      - name: Wait for job and copy files from AWS s3
+        if: ${{ failure() || success() }}
+        run: |
+          head -100 batch_job.log | grep -oP -m 1 'jobId: \K(.*)' > jobid.log
+          echo "Job ID is"
+          cat jobid.log
+          cat jobid.log | xargs -i python ./tools/batch/wait-job.py --job-id {}
+          echo "Copy Codecov file"
+          cat jobid.log | xargs -i aws s3api wait object-exists --bucket gluon-nlp-dev --key batch/{}/coverage.xml
+          cat jobid.log | xargs -i aws s3 cp s3://gluon-nlp-dev/batch/{}/coverage.xml ./coverage.xml
+
+      - name: Upload coverage to Codecov
+        run: |
+          curl -s https://codecov.io/bash -o codecov.sh
+          if [ "$EVENT_NAME" == "push" ]; then \
+              bash codecov.sh -f ./coverage.xml -n -F -B v0.x; \
+          else \
+              bash codecov.sh -f ./coverage.xml -n -F -B v0.x -C ${{ github.event.pull_request.head.sha }} -P ${{ github.event.pull_request.number }}; \
+          fi
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+
+      - name: Upload Cloud Watch Log Stream
+        if: ${{ failure() || success() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: Test_Logs
+          path: ./batch_job.log

.github/workflows/unittests.yml

@@ -0,0 +1,105 @@
+name: v0.x continuous build
+
+on: [push, pull_request_target]
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  unittest-gpu:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Install Linux dependencies
+        run: sudo apt-get install libopenblas-dev
+
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+          architecture: x64
+
+      - name: Install Other Dependencies
+        run: |
+          python -m pip install --quiet --upgrade pip
+          python -m pip install --quiet boto3
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      - name: Test project on AWS Batch(For push)
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          echo "Start submitting job"
+          python ./ci/batch/submit-job.py --region us-east-1 \
+                                             --job-type c5n.4x \
+                                             --name GluonNLP-GPU-v0.x-Test-${{ github.ref }} \
+                                             --source-ref ${{ github.ref }} \
+                                             --work-dir . \
+                                             --saved-output coverage.xml \
+                                             --save-path coverage.xml \
+                                             --remote https://github.com/${{ github.repository }} \
+                                             --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+                                             && chmod +x Miniconda3-latest-Linux-x86_64.sh \
+                                             && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \
+                                             && source ci/prepare_clean_env.sh gpu/cy3 \
+                                             && python3 -m pip install mxnet==1.6.0 flaky numpy==1.18.4 \
+                                             && pytest -m 'not (gpu or serial)' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \
+                                             --wait | tee batch_job.log
+
+      - name: Test project on AWS Batch(For pull request)
+        if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}
+        run: |
+          echo "Start submitting job"
+          python ./ci/batch/submit-job.py --region us-east-1 \
+                                             --job-type c5n.4x \
+                                             --name GluonNLP-GPU-v0.x-Test-PR#${{ github.event.number }} \
+                                             --source-ref ${{ github.event.pull_request.head.sha }} \
+                                             --work-dir . \
+                                             --saved-output coverage.xml \
+                                             --save-path coverage.xml \
+                                             --remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \
+                                             --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+                                             && chmod +x Miniconda3-latest-Linux-x86_64.sh \
+                                             && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \
+                                             && source ci/prepare_clean_env.sh gpu/cy3 \
+                                             && python3 -m pip install mxnet==1.6.0 flaky numpy==1.18.4 \
+                                             && pytest -m 'not (gpu or serial)' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \
+                                             --wait | tee batch_job.log
+
+      - name: Wait for job and copy files from AWS s3
+        if: ${{ failure() || success() }}
+        run: |
+          head -100 batch_job.log | grep -oP -m 1 'jobId: \K(.*)' > jobid.log
+          echo "Job ID is"
+          cat jobid.log
+          cat jobid.log | xargs -i python ./tools/batch/wait-job.py --job-id {}
+          echo "Copy Codecov file"
+          cat jobid.log | xargs -i aws s3api wait object-exists --bucket gluon-nlp-dev --key batch/{}/coverage.xml
+          cat jobid.log | xargs -i aws s3 cp s3://gluon-nlp-dev/batch/{}/coverage.xml ./coverage.xml
+
+      - name: Upload coverage to Codecov
+        run: |
+          curl -s https://codecov.io/bash -o codecov.sh
+          if [ "$EVENT_NAME" == "push" ]; then \
+              bash codecov.sh -f ./coverage.xml -n -F -B v0.x; \
+          else \
+              bash codecov.sh -f ./coverage.xml -n -F -B v0.x -C ${{ github.event.pull_request.head.sha }} -P ${{ github.event.pull_request.number }}; \
+          fi
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+
+      - name: Upload Cloud Watch Log Stream
+        if: ${{ failure() || success() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: Test_Logs
+          path: ./batch_job.log

ci/batch/compile_notebooks.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+# Shell script for submitting AWS Batch jobs to compile notebooks
+
+event=$1
+ref=$2
+
+FAIL=0
+
+compile_notebook () {
+    local MDFILE=$1
+    DIR=$(dirname $MDFILE)
+    BASENAME=$(basename $MDFILE)
+    TARGETNAME=$(dirname $MDFILE)/${BASENAME%.md}.ipynb
+    LOGNAME=$(dirname $MDFILE)/${BASENAME%.md}.stdout.log
+
+    echo Compiling $BASENAME ...
+
+    python3 docs/md2ipynb.py ${MDFILE} &> $LOGNAME
+
+    EXIT_CODE=$?
+
+    if [ $EXIT_CODE -ne 0 ]; then
+        echo Compiling $BASENAME Failed, please download Notebook_Logs in build Artifacts for more details.
+    else
+        echo Compiling $BASENAME Succeeded
+    fi
+    exit $EXIT_CODE
+}
+
+pids=()
+
+for f in $(find docs/tutorials -type f -name '*.md' -print); do
+    compile_notebook "$f" &
+    pids+=($!)
+done;
+
+for pid in "${pids[@]}"; do
+    wait "$pid" || let "FAIL+=1"
+done;
+
+if [ "$FAIL" == "0" ]; then
+    echo Building Website
+    make docs
+    EXIT_CODE=$?
+    if [ $EXIT_CODE -ne 0 ]; then
+        echo Building Website Failed.
+        exit $EXIT_CODE
+    else
+        echo Building Website Succeeded.
+        if [ "$1" == "push" ]; then
+            echo "Uploading docs to s3://gluon-nlp/$2/"
+            aws s3 sync --delete ./docs/_build/html/ s3://gluon-nlp/$2/ --quiet --acl public-read
+        else
+            echo "Uploading docs to s3://gluon-nlp-staging/PR$1/$2/"
+            aws s3 sync --delete ./docs/_build/html/ s3://gluon-nlp-staging/PR$1/$2/ --quiet --acl public-read
+        fi
+    fi
+else
+    exit 1
+fi