This repository has been archived by the owner on Jan 15, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 538
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Switch from Jenkins to Github Actions CI (#1531)
- Loading branch information
Showing
17 changed files
with
440 additions
and
2,466 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,107 @@ | ||
name: v0.x continuous build - docs | ||
|
||
on: [push, pull_request_target] | ||
|
||
defaults: | ||
run: | ||
shell: bash | ||
|
||
jobs: | ||
website-build: | ||
runs-on: ubuntu-latest | ||
strategy: | ||
fail-fast: false | ||
steps: | ||
- name: Checkout repository | ||
uses: actions/checkout@v2 | ||
|
||
- name: Setup python | ||
uses: actions/setup-python@v2 | ||
with: | ||
python-version: 3.7 | ||
architecture: x64 | ||
|
||
- name: Configure AWS Credentials | ||
uses: aws-actions/configure-aws-credentials@v1 | ||
with: | ||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | ||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | ||
aws-region: us-east-1 | ||
|
||
- name: Install Other Dependencies | ||
run: | | ||
python -m pip install --quiet --upgrade pip | ||
python -m pip install --quiet boto3 | ||
- name: Build Website (For push) | ||
if: ${{ (failure() || success()) && github.event_name == 'push' }} | ||
run: | | ||
python ./ci/batch/submit-job.py --region us-east-1 \ | ||
--job-type g4dn.4x \ | ||
--name GluonNLP-Website-v0.x-${{ github.ref }} \ | ||
--remote https://github.com/${{ github.repository }} \ | ||
--source-ref ${{ github.ref }} \ | ||
--work-dir . \ | ||
--saved-output docs/tutorials \ | ||
--save-path tutorials \ | ||
--command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ | ||
&& chmod +x Miniconda3-latest-Linux-x86_64.sh \ | ||
&& bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ | ||
&& source ci/prepare_clean_env.sh gpu/cy3 \ | ||
&& python3 -m pip install mxnet-cu102==1.6.0.post0 flaky numpy==1.18.4 \ | ||
&& ./ci/batch/compile_notebooks.sh \ | ||
'push' ${GITHUB_REF##*/}" \ | ||
--wait | tee build_website.log | ||
- name: Build Website (For pull request) | ||
if: ${{ (failure() || success()) && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') }} | ||
run: | | ||
python ./ci/batch/submit-job.py --region us-east-1 \ | ||
--job-type g4dn.4x \ | ||
--name GluonNLP-Website-v0.x-PR#${{ github.event.number }} \ | ||
--remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \ | ||
--source-ref ${{ github.event.pull_request.head.sha }} \ | ||
--work-dir . \ | ||
--saved-output docs/tutorials \ | ||
--save-path tutorials \ | ||
--command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ | ||
&& chmod +x Miniconda3-latest-Linux-x86_64.sh \ | ||
&& bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ | ||
&& source ci/prepare_clean_env.sh gpu/cy3 \ | ||
&& python3 -m pip install mxnet-cu102==1.6.0.post0 flaky numpy==1.18.4 \ | ||
&& ./ci/batch/compile_notebooks.sh \ | ||
${{ github.event.number }} ${{ github.event.pull_request.head.sha }}" \ | ||
--wait | tee build_website.log | ||
- name: Wait for job and copy logs from AWS s3 | ||
if: ${{ failure() || success() }} | ||
run: | | ||
head -100 build_website.log | grep -oP -m 1 'jobId: \K(.*)' > jobid.log | ||
echo "Job ID is" | ||
cat jobid.log | ||
cat jobid.log | xargs -i python ./tools/batch/wait-job.py --job-id {} | ||
echo "Copy Log file" | ||
cat jobid.log | xargs -i aws s3api wait object-exists --bucket gluon-nlp-dev --key batch/{}/tutorials/index.rst | ||
cat jobid.log | xargs -i aws s3 cp s3://gluon-nlp-dev/batch/{}/tutorials tutorials --recursive | ||
- name: Upload Notebook Compile Logs | ||
if: ${{ failure() || success() }} | ||
uses: actions/upload-artifact@v2 | ||
with: | ||
name: Notebook_Logs | ||
path: ./tutorials/**/*.stdout.log | ||
|
||
- name: Upload Website Build Log | ||
if: ${{ failure() || success() }} | ||
uses: actions/upload-artifact@v2 | ||
with: | ||
name: Website_Log | ||
path: ./build_website.log | ||
|
||
- name: Create comment | ||
if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} | ||
uses: peter-evans/create-or-update-comment@v1 | ||
with: | ||
issue-number: ${{ github.event.number }} | ||
body: | | ||
The documentation website for preview: http://gluon-nlp-staging.s3-accelerate.dualstack.amazonaws.com/PR${{ github.event.number }}/${{ github.event.pull_request.head.sha }}/index.html |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,106 @@ | ||
name: v0.x continuous build - gpu | ||
|
||
on: [push, pull_request_target] | ||
|
||
defaults: | ||
run: | ||
shell: bash | ||
|
||
jobs: | ||
unittest-gpu: | ||
runs-on: ubuntu-latest | ||
strategy: | ||
fail-fast: false | ||
steps: | ||
- name: Checkout repository | ||
uses: actions/checkout@v2 | ||
|
||
- name: Install Linux dependencies | ||
run: sudo apt-get install libopenblas-dev | ||
|
||
- name: Setup python | ||
uses: actions/setup-python@v2 | ||
with: | ||
python-version: 3.7 | ||
architecture: x64 | ||
|
||
- name: Install Other Dependencies | ||
run: | | ||
python -m pip install --quiet --upgrade pip | ||
python -m pip install --quiet boto3 | ||
- name: Configure AWS Credentials | ||
uses: aws-actions/configure-aws-credentials@v1 | ||
with: | ||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | ||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | ||
aws-region: us-east-1 | ||
|
||
- name: Test project on AWS Batch(For push) | ||
if: ${{ github.event_name == 'push' }} | ||
run: | | ||
echo "Start submitting job" | ||
python ./ci/batch/submit-job.py --region us-east-1 \ | ||
--job-type g4dn.4x \ | ||
--name GluonNLP-GPU-v0.x-Test-${{ github.ref }} \ | ||
--source-ref ${{ github.ref }} \ | ||
--work-dir . \ | ||
--saved-output coverage.xml \ | ||
--save-path coverage.xml \ | ||
--remote https://github.com/${{ github.repository }} \ | ||
--command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ | ||
&& chmod +x Miniconda3-latest-Linux-x86_64.sh \ | ||
&& bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ | ||
&& source ci/prepare_clean_env.sh gpu/cy3 \ | ||
&& python3 -m pip install mxnet-cu102==1.6.0.post0 flaky numpy==1.18.4 \ | ||
&& pytest -m 'gpu and not serial' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \ | ||
--wait | tee batch_job.log | ||
- name: Test project on AWS Batch(For pull request) | ||
if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} | ||
run: | | ||
echo "Start submitting job" | ||
python ./ci/batch/submit-job.py --region us-east-1 \ | ||
--job-type g4dn.4x \ | ||
--name GluonNLP-GPU-v0.x-Test-PR#${{ github.event.number }} \ | ||
--source-ref ${{ github.event.pull_request.head.sha }} \ | ||
--work-dir . \ | ||
--saved-output coverage.xml \ | ||
--save-path coverage.xml \ | ||
--remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \ | ||
--command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ | ||
&& chmod +x Miniconda3-latest-Linux-x86_64.sh \ | ||
&& bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ | ||
&& source ci/prepare_clean_env.sh gpu/cy3 \ | ||
&& python3 -m pip install mxnet-cu102==1.6.0.post0 flaky numpy==1.18.4 \ | ||
&& pytest -m 'gpu and not serial' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \ | ||
--wait | tee batch_job.log | ||
- name: Wait for job and copy files from AWS s3 | ||
if: ${{ failure() || success() }} | ||
run: | | ||
head -100 batch_job.log | grep -oP -m 1 'jobId: \K(.*)' > jobid.log | ||
echo "Job ID is" | ||
cat jobid.log | ||
cat jobid.log | xargs -i python ./tools/batch/wait-job.py --job-id {} | ||
echo "Copy Codecov file" | ||
cat jobid.log | xargs -i aws s3api wait object-exists --bucket gluon-nlp-dev --key batch/{}/coverage.xml | ||
cat jobid.log | xargs -i aws s3 cp s3://gluon-nlp-dev/batch/{}/coverage.xml ./coverage.xml | ||
- name: Upload coverage to Codecov | ||
run: | | ||
curl -s https://codecov.io/bash -o codecov.sh | ||
if [ "$EVENT_NAME" == "push" ]; then \ | ||
bash codecov.sh -f ./coverage.xml -n -F -B v0.x; \ | ||
else \ | ||
bash codecov.sh -f ./coverage.xml -n -F -B v0.x -C ${{ github.event.pull_request.head.sha }} -P ${{ github.event.pull_request.number }}; \ | ||
fi | ||
env: | ||
EVENT_NAME: ${{ github.event_name }} | ||
|
||
- name: Upload Cloud Watch Log Stream | ||
if: ${{ failure() || success() }} | ||
uses: actions/upload-artifact@v2 | ||
with: | ||
name: Test_Logs | ||
path: ./batch_job.log |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,105 @@ | ||
name: v0.x continuous build | ||
|
||
on: [push, pull_request_target] | ||
|
||
defaults: | ||
run: | ||
shell: bash | ||
|
||
jobs: | ||
unittest-gpu: | ||
runs-on: ubuntu-latest | ||
strategy: | ||
fail-fast: false | ||
steps: | ||
- name: Checkout repository | ||
uses: actions/checkout@v2 | ||
|
||
- name: Install Linux dependencies | ||
run: sudo apt-get install libopenblas-dev | ||
|
||
- name: Setup python | ||
uses: actions/setup-python@v2 | ||
with: | ||
python-version: 3.7 | ||
architecture: x64 | ||
|
||
- name: Install Other Dependencies | ||
run: | | ||
python -m pip install --quiet --upgrade pip | ||
python -m pip install --quiet boto3 | ||
- name: Configure AWS Credentials | ||
uses: aws-actions/configure-aws-credentials@v1 | ||
with: | ||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | ||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | ||
aws-region: us-east-1 | ||
|
||
- name: Test project on AWS Batch(For push) | ||
if: ${{ github.event_name == 'push' }} | ||
run: | | ||
echo "Start submitting job" | ||
python ./ci/batch/submit-job.py --region us-east-1 \ | ||
--job-type c5n.4x \ | ||
--name GluonNLP-GPU-v0.x-Test-${{ github.ref }} \ | ||
--source-ref ${{ github.ref }} \ | ||
--work-dir . \ | ||
--saved-output coverage.xml \ | ||
--save-path coverage.xml \ | ||
--remote https://github.com/${{ github.repository }} \ | ||
--command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ | ||
&& chmod +x Miniconda3-latest-Linux-x86_64.sh \ | ||
&& bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ | ||
&& source ci/prepare_clean_env.sh gpu/cy3 \ | ||
&& python3 -m pip install mxnet==1.6.0 flaky numpy==1.18.4 \ | ||
&& pytest -m 'not (gpu or serial)' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \ | ||
--wait | tee batch_job.log | ||
- name: Test project on AWS Batch(For pull request) | ||
if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} | ||
run: | | ||
echo "Start submitting job" | ||
python ./ci/batch/submit-job.py --region us-east-1 \ | ||
--job-type c5n.4x \ | ||
--name GluonNLP-GPU-v0.x-Test-PR#${{ github.event.number }} \ | ||
--source-ref ${{ github.event.pull_request.head.sha }} \ | ||
--work-dir . \ | ||
--saved-output coverage.xml \ | ||
--save-path coverage.xml \ | ||
--remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \ | ||
--command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ | ||
&& chmod +x Miniconda3-latest-Linux-x86_64.sh \ | ||
&& bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ | ||
&& source ci/prepare_clean_env.sh gpu/cy3 \ | ||
&& python3 -m pip install mxnet==1.6.0 flaky numpy==1.18.4 \ | ||
&& pytest -m 'not (gpu or serial)' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \ | ||
--wait | tee batch_job.log | ||
- name: Wait for job and copy files from AWS s3 | ||
if: ${{ failure() || success() }} | ||
run: | | ||
head -100 batch_job.log | grep -oP -m 1 'jobId: \K(.*)' > jobid.log | ||
echo "Job ID is" | ||
cat jobid.log | ||
cat jobid.log | xargs -i python ./tools/batch/wait-job.py --job-id {} | ||
echo "Copy Codecov file" | ||
cat jobid.log | xargs -i aws s3api wait object-exists --bucket gluon-nlp-dev --key batch/{}/coverage.xml | ||
cat jobid.log | xargs -i aws s3 cp s3://gluon-nlp-dev/batch/{}/coverage.xml ./coverage.xml | ||
- name: Upload coverage to Codecov | ||
run: | | ||
curl -s https://codecov.io/bash -o codecov.sh | ||
if [ "$EVENT_NAME" == "push" ]; then \ | ||
bash codecov.sh -f ./coverage.xml -n -F -B v0.x; \ | ||
else \ | ||
bash codecov.sh -f ./coverage.xml -n -F -B v0.x -C ${{ github.event.pull_request.head.sha }} -P ${{ github.event.pull_request.number }}; \ | ||
fi | ||
env: | ||
EVENT_NAME: ${{ github.event_name }} | ||
|
||
- name: Upload Cloud Watch Log Stream | ||
if: ${{ failure() || success() }} | ||
uses: actions/upload-artifact@v2 | ||
with: | ||
name: Test_Logs | ||
path: ./batch_job.log |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
#!/bin/bash | ||
# Shell script for submitting AWS Batch jobs to compile notebooks | ||
|
||
event=$1 | ||
ref=$2 | ||
|
||
FAIL=0 | ||
|
||
compile_notebook () { | ||
local MDFILE=$1 | ||
DIR=$(dirname $MDFILE) | ||
BASENAME=$(basename $MDFILE) | ||
TARGETNAME=$(dirname $MDFILE)/${BASENAME%.md}.ipynb | ||
LOGNAME=$(dirname $MDFILE)/${BASENAME%.md}.stdout.log | ||
|
||
echo Compiling $BASENAME ... | ||
|
||
python3 docs/md2ipynb.py ${MDFILE} &> $LOGNAME | ||
|
||
EXIT_CODE=$? | ||
|
||
if [ $EXIT_CODE -ne 0 ]; then | ||
echo Compiling $BASENAME Failed, please download Notebook_Logs in build Artifacts for more details. | ||
else | ||
echo Compiling $BASENAME Succeeded | ||
fi | ||
exit $EXIT_CODE | ||
} | ||
|
||
pids=() | ||
|
||
for f in $(find docs/tutorials -type f -name '*.md' -print); do | ||
compile_notebook "$f" & | ||
pids+=($!) | ||
done; | ||
|
||
for pid in "${pids[@]}"; do | ||
wait "$pid" || let "FAIL+=1" | ||
done; | ||
|
||
if [ "$FAIL" == "0" ]; then | ||
echo Building Website | ||
make docs | ||
EXIT_CODE=$? | ||
if [ $EXIT_CODE -ne 0 ]; then | ||
echo Building Website Failed. | ||
exit $EXIT_CODE | ||
else | ||
echo Building Website Succeeded. | ||
if [ "$1" == "push" ]; then | ||
echo "Uploading docs to s3://gluon-nlp/$2/" | ||
aws s3 sync --delete ./docs/_build/html/ s3://gluon-nlp/$2/ --quiet --acl public-read | ||
else | ||
echo "Uploading docs to s3://gluon-nlp-staging/PR$1/$2/" | ||
aws s3 sync --delete ./docs/_build/html/ s3://gluon-nlp-staging/PR$1/$2/ --quiet --acl public-read | ||
fi | ||
fi | ||
else | ||
exit 1 | ||
fi |
Oops, something went wrong.