From dbfb1ed6c1d245df0c4abcf6c4065eae8580504b Mon Sep 17 00:00:00 2001 From: barry-jin Date: Wed, 24 Feb 2021 09:43:03 -0800 Subject: [PATCH 01/14] backport ci from master --- .github/workflows/unittests.yml | 65 +++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 .github/workflows/unittests.yml diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml new file mode 100644 index 0000000000..9315dc6536 --- /dev/null +++ b/.github/workflows/unittests.yml @@ -0,0 +1,65 @@ +name: continuous build + +on: [push, pull_request] + +defaults: + run: + shell: bash + +jobs: + unittest: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # TODO Add windows test by using "windows-latest" + os: [ubuntu-latest, macos-latest] + python-version: [ '3.6', '3.7', '3.8'] + exclude: + - os: macos-latest + python-version: 3.6 + - os: macos-latest + python-version: 3.8 + - os: ubuntu-latest + python-version: 3.7 + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Compilation cache + uses: actions/cache@v2 + with: + path: ~/.ccache + # We include the commit sha in the cache key, as new cache entries are + # only created if there is no existing entry for the key yet. + key: ${{ runner.os }}-ccache-${{ github.sha }} + # Restore any ccache cache entry, if none for + # ${{ runner.os }}-ccache-${{ github.sha }} exists + restore-keys: | + ${{ runner.os }}-ccache + # Install Linux specific dependencies + - name: Install Linux dependencies + if: matrix.os == 'ubuntu-latest' + # TODO https://github.com/apache/incubator-mxnet/issues/18293 + run: | + sudo apt-get install -y libopenblas-dev ninja-build libedit-dev libxml2-dev + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + - name: Install Other Dependencies + run: | + python -m pip install --upgrade pip + python -m pip install setuptools pytest pytest-cov contextvars + python -m pip install --upgrade cython + python -m pip install mxnet<2 + python -m pip install -U -e .[extras,dev] + + - name: Run Unittests + run: | + pytest -n 4 -m 'not (gpu or serial)' --durations=30 --cov=./ --cov-report=xml tests/unittest + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1.0.10 + with: + env_vars: OS,PYTHON From 742103c60e7e4fd33a6de9cf6923e2080b23143a Mon Sep 17 00:00:00 2001 From: barry-jin Date: Wed, 24 Feb 2021 09:49:53 -0800 Subject: [PATCH 02/14] update mxnet version --- .github/workflows/unittests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 9315dc6536..14bc834437 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -53,7 +53,7 @@ jobs: python -m pip install --upgrade pip python -m pip install setuptools pytest pytest-cov contextvars python -m pip install --upgrade cython - python -m pip install mxnet<2 + python -m pip install mxnet>=1.6.0 python -m pip install -U -e .[extras,dev] - name: Run Unittests From f49468e7a3e23e72dae877c47937330bedc88bc1 Mon Sep 17 00:00:00 2001 From: barry-jin Date: Wed, 24 Feb 2021 10:01:40 -0800 Subject: [PATCH 03/14] update mxnet version --- .github/workflows/unittests.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 14bc834437..b7d93930b4 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -52,9 +52,13 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install setuptools pytest pytest-cov contextvars - python -m pip install --upgrade cython - python -m pip install mxnet>=1.6.0 + python -m pip install mxnet==1.7.0 python -m pip install -U -e .[extras,dev] + python -m pip install -v -e . + python -m pip install horovod --no-cache-dir -U + python -m spacy download en + python -m spacy download de + python -m nltk.downloader all - name: Run Unittests run: | From 43a012c34c09c7d2fbdc2c7ba1a911c8e9d94bc8 Mon Sep 17 00:00:00 2001 From: barry-jin Date: Wed, 24 Feb 2021 10:08:29 -0800 Subject: [PATCH 04/14] update mxnet version --- .github/workflows/unittests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index b7d93930b4..f73a2a5811 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -52,7 +52,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install setuptools pytest pytest-cov contextvars - python -m pip install mxnet==1.7.0 + python -m pip install --pre "mxnet>=1.9.0b20210220" -f https://dist.mxnet.io/python python -m pip install -U -e .[extras,dev] python -m pip install -v -e . python -m pip install horovod --no-cache-dir -U From 6b263d4fb2993f420f61a5228a1d5787d8ded8de Mon Sep 17 00:00:00 2001 From: barry-jin Date: Wed, 24 Feb 2021 10:12:22 -0800 Subject: [PATCH 05/14] remove some dependecies --- .github/workflows/unittests.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index f73a2a5811..a879d21589 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -37,12 +37,6 @@ jobs: # ${{ runner.os }}-ccache-${{ github.sha }} exists restore-keys: | ${{ runner.os }}-ccache - # Install Linux specific dependencies - - name: Install Linux dependencies - if: matrix.os == 'ubuntu-latest' - # TODO https://github.com/apache/incubator-mxnet/issues/18293 - run: | - sudo apt-get install -y libopenblas-dev ninja-build libedit-dev libxml2-dev - name: Setup python uses: actions/setup-python@v2 with: From 7047a6290dbd3bd37affcb8c6b98523483acea91 Mon Sep 17 00:00:00 2001 From: barry-jin Date: Wed, 24 Feb 2021 10:25:33 -0800 Subject: [PATCH 06/14] update mxnet version --- .github/workflows/unittests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index a879d21589..d4860be540 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -46,7 +46,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install setuptools pytest pytest-cov contextvars - python -m pip install --pre "mxnet>=1.9.0b20210220" -f https://dist.mxnet.io/python + python -m pip install --pre "mxnet>=1.9.0b20210220, <2.0.0" -f https://dist.mxnet.io/python python -m pip install -U -e .[extras,dev] python -m pip install -v -e . python -m pip install horovod --no-cache-dir -U From be43e866650d2da7248f689a6bf648c0cc2e6db1 Mon Sep 17 00:00:00 2001 From: barry-jin Date: Wed, 24 Feb 2021 16:55:20 -0800 Subject: [PATCH 07/14] update --- .github/workflows/buildwebsite.yml | 99 ++++++++++++++++++++++++++ .github/workflows/unittests-gpu.yml | 106 ++++++++++++++++++++++++++++ .github/workflows/unittests.yml | 40 +++++------ ci/batch/compile_notebooks.sh | 60 ++++++++++++++++ ci/batch/submit-job.py | 87 +++++++++++++++-------- 5 files changed, 338 insertions(+), 54 deletions(-) create mode 100644 .github/workflows/buildwebsite.yml create mode 100644 .github/workflows/unittests-gpu.yml create mode 100755 ci/batch/compile_notebooks.sh diff --git a/.github/workflows/buildwebsite.yml b/.github/workflows/buildwebsite.yml new file mode 100644 index 0000000000..8eba2f1ac9 --- /dev/null +++ b/.github/workflows/buildwebsite.yml @@ -0,0 +1,99 @@ +name: continuous build - docs + +on: [push, pull_request_target] + +defaults: + run: + shell: bash + +jobs: + website-build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + architecture: x64 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Install Other Dependencies + run: | + python -m pip install --quiet --upgrade pip + python -m pip install --quiet boto3 + - name: Build Website (For push) + if: ${{ (failure() || success()) && github.event_name == 'push' }} + run: | + python ./ci/batch/submit-job.py --region us-east-1 \ + --job-type g4dn.4x \ + --name GluonNLP-Website-v0.x-${{ github.ref }} \ + --remote https://github.com/${{ github.repository }} \ + --source-ref ${{ github.ref }} \ + --work-dir . \ + --saved-output docs/tutorials \ + --save-path tutorials \ + --command "source ci/prepare_clean_env.sh cpu/py3-master \ + && ./tools/batch/batch_states/compile_notebooks.sh \ + 'push' ${GITHUB_REF##*/}" \ + --wait | tee build_website.log + + - name: Build Website (For pull request) + if: ${{ (failure() || success()) && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') }} + run: | + python ./ci/batch/submit-job.py --region us-east-1 \ + --job-type g4dn.4x \ + --name GluonNLP-Website-v0.x-PR#${{ github.event.number }} \ + --remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \ + --source-ref ${{ github.event.pull_request.head.sha }} \ + --work-dir . \ + --saved-output docs/tutorials \ + --save-path tutorials \ + --command "source ci/prepare_clean_env.sh cpu/py3-master \ + && ./tools/batch/batch_states/compile_notebooks.sh \ + ${{ github.event.number }} ${{ github.event.pull_request.head.sha }}" \ + --wait | tee build_website.log + + - name: Wait for job and copy logs from AWS s3 + if: ${{ failure() || success() }} + run: | + head -100 build_website.log | grep -oP -m 1 'jobId: \K(.*)' > jobid.log + + echo "Job ID is" + cat jobid.log + cat jobid.log | xargs -i python ./tools/batch/wait-job.py --job-id {} + echo "Copy Log file" + cat jobid.log | xargs -i aws s3api wait object-exists --bucket gluon-nlp-dev --key batch/{}/tutorials/index.rst + cat jobid.log | xargs -i aws s3 cp s3://gluon-nlp-dev/batch/{}/tutorials tutorials --recursive + + - name: Upload Notebook Compile Logs + if: ${{ failure() || success() }} + uses: actions/upload-artifact@v2 + with: + name: Notebook_Logs + path: ./tutorials/**/*.stdout.log + + - name: Upload Website Build Log + if: ${{ failure() || success() }} + uses: actions/upload-artifact@v2 + with: + name: Website_Log + path: ./build_website.log + + - name: Create comment + if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} + uses: peter-evans/create-or-update-comment@v1 + with: + issue-number: ${{ github.event.number }} + body: | + The documentation website for preview: http://gluon-nlp-staging.s3-accelerate.dualstack.amazonaws.com/PR${{ github.event.number }}/${{ github.event.pull_request.head.sha }}/index.html diff --git a/.github/workflows/unittests-gpu.yml b/.github/workflows/unittests-gpu.yml new file mode 100644 index 0000000000..d18ebaf087 --- /dev/null +++ b/.github/workflows/unittests-gpu.yml @@ -0,0 +1,106 @@ +name: continuous build - gpu + +on: [push, pull_request_target] + +defaults: + run: + shell: bash + +jobs: + unittest-gpu: + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install Linux dependencies + run: sudo apt-get install libopenblas-dev + + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + architecture: x64 + + - name: Install Other Dependencies + run: | + python -m pip install --quiet --upgrade pip + python -m pip install --quiet boto3 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Test project on AWS Batch(For push) + if: ${{ github.event_name == 'push' }} + run: | + echo "Start submitting job" + python ./ci/batch/submit-job.py --region us-east-1 \ + --job-type g4dn.4x \ + --name GluonNLP-GPU-v0.x-Test-${{ github.ref }} \ + --source-ref ${{ github.ref }} \ + --work-dir . \ + --saved-output coverage.xml \ + --save-path coverage.xml \ + --remote https://github.com/${{ github.repository }} \ + --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && chmod +x Miniconda3-latest-Linux-x86_64.sh \ + && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ + && source ci/prepare_clean_env.sh gpu/cy3 \ + && python3 -m pip install mxnet-cu102==1.6.0.post0 flaky \ + && pytest -m 'gpu and not serial' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \ + --wait | tee batch_job.log + + - name: Test project on AWS Batch(For pull request) + if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} + run: | + echo "Start submitting job" + python ./ci/batch/submit-job.py --region us-east-1 \ + --job-type g4dn.4x \ + --name GluonNLP-GPU-v0.x-Test-PR#${{ github.event.number }} \ + --source-ref ${{ github.event.pull_request.head.sha }} \ + --work-dir . \ + --saved-output coverage.xml \ + --save-path coverage.xml \ + --remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \ + --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && chmod +x Miniconda3-latest-Linux-x86_64.sh \ + && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ + && source ci/prepare_clean_env.sh gpu/cy3 \ + && python3 -m pip install mxnet-cu102==1.6.0.post0 flaky \ + && pytest -m 'gpu and not serial' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \ + --wait | tee batch_job.log + + - name: Wait for job and copy files from AWS s3 + if: ${{ failure() || success() }} + run: | + head -100 batch_job.log | grep -oP -m 1 'jobId: \K(.*)' > jobid.log + echo "Job ID is" + cat jobid.log + cat jobid.log | xargs -i python ./tools/batch/wait-job.py --job-id {} + echo "Copy Codecov file" + cat jobid.log | xargs -i aws s3api wait object-exists --bucket gluon-nlp-dev --key batch/{}/coverage.xml + cat jobid.log | xargs -i aws s3 cp s3://gluon-nlp-dev/batch/{}/coverage.xml ./coverage.xml + + - name: Upload coverage to Codecov + run: | + curl -s https://codecov.io/bash -o codecov.sh + if [ "$EVENT_NAME" == "push" ]; then \ + bash codecov.sh -f ./coverage.xml -n -F; \ + else \ + bash codecov.sh -f ./coverage.xml -n -F -C ${{ github.event.pull_request.head.sha }} -P ${{ github.event.pull_request.number }}; \ + fi + env: + EVENT_NAME: ${{ github.event_name }} + + - name: Upload Cloud Watch Log Stream + if: ${{ failure() || success() }} + uses: actions/upload-artifact@v2 + with: + name: Test_Logs + path: ./batch_job.log diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index d4860be540..bf9304e3b4 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -13,19 +13,12 @@ jobs: fail-fast: false matrix: # TODO Add windows test by using "windows-latest" - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest] python-version: [ '3.6', '3.7', '3.8'] - exclude: - - os: macos-latest - python-version: 3.6 - - os: macos-latest - python-version: 3.8 - - os: ubuntu-latest - python-version: 3.7 + steps: - name: Checkout repository uses: actions/checkout@v2 - - name: Compilation cache uses: actions/cache@v2 with: @@ -37,26 +30,25 @@ jobs: # ${{ runner.os }}-ccache-${{ github.sha }} exists restore-keys: | ${{ runner.os }}-ccache - - name: Setup python - uses: actions/setup-python@v2 + + - name: Setup Miniconda + uses: conda-incubator/setup-miniconda@v2.0.0 with: + auto-update-conda: true python-version: ${{ matrix.python-version }} - architecture: x64 - - name: Install Other Dependencies + + - name: Sanity Check + shell: bash -l {0} run: | - python -m pip install --upgrade pip - python -m pip install setuptools pytest pytest-cov contextvars - python -m pip install --pre "mxnet>=1.9.0b20210220, <2.0.0" -f https://dist.mxnet.io/python - python -m pip install -U -e .[extras,dev] - python -m pip install -v -e . - python -m pip install horovod --no-cache-dir -U - python -m spacy download en - python -m spacy download de - python -m nltk.downloader all - - - name: Run Unittests + source ci/prepare_clean_env.sh cpu/py3 + make lintdir=src/gluonnlp lint + + - name: Unittest + shell: bash -l {0} run: | + source ci/prepare_clean_env.sh cpu/py3 pytest -n 4 -m 'not (gpu or serial)' --durations=30 --cov=./ --cov-report=xml tests/unittest + - name: Upload coverage to Codecov uses: codecov/codecov-action@v1.0.10 with: diff --git a/ci/batch/compile_notebooks.sh b/ci/batch/compile_notebooks.sh new file mode 100755 index 0000000000..7f3ffd5e37 --- /dev/null +++ b/ci/batch/compile_notebooks.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Shell script for submitting AWS Batch jobs to compile notebooks + +event=$1 +ref=$2 + +FAIL=0 + +compile_notebook () { + local MDFILE=$1 + DIR=$(dirname $MDFILE) + BASENAME=$(basename $MDFILE) + TARGETNAME=$(dirname $MDFILE)/${BASENAME%.md}.ipynb + LOGNAME=$(dirname $MDFILE)/${BASENAME%.md}.stdout.log + + echo Compiling $BASENAME ... + + python3 docs/md2ipynb.py ${MDFILE} &> $LOGNAME + + EXIT_CODE=$? + + if [ $EXIT_CODE -ne 0 ]; then + echo Compiling $BASENAME Failed, please download Notebook_Logs in build Artifacts for more details. + else + echo Compiling $BASENAME Succeeded + fi + exit $EXIT_CODE +} + +pids=() + +for f in $(find docs/tutorials -type f -name '*.md' -print); do + compile_notebook "$f" & + pids+=($!) +done; + +for pid in "${pids[@]}"; do + wait "$pid" || let "FAIL+=1" +done; + +if [ "$FAIL" == "0" ]; then + echo Building Website + make docs + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ]; then + echo Building Website Failed. + exit $EXIT_CODE + else + echo Building Website Succeeded. + if [ "$1" == "push" ]; then + echo "Uploading docs to s3://gluon-nlp/$2/" + aws s3 sync --delete ./docs/_build/html/ s3://gluon-nlp/$2/ --quiet --acl public-read + else + echo "Uploading docs to s3://gluon-nlp-staging/PR$1/$2/" + aws s3 sync --delete ./docs/_build/html/ s3://gluon-nlp-staging/PR$1/$2/ --quiet --acl public-read + fi + fi +else + exit 1 +fi diff --git a/ci/batch/submit-job.py b/ci/batch/submit-job.py index ec99e44f47..cbaf0809fc 100644 --- a/ci/batch/submit-job.py +++ b/ci/batch/submit-job.py @@ -8,23 +8,60 @@ import boto3 from botocore.compat import total_seconds +instance_type_info = { + 'g4dn.4x': { + 'job_definition': 'gluon-nlp-g4dn_4xlarge:5', + 'job_queue': 'g4dn' + }, + 'g4dn.8x': { + 'job_definition': 'gluon-nlp-g4dn_8xlarge:5', + 'job_queue': 'g4dn' + }, + 'g4dn.12x': { + 'job_definition': 'gluon-nlp-g4dn_12xlarge:5', + 'job_queue': 'g4dn-multi-gpu' + }, + 'p3.2x': { + 'job_definition': 'gluon-nlp-p3_2xlarge:5', + 'job_queue': 'p3' + }, + 'p3.8x': { + 'job_definition': 'gluon-nlp-p3_8xlarge:5', + 'job_queue': 'p3-4gpu' + }, + 'p3.16x': { + 'job_definition': 'gluon-nlp-p3_16xlarge:5', + 'job_queue': 'p3-8gpu' + }, + 'p3dn.24x': { + 'job_definition': 'gluon-nlp-p3_24xlarge:5', + 'job_queue': 'p3dn-8gpu' + }, + 'c5n.4x': { + 'job_definition': 'gluon-nlp-c5_4xlarge:3', + 'job_queue': 'c5n' + }, + 'c5n.18x': { + 'job_definition': 'gluon-nlp-c5_18xlarge:3', + 'job_queue': 'c5n' + } +} + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--profile', help='profile name of aws account.', type=str, default=None) parser.add_argument('--region', help='Default region when creating new connections', type=str, - default=None) + default='us-east-1') parser.add_argument('--name', help='name of the job', type=str, default='dummy') -parser.add_argument('--job-queue', help='name of the job queue to submit this job', type=str, - default='gluon-nlp-jobs') -parser.add_argument('--job-definition', help='name of the job job definition', type=str, - default='gluon-nlp-jobs:8') +parser.add_argument('--job-type', help='type of job to submit.', type=str, + choices=instance_type_info.keys(), default='g4dn.4x') parser.add_argument('--source-ref', help='ref in GluonNLP main github. e.g. master, refs/pull/500/head', type=str, default='master') parser.add_argument('--work-dir', - help='working directory inside the repo. e.g. scripts/sentiment_analysis', - type=str, default='scripts/bert') + help='working directory inside the repo. e.g. scripts/preprocess', + type=str, default='scripts/preprocess') parser.add_argument('--saved-output', help='output to be saved, relative to working directory. ' 'it can be either a single file or a directory', @@ -32,9 +69,6 @@ parser.add_argument('--save-path', help='s3 path where files are saved.', type=str, default='batch/temp/{}'.format(datetime.now().isoformat())) -parser.add_argument('--conda-env', - help='conda environment preset to use.', - type=str, default='gpu/py3') parser.add_argument('--command', help='command to run', type=str, default='git rev-parse HEAD | tee stdout.log') parser.add_argument('--remote', @@ -44,18 +78,20 @@ 'Non-zero exit code if job fails.', action='store_true') parser.add_argument('--timeout', help='job timeout in seconds', default=None, type=int) + args = parser.parse_args() session = boto3.Session(profile_name=args.profile, region_name=args.region) batch, cloudwatch = [session.client(service_name=sn) for sn in ['batch', 'logs']] + def printLogs(logGroupName, logStreamName, startTime): kwargs = {'logGroupName': logGroupName, 'logStreamName': logStreamName, 'startTime': startTime, 'startFromHead': True} - lastTimestamp = 0 + lastTimestamp = startTime - 1 while True: logEvents = cloudwatch.get_log_events(**kwargs) @@ -72,17 +108,6 @@ def printLogs(logGroupName, logStreamName, startTime): return lastTimestamp -def getLogStream(logGroupName, jobName, jobId): - response = cloudwatch.describe_log_streams( - logGroupName=logGroupName, - logStreamNamePrefix=jobName + '/' + jobId - ) - logStreams = response['logStreams'] - if not logStreams: - return '' - else: - return logStreams[0]['logStreamName'] - def nowInMillis(): endTime = long(total_seconds(datetime.utcnow() - datetime(1970, 1, 1))) * 1000 return endTime @@ -93,17 +118,17 @@ def main(): logGroupName = '/aws/batch/job' jobName = re.sub('[^A-Za-z0-9_\-]', '', args.name)[:128] # Enforce AWS Batch jobName rules - jobQueue = args.job_queue - jobDefinition = args.job_definition + jobType = args.job_type + jobQueue = instance_type_info[jobType]['job_queue'] + jobDefinition = instance_type_info[jobType]['job_definition'] command = args.command.split() wait = args.wait - parameters={ + parameters = { 'SOURCE_REF': args.source_ref, 'WORK_DIR': args.work_dir, 'SAVED_OUTPUT': args.saved_output, 'SAVE_PATH': args.save_path, - 'CONDA_ENV': args.conda_env, 'COMMAND': args.command, 'REMOTE': args.remote } @@ -124,22 +149,23 @@ def main(): running = False status_set = set() startTime = 0 - + logStreamName = None while wait: time.sleep(random.randint(5, 10)) describeJobsResponse = batch.describe_jobs(jobs=[jobId]) status = describeJobsResponse['jobs'][0]['status'] if status == 'SUCCEEDED' or status == 'FAILED': + if logStreamName: + startTime = printLogs(logGroupName, logStreamName, startTime) + 1 print('=' * 80) print('Job [{} - {}] {}'.format(jobName, jobId, status)) - sys.exit(status == 'FAILED') elif status == 'RUNNING': - logStreamName = getLogStream(logGroupName, jobName, jobId) + logStreamName = describeJobsResponse['jobs'][0]['container']['logStreamName'] if not running: running = True - print('\rJob [{} - {}] is RUNNING.'.format(jobName, jobId)) + print('\rJob [{}, {}] is RUNNING.'.format(jobName, jobId)) if logStreamName: print('Output [{}]:\n {}'.format(logStreamName, '=' * 80)) if logStreamName: @@ -150,5 +176,6 @@ def main(): sys.stdout.flush() spinner += 1 + if __name__ == '__main__': main() From 708780f313f5bc054ad7a4a98451802c01d6a436 Mon Sep 17 00:00:00 2001 From: barry-jin Date: Thu, 25 Feb 2021 09:23:20 -0800 Subject: [PATCH 08/14] update unittests.yml --- .github/workflows/unittests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index bf9304e3b4..81cf09ea94 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -47,7 +47,7 @@ jobs: shell: bash -l {0} run: | source ci/prepare_clean_env.sh cpu/py3 - pytest -n 4 -m 'not (gpu or serial)' --durations=30 --cov=./ --cov-report=xml tests/unittest + pytest -m 'not (gpu or serial)' --durations=30 --cov=./ --cov-report=xml tests/unittest - name: Upload coverage to Codecov uses: codecov/codecov-action@v1.0.10 From 920cbafcba375591025d740de645e6a67c283aa2 Mon Sep 17 00:00:00 2001 From: barry-jin Date: Fri, 26 Feb 2021 14:21:51 -0800 Subject: [PATCH 09/14] update workflow --- .github/workflows/buildwebsite.yml | 18 +++-- .github/workflows/unittests-gpu.yml | 10 +-- .github/workflows/unittests.yml | 119 ++++++++++++++++++++-------- 3 files changed, 103 insertions(+), 44 deletions(-) diff --git a/.github/workflows/buildwebsite.yml b/.github/workflows/buildwebsite.yml index 8eba2f1ac9..eabcfbec0f 100644 --- a/.github/workflows/buildwebsite.yml +++ b/.github/workflows/buildwebsite.yml @@ -1,4 +1,4 @@ -name: continuous build - docs +name: v0.x continuous build - docs on: [push, pull_request_target] @@ -43,8 +43,12 @@ jobs: --work-dir . \ --saved-output docs/tutorials \ --save-path tutorials \ - --command "source ci/prepare_clean_env.sh cpu/py3-master \ - && ./tools/batch/batch_states/compile_notebooks.sh \ + --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && chmod +x Miniconda3-latest-Linux-x86_64.sh \ + && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ + && source ci/prepare_clean_env.sh gpu/cy3 \ + && python3 -m pip install mxnet-cu102==1.6.0.post0 flaky numpy==1.18.4 \ + && ./ci/batch/compile_notebooks.sh \ 'push' ${GITHUB_REF##*/}" \ --wait | tee build_website.log @@ -59,8 +63,12 @@ jobs: --work-dir . \ --saved-output docs/tutorials \ --save-path tutorials \ - --command "source ci/prepare_clean_env.sh cpu/py3-master \ - && ./tools/batch/batch_states/compile_notebooks.sh \ + --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && chmod +x Miniconda3-latest-Linux-x86_64.sh \ + && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ + && source ci/prepare_clean_env.sh gpu/cy3 \ + && python3 -m pip install mxnet-cu102==1.6.0.post0 flaky numpy==1.18.4 \ + && ./ci/batch/compile_notebooks.sh \ ${{ github.event.number }} ${{ github.event.pull_request.head.sha }}" \ --wait | tee build_website.log diff --git a/.github/workflows/unittests-gpu.yml b/.github/workflows/unittests-gpu.yml index d18ebaf087..19965540c3 100644 --- a/.github/workflows/unittests-gpu.yml +++ b/.github/workflows/unittests-gpu.yml @@ -1,4 +1,4 @@ -name: continuous build - gpu +name: v0.x continuous build - gpu on: [push, pull_request_target] @@ -52,7 +52,7 @@ jobs: && chmod +x Miniconda3-latest-Linux-x86_64.sh \ && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ && source ci/prepare_clean_env.sh gpu/cy3 \ - && python3 -m pip install mxnet-cu102==1.6.0.post0 flaky \ + && python3 -m pip install mxnet-cu102==1.6.0.post0 flaky numpy==1.18.4 \ && pytest -m 'gpu and not serial' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \ --wait | tee batch_job.log @@ -72,7 +72,7 @@ jobs: && chmod +x Miniconda3-latest-Linux-x86_64.sh \ && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ && source ci/prepare_clean_env.sh gpu/cy3 \ - && python3 -m pip install mxnet-cu102==1.6.0.post0 flaky \ + && python3 -m pip install mxnet-cu102==1.6.0.post0 flaky numpy==1.18.4 \ && pytest -m 'gpu and not serial' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \ --wait | tee batch_job.log @@ -91,9 +91,9 @@ jobs: run: | curl -s https://codecov.io/bash -o codecov.sh if [ "$EVENT_NAME" == "push" ]; then \ - bash codecov.sh -f ./coverage.xml -n -F; \ + bash codecov.sh -f ./coverage.xml -n -F -B v0.x; \ else \ - bash codecov.sh -f ./coverage.xml -n -F -C ${{ github.event.pull_request.head.sha }} -P ${{ github.event.pull_request.number }}; \ + bash codecov.sh -f ./coverage.xml -n -F -B v0.x -C ${{ github.event.pull_request.head.sha }} -P ${{ github.event.pull_request.number }}; \ fi env: EVENT_NAME: ${{ github.event_name }} diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 81cf09ea94..5b842bb620 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -1,55 +1,106 @@ -name: continuous build +name: v0.x continuous build - gpu -on: [push, pull_request] +on: [push, pull_request_target] defaults: run: shell: bash jobs: - unittest: - runs-on: ${{ matrix.os }} + unittest-gpu: + runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - # TODO Add windows test by using "windows-latest" - os: [ubuntu-latest] - python-version: [ '3.6', '3.7', '3.8'] - steps: - name: Checkout repository uses: actions/checkout@v2 - - name: Compilation cache - uses: actions/cache@v2 + + - name: Install Linux dependencies + run: sudo apt-get install libopenblas-dev + + - name: Setup python + uses: actions/setup-python@v2 with: - path: ~/.ccache - # We include the commit sha in the cache key, as new cache entries are - # only created if there is no existing entry for the key yet. - key: ${{ runner.os }}-ccache-${{ github.sha }} - # Restore any ccache cache entry, if none for - # ${{ runner.os }}-ccache-${{ github.sha }} exists - restore-keys: | - ${{ runner.os }}-ccache - - - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2.0.0 + python-version: 3.7 + architecture: x64 + + - name: Install Other Dependencies + run: | + python -m pip install --quiet --upgrade pip + python -m pip install --quiet boto3 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 with: - auto-update-conda: true - python-version: ${{ matrix.python-version }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Test project on AWS Batch(For push) + if: ${{ github.event_name == 'push' }} + run: | + echo "Start submitting job" + python ./ci/batch/submit-job.py --region us-east-1 \ + --job-type c5n.4x \ + --name GluonNLP-GPU-v0.x-Test-${{ github.ref }} \ + --source-ref ${{ github.ref }} \ + --work-dir . \ + --saved-output coverage.xml \ + --save-path coverage.xml \ + --remote https://github.com/${{ github.repository }} \ + --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && chmod +x Miniconda3-latest-Linux-x86_64.sh \ + && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ + && source ci/prepare_clean_env.sh gpu/cy3 \ + && python3 -m pip install mxnet==1.6.0 flaky numpy==1.18.4 \ + && pytest -m 'not (gpu or serial)' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \ + --wait | tee batch_job.log - - name: Sanity Check - shell: bash -l {0} + - name: Test project on AWS Batch(For pull request) + if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} run: | - source ci/prepare_clean_env.sh cpu/py3 - make lintdir=src/gluonnlp lint + echo "Start submitting job" + python ./ci/batch/submit-job.py --region us-east-1 \ + --job-type c5n.4x \ + --name GluonNLP-GPU-v0.x-Test-PR#${{ github.event.number }} \ + --source-ref ${{ github.event.pull_request.head.sha }} \ + --work-dir . \ + --saved-output coverage.xml \ + --save-path coverage.xml \ + --remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \ + --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && chmod +x Miniconda3-latest-Linux-x86_64.sh \ + && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ + && source ci/prepare_clean_env.sh gpu/cy3 \ + && python3 -m pip install mxnet==1.6.0 flaky numpy==1.18.4 \ + && pytest -m 'not (gpu or serial)' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \ + --wait | tee batch_job.log - - name: Unittest - shell: bash -l {0} + - name: Wait for job and copy files from AWS s3 + if: ${{ failure() || success() }} run: | - source ci/prepare_clean_env.sh cpu/py3 - pytest -m 'not (gpu or serial)' --durations=30 --cov=./ --cov-report=xml tests/unittest + head -100 batch_job.log | grep -oP -m 1 'jobId: \K(.*)' > jobid.log + echo "Job ID is" + cat jobid.log + cat jobid.log | xargs -i python ./tools/batch/wait-job.py --job-id {} + echo "Copy Codecov file" + cat jobid.log | xargs -i aws s3api wait object-exists --bucket gluon-nlp-dev --key batch/{}/coverage.xml + cat jobid.log | xargs -i aws s3 cp s3://gluon-nlp-dev/batch/{}/coverage.xml ./coverage.xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1.0.10 + run: | + curl -s https://codecov.io/bash -o codecov.sh + if [ "$EVENT_NAME" == "push" ]; then \ + bash codecov.sh -f ./coverage.xml -n -F -B v0.x; \ + else \ + bash codecov.sh -f ./coverage.xml -n -F -B v0.x -C ${{ github.event.pull_request.head.sha }} -P ${{ github.event.pull_request.number }}; \ + fi + env: + EVENT_NAME: ${{ github.event_name }} + + - name: Upload Cloud Watch Log Stream + if: ${{ failure() || success() }} + uses: actions/upload-artifact@v2 with: - env_vars: OS,PYTHON + name: Test_Logs + path: ./batch_job.log From 09369e032ff4dffb84b947f1e1e3995dcfccbfbe Mon Sep 17 00:00:00 2001 From: barry-jin Date: Mon, 1 Mar 2021 15:22:05 -0800 Subject: [PATCH 10/14] update unittests.yml --- .github/workflows/unittests.yml | 119 +++++++++----------------------- 1 file changed, 34 insertions(+), 85 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 5b842bb620..4ac0030132 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -1,106 +1,55 @@ -name: v0.x continuous build - gpu +name: v0.x continuous build -on: [push, pull_request_target] +on: [push, pull_request] defaults: run: shell: bash jobs: - unittest-gpu: - runs-on: ubuntu-latest + unittest: + runs-on: ${{ matrix.os }} strategy: fail-fast: false + matrix: + # TODO Add windows test by using "windows-latest" + os: [ubuntu-latest] + python-version: [ '3.6', '3.7', '3.8'] + steps: - name: Checkout repository uses: actions/checkout@v2 - - - name: Install Linux dependencies - run: sudo apt-get install libopenblas-dev - - - name: Setup python - uses: actions/setup-python@v2 + - name: Compilation cache + uses: actions/cache@v2 with: - python-version: 3.7 - architecture: x64 - - - name: Install Other Dependencies - run: | - python -m pip install --quiet --upgrade pip - python -m pip install --quiet boto3 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 + path: ~/.ccache + # We include the commit sha in the cache key, as new cache entries are + # only created if there is no existing entry for the key yet. + key: ${{ runner.os }}-ccache-${{ github.sha }} + # Restore any ccache cache entry, if none for + # ${{ runner.os }}-ccache-${{ github.sha }} exists + restore-keys: | + ${{ runner.os }}-ccache + - name: Setup Miniconda + uses: conda-incubator/setup-miniconda@v2.0.0 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Test project on AWS Batch(For push) - if: ${{ github.event_name == 'push' }} - run: | - echo "Start submitting job" - python ./ci/batch/submit-job.py --region us-east-1 \ - --job-type c5n.4x \ - --name GluonNLP-GPU-v0.x-Test-${{ github.ref }} \ - --source-ref ${{ github.ref }} \ - --work-dir . \ - --saved-output coverage.xml \ - --save-path coverage.xml \ - --remote https://github.com/${{ github.repository }} \ - --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - && chmod +x Miniconda3-latest-Linux-x86_64.sh \ - && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ - && source ci/prepare_clean_env.sh gpu/cy3 \ - && python3 -m pip install mxnet==1.6.0 flaky numpy==1.18.4 \ - && pytest -m 'not (gpu or serial)' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \ - --wait | tee batch_job.log + auto-update-conda: true + python-version: ${{ matrix.python-version }} - - name: Test project on AWS Batch(For pull request) - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} + - name: Sanity Check + shell: bash -l {0} run: | - echo "Start submitting job" - python ./ci/batch/submit-job.py --region us-east-1 \ - --job-type c5n.4x \ - --name GluonNLP-GPU-v0.x-Test-PR#${{ github.event.number }} \ - --source-ref ${{ github.event.pull_request.head.sha }} \ - --work-dir . \ - --saved-output coverage.xml \ - --save-path coverage.xml \ - --remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \ - --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - && chmod +x Miniconda3-latest-Linux-x86_64.sh \ - && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ - && source ci/prepare_clean_env.sh gpu/cy3 \ - && python3 -m pip install mxnet==1.6.0 flaky numpy==1.18.4 \ - && pytest -m 'not (gpu or serial)' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \ - --wait | tee batch_job.log + source ci/prepare_clean_env.sh cpu/py3 + make lintdir=src/gluonnlp lint - - name: Wait for job and copy files from AWS s3 - if: ${{ failure() || success() }} + - name: Unittest + shell: bash -l {0} run: | - head -100 batch_job.log | grep -oP -m 1 'jobId: \K(.*)' > jobid.log - echo "Job ID is" - cat jobid.log - cat jobid.log | xargs -i python ./tools/batch/wait-job.py --job-id {} - echo "Copy Codecov file" - cat jobid.log | xargs -i aws s3api wait object-exists --bucket gluon-nlp-dev --key batch/{}/coverage.xml - cat jobid.log | xargs -i aws s3 cp s3://gluon-nlp-dev/batch/{}/coverage.xml ./coverage.xml + source ci/prepare_clean_env.sh cpu/py3 + python -m pip install pytest-forked + pytest --forked -m 'not (gpu or serial)' --durations=30 --cov=./ --cov-report=xml tests/unittest - name: Upload coverage to Codecov - run: | - curl -s https://codecov.io/bash -o codecov.sh - if [ "$EVENT_NAME" == "push" ]; then \ - bash codecov.sh -f ./coverage.xml -n -F -B v0.x; \ - else \ - bash codecov.sh -f ./coverage.xml -n -F -B v0.x -C ${{ github.event.pull_request.head.sha }} -P ${{ github.event.pull_request.number }}; \ - fi - env: - EVENT_NAME: ${{ github.event_name }} - - - name: Upload Cloud Watch Log Stream - if: ${{ failure() || success() }} - uses: actions/upload-artifact@v2 + uses: codecov/codecov-action@v1.0.10 with: - name: Test_Logs - path: ./batch_job.log + env_vars: OS,PYTHON \ No newline at end of file From 1b52d1edc7617c2cd7d1482609d07b30e2c5f560 Mon Sep 17 00:00:00 2001 From: barry-jin Date: Tue, 2 Mar 2021 09:12:03 -0800 Subject: [PATCH 11/14] use batch --- .github/workflows/unittests.yml | 117 +++++++++++++++++++++++--------- 1 file changed, 84 insertions(+), 33 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 4ac0030132..df6236cef0 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -1,55 +1,106 @@ name: v0.x continuous build -on: [push, pull_request] +on: [push, pull_request_target] defaults: run: shell: bash jobs: - unittest: - runs-on: ${{ matrix.os }} + unittest-gpu: + runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - # TODO Add windows test by using "windows-latest" - os: [ubuntu-latest] - python-version: [ '3.6', '3.7', '3.8'] - steps: - name: Checkout repository uses: actions/checkout@v2 - - name: Compilation cache - uses: actions/cache@v2 + + - name: Install Linux dependencies + run: sudo apt-get install libopenblas-dev + + - name: Setup python + uses: actions/setup-python@v2 with: - path: ~/.ccache - # We include the commit sha in the cache key, as new cache entries are - # only created if there is no existing entry for the key yet. - key: ${{ runner.os }}-ccache-${{ github.sha }} - # Restore any ccache cache entry, if none for - # ${{ runner.os }}-ccache-${{ github.sha }} exists - restore-keys: | - ${{ runner.os }}-ccache - - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2.0.0 + python-version: 3.7 + architecture: x64 + + - name: Install Other Dependencies + run: | + python -m pip install --quiet --upgrade pip + python -m pip install --quiet boto3 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 with: - auto-update-conda: true - python-version: ${{ matrix.python-version }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Test project on AWS Batch(For push) + if: ${{ github.event_name == 'push' }} + run: | + echo "Start submitting job" + python ./ci/batch/submit-job.py --region us-east-1 \ + --job-type c5n.4x \ + --name GluonNLP-GPU-v0.x-Test-${{ github.ref }} \ + --source-ref ${{ github.ref }} \ + --work-dir . \ + --saved-output coverage.xml \ + --save-path coverage.xml \ + --remote https://github.com/${{ github.repository }} \ + --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && chmod +x Miniconda3-latest-Linux-x86_64.sh \ + && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ + && source ci/prepare_clean_env.sh gpu/cy3 \ + && python3 -m pip install mxnet==1.6.0 flaky numpy==1.18.4 \ + && pytest -m 'not (gpu or serial)' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \ + --wait | tee batch_job.log - - name: Sanity Check - shell: bash -l {0} + - name: Test project on AWS Batch(For pull request) + if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} run: | - source ci/prepare_clean_env.sh cpu/py3 - make lintdir=src/gluonnlp lint + echo "Start submitting job" + python ./ci/batch/submit-job.py --region us-east-1 \ + --job-type c5n.4x \ + --name GluonNLP-GPU-v0.x-Test-PR#${{ github.event.number }} \ + --source-ref ${{ github.event.pull_request.head.sha }} \ + --work-dir . \ + --saved-output coverage.xml \ + --save-path coverage.xml \ + --remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \ + --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && chmod +x Miniconda3-latest-Linux-x86_64.sh \ + && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ + && source ci/prepare_clean_env.sh gpu/cy3 \ + && python3 -m pip install mxnet==1.6.0 flaky numpy==1.18.4 \ + && pytest -m 'not (gpu or serial)' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \ + --wait | tee batch_job.log - - name: Unittest - shell: bash -l {0} + - name: Wait for job and copy files from AWS s3 + if: ${{ failure() || success() }} run: | - source ci/prepare_clean_env.sh cpu/py3 - python -m pip install pytest-forked - pytest --forked -m 'not (gpu or serial)' --durations=30 --cov=./ --cov-report=xml tests/unittest + head -100 batch_job.log | grep -oP -m 1 'jobId: \K(.*)' > jobid.log + echo "Job ID is" + cat jobid.log + cat jobid.log | xargs -i python ./tools/batch/wait-job.py --job-id {} + echo "Copy Codecov file" + cat jobid.log | xargs -i aws s3api wait object-exists --bucket gluon-nlp-dev --key batch/{}/coverage.xml + cat jobid.log | xargs -i aws s3 cp s3://gluon-nlp-dev/batch/{}/coverage.xml ./coverage.xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1.0.10 + run: | + curl -s https://codecov.io/bash -o codecov.sh + if [ "$EVENT_NAME" == "push" ]; then \ + bash codecov.sh -f ./coverage.xml -n -F -B v0.x; \ + else \ + bash codecov.sh -f ./coverage.xml -n -F -B v0.x -C ${{ github.event.pull_request.head.sha }} -P ${{ github.event.pull_request.number }}; \ + fi + env: + EVENT_NAME: ${{ github.event_name }} + + - name: Upload Cloud Watch Log Stream + if: ${{ failure() || success() }} + uses: actions/upload-artifact@v2 with: - env_vars: OS,PYTHON \ No newline at end of file + name: Test_Logs + path: ./batch_job.log From af98630e408ad5ed5b4cd65ce00efe8a9bebba4a Mon Sep 17 00:00:00 2001 From: barry-jin Date: Tue, 2 Mar 2021 14:21:56 -0800 Subject: [PATCH 12/14] update cpu pytest with --verbose --- .github/workflows/unittests.yml | 118 +++++++++----------------------- 1 file changed, 34 insertions(+), 84 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index df6236cef0..d5287f7437 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -1,106 +1,56 @@ name: v0.x continuous build -on: [push, pull_request_target] +on: [push, pull_request] defaults: run: shell: bash jobs: - unittest-gpu: - runs-on: ubuntu-latest + unittest: + runs-on: ${{ matrix.os }} strategy: fail-fast: false + matrix: + # TODO Add windows test by using "windows-latest" + os: [ubuntu-latest] + python-version: [ '3.6', '3.7', '3.8'] + steps: - name: Checkout repository uses: actions/checkout@v2 - - - name: Install Linux dependencies - run: sudo apt-get install libopenblas-dev - - - name: Setup python - uses: actions/setup-python@v2 + - name: Compilation cache + uses: actions/cache@v2 with: - python-version: 3.7 - architecture: x64 - - - name: Install Other Dependencies - run: | - python -m pip install --quiet --upgrade pip - python -m pip install --quiet boto3 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 + path: ~/.ccache + # We include the commit sha in the cache key, as new cache entries are + # only created if there is no existing entry for the key yet. + key: ${{ runner.os }}-ccache-${{ github.sha }} + # Restore any ccache cache entry, if none for + # ${{ runner.os }}-ccache-${{ github.sha }} exists + restore-keys: | + ${{ runner.os }}-ccache + + - name: Setup Miniconda + uses: conda-incubator/setup-miniconda@v2.0.0 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Test project on AWS Batch(For push) - if: ${{ github.event_name == 'push' }} - run: | - echo "Start submitting job" - python ./ci/batch/submit-job.py --region us-east-1 \ - --job-type c5n.4x \ - --name GluonNLP-GPU-v0.x-Test-${{ github.ref }} \ - --source-ref ${{ github.ref }} \ - --work-dir . \ - --saved-output coverage.xml \ - --save-path coverage.xml \ - --remote https://github.com/${{ github.repository }} \ - --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - && chmod +x Miniconda3-latest-Linux-x86_64.sh \ - && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ - && source ci/prepare_clean_env.sh gpu/cy3 \ - && python3 -m pip install mxnet==1.6.0 flaky numpy==1.18.4 \ - && pytest -m 'not (gpu or serial)' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \ - --wait | tee batch_job.log + auto-update-conda: true + python-version: ${{ matrix.python-version }} - - name: Test project on AWS Batch(For pull request) - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} + - name: Sanity Check + shell: bash -l {0} run: | - echo "Start submitting job" - python ./ci/batch/submit-job.py --region us-east-1 \ - --job-type c5n.4x \ - --name GluonNLP-GPU-v0.x-Test-PR#${{ github.event.number }} \ - --source-ref ${{ github.event.pull_request.head.sha }} \ - --work-dir . \ - --saved-output coverage.xml \ - --save-path coverage.xml \ - --remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \ - --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - && chmod +x Miniconda3-latest-Linux-x86_64.sh \ - && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ - && source ci/prepare_clean_env.sh gpu/cy3 \ - && python3 -m pip install mxnet==1.6.0 flaky numpy==1.18.4 \ - && pytest -m 'not (gpu or serial)' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \ - --wait | tee batch_job.log + source ci/prepare_clean_env.sh cpu/py3 + make lintdir=src/gluonnlp lint - - name: Wait for job and copy files from AWS s3 - if: ${{ failure() || success() }} + - name: Unittest + shell: bash -l {0} run: | - head -100 batch_job.log | grep -oP -m 1 'jobId: \K(.*)' > jobid.log - echo "Job ID is" - cat jobid.log - cat jobid.log | xargs -i python ./tools/batch/wait-job.py --job-id {} - echo "Copy Codecov file" - cat jobid.log | xargs -i aws s3api wait object-exists --bucket gluon-nlp-dev --key batch/{}/coverage.xml - cat jobid.log | xargs -i aws s3 cp s3://gluon-nlp-dev/batch/{}/coverage.xml ./coverage.xml + source ci/prepare_clean_env.sh cpu/py3 + python -m pip install pytest-forked + pytest --forked --verbose -m 'not (gpu or serial)' --durations=30 --cov=./ --cov-report=xml tests/unittest - name: Upload coverage to Codecov - run: | - curl -s https://codecov.io/bash -o codecov.sh - if [ "$EVENT_NAME" == "push" ]; then \ - bash codecov.sh -f ./coverage.xml -n -F -B v0.x; \ - else \ - bash codecov.sh -f ./coverage.xml -n -F -B v0.x -C ${{ github.event.pull_request.head.sha }} -P ${{ github.event.pull_request.number }}; \ - fi - env: - EVENT_NAME: ${{ github.event_name }} - - - name: Upload Cloud Watch Log Stream - if: ${{ failure() || success() }} - uses: actions/upload-artifact@v2 + uses: codecov/codecov-action@v1.0.10 with: - name: Test_Logs - path: ./batch_job.log + env_vars: OS,PYTHON From a33cc9f78fceeef6de2562cec99d7baa5b27be0c Mon Sep 17 00:00:00 2001 From: barry-jin Date: Thu, 4 Mar 2021 13:49:24 -0800 Subject: [PATCH 13/14] use batch --- .github/workflows/unittests.yml | 117 ++++++++++++++++++++++---------- 1 file changed, 83 insertions(+), 34 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index d5287f7437..ee306c6965 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -1,56 +1,105 @@ name: v0.x continuous build -on: [push, pull_request] +on: [push, pull_request_target] defaults: run: shell: bash jobs: - unittest: - runs-on: ${{ matrix.os }} + unittest-gpu: + runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - # TODO Add windows test by using "windows-latest" - os: [ubuntu-latest] - python-version: [ '3.6', '3.7', '3.8'] - steps: - name: Checkout repository uses: actions/checkout@v2 - - name: Compilation cache - uses: actions/cache@v2 + + - name: Install Linux dependencies + run: sudo apt-get install libopenblas-dev + + - name: Setup python + uses: actions/setup-python@v2 with: - path: ~/.ccache - # We include the commit sha in the cache key, as new cache entries are - # only created if there is no existing entry for the key yet. - key: ${{ runner.os }}-ccache-${{ github.sha }} - # Restore any ccache cache entry, if none for - # ${{ runner.os }}-ccache-${{ github.sha }} exists - restore-keys: | - ${{ runner.os }}-ccache - - - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2.0.0 + python-version: 3.7 + architecture: x64 + + - name: Install Other Dependencies + run: | + python -m pip install --quiet --upgrade pip + python -m pip install --quiet boto3 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 with: - auto-update-conda: true - python-version: ${{ matrix.python-version }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 - - name: Sanity Check - shell: bash -l {0} + - name: Test project on AWS Batch(For push) + if: ${{ github.event_name == 'push' }} run: | - source ci/prepare_clean_env.sh cpu/py3 - make lintdir=src/gluonnlp lint + echo "Start submitting job" + python ./ci/batch/submit-job.py --region us-east-1 \ + --job-type c5n.4x \ + --name GluonNLP-GPU-v0.x-Test-${{ github.ref }} \ + --source-ref ${{ github.ref }} \ + --work-dir . \ + --saved-output coverage.xml \ + --save-path coverage.xml \ + --remote https://github.com/${{ github.repository }} \ + --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && chmod +x Miniconda3-latest-Linux-x86_64.sh \ + && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ + && source ci/prepare_clean_env.sh gpu/cy3 \ + && python3 -m pip install mxnet==1.6.0 flaky numpy==1.18.4 \ + && pytest -m 'not (gpu or serial)' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \ + --wait | tee batch_job.log - - name: Unittest - shell: bash -l {0} + - name: Test project on AWS Batch(For pull request) + if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} run: | - source ci/prepare_clean_env.sh cpu/py3 - python -m pip install pytest-forked - pytest --forked --verbose -m 'not (gpu or serial)' --durations=30 --cov=./ --cov-report=xml tests/unittest + echo "Start submitting job" + python ./ci/batch/submit-job.py --region us-east-1 \ + --job-type c5n.4x \ + --name GluonNLP-GPU-v0.x-Test-PR#${{ github.event.number }} \ + --source-ref ${{ github.event.pull_request.head.sha }} \ + --work-dir . \ + --saved-output coverage.xml \ + --save-path coverage.xml \ + --remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \ + --command "wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && chmod +x Miniconda3-latest-Linux-x86_64.sh \ + && bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda \ + && source ci/prepare_clean_env.sh gpu/cy3 \ + && python3 -m pip install mxnet==1.6.0 flaky numpy==1.18.4 \ + && pytest -m 'not (gpu or serial)' --durations=30 --cov=. --cov-config=./.coveragerc --cov-report=xml tests/unittest" \ + --wait | tee batch_job.log + + - name: Wait for job and copy files from AWS s3 + if: ${{ failure() || success() }} + run: | + head -100 batch_job.log | grep -oP -m 1 'jobId: \K(.*)' > jobid.log + echo "Job ID is" + cat jobid.log + cat jobid.log | xargs -i python ./tools/batch/wait-job.py --job-id {} + echo "Copy Codecov file" + cat jobid.log | xargs -i aws s3api wait object-exists --bucket gluon-nlp-dev --key batch/{}/coverage.xml + cat jobid.log | xargs -i aws s3 cp s3://gluon-nlp-dev/batch/{}/coverage.xml ./coverage.xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1.0.10 + run: | + curl -s https://codecov.io/bash -o codecov.sh + if [ "$EVENT_NAME" == "push" ]; then \ + bash codecov.sh -f ./coverage.xml -n -F -B v0.x; \ + else \ + bash codecov.sh -f ./coverage.xml -n -F -B v0.x -C ${{ github.event.pull_request.head.sha }} -P ${{ github.event.pull_request.number }}; \ + fi + env: + EVENT_NAME: ${{ github.event_name }} + + - name: Upload Cloud Watch Log Stream + if: ${{ failure() || success() }} + uses: actions/upload-artifact@v2 with: - env_vars: OS,PYTHON + name: Test_Logs + path: ./batch_job.log From 15e4e20e6981143692668d612f68758fe388dce0 Mon Sep 17 00:00:00 2001 From: barry-jin Date: Fri, 5 Mar 2021 11:12:03 -0800 Subject: [PATCH 14/14] remove jenkins --- ci/codecov.sh | 1550 ----------------- .../Jenkinsfile_py3-master_cpu_unittest | 69 - ci/jenkins/Jenkinsfile_py3-master_gpu_doc | 168 -- .../Jenkinsfile_py3-master_gpu_integration | 53 - .../Jenkinsfile_py3-master_gpu_unittest | 61 - ci/jenkins/Jenkinsfile_py3_cpu_unittest | 69 - ci/jenkins/Jenkinsfile_py3_gpu_integration | 53 - ci/jenkins/Jenkinsfile_py3_gpu_unittest | 61 - ci/jenkins/build_steps.groovy | 127 -- ci/jenkins/utils.groovy | 214 --- ci/upload_doc.sh | 6 - docs/website/contribute.rst | 10 +- 12 files changed, 5 insertions(+), 2436 deletions(-) delete mode 100755 ci/codecov.sh delete mode 100644 ci/jenkins/Jenkinsfile_py3-master_cpu_unittest delete mode 100644 ci/jenkins/Jenkinsfile_py3-master_gpu_doc delete mode 100644 ci/jenkins/Jenkinsfile_py3-master_gpu_integration delete mode 100644 ci/jenkins/Jenkinsfile_py3-master_gpu_unittest delete mode 100644 ci/jenkins/Jenkinsfile_py3_cpu_unittest delete mode 100644 ci/jenkins/Jenkinsfile_py3_gpu_integration delete mode 100644 ci/jenkins/Jenkinsfile_py3_gpu_unittest delete mode 100644 ci/jenkins/build_steps.groovy delete mode 100644 ci/jenkins/utils.groovy delete mode 100755 ci/upload_doc.sh diff --git a/ci/codecov.sh b/ci/codecov.sh deleted file mode 100755 index 1ef332b1b3..0000000000 --- a/ci/codecov.sh +++ /dev/null @@ -1,1550 +0,0 @@ -#!/usr/bin/env bash - -# Apache License Version 2.0, January 2004 -# https://github.com/codecov/codecov-bash/blob/master/LICENSE - - -set -e +o pipefail - -VERSION="0b37652" - -url="https://codecov.io" -env="$CODECOV_ENV" -service="" -token="" -search_in="" -flags="" -exit_with=0 -curlargs="" -curlawsargs="" -dump="0" -clean="0" -curl_s="-s" -name="$CODECOV_NAME" -include_cov="" -exclude_cov="" -ddp="$(echo ~)/Library/Developer/Xcode/DerivedData" -xp="" -files="" -cacert="$CODECOV_CA_BUNDLE" -gcov_ignore="-not -path './bower_components/**' -not -path './node_modules/**' -not -path './vendor/**'" -gcov_include="" - -ft_gcov="1" -ft_coveragepy="1" -ft_fix="1" -ft_search="1" -ft_s3="1" -ft_network="1" -ft_xcodellvm="1" -ft_xcodeplist="0" - -_git_root=$(git rev-parse --show-toplevel 2>/dev/null || hg root 2>/dev/null || echo $PWD) -git_root="$_git_root" -codecov_yml="" -remote_addr="" -if [ "$git_root" = "$PWD" ]; -then - git_root="." -fi - -url_o="" -pr_o="" -build_o="" -commit_o="" -search_in_o="" -tag_o="" -branch_o="" -slug_o="" -prefix_o="" - -commit="$VCS_COMMIT_ID" -branch="$VCS_BRANCH_NAME" -pr="$VCS_PULL_REQUEST" -slug="$VCS_SLUG" -tag="$VCS_TAG" -build_url="$CI_BUILD_URL" -build="$CI_BUILD_ID" -job="$CI_JOB_ID" - -beta_xcode_partials="" - -proj_root="$git_root" -gcov_exe="gcov" -gcov_arg="" - -b="\033[0;36m" -g="\033[0;32m" -r="\033[0;31m" -e="\033[0;90m" -x="\033[0m" - -show_help() { -cat << EOF - - Codecov Bash $VERSION - - Global report uploading tool for Codecov - Documentation at https://docs.codecov.io/docs - Contribute at https://github.com/codecov/codecov-bash - - - -h Display this help and exit - -f FILE Target file(s) to upload - - -f "path/to/file" only upload this file - skips searching unless provided patterns below - - -f '!*.bar' ignore all files at pattern *.bar - -f '*.foo' include all files at pattern *.foo - Must use single quotes. - This is non-exclusive, use -s "*.foo" to match specific paths. - - -s DIR Directory to search for coverage reports. - Already searches project root and artifact folders. - -t TOKEN Set the private repository token - (option) set environment variable CODECOV_TOKEN=:uuid - - -t @/path/to/token_file - -t uuid - - -n NAME Custom defined name of the upload. Visible in Codecov UI - - -e ENV Specify environment variables to be included with this build - Also accepting environment variables: CODECOV_ENV=VAR,VAR2 - - -e VAR,VAR2 - - -X feature Toggle functionalities - - -X gcov Disable gcov - -X coveragepy Disable python coverage - -X fix Disable report fixing - -X search Disable searching for reports - -X xcode Disable xcode processing - -X network Disable uploading the file network - - -R root dir Used when not in git/hg project to identify project root directory - -y conf file Used to specify the location of the .codecov.yml config file - -F flag Flag the upload to group coverage metrics - - -F unittests This upload is only unittests - -F integration This upload is only integration tests - -F ui,chrome This upload is Chrome - UI tests - - -c Move discovered coverage reports to the trash - -Z Exit with 1 if not successful. Default will Exit with 0 - - -- xcode -- - -D Custom Derived Data Path for Coverage.profdata and gcov processing - Default '~/Library/Developer/Xcode/DerivedData' - -J Specify packages to build coverage. - This can significantly reduces time to build coverage reports. - - -J 'MyAppName' Will match "MyAppName" and "MyAppNameTests" - -J '^ExampleApp$' Will match only "ExampleApp" not "ExampleAppTests" - - -- gcov -- - -g GLOB Paths to ignore during gcov gathering - -G GLOB Paths to include during gcov gathering - -p dir Project root directory - Also used when preparing gcov - -k prefix Prefix filepaths to help resolve path fixing: https://github.com/codecov/support/issues/472 - -x gcovexe gcov executable to run. Defaults to 'gcov' - -a gcovargs extra arguments to pass to gcov - - -- Override CI Environment Variables -- - These variables are automatically detected by popular CI providers - - -B branch Specify the branch name - -C sha Specify the commit sha - -P pr Specify the pull request number - -b build Specify the build number - -T tag Specify the git tag - - -- Enterprise -- - -u URL Set the target url for Enterprise customers - Not required when retrieving the bash uploader from your CCE - (option) Set environment variable CODECOV_URL=https://my-hosted-codecov.com - -r SLUG owner/repo slug used instead of the private repo token in Enterprise - (option) set environment variable CODECOV_SLUG=:owner/:repo - (option) set in your codecov.yml "codecov.slug" - -S PATH File path to your cacert.pem file used to verify ssl with Codecov Enterprise (optional) - (option) Set environment variable: CODECOV_CA_BUNDLE="/path/to/ca.pem" - -U curlargs Extra curl arguments to communicate with Codecov. e.g., -U "--proxy http://http-proxy" - -A curlargs Extra curl arguments to communicate with AWS. - - -- Debugging -- - -d Don't upload, but dump upload file to stdout - -K Remove color from the output - -v Verbose mode - -EOF -} - - -say() { - echo -e "$1" -} - - -urlencode() { - echo "$1" | curl -Gso /dev/null -w %{url_effective} --data-urlencode @- "" | cut -c 3- | sed -e 's/%0A//' -} - - -swiftcov() { - _dir=$(dirname "$1" | sed 's/\(Build\).*/\1/g') - for _type in app framework xctest - do - find "$_dir" -name "*.$_type" | while read f - do - _proj=${f##*/} - _proj=${_proj%."$_type"} - if [ "$2" = "" ] || [ "$(echo "$_proj" | grep -i "$2")" != "" ]; - then - say " $g+$x Building reports for $_proj $_type" - dest=$([ -f "$f/$_proj" ] && echo "$f/$_proj" || echo "$f/Contents/MacOS/$_proj") - _proj_name=$(echo "$_proj" | sed -e 's/[[:space:]]//g') - xcrun llvm-cov show $beta_xcode_partials -instr-profile "$1" "$dest" > "$_proj_name.$_type.coverage.txt" \ - || say " ${r}x>${x} llvm-cov failed to produce results for $dest" - fi - done - done -} - - -# Credits to: https://gist.github.com/pkuczynski/8665367 -parse_yaml() { - local prefix=$2 - local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034') - sed -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \ - -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $1 | - awk -F$fs '{ - indent = length($1)/2; - vname[indent] = $2; - for (i in vname) {if (i > indent) {delete vname[i]}} - if (length($3) > 0) { - vn=""; if (indent > 0) {vn=(vn)(vname[0])("_")} - printf("%s%s%s=\"%s\"\n", "'$prefix'",vn, $2, $3); - } - }' -} - - -if [ $# != 0 ]; -then - while getopts "a:A:b:B:cC:dD:e:f:F:g:G:hJ:k:Kn:p:P:r:R:y:s:S:t:T:u:U:vx:X:Z" o - do - case "$o" in - "a") - gcov_arg=$OPTARG - ;; - "A") - curlawsargs="$OPTARG" - ;; - "b") - build_o="$OPTARG" - ;; - "B") - branch_o="$OPTARG" - ;; - "c") - clean="1" - ;; - "C") - commit_o="$OPTARG" - ;; - "d") - dump="1" - ;; - "D") - ddp="$OPTARG" - ;; - "e") - env="$env,$OPTARG" - ;; - "f") - if [ "${OPTARG::1}" = "!" ]; - then - exclude_cov="$exclude_cov -not -path '${OPTARG:1}'" - - elif [[ "$OPTARG" = *"*"* ]]; - then - include_cov="$include_cov -or -name '$OPTARG'" - - else - ft_search=0 - if [ "$files" = "" ]; - then - files="$OPTARG" - else - files="$files -$OPTARG" - fi - fi - ;; - "F") - if [ "$flags" = "" ]; - then - flags="$OPTARG" - else - flags="$flags,$OPTARG" - fi - ;; - "g") - gcov_ignore="$gcov_ignore -not -path '$OPTARG'" - ;; - "G") - gcov_include="$gcov_include -path '$OPTARG'" - ;; - "h") - show_help - exit 0; - ;; - "J") - ft_xcodellvm="1" - ft_xcodeplist="0" - if [ "$xp" = "" ]; - then - xp="$OPTARG" - else - xp="$xp\|$OPTARG" - fi - ;; - "k") - prefix_o=$(echo "$OPTARG" | sed -e 's:^/*::' -e 's:/*$::') - ;; - "K") - b="" - g="" - r="" - e="" - x="" - ;; - "n") - name="$OPTARG" - ;; - "p") - proj_root="$OPTARG" - ;; - "P") - pr_o="$OPTARG" - ;; - "r") - slug_o="$OPTARG" - ;; - "R") - git_root="$OPTARG" - ;; - "s") - if [ "$search_in_o" = "" ]; - then - search_in_o="$OPTARG" - else - search_in_o="$search_in_o $OPTARG" - fi - ;; - "S") - cacert="--cacert \"$OPTARG\"" - ;; - "t") - if [ "${OPTARG::1}" = "@" ]; - then - token=$(cat "${OPTARG:1}" | tr -d ' \n') - else - token="$OPTARG" - fi - ;; - "T") - tag_o="$OPTARG" - ;; - "u") - url_o=$(echo "$OPTARG" | sed -e 's/\/$//') - ;; - "U") - curlargs="$OPTARG" - ;; - "v") - set -x - curl_s="" - ;; - "x") - gcov_exe=$OPTARG - ;; - "X") - if [ "$OPTARG" = "gcov" ]; - then - ft_gcov="0" - elif [ "$OPTARG" = "coveragepy" ] || [ "$OPTARG" = "py" ]; - then - ft_coveragepy="0" - elif [ "$OPTARG" = "xcodellvm" ]; - then - ft_xcodellvm="1" - ft_xcodeplist="0" - elif [ "$OPTARG" = "fix" ] || [ "$OPTARG" = "fixes" ]; - then - ft_fix="0" - elif [ "$OPTARG" = "xcode" ]; - then - ft_xcodellvm="0" - ft_xcodeplist="0" - elif [ "$OPTARG" = "search" ]; - then - ft_search="0" - elif [ "$OPTARG" = "xcodepartials" ]; - then - beta_xcode_partials="-use-color" - elif [ "$OPTARG" = "network" ]; - then - ft_network="0" - elif [ "$OPTARG" = "s3" ]; - then - ft_s3="0" - fi - ;; - "y") - codecov_yml="$OPTARG" - ;; - "Z") - exit_with=1 - ;; - esac - done -fi - -say " - _____ _ - / ____| | | -| | ___ __| | ___ ___ _____ __ -| | / _ \\ / _\` |/ _ \\/ __/ _ \\ \\ / / -| |___| (_) | (_| | __/ (_| (_) \\ V / - \\_____\\___/ \\__,_|\\___|\\___\\___/ \\_/ - Bash-$VERSION - -" - -search_in="$proj_root" - -if [ "$JENKINS_URL" != "" ]; -then - say "$e==>$x Jenkins CI detected." - # https://wiki.jenkins-ci.org/display/JENKINS/Building+a+software+project - # https://wiki.jenkins-ci.org/display/JENKINS/GitHub+pull+request+builder+plugin#GitHubpullrequestbuilderplugin-EnvironmentVariables - service="jenkins" - - if [ "$ghprbSourceBranch" != "" ]; - then - branch="$ghprbSourceBranch" - elif [ "$GIT_BRANCH" != "" ]; - then - branch="$GIT_BRANCH" - elif [ "$BRANCH_NAME" != "" ]; - then - branch="$BRANCH_NAME" - fi - - if [ "$ghprbActualCommit" != "" ]; - then - commit="$ghprbActualCommit" - elif [ "$GIT_COMMIT" != "" ]; - then - commit="$GIT_COMMIT" - fi - - if [ "$ghprbPullId" != "" ]; - then - pr="$ghprbPullId" - elif [ "$CHANGE_ID" != "" ]; - then - pr="$CHANGE_ID" - fi - - build="$BUILD_NUMBER" - build_url=$(urlencode "$BUILD_URL") - -elif [ "$CI" = "true" ] && [ "$TRAVIS" = "true" ] && [ "$SHIPPABLE" != "true" ]; -then - say "$e==>$x Travis CI detected." - # https://docs.travis-ci.com/user/environment-variables/ - service="travis" - commit="${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT}" - build="$TRAVIS_JOB_NUMBER" - pr="$TRAVIS_PULL_REQUEST" - job="$TRAVIS_JOB_ID" - slug="$TRAVIS_REPO_SLUG" - env="$env,TRAVIS_OS_NAME" - tag="$TRAVIS_TAG" - if [ "$TRAVIS_BRANCH" != "$TRAVIS_TAG" ]; - then - branch="$TRAVIS_BRANCH" - fi - - language=$(printenv | grep "TRAVIS_.*_VERSION" | head -1) - if [ "$language" != "" ]; - then - env="$env,${language%=*}" - fi - -elif [ "$DOCKER_REPO" != "" ]; -then - say "$e==>$x Docker detected." - # https://docs.docker.com/docker-cloud/builds/advanced/ - service="docker" - branch="$SOURCE_BRANCH" - commit="$SOURCE_COMMIT" - slug="$DOCKER_REPO" - tag="$CACHE_TAG" - env="$env,IMAGE_NAME" - -elif [ "$CI" = "true" ] && [ "$CI_NAME" = "codeship" ]; -then - say "$e==>$x Codeship CI detected." - # https://www.codeship.io/documentation/continuous-integration/set-environment-variables/ - service="codeship" - branch="$CI_BRANCH" - build="$CI_BUILD_NUMBER" - build_url=$(urlencode "$CI_BUILD_URL") - commit="$CI_COMMIT_ID" - -elif [ ! -z "$CF_BUILD_URL" ] && [ ! -z "$CF_BUILD_ID" ]; -then - say "$e==>$x Codefresh CI detected." - # https://docs.codefresh.io/v1.0/docs/variables - service="codefresh" - branch="$CF_BRANCH" - build="$CF_BUILD_ID" - build_url=$(urlencode "$CF_BUILD_URL") - commit="$CF_REVISION" - -elif [ "$TEAMCITY_VERSION" != "" ]; -then - say "$e==>$x TeamCity CI detected." - # https://confluence.jetbrains.com/display/TCD8/Predefined+Build+Parameters - # https://confluence.jetbrains.com/plugins/servlet/mobile#content/view/74847298 - if [ "$TEAMCITY_BUILD_BRANCH" = '' ]; - then - echo " Teamcity does not automatically make build parameters available as environment variables." - echo " Add the following environment parameters to the build configuration" - echo " env.TEAMCITY_BUILD_BRANCH = %teamcity.build.branch%" - echo " env.TEAMCITY_BUILD_ID = %teamcity.build.id%" - echo " env.TEAMCITY_BUILD_URL = %teamcity.serverUrl%/viewLog.html?buildId=%teamcity.build.id%" - echo " env.TEAMCITY_BUILD_COMMIT = %system.build.vcs.number%" - echo " env.TEAMCITY_BUILD_REPOSITORY = %vcsroot..url%" - fi - service="teamcity" - branch="$TEAMCITY_BUILD_BRANCH" - build="$TEAMCITY_BUILD_ID" - build_url=$(urlencode "$TEAMCITY_BUILD_URL") - if [ "$TEAMCITY_BUILD_COMMIT" != "" ]; - then - commit="$TEAMCITY_BUILD_COMMIT" - else - commit="$BUILD_VCS_NUMBER" - fi - remote_addr="$TEAMCITY_BUILD_REPOSITORY" - -elif [ "$CI" = "true" ] && [ "$CIRCLECI" = "true" ]; -then - say "$e==>$x Circle CI detected." - # https://circleci.com/docs/environment-variables - service="circleci" - branch="$CIRCLE_BRANCH" - build="$CIRCLE_BUILD_NUM" - job="$CIRCLE_NODE_INDEX" - if [ "$CIRCLE_PROJECT_REPONAME" != "" ]; - then - slug="$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME" - else - # git@github.com:owner/repo.git - slug="${CIRCLE_REPOSITORY_URL##*:}" - # owner/repo.git - slug="${slug%%.git}" - fi - pr="$CIRCLE_PR_NUMBER" - commit="$CIRCLE_SHA1" - search_in="$search_in $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS" - -elif [ "$BUDDYBUILD_BRANCH" != "" ]; -then - say "$e==>$x buddybuild detected" - # http://docs.buddybuild.com/v6/docs/custom-prebuild-and-postbuild-steps - service="buddybuild" - branch="$BUDDYBUILD_BRANCH" - build="$BUDDYBUILD_BUILD_NUMBER" - build_url="https://dashboard.buddybuild.com/public/apps/$BUDDYBUILD_APP_ID/build/$BUDDYBUILD_BUILD_ID" - # BUDDYBUILD_TRIGGERED_BY - if [ "$ddp" = "$(echo ~)/Library/Developer/Xcode/DerivedData" ]; - then - ddp="/private/tmp/sandbox/${BUDDYBUILD_APP_ID}/bbtest" - fi - -elif [ "${bamboo_planRepository_revision}" != "" ]; -then - say "$e==>$x Bamboo detected" - # https://confluence.atlassian.com/bamboo/bamboo-variables-289277087.html#Bamboovariables-Build-specificvariables - service="bamboo" - commit="${bamboo_planRepository_revision}" - branch="${bamboo_planRepository_branch}" - build="${bamboo_buildNumber}" - build_url="${bamboo_buildResultsUrl}" - remote_addr="${bamboo_planRepository_repositoryUrl}" - -elif [ "$CI" = "true" ] && [ "$BITRISE_IO" = "true" ]; -then - # http://devcenter.bitrise.io/faq/available-environment-variables/ - say "$e==>$x Bitrise CI detected." - service="bitrise" - branch="$BITRISE_GIT_BRANCH" - build="$BITRISE_BUILD_NUMBER" - build_url=$(urlencode "$BITRISE_BUILD_URL") - pr="$BITRISE_PULL_REQUEST" - if [ "$GIT_CLONE_COMMIT_HASH" != "" ]; - then - commit="$GIT_CLONE_COMMIT_HASH" - fi - -elif [ "$CI" = "true" ] && [ "$SEMAPHORE" = "true" ]; -then - say "$e==>$x Semaphore CI detected." - # https://semaphoreapp.com/docs/available-environment-variables.html - service="semaphore" - branch="$BRANCH_NAME" - build="$SEMAPHORE_BUILD_NUMBER" - job="$SEMAPHORE_CURRENT_THREAD" - pr="$PULL_REQUEST_NUMBER" - slug="$SEMAPHORE_REPO_SLUG" - commit="$REVISION" - env="$env,SEMAPHORE_TRIGGER_SOURCE" - -elif [ "$CI" = "true" ] && [ "$BUILDKITE" = "true" ]; -then - say "$e==>$x Buildkite CI detected." - # https://buildkite.com/docs/guides/environment-variables - service="buildkite" - branch="$BUILDKITE_BRANCH" - build="$BUILDKITE_BUILD_NUMBER" - job="$BUILDKITE_JOB_ID" - build_url=$(urlencode "$BUILDKITE_BUILD_URL") - slug="$BUILDKITE_PROJECT_SLUG" - commit="$BUILDKITE_COMMIT" - if [[ "$BUILDKITE_PULL_REQUEST" != "false" ]]; then - pr="$BUILDKITE_PULL_REQUEST" - fi - tag="$BUILDKITE_TAG" - -elif [ "$CI" = "drone" ] || [ "$DRONE" = "true" ]; -then - say "$e==>$x Drone CI detected." - # http://docs.drone.io/env.html - # drone commits are not full shas - service="drone.io" - branch="$DRONE_BRANCH" - build="$DRONE_BUILD_NUMBER" - build_url=$(urlencode "${DRONE_BUILD_LINK}") - pr="$DRONE_PULL_REQUEST" - job="$DRONE_JOB_NUMBER" - tag="$DRONE_TAG" - -elif [ "$HEROKU_TEST_RUN_BRANCH" != "" ]; -then - say "$e==>$x Heroku CI detected." - # https://devcenter.heroku.com/articles/heroku-ci#environment-variables - service="heroku" - branch="$HEROKU_TEST_RUN_BRANCH" - build="$HEROKU_TEST_RUN_ID" - -elif [ "$CI" = "True" ] && [ "$APPVEYOR" = "True" ]; -then - say "$e==>$x Appveyor CI detected." - # http://www.appveyor.com/docs/environment-variables - service="appveyor" - branch="$APPVEYOR_REPO_BRANCH" - build=$(urlencode "$APPVEYOR_JOB_ID") - pr="$APPVEYOR_PULL_REQUEST_NUMBER" - job="$APPVEYOR_ACCOUNT_NAME%2F$APPVEYOR_PROJECT_SLUG%2F$APPVEYOR_BUILD_VERSION" - slug="$APPVEYOR_REPO_NAME" - commit="$APPVEYOR_REPO_COMMIT" - -elif [ "$CI" = "true" ] && [ "$WERCKER_GIT_BRANCH" != "" ]; -then - say "$e==>$x Wercker CI detected." - # http://devcenter.wercker.com/articles/steps/variables.html - service="wercker" - branch="$WERCKER_GIT_BRANCH" - build="$WERCKER_MAIN_PIPELINE_STARTED" - slug="$WERCKER_GIT_OWNER/$WERCKER_GIT_REPOSITORY" - commit="$WERCKER_GIT_COMMIT" - -elif [ "$CI" = "true" ] && [ "$MAGNUM" = "true" ]; -then - say "$e==>$x Magnum CI detected." - # https://magnum-ci.com/docs/environment - service="magnum" - branch="$CI_BRANCH" - build="$CI_BUILD_NUMBER" - commit="$CI_COMMIT" - -elif [ "$SHIPPABLE" = "true" ]; -then - say "$e==>$x Shippable CI detected." - # http://docs.shippable.com/ci_configure/ - service="shippable" - branch=$([ "$HEAD_BRANCH" != "" ] && echo "$HEAD_BRANCH" || echo "$BRANCH") - build="$BUILD_NUMBER" - build_url=$(urlencode "$BUILD_URL") - pr="$PULL_REQUEST" - slug="$REPO_FULL_NAME" - commit="$COMMIT" - -elif [ "$TDDIUM" = "true" ]; -then - say "Solano CI detected." - # http://docs.solanolabs.com/Setup/tddium-set-environment-variables/ - service="solano" - commit="$TDDIUM_CURRENT_COMMIT" - branch="$TDDIUM_CURRENT_BRANCH" - build="$TDDIUM_TID" - pr="$TDDIUM_PR_ID" - -elif [ "$GREENHOUSE" = "true" ]; -then - say "$e==>$x Greenhouse CI detected." - # http://docs.greenhouseci.com/docs/environment-variables-files - service="greenhouse" - branch="$GREENHOUSE_BRANCH" - build="$GREENHOUSE_BUILD_NUMBER" - build_url=$(urlencode "$GREENHOUSE_BUILD_URL") - pr="$GREENHOUSE_PULL_REQUEST" - commit="$GREENHOUSE_COMMIT" - search_in="$search_in $GREENHOUSE_EXPORT_DIR" - -elif [ "$GITLAB_CI" != "" ]; -then - say "$e==>$x GitLab CI detected." - # http://doc.gitlab.com/ce/ci/variables/README.html - service="gitlab" - branch="${CI_BUILD_REF_NAME:-$CI_COMMIT_REF_NAME}" - build="${CI_BUILD_ID:-$CI_JOB_ID}" - remote_addr="${CI_BUILD_REPO:-$CI_REPOSITORY_URL}" - commit="${CI_BUILD_REF:-$CI_COMMIT_SHA}" - -else - say "${r}x>${x} No CI provider detected." - say " Testing inside Docker? ${b}http://docs.codecov.io/docs/testing-with-docker${x}" - say " Testing with Tox? ${b}https://docs.codecov.io/docs/python#section-testing-with-tox${x}" - -fi - -say " ${e}project root:${x} $git_root" - -# find branch, commit, repo from git command -if [ "$GIT_BRANCH" != "" ]; -then - branch="$GIT_BRANCH" - -elif [ "$branch" = "" ]; -then - branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || hg branch 2>/dev/null || echo "") - if [ "$branch" = "HEAD" ]; - then - branch="" - fi -fi - -if [ "$commit_o" = "" ]; -then - # merge commit -> actual commit - mc= - if [ -n "$pr" ] && [ "$pr" != false ]; - then - mc=$(git show --no-patch --format="%P" 2>/dev/null || echo "") - fi - if [[ "$mc" =~ ^[a-z0-9]{40}[[:space:]][a-z0-9]{40}$ ]]; - then - say " Fixing merge commit SHA" - commit=$(echo "$mc" | cut -d' ' -f2) - elif [ "$GIT_COMMIT" != "" ]; - then - commit="$GIT_COMMIT" - elif [ "$commit" = "" ]; - then - commit=$(git log -1 --format="%H" 2>/dev/null || hg id -i --debug 2>/dev/null | tr -d '+' || echo "") - fi -else - commit="$commit_o" -fi - -if [ "$CODECOV_TOKEN" != "" ] && [ "$token" = "" ]; -then - say "${e}-->${x} token set from env" - token="$CODECOV_TOKEN" -fi - -if [ "$CODECOV_URL" != "" ] && [ "$url_o" = "" ]; -then - say "${e}-->${x} url set from env" - url_o=$(echo "$CODECOV_URL" | sed -e 's/\/$//') -fi - -if [ "$CODECOV_SLUG" != "" ]; -then - say "${e}-->${x} slug set from env" - slug_o="$CODECOV_SLUG" - -elif [ "$slug" = "" ]; -then - if [ "$remote_addr" = "" ]; - then - remote_addr=$(git config --get remote.origin.url || hg paths default || echo '') - fi - if [ "$remote_addr" != "" ]; - then - if echo "$remote_addr" | grep -q "//"; then - # https - slug=$(echo "$remote_addr" | cut -d / -f 4,5 | sed -e 's/\.git$//') - else - # ssh - slug=$(echo "$remote_addr" | cut -d : -f 2 | sed -e 's/\.git$//') - fi - fi - if [ "$slug" = "/" ]; - then - slug="" - fi -fi - -yaml=$(test -n "$codecov_yml" && echo "$codecov_yml" \ - || cd "$git_root" && \ - git ls-files "*codecov.yml" "*codecov.yaml" 2>/dev/null \ - || hg locate "*codecov.yml" "*codecov.yaml" 2>/dev/null \ - || cd $proj_root && find . -type f -name '*codecov.y*ml' -depth 1 2>/dev/null \ - || echo '') -yaml=$(echo "$yaml" | head -1) - -if [ "$yaml" != "" ]; -then - say " ${e}Yaml found at:${x} $yaml" - config=$(parse_yaml "$git_root/$yaml" || echo '') - - # TODO validate the yaml here - - if [ "$(echo "$config" | grep 'codecov_token="')" != "" ] && [ "$token" = "" ]; - then - say "${e}-->${x} token set from yaml" - token="$(echo "$config" | grep 'codecov_token="' | sed -e 's/codecov_token="//' | sed -e 's/"\.*//')" - fi - - if [ "$(echo "$config" | grep 'codecov_url="')" != "" ] && [ "$url_o" = "" ]; - then - say "${e}-->${x} url set from yaml" - url_o="$(echo "$config" | grep 'codecov_url="' | sed -e 's/codecov_url="//' | sed -e 's/"\.*//')" - fi - - if [ "$(echo "$config" | grep 'codecov_slug="')" != "" ] && [ "$slug_o" = "" ]; - then - say "${e}-->${x} slug set from yaml" - slug_o="$(echo "$config" | grep 'codecov_slug="' | sed -e 's/codecov_slug="//' | sed -e 's/"\.*//')" - fi -else - say " ${g}Yaml not found, that's ok! Learn more at${x} ${b}http://docs.codecov.io/docs/codecov-yaml${x}" - -fi - -if [ "$branch_o" != "" ]; -then - branch=$(urlencode "$branch_o") -else - branch=$(urlencode "$branch") -fi - -query="branch=$branch\ - &commit=$commit\ - &build=$([ "$build_o" = "" ] && echo "$build" || echo "$build_o")\ - &build_url=$build_url\ - &name=$(urlencode "$name")\ - &tag=$([ "$tag_o" = "" ] && echo "$tag" || echo "$tag_o")\ - &slug=$([ "$slug_o" = "" ] && urlencode "$slug" || urlencode "$slug_o")\ - &service=$service\ - &flags=$flags\ - &pr=$([ "$pr_o" = "" ] && echo "${pr##\#}" || echo "${pr_o##\#}")\ - &job=$job" - -if [ "$ft_search" = "1" ]; -then - # detect bower comoponents location - bower_components="bower_components" - bower_rc=$(cd "$git_root" && cat .bowerrc 2>/dev/null || echo "") - if [ "$bower_rc" != "" ]; - then - bower_components=$(echo "$bower_rc" | tr -d '\n' | grep '"directory"' | cut -d'"' -f4 | sed -e 's/\/$//') - if [ "$bower_components" = "" ]; - then - bower_components="bower_components" - fi - fi - - # Swift Coverage - if [ "$ft_xcodellvm" = "1" ] && [ -d "$ddp" ]; - then - say "${e}==>${x} Processing Xcode reports via llvm-cov" - say " DerivedData folder: $ddp" - profdata_files=$(find "$ddp" -name '*.profdata' 2>/dev/null || echo '') - if [ "$profdata_files" != "" ]; - then - # xcode via profdata - if [ "$xp" = "" ]; - then - # xp=$(xcodebuild -showBuildSettings 2>/dev/null | grep -i "^\s*PRODUCT_NAME" | sed -e 's/.*= \(.*\)/\1/') - # say " ${e}->${x} Speed up Xcode processing by adding ${e}-J '$xp'${x}" - say " ${g}hint${x} Speed up Swift processing by using use ${g}-J 'AppName'${x} (regexp accepted)" - say " ${g}hint${x} This will remove Pods/ from your report. Also ${b}https://docs.codecov.io/docs/ignoring-paths${x}" - fi - while read -r profdata; - do - if [ "$profdata" != "" ]; - then - swiftcov "$profdata" "$xp" - fi - done <<< "$profdata_files" - else - say " ${e}->${x} No Swift coverage found" - fi - - # Obj-C Gcov Coverage - if [ "$ft_gcov" = "1" ]; - then - say " ${e}->${x} Running $gcov_exe for Obj-C" - bash -c "find $ddp -type f -name '*.gcda' $gcov_include $gcov_ignore -exec $gcov_exe -p $gcov_arg {} +" || true - fi - fi - - if [ "$ft_xcodeplist" = "1" ] && [ -d "$ddp" ]; - then - say "${e}==>${x} Processing Xcode plists" - plists_files=$(find "$ddp" -name '*.xccoverage' 2>/dev/null || echo '') - if [ "$plists_files" != "" ]; - then - while read -r plist; - do - if [ "$plist" != "" ]; - then - say " ${g}Found${x} plist file at $plist" - plutil -convert xml1 -o "$(basename "$plist").plist" -- $plist - fi - done <<< "$plists_files" - fi - fi - - # Gcov Coverage - if [ "$ft_gcov" = "1" ]; - then - say "${e}==>${x} Running gcov in $proj_root ${e}(disable via -X gcov)${x}" - bash -c "find $proj_root -type f -name '*.gcno' $gcov_include $gcov_ignore -exec $gcov_exe -pb $gcov_arg {} +" || true - else - say "${e}==>${x} gcov disabled" - fi - - # Python Coverage - if [ "$ft_coveragepy" = "1" ]; - then - if [ ! -f coverage.xml ]; - then - if which coverage >/dev/null 2>&1; - then - say "${e}==>${x} Python coveragepy exists ${e}disable via -X coveragepy${x}" - - dotcoverage=$(find "$git_root" -name '.coverage' -or -name '.coverage.*' | head -1 || echo '') - if [ "$dotcoverage" != "" ]; - then - cd "$(dirname "$dotcoverage")" - if [ ! -f .coverage ]; - then - say " ${e}->${x} Running coverage combine" - coverage combine -a - fi - say " ${e}->${x} Running coverage xml" - if [ "$(coverage xml -i)" != "No data to report." ]; - then - files="$files -$PWD/coverage.xml" - else - say " ${r}No data to report.${x}" - fi - cd "$proj_root" - else - say " ${r}No .coverage file found.${x}" - fi - else - say "${e}==>${x} Python coveragepy not found" - fi - fi - else - say "${e}==>${x} Python coveragepy disabled" - fi - - if [ "$search_in_o" != "" ]; - then - # location override - search_in="$search_in_o" - fi - - say "$e==>$x Searching for coverage reports in:" - for _path in $search_in - do - say " ${g}+${x} $_path" - done - - patterns="find $search_in \( \ - -name vendor \ - -or -name htmlcov \ - -or -name virtualenv \ - -or -name js/generated/coverage \ - -or -name .virtualenv \ - -or -name virtualenvs \ - -or -name .virtualenvs \ - -or -name .env \ - -or -name .envs \ - -or -name env \ - -or -name .yarn-cache \ - -or -name envs \ - -or -name .venv \ - -or -name .venvs \ - -or -name venv \ - -or -name venvs \ - -or -name .git \ - -or -name .hg \ - -or -name .tox \ - -or -name __pycache__ \ - -or -name '.egg-info*' \ - -or -name '$bower_components' \ - -or -name node_modules \ - -or -name 'conftest_*.c.gcov' \ - \) -prune -or \ - -type f \( -name '*coverage*.*' \ - -or -name 'nosetests.xml' \ - -or -name 'jacoco*.xml' \ - -or -name 'clover.xml' \ - -or -name 'report.xml' \ - -or -name '*.codecov.*' \ - -or -name 'codecov.*' \ - -or -name 'cobertura.xml' \ - -or -name 'excoveralls.json' \ - -or -name 'luacov.report.out' \ - -or -name 'coverage-final.json' \ - -or -name 'naxsi.info' \ - -or -name 'lcov.info' \ - -or -name 'lcov.dat' \ - -or -name '*.lcov' \ - -or -name '*.clover' \ - -or -name 'cover.out' \ - -or -name 'gcov.info' \ - -or -name '*.gcov' \ - -or -name '*.lst' \ - $include_cov \) \ - $exclude_cov \ - -not -name '*.profdata' \ - -not -name 'coverage-summary.json' \ - -not -name 'phpunit-code-coverage.xml' \ - -not -name '*/classycle/report.xml' \ - -not -name 'remapInstanbul.coverage*.json' \ - -not -name 'phpunit-coverage.xml' \ - -not -name '*codecov.yml' \ - -not -name '*.serialized' \ - -not -name '.coverage*' \ - -not -name '.*coveragerc' \ - -not -name '*.sh' \ - -not -name '*.bat' \ - -not -name '*.ps1' \ - -not -name '*.env' \ - -not -name '*.cmake' \ - -not -name '*.dox' \ - -not -name '*.ec' \ - -not -name '*.rst' \ - -not -name '*.h' \ - -not -name '*.scss' \ - -not -name '*.o' \ - -not -name '*.proto' \ - -not -name '*.sbt' \ - -not -name '*.xcoverage.*' \ - -not -name '*.gz' \ - -not -name '*.conf' \ - -not -name '*.p12' \ - -not -name '*.csv' \ - -not -name '*.rsp' \ - -not -name '*.m4' \ - -not -name '*.pem' \ - -not -name '*~' \ - -not -name '*.exe' \ - -not -name '*.am' \ - -not -name '*.template' \ - -not -name '*.cp' \ - -not -name '*.bw' \ - -not -name '*.crt' \ - -not -name '*.log' \ - -not -name '*.cmake' \ - -not -name '*.pth' \ - -not -name '*.in' \ - -not -name '*.jar*' \ - -not -name '*.pom*' \ - -not -name '*.png' \ - -not -name '*.jpg' \ - -not -name '*.sql' \ - -not -name '*.jpeg' \ - -not -name '*.svg' \ - -not -name '*.gif' \ - -not -name '*.csv' \ - -not -name '*.snapshot' \ - -not -name '*.mak*' \ - -not -name '*.bash' \ - -not -name '*.data' \ - -not -name '*.py' \ - -not -name '*.class' \ - -not -name '*.xcconfig' \ - -not -name '*.ec' \ - -not -name '*.coverage' \ - -not -name '*.pyc' \ - -not -name '*.cfg' \ - -not -name '*.egg' \ - -not -name '*.ru' \ - -not -name '*.css' \ - -not -name '*.less' \ - -not -name '*.pyo' \ - -not -name '*.whl' \ - -not -name '*.html' \ - -not -name '*.ftl' \ - -not -name '*.erb' \ - -not -name '*.rb' \ - -not -name '*.js' \ - -not -name '*.jade' \ - -not -name '*.db' \ - -not -name '*.md' \ - -not -name '*.cpp' \ - -not -name '*.gradle' \ - -not -name '*.tar.tz' \ - -not -name '*.scss' \ - -not -name 'include.lst' \ - -not -name 'fullLocaleNames.lst' \ - -not -name 'inputFiles.lst' \ - -not -name 'createdFiles.lst' \ - -not -name 'scoverage.measurements.*' \ - -not -name 'test_*_coverage.txt' \ - -not -name 'testrunner-coverage*' \ - -print 2>/dev/null" - files=$(eval "$patterns" || echo '') - -elif [ "$include_cov" != "" ]; -then - files=$(eval "find $search_in -type f \( ${include_cov:5} \)$exclude_cov 2>/dev/null" || echo '') -fi - -num_of_files=$(echo "$files" | wc -l | tr -d ' ') -if [ "$num_of_files" != '' ] && [ "$files" != '' ]; -then - say " ${e}->${x} Found $num_of_files reports" -fi - -# no files found -if [ "$files" = "" ]; -then - say "${r}-->${x} No coverage report found." - say " Please visit ${b}http://docs.codecov.io/docs/supported-languages${x}" - exit ${exit_with}; -fi - -if [ "$ft_network" == "1" ]; -then - say "${e}==>${x} Detecting git/mercurial file structure" - network=$(cd "$git_root" && git ls-files 2>/dev/null || hg locate 2>/dev/null || echo "") - if [ "$network" = "" ]; - then - network=$(find "$git_root" \( \ - -name virtualenv \ - -name .virtualenv \ - -name virtualenvs \ - -name .virtualenvs \ - -name '*.png' \ - -name '*.gif' \ - -name '*.jpg' \ - -name '*.jpeg' \ - -name '*.md' \ - -name .env \ - -name .envs \ - -name env \ - -name envs \ - -name .venv \ - -name .venvs \ - -name venv \ - -name venvs \ - -name .git \ - -name .egg-info \ - -name shunit2-2.1.6 \ - -name vendor \ - -name __pycache__ \ - -name node_modules \ - -path '*/$bower_components/*' \ - -path '*/target/delombok/*' \ - -path '*/build/lib/*' \ - -path '*/js/generated/coverage/*' \ - \) -prune -or \ - -type f -print 2>/dev/null || echo '') - fi - - if [ "$prefix_o" != "" ]; - then - network=$(echo "$network" | awk "{print \"$prefix_o/\"\$0}") - fi -fi - -upload_file=`mktemp /tmp/codecov.XXXXXX` -adjustments_file=`mktemp /tmp/codecov.adjustments.XXXXXX` - -cleanup() { - rm -f $upload_file $adjustments_file $upload_file.gz -} - -trap cleanup INT ABRT TERM - -if [ "$env" != "" ]; -then - inc_env="" - say "${e}==>${x} Appending build variables" - for varname in $(echo "$env" | tr ',' ' ') - do - if [ "$varname" != "" ]; - then - say " ${g}+${x} $varname" - inc_env="${inc_env}${varname}=$(eval echo "\$${varname}") -" - fi - done - -echo "$inc_env<<<<<< ENV" >> $upload_file -fi - -# Append git file list -# write discovered yaml location -echo "$yaml" >> $upload_file -if [ "$ft_network" == "1" ]; -then - i="woff|eot|otf" # fonts - i="$i|gif|png|jpg|jpeg|psd" # images - i="$i|ptt|pptx|numbers|pages|md|txt|xlsx|docx|doc|pdf|html|csv" # docs - i="$i|yml|yaml|.gitignore" # supporting docs - echo "$network" | grep -vwE "($i)$" >> $upload_file -fi -echo "<<<<<< network" >> $upload_file - -fr=0 -say "${e}==>${x} Reading reports" -while IFS='' read -r file; -do - # read the coverage file - if [ "$(echo "$file" | tr -d ' ')" != '' ]; - then - if [ -f "$file" ]; - then - report_len=$(wc -c < "$file") - if [ "$report_len" -ne 0 ]; - then - say " ${g}+${x} $file ${e}bytes=$(echo "$report_len" | tr -d ' ')${x}" - # append to to upload - _filename=$(basename "$file") - if [ "${_filename##*.}" = 'gcov' ]; - then - echo "# path=$(echo "$file.reduced" | sed "s|^$git_root/||")" >> $upload_file - # get file name - head -1 $file >> $upload_file - # 1. remove source code - # 2. remove ending bracket lines - # 3. remove whitespace - # 4. remove contextual lines - # 5. remove function names - awk -F': *' '{print $1":"$2":"}' $file \ - | sed '\/: *} *$/d' \ - | sed 's/^ *//' \ - | sed '/^-/d' \ - | sed 's/^function.*/func/' >> $upload_file - else - echo "# path=$(echo "$file" | sed "s|^$git_root/||")" >> $upload_file - cat "$file" >> $upload_file - fi - echo "<<<<<< EOF" >> $upload_file - fr=1 - if [ "$clean" = "1" ]; - then - rm "$file" - fi - else - say " ${r}-${x} Skipping empty file $file" - fi - else - say " ${r}-${x} file not found at $file" - fi - fi -done <<< "$(echo -e "$files")" - -if [ "$fr" = "0" ]; -then - say "${r}-->${x} No coverage data found." - say " Please visit ${b}http://docs.codecov.io/docs/supported-languages${x}" - say " search for your projects language to learn how to collect reports." - exit ${exit_with}; -fi - -if [ "$ft_fix" = "1" ]; -then - say "${e}==>${x} Appending adjustments" - say " ${b}http://docs.codecov.io/docs/fixing-reports${x}" - - empty_line='^[[:space:]]*$' - # // - syntax_comment='^[[:space:]]*//.*' - # /* or */ - syntax_comment_block='^[[:space:]]*(\/\*|\*\/)[[:space:]]*$' - # { or } - syntax_bracket='^[[:space:]]*[\{\}][[:space:]]*(//.*)?$' - # [ or ] - syntax_list='^[[:space:]]*[][][[:space:]]*(//.*)?$' - - skip_dirs="-not -path '*/$bower_components/*' \ - -not -path '*/node_modules/*'" - - cut_and_join() { - awk 'BEGIN { FS=":" } - $3 ~ /\/\*/ || $3 ~ /\*\// { print $0 ; next } - $1!=key { if (key!="") print out ; key=$1 ; out=$1":"$2 ; next } - { out=out","$2 } - END { print out }' 2>/dev/null - } - - if echo "$network" | grep -m1 '.kt$' 1>/dev/null; - then - # skip brackets and comments - find "$git_root" -type f \ - -name '*.kt' \ - -exec \ - grep -nIHE -e $syntax_bracket \ - -e $syntax_comment_block {} \; \ - | cut_and_join \ - >> $adjustments_file \ - || echo '' - - # last line in file - find "$git_root" -type f \ - -name '*.kt' -exec \ - wc -l {} \; \ - | while read l; do echo "EOF: $l"; done \ - 2>/dev/null \ - >> $adjustments_file \ - || echo '' - - fi - - if echo "$network" | grep -m1 '.go$' 1>/dev/null; - then - # skip empty lines, comments, and brackets - find "$git_root" -not -path '*/vendor/*' \ - -type f \ - -name '*.go' \ - -exec \ - grep -nIHE \ - -e $empty_line \ - -e $syntax_comment \ - -e $syntax_comment_block \ - -e $syntax_bracket \ - {} \; \ - | cut_and_join \ - >> $adjustments_file \ - || echo '' - fi - - if echo "$network" | grep -m1 '.dart$' 1>/dev/null; - then - # skip brackets - find "$git_root" -type f \ - -name '*.dart' \ - -exec \ - grep -nIHE \ - -e $syntax_bracket \ - {} \; \ - | cut_and_join \ - >> $adjustments_file \ - || echo '' - fi - - if echo "$network" | grep -m1 '.php$' 1>/dev/null; - then - # skip empty lines, comments, and brackets - find "$git_root" -not -path "*/vendor/*" \ - -type f \ - -name '*.php' \ - -exec \ - grep -nIHE \ - -e $syntax_list \ - -e $syntax_bracket \ - -e '^[[:space:]]*\);[[:space:]]*(//.*)?$' \ - {} \; \ - | cut_and_join \ - >> $adjustments_file \ - || echo '' - fi - - if echo "$network" | grep -m1 '\(.cpp\|.h\|.cxx\|.c\|.hpp\|.m\)$' 1>/dev/null; - then - # skip brackets - find "$git_root" -type f \ - $skip_dirs \ - \( \ - -name '*.h' \ - -or -name '*.cpp' \ - -or -name '*.cxx' \ - -or -name '*.m' \ - -or -name '*.c' \ - -or -name '*.hpp' \ - \) -exec \ - grep -nIHE \ - -e $empty_line \ - -e $syntax_bracket \ - -e '// LCOV_EXCL' \ - {} \; \ - | cut_and_join \ - >> $adjustments_file \ - || echo '' - - # skip brackets - find "$git_root" -type f \ - $skip_dirs \ - \( \ - -name '*.h' \ - -or -name '*.cpp' \ - -or -name '*.cxx' \ - -or -name '*.m' \ - -or -name '*.c' \ - -or -name '*.hpp' \ - \) -exec \ - grep -nIH '// LCOV_EXCL' \ - {} \; \ - >> $adjustments_file \ - || echo '' - - fi - - found=$(cat $adjustments_file | tr -d ' ') - - if [ "$found" != "" ]; - then - say " ${g}+${x} Found adjustments" - echo "# path=fixes" >> $upload_file - cat $adjustments_file >> $upload_file - echo "<<<<<< EOF" >> $upload_file - rm -rf $adjustments_file - else - say " ${e}->${x} No adjustments found" - fi -fi - -if [ "$url_o" != "" ]; -then - url="$url_o" -fi - -if [ "$dump" != "0" ]; -then - # trim whitespace from query - say " ${e}->${x} Dumping upload file (no upload)" - echo "$url/upload/v4?$(echo "package=bash-$VERSION&token=$token&$query" | tr -d ' ')" - cat $upload_file -else - - say "${e}==>${x} Gzipping contents" - gzip -nf9 $upload_file - - query=$(echo "${query}" | tr -d ' ') - say "${e}==>${x} Uploading reports" - say " ${e}url:${x} $url" - say " ${e}query:${x} $query" - - # now add token to query - query=$(echo "package=bash-$VERSION&token=$token&$query" | tr -d ' ') - - if [ "$ft_s3" = "1" ]; - then - i="0" - while [ $i -lt 4 ] - do - i=$[$i+1] - say " ${e}->${x} Pinging Codecov" - res=$(curl $curl_s -X POST $curlargs $cacert \ - -H 'X-Reduced-Redundancy: false' \ - -H 'X-Content-Type: application/x-gzip' \ - "$url/upload/v4?$query" || true) - # a good replay is "https://codecov.io" + "\n" + "https://codecov.s3.amazonaws.com/..." - status=$(echo "$res" | head -1 | grep 'HTTP ' | cut -d' ' -f2) - if [ "$status" = "" ]; - then - s3target=$(echo "$res" | sed -n 2p) - say " ${e}->${x} Uploading" - s3=$(curl $curl_s -fiX PUT $curlawsargs \ - --data-binary @$upload_file.gz \ - -H 'Content-Type: application/x-gzip' \ - -H 'Content-Encoding: gzip' \ - -H 'x-amz-acl: public-read' \ - "$s3target" || true) - if [ "$s3" != "" ]; - then - say " ${g}->${x} View reports at ${b}$(echo "$res" | sed -n 1p)${x}" - exit 0 - else - say " ${r}X>${x} Failed to upload" - fi - elif [ "$status" = "400" ]; - then - # 400 Error - say "${g}${res}${x}" - exit ${exit_with} - fi - say " ${e}->${x} Sleeping for 30s and trying again..." - sleep 30 - done - fi - - say " ${e}->${x} Uploading to Codecov" - i="0" - while [ $i -lt 4 ] - do - i=$[$i+1] - - res=$(curl $curl_s -X POST $curlargs $cacert \ - --data-binary @$upload_file.gz \ - -H 'Content-Type: text/plain' \ - -H 'Content-Encoding: gzip' \ - -H 'X-Content-Encoding: gzip' \ - -H 'Accept: text/plain' \ - "$url/upload/v2?$query" || echo 'HTTP 500') - # HTTP 200 - # http://.... - status=$(echo "$res" | head -1 | cut -d' ' -f2) - if [ "$status" = "" ]; - then - say " View reports at ${b}$(echo "$res" | head -2 | tail -1)${x}" - exit 0 - - elif [ "${status:0:1}" = "5" ]; - then - say " ${e}->${x} Sleeping for 30s and trying again..." - sleep 30 - - else - say " ${g}${res}${x}" - exit 0 - exit ${exit_with} - fi - - done - - say " ${r}X> Failed to upload coverage reports${x}" -fi - -exit ${exit_with} diff --git a/ci/jenkins/Jenkinsfile_py3-master_cpu_unittest b/ci/jenkins/Jenkinsfile_py3-master_cpu_unittest deleted file mode 100644 index cf2ddb402d..0000000000 --- a/ci/jenkins/Jenkinsfile_py3-master_cpu_unittest +++ /dev/null @@ -1,69 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 180 - -node { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/jenkins/utils.groovy') - build_steps = load('ci/jenkins/build_steps.groovy') -} -utils.assign_node_labels(linux_gpu: 'linux-gpu', linux_cpu: 'linux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Sanity', [ - build_steps.sanity_lint('gluon-nlp-cpu-py3-master', 'cpu/py3-master', 'src/gluonnlp') - ]) - - utils.parallel_stage('Tests', [ - build_steps.test_unittest('gluon-nlp-cpu-py3-master', 'cpu/py3-master', - 'tests/unittest', 'src/gluonnlp', - 'not (gpu or serial or skip_master)', - 4, false, false), - build_steps.test_unittest('gluon-nlp-cpu-py3-master', 'cpu/py3-master', - 'tests/unittest', 'src/gluonnlp', - 'not (gpu or skip_master) and serial', - 0, false, false), - build_steps.test_unittest('gluon-nlp-cpu-py3-master', 'cpu/py3-master', - 'scripts/tests', 'src/gluonnlp', - 'not (gpu or serial or integration or skip_master)', - 4, false, false), - build_steps.test_unittest('gluon-nlp-cpu-py3-master', 'cpu/py3-master', - 'scripts/tests', 'src/gluonnlp', - '(not (gpu or integration or skip_master)) and serial', - 0, false, false), - build_steps.test_unittest('gluon-nlp-cpu-py3-master', 'cpu/py3-master', - 'scripts/tests', 'src/gluonnlp', - 'not (gpu or serial or skip_master) and integration', - 4, false, false), - build_steps.test_unittest('gluon-nlp-cpu-py3-master', 'cpu/py3-master', - 'scripts/tests', 'src/gluonnlp', - 'not (gpu or skip_master) and serial and integration', - 0, false, false) - ]) -} -, -failure_handler: {} -) diff --git a/ci/jenkins/Jenkinsfile_py3-master_gpu_doc b/ci/jenkins/Jenkinsfile_py3-master_gpu_doc deleted file mode 100644 index 745608d052..0000000000 --- a/ci/jenkins/Jenkinsfile_py3-master_gpu_doc +++ /dev/null @@ -1,168 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 120 - -node { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/jenkins/utils.groovy') - build_steps = load('ci/jenkins/build_steps.groovy') -} -utils.assign_node_labels(linux_gpu: 'linux-gpu', linux_cpu: 'linux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Doc Test', [ - build_steps.test_doctest('gluon-nlp-cpu-py3-master', 'cpu/py3-master', - 'src/gluonnlp', 'src/gluonnlp', 4) - ]) - - // Compile example notebooks, Doctest & Create Website - node { // Single node parallelism - ws('gluon-nlp-cpu-py3-master') { - stage("Prepare conda environment for website") { - utils.init_git() - // Require a full environment here due to sphinx build step - // after compiling and downloading the notebooks - sh 'source ci/prepare_clean_env.sh cpu/py3-master' - } - - stage("Create Website") { - def tests = [:] - for (f in findFiles(glob: '**/docs/examples/*/*.md')) { - def md_file = f.toString() // Convert FileWrapper to String - def short_name = md_file["docs/examples/".length()..-1] - tests[short_name] = { -> - def base_name = md_file[0..-4] + '' - def ipynb_file = base_name + '.ipynb' - def stdout_file = base_name + '.stdout.log' - def stderr_file = base_name + '.stderr.log' - stage(short_name) { // remove common path from name - // Submit AWS Batch jobs for each example notebook - // The converted notebooks and the conversion logs are - // saved to S3 and retrieved on the CI server once the jobs - // finished. - - if (env.BRANCH_NAME.startsWith('PR-')){ - sh """ - set +e - conda activate ./conda/cpu/py3-master - - python3 ci/batch/submit-job.py --region us-east-1 --wait \ - --timeout 3600 --saved-output ./docs/examples --conda-env docker/py3 \ - --name GluonNLP-${env.BRANCH_NAME}-${env.BUILD_NUMBER} \ - --save-path batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/docs/examples \ - --work-dir . --source-ref refs/pull/${env.CHANGE_ID}/head \ - --command \"(python3 docs/md2ipynb.py ${md_file} | tee ${stdout_file}) 3>&1 1>&2 2>&3 | tee ${stderr_file} \" - BATCH_EXIT_CODE=\$? - - aws s3api wait object-exists --bucket gluon-nlp-staging \ - --key batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${stderr_file} - aws s3 cp s3://gluon-nlp-staging/batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${stderr_file} ${stderr_file} - cat ${stderr_file} - - aws s3api wait object-exists --bucket gluon-nlp-staging \ - --key batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${stdout_file} - aws s3 cp s3://gluon-nlp-staging/batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${stdout_file} ${stdout_file} - cat ${stdout_file} - - if [ \$BATCH_EXIT_CODE -ne 0 ]; then - echo AWS Batch Task Failed - else - aws s3api wait object-exists --bucket gluon-nlp-staging \ - --key batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${ipynb_file} - aws s3 cp s3://gluon-nlp-staging/batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${ipynb_file} ${ipynb_file} - fi - - exit \$BATCH_EXIT_CODE - """ - } else { - sh """ - set +e - conda activate ./conda/cpu/py3-master - - python3 ci/batch/submit-job.py --region us-east-1 --wait \ - --timeout 3600 --saved-output ./docs/examples --conda-env docker/py3 \ - --name GluonNLP-${env.BRANCH_NAME}-${env.BUILD_NUMBER} \ - --save-path batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/docs/examples \ - --work-dir . --source-ref ${env.BRANCH_NAME} \ - --command \"(python3 docs/md2ipynb.py ${md_file} | tee ${stdout_file}) 3>&1 1>&2 2>&3 | tee ${stderr_file} \" - BATCH_EXIT_CODE=\$? - - aws s3api wait object-exists --bucket gluon-nlp-staging \ - --key batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${stderr_file} - aws s3 cp s3://gluon-nlp-staging/batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${stderr_file} ${stderr_file} - cat ${stderr_file} - - aws s3api wait object-exists --bucket gluon-nlp-staging \ - --key batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${stdout_file} - aws s3 cp s3://gluon-nlp-staging/batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${stdout_file} ${stdout_file} - cat ${stdout_file} - - if [ \$BATCH_EXIT_CODE -ne 0 ]; then - echo AWS Batch Task Failed - else - aws s3api wait object-exists --bucket gluon-nlp-staging \ - --key batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${ipynb_file} - aws s3 cp s3://gluon-nlp-staging/batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${ipynb_file} ${ipynb_file} - fi - - exit \$BATCH_EXIT_CODE - """ - } - } - } - } - - parallel tests - } - - stage("Upload Website") { - if (env.BRANCH_NAME.startsWith('PR-')){ - bucket = 'gluon-nlp-staging' - path = env.BRANCH_NAME+'/'+env.BUILD_NUMBER - } else { - bucket = 'gluon-nlp' - path = env.BRANCH_NAME - } - sh """ - conda activate ./conda/cpu/py3-master - make docs - ci/upload_doc.sh ${bucket} ${path} - """ - } - } - } - - utils.parallel_stage('Documentation', [ - build_steps.website_linkcheck('gluon-nlp-cpu-py3-master', 'cpu/py3-master') - ]) - - utils.parallel_stage('Deploy', [ - build_steps.post_website_link() - ]) -} -, -failure_handler: {} -) diff --git a/ci/jenkins/Jenkinsfile_py3-master_gpu_integration b/ci/jenkins/Jenkinsfile_py3-master_gpu_integration deleted file mode 100644 index 31002e4bdd..0000000000 --- a/ci/jenkins/Jenkinsfile_py3-master_gpu_integration +++ /dev/null @@ -1,53 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 120 - -node { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/jenkins/utils.groovy') - build_steps = load('ci/jenkins/build_steps.groovy') -} -utils.assign_node_labels(linux_gpu: 'linux-gpu', linux_cpu: 'linux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Sanity', [ - build_steps.sanity_lint('gluon-nlp-gpu-py3-master', 'gpu/py3-master', 'scripts') - ]) - - utils.parallel_stage('Scripts', [ - build_steps.test_unittest('gluon-nlp-gpu-py3-master', 'gpu/py3-master', - 'scripts/tests', 'src/gluonnlp', - 'gpu and (not (serial or skip_master)) and integration', - 4, true, true), - build_steps.test_unittest('gluon-nlp-gpu-py3-master', 'gpu/py3-master', - 'scripts/tests', 'src/gluonnlp', - 'gpu and serial and integration and (not skip_master)', - 0, true, true) - ]) -} -, -failure_handler: {} -) diff --git a/ci/jenkins/Jenkinsfile_py3-master_gpu_unittest b/ci/jenkins/Jenkinsfile_py3-master_gpu_unittest deleted file mode 100644 index 6275e40d58..0000000000 --- a/ci/jenkins/Jenkinsfile_py3-master_gpu_unittest +++ /dev/null @@ -1,61 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 120 - -node { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/jenkins/utils.groovy') - build_steps = load('ci/jenkins/build_steps.groovy') -} -utils.assign_node_labels(linux_gpu: 'linux-gpu', linux_cpu: 'linux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Sanity', [ - build_steps.sanity_lint('gluon-nlp-gpu-py3-master', 'gpu/py3-master', 'src/gluonnlp') - ]) - - utils.parallel_stage('Tests', [ - build_steps.test_unittest('gluon-nlp-gpu-py3-master', 'gpu/py3-master', - 'tests/unittest', 'src/gluonnlp', - 'gpu and (not (serial or skip_master))', - 4, true, false), - build_steps.test_unittest('gluon-nlp-gpu-py3-master', 'gpu/py3-master', - 'tests/unittest', 'src/gluonnlp', - 'gpu and serial and not skip_master', - 0, true, false), - build_steps.test_unittest('gluon-nlp-gpu-py3-master', 'gpu/py3-master', - 'scripts/tests', 'src/gluonnlp', - 'gpu and (not (serial or skip_master or integration))', - 4, true, false), - build_steps.test_unittest('gluon-nlp-gpu-py3-master', 'gpu/py3-master', - 'scripts/tests', 'src/gluonnlp', - 'gpu and serial and not (skip_master or integration)', - 0, true, false) - ]) -} -, -failure_handler: {} -) diff --git a/ci/jenkins/Jenkinsfile_py3_cpu_unittest b/ci/jenkins/Jenkinsfile_py3_cpu_unittest deleted file mode 100644 index 56478e1a0a..0000000000 --- a/ci/jenkins/Jenkinsfile_py3_cpu_unittest +++ /dev/null @@ -1,69 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 180 - -node { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/jenkins/utils.groovy') - build_steps = load('ci/jenkins/build_steps.groovy') -} -utils.assign_node_labels(linux_gpu: 'linux-gpu', linux_cpu: 'linux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Sanity', [ - build_steps.sanity_lint('gluon-nlp-cpu-py3', 'cpu/py3', 'src/gluonnlp') - ]) - - utils.parallel_stage('Tests', [ - build_steps.test_unittest('gluon-nlp-cpu-py3', 'cpu/py3', - 'tests/unittest', 'src/gluonnlp', - 'not (gpu or serial)', - 4, false, false), - build_steps.test_unittest('gluon-nlp-cpu-py3', 'cpu/py3', - 'tests/unittest', 'src/gluonnlp', - '(not gpu) and serial', - 0, false, false), - build_steps.test_unittest('gluon-nlp-cpu-py3', 'cpu/py3', - 'scripts/tests', 'src/gluonnlp', - 'not (gpu or serial or integration)', - 4, false, false), - build_steps.test_unittest('gluon-nlp-cpu-py3', 'cpu/py3', - 'scripts/tests', 'src/gluonnlp', - '(not (gpu or integration)) and serial', - 0, false, false), - build_steps.test_unittest('gluon-nlp-cpu-py3', 'cpu/py3', - 'scripts/tests', 'src/gluonnlp', - 'not (gpu or serial) and integration', - 4, false, false), - build_steps.test_unittest('gluon-nlp-cpu-py3', 'cpu/py3', - 'scripts/tests', 'src/gluonnlp', - '(not gpu) and serial and integration', - 0, false, false) - ]) -} -, -failure_handler: {} -) diff --git a/ci/jenkins/Jenkinsfile_py3_gpu_integration b/ci/jenkins/Jenkinsfile_py3_gpu_integration deleted file mode 100644 index e683f5f14d..0000000000 --- a/ci/jenkins/Jenkinsfile_py3_gpu_integration +++ /dev/null @@ -1,53 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 120 - -node { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/jenkins/utils.groovy') - build_steps = load('ci/jenkins/build_steps.groovy') -} -utils.assign_node_labels(linux_gpu: 'linux-gpu', linux_cpu: 'linux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Sanity', [ - build_steps.sanity_lint('gluon-nlp-gpu-py3', 'gpu/py3', 'scripts') - ]) - - utils.parallel_stage('Scripts', [ - build_steps.test_unittest('gluon-nlp-gpu-py3', 'gpu/py3', - 'scripts/tests', 'src/gluonnlp', - 'gpu and (not serial) and integration', - 4, true, true), - build_steps.test_unittest('gluon-nlp-gpu-py3', 'gpu/py3', - 'scripts/tests', 'src/gluonnlp', - 'gpu and serial and integration', - 0, true, true) - ]) -} -, -failure_handler: {} -) diff --git a/ci/jenkins/Jenkinsfile_py3_gpu_unittest b/ci/jenkins/Jenkinsfile_py3_gpu_unittest deleted file mode 100644 index 8430ca0a36..0000000000 --- a/ci/jenkins/Jenkinsfile_py3_gpu_unittest +++ /dev/null @@ -1,61 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 120 - -node { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/jenkins/utils.groovy') - build_steps = load('ci/jenkins/build_steps.groovy') -} -utils.assign_node_labels(linux_gpu: 'linux-gpu', linux_cpu: 'linux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Sanity', [ - build_steps.sanity_lint('gluon-nlp-gpu-py3', 'gpu/py3', 'src/gluonnlp') - ]) - - utils.parallel_stage('Tests', [ - build_steps.test_unittest('gluon-nlp-gpu-py3', 'gpu/py3', - 'tests/unittest', 'src/gluonnlp', - 'gpu and not serial', - 4, true, false), - build_steps.test_unittest('gluon-nlp-gpu-py3', 'gpu/py3', - 'tests/unittest', 'src/gluonnlp', - 'gpu and serial', - 0, true, false), - build_steps.test_unittest('gluon-nlp-gpu-py3', 'gpu/py3', - 'scripts/tests', 'src/gluonnlp', - 'gpu and not (serial or integration)', - 4, true, false), - build_steps.test_unittest('gluon-nlp-gpu-py3', 'gpu/py3', - 'scripts/tests', 'src/gluonnlp', - 'gpu and serial and not integration', - 0, true, false) - ]) -} -, -failure_handler: {} -) diff --git a/ci/jenkins/build_steps.groovy b/ci/jenkins/build_steps.groovy deleted file mode 100644 index 63bd59e81d..0000000000 --- a/ci/jenkins/build_steps.groovy +++ /dev/null @@ -1,127 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// This file contains the steps that will be used in the -// Jenkins pipelines - -utils = load('ci/jenkins/utils.groovy') - -def sanity_lint(workspace_name, conda_env_name, path) { - return ['Lint': { - node { - ws(workspace_name) { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - sh """ - set -ex - source ci/prepare_clean_env.sh ${conda_env_name} - make lintdir=${path} lint - set +ex - """ - } - } - } - }] -} - -def test_unittest(workspace_name, conda_env_name, - test_path, cov_path, - mark, - threads, gpu, skip_report) { - capture_flag = env.BRANCH_NAME.startsWith('PR-')?'':'--capture=no' - node_type = gpu?NODE_LINUX_GPU:NODE_LINUX_CPU - return ["${conda_env_name}: ${test_path} -m '${mark}'": { - node(node_type) { - ws(workspace_name) { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - sh """ - set -ex - source ci/prepare_clean_env.sh ${conda_env_name} - pytest -v ${capture_flag} -n ${threads} -m '${mark}' --durations=30 --cov ${cov_path} --cov-report=term --cov-report xml ${test_path} - set +ex - """ - if (!skip_report) utils.publish_test_coverage('GluonNLPCodeCov') - } - } - } - }] -} - -def test_doctest(workspace_name, conda_env_name, - test_path, cov_path, threads) { - capture_flag = env.BRANCH_NAME.startsWith('PR-')?'':'--capture=no' - return ["${conda_env_name}: doctest ${test_path}": { - node(NODE_LINUX_CPU) { - ws(workspace_name) { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - sh """ - set -ex - source ci/prepare_clean_env.sh ${conda_env_name} - pytest -v ${capture_flag} -n ${threads} --durations=30 --cov ${cov_path} --cov-report=term --cov-report xml --doctest-modules ${test_path} - set +ex - """ - utils.publish_test_coverage('GluonNLPCodeCov') - } - } - } - }] -} - -def website_linkcheck(workspace_name, conda_env_name) { - return ["${conda_env_name}: website link check": { - node(NODE_LINUX_CPU) { - ws(workspace_name) { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - sh """ - set -ex - source ci/prepare_clean_env.sh ${conda_env_name} - make distribute - set +ex - """ - linkcheck_errors = sh returnStdout: true, script: """ - conda activate ./conda/${conda_env_name} - """ - linkcheck_errors = linkcheck_errors.split('\n').findAll {it ==~ '/^(line *[0-9]*) broken.*$/'} - linkcheck_errors = linkcheck_errors.join('\n') - linkcheck_errors = linkcheck_errors.trim() - if (linkcheck_errors && env.BRANCH_NAME.startsWith("PR-")) { - pullRequest.comment("Found link check problems in job ${env.BRANCH_NAME}/${env.BUILD_NUMBER}:\n"+linkcheck_errors) - } - } - } - } - }] -} - -def post_website_link() { - return ["Deploy: ": { - node { - timeout(time: max_time, unit: 'MINUTES') { - if (env.BRANCH_NAME.startsWith("PR-")) { - pullRequest.comment("Job ${env.BRANCH_NAME}/${env.BUILD_NUMBER} is complete. \nDocs are uploaded to http://gluon-nlp-staging.s3-accelerate.dualstack.amazonaws.com/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/index.html") - } - } - } - }] -} - -return this diff --git a/ci/jenkins/utils.groovy b/ci/jenkins/utils.groovy deleted file mode 100644 index ddbde419d5..0000000000 --- a/ci/jenkins/utils.groovy +++ /dev/null @@ -1,214 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// initialize source codes -def init_git() { - deleteDir() - retry(5) { - try { - // Make sure wait long enough for api.github.com request quota. Important: Don't increase the amount of - // retries as this will increase the amount of requests and worsen the throttling - timeout(time: 15, unit: 'MINUTES') { - checkout scm - sh 'git clean -xdff' - sh 'git reset --hard' - sh 'git submodule update --init --recursive' - sh 'git submodule foreach --recursive git clean -ffxd' - sh 'git submodule foreach --recursive git reset --hard' - } - } catch (exc) { - deleteDir() - error "Failed to fetch source codes with ${exc}" - sleep 2 - } - } -} - - -def get_git_commit_hash() { - lastCommitMessage = sh (script: "git log -1 --pretty=%B", returnStdout: true) - lastCommitMessage = lastCommitMessage.trim() - if (lastCommitMessage.startsWith("Merge commit '") && lastCommitMessage.endsWith("' into HEAD")) { - // Merge commit applied by Jenkins, skip that commit - git_commit_hash = sh (script: "git rev-parse @~", returnStdout: true) - } else { - git_commit_hash = sh (script: "git rev-parse @", returnStdout: true) - } - return git_commit_hash.trim() -} - -def publish_test_coverage(codecov_credential) { - // CodeCovs auto detection has trouble with our CIs PR validation due the merging strategy - git_commit_hash = get_git_commit_hash() - - if (env.CHANGE_ID) { - // PR execution - codecovArgs = "-B ${env.CHANGE_TARGET} -C ${git_commit_hash} -P ${env.CHANGE_ID}" - } else { - // Branch execution - codecovArgs = "-B ${env.BRANCH_NAME} -C ${git_commit_hash}" - } - - // To make sure we never fail because test coverage reporting is not available - // Fall back to our own copy of the bash helper if it failed to download the public version - withCredentials([string(credentialsId: codecov_credential, variable: 'CODECOV_TOKEN')]) { - sh "(curl --retry 10 -s https://codecov.io/bash | bash -s - ${codecovArgs}) || (curl --retry 10 -s https://s3-us-west-2.amazonaws.com/mxnet-ci-prod-slave-data/codecov-bash.txt | bash -s - ${codecovArgs}) || true" - } -} - -// Allow publishing to GitHub with a custom context (the status shown under a PR) -// Credit to https://plugins.jenkins.io/github -def get_repo_url() { - checkout scm - return sh(returnStdout: true, script: "git config --get remote.origin.url").trim() -} - -def update_github_commit_status(state, message) { - node { - // NOTE: https://issues.jenkins-ci.org/browse/JENKINS-39482 - //The GitHubCommitStatusSetter requires that the Git Server is defined under - //*Manage Jenkins > Configure System > GitHub > GitHub Servers*. - //Otherwise the GitHubCommitStatusSetter is not able to resolve the repository name - //properly and you would see an empty list of repos: - //[Set GitHub commit status (universal)] PENDING on repos [] (sha:xxxxxxx) with context:test/mycontext - //See https://cwiki.apache.org/confluence/display/MXNET/Troubleshooting#Troubleshooting-GitHubcommit/PRstatusdoesnotgetpublished - - echo "Publishing commit status..." - - repoUrl = get_repo_url() - echo "repoUrl=${repoUrl}" - - commitSha = get_git_commit_hash() - echo "commitSha=${commitSha}" - - context = get_github_context() - echo "context=${context}" - - // a few attempts need to be made: https://github.com/apache/incubator-mxnet/issues/11654 - for (int attempt = 1; attempt <= 3; attempt++) { - echo "Sending GitHub status attempt ${attempt}..." - - step([ - $class: 'GitHubCommitStatusSetter', - reposSource: [$class: "ManuallyEnteredRepositorySource", url: repoUrl], - contextSource: [$class: "ManuallyEnteredCommitContextSource", context: context], - commitShaSource: [$class: "ManuallyEnteredShaSource", sha: commitSha], - statusBackrefSource: [$class: "ManuallyEnteredBackrefSource", backref: "${env.RUN_DISPLAY_URL}"], - errorHandlers: [[$class: 'ShallowAnyErrorHandler']], - statusResultSource: [ - $class: 'ConditionalStatusResultSource', - results: [[$class: "AnyBuildResult", message: message, state: state]] - ] - ]) - - if (attempt <= 2) { - sleep 1 - } - } - - echo "Publishing commit status done." - - } -} - -def get_github_context() { - // Since we use multi-branch pipelines, Jenkins appends the branch name to the job name - if (env.BRANCH_NAME) { - short_job_name = JOB_NAME.substring(0, JOB_NAME.lastIndexOf('/')) - } else { - short_job_name = JOB_NAME - } - - return "ci/jenkins/${short_job_name}" -} - -def parallel_stage(stage_name, steps) { - // Allow to pass an array of steps that will be executed in parallel in a stage - new_map = [:] - - for (def step in steps) { - new_map = new_map << step - } - - stage(stage_name) { - parallel new_map - } -} - -def assign_node_labels(args) { - // This function allows to assign instance labels to the generalized placeholders. - // This serves two purposes: - // 1. Allow generalized placeholders (e.g. NODE_WINDOWS_CPU) in the job definition - // in order to abstract away the underlying node label. This allows to schedule a job - // onto a different node for testing or security reasons. This could be, for example, - // when you want to test a new set of slaves on separate labels or when a job should - // only be run on restricted slaves - // 2. Restrict the allowed job types within a Jenkinsfile. For example, a UNIX-CPU-only - // Jenkinsfile should not allowed access to Windows or GPU instances. This prevents - // users from just copy&pasting something into an existing Jenkinsfile without - // knowing about the limitations. - NODE_LINUX_GPU = args.linux_gpu - NODE_LINUX_CPU = args.linux_cpu -} - -def main_wrapper(args) { - // Main Jenkinsfile pipeline wrapper handler that allows to wrap core logic into a format - // that supports proper failure handling - // args: - // - core_logic: Jenkins pipeline containing core execution logic - // - failure_handler: Failure handler - - // assign any caught errors here - err = null - try { - update_github_commit_status('PENDING', 'Job has been enqueued') - - timestamps { - args['core_logic']() - } - - // set build status to success at the end - currentBuild.result = "SUCCESS" - update_github_commit_status('SUCCESS', 'Job succeeded') - } catch (caughtError) { - node { - sh "echo caught ${caughtError}" - err = caughtError - currentBuild.result = "FAILURE" - update_github_commit_status('FAILURE', 'Job failed') - } - } finally { - timestamps { - node { - // Call failure handler - args['failure_handler']() - - // Clean workspace to reduce space requirements - cleanWs() - - // Remember to rethrow so the build is marked as failing - if (err) { - throw err - } - } - } - } -} - -return this diff --git a/ci/upload_doc.sh b/ci/upload_doc.sh deleted file mode 100755 index efa5e5d904..0000000000 --- a/ci/upload_doc.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -bucket=$1 -path=$2 -echo "Uploading doc to s3://${bucket}/${path}/" -aws s3 sync --delete docs/_build/html/ s3://${bucket}/${path}/ --acl public-read -echo "Uploaded doc to http://${bucket}.s3-accelerate.dualstack.amazonaws.com/${path}/index.html" diff --git a/docs/website/contribute.rst b/docs/website/contribute.rst index f0542385d6..56aef56922 100644 --- a/docs/website/contribute.rst +++ b/docs/website/contribute.rst @@ -43,13 +43,13 @@ Make changes Our package uses continuous integration and code coverage tools for verifying pull requests. Before submitting, contributor should ensure that the following checks do not fail: -- Lint (code style) -- Unittest -- Doctest +- CPU unittest +- GPU unittest +- Website build The commands executed by the continuous integration server to perform the tests -are listed in the `build_steps.groovy file -`__. +are listed in the `github workflows files +`__. Contribute to model zoo -----------------------