From ec3d037abbb1c9cb51995f0b5a9c8e65467e4a3d Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Wed, 24 Aug 2022 16:25:25 -0700 Subject: [PATCH 1/6] testing github secrets --- .github/workflows/storage-test.yml | 45 ++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/storage-test.yml diff --git a/.github/workflows/storage-test.yml b/.github/workflows/storage-test.yml new file mode 100644 index 000000000000..b876e83f6d43 --- /dev/null +++ b/.github/workflows/storage-test.yml @@ -0,0 +1,45 @@ +name: Formatting + +on: + push: + branches: + - 'master' + - 'staging**' + pull_request: + branches: + '**' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + + # formatting and basic install on cpu-only machine + formatting: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + + - name: environment + run: | + which python + python --version + + - name: Install deepspeed + run: | + pip install .[dev,autotuning] + ds_report + + - name: Formatting checks + run: | + pre-commit run --all-files + + - name: secret + env: + MYSAS: ${{ secrets.BLOB_SAS_TOKEN }} + run: | + echo "MY SECRET IS:" + echo ${{ secrets.BLOB_SAS_TOKEN }} + echo "$MYSAS" From b359d8f9b805fe3e72202f50638b0d8c84fc9587 Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Wed, 24 Aug 2022 16:26:23 -0700 Subject: [PATCH 2/6] fix duplicate name --- .github/workflows/storage-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/storage-test.yml b/.github/workflows/storage-test.yml index b876e83f6d43..3a4a47a0e646 100644 --- a/.github/workflows/storage-test.yml +++ b/.github/workflows/storage-test.yml @@ -1,4 +1,4 @@ -name: Formatting +name: Storage on: push: From 51646dd4d6fac4965f5a30af7bdcb70460f0edbd Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Thu, 25 Aug 2022 16:04:24 -0700 Subject: [PATCH 3/6] testing new trigger --- .github/workflows/storage-test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/storage-test.yml b/.github/workflows/storage-test.yml index 3a4a47a0e646..09eaaca8d8cf 100644 --- a/.github/workflows/storage-test.yml +++ b/.github/workflows/storage-test.yml @@ -5,9 +5,7 @@ on: branches: - 'master' - 'staging**' - pull_request: - branches: - '**' + pull_request_target concurrency: group: ${{ github.workflow }}-${{ github.ref }} From afcae58a3b2f29ada234ce3dd9a90963e3d0b2b4 Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Thu, 25 Aug 2022 16:06:25 -0700 Subject: [PATCH 4/6] testing new trigger --- .github/workflows/storage-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/storage-test.yml b/.github/workflows/storage-test.yml index 09eaaca8d8cf..88aa062719b7 100644 --- a/.github/workflows/storage-test.yml +++ b/.github/workflows/storage-test.yml @@ -5,7 +5,9 @@ on: branches: - 'master' - 'staging**' - pull_request_target + pull_request_target: + branches: + '**' concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 55f248da02cd48f75b98c078788f61f6f0953a50 Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Thu, 25 Aug 2022 17:08:10 -0700 Subject: [PATCH 5/6] add blob storage for transformers cache --- .github/workflows/nv-inference.yml | 2 +- .github/workflows/storage-test.yml | 45 ------------------------------ 2 files changed, 1 insertion(+), 46 deletions(-) delete mode 100644 .github/workflows/storage-test.yml diff --git a/.github/workflows/nv-inference.yml b/.github/workflows/nv-inference.yml index 1df8142697cf..7ffcffb7b219 100644 --- a/.github/workflows/nv-inference.yml +++ b/.github/workflows/nv-inference.yml @@ -61,4 +61,4 @@ jobs: if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi cd tests EXPECTED_TORCH=$(pip index versions torch | grep -oP -m1 "^\s*LATEST.*\s\K\d+\.\d+") - TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose -m 'inference' unit/ --torch_ver=$EXPECTED_TORCH --cuda_ver="11.3" + TRANSFORMERS_CACHE=/blob/transformers_cache/ TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose -m 'inference' unit/ --torch_ver=$EXPECTED_TORCH --cuda_ver="11.3" diff --git a/.github/workflows/storage-test.yml b/.github/workflows/storage-test.yml deleted file mode 100644 index 88aa062719b7..000000000000 --- a/.github/workflows/storage-test.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Storage - -on: - push: - branches: - - 'master' - - 'staging**' - pull_request_target: - branches: - '**' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - - # formatting and basic install on cpu-only machine - formatting: - runs-on: ubuntu-20.04 - - steps: - - uses: actions/checkout@v2 - - - name: environment - run: | - which python - python --version - - - name: Install deepspeed - run: | - pip install .[dev,autotuning] - ds_report - - - name: Formatting checks - run: | - pre-commit run --all-files - - - name: secret - env: - MYSAS: ${{ secrets.BLOB_SAS_TOKEN }} - run: | - echo "MY SECRET IS:" - echo ${{ secrets.BLOB_SAS_TOKEN }} - echo "$MYSAS" From 320469856d611ad4dec6da061706922971b7eaee Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Fri, 26 Aug 2022 09:27:59 -0700 Subject: [PATCH 6/6] run 4 tests in parallel --- .github/workflows/nv-inference.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nv-inference.yml b/.github/workflows/nv-inference.yml index 7ffcffb7b219..5d588f08f3e9 100644 --- a/.github/workflows/nv-inference.yml +++ b/.github/workflows/nv-inference.yml @@ -61,4 +61,4 @@ jobs: if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi cd tests EXPECTED_TORCH=$(pip index versions torch | grep -oP -m1 "^\s*LATEST.*\s\K\d+\.\d+") - TRANSFORMERS_CACHE=/blob/transformers_cache/ TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose -m 'inference' unit/ --torch_ver=$EXPECTED_TORCH --cuda_ver="11.3" + TRANSFORMERS_CACHE=/blob/transformers_cache/ TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked -n 4 --verbose -m 'inference' unit/ --torch_ver=$EXPECTED_TORCH --cuda_ver="11.3"