From 6ad15fc069de9fe8fb3098aa509969b4581d649d Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Fri, 13 Nov 2020 12:11:03 +0100 Subject: [PATCH 1/8] HDDS-4461. Reuse compiled binaries in acceptance test --- .github/workflows/post-commit.yml | 91 +++++++++++-------------------- 1 file changed, 32 insertions(+), 59 deletions(-) diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml index 1aaf27c6da23..165ffef9fe86 100644 --- a/.github/workflows/post-commit.yml +++ b/.github/workflows/post-commit.yml @@ -19,8 +19,7 @@ on: schedule: - cron: 30 0,12 * * * jobs: - build: - name: compile + compile: runs-on: ubuntu-18.04 strategy: matrix: @@ -49,7 +48,17 @@ jobs: with: java-version: ${{ matrix.java }} - name: Run a full build - run: hadoop-ozone/dev-support/checks/build.sh + run: hadoop-ozone/dev-support/checks/build.sh -Pcoverage + - name: Tar binaries + # to maintain permissions (needed for executables) + if: matrix.java == '8' + run: tar cvf ozone-bin.tar hadoop-ozone/dist/target/ozone-* + - name: Store binaries for tests + uses: actions/upload-artifact@v2 + if: matrix.java == '8' + with: + name: ozone-bin + path: ozone-bin.tar bats: runs-on: ubuntu-18.04 steps: @@ -167,6 +176,7 @@ jobs: path: target/findbugs continue-on-error: true acceptance: + needs: compile runs-on: ubuntu-18.04 strategy: matrix: @@ -178,37 +188,18 @@ jobs: steps: - name: Checkout project uses: actions/checkout@v2 - - name: Cache for maven dependencies - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: maven-repo- - - name: Cache for npm dependencies - uses: actions/cache@v2 with: - path: | - ~/.pnpm-store - **/node_modules - key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm- - - name: Checkout to /mnt/ozone + path: ozone + - name: Move Ozone to /mnt run: | sudo chmod 777 /mnt - git clone 'https://github.com/${{ github.repository }}.git' /mnt/ozone - cd /mnt/ozone - if [[ '${{ github.event_name }}' == 'pull_request' ]]; then - git fetch --verbose origin '${{ github.ref }}' - else - git fetch --verbose origin '${{ github.sha }}' - fi - git checkout FETCH_HEAD - git reset --hard - - name: Run a full build - run: | - cd /mnt/ozone - hadoop-ozone/dev-support/checks/build.sh -Pcoverage + mv ozone /mnt/ + - name: Download compiled Ozone binaries + uses: actions/download-artifact@v2 + with: + name: ozone-bin + - name: Untar binaries + run: tar xvf ozone-bin.tar -C /mnt/ozone - name: Install robotframework run: sudo pip install robotframework - name: Execute tests @@ -304,37 +295,23 @@ jobs: path: target/coverage continue-on-error: true kubernetes: + needs: compile runs-on: ubuntu-18.04 steps: - name: Checkout project uses: actions/checkout@v2 - - name: Cache for maven dependencies - uses: actions/cache@v2 with: - path: ~/.m2/repository - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: maven-repo- - - name: Cache for npm dependencies - uses: actions/cache@v2 - with: - path: | - ~/.pnpm-store - **/node_modules - key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm- - - name: Checkout to /mnt/ozone + path: ozone + - name: Move Ozone to /mnt run: | sudo chmod 777 /mnt - git clone 'https://github.com/${{ github.repository }}.git' /mnt/ozone - cd /mnt/ozone - if [[ '${{ github.event_name }}' == 'pull_request' ]]; then - git fetch --verbose origin '${{ github.ref }}' - else - git fetch --verbose origin '${{ github.sha }}' - fi - git checkout FETCH_HEAD - git reset --hard + mv ozone /mnt/ + - name: Download compiled Ozone binaries + uses: actions/download-artifact@v2 + with: + name: ozone-bin + - name: Untar binaries + run: tar xvf ozone-bin.tar -C /mnt/ozone - name: Install robotframework run: sudo pip install robotframework - name: Install k3s @@ -350,10 +327,6 @@ jobs: wget https://github.com/elek/flekszible/releases/download/v1.8.1/flekszible_1.8.1_Linux_x86_64.tar.gz -O - | tar -zx chmod +x flekszible sudo mv flekszible /usr/bin/flekszible - - name: Run a full build - run: | - cd /mnt/ozone - hadoop-ozone/dev-support/checks/build.sh -Pcoverage - name: Execute tests run: | cd /mnt/ozone/hadoop-ozone/dist/target/ozone-* && sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws From 243ee5250e9221bdbc18e547bf4af78be597c3dd Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Fri, 13 Nov 2020 13:23:10 +0100 Subject: [PATCH 2/8] trigger new CI check From 99f541479dba71806141d1565d50409b19d247d0 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 16 Nov 2020 14:23:30 +0100 Subject: [PATCH 3/8] Create tar via -Pdist instead of custom step --- .github/workflows/post-commit.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml index 165ffef9fe86..08036dc852db 100644 --- a/.github/workflows/post-commit.yml +++ b/.github/workflows/post-commit.yml @@ -48,17 +48,13 @@ jobs: with: java-version: ${{ matrix.java }} - name: Run a full build - run: hadoop-ozone/dev-support/checks/build.sh -Pcoverage - - name: Tar binaries - # to maintain permissions (needed for executables) - if: matrix.java == '8' - run: tar cvf ozone-bin.tar hadoop-ozone/dist/target/ozone-* + run: hadoop-ozone/dev-support/checks/build.sh -Pcoverage -Pdist - name: Store binaries for tests uses: actions/upload-artifact@v2 if: matrix.java == '8' with: name: ozone-bin - path: ozone-bin.tar + path: hadoop-ozone/dist/target/hadoop-ozone*.tar.gz bats: runs-on: ubuntu-18.04 steps: @@ -199,7 +195,7 @@ jobs: with: name: ozone-bin - name: Untar binaries - run: tar xvf ozone-bin.tar -C /mnt/ozone + run: tar xzvf hadoop-ozone*.tar.gz -C /mnt/ozone - name: Install robotframework run: sudo pip install robotframework - name: Execute tests From e5d977cb4659c41fcd0ac94c1d147106b034bfd8 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 16 Nov 2020 18:21:59 +0100 Subject: [PATCH 4/8] Extract to hadoop-ozone/dist/target --- .github/workflows/post-commit.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml index 08036dc852db..e9bb4f0db864 100644 --- a/.github/workflows/post-commit.yml +++ b/.github/workflows/post-commit.yml @@ -195,7 +195,9 @@ jobs: with: name: ozone-bin - name: Untar binaries - run: tar xzvf hadoop-ozone*.tar.gz -C /mnt/ozone + run: | + mkdir -p /mnt/ozone/hadoop-ozone/dist/target + tar xzvf hadoop-ozone*.tar.gz -C /mnt/ozone/hadoop-ozone/dist/target - name: Install robotframework run: sudo pip install robotframework - name: Execute tests @@ -307,7 +309,9 @@ jobs: with: name: ozone-bin - name: Untar binaries - run: tar xvf ozone-bin.tar -C /mnt/ozone + run: | + mkdir -p /mnt/ozone/hadoop-ozone/dist/target + tar xzvf hadoop-ozone*.tar.gz -C /mnt/ozone/hadoop-ozone/dist/target - name: Install robotframework run: sudo pip install robotframework - name: Install k3s From d1ceb8ee96f9f4feb0cdf28e3d8010a18cbdc22a Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 23 Nov 2020 15:18:20 +0100 Subject: [PATCH 5/8] Minimize retention of ozone-bin --- .github/workflows/post-commit.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml index e9bb4f0db864..44d14c04b7e4 100644 --- a/.github/workflows/post-commit.yml +++ b/.github/workflows/post-commit.yml @@ -55,6 +55,7 @@ jobs: with: name: ozone-bin path: hadoop-ozone/dist/target/hadoop-ozone*.tar.gz + retention-days: 1 bats: runs-on: ubuntu-18.04 steps: From 65388cf613f5349fcc2f8f55cb548609c4888e45 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 23 Nov 2020 16:32:26 +0100 Subject: [PATCH 6/8] trigger new CI check From c1b7d485520603894ff69ba45ddc7ebbe4122c8a Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 23 Nov 2020 19:08:09 +0100 Subject: [PATCH 7/8] trigger new CI check From c886dc2117643be8ef74487f3659ed659f6e7706 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 23 Nov 2020 22:48:36 +0100 Subject: [PATCH 8/8] trigger new CI check