From 13906d82b4578e9d362a9151a956d8c99f56e62f Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Tue, 29 Oct 2024 17:25:34 +0000 Subject: [PATCH 1/9] When PR comes from a forked repo we also need to get the originating repo --- .github/workflows/pw_aws_ci.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pw_aws_ci.yaml b/.github/workflows/pw_aws_ci.yaml index 245e219dd44..c2f4627b53f 100644 --- a/.github/workflows/pw_aws_ci.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -31,24 +31,29 @@ env: MACHINE_ID: noaacloud jobs: + fetch-branch: runs-on: ubuntu-latest env: GH_TOKEN: ${{ secrets.GITHUBTOKEN }} outputs: branch: ${{ steps.get-branch.outputs.branch }} + repo: ${{ steps.get-branch.outputs.repo }} steps: - - name: Fetch branch name for PR + - name: Fetch branch name and repo for PR id: get-branch run: | pr_number=${{ github.event.inputs.pr_number }} repo=${{ github.repository }} if [ "$pr_number" -eq "0" ]; then branch=${{ github.event.inputs.ref }} + repo_url="https://github.com/${{ github.repository_owner }}/${{ github.repository }}.git" else branch=$(gh pr view $pr_number --repo $repo --json headRefName --jq '.headRefName') + repo_url=$(gh pr view $pr_number --repo $repo --json headRepository --jq '.headRepository.url') fi echo "::set-output name=branch::$branch" + echo "::set-output name=repo::$repo_url" checkout: needs: fetch-branch @@ -64,6 +69,7 @@ jobs: with: path: ${{ github.run_id }}/HOMEgfs submodules: 'recursive' + repository: ${{ needs.fetch-branch.outputs.repo }} ref: ${{ needs.fetch-branch.outputs.branch }} build-link: From 0edf759378ae968a318004ec6c2c6a16a9523d57 Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Wed, 30 Oct 2024 15:59:15 -0500 Subject: [PATCH 2/9] added section to auttomatically start up a cluster --- .github/workflows/pw_aws_ci.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/pw_aws_ci.yaml b/.github/workflows/pw_aws_ci.yaml index c2f4627b53f..84d57153c30 100644 --- a/.github/workflows/pw_aws_ci.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -32,7 +32,34 @@ env: jobs: + run-start-clusters: + runs-on: ubuntu-latest + steps: + - name: Checkout pw-cluster-automation repository + uses: actions/checkout@v4 + with: + repository: TerrenceMcGuinness-NOAA/pw-cluster-automation + path: pw-cluster-automation + + - name: Add SSH keys + run: | + mkdir -p ~/.ssh + echo "${{ secrets.ID_RSA_AWS }}" > ~/.ssh/id_rsa_aws + echo "${{ secrets.PW_API_KEY }}" > ~/.ssh/pw_api.key + chmod 600 ~/.ssh/id_rsa_aws + chmod 600 ~/.ssh/pw_api.key + + - name: Run startClusters.py + run: | + if [ "${{ github.event.inputs.os }}" == "rocky" ]; then + clustername="globalworkflowciplatformrocky8" + elif [ "${{ github.event.inputs.os }}" == "centos" ]; then + clustername="awsemctmcgc7i48xlargeciplatform" + fi + python3 pw-cluster-automation/startClusters.py $clustername + fetch-branch: + needs: run-start-clusters runs-on: ubuntu-latest env: GH_TOKEN: ${{ secrets.GITHUBTOKEN }} From 54c51324edb23081d42c6fbb4f3268e8254c531f Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Wed, 30 Oct 2024 17:07:47 -0500 Subject: [PATCH 3/9] added env var for pw url --- .github/workflows/pw_aws_ci.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pw_aws_ci.yaml b/.github/workflows/pw_aws_ci.yaml index 84d57153c30..a2baca00617 100644 --- a/.github/workflows/pw_aws_ci.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -34,6 +34,8 @@ jobs: run-start-clusters: runs-on: ubuntu-latest + env: + PW_PLATFORM_HOST=noaa.parallel.works steps: - name: Checkout pw-cluster-automation repository uses: actions/checkout@v4 @@ -49,7 +51,7 @@ jobs: chmod 600 ~/.ssh/id_rsa_aws chmod 600 ~/.ssh/pw_api.key - - name: Run startClusters.py + - name: Run startClusters run: | if [ "${{ github.event.inputs.os }}" == "rocky" ]; then clustername="globalworkflowciplatformrocky8" From fff0cfe3553c07e4a67a71f1d1f9ab0fbc89d449 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Wed, 30 Oct 2024 17:09:56 -0500 Subject: [PATCH 4/9] added env var for pw url correctly --- .github/workflows/pw_aws_ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pw_aws_ci.yaml b/.github/workflows/pw_aws_ci.yaml index a2baca00617..d07f2b3f0c1 100644 --- a/.github/workflows/pw_aws_ci.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -35,7 +35,7 @@ jobs: run-start-clusters: runs-on: ubuntu-latest env: - PW_PLATFORM_HOST=noaa.parallel.works + PW_PLATFORM_HOST: noaa.parallel.works steps: - name: Checkout pw-cluster-automation repository uses: actions/checkout@v4 From 47bb8af0a8ad9806076bcf6a7e9c697e204f3720 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 30 Oct 2024 18:48:38 -0400 Subject: [PATCH 5/9] Update pw_aws_ci.yaml chmod .ssh dir --- .github/workflows/pw_aws_ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pw_aws_ci.yaml b/.github/workflows/pw_aws_ci.yaml index d07f2b3f0c1..20f9150ecda 100644 --- a/.github/workflows/pw_aws_ci.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -48,6 +48,7 @@ jobs: mkdir -p ~/.ssh echo "${{ secrets.ID_RSA_AWS }}" > ~/.ssh/id_rsa_aws echo "${{ secrets.PW_API_KEY }}" > ~/.ssh/pw_api.key + chmod 700 ~/.ssh chmod 600 ~/.ssh/id_rsa_aws chmod 600 ~/.ssh/pw_api.key From da1b1a31ea3e21abe78047eec8068d9bef6717ad Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 31 Oct 2024 11:28:11 -0500 Subject: [PATCH 6/9] added branch to startClusters --- .github/workflows/pw_aws_ci.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pw_aws_ci.yaml b/.github/workflows/pw_aws_ci.yaml index d07f2b3f0c1..59b8763ff20 100644 --- a/.github/workflows/pw_aws_ci.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -42,17 +42,15 @@ jobs: with: repository: TerrenceMcGuinness-NOAA/pw-cluster-automation path: pw-cluster-automation + ref: pw_cluster_noaa - - name: Add SSH keys + - name: Run startClusters run: | mkdir -p ~/.ssh echo "${{ secrets.ID_RSA_AWS }}" > ~/.ssh/id_rsa_aws echo "${{ secrets.PW_API_KEY }}" > ~/.ssh/pw_api.key chmod 600 ~/.ssh/id_rsa_aws chmod 600 ~/.ssh/pw_api.key - - - name: Run startClusters - run: | if [ "${{ github.event.inputs.os }}" == "rocky" ]; then clustername="globalworkflowciplatformrocky8" elif [ "${{ github.event.inputs.os }}" == "centos" ]; then From 5b2569ff94134e3c6c28c904481a21073e1cee86 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 31 Oct 2024 12:09:38 -0500 Subject: [PATCH 7/9] debugging ssh command directly --- .github/workflows/pw_aws_ci.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pw_aws_ci.yaml b/.github/workflows/pw_aws_ci.yaml index 90fb01e60ea..a42cbd0d594 100644 --- a/.github/workflows/pw_aws_ci.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -52,6 +52,9 @@ jobs: chmod 700 ~/.ssh chmod 600 ~/.ssh/id_rsa_aws chmod 600 ~/.ssh/pw_api.key + ls -l ~/.ssh + ssh -v -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null Terry.McGuinness@107.22.156.241 sinfo + exit 0 if [ "${{ github.event.inputs.os }}" == "rocky" ]; then clustername="globalworkflowciplatformrocky8" elif [ "${{ github.event.inputs.os }}" == "centos" ]; then From 5814d292aafda89522e162564a8bd823470590ef Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 31 Oct 2024 12:14:08 -0500 Subject: [PATCH 8/9] looks like ssh in runner/home only checks stardard names for private keys --- .github/workflows/pw_aws_ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pw_aws_ci.yaml b/.github/workflows/pw_aws_ci.yaml index a42cbd0d594..310cbf30d0e 100644 --- a/.github/workflows/pw_aws_ci.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -47,10 +47,10 @@ jobs: - name: Run startClusters run: | mkdir -p ~/.ssh - echo "${{ secrets.ID_RSA_AWS }}" > ~/.ssh/id_rsa_aws + echo "${{ secrets.ID_RSA_AWS }}" > ~/.ssh/id_rsa echo "${{ secrets.PW_API_KEY }}" > ~/.ssh/pw_api.key chmod 700 ~/.ssh - chmod 600 ~/.ssh/id_rsa_aws + chmod 600 ~/.ssh/id_rsa chmod 600 ~/.ssh/pw_api.key ls -l ~/.ssh ssh -v -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null Terry.McGuinness@107.22.156.241 sinfo From 4ecb2497c13a42748df532c3f5c13a065ea8542b Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 31 Oct 2024 12:17:31 -0500 Subject: [PATCH 9/9] fixed issu with ssh auth in startClusters (removed debut statementns) --- .github/workflows/pw_aws_ci.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/pw_aws_ci.yaml b/.github/workflows/pw_aws_ci.yaml index 310cbf30d0e..c59f0279201 100644 --- a/.github/workflows/pw_aws_ci.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -52,9 +52,6 @@ jobs: chmod 700 ~/.ssh chmod 600 ~/.ssh/id_rsa chmod 600 ~/.ssh/pw_api.key - ls -l ~/.ssh - ssh -v -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null Terry.McGuinness@107.22.156.241 sinfo - exit 0 if [ "${{ github.event.inputs.os }}" == "rocky" ]; then clustername="globalworkflowciplatformrocky8" elif [ "${{ github.event.inputs.os }}" == "centos" ]; then