From c001aa8997892b16fd60e04b71f31275607fece9 Mon Sep 17 00:00:00 2001
From: Gautham Banasandra <gautham.bangalore@gmail.com>
Date: Tue, 13 Jul 2021 20:22:39 +0530
Subject: [PATCH 01/11] [Do not commit] Use exclusive src for each platform

* Need to ensure that each platform build stage
  runs on its own copy of the source directory.
  This is needed to parallellize the multi-platform
  build.
---
 dev-support/Jenkinsfile | 161 ++++++++++++++++++++++------------------
 1 file changed, 87 insertions(+), 74 deletions(-)

diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 6cabbfe7b54c4..9f3ca2b443aa7 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -56,86 +56,99 @@ pipeline {
             }
         }
 
-        // This is an optional stage which runs only when there's a change in
-        // C++/C++ build/platform.
-        // This stage serves as a means of cross platform validation, which is
-        // really needed to ensure that any C++ related/platform change doesn't
-        // break the Hadoop build on Centos 8.
-        stage ('precommit-run Centos 8') {
-            environment {
-                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
-                IS_OPTIONAL = 1
-            }
-
+        stage ('setup sources') {
             steps {
-                withCredentials(
-                    [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
-                                  passwordVariable: 'GITHUB_TOKEN',
-                                  usernameVariable: 'GITHUB_USER'),
-                    usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
-                                        passwordVariable: 'JIRA_PASSWORD',
-                                        usernameVariable: 'JIRA_USER')]) {
-                        sh '''#!/usr/bin/env bash
-
-                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                        "${SOURCEDIR}/dev-support/jenkins.sh"
-                        '''
-                }
-            }
-        }
+                dir("${WORKSPACE}/hadoop-centos-8") {
+                    sh '''#!/usr/bin/env bash
 
-        // This is an optional stage which runs only when there's a change in
-        // C++/C++ build/platform.
-        // This stage serves as a means of cross platform validation, which is
-        // really needed to ensure that any C++ related/platform change doesn't
-        // break the Hadoop build on Debian 10.
-        stage ('precommit-run Debian 10') {
-            environment {
-                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10"
-                IS_OPTIONAL = 1
-            }
-
-            steps {
-                withCredentials(
-                    [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
-                                  passwordVariable: 'GITHUB_TOKEN',
-                                  usernameVariable: 'GITHUB_USER'),
-                    usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
-                                        passwordVariable: 'JIRA_PASSWORD',
-                                        usernameVariable: 'JIRA_USER')]) {
-                        sh '''#!/usr/bin/env bash
-
-                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                        "${SOURCEDIR}/dev-support/jenkins.sh"
-                        '''
+                    pwd
+                    cp -Rp ${WORKSPACE}/${SOURCEDIR} ${WORKSPACE}/hadoop-centos-8
+                    ls -l
+                    '''
                 }
             }
         }
 
-        // We want to use Ubuntu Focal as our main CI and thus, this stage
-        // isn't optional (runs for all the PRs).
-        stage ('precommit-run Ubuntu focal') {
-            environment {
-                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
-                IS_OPTIONAL = 0
-            }
-
-            steps {
-                withCredentials(
-                    [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
-                                  passwordVariable: 'GITHUB_TOKEN',
-                                  usernameVariable: 'GITHUB_USER'),
-                    usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
-                                        passwordVariable: 'JIRA_PASSWORD',
-                                        usernameVariable: 'JIRA_USER')]) {
-                        sh '''#!/usr/bin/env bash
-
-                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                        "${SOURCEDIR}/dev-support/jenkins.sh"
-                        '''
-                }
-            }
-        }
+//         // This is an optional stage which runs only when there's a change in
+//         // C++/C++ build/platform.
+//         // This stage serves as a means of cross platform validation, which is
+//         // really needed to ensure that any C++ related/platform change doesn't
+//         // break the Hadoop build on Centos 8.
+//         stage ('precommit-run Centos 8') {
+//             environment {
+//                 DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
+//                 IS_OPTIONAL = 1
+//             }
+//
+//             steps {
+//                 withCredentials(
+//                     [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
+//                                   passwordVariable: 'GITHUB_TOKEN',
+//                                   usernameVariable: 'GITHUB_USER'),
+//                     usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
+//                                         passwordVariable: 'JIRA_PASSWORD',
+//                                         usernameVariable: 'JIRA_USER')]) {
+//                         sh '''#!/usr/bin/env bash
+//
+//                         chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+//                         "${SOURCEDIR}/dev-support/jenkins.sh"
+//                         '''
+//                 }
+//             }
+//         }
+//
+//         // This is an optional stage which runs only when there's a change in
+//         // C++/C++ build/platform.
+//         // This stage serves as a means of cross platform validation, which is
+//         // really needed to ensure that any C++ related/platform change doesn't
+//         // break the Hadoop build on Debian 10.
+//         stage ('precommit-run Debian 10') {
+//             environment {
+//                 DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10"
+//                 IS_OPTIONAL = 1
+//             }
+//
+//             steps {
+//                 withCredentials(
+//                     [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
+//                                   passwordVariable: 'GITHUB_TOKEN',
+//                                   usernameVariable: 'GITHUB_USER'),
+//                     usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
+//                                         passwordVariable: 'JIRA_PASSWORD',
+//                                         usernameVariable: 'JIRA_USER')]) {
+//                         sh '''#!/usr/bin/env bash
+//
+//                         chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+//                         "${SOURCEDIR}/dev-support/jenkins.sh"
+//                         '''
+//                 }
+//             }
+//         }
+//
+//         // We want to use Ubuntu Focal as our main CI and thus, this stage
+//         // isn't optional (runs for all the PRs).
+//         stage ('precommit-run Ubuntu focal') {
+//             environment {
+//                 DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
+//                 IS_OPTIONAL = 0
+//             }
+//
+//             steps {
+//                 withCredentials(
+//                     [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
+//                                   passwordVariable: 'GITHUB_TOKEN',
+//                                   usernameVariable: 'GITHUB_USER'),
+//                     usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
+//                                         passwordVariable: 'JIRA_PASSWORD',
+//                                         usernameVariable: 'JIRA_USER')]) {
+//                         sh '''#!/usr/bin/env bash
+//
+//                         chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+//                         "${SOURCEDIR}/dev-support/jenkins.sh"
+//                         '''
+//                 }
+//             }
+//         }
 
     }
 

From 9f0f007287b3dd55649dd409d0a745a484eb7945 Mon Sep 17 00:00:00 2001
From: Gautham Banasandra <gautham.bangalore@gmail.com>
Date: Tue, 13 Jul 2021 20:31:24 +0530
Subject: [PATCH 02/11] Show head

---
 dev-support/Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 9f3ca2b443aa7..1d2aba72759e3 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -63,7 +63,8 @@ pipeline {
 
                     pwd
                     cp -Rp ${WORKSPACE}/${SOURCEDIR} ${WORKSPACE}/hadoop-centos-8
-                    ls -l
+                    cd src
+                    git show HEAD
                     '''
                 }
             }

From 95d835ecda6749e527bcf6b199856ff6d0817178 Mon Sep 17 00:00:00 2001
From: Gautham Banasandra <gautham.bangalore@gmail.com>
Date: Tue, 13 Jul 2021 23:24:30 +0530
Subject: [PATCH 03/11] Setup sources

---
 dev-support/Jenkinsfile | 164 +++++++++++++++++++++++++---------------
 1 file changed, 105 insertions(+), 59 deletions(-)

diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 1d2aba72759e3..d430774439aeb 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -61,71 +61,117 @@ pipeline {
                 dir("${WORKSPACE}/hadoop-centos-8") {
                     sh '''#!/usr/bin/env bash
 
-                    pwd
                     cp -Rp ${WORKSPACE}/${SOURCEDIR} ${WORKSPACE}/hadoop-centos-8
-                    cd src
-                    git show HEAD
+                    '''
+                }
+
+                dir("${WORKSPACE}/debian-10") {
+                    sh '''#!/usr/bin/env bash
+
+                    cp -Rp ${WORKSPACE}/${SOURCEDIR} ${WORKSPACE}/debian-10
                     '''
                 }
             }
         }
 
-//         // This is an optional stage which runs only when there's a change in
-//         // C++/C++ build/platform.
-//         // This stage serves as a means of cross platform validation, which is
-//         // really needed to ensure that any C++ related/platform change doesn't
-//         // break the Hadoop build on Centos 8.
-//         stage ('precommit-run Centos 8') {
-//             environment {
-//                 DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
-//                 IS_OPTIONAL = 1
-//             }
-//
-//             steps {
-//                 withCredentials(
-//                     [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
-//                                   passwordVariable: 'GITHUB_TOKEN',
-//                                   usernameVariable: 'GITHUB_USER'),
-//                     usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
-//                                         passwordVariable: 'JIRA_PASSWORD',
-//                                         usernameVariable: 'JIRA_USER')]) {
-//                         sh '''#!/usr/bin/env bash
-//
-//                         chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-//                         "${SOURCEDIR}/dev-support/jenkins.sh"
-//                         '''
-//                 }
-//             }
-//         }
-//
-//         // This is an optional stage which runs only when there's a change in
-//         // C++/C++ build/platform.
-//         // This stage serves as a means of cross platform validation, which is
-//         // really needed to ensure that any C++ related/platform change doesn't
-//         // break the Hadoop build on Debian 10.
-//         stage ('precommit-run Debian 10') {
-//             environment {
-//                 DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10"
-//                 IS_OPTIONAL = 1
-//             }
-//
-//             steps {
-//                 withCredentials(
-//                     [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
-//                                   passwordVariable: 'GITHUB_TOKEN',
-//                                   usernameVariable: 'GITHUB_USER'),
-//                     usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
-//                                         passwordVariable: 'JIRA_PASSWORD',
-//                                         usernameVariable: 'JIRA_USER')]) {
-//                         sh '''#!/usr/bin/env bash
-//
-//                         chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-//                         "${SOURCEDIR}/dev-support/jenkins.sh"
-//                         '''
-//                 }
-//             }
-//         }
-//
+        // This is an optional stage which runs only when there's a change in
+        // C++/C++ build/platform.
+        // This stage serves as a means of cross platform validation, which is
+        // really needed to ensure that any C++ related/platform change doesn't
+        // break the Hadoop build on Centos 8.
+        stage ('precommit-run Centos 8') {
+            environment {
+                SOURCEDIR = "${WORKSPACE}/hadoop-centos-8/src"
+                PATCHDIR = "${WORKSPACE}/hadoop-centos-8/out"
+                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
+                IS_OPTIONAL = 1
+            }
+
+            steps {
+                withCredentials(
+                    [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
+                                  passwordVariable: 'GITHUB_TOKEN',
+                                  usernameVariable: 'GITHUB_USER'),
+                    usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
+                                        passwordVariable: 'JIRA_PASSWORD',
+                                        usernameVariable: 'JIRA_USER')]) {
+                        sh '''#!/usr/bin/env bash
+
+                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+                        "${SOURCEDIR}/dev-support/jenkins.sh"
+                        '''
+                }
+            }
+
+            post {
+                cleanup() {
+                    script {
+                        sh '''
+                            # See YETUS-764
+                            if [ -f "${PATCHDIR}/pidfile.txt" ]; then
+                              echo "test-patch process appears to still be running: killing"
+                              kill `cat "${PATCHDIR}/pidfile.txt"` || true
+                              sleep 10
+                            fi
+                            if [ -f "${PATCHDIR}/cidfile.txt" ]; then
+                              echo "test-patch container appears to still be running: killing"
+                              docker kill `cat "/${PATCHDIR}/cidfile.txt"` || true
+                            fi
+                            '''
+                    }
+                }
+            }
+        }
+
+        // This is an optional stage which runs only when there's a change in
+        // C++/C++ build/platform.
+        // This stage serves as a means of cross platform validation, which is
+        // really needed to ensure that any C++ related/platform change doesn't
+        // break the Hadoop build on Debian 10.
+        stage ('precommit-run Debian 10') {
+            environment {
+                SOURCEDIR = "${WORKSPACE}/debian-10/src"
+                PATCHDIR = "${WORKSPACE}/debian-10/out"
+                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10"
+                IS_OPTIONAL = 1
+            }
+
+            steps {
+                withCredentials(
+                    [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
+                                  passwordVariable: 'GITHUB_TOKEN',
+                                  usernameVariable: 'GITHUB_USER'),
+                    usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
+                                        passwordVariable: 'JIRA_PASSWORD',
+                                        usernameVariable: 'JIRA_USER')]) {
+                        sh '''#!/usr/bin/env bash
+
+                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+                        "${SOURCEDIR}/dev-support/jenkins.sh"
+                        '''
+                }
+            }
+
+            post {
+                cleanup() {
+                    script {
+                        sh '''
+                            # See YETUS-764
+                            if [ -f "${PATCHDIR}/pidfile.txt" ]; then
+                              echo "test-patch process appears to still be running: killing"
+                              kill `cat "${PATCHDIR}/pidfile.txt"` || true
+                              sleep 10
+                            fi
+                            if [ -f "${PATCHDIR}/cidfile.txt" ]; then
+                              echo "test-patch container appears to still be running: killing"
+                              docker kill `cat "/${PATCHDIR}/cidfile.txt"` || true
+                            fi
+                            '''
+                    }
+                }
+            }
+        }
+
 //         // We want to use Ubuntu Focal as our main CI and thus, this stage
 //         // isn't optional (runs for all the PRs).
 //         stage ('precommit-run Ubuntu focal') {

From dd351e7517bf08762db71c9a43a325f97b28a5a8 Mon Sep 17 00:00:00 2001
From: Gautham Banasandra <gautham.bangalore@gmail.com>
Date: Tue, 13 Jul 2021 23:32:09 +0530
Subject: [PATCH 04/11] Trigger CI

---
 .../src/main/native/libhdfspp/tests/configuration_test.cc        | 1 -
 1 file changed, 1 deletion(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
index 3bf2524354b6b..e173d87f351be 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
@@ -28,7 +28,6 @@ using ::testing::_;
 using namespace hdfs;
 
 namespace hdfs {
-
 TEST(ConfigurationTest, TestDegenerateInputs) {
   /* Completely empty stream */
   {

From 13156d0e8bf4447ebb56e10229a1dd867d7e2e24 Mon Sep 17 00:00:00 2001
From: Gautham Banasandra <gautham.bangalore@gmail.com>
Date: Wed, 14 Jul 2021 19:55:39 +0530
Subject: [PATCH 05/11] Set basedir correctly

---
 dev-support/Jenkinsfile |  4 ++--
 dev-support/jenkins.sh  | 20 ++++++++++----------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index d430774439aeb..ee2321bb5a54e 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -115,7 +115,7 @@ pipeline {
                             fi
                             if [ -f "${PATCHDIR}/cidfile.txt" ]; then
                               echo "test-patch container appears to still be running: killing"
-                              docker kill `cat "/${PATCHDIR}/cidfile.txt"` || true
+                              docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
                             fi
                             '''
                     }
@@ -164,7 +164,7 @@ pipeline {
                             fi
                             if [ -f "${PATCHDIR}/cidfile.txt" ]; then
                               echo "test-patch container appears to still be running: killing"
-                              docker kill `cat "/${PATCHDIR}/cidfile.txt"` || true
+                              docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
                             fi
                             '''
                     }
diff --git a/dev-support/jenkins.sh b/dev-support/jenkins.sh
index 09905dd320cb7..e9eb32b027083 100644
--- a/dev-support/jenkins.sh
+++ b/dev-support/jenkins.sh
@@ -115,10 +115,10 @@ function run_ci() {
   TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
 
   # this must be clean for every run
-  if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then
-    rm -rf "${WORKSPACE:?}/${PATCHDIR}"
+  if [[ -d "${PATCHDIR}" ]]; then
+    rm -rf "${PATCHDIR:?}"
   fi
-  mkdir -p "${WORKSPACE}/${PATCHDIR}"
+  mkdir -p "${PATCHDIR}"
 
   # if given a JIRA issue, process it. If CHANGE_URL is set
   # (e.g., Github Branch Source plugin), process it.
@@ -128,23 +128,23 @@ function run_ci() {
   if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
     YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
   elif [[ -z "${CHANGE_URL}" ]]; then
-    echo "Full build skipped" >"${WORKSPACE}/${PATCHDIR}/report.html"
+    echo "Full build skipped" >"${PATCHDIR}/report.html"
     exit 0
   fi
 
-  YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
+  YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
 
   # where the source is located
-  YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
+  YETUS_ARGS+=("--basedir=${SOURCEDIR}")
 
   # our project defaults come from a personality file
   YETUS_ARGS+=("--project=hadoop")
-  YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/hadoop.sh")
+  YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/bin/hadoop.sh")
 
   # lots of different output formats
-  YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
-  YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
-  YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
+  YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt")
+  YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
+  YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")
 
   # enable writing back to Github
   YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")

From 4980188aeb1309ff49d636a486a6b173303dd790 Mon Sep 17 00:00:00 2001
From: Gautham Banasandra <gautham.bangalore@gmail.com>
Date: Wed, 14 Jul 2021 23:44:12 +0530
Subject: [PATCH 06/11] Add clean up and github recovery

---
 dev-support/Jenkinsfile | 195 +++++++++++++++++++---------------------
 dev-support/jenkins.sh  |  45 +++++++++-
 2 files changed, 134 insertions(+), 106 deletions(-)

diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index ee2321bb5a54e..1d55dfc2e9dba 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -29,9 +29,6 @@ pipeline {
     }
 
     environment {
-        SOURCEDIR = 'src'
-        // will also need to change notification section below
-        PATCHDIR = 'out'
         YETUS='yetus'
         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
         YETUS_VERSION='f9ba0170a5787a5f4662d3769804fef0226a182f'
@@ -61,14 +58,21 @@ pipeline {
                 dir("${WORKSPACE}/hadoop-centos-8") {
                     sh '''#!/usr/bin/env bash
 
-                    cp -Rp ${WORKSPACE}/${SOURCEDIR} ${WORKSPACE}/hadoop-centos-8
+                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/hadoop-centos-8
                     '''
                 }
 
                 dir("${WORKSPACE}/debian-10") {
                     sh '''#!/usr/bin/env bash
 
-                    cp -Rp ${WORKSPACE}/${SOURCEDIR} ${WORKSPACE}/debian-10
+                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-10
+                    '''
+                }
+
+                dir("${WORKSPACE}/ubuntu-focal") {
+                    sh '''#!/usr/bin/env bash
+
+                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/ubuntu-focal
                     '''
                 }
             }
@@ -98,7 +102,7 @@ pipeline {
                         sh '''#!/usr/bin/env bash
 
                         chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                        "${SOURCEDIR}/dev-support/jenkins.sh"
+                        "${SOURCEDIR}/dev-support/jenkins.sh" run_ci
                         '''
                 }
             }
@@ -106,18 +110,10 @@ pipeline {
             post {
                 cleanup() {
                     script {
-                        sh '''
-                            # See YETUS-764
-                            if [ -f "${PATCHDIR}/pidfile.txt" ]; then
-                              echo "test-patch process appears to still be running: killing"
-                              kill `cat "${PATCHDIR}/pidfile.txt"` || true
-                              sleep 10
-                            fi
-                            if [ -f "${PATCHDIR}/cidfile.txt" ]; then
-                              echo "test-patch container appears to still be running: killing"
-                              docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
-                            fi
-                            '''
+                        sh '''#!/usr/bin/env bash
+
+                        "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
+                        '''
                     }
                 }
             }
@@ -147,7 +143,7 @@ pipeline {
                         sh '''#!/usr/bin/env bash
 
                         chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                        "${SOURCEDIR}/dev-support/jenkins.sh"
+                        "${SOURCEDIR}/dev-support/jenkins.sh" run_ci
                         '''
                 }
             }
@@ -155,105 +151,98 @@ pipeline {
             post {
                 cleanup() {
                     script {
-                        sh '''
-                            # See YETUS-764
-                            if [ -f "${PATCHDIR}/pidfile.txt" ]; then
-                              echo "test-patch process appears to still be running: killing"
-                              kill `cat "${PATCHDIR}/pidfile.txt"` || true
-                              sleep 10
-                            fi
-                            if [ -f "${PATCHDIR}/cidfile.txt" ]; then
-                              echo "test-patch container appears to still be running: killing"
-                              docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
-                            fi
-                            '''
+                        sh '''#!/usr/bin/env bash
+
+                        "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
+                        '''
                     }
                 }
             }
         }
 
-//         // We want to use Ubuntu Focal as our main CI and thus, this stage
-//         // isn't optional (runs for all the PRs).
-//         stage ('precommit-run Ubuntu focal') {
-//             environment {
-//                 DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
-//                 IS_OPTIONAL = 0
-//             }
-//
-//             steps {
-//                 withCredentials(
-//                     [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
-//                                   passwordVariable: 'GITHUB_TOKEN',
-//                                   usernameVariable: 'GITHUB_USER'),
-//                     usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
-//                                         passwordVariable: 'JIRA_PASSWORD',
-//                                         usernameVariable: 'JIRA_USER')]) {
-//                         sh '''#!/usr/bin/env bash
-//
-//                         chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-//                         "${SOURCEDIR}/dev-support/jenkins.sh"
-//                         '''
-//                 }
-//             }
-//         }
-
-    }
+        // We want to use Ubuntu Focal as our main CI and thus, this stage
+        // isn't optional (runs for all the PRs).
+        stage ('precommit-run Ubuntu focal') {
+            environment {
+                SOURCEDIR = "${WORKSPACE}/ubuntu-focal/src"
+                PATCHDIR = "${WORKSPACE}/ubuntu-focal/out"
+                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
+                IS_OPTIONAL = 0
+            }
 
-    post {
-        always {
-          script {
-            // Publish status if it was missed (YETUS-1059)
-            withCredentials(
-                [usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd',
+            steps {
+                withCredentials(
+                    [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
                                   passwordVariable: 'GITHUB_TOKEN',
-                                  usernameVariable: 'GITHUB_USER')]) {
-                sh '''#!/usr/bin/env bash
-                    YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
-                    YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
-                    TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/github-status-recovery.sh"
-                    /usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" ${EXTRA_ARGS} || true
-                    '''
+                                  usernameVariable: 'GITHUB_USER'),
+                    usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
+                                        passwordVariable: 'JIRA_PASSWORD',
+                                        usernameVariable: 'JIRA_USER')]) {
+                        sh '''#!/usr/bin/env bash
+
+                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+                        "${SOURCEDIR}/dev-support/jenkins.sh" run_ci
+                        '''
+                }
             }
 
-            // Yetus output
-            archiveArtifacts "${env.PATCHDIR}/**"
-            // Publish the HTML report so that it can be looked at
-            // Has to be relative to WORKSPACE.
-            publishHTML (target: [
-                          allowMissing: true,
-                          keepAll: true,
-                          alwaysLinkToLastBuild: true,
-                          // Has to be relative to WORKSPACE
-                          reportDir: "${env.PATCHDIR}",
-                          reportFiles: 'report.html',
-                          reportName: 'Yetus Report'
-            ])
-            // Publish JUnit results
-            try {
-                junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml"
-            } catch(e) {
-                echo 'junit processing: ' + e.toString()
+            post {
+                always {
+                    script {
+                        // Publish status if it was missed (YETUS-1059)
+                        withCredentials(
+                            [usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd',
+                                              passwordVariable: 'GITHUB_TOKEN',
+                                              usernameVariable: 'GITHUB_USER')]) {
+                            sh '''#!/usr/bin/env bash
+
+                            "${SOURCEDIR}/dev-support/jenkins.sh" github_status_recovery
+                            '''
+                        }
+
+                        // Yetus output
+                        archiveArtifacts "${PATCHDIR}/**"
+                        // Publish the HTML report so that it can be looked at
+                        // Has to be relative to WORKSPACE.
+                        publishHTML (target: [
+                                      allowMissing: true,
+                                      keepAll: true,
+                                      alwaysLinkToLastBuild: true,
+                                      // Has to be relative to WORKSPACE
+                                      reportDir: "ubuntu-focal/out",
+                                      reportFiles: 'report.html',
+                                      reportName: 'Yetus Report'
+                        ])
+                        // Publish JUnit results
+                        try {
+                            junit "${SOURCEDIR}/**/target/surefire-reports/*.xml"
+                        } catch(e) {
+                            echo 'junit processing: ' + e.toString()
+                        }
+                    }
+                }
+
+                cleanup() {
+                    script {
+                        sh '''#!/usr/bin/env bash
+
+                        "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
+                        '''
+                    }
+                }
             }
-          }
         }
+    }
 
+    post {
         // Jenkins pipeline jobs fill slaves on PRs without this :(
         cleanup() {
             script {
-                sh '''
-                    # See YETUS-764
-                    if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
-                      echo "test-patch process appears to still be running: killing"
-                      kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
-                      sleep 10
-                    fi
-                    if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
-                      echo "test-patch container appears to still be running: killing"
-                      docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true
-                    fi
-                    # See HADOOP-13951
-                    chmod -R u+rxw "${WORKSPACE}"
-                    '''
+                sh '''#!/usr/bin/env bash
+
+                # See HADOOP-13951
+                chmod -R u+rxw "${WORKSPACE}"
+                '''
                 deleteDir()
             }
         }
diff --git a/dev-support/jenkins.sh b/dev-support/jenkins.sh
index e9eb32b027083..20fd963626df3 100644
--- a/dev-support/jenkins.sh
+++ b/dev-support/jenkins.sh
@@ -206,7 +206,46 @@ function run_ci() {
   "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
 }
 
-# Check if the CI needs to be run, if so, do so :)
-if check_ci_run; then
-  run_ci
+function cleanup_ci_proc() {
+  # See YETUS-764
+  if [ -f "${PATCHDIR}/pidfile.txt" ]; then
+    echo "test-patch process appears to still be running: killing"
+    kill "$(cat "${PATCHDIR}/pidfile.txt")" || true
+    sleep 10
+  fi
+  if [ -f "${PATCHDIR}/cidfile.txt" ]; then
+    echo "test-patch container appears to still be running: killing"
+    docker kill "$(cat "${PATCHDIR}/cidfile.txt")" || true
+  fi
+}
+
+function github_status_recovery() {
+  YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
+  YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
+  TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/github-status-recovery.sh"
+  /usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" "${EXTRA_ARGS}" || true
+}
+
+if [ -z "$1" ]; then
+  echo "Must specify an argument for jenkins.sh"
+  echo "run_ci          - Runs the CI based on platform image as defined by DOCKERFILE"
+  echo "cleanup_ci_proc - Cleans up the processes spawned for running the CI"
+  exit 1
+fi
+
+# Process arguments to jenkins.sh
+if [ "$1" == "run_ci" ]; then
+  # Check if the CI needs to be run, if so, do so :)
+  if check_ci_run; then
+    run_ci
+  else
+    echo "No C++ file/C++ build/platform changes found, will not run CI"
+  fi
+elif [ "$1" == "cleanup_ci_proc" ]; then
+  cleanup_ci_proc
+elif [ "$1" == "github_status_recovery" ]; then
+  github_status_recovery
+else
+  echo "Don't know how to process $1"
+  exit 1
 fi

From 27609cc7c43fa6898f71cd10e88d2fb7f95346e0 Mon Sep 17 00:00:00 2001
From: Gautham B A <gautham.bangalore@gmail.com>
Date: Thu, 15 Jul 2021 10:21:41 +0530
Subject: [PATCH 07/11] Add note for github status

---
 dev-support/jenkins.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/dev-support/jenkins.sh b/dev-support/jenkins.sh
index 20fd963626df3..5a9ae1f0d296f 100644
--- a/dev-support/jenkins.sh
+++ b/dev-support/jenkins.sh
@@ -228,8 +228,9 @@ function github_status_recovery() {
 
 if [ -z "$1" ]; then
   echo "Must specify an argument for jenkins.sh"
-  echo "run_ci          - Runs the CI based on platform image as defined by DOCKERFILE"
-  echo "cleanup_ci_proc - Cleans up the processes spawned for running the CI"
+  echo "run_ci                  - Runs the CI based on platform image as defined by DOCKERFILE"
+  echo "cleanup_ci_proc         - Cleans up the processes spawned for running the CI"
+  echo "github_status_recovery  - Sends Github status"
   exit 1
 fi
 

From 5a238e0aada1e409a95d1f972cdb1f4010bfbab6 Mon Sep 17 00:00:00 2001
From: Gautham B A <gautham.bangalore@gmail.com>
Date: Thu, 15 Jul 2021 10:50:24 +0530
Subject: [PATCH 08/11] Trigger CI

---
 .../src/main/native/libhdfspp/tests/configuration_test.cc        | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
index e173d87f351be..912b8f078827d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
@@ -23,6 +23,7 @@
 #include <cstdio>
 #include <fstream>
 
+
 using ::testing::_;
 
 using namespace hdfs;

From c3db91650b4b5789c27d938ab838bafb5866b3ec Mon Sep 17 00:00:00 2001
From: Gautham Banasandra <gautham.bangalore@gmail.com>
Date: Thu, 15 Jul 2021 18:07:05 +0530
Subject: [PATCH 09/11] Set permissions for jenkins.sh

---
 dev-support/Jenkinsfile | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 1d55dfc2e9dba..27a26b0fcd65d 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -112,6 +112,7 @@ pipeline {
                     script {
                         sh '''#!/usr/bin/env bash
 
+                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
                         "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
                         '''
                     }
@@ -153,6 +154,7 @@ pipeline {
                     script {
                         sh '''#!/usr/bin/env bash
 
+                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
                         "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
                         '''
                     }
@@ -196,6 +198,7 @@ pipeline {
                                               usernameVariable: 'GITHUB_USER')]) {
                             sh '''#!/usr/bin/env bash
 
+                            chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
                             "${SOURCEDIR}/dev-support/jenkins.sh" github_status_recovery
                             '''
                         }
@@ -226,6 +229,7 @@ pipeline {
                     script {
                         sh '''#!/usr/bin/env bash
 
+                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
                         "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
                         '''
                     }

From ba2a2c6d650b755bd5ad86227a9c84f4a3e14fb9 Mon Sep 17 00:00:00 2001
From: Gautham B A <gautham.bangalore@gmail.com>
Date: Thu, 15 Jul 2021 21:13:24 +0530
Subject: [PATCH 10/11] Trigger CI

---
 .../src/main/native/libhdfspp/tests/configuration_test.cc        | 1 -
 1 file changed, 1 deletion(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
index 912b8f078827d..e173d87f351be 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
@@ -23,7 +23,6 @@
 #include <cstdio>
 #include <fstream>
 
-
 using ::testing::_;
 
 using namespace hdfs;

From 53654661ce216681729b4b85727bded170a761a0 Mon Sep 17 00:00:00 2001
From: Gautham B A <gautham.bangalore@gmail.com>
Date: Fri, 16 Jul 2021 11:01:35 +0530
Subject: [PATCH 11/11] Use relative path for archiving artifacts

---
 dev-support/Jenkinsfile | 168 ++++++++++++++++++++--------------------
 1 file changed, 84 insertions(+), 84 deletions(-)

diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 27a26b0fcd65d..aef732dd442be 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -78,89 +78,89 @@ pipeline {
             }
         }
 
-        // This is an optional stage which runs only when there's a change in
-        // C++/C++ build/platform.
-        // This stage serves as a means of cross platform validation, which is
-        // really needed to ensure that any C++ related/platform change doesn't
-        // break the Hadoop build on Centos 8.
-        stage ('precommit-run Centos 8') {
-            environment {
-                SOURCEDIR = "${WORKSPACE}/hadoop-centos-8/src"
-                PATCHDIR = "${WORKSPACE}/hadoop-centos-8/out"
-                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
-                IS_OPTIONAL = 1
-            }
-
-            steps {
-                withCredentials(
-                    [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
-                                  passwordVariable: 'GITHUB_TOKEN',
-                                  usernameVariable: 'GITHUB_USER'),
-                    usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
-                                        passwordVariable: 'JIRA_PASSWORD',
-                                        usernameVariable: 'JIRA_USER')]) {
-                        sh '''#!/usr/bin/env bash
-
-                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                        "${SOURCEDIR}/dev-support/jenkins.sh" run_ci
-                        '''
-                }
-            }
-
-            post {
-                cleanup() {
-                    script {
-                        sh '''#!/usr/bin/env bash
-
-                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                        "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
-                        '''
-                    }
-                }
-            }
-        }
-
-        // This is an optional stage which runs only when there's a change in
-        // C++/C++ build/platform.
-        // This stage serves as a means of cross platform validation, which is
-        // really needed to ensure that any C++ related/platform change doesn't
-        // break the Hadoop build on Debian 10.
-        stage ('precommit-run Debian 10') {
-            environment {
-                SOURCEDIR = "${WORKSPACE}/debian-10/src"
-                PATCHDIR = "${WORKSPACE}/debian-10/out"
-                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10"
-                IS_OPTIONAL = 1
-            }
-
-            steps {
-                withCredentials(
-                    [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
-                                  passwordVariable: 'GITHUB_TOKEN',
-                                  usernameVariable: 'GITHUB_USER'),
-                    usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
-                                        passwordVariable: 'JIRA_PASSWORD',
-                                        usernameVariable: 'JIRA_USER')]) {
-                        sh '''#!/usr/bin/env bash
-
-                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                        "${SOURCEDIR}/dev-support/jenkins.sh" run_ci
-                        '''
-                }
-            }
-
-            post {
-                cleanup() {
-                    script {
-                        sh '''#!/usr/bin/env bash
-
-                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                        "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
-                        '''
-                    }
-                }
-            }
-        }
+//         // This is an optional stage which runs only when there's a change in
+//         // C++/C++ build/platform.
+//         // This stage serves as a means of cross platform validation, which is
+//         // really needed to ensure that any C++ related/platform change doesn't
+//         // break the Hadoop build on Centos 8.
+//         stage ('precommit-run Centos 8') {
+//             environment {
+//                 SOURCEDIR = "${WORKSPACE}/hadoop-centos-8/src"
+//                 PATCHDIR = "${WORKSPACE}/hadoop-centos-8/out"
+//                 DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
+//                 IS_OPTIONAL = 1
+//             }
+
+//             steps {
+//                 withCredentials(
+//                     [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
+//                                   passwordVariable: 'GITHUB_TOKEN',
+//                                   usernameVariable: 'GITHUB_USER'),
+//                     usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
+//                                         passwordVariable: 'JIRA_PASSWORD',
+//                                         usernameVariable: 'JIRA_USER')]) {
+//                         sh '''#!/usr/bin/env bash
+
+//                         chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+//                         "${SOURCEDIR}/dev-support/jenkins.sh" run_ci
+//                         '''
+//                 }
+//             }
+
+//             post {
+//                 cleanup() {
+//                     script {
+//                         sh '''#!/usr/bin/env bash
+
+//                         chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+//                         "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
+//                         '''
+//                     }
+//                 }
+//             }
+//         }
+
+//         // This is an optional stage which runs only when there's a change in
+//         // C++/C++ build/platform.
+//         // This stage serves as a means of cross platform validation, which is
+//         // really needed to ensure that any C++ related/platform change doesn't
+//         // break the Hadoop build on Debian 10.
+//         stage ('precommit-run Debian 10') {
+//             environment {
+//                 SOURCEDIR = "${WORKSPACE}/debian-10/src"
+//                 PATCHDIR = "${WORKSPACE}/debian-10/out"
+//                 DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10"
+//                 IS_OPTIONAL = 1
+//             }
+
+//             steps {
+//                 withCredentials(
+//                     [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
+//                                   passwordVariable: 'GITHUB_TOKEN',
+//                                   usernameVariable: 'GITHUB_USER'),
+//                     usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
+//                                         passwordVariable: 'JIRA_PASSWORD',
+//                                         usernameVariable: 'JIRA_USER')]) {
+//                         sh '''#!/usr/bin/env bash
+
+//                         chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+//                         "${SOURCEDIR}/dev-support/jenkins.sh" run_ci
+//                         '''
+//                 }
+//             }
+
+//             post {
+//                 cleanup() {
+//                     script {
+//                         sh '''#!/usr/bin/env bash
+
+//                         chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+//                         "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
+//                         '''
+//                     }
+//                 }
+//             }
+//         }
 
         // We want to use Ubuntu Focal as our main CI and thus, this stage
         // isn't optional (runs for all the PRs).
@@ -204,7 +204,7 @@ pipeline {
                         }
 
                         // Yetus output
-                        archiveArtifacts "${PATCHDIR}/**"
+                        archiveArtifacts "ubuntu-focal/out/**"
                         // Publish the HTML report so that it can be looked at
                         // Has to be relative to WORKSPACE.
                         publishHTML (target: [