HDDS-11057. Enable reproducible builds

.github/workflows/ci.yml

@@ -369,6 +369,57 @@ jobs:
           name: ${{ github.job }}
           path: target/${{ github.job }}
         continue-on-error: true
+  repro:
+    needs:
+      - build-info
+      - build
+    runs-on: ubuntu-20.04
+    timeout-minutes: 30
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v4
+      - name: Cache for maven dependencies
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            maven-repo-
+      - name: Download Ozone repo
+        id: download-ozone-repo
+        uses: actions/download-artifact@v4
+        with:
+          name: ozone-repo
+          path: |
+            ~/.m2/repository/org/apache/ozone
+      - name: Download Ratis repo
+        if: ${{ inputs.ratis_args != '' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ratis-jars
+          path: |
+            ~/.m2/repository/org/apache/ratis
+      - name: Setup java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 8
+      - name: Execute tests
+        run: |
+          hadoop-ozone/dev-support/checks/${{ github.job }}.sh
+        continue-on-error: true
+      - name: Summary of failures
+        run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
+        if: ${{ !cancelled() }}
+      - name: Archive build results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: ${{ github.job }}
+          path: target/${{ github.job }}
+        continue-on-error: true
   acceptance:
     needs:
       - build-info

hadoop-hdds/common/src/main/resources/hdds-version-info.properties

@@ -19,8 +19,6 @@
 version=${declared.hdds.version}
 revision=${version-info.scm.commit}
 branch=${version-info.scm.branch}
-user=${user.name}
-date=${version-info.build.time}
 url=${version-info.scm.uri}
 srcChecksum=${version-info.source.md5}
 hadoopProtoc2Version=${proto2.hadooprpc.protobuf.version}

hadoop-ozone/common/src/main/resources/ozone-version-info.properties

@@ -20,8 +20,6 @@ version=${declared.ozone.version}
 release=${ozone.release}
 revision=${version-info.scm.commit}
 branch=${version-info.scm.branch}
-user=${user.name}
-date=${version-info.build.time}
 url=${version-info.scm.uri}
 srcChecksum=${version-info.source.md5}
 hadoopProtoc2Version=${proto2.hadooprpc.protobuf.version}

hadoop-ozone/dev-support/checks/_build.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
+: ${OZONE_WITH_COVERAGE:="false"}
+
+MAVEN_OPTIONS='-V -B -Dmaven.javadoc.skip=true -DskipTests -DskipDocs --no-transfer-progress'
+
+if [[ "${OZONE_WITH_COVERAGE}" == "true" ]]; then
+  MAVEN_OPTIONS="${MAVEN_OPTIONS} -Pcoverage"
+else
+  MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip"
+fi
+
+export MAVEN_OPTS="-Xmx4096m $MAVEN_OPTS"
+mvn ${MAVEN_OPTIONS} clean "$@"

hadoop-ozone/dev-support/checks/build.sh

@@ -13,20 +13,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR/../../.." || exit 1
-
-: ${OZONE_WITH_COVERAGE:="false"}
-
-MAVEN_OPTIONS='-V -B -Dmaven.javadoc.skip=true -DskipTests -DskipDocs --no-transfer-progress'
 
-if [[ "${OZONE_WITH_COVERAGE}" == "true" ]]; then
-  MAVEN_OPTIONS="${MAVEN_OPTIONS} -Pcoverage"
-else
-  MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip"
-fi
-
-export MAVEN_OPTS="-Xmx4096m $MAVEN_OPTS"
-echo "${MAVEN_OPTIONS}"
-mvn ${MAVEN_OPTIONS} clean install "$@"
-exit $?
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source "${DIR}"/_build.sh install "$@"

hadoop-ozone/dev-support/checks/repro.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This check verifies build reproducibility.
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
+BASE_DIR="$(pwd -P)"
+REPORT_DIR=${OUTPUT_DIR:-"${BASE_DIR}/target/repro"}
+
+source "${DIR}"/_build.sh verify artifact:compare "$@" | tee "${REPORT_DIR}/output.log"
+
+mkdir -p "$REPORT_DIR"
+REPORT_FILE="$REPORT_DIR/summary.txt"
+grep 'ERROR.*mismatch' "${REPORT_DIR}/output.log" > "${REPORT_FILE}"
+
+wc -l "${REPORT_FILE}" | awk '{ print $1 }' > "${REPORT_DIR}/failures"
+
+if [[ -s "${REPORT_FILE}" ]]; then
+   exit 1
+fi

hadoop-ozone/httpfsgateway/src/main/resources/httpfs.properties

@@ -16,6 +16,3 @@ httpfs.version=${project.version}
 
 httpfs.source.repository=${httpfs.source.repository}
 httpfs.source.revision=${httpfs.source.revision}
-
-httpfs.build.username=${user.name}
-httpfs.build.timestamp=${httpfs.build.timestamp}

pom.xml

@@ -62,6 +62,9 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
   </organization>
 
   <properties>
+    <!-- Enable Reproducible Builds mode -->
+    <project.build.outputTimestamp>2023-01-01T00:00:00Z</project.build.outputTimestamp>
+
     <hadoop2.version>2.10.2</hadoop2.version>
     <hadoop.version>3.3.6</hadoop.version>
 
@@ -257,7 +260,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
     <maven-pdf-plugin.version>1.6.1</maven-pdf-plugin.version>
     <maven-remote-resources-plugin.version>1.7.0</maven-remote-resources-plugin.version>
     <maven-enforcer-plugin.version>3.5.0</maven-enforcer-plugin.version>
-    <maven-javadoc-plugin.version>3.0.1</maven-javadoc-plugin.version>
+    <maven-javadoc-plugin.version>3.7.0</maven-javadoc-plugin.version>
     <maven-assembly-plugin.version>3.7.1</maven-assembly-plugin.version>
     <apache-rat-plugin.version>0.16.1</apache-rat-plugin.version>
     <maven-deploy-plugin.version>3.1.2</maven-deploy-plugin.version>
@@ -1341,6 +1344,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
           <version>${maven-javadoc-plugin.version}</version>
           <configuration>
             <additionalJOption>-Xdoclint:none</additionalJOption>
+            <notimestamp>true</notimestamp>
           </configuration>
         </plugin>
         <plugin>