HDDS-11057. Enable reproducible builds

.github/workflows/ci.yml

@@ -153,6 +153,7 @@ jobs:
           retention-days: 1
       - name: Store source tarball for compilation
         uses: actions/upload-artifact@v4
+        if: needs.build-info.outputs.needs-compile == 'true'
         with:
           name: ozone-src
           path: hadoop-ozone/dist/target/ozone-*-src.tar.gz
@@ -405,6 +406,66 @@ jobs:
           name: ${{ github.job }}
           path: target/${{ github.job }}
         continue-on-error: true
+  repro:
+    needs:
+      - build-info
+      - build
+    runs-on: ubuntu-20.04
+    timeout-minutes: 30
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v4
+      - name: Cache for maven dependencies
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            maven-repo-
+      - name: Download Ozone repo
+        id: download-ozone-repo
+        uses: actions/download-artifact@v4
+        with:
+          name: ozone-repo
+          path: |
+            ~/.m2/repository/org/apache/ozone
+      - name: Download Ratis repo
+        if: ${{ inputs.ratis_args != '' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ratis-jars
+          path: |
+            ~/.m2/repository/org/apache/ratis
+      - name: Setup java ${{ env.TEST_JAVA_VERSION }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: ${{ env.TEST_JAVA_VERSION }}
+      - name: Execute tests
+        run: |
+          hadoop-ozone/dev-support/checks/${{ github.job }}.sh -Pdist -Psrc -Dmaven.javadoc.skip=true ${{ inputs.ratis_args }}
+        continue-on-error: true
+      - name: Summary of failures
+        run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
+        if: ${{ !cancelled() }}
+      - name: Install diffoscope
+        run: |
+          sudo apt update -q
+          sudo apt install -y diffoscope
+        if: ${{ failure() }}
+      - name: Check artifact differences
+        run: |
+          hadoop-ozone/dev-support/checks/_diffoscope.sh
+        if: ${{ failure() }}
+      - name: Archive build results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: ${{ github.job }}
+          path: target/${{ github.job }}
+        continue-on-error: true
   acceptance:
     needs:
       - build-info

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java

@@ -93,7 +93,6 @@ public static Codec<DatanodeDetails> getCodec() {
   private String version;
   private long setupTime;
   private String revision;
-  private String buildDate;
   private volatile HddsProtos.NodeOperationalState persistedOpState;
   private volatile long persistedOpStateExpiryEpochSec;
   private int initialVersion;
@@ -111,7 +110,6 @@ private DatanodeDetails(Builder b) {
     version = b.version;
     setupTime = b.setupTime;
     revision = b.revision;
-    buildDate = b.buildDate;
     persistedOpState = b.persistedOpState;
     persistedOpStateExpiryEpochSec = b.persistedOpStateExpiryEpochSec;
     initialVersion = b.initialVersion;
@@ -140,7 +138,6 @@ public DatanodeDetails(DatanodeDetails datanodeDetails) {
     this.version = datanodeDetails.version;
     this.setupTime = datanodeDetails.setupTime;
     this.revision = datanodeDetails.revision;
-    this.buildDate = datanodeDetails.buildDate;
     this.persistedOpState = datanodeDetails.getPersistedOpState();
     this.persistedOpStateExpiryEpochSec =
         datanodeDetails.getPersistedOpStateExpiryEpochSec();
@@ -432,9 +429,6 @@ public static DatanodeDetails getFromProtoBuf(
     if (extendedDetailsProto.hasRevision()) {
       builder.setRevision(extendedDetailsProto.getRevision());
     }
-    if (extendedDetailsProto.hasBuildDate()) {
-      builder.setBuildDate(extendedDetailsProto.getBuildDate());
-    }
     return builder.build();
   }
 
@@ -526,9 +520,6 @@ public ExtendedDatanodeDetailsProto getExtendedProtoBufMessage() {
     if (!Strings.isNullOrEmpty(getRevision())) {
       extendedBuilder.setRevision(getRevision());
     }
-    if (!Strings.isNullOrEmpty(getBuildDate())) {
-      extendedBuilder.setBuildDate(getBuildDate());
-    }
 
     return extendedBuilder.build();
   }
@@ -621,7 +612,6 @@ public static final class Builder {
     private String version;
     private long setupTime;
     private String revision;
-    private String buildDate;
     private HddsProtos.NodeOperationalState persistedOpState;
     private long persistedOpStateExpiryEpochSec = 0;
     private int initialVersion;
@@ -653,7 +643,6 @@ public Builder setDatanodeDetails(DatanodeDetails details) {
       this.version = details.getVersion();
       this.setupTime = details.getSetupTime();
       this.revision = details.getRevision();
-      this.buildDate = details.getBuildDate();
       this.persistedOpState = details.getPersistedOpState();
       this.persistedOpStateExpiryEpochSec =
           details.getPersistedOpStateExpiryEpochSec();
@@ -800,18 +789,6 @@ public Builder setRevision(String rev) {
       return this;
     }
 
-    /**
-     * Sets the DataNode build date.
-     *
-     * @param date the build date of DataNode.
-     *
-     * @return DatanodeDetails.Builder
-     */
-    public Builder setBuildDate(String date) {
-      this.buildDate = date;
-      return this;
-    }
-
     /**
      * Sets the DataNode setup time.
      *
@@ -1053,24 +1030,6 @@ public void setRevision(String rev) {
     this.revision = rev;
   }
 
-  /**
-   * Returns the DataNode build date.
-   *
-   * @return DataNode build date
-   */
-  public String getBuildDate() {
-    return buildDate;
-  }
-
-  /**
-   * Set DataNode build date.
-   *
-   * @param date DataNode build date
-   */
-  public void setBuildDate(String date) {
-    this.buildDate = date;
-  }
-
   @Override
   public HddsProtos.NetworkNode toProtobuf(
       int clientVersion) {

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/HddsVersionInfo.java

@@ -44,8 +44,6 @@ public static void main(String[] args) {
     System.out.println(
         "Source code repository " + HDDS_VERSION_INFO.getUrl() + " -r " +
             HDDS_VERSION_INFO.getRevision());
-    System.out.println("Compiled by " + HDDS_VERSION_INFO.getUser() + " on "
-        + HDDS_VERSION_INFO.getDate());
     System.out.println(
         "Compiled with protoc " + HDDS_VERSION_INFO.getHadoopProtoc2Version() +
             ", " + HDDS_VERSION_INFO.getGrpcProtocVersion() +

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/VersionInfo.java

@@ -65,18 +65,6 @@ public String getRevision() {
     return info.getProperty("revision", "Unknown");
   }
 
-  public String getBranch() {
-    return info.getProperty("branch", "Unknown");
-  }
-
-  public String getDate() {
-    return info.getProperty("date", "Unknown");
-  }
-
-  public String getUser() {
-    return info.getProperty("user", "Unknown");
-  }
-
   public String getUrl() {
     return info.getProperty("url", "Unknown");
   }
@@ -108,7 +96,6 @@ public String getCompilePlatform() {
   public String getBuildVersion() {
     return getVersion() +
         " from " + getRevision() +
-        " by " + getUser() +
         " source checksum " + getSrcChecksum();
   }
 }

hadoop-hdds/common/src/main/resources/hdds-version-info.properties

@@ -18,9 +18,6 @@
 
 version=${declared.hdds.version}
 revision=${version-info.scm.commit}
-branch=${version-info.scm.branch}
-user=${user.name}
-date=${version-info.build.time}
 url=${version-info.scm.uri}
 srcChecksum=${version-info.source.md5}
 hadoopProtoc2Version=${proto2.hadooprpc.protobuf.version}

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java

@@ -228,7 +228,6 @@ public void start() {
       datanodeDetails.setSetupTime(Time.now());
       datanodeDetails.setRevision(
           HddsVersionInfo.HDDS_VERSION_INFO.getRevision());
-      datanodeDetails.setBuildDate(HddsVersionInfo.HDDS_VERSION_INFO.getDate());
       TracingUtil.initTracing(
           "HddsDatanodeService." + datanodeDetails.getUuidString()
               .substring(0, 8), conf);

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServiceRuntimeInfo.java

@@ -47,13 +47,6 @@ public interface ServiceRuntimeInfo {
    */
   String getSoftwareVersion();
 
-  /**
-   * Get the compilation information which contains date, user and branch.
-   *
-   * @return the compilation information, as a JSON string.
-   */
-  String getCompileInfo();
-
   /**
    * Gets the NN start time in milliseconds.
    *

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServiceRuntimeInfoImpl.java

@@ -42,12 +42,6 @@ public String getSoftwareVersion() {
     return versionInfo.getVersion();
   }
 
-  @Override
-  public String getCompileInfo() {
-    return versionInfo.getDate() + " by " + versionInfo.getUser() + " from "
-        + versionInfo.getBranch();
-  }
-
   @Override
   public long getStartedTimeInMillis() {
     return startedTimeInMillis;

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java

@@ -742,9 +742,7 @@ public static String createStartupShutdownMessage(VersionInfo versionInfo,
         "  version = " + versionInfo.getVersion(),
         "  classpath = " + System.getProperty("java.class.path"),
         "  build = " + versionInfo.getUrl() + "/"
-            + versionInfo.getRevision()
-            + " ; compiled by '" + versionInfo.getUser()
-            + "' on " + versionInfo.getDate(),
+            + versionInfo.getRevision(),
         "  java = " + System.getProperty("java.version"),
         "  conf = " + conf);
   }

hadoop-hdds/interface-client/src/main/proto/hdds.proto

@@ -61,7 +61,7 @@ message ExtendedDatanodeDetailsProto {
     optional string version = 2;
     optional int64 setupTime = 3;
     optional string revision = 4;
-    optional string buildDate = 5;
+    optional string buildDate = 5; // unused, reserved for compatibility
 }
 
 message MoveDataNodePairProto {

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/OzoneVersionInfo.java

@@ -69,8 +69,6 @@ public static void main(String[] args) {
     System.out.println(
         "Source code repository " + OZONE_VERSION_INFO.getUrl() + " -r " +
             OZONE_VERSION_INFO.getRevision());
-    System.out.println("Compiled by " + OZONE_VERSION_INFO.getUser() + " on "
-        + OZONE_VERSION_INFO.getDate());
     System.out.println(
         "Compiled with protoc " + OZONE_VERSION_INFO.getHadoopProtoc2Version() +
             ", " + OZONE_VERSION_INFO.getGrpcProtocVersion() +

hadoop-ozone/common/src/main/resources/ozone-version-info.properties

@@ -19,9 +19,6 @@
 version=${declared.ozone.version}
 release=${ozone.release}
 revision=${version-info.scm.commit}
-branch=${version-info.scm.branch}
-user=${user.name}
-date=${version-info.build.time}
 url=${version-info.scm.uri}
 srcChecksum=${version-info.source.md5}
 hadoopProtoc2Version=${proto2.hadooprpc.protobuf.version}

hadoop-ozone/dev-support/checks/_build.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
+: ${OZONE_WITH_COVERAGE:="false"}
+
+MAVEN_OPTIONS='-V -B -DskipTests -DskipDocs --no-transfer-progress'
+
+if [[ "${OZONE_WITH_COVERAGE}" == "true" ]]; then
+  MAVEN_OPTIONS="${MAVEN_OPTIONS} -Pcoverage"
+else
+  MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip"
+fi
+
+export MAVEN_OPTS="-Xmx4096m ${MAVEN_OPTS:-}"
+mvn ${MAVEN_OPTIONS} clean "$@"
+rc=$?

hadoop-ozone/dev-support/checks/_diffoscope.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Helper script to compare jars reported by maven-artifact-plugin
+
+set -e -u -o pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
+BASE_DIR="$(pwd -P)"
+: ${OUTPUT_LOG:="${BASE_DIR}/target/repro/output.log"}
+
+for jar in $(grep -o "investigate with diffoscope [^ ]*\.jar [^ ]*\.jar" "${OUTPUT_LOG}" | awk '{ print $NF }'); do
+  jarname=$(basename "$jar")
+  if [[ ! -e "$jar" ]]; then
+    echo "$jar does not exist"
+    continue
+  fi
+
+  ref=$(find target/reference -name "$jarname")
+  if [[ -z "$ref" ]]; then
+    ref=$(find ~/.m2/repository -name "$jarname")
+  fi
+
+  if [[ ! -e "$ref" ]]; then
+    echo "Reference not found for: $jarname"
+    continue
+  fi
+
+  diffoscope "$ref" "$jar"
+done

hadoop-ozone/dev-support/checks/build.sh

@@ -13,20 +13,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR/../../.." || exit 1
-
-: ${OZONE_WITH_COVERAGE:="false"}
-
-MAVEN_OPTIONS='-V -B -DskipTests -DskipDocs --no-transfer-progress'
 
-if [[ "${OZONE_WITH_COVERAGE}" == "true" ]]; then
-  MAVEN_OPTIONS="${MAVEN_OPTIONS} -Pcoverage"
-else
-  MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip"
-fi
-
-export MAVEN_OPTS="-Xmx4096m ${MAVEN_OPTS:-}"
-echo "${MAVEN_OPTIONS}"
-mvn ${MAVEN_OPTIONS} clean install "$@"
-exit $?
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source "${DIR}"/_build.sh install "$@"