[DO NOT MERGE] Backport some fixes from master to ozone-1.4

.github/workflows/ci.yml

@@ -92,10 +92,10 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pnpm-
       - name: Cache for maven dependencies
-        uses: actions/cache@v4
+        uses: actions/cache/restore@v4
         with:
           path: |
-            ~/.m2/repository
+            ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/ozone
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
@@ -160,7 +160,7 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: |
-            ~/.m2/repository
+            ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/ozone
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
@@ -198,7 +198,7 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: |
-            ~/.m2/repository
+            ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/ozone
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
@@ -242,7 +242,7 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: |
-            ~/.m2/repository
+            ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/ozone
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
@@ -310,7 +310,7 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: |
-            ~/.m2/repository
+            ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/ozone
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
@@ -446,7 +446,7 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: |
-            ~/.m2/repository
+            ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/ozone
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
@@ -506,7 +506,7 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: |
-            ~/.m2/repository
+            ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/ozone
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |

.github/workflows/intermittent-test-check.yml

@@ -105,13 +105,13 @@ jobs:
         with:
           ref: ${{ github.event.inputs.ref }}
       - name: Cache for maven dependencies
-        uses: actions/cache@v4
+        uses: actions/cache/restore@v4
         with:
-          path: ~/.m2/repository
-          key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-single
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
-            maven-repo-${{ hashFiles('**/pom.xml') }}-8
-            maven-repo-${{ hashFiles('**/pom.xml') }}
             maven-repo-
       - name: Setup java
         uses: actions/setup-java@v4

.github/workflows/populate-cache.yml

@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This workflow creates cache with Maven dependencies for Ozone build.
+
+name: populate-cache
+
+on:
+  push:
+    branches:
+      - master
+      - ozone-1.4
+    paths:
+      - 'pom.xml'
+      - '**/pom.xml'
+      - '.github/workflows/populate-cache.yml'
+  schedule:
+    - cron: '20 3 * * *'
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v4
+
+      - name: Restore cache for Maven dependencies
+        id: restore-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
+
+      - name: Setup Java
+        if: steps.restore-cache.outputs.cache-hit != 'true'
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 8
+
+      - name: Fetch dependencies
+        if: steps.restore-cache.outputs.cache-hit != 'true'
+        run: mvn --batch-mode --fail-never --no-transfer-progress --show-version dependency:go-offline
+
+      - name: Delete Ozone jars from repo
+        if: steps.restore-cache.outputs.cache-hit != 'true'
+        run: rm -fr ~/.m2/repository/org/apache/ozone
+
+      - name: List repo contents
+        if: steps.restore-cache.outputs.cache-hit != 'true'
+        run: find ~/.m2/repository -type f | sort | xargs ls -lh
+
+      - name: Save cache for Maven dependencies
+        if: steps.restore-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}

.github/workflows/repeat-acceptance.yml

@@ -91,9 +91,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pnpm-
       - name: Cache for maven dependencies
-        uses: actions/cache@v4
+        uses: actions/cache/restore@v4
         with:
-          path: ~/.m2/repository
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/ozone
           key: maven-repo-${{ hashFiles('**/pom.xml') }}-${{ env.JAVA_VERSION }}
           restore-keys: |
             maven-repo-${{ hashFiles('**/pom.xml') }}
@@ -115,12 +117,6 @@ jobs:
             hadoop-ozone/dist/target/ozone-*.tar.gz
             !hadoop-ozone/dist/target/ozone-*-src.tar.gz
           retention-days: 1
-      - name: Delete temporary build artifacts before caching
-        run: |
-          #Never cache local artifacts
-          rm -rf ~/.m2/repository/org/apache/ozone/hdds*
-          rm -rf ~/.m2/repository/org/apache/ozone/ozone*
-        if: always()
   acceptance:
     needs:
       - prepare-job

SECURITY.md

@@ -5,13 +5,16 @@
 The first stable release of Apache Ozone is 1.0, the previous alpha and beta releases are not supported by the community.
 
 | Version       | Supported          |
-| ------------- | ------------------ |
+|---------------| ------------------ |
 | 0.3.0 (alpha) | :x:                |
 | 0.4.0 (alpha) | :x:                |
 | 0.4.1 (alpha) | :x:                |
 | 0.5.0 (beta)  | :x:                |
-| 1.0           | :white_check_mark: |
-| 1.1           | :white_check_mark: |
+| 1.0.0         | :x:                |
+| 1.1.0         | :x:                |
+| 1.2.1         | :x:                |
+| 1.3.0         | :x:                |
+| 1.4.0         | :white_check_mark: |
 
 ## Reporting a Vulnerability
 

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java

@@ -224,7 +224,7 @@ public enum ChecksumCombineMode {
   private String fsDefaultBucketLayout = "FILE_SYSTEM_OPTIMIZED";
 
   @PostConstruct
-  private void validate() {
+  public void validate() {
     Preconditions.checkState(streamBufferSize > 0);
     Preconditions.checkState(streamBufferFlushSize > 0);
     Preconditions.checkState(streamBufferMaxSize > 0);

hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ByteBufferOutputStream.java

@@ -39,6 +39,11 @@ public void write(@Nonnull byte[] byteArray) throws IOException {
     write(ByteBuffer.wrap(byteArray));
   }
 
+  @Override
+  public void write(@Nonnull byte[] byteArray, int off, int len) throws IOException {
+    write(ByteBuffer.wrap(byteArray), off, len);
+  }
+
   @Override
   public void write(int b) throws IOException {
     write(new byte[]{(byte) b});

hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/TestOzoneClientConfig.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.junit.jupiter.api.Test;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM_MIN_SIZE;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class TestOzoneClientConfig {
+
+  @Test
+  void missingSizeSuffix() {
+    final int bytes = 1024;
+
+    OzoneConfiguration conf = new OzoneConfiguration();
+    conf.setInt("ozone.client.bytes.per.checksum", bytes);
+
+    OzoneClientConfig subject = conf.getObject(OzoneClientConfig.class);
+
+    assertEquals(OZONE_CLIENT_BYTES_PER_CHECKSUM_MIN_SIZE, subject.getBytesPerChecksum());
+  }
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/LeakDetector.java

@@ -18,13 +18,15 @@
  */
 package org.apache.hadoop.hdds.utils;
 
+import org.apache.ratis.util.UncheckedAutoCloseable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.lang.ref.ReferenceQueue;
 import java.util.Collections;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * Simple general resource leak detector using {@link ReferenceQueue} and {@link java.lang.ref.WeakReference} to
@@ -37,7 +39,7 @@
  * class MyResource implements AutoClosable {
  *   static final LeakDetector LEAK_DETECTOR = new LeakDetector("MyResource");
  *
- *   private final LeakTracker leakTracker = LEAK_DETECTOR.track(this, () -> {
+ *   private final UncheckedAutoCloseable leakTracker = LEAK_DETECTOR.track(this, () -> {
  *      // report leaks, don't refer to the original object (MyResource) here.
  *      System.out.println("MyResource is not closed before being discarded.");
  *   });
@@ -53,13 +55,14 @@
  * }</pre>
  */
 public class LeakDetector {
-  public static final Logger LOG = LoggerFactory.getLogger(LeakDetector.class);
+  private static final Logger LOG = LoggerFactory.getLogger(LeakDetector.class);
+  private static final AtomicLong COUNTER = new AtomicLong();
   private final ReferenceQueue<Object> queue = new ReferenceQueue<>();
   private final Set<LeakTracker> allLeaks = Collections.newSetFromMap(new ConcurrentHashMap<>());
   private final String name;
 
   public LeakDetector(String name) {
-    this.name = name;
+    this.name = name + COUNTER.getAndIncrement();
     start();
   }
 
@@ -89,7 +92,7 @@ private void run() {
     LOG.warn("Exiting leak detector {}.", name);
   }
 
-  public LeakTracker track(Object leakable, Runnable reportLeak) {
+  public UncheckedAutoCloseable track(Object leakable, Runnable reportLeak) {
     // A rate filter can be put here to only track a subset of all objects, e.g. 5%, 10%,
     // if we have proofs that leak tracking impacts performance, or a single LeakDetector
     // thread can't keep up with the pace of object allocation.

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/LeakTracker.java

@@ -18,6 +18,8 @@
  */
 package org.apache.hadoop.hdds.utils;
 
+import org.apache.ratis.util.UncheckedAutoCloseable;
+
 import java.lang.ref.ReferenceQueue;
 import java.lang.ref.WeakReference;
 import java.util.Set;
@@ -27,7 +29,7 @@
  *
  * @see LeakDetector
  */
-public class LeakTracker extends WeakReference<Object> {
+final class LeakTracker extends WeakReference<Object> implements UncheckedAutoCloseable {
   private final Set<LeakTracker> allLeaks;
   private final Runnable leakReporter;
   LeakTracker(Object referent, ReferenceQueue<Object> referenceQueue,
@@ -40,6 +42,7 @@ public class LeakTracker extends WeakReference<Object> {
   /**
    * Called by the tracked resource when closing.
    */
+  @Override
   public void close() {
     allLeaks.remove(this);
   }