HDDS-11081. Use thread-local instance of FileSystem in Freon tests

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGenerator.java

@@ -152,10 +152,10 @@ private void verifyDirTree(String volumeName, String bucketName, int depth,
     Path rootDir = new Path(rootPath.concat("/"));
     // verify root path details
     FileStatus[] fileStatuses = fileSystem.listStatus(rootDir);
+    // verify the num of peer directories, expected span count is 1
+    // as it has only one dir at root.
+    verifyActualSpan(1, fileStatuses);
     for (FileStatus fileStatus : fileStatuses) {
-      // verify the num of peer directories, expected span count is 1
-      // as it has only one dir at root.
-      verifyActualSpan(1, fileStatuses);
       int actualDepth =
           traverseToLeaf(fileSystem, fileStatus.getPath(), 1, depth, span,
               fileCount, StorageSize.parse(perFileSize, StorageUnit.BYTES));

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopBaseFreonGenerator.java

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.freon;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import picocli.CommandLine.Option;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.util.Optional;
+
+/**
+ * Base class for Freon generator tests that requires  {@link FileSystem} instance.
+ */
+public abstract class HadoopBaseFreonGenerator extends BaseFreonGenerator {
+
+  @Option(names = {"-r", "--rpath", "--path"},
+      description = "Hadoop FS file system path. Use full path.",
+      defaultValue = "o3fs://bucket1.vol1")
+  private String rootPath;
+
+  private final ThreadLocal<FileSystem>  threadLocalFileSystem =
+      ThreadLocal.withInitial(this::createFS);
+
+  private OzoneConfiguration configuration;
+  private URI uri;
+
+  @Override
+  public void init() {
+    super.init();
+    configuration = createOzoneConfiguration();
+    uri = URI.create(rootPath);
+    String scheme = Optional.ofNullable(uri.getScheme())
+        .orElseGet(() -> FileSystem.getDefaultUri(configuration)
+            .getScheme());
+    String disableCacheName =
+        String.format("fs.%s.impl.disable.cache", scheme);
+    print("Disabling FS cache: " + disableCacheName);
+    configuration.setBoolean(disableCacheName, true);
+  }
+
+  @Override
+  protected void taskLoopCompleted() {
+    FileSystem fileSystem = threadLocalFileSystem.get();
+    try {
+      fileSystem.close();
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
+  }
+
+  protected String getRootPath() {
+    return rootPath;
+  }
+
+  protected FileSystem getFileSystem() {
+    return threadLocalFileSystem.get();
+  }
+
+  private org.apache.hadoop.fs.FileSystem createFS() {
+    try {
+      return org.apache.hadoop.fs.FileSystem.get(uri, configuration);
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
+  }
+
+}

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopDirTreeGenerator.java

@@ -19,17 +19,14 @@
 import com.codahale.metrics.Timer;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.conf.StorageSize;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Option;
 
-import java.net.URI;
 import java.util.concurrent.Callable;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -45,17 +42,12 @@
     mixinStandardHelpOptions = true,
     showDefaultValues = true)
 @SuppressWarnings("java:S2245") // no need for secure random
-public class HadoopDirTreeGenerator extends BaseFreonGenerator
+public class HadoopDirTreeGenerator extends HadoopBaseFreonGenerator
     implements Callable<Void> {
 
   private static final Logger LOG =
       LoggerFactory.getLogger(HadoopDirTreeGenerator.class);
 
-  @Option(names = {"-r", "--rpath"},
-      description = "Hadoop FS root path",
-      defaultValue = "o3fs://bucket2.vol2")
-  private String rootPath;
-
   @Option(names = {"-d", "--depth"},
       description = "Number of directories to be generated recursively",
       defaultValue = "5")
@@ -99,8 +91,6 @@ public class HadoopDirTreeGenerator extends BaseFreonGenerator
 
   private ContentGenerator contentGenerator;
 
-  private FileSystem fileSystem;
-
   @Override
   public Void call() throws Exception {
     String s;
@@ -111,10 +101,7 @@ public Void call() throws Exception {
       s = "Invalid span value, span value should be greater than zero!";
       print(s);
     } else {
-      init();
-      OzoneConfiguration configuration = createOzoneConfiguration();
-      fileSystem = FileSystem.get(URI.create(rootPath), configuration);
-
+      super.init();
       contentGenerator = new ContentGenerator(fileSize.toBytes(), bufferSize);
       timer = getMetrics().timer("file-create");
 
@@ -152,7 +139,7 @@ public Void call() throws Exception {
      created.
    */
   private void createDir(long counter) throws Exception {
-    String dir = makeDirWithGivenNumberOfFiles(rootPath);
+    String dir = makeDirWithGivenNumberOfFiles(getRootPath());
     if (depth > 1) {
       createSubDirRecursively(dir, 1, 1);
     }
@@ -196,8 +183,8 @@ private void createSubDirRecursively(String parent, int depthIndex,
   private String makeDirWithGivenNumberOfFiles(String parent)
           throws Exception {
     String dir = RandomStringUtils.randomAlphanumeric(length);
-    dir = parent.toString().concat("/").concat(dir);
-    fileSystem.mkdirs(new Path(dir));
+    dir = parent.concat("/").concat(dir);
+    getFileSystem().mkdirs(new Path(dir));
     totalDirsCnt.incrementAndGet();
     // Add given number of files into the created directory.
     createFiles(dir);
@@ -212,7 +199,7 @@ private void createFile(String dir, long counter) throws Exception {
       LOG.debug("FilePath:{}", file);
     }
     timer.time(() -> {
-      try (FSDataOutputStream output = fileSystem.create(file)) {
+      try (FSDataOutputStream output = getFileSystem().create(file)) {
         contentGenerator.write(output);
       }
       return null;

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java

@@ -16,17 +16,12 @@
  */
 package org.apache.hadoop.ozone.freon;
 
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.net.URI;
-import java.util.Optional;
 import java.util.concurrent.Callable;
 
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 
 import com.codahale.metrics.Timer;
 import org.apache.hadoop.hdds.conf.StorageSize;
@@ -42,14 +37,9 @@
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true,
     showDefaultValues = true)
-public class HadoopFsGenerator extends BaseFreonGenerator
+public class HadoopFsGenerator extends HadoopBaseFreonGenerator
     implements Callable<Void> {
 
-  @Option(names = {"--path"},
-      description = "Hadoop FS file system path. Use full path.",
-      defaultValue = "o3fs://bucket1.vol1")
-  private String rootPath;
-
   @Option(names = {"-s", "--size"},
       description = "Size of the generated files. " +
           StorageSizeConverter.STORAGE_SIZE_DESCRIPTION,
@@ -77,29 +67,12 @@ public class HadoopFsGenerator extends BaseFreonGenerator
 
   private Timer timer;
 
-  private OzoneConfiguration configuration;
-  private URI uri;
-  private final ThreadLocal<FileSystem> threadLocalFileSystem =
-      ThreadLocal.withInitial(this::createFS);
-
   @Override
   public Void call() throws Exception {
-    init();
-
-    configuration = createOzoneConfiguration();
-    uri = URI.create(rootPath);
-    String scheme = Optional.ofNullable(uri.getScheme())
-            .orElseGet(() -> FileSystem.getDefaultUri(configuration)
-                    .getScheme());
-    String disableCacheName =
-            String.format("fs.%s.impl.disable.cache", scheme);
-    print("Disabling FS cache: " + disableCacheName);
-    configuration.setBoolean(disableCacheName, true);
-
-    Path file = new Path(rootPath + "/" + generateObjectName(0));
-    try (FileSystem fileSystem = threadLocalFileSystem.get()) {
-      fileSystem.mkdirs(file.getParent());
-    }
+    super.init();
+
+    Path file = new Path(getRootPath() + "/" + generateObjectName(0));
+    getFileSystem().mkdirs(file.getParent());
 
     contentGenerator =
         new ContentGenerator(fileSize.toBytes(), bufferSize, copyBufferSize,
@@ -113,8 +86,8 @@ public Void call() throws Exception {
   }
 
   private void createFile(long counter) throws Exception {
-    Path file = new Path(rootPath + "/" + generateObjectName(counter));
-    FileSystem fileSystem = threadLocalFileSystem.get();
+    Path file = new Path(getRootPath() + "/" + generateObjectName(counter));
+    FileSystem fileSystem = getFileSystem();
 
     timer.time(() -> {
       try (FSDataOutputStream output = fileSystem.create(file)) {
@@ -123,22 +96,4 @@ private void createFile(long counter) throws Exception {
       return null;
     });
   }
-
-  private FileSystem createFS() {
-    try {
-      return FileSystem.get(uri, configuration);
-    } catch (IOException e) {
-      throw new UncheckedIOException(e);
-    }
-  }
-
-  @Override
-  protected void taskLoopCompleted() {
-    FileSystem fileSystem = threadLocalFileSystem.get();
-    try {
-      fileSystem.close();
-    } catch (IOException e) {
-      throw new UncheckedIOException(e);
-    }
-  }
 }

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsValidator.java

@@ -16,22 +16,18 @@
  */
 package org.apache.hadoop.ozone.freon;
 
-import java.net.URI;
 import java.security.MessageDigest;
 import java.util.concurrent.Callable;
 
 import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 
 import com.codahale.metrics.Timer;
 import org.apache.commons.io.IOUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine.Command;
-import picocli.CommandLine.Option;
 
 /**
  * Data generator tool test om performance.
@@ -42,36 +38,24 @@
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true,
     showDefaultValues = true)
-public class HadoopFsValidator extends BaseFreonGenerator
+public class HadoopFsValidator extends HadoopBaseFreonGenerator
     implements Callable<Void> {
 
   private static final Logger LOG =
       LoggerFactory.getLogger(HadoopFsValidator.class);
 
-  @Option(names = {"--path"},
-      description = "Hadoop FS file system path",
-      defaultValue = "o3fs://bucket1.vol1")
-  private String rootPath;
-
   private ContentGenerator contentGenerator;
 
   private Timer timer;
 
-  private FileSystem fileSystem;
-
   private byte[] referenceDigest;
 
   @Override
   public Void call() throws Exception {
+    super.init();
 
-    init();
-
-    OzoneConfiguration configuration = createOzoneConfiguration();
-
-    fileSystem = FileSystem.get(URI.create(rootPath), configuration);
-
-    Path file = new Path(rootPath + "/" + generateObjectName(0));
-    try (FSDataInputStream stream = fileSystem.open(file)) {
+    Path file = new Path(getRootPath() + "/" + generateObjectName(0));
+    try (FSDataInputStream stream = getFileSystem().open(file)) {
       referenceDigest = getDigest(stream);
     }
 
@@ -83,10 +67,10 @@ public Void call() throws Exception {
   }
 
   private void validateFile(long counter) throws Exception {
-    Path file = new Path(rootPath + "/" + generateObjectName(counter));
+    Path file = new Path(getRootPath() + "/" + generateObjectName(counter));
 
     byte[] content = timer.time(() -> {
-      try (FSDataInputStream input = fileSystem.open(file)) {
+      try (FSDataInputStream input = getFileSystem().open(file)) {
         return IOUtils.toByteArray(input);
       }
     });