From c38ead23d9a279aca417cca4fa85fa7266a66b82 Mon Sep 17 00:00:00 2001 From: muskan mishra Date: Thu, 7 Dec 2023 17:51:23 +0530 Subject: [PATCH 01/25] added ops per sec and average pagination metric for listkeys --- .../hadoop/ozone/om/OMPerformanceMetrics.java | 13 +++++++++++++ .../ozone/om/OmMetadataManagerImpl.java | 19 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java index 25223c502c2d..df5bd513cae8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java @@ -113,6 +113,11 @@ public static void unregister() { @Metric(about = "Ratis local command execution latency in nano seconds") private MutableRate validateAndUpdateCacneLatencyNs; + @Metric(about = "average pagination for listKeys") + private MutableRate listKeysAveragePagination; + + @Metric(about = "ops per second for listKeys") + private MutableRate listKeysOpsPerSec; public void addLookupLatency(long latencyInNs) { lookupLatencyNs.add(latencyInNs); @@ -210,4 +215,12 @@ public MutableRate getCreateOmResponseLatencyNs() { public MutableRate getValidateAndUpdateCacneLatencyNs() { return validateAndUpdateCacneLatencyNs; } + + public void setListKeysAveragePagination(long averagePagination) { + listKeysAveragePagination.add(averagePagination); + } + + public void setListKeysOpsPerSec(long opsPerSec) { + listKeysOpsPerSec.add(opsPerSec); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 7a7101152181..56cc9f5c39f9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -315,6 +315,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager, private final Map tableCacheMetricsMap = new HashMap<>(); private SnapshotChainManager snapshotChainManager; + private final OMPerformanceMetrics perfMetrics; /** * OmMetadataManagerImpl constructor. @@ -334,6 +335,7 @@ public OmMetadataManagerImpl(OzoneConfiguration conf, OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT); this.omEpoch = OmUtils.getOMEpoch(isRatisEnabled); + this.perfMetrics = ozoneManager.getPerfMetrics(); // For test purpose only ignorePipelineinKey = conf.getBoolean( "ozone.om.ignore.pipeline", Boolean.TRUE); @@ -347,6 +349,7 @@ protected OmMetadataManagerImpl() { OzoneConfiguration conf = new OzoneConfiguration(); this.lock = new OzoneManagerLock(conf); this.omEpoch = 0; + perfMetrics = null; } public static OmMetadataManagerImpl createCheckpointMetadataManager( @@ -381,6 +384,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) setStore(loadDB(conf, dir, name, true, java.util.Optional.of(Boolean.TRUE), Optional.empty())); initializeOmTables(CacheType.PARTIAL_CACHE, false); + perfMetrics = null; } @@ -418,6 +422,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) stop(); throw e; } + perfMetrics = null; } @Override @@ -1177,7 +1182,7 @@ public List listBuckets(final String volumeName, public ListKeysResult listKeys(String volumeName, String bucketName, String startKey, String keyPrefix, int maxKeys) throws IOException { - + long startNanos = Time.monotonicNowNanos(); List result = new ArrayList<>(); if (maxKeys <= 0) { return new ListKeysResult(result, false); @@ -1276,6 +1281,18 @@ public ListKeysResult listKeys(String volumeName, String bucketName, } boolean isTruncated = cacheKeyMap.size() > maxKeys; + long averagePagination; + if(isTruncated) + { + averagePagination = maxKeys; + } + else + { + averagePagination = cacheKeyMap.size(); + } + perfMetrics.setListKeysAveragePagination(averagePagination); + long opsPerSec = averagePagination/(Time.monotonicNowNanos() - startNanos); + perfMetrics.setListKeysOpsPerSec(opsPerSec); // Finally DB entries and cache entries are merged, then return the count // of maxKeys from the sorted map. From 36ca4fa5ecb61e7329d1d04a453ae3f31a64112d Mon Sep 17 00:00:00 2001 From: muskan mishra Date: Tue, 12 Dec 2023 11:41:11 +0530 Subject: [PATCH 02/25] updated test cases --- .../ozone/om/OmMetadataManagerImpl.java | 26 +++++++++++++------ .../ozone/om/TestOmMetadataManager.java | 2 +- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 56cc9f5c39f9..f0d6b226b63a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -341,6 +341,20 @@ public OmMetadataManagerImpl(OzoneConfiguration conf, "ozone.om.ignore.pipeline", Boolean.TRUE); start(conf); } + public OmMetadataManagerImpl(OzoneConfiguration conf, + OzoneManager ozoneManager, OMPerformanceMetrics perfMetrics) throws IOException { + this.ozoneManager = ozoneManager; + this.lock = new OzoneManagerLock(conf); + isRatisEnabled = conf.getBoolean( + OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, + OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT); + this.omEpoch = OmUtils.getOMEpoch(isRatisEnabled); + this.perfMetrics = perfMetrics; + // For test purpose only + ignorePipelineinKey = conf.getBoolean( + "ozone.om.ignore.pipeline", Boolean.TRUE); + start(conf); + } /** * For subclass overriding. @@ -1282,16 +1296,12 @@ public ListKeysResult listKeys(String volumeName, String bucketName, boolean isTruncated = cacheKeyMap.size() > maxKeys; long averagePagination; - if(isTruncated) - { + if (isTruncated) { averagePagination = maxKeys; - } - else - { + } else { averagePagination = cacheKeyMap.size(); - } - perfMetrics.setListKeysAveragePagination(averagePagination); - long opsPerSec = averagePagination/(Time.monotonicNowNanos() - startNanos); + } perfMetrics.setListKeysAveragePagination(averagePagination); + long opsPerSec = averagePagination / (Time.monotonicNowNanos() - startNanos); perfMetrics.setListKeysOpsPerSec(opsPerSec); // Finally DB entries and cache entries are merged, then return the count diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 58dfeb7b831e..0c491891e14e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -90,7 +90,7 @@ public void setup() throws Exception { ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OZONE_OM_DB_DIRS, folder.getAbsolutePath()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, new OMPerformanceMetrics()); } @Test From 1818a49bb43fe1bac76528404f4e5d013fc76ca9 Mon Sep 17 00:00:00 2001 From: muskan mishra Date: Wed, 13 Dec 2023 17:04:36 +0530 Subject: [PATCH 03/25] Updated all test cases --- .../apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 9 ++++++--- .../apache/hadoop/ozone/om/TestOmMetadataManager.java | 3 ++- .../org/apache/hadoop/ozone/om/TestSnapshotChain.java | 3 ++- .../org/apache/hadoop/ozone/om/TestSnapshotInfo.java | 3 ++- .../TestOzoneManagerDoubleBufferWithDummyResponse.java | 5 +++-- .../hadoop/ozone/om/response/TestCleanupTableInfo.java | 9 +++------ .../om/response/bucket/TestOMBucketCreateResponse.java | 4 +++- .../om/response/bucket/TestOMBucketDeleteResponse.java | 4 +++- .../response/bucket/TestOMBucketSetPropertyResponse.java | 4 +++- .../om/response/file/TestOMDirectoryCreateResponse.java | 4 +++- .../file/TestOMDirectoryCreateResponseWithFSO.java | 4 +++- .../hadoop/ozone/om/response/key/TestOMKeyResponse.java | 4 +++- .../response/s3/multipart/TestS3MultipartResponse.java | 4 +++- .../response/security/TestOMDelegationTokenResponse.java | 3 ++- .../response/snapshot/TestOMSnapshotCreateResponse.java | 4 +++- .../response/snapshot/TestOMSnapshotDeleteResponse.java | 8 +++----- .../om/response/volume/TestOMVolumeCreateResponse.java | 4 +++- .../om/response/volume/TestOMVolumeDeleteResponse.java | 4 +++- .../om/response/volume/TestOMVolumeSetOwnerResponse.java | 4 +++- .../om/response/volume/TestOMVolumeSetQuotaResponse.java | 4 +++- 20 files changed, 59 insertions(+), 32 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index f0d6b226b63a..84e5a4cb2a47 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -342,7 +342,8 @@ public OmMetadataManagerImpl(OzoneConfiguration conf, start(conf); } public OmMetadataManagerImpl(OzoneConfiguration conf, - OzoneManager ozoneManager, OMPerformanceMetrics perfMetrics) throws IOException { + OzoneManager ozoneManager, OMPerformanceMetrics perfMetrics) + throws IOException { this.ozoneManager = ozoneManager; this.lock = new OzoneManagerLock(conf); isRatisEnabled = conf.getBoolean( @@ -1300,8 +1301,10 @@ public ListKeysResult listKeys(String volumeName, String bucketName, averagePagination = maxKeys; } else { averagePagination = cacheKeyMap.size(); - } perfMetrics.setListKeysAveragePagination(averagePagination); - long opsPerSec = averagePagination / (Time.monotonicNowNanos() - startNanos); + } + perfMetrics.setListKeysAveragePagination(averagePagination); + long opsPerSec = + averagePagination / (Time.monotonicNowNanos() - startNanos); perfMetrics.setListKeysOpsPerSec(opsPerSec); // Finally DB entries and cache entries are merged, then return the count diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 0c491891e14e..725ee5c7be87 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -90,7 +90,8 @@ public void setup() throws Exception { ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OZONE_OM_DB_DIRS, folder.getAbsolutePath()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, + new OMPerformanceMetrics()); } @Test diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotChain.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotChain.java index 1821b6f9af32..d1e5b6a50b20 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotChain.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotChain.java @@ -69,7 +69,8 @@ public class TestSnapshotChain { public void setup() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OZONE_OM_DB_DIRS, folder.toString()); - omMetadataManager = new OmMetadataManagerImpl(conf, null); + omMetadataManager = new OmMetadataManagerImpl(conf, null, + new OMPerformanceMetrics()); snapshotIdToSnapshotInfoMap = new HashMap<>(); chainManager = new SnapshotChainManager(omMetadataManager); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotInfo.java index 3d78764ae43d..7198fc30c967 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotInfo.java @@ -51,7 +51,8 @@ public void setup() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(conf, null); + omMetadataManager = new OmMetadataManagerImpl(conf, null, + new OMPerformanceMetrics()); } private SnapshotInfo createSnapshotInfo() { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java index 0b01cd24afb7..ffd1a5e8676e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java @@ -24,6 +24,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.hdds.utils.TransactionInfo; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -74,8 +75,8 @@ public void setup() throws IOException { OzoneConfiguration configuration = new OzoneConfiguration(); configuration.set(OZONE_METADATA_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = - new OmMetadataManagerImpl(configuration, null); + omMetadataManager = new OmMetadataManagerImpl(configuration, null, + new OMPerformanceMetrics()); OzoneManagerRatisSnapshot ozoneManagerRatisSnapshot = index -> { lastAppliedIndex = index.get(index.size() - 1); }; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java index 3bacc7c5a605..82608ae6d650 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java @@ -29,11 +29,7 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.audit.AuditLogger; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMMetrics; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.om.ResolvedBucket; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; @@ -283,7 +279,8 @@ private OMMetadataManager createOMMetadataManagerSpy() throws IOException { Assertions.assertTrue(newFolder.mkdirs()); } ServerUtils.setOzoneMetaDirPath(conf, newFolder.toString()); - return spy(new OmMetadataManagerImpl(conf, null)); + return spy(new OmMetadataManagerImpl(conf, null, + new OMPerformanceMetrics())); } private OMFileCreateRequest anOMFileCreateRequest() { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketCreateResponse.java index bd8be1e0b129..fb6710681cfc 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketCreateResponse.java @@ -23,6 +23,7 @@ import java.util.UUID; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -58,7 +59,8 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java index a9973439079a..3b825fb414c7 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java @@ -22,6 +22,7 @@ import java.nio.file.Path; import java.util.UUID; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -60,7 +61,8 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketSetPropertyResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketSetPropertyResponse.java index 1a203c4f5d3e..ee467a97bb70 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketSetPropertyResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketSetPropertyResponse.java @@ -23,6 +23,7 @@ import java.util.UUID; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -58,7 +59,8 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java index bc302fb761fb..4b47a5722dcc 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -61,7 +62,8 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseWithFSO.java index dc4ad90c05a0..e70d2ebbec69 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseWithFSO.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -62,7 +63,8 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyResponse.java index 9a0fe7077f15..5063657bd95e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyResponse.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -72,7 +73,8 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = getOzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); volumeName = UUID.randomUUID().toString(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java index 23b543b6ec12..876cf65f7e7c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; @@ -77,7 +78,8 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMDelegationTokenResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMDelegationTokenResponse.java index a5545f5a1d8b..27313b10b05c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMDelegationTokenResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMDelegationTokenResponse.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -47,7 +48,7 @@ public void setup() throws IOException { ((OzoneConfiguration) conf).set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl((OzoneConfiguration) conf, - null); + null, new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java index f0b729d103ea..a7114eecd5de 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; @@ -73,7 +74,8 @@ public void setup() throws Exception { String fsPath = folder.getAbsolutePath(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, fsPath); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java index b589665e84a5..651079b48c9b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java @@ -24,10 +24,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.Table.KeyValue; import org.apache.hadoop.hdds.utils.db.TableIterator; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateSnapshotResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteSnapshotResponse; @@ -66,7 +63,8 @@ public void setup() throws Exception { String fsPath = folder.toAbsolutePath().toString(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, fsPath); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeCreateResponse.java index df4a52fb91e1..ed3d630e14b5 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeCreateResponse.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -60,7 +61,8 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeDeleteResponse.java index 43c897dbc0ae..db147a408a90 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeDeleteResponse.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -59,7 +60,8 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetOwnerResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetOwnerResponse.java index 265e0aef0eff..ef5f73929aeb 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetOwnerResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetOwnerResponse.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -60,7 +61,8 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetQuotaResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetQuotaResponse.java index af17f25c3683..5dd18508db35 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetQuotaResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetQuotaResponse.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -59,7 +60,8 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } From f731a19836ed4583aad41fa8f13fd253b3897c39 Mon Sep 17 00:00:00 2001 From: muskan mishra Date: Wed, 13 Dec 2023 19:02:22 +0530 Subject: [PATCH 04/25] refactored code --- .../org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 6 ++++-- .../org/apache/hadoop/ozone/om/TestOmMetadataManager.java | 2 +- .../java/org/apache/hadoop/ozone/om/TestSnapshotChain.java | 2 +- .../java/org/apache/hadoop/ozone/om/TestSnapshotInfo.java | 2 +- .../TestOzoneManagerDoubleBufferWithDummyResponse.java | 6 +++--- .../hadoop/ozone/om/response/TestCleanupTableInfo.java | 7 ++++--- .../om/response/bucket/TestOMBucketCreateResponse.java | 2 +- .../om/response/bucket/TestOMBucketDeleteResponse.java | 4 ++-- .../response/bucket/TestOMBucketSetPropertyResponse.java | 2 +- .../om/response/file/TestOMDirectoryCreateResponse.java | 2 +- .../file/TestOMDirectoryCreateResponseWithFSO.java | 2 +- .../hadoop/ozone/om/response/key/TestOMKeyResponse.java | 2 +- .../om/response/s3/multipart/TestS3MultipartResponse.java | 2 +- .../om/response/snapshot/TestOMSnapshotCreateResponse.java | 2 +- .../om/response/snapshot/TestOMSnapshotDeleteResponse.java | 5 +++-- .../om/response/volume/TestOMVolumeCreateResponse.java | 2 +- .../om/response/volume/TestOMVolumeDeleteResponse.java | 2 +- .../om/response/volume/TestOMVolumeSetOwnerResponse.java | 2 +- .../om/response/volume/TestOMVolumeSetQuotaResponse.java | 2 +- 19 files changed, 30 insertions(+), 26 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 84e5a4cb2a47..0cc452d15834 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -341,9 +341,11 @@ public OmMetadataManagerImpl(OzoneConfiguration conf, "ozone.om.ignore.pipeline", Boolean.TRUE); start(conf); } + public OmMetadataManagerImpl(OzoneConfiguration conf, - OzoneManager ozoneManager, OMPerformanceMetrics perfMetrics) - throws IOException { + OzoneManager ozoneManager, + OMPerformanceMetrics perfMetrics) + throws IOException { this.ozoneManager = ozoneManager; this.lock = new OzoneManagerLock(conf); isRatisEnabled = conf.getBoolean( diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 725ee5c7be87..af898338ff62 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -91,7 +91,7 @@ public void setup() throws Exception { ozoneConfiguration.set(OZONE_OM_DB_DIRS, folder.getAbsolutePath()); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + new OMPerformanceMetrics()); } @Test diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotChain.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotChain.java index d1e5b6a50b20..471f1a72c55d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotChain.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotChain.java @@ -70,7 +70,7 @@ public void setup() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OZONE_OM_DB_DIRS, folder.toString()); omMetadataManager = new OmMetadataManagerImpl(conf, null, - new OMPerformanceMetrics()); + new OMPerformanceMetrics()); snapshotIdToSnapshotInfoMap = new HashMap<>(); chainManager = new SnapshotChainManager(omMetadataManager); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotInfo.java index 7198fc30c967..454e753d26f1 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotInfo.java @@ -52,7 +52,7 @@ public void setup() throws Exception { conf.set(OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(conf, null, - new OMPerformanceMetrics()); + new OMPerformanceMetrics()); } private SnapshotInfo createSnapshotInfo() { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java index ffd1a5e8676e..db325062f64e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java @@ -76,7 +76,7 @@ public void setup() throws IOException { configuration.set(OZONE_METADATA_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(configuration, null, - new OMPerformanceMetrics()); + new OMPerformanceMetrics()); OzoneManagerRatisSnapshot ozoneManagerRatisSnapshot = index -> { lastAppliedIndex = index.get(index.size() - 1); }; @@ -176,14 +176,14 @@ private static class OMDummyCreateBucketResponse extends OMClientResponse { private final OmBucketInfo omBucketInfo; OMDummyCreateBucketResponse(OmBucketInfo omBucketInfo, - OMResponse omResponse) { + OMResponse omResponse) { super(omResponse); this.omBucketInfo = omBucketInfo; } @Override public void addToDBBatch(OMMetadataManager omMetadataManager, - BatchOperation batchOperation) throws IOException { + BatchOperation batchOperation) throws IOException { String dbBucketKey = omMetadataManager.getBucketKey(omBucketInfo.getVolumeName(), omBucketInfo.getBucketName()); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java index 82608ae6d650..ea5b9763b7b6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.om.OMPerformanceMetrics; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; @@ -272,15 +273,15 @@ private void addBucketToMetaTable(OmBucketInfo bucketInfo) * @throws IOException if I/O error occurs in setting up data store for the * metadata manager. */ - private OMMetadataManager createOMMetadataManagerSpy() throws IOException { + private OmMetadataManagerImpl createOMMetadataManagerSpy() throws IOException { OzoneConfiguration conf = new OzoneConfiguration(); File newFolder = folder.toFile(); if (!newFolder.exists()) { Assertions.assertTrue(newFolder.mkdirs()); } ServerUtils.setOzoneMetaDirPath(conf, newFolder.toString()); - return spy(new OmMetadataManagerImpl(conf, null, - new OMPerformanceMetrics())); + return spy( + new OmMetadataManagerImpl(conf, null, new OMPerformanceMetrics())); } private OMFileCreateRequest anOMFileCreateRequest() { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketCreateResponse.java index fb6710681cfc..6bb1333ad915 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketCreateResponse.java @@ -60,7 +60,7 @@ public void setup() throws Exception { ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java index 3b825fb414c7..f4ba17ed18ce 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java @@ -62,7 +62,7 @@ public void setup() throws Exception { ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } @@ -102,7 +102,7 @@ public void testAddToDBBatch() throws Exception { omMetadataManager.getStore().commitBatchOperation(batchOperation); Assertions.assertNull(omMetadataManager.getBucketTable().get( - omMetadataManager.getBucketKey(volumeName, bucketName))); + omMetadataManager.getBucketKey(volumeName, bucketName))); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketSetPropertyResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketSetPropertyResponse.java index ee467a97bb70..e57772d57da4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketSetPropertyResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketSetPropertyResponse.java @@ -60,7 +60,7 @@ public void setup() throws Exception { ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java index 4b47a5722dcc..d22983ec5f44 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java @@ -63,7 +63,7 @@ public void setup() throws Exception { ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseWithFSO.java index e70d2ebbec69..8eff4561ee72 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseWithFSO.java @@ -64,7 +64,7 @@ public void setup() throws Exception { ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyResponse.java index 5063657bd95e..062e6808a9bf 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyResponse.java @@ -74,7 +74,7 @@ public void setup() throws Exception { ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); volumeName = UUID.randomUUID().toString(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java index 876cf65f7e7c..b4c5cf400b69 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java @@ -79,7 +79,7 @@ public void setup() throws Exception { ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java index a7114eecd5de..9a6072cdd12f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java @@ -75,7 +75,7 @@ public void setup() throws Exception { ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, fsPath); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java index 651079b48c9b..f4c0fe67d4ea 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.utils.db.Table.KeyValue; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.om.OMPerformanceMetrics; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateSnapshotResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteSnapshotResponse; @@ -53,7 +54,7 @@ public class TestOMSnapshotDeleteResponse { @TempDir private Path folder; - private OMMetadataManager omMetadataManager; + private OmMetadataManagerImpl omMetadataManager; private BatchOperation batchOperation; private OzoneConfiguration ozoneConfiguration; @@ -64,7 +65,7 @@ public void setup() throws Exception { ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, fsPath); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeCreateResponse.java index ed3d630e14b5..9b2c0dc14451 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeCreateResponse.java @@ -62,7 +62,7 @@ public void setup() throws Exception { ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeDeleteResponse.java index db147a408a90..236e4373eb48 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeDeleteResponse.java @@ -61,7 +61,7 @@ public void setup() throws Exception { ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetOwnerResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetOwnerResponse.java index ef5f73929aeb..46431f2482f0 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetOwnerResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetOwnerResponse.java @@ -62,7 +62,7 @@ public void setup() throws Exception { ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetQuotaResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetQuotaResponse.java index 5dd18508db35..3960cec130cf 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetQuotaResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetQuotaResponse.java @@ -61,7 +61,7 @@ public void setup() throws Exception { ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + new OMPerformanceMetrics()); batchOperation = omMetadataManager.getStore().initBatchOperation(); } From 6d3d979da311aa043ea155b06ef400f3fe41199b Mon Sep 17 00:00:00 2001 From: muskan mishra Date: Thu, 14 Dec 2023 00:50:50 +0530 Subject: [PATCH 05/25] refactored code --- .../hadoop/ozone/om/response/TestCleanupTableInfo.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java index ea5b9763b7b6..2bd10adae48b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java @@ -29,8 +29,10 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.audit.AuditLogger; +import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; @@ -273,7 +275,8 @@ private void addBucketToMetaTable(OmBucketInfo bucketInfo) * @throws IOException if I/O error occurs in setting up data store for the * metadata manager. */ - private OmMetadataManagerImpl createOMMetadataManagerSpy() throws IOException { + private OmMetadataManagerImpl createOMMetadataManagerSpy() + throws IOException { OzoneConfiguration conf = new OzoneConfiguration(); File newFolder = folder.toFile(); if (!newFolder.exists()) { From 18e1e00d091d496f2e844fceead57c2256840f6d Mon Sep 17 00:00:00 2001 From: muskan mishra Date: Thu, 14 Dec 2023 10:42:57 +0530 Subject: [PATCH 06/25] fixed errors --- .../ozone/om/response/TestCleanupTableInfo.java | 17 ++++------------- .../snapshot/TestOMSnapshotDeleteResponse.java | 3 ++- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java index 2bd10adae48b..16e51540648c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java @@ -32,6 +32,8 @@ import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.ResolvedBucket; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -63,23 +65,12 @@ import java.io.File; import java.io.IOException; import java.nio.file.Path; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; +import java.util.*; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.*; /** * The test checks whether all {@link OMClientResponse} have defined the diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java index f4c0fe67d4ea..b58de2c2e1e2 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateSnapshotResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteSnapshotResponse; @@ -38,7 +39,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; - +import org.apache.hadoop.ozone.om.OMConfigKeys; import java.io.File; import java.util.UUID; import java.nio.file.Path; From 6744486c4486cd715d15979629369fae828cd980 Mon Sep 17 00:00:00 2001 From: muskan mishra Date: Thu, 14 Dec 2023 11:08:11 +0530 Subject: [PATCH 07/25] fixed import errors --- .../ozone/om/response/TestCleanupTableInfo.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java index 16e51540648c..503a0e1575e4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java @@ -65,12 +65,23 @@ import java.io.File; import java.io.IOException; import java.nio.file.Path; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; /** * The test checks whether all {@link OMClientResponse} have defined the From 3375fce9f3d22138bcaeda762d88d50394533659 Mon Sep 17 00:00:00 2001 From: muskan mishra Date: Mon, 18 Dec 2023 16:56:31 +0530 Subject: [PATCH 08/25] rearranged imports and refactored code --- .../hadoop/ozone/om/response/TestCleanupTableInfo.java | 4 ++-- .../om/response/snapshot/TestOMSnapshotDeleteResponse.java | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java index 383b9daf3ca2..bea942c66c66 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java @@ -29,12 +29,12 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.audit.AuditLogger; +import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.ResolvedBucket; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.ResolvedBucket; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java index b58de2c2e1e2..1741127232dd 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java @@ -24,6 +24,8 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.Table.KeyValue; import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshotManager; @@ -39,7 +41,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.apache.hadoop.ozone.om.OMConfigKeys; import java.io.File; import java.util.UUID; import java.nio.file.Path; @@ -55,7 +56,7 @@ public class TestOMSnapshotDeleteResponse { @TempDir private Path folder; - private OmMetadataManagerImpl omMetadataManager; + private OMMetadataManager omMetadataManager; private BatchOperation batchOperation; private OzoneConfiguration ozoneConfiguration; From e879fc9fde2423274361ee908afd695a4cd5e9a8 Mon Sep 17 00:00:00 2001 From: muskan mishra Date: Mon, 8 Jan 2024 13:14:11 +0530 Subject: [PATCH 09/25] removed duplicate code --- .../ozone/om/response/bucket/TestOMBucketDeleteResponse.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java index 44c03594a828..d9f23fe2f90e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java @@ -100,9 +100,6 @@ public void testAddToDBBatch() throws Exception { // Do manual commit and see whether addToBatch is successful or not. omMetadataManager.getStore().commitBatchOperation(batchOperation); - - Assertions.assertNull(omMetadataManager.getBucketTable().get( - omMetadataManager.getBucketKey(volumeName, bucketName))); assertNull(omMetadataManager.getBucketTable().get( omMetadataManager.getBucketKey(volumeName, bucketName))); From 94545b62541e1d35befe7534c1ea96ac04f2f9d7 Mon Sep 17 00:00:00 2001 From: muskan mishra Date: Wed, 10 Jan 2024 12:52:55 +0530 Subject: [PATCH 10/25] code change to avoid build failure --- .../om/response/bucket/TestOMBucketDeleteResponse.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java index d9f23fe2f90e..ad93f26600f1 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java @@ -19,7 +19,7 @@ package org.apache.hadoop.ozone.om.response.bucket; -import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.nio.file.Path; import java.util.UUID; @@ -100,9 +100,9 @@ public void testAddToDBBatch() throws Exception { // Do manual commit and see whether addToBatch is successful or not. omMetadataManager.getStore().commitBatchOperation(batchOperation); - - assertNull(omMetadataManager.getBucketTable().get( - omMetadataManager.getBucketKey(volumeName, bucketName))); + + assertEquals(null, omMetadataManager.getBucketTable().get( + omMetadataManager.getBucketKey(volumeName, bucketName))); } } From 3c1595685acd3ae8761e207c238c0d1fbdcbf144 Mon Sep 17 00:00:00 2001 From: muskan mishra Date: Wed, 7 Feb 2024 14:23:37 +0530 Subject: [PATCH 11/25] fixed test cases --- .../TestOzoneFileSystemPrefixParser.java | 13 +++-- .../ozone/fsck/TestContainerMapper.java | 3 +- .../snapshot/TestOmSnapshotObjectStore.java | 1 + .../ozone/om/OmMetadataManagerImpl.java | 52 +++++++++---------- .../om/TestOMMultiTenantManagerImpl.java | 2 +- .../TestSetRangerServiceVersionRequest.java | 3 +- .../hadoop/ozone/debug/PrefixParser.java | 17 ++++-- .../hadoop/ozone/fsck/ContainerMapper.java | 19 +++++++ 8 files changed, 72 insertions(+), 38 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java index 972ad7dd2e78..4ca082084033 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java @@ -29,6 +29,7 @@ import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.debug.PrefixParser; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OMStorage; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.junit.jupiter.api.AfterAll; @@ -59,13 +60,15 @@ public class TestOzoneFileSystemPrefixParser { private static Path dir; private static Path file; + private static OMPerformanceMetrics perfMetrics; + @BeforeAll public static void init() throws Exception { volumeName = RandomStringUtils.randomAlphabetic(10).toLowerCase(); bucketName = RandomStringUtils.randomAlphabetic(10).toLowerCase(); configuration = new OzoneConfiguration(); - + perfMetrics = new OMPerformanceMetrics(); cluster = MiniOzoneCluster.newBuilder(configuration) .setNumDatanodes(3) .build(); @@ -135,7 +138,7 @@ private void testPrefixParseWithInvalidPaths() throws Exception { RandomStringUtils.randomAlphabetic(10).toLowerCase(); invalidVolumeParser.parse(invalidVolumeName, bucketName, OMStorage.getOmDbDir(configuration).getPath(), - file.toString()); + file.toString(), perfMetrics); assertPrefixStats(invalidVolumeParser, 0, 0, 0, 0, 0, 0); PrefixParser invalidBucketParser = new PrefixParser(); @@ -143,7 +146,7 @@ private void testPrefixParseWithInvalidPaths() throws Exception { RandomStringUtils.randomAlphabetic(10).toLowerCase(); invalidBucketParser.parse(volumeName, invalidBucketName, OMStorage.getOmDbDir(configuration).getPath(), - file.toString()); + file.toString(), perfMetrics); assertPrefixStats(invalidBucketParser, 1, 0, 0, 0, 0, 0); @@ -151,7 +154,7 @@ private void testPrefixParseWithInvalidPaths() throws Exception { PrefixParser invalidIntermediateDirParser = new PrefixParser(); invalidIntermediateDirParser.parse(volumeName, bucketName, OMStorage.getOmDbDir(configuration).getPath(), - invalidIntermediateDir.toString()); + invalidIntermediateDir.toString(), perfMetrics); assertPrefixStats(invalidIntermediateDirParser, 1, 1, 3, 1, 1, 1); @@ -162,7 +165,7 @@ private void verifyPrefixParsePath(Path parent, int intermediateDirCount, PrefixParser parser = new PrefixParser(); parser.parse(volumeName, bucketName, - OMStorage.getOmDbDir(configuration).getPath(), parent.toString()); + OMStorage.getOmDbDir(configuration).getPath(), parent.toString(), perfMetrics); assertPrefixStats(parser, 1, 1, intermediateDirCount, nonExistentDirCount, fileCount, dirCount); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/fsck/TestContainerMapper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/fsck/TestContainerMapper.java index 77815698c3b8..f0aa5f7f26a2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/fsck/TestContainerMapper.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/fsck/TestContainerMapper.java @@ -33,6 +33,7 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.ozone.test.GenericTestUtils; @@ -122,7 +123,7 @@ public static void init() throws Exception { public void testContainerMapper() throws Exception { ContainerMapper containerMapper = new ContainerMapper(); Map>> dataMap = - containerMapper.parseOmDB(conf); + containerMapper.parseOmDB(conf, new OMPerformanceMetrics()); // As we have created 20 keys with 10 MB size, and each // container max size is 100 MB, it should create 3 containers because // containers are closing before reaching the threshold diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java index 13c8cb5fca32..b5ebf654690b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om.snapshot; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.junit.jupiter.api.Timeout; import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 519d6a3dcc7c..93751349a793 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -328,19 +328,10 @@ public class OmMetadataManagerImpl implements OMMetadataManager, */ public OmMetadataManagerImpl(OzoneConfiguration conf, OzoneManager ozoneManager) throws IOException { - this.ozoneManager = ozoneManager; + init(conf, ozoneManager); this.lock = new OzoneManagerLock(conf); - // TODO: This is a temporary check. Once fully implemented, all OM state - // change should go through Ratis - be it standalone (for non-HA) or - // replicated (for HA). - isRatisEnabled = conf.getBoolean( - OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, - OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT); this.omEpoch = OmUtils.getOMEpoch(isRatisEnabled); this.perfMetrics = ozoneManager.getPerfMetrics(); - // For test purpose only - ignorePipelineinKey = conf.getBoolean( - "ozone.om.ignore.pipeline", Boolean.TRUE); start(conf); } @@ -348,11 +339,8 @@ public OmMetadataManagerImpl(OzoneConfiguration conf, OzoneManager ozoneManager, OMPerformanceMetrics perfMetrics) throws IOException { - this.ozoneManager = ozoneManager; + init(conf, ozoneManager); this.lock = new OzoneManagerLock(conf); - isRatisEnabled = conf.getBoolean( - OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, - OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT); this.omEpoch = OmUtils.getOMEpoch(isRatisEnabled); this.perfMetrics = perfMetrics; // For test purpose only @@ -360,7 +348,19 @@ public OmMetadataManagerImpl(OzoneConfiguration conf, "ozone.om.ignore.pipeline", Boolean.TRUE); start(conf); } - +private void init(OzoneConfiguration conf, + OzoneManager ozoneManager){ + this.ozoneManager = ozoneManager; + // TODO: This is a temporary check. Once fully implemented, all OM state + // change should go through Ratis - be it standalone (for non-HA) or + // replicated (for HA). + isRatisEnabled = conf.getBoolean( + OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, + OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT); + // For test purpose only + ignorePipelineinKey = conf.getBoolean( + "ozone.om.ignore.pipeline", Boolean.TRUE); +} /** * For subclass overriding. */ @@ -1298,19 +1298,19 @@ public ListKeysResult listKeys(String volumeName, String bucketName, } } } - boolean isTruncated = cacheKeyMap.size() > maxKeys; - long averagePagination; - if (isTruncated) { - averagePagination = maxKeys; - } else { - averagePagination = cacheKeyMap.size(); + if (perfMetrics!=null) { + long averagePagination; + if (isTruncated) { + averagePagination = maxKeys; + } else { + averagePagination = cacheKeyMap.size(); + } + perfMetrics.setListKeysAveragePagination(averagePagination); + long opsPerSec = + averagePagination / (Time.monotonicNowNanos() - startNanos); + perfMetrics.setListKeysOpsPerSec(opsPerSec); } - perfMetrics.setListKeysAveragePagination(averagePagination); - long opsPerSec = - averagePagination / (Time.monotonicNowNanos() - startNanos); - perfMetrics.setListKeysOpsPerSec(opsPerSec); - // Finally DB entries and cache entries are merged, then return the count // of maxKeys from the sorted map. currentCount = 0; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMultiTenantManagerImpl.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMultiTenantManagerImpl.java index 70c204c04c50..db1ec3fcfa8b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMultiTenantManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMultiTenantManagerImpl.java @@ -67,7 +67,7 @@ public void setUp() throws IOException { conf = new OzoneConfiguration(); conf.set(OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); conf.set(OZONE_OM_TENANT_DEV_SKIP_RANGER, "true"); - omMetadataManager = new OmMetadataManagerImpl(conf, ozoneManager); + omMetadataManager = new OmMetadataManagerImpl(conf, ozoneManager, new OMPerformanceMetrics()); createTenantInDB(TENANT_ID); assignUserToTenantInDB(TENANT_ID, "seed-accessId1", "seed-user1", false, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestSetRangerServiceVersionRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestSetRangerServiceVersionRequest.java index a6b87b173507..76bd504c0444 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestSetRangerServiceVersionRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestSetRangerServiceVersionRequest.java @@ -24,6 +24,7 @@ import static org.mockito.Mockito.when; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -61,7 +62,7 @@ public void setUp() throws Exception { conf.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); when(ozoneManager.getMetadataManager()) - .thenReturn(new OmMetadataManagerImpl(conf, ozoneManager)); + .thenReturn(new OmMetadataManagerImpl(conf, ozoneManager, new OMPerformanceMetrics())); } @AfterEach diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java index fabc7f456aec..10a5d0449db4 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.Table.KeyValue; import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManagerUtils; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -108,7 +109,7 @@ public Class getParentType() { @Override public Void call() throws Exception { - parse(volume, bucket, dbPath, filePath); + parse(volume, bucket, dbPath, filePath, null); return null; } @@ -117,7 +118,7 @@ public static void main(String[] args) throws Exception { } public void parse(String vol, String buck, String db, - String file) throws Exception { + String file, OMPerformanceMetrics perfMetrics) throws Exception { if (!Files.exists(Paths.get(db))) { System.out.println("DB path not exist:" + db); return; @@ -128,9 +129,17 @@ public void parse(String vol, String buck, String db, OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OMConfigKeys.OZONE_OM_DB_DIRS, db); + OmMetadataManagerImpl metadataManager; + +if (perfMetrics!= null) { + metadataManager = + new OmMetadataManagerImpl(conf, null, perfMetrics); +} +else { + metadataManager = + new OmMetadataManagerImpl(conf, null); +} - OmMetadataManagerImpl metadataManager = - new OmMetadataManagerImpl(conf, null); metadataManager.start(conf); org.apache.hadoop.fs.Path effectivePath = diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java index a7a74c2e3729..7376549503fc 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -29,6 +30,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.jetbrains.annotations.NotNull; import java.io.IOException; import java.util.ArrayList; @@ -79,6 +81,23 @@ public static void main(String[] args) throws IOException { } OmMetadataManagerImpl metadataManager = new OmMetadataManagerImpl(configuration, null); + return getBlockIdDetailsUtils(metadataManager); + } + + public Map>> + parseOmDB(OzoneConfiguration configuration, OMPerformanceMetrics perfMetrics) throws IOException { + String path = configuration.get(OZONE_OM_DB_DIRS); + if (path == null || path.isEmpty()) { + throw new IOException(OZONE_OM_DB_DIRS + "should be set "); + } + OmMetadataManagerImpl metadataManager = + new OmMetadataManagerImpl(configuration, null, perfMetrics); + return getBlockIdDetailsUtils(metadataManager); + } + + @NotNull + private static Map>> getBlockIdDetailsUtils( + OmMetadataManagerImpl metadataManager) throws IOException { try { Table keyTable = metadataManager.getKeyTable(getBucketLayout()); From 1bd6fd7a659720002546e922206ccfebddebe206 Mon Sep 17 00:00:00 2001 From: muskan mishra Date: Wed, 7 Feb 2024 16:39:31 +0530 Subject: [PATCH 12/25] fixed checkstyle and other issues --- .../om/snapshot/TestOmSnapshotObjectStore.java | 1 - .../hadoop/ozone/om/OmMetadataManagerImpl.java | 14 +++++++------- ...tOzoneManagerDoubleBufferWithDummyResponse.java | 5 ----- .../apache/hadoop/ozone/debug/PrefixParser.java | 13 +++++-------- .../apache/hadoop/ozone/fsck/ContainerMapper.java | 4 +--- 5 files changed, 13 insertions(+), 24 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java index b5ebf654690b..13c8cb5fca32 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.om.snapshot; -import org.apache.hadoop.ozone.om.exceptions.OMException; import org.junit.jupiter.api.Timeout; import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index d8564e10cf54..16e8838dd3fb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -350,19 +350,19 @@ public OmMetadataManagerImpl(OzoneConfiguration conf, "ozone.om.ignore.pipeline", Boolean.TRUE); start(conf); } -private void init(OzoneConfiguration conf, - OzoneManager ozoneManager){ - this.ozoneManager = ozoneManager; + private void init(OzoneConfiguration conf, + OzoneManager ozoneManager) { + this.ozoneManager = ozoneManager; // TODO: This is a temporary check. Once fully implemented, all OM state // change should go through Ratis - be it standalone (for non-HA) or // replicated (for HA). - isRatisEnabled = conf.getBoolean( + isRatisEnabled = conf.getBoolean( OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT); // For test purpose only - ignorePipelineinKey = conf.getBoolean( + ignorePipelineinKey = conf.getBoolean( "ozone.om.ignore.pipeline", Boolean.TRUE); -} + } /** * For subclass overriding. */ @@ -1318,7 +1318,7 @@ public ListKeysResult listKeys(String volumeName, String bucketName, } } boolean isTruncated = cacheKeyMap.size() > maxKeys; - if (perfMetrics!=null) { + if (perfMetrics != null) { long averagePagination; if (isTruncated) { averagePagination = maxKeys; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java index 4a272f3c6220..5eba25e4ddea 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java @@ -74,13 +74,8 @@ public void setup() throws IOException { OzoneConfiguration configuration = new OzoneConfiguration(); configuration.set(OZONE_METADATA_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(configuration, null, new OMPerformanceMetrics()); - OzoneManagerRatisSnapshot ozoneManagerRatisSnapshot = index -> { - lastAppliedIndex = index.get(index.size() - 1); - }; - doubleBuffer = OzoneManagerDoubleBuffer.newBuilder() .setOmMetadataManager(omMetadataManager) .setMaxUnFlushedTransactionCount(10000) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java index 10a5d0449db4..f108d8f142fc 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java @@ -131,14 +131,11 @@ public void parse(String vol, String buck, String db, conf.set(OMConfigKeys.OZONE_OM_DB_DIRS, db); OmMetadataManagerImpl metadataManager; -if (perfMetrics!= null) { - metadataManager = - new OmMetadataManagerImpl(conf, null, perfMetrics); -} -else { - metadataManager = - new OmMetadataManagerImpl(conf, null); -} + if (perfMetrics != null) { + metadataManager = new OmMetadataManagerImpl(conf, null, perfMetrics); + } else { + metadataManager = new OmMetadataManagerImpl(conf, null); + } metadataManager.start(conf); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java index 7376549503fc..29dedfea8c49 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java @@ -30,7 +30,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; -import org.jetbrains.annotations.NotNull; import java.io.IOException; import java.util.ArrayList; @@ -85,7 +84,7 @@ public static void main(String[] args) throws IOException { } public Map>> - parseOmDB(OzoneConfiguration configuration, OMPerformanceMetrics perfMetrics) throws IOException { + parseOmDB(OzoneConfiguration configuration, OMPerformanceMetrics perfMetrics) throws IOException { String path = configuration.get(OZONE_OM_DB_DIRS); if (path == null || path.isEmpty()) { throw new IOException(OZONE_OM_DB_DIRS + "should be set "); @@ -95,7 +94,6 @@ public static void main(String[] args) throws IOException { return getBlockIdDetailsUtils(metadataManager); } - @NotNull private static Map>> getBlockIdDetailsUtils( OmMetadataManagerImpl metadataManager) throws IOException { try { From 31f8a3cc807f781c96ad4114ead22a696819bb92 Mon Sep 17 00:00:00 2001 From: muskan mishra Date: Wed, 7 Feb 2024 18:07:21 +0530 Subject: [PATCH 13/25] checkstyle issue resolved --- .../ozone/om/OmMetadataManagerImpl.java | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 16e8838dd3fb..bf96d236d180 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -350,10 +350,9 @@ public OmMetadataManagerImpl(OzoneConfiguration conf, "ozone.om.ignore.pipeline", Boolean.TRUE); start(conf); } - private void init(OzoneConfiguration conf, - OzoneManager ozoneManager) { - this.ozoneManager = ozoneManager; - // TODO: This is a temporary check. Once fully implemented, all OM state + private void init(OzoneConfiguration conf, OzoneManager manager) { + this.ozoneManager = manager; + // TODO: This is a temporary check. Once fully implemented, all OM state // change should go through Ratis - be it standalone (for non-HA) or // replicated (for HA). isRatisEnabled = conf.getBoolean( @@ -403,7 +402,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) lock = new OmReadOnlyLock(); omEpoch = 0; setStore(loadDB(conf, dir, name, true, - java.util.Optional.of(Boolean.TRUE), Optional.empty())); + Optional.of(Boolean.TRUE), Optional.empty())); initializeOmTables(CacheType.PARTIAL_CACHE, false); perfMetrics = null; } @@ -436,7 +435,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) checkSnapshotDirExist(checkpoint); } setStore(loadDB(conf, metaDir, dbName, false, - java.util.Optional.of(Boolean.TRUE), + Optional.of(Boolean.TRUE), Optional.of(maxOpenFiles), false, false)); initializeOmTables(CacheType.PARTIAL_CACHE, false); } catch (IOException e) { @@ -592,14 +591,14 @@ public void start(OzoneConfiguration configuration) throws IOException { public static DBStore loadDB(OzoneConfiguration configuration, File metaDir) throws IOException { return loadDB(configuration, metaDir, OM_DB_NAME, false, - java.util.Optional.empty(), Optional.empty(), true, true); + Optional.empty(), Optional.empty(), true, true); } public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, String dbName, boolean readOnly, - java.util.Optional + Optional disableAutoCompaction, - java.util.Optional maxOpenFiles) + Optional maxOpenFiles) throws IOException { return loadDB(configuration, metaDir, dbName, readOnly, disableAutoCompaction, maxOpenFiles, true, true); @@ -608,9 +607,9 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, @SuppressWarnings("checkstyle:parameternumber") public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, String dbName, boolean readOnly, - java.util.Optional + Optional disableAutoCompaction, - java.util.Optional maxOpenFiles, + Optional maxOpenFiles, boolean enableCompactionDag, boolean createCheckpointDirs) throws IOException { @@ -821,7 +820,7 @@ public DBStore getStore() { */ @Override public String getVolumeKey(String volume) { - return OzoneConsts.OM_KEY_PREFIX + volume; + return OM_KEY_PREFIX + volume; } /** @@ -1129,13 +1128,13 @@ public List listBuckets(final String volumeName, List result = new ArrayList<>(); if (Strings.isNullOrEmpty(volumeName)) { throw new OMException("Volume name is required.", - ResultCodes.VOLUME_NOT_FOUND); + VOLUME_NOT_FOUND); } String volumeNameBytes = getVolumeKey(volumeName); if (volumeTable.get(volumeNameBytes) == null) { throw new OMException("Volume " + volumeName + " not found.", - ResultCodes.VOLUME_NOT_FOUND); + VOLUME_NOT_FOUND); } String startKey; @@ -1228,18 +1227,18 @@ public ListKeysResult listKeys(String volumeName, String bucketName, if (Strings.isNullOrEmpty(volumeName)) { throw new OMException("Volume name is required.", - ResultCodes.VOLUME_NOT_FOUND); + VOLUME_NOT_FOUND); } if (Strings.isNullOrEmpty(bucketName)) { throw new OMException("Bucket name is required.", - ResultCodes.BUCKET_NOT_FOUND); + BUCKET_NOT_FOUND); } String bucketNameBytes = getBucketKey(volumeName, bucketName); if (getBucketTable().get(bucketNameBytes) == null) { throw new OMException("Bucket " + bucketName + " not found.", - ResultCodes.BUCKET_NOT_FOUND); + BUCKET_NOT_FOUND); } String seekKey; @@ -1470,7 +1469,7 @@ public List listVolumes(String userName, int index = 0; if (!Strings.isNullOrEmpty(startKey)) { index = volumes.indexOf( - startKey.startsWith(OzoneConsts.OM_KEY_PREFIX) ? + startKey.startsWith(OM_KEY_PREFIX) ? startKey.substring(1) : startKey); @@ -1490,7 +1489,7 @@ public List listVolumes(String userName, // this probably means om db is corrupted or some entries are // accidentally removed. throw new OMException("Volume info not found for " + volumeName, - ResultCodes.VOLUME_NOT_FOUND); + VOLUME_NOT_FOUND); } result.add(volumeArgs); } @@ -1817,7 +1816,7 @@ public ExpiredOpenKeys getExpiredOpenKeys(Duration expireThreshold, = dbOpenKeyName.substring(lastPrefix + 1); final OmKeyInfo info = kt.get(dbKeyName); - final boolean isHsync = java.util.Optional.ofNullable(info) + final boolean isHsync = Optional.ofNullable(info) .map(WithMetadata::getMetadata) .map(meta -> meta.get(OzoneConsts.HSYNC_CLIENT_ID)) .filter(id -> id.equals(clientIdString)) @@ -1833,7 +1832,7 @@ public ExpiredOpenKeys getExpiredOpenKeys(Duration expireThreshold, .setBucketName(info.getBucketName()) .setKeyName(info.getKeyName()) .setDataSize(info.getDataSize()); - java.util.Optional.ofNullable(info.getLatestVersionLocations()) + Optional.ofNullable(info.getLatestVersionLocations()) .map(OmKeyLocationInfoGroup::getLocationList) .map(Collection::stream) .orElseGet(Stream::empty) From 3a7385950242bb818603703e6221d856aebcf1a4 Mon Sep 17 00:00:00 2001 From: muskan mishra Date: Wed, 7 Feb 2024 21:56:52 +0530 Subject: [PATCH 14/25] unit test resolution --- .../org/apache/hadoop/ozone/om/TestOmMetadataManager.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index f1caf58997aa..82b8d1104b80 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -75,6 +75,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.mockito.Mockito.mock; /** * Tests OzoneManager MetadataManager. @@ -92,8 +93,9 @@ public void setup() throws Exception { ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OZONE_OM_DB_DIRS, folder.getAbsolutePath()); + OMPerformanceMetrics perfMetrics = mock(OMPerformanceMetrics.class); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + perfMetrics); } @Test From 6bf2a5e2b946a633b27b481b93f0716885b6a999 Mon Sep 17 00:00:00 2001 From: muskan mishra Date: Thu, 8 Feb 2024 11:01:40 +0530 Subject: [PATCH 15/25] unit test case failures resolutions --- .../hadoop/ozone/recon/OMMetadataManagerTestUtils.java | 5 +++-- .../hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java | 3 ++- .../ozone/recon/api/TestNSSummaryEndpointWithLegacy.java | 3 ++- .../ozone/recon/recovery/TestReconOmMetadataManagerImpl.java | 3 ++- .../apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java | 3 ++- .../ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java | 3 ++- .../hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java | 5 +++-- 7 files changed, 16 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java index 42d69e030f31..b11d3140a2d7 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -79,7 +80,7 @@ public static OMMetadataManager initializeNewOmMetadataManager( omConfiguration.set(OZONE_OM_DB_DIRS, omDbDir.getAbsolutePath()); OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null); + omConfiguration, null, new OMPerformanceMetrics()); String volumeKey = omMetadataManager.getVolumeKey("sampleVol"); OmVolumeArgs args = @@ -113,7 +114,7 @@ public static OMMetadataManager initializeEmptyOmMetadataManager( OzoneConfiguration omConfiguration = new OzoneConfiguration(); omConfiguration.set(OZONE_OM_DB_DIRS, omDbDir.getAbsolutePath()); - return new OmMetadataManagerImpl(omConfiguration, null); + return new OmMetadataManagerImpl(omConfiguration, null, new OMPerformanceMetrics()); } /** diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java index cbe850b918f0..3672ab3acec0 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; @@ -838,7 +839,7 @@ private static OMMetadataManager initializeNewOmMetadataManager( omConfiguration.set(OZONE_OM_DB_DIRS, omDbDir.getAbsolutePath()); OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null); + omConfiguration, null, new OMPerformanceMetrics()); String volumeKey = omMetadataManager.getVolumeKey(VOL); OmVolumeArgs args = diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java index ba00f843f447..7772a8733d23 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java @@ -35,6 +35,7 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; @@ -876,7 +877,7 @@ private static OMMetadataManager initializeNewOmMetadataManager( omConfiguration.set(OMConfigKeys .OZONE_OM_ENABLE_FILESYSTEM_PATHS, "true"); OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null); + omConfiguration, null, new OMPerformanceMetrics()); String volumeKey = omMetadataManager.getVolumeKey(VOL); OmVolumeArgs args = diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/recovery/TestReconOmMetadataManagerImpl.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/recovery/TestReconOmMetadataManagerImpl.java index b1e7e54d6d5c..5680737b0973 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/recovery/TestReconOmMetadataManagerImpl.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/recovery/TestReconOmMetadataManagerImpl.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -162,7 +163,7 @@ private OMMetadataManager getOMMetadataManager() throws IOException { omConfiguration.set(OZONE_OM_DB_DIRS, omDbDir.getAbsolutePath()); OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null); + omConfiguration, null, new OMPerformanceMetrics()); //Create a volume + bucket + 2 keys. String volumeKey = omMetadataManager.getVolumeKey("sampleVol"); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java index 6992c3100fb9..f5c3044b4cb7 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.utils.db.RDBBatchOperation; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -429,7 +430,7 @@ private static void initializeNewOmMetadataManager( omConfiguration.set(OMConfigKeys .OZONE_OM_ENABLE_FILESYSTEM_PATHS, "true"); omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null); + omConfiguration, null, new OMPerformanceMetrics()); String volumeKey = omMetadataManager.getVolumeKey(VOL); OmVolumeArgs args = diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java index 5ffd03cbb88a..7748d39b7330 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.utils.db.RDBBatchOperation; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -695,7 +696,7 @@ private static void initializeNewOmMetadataManager( omConfiguration.set(OMConfigKeys .OZONE_OM_ENABLE_FILESYSTEM_PATHS, "true"); omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null); + omConfiguration, null, new OMPerformanceMetrics()); String volumeKey = omMetadataManager.getVolumeKey(VOL); OmVolumeArgs args = diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java index 9676af015747..1428d0c8f5ba 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedTransactionLogIterator; import org.apache.hadoop.io.Text; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.codec.OMDBDefinition; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -81,11 +82,11 @@ private OzoneConfiguration createNewTestPath(String folderName) @BeforeEach public void setUp() throws Exception { OzoneConfiguration configuration = createNewTestPath("config"); - omMetadataManager = new OmMetadataManagerImpl(configuration, null); + omMetadataManager = new OmMetadataManagerImpl(configuration, null, new OMPerformanceMetrics()); OzoneConfiguration reconConfiguration = createNewTestPath("reconConfig"); reconOmMetadataManager = new OmMetadataManagerImpl(reconConfiguration, - null); + null, new OMPerformanceMetrics()); } @Test From 86717121d1effaf68541d8cd675d15db49b2430f Mon Sep 17 00:00:00 2001 From: muskan mishra Date: Thu, 8 Feb 2024 14:23:48 +0530 Subject: [PATCH 16/25] listKeysReadFromRocksDbLatency metric added --- .../org/apache/hadoop/ozone/om/OMPerformanceMetrics.java | 7 +++++++ .../org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java index 961eabca5ffc..b0a518de4cf1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java @@ -125,6 +125,9 @@ public static void unregister() { @Metric(about = "resolveBucketLink latency in listKeys") private MutableRate listKeysResolveBucketLatencyNs; + @Metric(about = "readFromRockDb latency in listKeys") + private MutableRate listKeysReadFromRocksDbLatencyNs; + public void addLookupLatency(long latencyInNs) { lookupLatencyNs.add(latencyInNs); } @@ -237,4 +240,8 @@ public MutableRate getListKeysAclCheckLatencyNs() { public MutableRate getListKeysResolveBucketLatencyNs() { return listKeysResolveBucketLatencyNs; } + + public void addListKeysReadFromRocksDbLatencyNs(long latencyInNs) { + listKeysReadFromRocksDbLatencyNs.add(latencyInNs); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index bf96d236d180..60aa458e43fa 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1290,7 +1290,7 @@ public ListKeysResult listKeys(String volumeName, String bucketName, } // Get maxKeys from DB if it has. - + long startNano = Time.monotonicNowNanos(); try (TableIterator> keyIter = getKeyTable(getBucketLayout()).iterator()) { KeyValue< String, OmKeyInfo > kv; @@ -1316,6 +1316,7 @@ public ListKeysResult listKeys(String volumeName, String bucketName, } } } + perfMetrics.addListKeysReadFromRocksDbLatencyNs(Time.monotonicNowNanos() - startNano); boolean isTruncated = cacheKeyMap.size() > maxKeys; if (perfMetrics != null) { long averagePagination; From a0397eb3d427a7f501589e0fb34f38184415408f Mon Sep 17 00:00:00 2001 From: muskan mishra Date: Thu, 8 Feb 2024 14:37:56 +0530 Subject: [PATCH 17/25] removed irrelevant code --- .../ozone/om/response/bucket/TestOMBucketDeleteResponse.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java index ad93f26600f1..e6d4575ef493 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java @@ -19,7 +19,7 @@ package org.apache.hadoop.ozone.om.response.bucket; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import java.nio.file.Path; import java.util.UUID; @@ -101,7 +101,7 @@ public void testAddToDBBatch() throws Exception { // Do manual commit and see whether addToBatch is successful or not. omMetadataManager.getStore().commitBatchOperation(batchOperation); - assertEquals(null, omMetadataManager.getBucketTable().get( + assertNull(omMetadataManager.getBucketTable().get( omMetadataManager.getBucketKey(volumeName, bucketName))); } From dfd427bddae33db08196ff6d7fd64bff0934fc30 Mon Sep 17 00:00:00 2001 From: muskan mishra Date: Thu, 8 Feb 2024 22:23:59 +0530 Subject: [PATCH 18/25] refactored code to avoid null pointer exception --- .../org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 60aa458e43fa..0611cc2fa7ca 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1288,11 +1288,11 @@ public ListKeysResult listKeys(String volumeName, String bucketName, cacheKeyMap.put(key, omKeyInfo); } } - + long startNano, stopNano = 0; // Get maxKeys from DB if it has. - long startNano = Time.monotonicNowNanos(); try (TableIterator> keyIter = getKeyTable(getBucketLayout()).iterator()) { + startNano = Time.monotonicNowNanos(); KeyValue< String, OmKeyInfo > kv; keyIter.seek(seekKey); // we need to iterate maxKeys + 1 here because if skipStartKey is true, @@ -1315,8 +1315,8 @@ public ListKeysResult listKeys(String volumeName, String bucketName, break; } } + stopNano = Time.monotonicNowNanos(); } - perfMetrics.addListKeysReadFromRocksDbLatencyNs(Time.monotonicNowNanos() - startNano); boolean isTruncated = cacheKeyMap.size() > maxKeys; if (perfMetrics != null) { long averagePagination; @@ -1329,6 +1329,7 @@ public ListKeysResult listKeys(String volumeName, String bucketName, long opsPerSec = averagePagination / (Time.monotonicNowNanos() - startNanos); perfMetrics.setListKeysOpsPerSec(opsPerSec); + perfMetrics.addListKeysReadFromRocksDbLatencyNs(stopNano - startNano); } // Finally DB entries and cache entries are merged, then return the count // of maxKeys from the sorted map. From 364de9b0679e9afbc144f688642f20a4e6e6d7b3 Mon Sep 17 00:00:00 2001 From: muskan1012 Date: Tue, 18 Jun 2024 21:00:04 +0530 Subject: [PATCH 19/25] updated code to avoid npe --- .../org/apache/hadoop/ozone/om/snapshot/TestSnapshotChain.java | 1 + .../org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java | 1 + .../hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotChain.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotChain.java index 226805b78a5e..a25823388d7e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotChain.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotChain.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java index df2d3fb87729..6755d8f42dea 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus; diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java index a244e4ff2ce2..f1c88fc08b42 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; @@ -126,7 +127,7 @@ private static OMMetadataManager initializeNewOmMetadataManager( omConfiguration.set(OZONE_OM_DB_DIRS, omDbDir.getAbsolutePath()); OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null); + omConfiguration, null, new OMPerformanceMetrics()); return omMetadataManager; } From d26e986f24484c57f6ad83a1c7e9986a9e6567fd Mon Sep 17 00:00:00 2001 From: muskan1012 Date: Tue, 18 Jun 2024 22:57:03 +0530 Subject: [PATCH 20/25] npe resolution for integration-recon --- .../ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java index 6a2f2c557db8..66b69d1b148c 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java @@ -37,6 +37,7 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -1099,7 +1100,7 @@ private static OMMetadataManager initializeNewOmMetadataManager( omConfiguration.set(OMConfigKeys .OZONE_OM_ENABLE_FILESYSTEM_PATHS, "false"); OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null); + omConfiguration, null, new OMPerformanceMetrics()); String volumeKey = omMetadataManager.getVolumeKey(VOL); OmVolumeArgs args = From 2bb0cdafd526d057666b1112e0494789566df3ee Mon Sep 17 00:00:00 2001 From: muskan1012 Date: Tue, 25 Jun 2024 03:07:27 +0530 Subject: [PATCH 21/25] refactored code as per review comments --- .../fs/ozone/TestOzoneFileSystemPrefixParser.java | 12 ++++-------- .../hadoop/ozone/fsck/TestContainerMapper.java | 3 +-- .../hadoop/ozone/om/OMPerformanceMetrics.java | 4 ++-- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 14 +++++++------- .../ozone/om/TestOMMultiTenantManagerImpl.java | 2 +- .../hadoop/ozone/om/TestOmMetadataManager.java | 5 +---- ...tOzoneManagerDoubleBufferWithDummyResponse.java | 8 +++----- .../tenant/TestSetRangerServiceVersionRequest.java | 3 +-- .../ozone/om/response/TestCleanupTableInfo.java | 3 +-- .../bucket/TestOMBucketCreateResponse.java | 4 +--- .../bucket/TestOMBucketDeleteResponse.java | 6 ++---- .../bucket/TestOMBucketSetPropertyResponse.java | 4 +--- .../file/TestOMDirectoryCreateResponse.java | 4 +--- .../file/TestOMDirectoryCreateResponseWithFSO.java | 4 +--- .../ozone/om/response/key/TestOMKeyResponse.java | 4 +--- .../s3/multipart/TestS3MultipartResponse.java | 4 +--- .../security/TestOMDelegationTokenResponse.java | 3 +-- .../snapshot/TestOMSnapshotCreateResponse.java | 4 +--- .../snapshot/TestOMSnapshotDeleteResponse.java | 4 +--- .../volume/TestOMVolumeCreateResponse.java | 4 +--- .../volume/TestOMVolumeDeleteResponse.java | 4 +--- .../volume/TestOMVolumeSetOwnerResponse.java | 4 +--- .../volume/TestOMVolumeSetQuotaResponse.java | 4 +--- .../ozone/om/snapshot/TestSnapshotChain.java | 4 +--- .../hadoop/ozone/om/snapshot/TestSnapshotInfo.java | 4 +--- .../ozone/recon/OMMetadataManagerTestUtils.java | 5 ++--- .../recon/api/TestNSSummaryDiskUsageOrdering.java | 3 +-- .../recon/api/TestNSSummaryEndpointWithFSO.java | 3 +-- .../recon/api/TestNSSummaryEndpointWithLegacy.java | 3 +-- .../api/TestNSSummaryEndpointWithOBSAndLegacy.java | 3 +-- .../recovery/TestReconOmMetadataManagerImpl.java | 3 +-- .../ozone/recon/tasks/TestNSSummaryTask.java | 3 +-- .../recon/tasks/TestNSSummaryTaskWithLegacy.java | 3 +-- .../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 5 ++--- .../apache/hadoop/ozone/debug/PrefixParser.java | 13 +++---------- .../apache/hadoop/ozone/fsck/ContainerMapper.java | 12 ------------ 36 files changed, 52 insertions(+), 123 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java index de36e82fa912..4617c42f551f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java @@ -29,7 +29,6 @@ import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.debug.PrefixParser; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OMStorage; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.junit.jupiter.api.AfterAll; @@ -61,15 +60,12 @@ public class TestOzoneFileSystemPrefixParser { private static Path dir; private static Path file; - private static OMPerformanceMetrics perfMetrics; - @BeforeAll public static void init() throws Exception { volumeName = RandomStringUtils.randomAlphabetic(10).toLowerCase(); bucketName = RandomStringUtils.randomAlphabetic(10).toLowerCase(); configuration = new OzoneConfiguration(); - perfMetrics = new OMPerformanceMetrics(); cluster = MiniOzoneCluster.newBuilder(configuration) .setNumDatanodes(3) .build(); @@ -133,7 +129,7 @@ private void testPrefixParseWithInvalidPaths() throws Exception { RandomStringUtils.randomAlphabetic(10).toLowerCase(); invalidVolumeParser.parse(invalidVolumeName, bucketName, OMStorage.getOmDbDir(configuration).getPath(), - file.toString(), perfMetrics); + file.toString()); assertPrefixStats(invalidVolumeParser, 0, 0, 0, 0, 0, 0); PrefixParser invalidBucketParser = new PrefixParser(); @@ -141,7 +137,7 @@ private void testPrefixParseWithInvalidPaths() throws Exception { RandomStringUtils.randomAlphabetic(10).toLowerCase(); invalidBucketParser.parse(volumeName, invalidBucketName, OMStorage.getOmDbDir(configuration).getPath(), - file.toString(), perfMetrics); + file.toString()); assertPrefixStats(invalidBucketParser, 1, 0, 0, 0, 0, 0); @@ -149,7 +145,7 @@ private void testPrefixParseWithInvalidPaths() throws Exception { PrefixParser invalidIntermediateDirParser = new PrefixParser(); invalidIntermediateDirParser.parse(volumeName, bucketName, OMStorage.getOmDbDir(configuration).getPath(), - invalidIntermediateDir.toString(), perfMetrics); + invalidIntermediateDir.toString()); assertPrefixStats(invalidIntermediateDirParser, 1, 1, 3, 1, 1, 1); @@ -160,7 +156,7 @@ private void verifyPrefixParsePath(Path parent, int intermediateDirCount, PrefixParser parser = new PrefixParser(); parser.parse(volumeName, bucketName, - OMStorage.getOmDbDir(configuration).getPath(), parent.toString(), perfMetrics); + OMStorage.getOmDbDir(configuration).getPath(), parent.toString()); assertPrefixStats(parser, 1, 1, intermediateDirCount, nonExistentDirCount, fileCount, dirCount); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/fsck/TestContainerMapper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/fsck/TestContainerMapper.java index ded6e5c867b3..7811470887dc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/fsck/TestContainerMapper.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/fsck/TestContainerMapper.java @@ -33,7 +33,6 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.ozone.test.GenericTestUtils; @@ -121,7 +120,7 @@ public static void init() throws Exception { public void testContainerMapper() throws Exception { ContainerMapper containerMapper = new ContainerMapper(); Map>> dataMap = - containerMapper.parseOmDB(conf, new OMPerformanceMetrics()); + containerMapper.parseOmDB(conf); // As we have created 20 keys with 10 MB size, and each // container max size is 100 MB, it should create 3 containers because // containers are closing before reaching the threshold diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java index 2b328b3a2813..89a0e47d115b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java @@ -229,8 +229,8 @@ public void setListKeysAveragePagination(long averagePagination) { listKeysAveragePagination.add(averagePagination); } - public void setListKeysOpsPerSec(long opsPerSec) { - listKeysOpsPerSec.add(opsPerSec); + public void setListKeysOpsPerNs(long opsPerNs) { + listKeysOpsPerSec.add(opsPerNs); } MutableRate getListKeysAclCheckLatencyNs() { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index a23393073ffe..3ee26e5c36a4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -332,7 +332,7 @@ public OmMetadataManagerImpl(OzoneConfiguration conf, init(conf, ozoneManager); this.lock = new OzoneManagerLock(conf); this.omEpoch = OmUtils.getOMEpoch(isRatisEnabled); - this.perfMetrics = ozoneManager.getPerfMetrics(); + this.perfMetrics = null; start(conf); } @@ -1253,11 +1253,11 @@ public ListKeysResult listKeys(String volumeName, String bucketName, cacheKeyMap.put(key, omKeyInfo); } } - long startNano, stopNano = 0; + long readFromRDbStartNs, readFromRDbStopNs = 0; // Get maxKeys from DB if it has. try (TableIterator> keyIter = getKeyTable(getBucketLayout()).iterator()) { - startNano = Time.monotonicNowNanos(); + readFromRDbStartNs = Time.monotonicNowNanos(); KeyValue< String, OmKeyInfo > kv; keyIter.seek(seekKey); // we need to iterate maxKeys + 1 here because if skipStartKey is true, @@ -1280,7 +1280,7 @@ public ListKeysResult listKeys(String volumeName, String bucketName, break; } } - stopNano = Time.monotonicNowNanos(); + readFromRDbStopNs = Time.monotonicNowNanos(); } boolean isTruncated = cacheKeyMap.size() > maxKeys; if (perfMetrics != null) { @@ -1291,10 +1291,10 @@ public ListKeysResult listKeys(String volumeName, String bucketName, averagePagination = cacheKeyMap.size(); } perfMetrics.setListKeysAveragePagination(averagePagination); - long opsPerSec = + long opsPerNs = averagePagination / (Time.monotonicNowNanos() - startNanos); - perfMetrics.setListKeysOpsPerSec(opsPerSec); - perfMetrics.addListKeysReadFromRocksDbLatencyNs(stopNano - startNano); + perfMetrics.setListKeysOpsPerNs(opsPerNs); + perfMetrics.addListKeysReadFromRocksDbLatencyNs(readFromRDbStopNs - readFromRDbStartNs); } // Finally DB entries and cache entries are merged, then return the count // of maxKeys from the sorted map. diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMultiTenantManagerImpl.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMultiTenantManagerImpl.java index d4b8898c5260..4530e526824d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMultiTenantManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMultiTenantManagerImpl.java @@ -66,7 +66,7 @@ public void setUp() throws IOException { conf = new OzoneConfiguration(); conf.set(OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); conf.set(OZONE_OM_TENANT_DEV_SKIP_RANGER, "true"); - omMetadataManager = new OmMetadataManagerImpl(conf, ozoneManager, new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(conf, ozoneManager); createTenantInDB(TENANT_ID); assignUserToTenantInDB(TENANT_ID, "seed-accessId1", "seed-user1", false, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index b64ab8e2c99c..5239fa009029 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -81,7 +81,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.params.provider.Arguments.arguments; -import static org.mockito.Mockito.mock; /** * Tests OzoneManager MetadataManager. @@ -99,9 +98,7 @@ public void setup() throws Exception { ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OZONE_OM_DB_DIRS, folder.getAbsolutePath()); - OMPerformanceMetrics perfMetrics = mock(OMPerformanceMetrics.class); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - perfMetrics); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); } @Test diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java index af406b2c0b15..05d30d544349 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java @@ -24,7 +24,6 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.hdds.utils.TransactionInfo; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.ratis.server.protocol.TermIndex; import org.junit.jupiter.api.AfterEach; @@ -73,8 +72,7 @@ public void setup() throws IOException { OzoneConfiguration configuration = new OzoneConfiguration(); configuration.set(OZONE_METADATA_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(configuration, null, - new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(configuration, null); doubleBuffer = OzoneManagerDoubleBuffer.newBuilder() .setOmMetadataManager(omMetadataManager) .setMaxUnFlushedTransactionCount(10000) @@ -164,14 +162,14 @@ private static class OMDummyCreateBucketResponse extends OMClientResponse { private final OmBucketInfo omBucketInfo; OMDummyCreateBucketResponse(OmBucketInfo omBucketInfo, - OMResponse omResponse) { + OMResponse omResponse) { super(omResponse); this.omBucketInfo = omBucketInfo; } @Override public void addToDBBatch(OMMetadataManager omMetadataManager, - BatchOperation batchOperation) throws IOException { + BatchOperation batchOperation) throws IOException { String dbBucketKey = omMetadataManager.getBucketKey(omBucketInfo.getVolumeName(), omBucketInfo.getBucketName()); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestSetRangerServiceVersionRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestSetRangerServiceVersionRequest.java index 76bd504c0444..a6b87b173507 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestSetRangerServiceVersionRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestSetRangerServiceVersionRequest.java @@ -24,7 +24,6 @@ import static org.mockito.Mockito.when; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -62,7 +61,7 @@ public void setUp() throws Exception { conf.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); when(ozoneManager.getMetadataManager()) - .thenReturn(new OmMetadataManagerImpl(conf, ozoneManager, new OMPerformanceMetrics())); + .thenReturn(new OmMetadataManagerImpl(conf, ozoneManager)); } @AfterEach diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java index 239eb317e2f3..e0ee5113f380 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java @@ -31,7 +31,6 @@ import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OMMetrics; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -270,7 +269,7 @@ private OmMetadataManagerImpl createOMMetadataManagerSpy() } ServerUtils.setOzoneMetaDirPath(conf, newFolder.toString()); return spy( - new OmMetadataManagerImpl(conf, null, new OMPerformanceMetrics())); + new OmMetadataManagerImpl(conf, null)); } private OMFileCreateRequest anOMFileCreateRequest() { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketCreateResponse.java index 64b476dbe7be..e81b0370920f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketCreateResponse.java @@ -24,7 +24,6 @@ import java.util.UUID; import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -59,8 +58,7 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java index e6d4575ef493..e96dd5528e68 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketDeleteResponse.java @@ -23,7 +23,6 @@ import java.nio.file.Path; import java.util.UUID; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -61,8 +60,7 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); batchOperation = omMetadataManager.getStore().initBatchOperation(); } @@ -102,7 +100,7 @@ public void testAddToDBBatch() throws Exception { omMetadataManager.getStore().commitBatchOperation(batchOperation); assertNull(omMetadataManager.getBucketTable().get( - omMetadataManager.getBucketKey(volumeName, bucketName))); + omMetadataManager.getBucketKey(volumeName, bucketName))); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketSetPropertyResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketSetPropertyResponse.java index 31e2c5b75b4f..b6cebfbc425f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketSetPropertyResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/bucket/TestOMBucketSetPropertyResponse.java @@ -24,7 +24,6 @@ import java.util.UUID; import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -59,8 +58,7 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java index f5f5eca76f6c..7d6487493861 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java @@ -27,7 +27,6 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -65,8 +64,7 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseWithFSO.java index 672ab7f368fc..8d0ce70a01e8 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseWithFSO.java @@ -28,7 +28,6 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -64,8 +63,7 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyResponse.java index 0b3ea976b0ce..bc4c34bd0db3 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyResponse.java @@ -28,7 +28,6 @@ import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -73,8 +72,7 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = getOzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); batchOperation = omMetadataManager.getStore().initBatchOperation(); volumeName = UUID.randomUUID().toString(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java index 908839d9a776..a2192ddb880c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java @@ -30,7 +30,6 @@ import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; @@ -78,8 +77,7 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMDelegationTokenResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMDelegationTokenResponse.java index 27313b10b05c..1c76ff8028c3 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMDelegationTokenResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMDelegationTokenResponse.java @@ -23,7 +23,6 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -48,7 +47,7 @@ public void setup() throws IOException { ((OzoneConfiguration) conf).set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl((OzoneConfiguration) conf, - null, new OMPerformanceMetrics()); + null); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java index 99c41bf90df1..7f74f3d17ecf 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; @@ -76,8 +75,7 @@ public void setup() throws Exception { String fsPath = folder.getAbsolutePath(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, fsPath); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java index f129c75b2ca0..4cb861bdf651 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; @@ -67,8 +66,7 @@ public void setup() throws Exception { String fsPath = folder.toAbsolutePath().toString(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, fsPath); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeCreateResponse.java index ae7641d18dd0..9d8de4bbb202 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeCreateResponse.java @@ -21,7 +21,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -59,8 +58,7 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeDeleteResponse.java index 1d61150ed968..70dd23a7b047 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeDeleteResponse.java @@ -21,7 +21,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -59,8 +58,7 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetOwnerResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetOwnerResponse.java index b817ec54ba27..aa640067ca45 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetOwnerResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetOwnerResponse.java @@ -21,7 +21,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -59,8 +58,7 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetQuotaResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetQuotaResponse.java index 442ce6e582c9..fbc8e3c944d9 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetQuotaResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/volume/TestOMVolumeSetQuotaResponse.java @@ -21,7 +21,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -58,8 +57,7 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null, - new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotChain.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotChain.java index a25823388d7e..c5ae809718e7 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotChain.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotChain.java @@ -21,7 +21,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; @@ -73,8 +72,7 @@ public class TestSnapshotChain { public void setup() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OZONE_OM_DB_DIRS, folder.toString()); - omMetadataManager = new OmMetadataManagerImpl(conf, null, - new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(conf, null); snapshotIdToSnapshotInfoMap = new HashMap<>(); chainManager = new SnapshotChainManager(omMetadataManager); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java index 6755d8f42dea..dc00433e179b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java @@ -21,7 +21,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus; @@ -56,8 +55,7 @@ public void setup() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(conf, null, - new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(conf, null); } private SnapshotInfo createSnapshotInfo() { diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java index fe7d9f4bf723..8a48fc3d5c6f 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java @@ -48,7 +48,6 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -86,7 +85,7 @@ public static OMMetadataManager initializeNewOmMetadataManager( omConfiguration.set(OMConfigKeys .OZONE_OM_ENABLE_FILESYSTEM_PATHS, "true"); OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null, new OMPerformanceMetrics()); + omConfiguration, null); String volumeKey = omMetadataManager.getVolumeKey("sampleVol"); OmVolumeArgs args = @@ -120,7 +119,7 @@ public static OMMetadataManager initializeEmptyOmMetadataManager( OzoneConfiguration omConfiguration = new OzoneConfiguration(); omConfiguration.set(OZONE_OM_DB_DIRS, omDbDir.getAbsolutePath()); - return new OmMetadataManagerImpl(omConfiguration, null, new OMPerformanceMetrics()); + return new OmMetadataManagerImpl(omConfiguration, null); } /** diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java index f1c88fc08b42..a244e4ff2ce2 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java @@ -20,7 +20,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; @@ -127,7 +126,7 @@ private static OMMetadataManager initializeNewOmMetadataManager( omConfiguration.set(OZONE_OM_DB_DIRS, omDbDir.getAbsolutePath()); OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null, new OMPerformanceMetrics()); + omConfiguration, null); return omMetadataManager; } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java index af2ee1d6ca3e..54da926601e5 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -998,7 +997,7 @@ private static OMMetadataManager initializeNewOmMetadataManager( omConfiguration.set(OZONE_OM_DB_DIRS, omDbDir.getAbsolutePath()); OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null, new OMPerformanceMetrics()); + omConfiguration, null); String volumeKey = omMetadataManager.getVolumeKey(VOL); OmVolumeArgs args = diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java index 5717a0d1f2a8..dba245ce8b80 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java @@ -35,7 +35,6 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -926,7 +925,7 @@ private static OMMetadataManager initializeNewOmMetadataManager( .OZONE_OM_ENABLE_FILESYSTEM_PATHS, "true"); setConfiguration(omConfiguration); OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null, new OMPerformanceMetrics()); + omConfiguration, null); String volumeKey = omMetadataManager.getVolumeKey(VOL); OmVolumeArgs args = diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java index 66b69d1b148c..6a2f2c557db8 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java @@ -37,7 +37,6 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -1100,7 +1099,7 @@ private static OMMetadataManager initializeNewOmMetadataManager( omConfiguration.set(OMConfigKeys .OZONE_OM_ENABLE_FILESYSTEM_PATHS, "false"); OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null, new OMPerformanceMetrics()); + omConfiguration, null); String volumeKey = omMetadataManager.getVolumeKey(VOL); OmVolumeArgs args = diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/recovery/TestReconOmMetadataManagerImpl.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/recovery/TestReconOmMetadataManagerImpl.java index 5680737b0973..b1e7e54d6d5c 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/recovery/TestReconOmMetadataManagerImpl.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/recovery/TestReconOmMetadataManagerImpl.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -163,7 +162,7 @@ private OMMetadataManager getOMMetadataManager() throws IOException { omConfiguration.set(OZONE_OM_DB_DIRS, omDbDir.getAbsolutePath()); OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null, new OMPerformanceMetrics()); + omConfiguration, null); //Create a volume + bucket + 2 keys. String volumeKey = omMetadataManager.getVolumeKey("sampleVol"); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java index 56414354d335..485804240d52 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hdds.utils.db.RDBBatchOperation; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -430,7 +429,7 @@ private static void initializeNewOmMetadataManager( omConfiguration.set(OMConfigKeys .OZONE_OM_ENABLE_FILESYSTEM_PATHS, "true"); omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null, new OMPerformanceMetrics()); + omConfiguration, null); String volumeKey = omMetadataManager.getVolumeKey(VOL); OmVolumeArgs args = diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java index 7748d39b7330..5ffd03cbb88a 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hdds.utils.db.RDBBatchOperation; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -696,7 +695,7 @@ private static void initializeNewOmMetadataManager( omConfiguration.set(OMConfigKeys .OZONE_OM_ENABLE_FILESYSTEM_PATHS, "true"); omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null, new OMPerformanceMetrics()); + omConfiguration, null); String volumeKey = omMetadataManager.getVolumeKey(VOL); OmVolumeArgs args = diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java index 1428d0c8f5ba..9676af015747 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java @@ -40,7 +40,6 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedTransactionLogIterator; import org.apache.hadoop.io.Text; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.codec.OMDBDefinition; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -82,11 +81,11 @@ private OzoneConfiguration createNewTestPath(String folderName) @BeforeEach public void setUp() throws Exception { OzoneConfiguration configuration = createNewTestPath("config"); - omMetadataManager = new OmMetadataManagerImpl(configuration, null, new OMPerformanceMetrics()); + omMetadataManager = new OmMetadataManagerImpl(configuration, null); OzoneConfiguration reconConfiguration = createNewTestPath("reconConfig"); reconOmMetadataManager = new OmMetadataManagerImpl(reconConfiguration, - null, new OMPerformanceMetrics()); + null); } @Test diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java index f108d8f142fc..5902064cacd2 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.Table.KeyValue; import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManagerUtils; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -109,7 +108,7 @@ public Class getParentType() { @Override public Void call() throws Exception { - parse(volume, bucket, dbPath, filePath, null); + parse(volume, bucket, dbPath, filePath); return null; } @@ -118,7 +117,7 @@ public static void main(String[] args) throws Exception { } public void parse(String vol, String buck, String db, - String file, OMPerformanceMetrics perfMetrics) throws Exception { + String file) throws Exception { if (!Files.exists(Paths.get(db))) { System.out.println("DB path not exist:" + db); return; @@ -129,13 +128,7 @@ public void parse(String vol, String buck, String db, OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OMConfigKeys.OZONE_OM_DB_DIRS, db); - OmMetadataManagerImpl metadataManager; - - if (perfMetrics != null) { - metadataManager = new OmMetadataManagerImpl(conf, null, perfMetrics); - } else { - metadataManager = new OmMetadataManagerImpl(conf, null); - } + OmMetadataManagerImpl metadataManager = new OmMetadataManagerImpl(conf, null); metadataManager.start(conf); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java index 29dedfea8c49..4a62b1bedb9a 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.ClientVersion; -import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -83,17 +82,6 @@ public static void main(String[] args) throws IOException { return getBlockIdDetailsUtils(metadataManager); } - public Map>> - parseOmDB(OzoneConfiguration configuration, OMPerformanceMetrics perfMetrics) throws IOException { - String path = configuration.get(OZONE_OM_DB_DIRS); - if (path == null || path.isEmpty()) { - throw new IOException(OZONE_OM_DB_DIRS + "should be set "); - } - OmMetadataManagerImpl metadataManager = - new OmMetadataManagerImpl(configuration, null, perfMetrics); - return getBlockIdDetailsUtils(metadataManager); - } - private static Map>> getBlockIdDetailsUtils( OmMetadataManagerImpl metadataManager) throws IOException { try { From bf9d575b4456912459d2abad424c3bf1ddb605a2 Mon Sep 17 00:00:00 2001 From: muskan1012 Date: Tue, 25 Jun 2024 11:32:11 +0530 Subject: [PATCH 22/25] minor nits fixed --- .../TestOzoneFileSystemPrefixParser.java | 1 + .../ozone/om/OmMetadataManagerImpl.java | 38 ++++++++++--------- ...eManagerDoubleBufferWithDummyResponse.java | 3 +- .../om/response/TestCleanupTableInfo.java | 3 +- .../TestOMDelegationTokenResponse.java | 2 +- .../TestOMSnapshotDeleteResponse.java | 1 + .../hadoop/ozone/debug/PrefixParser.java | 3 +- 7 files changed, 28 insertions(+), 23 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java index 4617c42f551f..37116f33e272 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java @@ -66,6 +66,7 @@ public static void init() throws Exception { bucketName = RandomStringUtils.randomAlphabetic(10).toLowerCase(); configuration = new OzoneConfiguration(); + cluster = MiniOzoneCluster.newBuilder(configuration) .setNumDatanodes(3) .build(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 3ee26e5c36a4..2009bed5812a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -346,7 +346,7 @@ public OmMetadataManagerImpl(OzoneConfiguration conf, this.perfMetrics = perfMetrics; // For test purpose only ignorePipelineinKey = conf.getBoolean( - "ozone.om.ignore.pipeline", Boolean.TRUE); + "ozone.om.ignore.pipeline", Boolean.TRUE); start(conf); } private void init(OzoneConfiguration conf, OzoneManager manager) { @@ -401,7 +401,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) lock = new OmReadOnlyLock(); omEpoch = 0; setStore(loadDB(conf, dir, name, true, - Optional.of(Boolean.TRUE), Optional.empty())); + java.util.Optional.of(Boolean.TRUE), Optional.empty())); initializeOmTables(CacheType.PARTIAL_CACHE, false); perfMetrics = null; } @@ -434,7 +434,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) checkSnapshotDirExist(checkpoint); } setStore(loadDB(conf, metaDir, dbName, false, - Optional.of(Boolean.TRUE), + java.util.Optional.of(Boolean.TRUE), Optional.of(maxOpenFiles), false, false)); initializeOmTables(CacheType.PARTIAL_CACHE, false); } catch (IOException e) { @@ -590,14 +590,14 @@ public void start(OzoneConfiguration configuration) throws IOException { public static DBStore loadDB(OzoneConfiguration configuration, File metaDir) throws IOException { return loadDB(configuration, metaDir, OM_DB_NAME, false, - Optional.empty(), Optional.empty(), true, true); + java.util.Optional.empty(), Optional.empty(), true, true); } public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, String dbName, boolean readOnly, - Optional + java.util.Optional disableAutoCompaction, - Optional maxOpenFiles) + java.util.Optional maxOpenFiles) throws IOException { return loadDB(configuration, metaDir, dbName, readOnly, disableAutoCompaction, maxOpenFiles, true, true); @@ -606,9 +606,9 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, @SuppressWarnings("checkstyle:parameternumber") public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, String dbName, boolean readOnly, - Optional + java.util.Optional disableAutoCompaction, - Optional maxOpenFiles, + java.util.Optional maxOpenFiles, boolean enableCompactionDag, boolean createCheckpointDirs) throws IOException { @@ -819,7 +819,7 @@ public DBStore getStore() { */ @Override public String getVolumeKey(String volume) { - return OM_KEY_PREFIX + volume; + return OzoneConsts.OM_KEY_PREFIX + volume; } /** @@ -1093,13 +1093,13 @@ public List listBuckets(final String volumeName, List result = new ArrayList<>(); if (Strings.isNullOrEmpty(volumeName)) { throw new OMException("Volume name is required.", - VOLUME_NOT_FOUND); + ResultCodes.VOLUME_NOT_FOUND); } String volumeNameBytes = getVolumeKey(volumeName); if (volumeTable.get(volumeNameBytes) == null) { throw new OMException("Volume " + volumeName + " not found.", - VOLUME_NOT_FOUND); + ResultCodes.VOLUME_NOT_FOUND); } String startKey; @@ -1192,18 +1192,18 @@ public ListKeysResult listKeys(String volumeName, String bucketName, if (Strings.isNullOrEmpty(volumeName)) { throw new OMException("Volume name is required.", - VOLUME_NOT_FOUND); + ResultCodes.VOLUME_NOT_FOUND); } if (Strings.isNullOrEmpty(bucketName)) { throw new OMException("Bucket name is required.", - BUCKET_NOT_FOUND); + ResultCodes.BUCKET_NOT_FOUND); } String bucketNameBytes = getBucketKey(volumeName, bucketName); if (getBucketTable().get(bucketNameBytes) == null) { throw new OMException("Bucket " + bucketName + " not found.", - BUCKET_NOT_FOUND); + ResultCodes.BUCKET_NOT_FOUND); } String seekKey; @@ -1282,7 +1282,9 @@ public ListKeysResult listKeys(String volumeName, String bucketName, } readFromRDbStopNs = Time.monotonicNowNanos(); } + boolean isTruncated = cacheKeyMap.size() > maxKeys; + if (perfMetrics != null) { long averagePagination; if (isTruncated) { @@ -1435,7 +1437,7 @@ public List listVolumes(String userName, int index = 0; if (!Strings.isNullOrEmpty(startKey)) { index = volumes.indexOf( - startKey.startsWith(OM_KEY_PREFIX) ? + startKey.startsWith(OzoneConsts.OM_KEY_PREFIX) ? startKey.substring(1) : startKey); @@ -1455,7 +1457,7 @@ public List listVolumes(String userName, // this probably means om db is corrupted or some entries are // accidentally removed. throw new OMException("Volume info not found for " + volumeName, - VOLUME_NOT_FOUND); + ResultCodes.VOLUME_NOT_FOUND); } result.add(volumeArgs); } @@ -1779,7 +1781,7 @@ public ExpiredOpenKeys getExpiredOpenKeys(Duration expireThreshold, = dbOpenKeyName.substring(lastPrefix + 1); final OmKeyInfo info = kt.get(dbKeyName); - final boolean isHsync = Optional.ofNullable(info) + final boolean isHsync = java.util.Optional.ofNullable(info) .map(WithMetadata::getMetadata) .map(meta -> meta.get(OzoneConsts.HSYNC_CLIENT_ID)) .filter(id -> id.equals(clientIdString)) @@ -1795,7 +1797,7 @@ public ExpiredOpenKeys getExpiredOpenKeys(Duration expireThreshold, .setBucketName(info.getBucketName()) .setKeyName(info.getKeyName()) .setDataSize(info.getDataSize()); - Optional.ofNullable(info.getLatestVersionLocations()) + java.util.Optional.ofNullable(info.getLatestVersionLocations()) .map(OmKeyLocationInfoGroup::getLocationList) .map(Collection::stream) .orElseGet(Stream::empty) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java index 05d30d544349..22272182997e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java @@ -72,7 +72,8 @@ public void setup() throws IOException { OzoneConfiguration configuration = new OzoneConfiguration(); configuration.set(OZONE_METADATA_DIRS, folder.toAbsolutePath().toString()); - omMetadataManager = new OmMetadataManagerImpl(configuration, null); + omMetadataManager = + new OmMetadataManagerImpl(configuration, null); doubleBuffer = OzoneManagerDoubleBuffer.newBuilder() .setOmMetadataManager(omMetadataManager) .setMaxUnFlushedTransactionCount(10000) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java index e0ee5113f380..1073e0323950 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java @@ -268,8 +268,7 @@ private OmMetadataManagerImpl createOMMetadataManagerSpy() assertTrue(newFolder.mkdirs()); } ServerUtils.setOzoneMetaDirPath(conf, newFolder.toString()); - return spy( - new OmMetadataManagerImpl(conf, null)); + return spy(new OmMetadataManagerImpl(conf, null)); } private OMFileCreateRequest anOMFileCreateRequest() { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMDelegationTokenResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMDelegationTokenResponse.java index 1c76ff8028c3..a5545f5a1d8b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMDelegationTokenResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMDelegationTokenResponse.java @@ -47,7 +47,7 @@ public void setup() throws IOException { ((OzoneConfiguration) conf).set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl((OzoneConfiguration) conf, - null); + null); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java index 4cb861bdf651..7c0cf29290f8 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java @@ -39,6 +39,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; + import java.io.File; import java.util.UUID; import java.nio.file.Path; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java index 5902064cacd2..784c71a9851d 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java @@ -128,7 +128,8 @@ public void parse(String vol, String buck, String db, OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OMConfigKeys.OZONE_OM_DB_DIRS, db); - OmMetadataManagerImpl metadataManager = new OmMetadataManagerImpl(conf, null); + OmMetadataManagerImpl metadataManager = + new OmMetadataManagerImpl(conf, null); metadataManager.start(conf); From b0a5832c06caada53d34ece5828af2ea9cb58ec2 Mon Sep 17 00:00:00 2001 From: muskan1012 Date: Mon, 1 Jul 2024 10:33:44 +0530 Subject: [PATCH 23/25] changes as per the review comment --- .../hadoop/ozone/om/OMPerformanceMetrics.java | 7 ++-- .../ozone/om/OmMetadataManagerImpl.java | 35 +++++++------------ .../om/response/TestCleanupTableInfo.java | 3 +- .../hadoop/ozone/debug/PrefixParser.java | 2 +- 4 files changed, 19 insertions(+), 28 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java index 89a0e47d115b..74bc5f87eade 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java @@ -19,6 +19,7 @@ import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.annotation.Metric; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MutableGaugeFloat; import org.apache.hadoop.metrics2.lib.MutableRate; /** @@ -117,7 +118,7 @@ public static void unregister() { private MutableRate listKeysAveragePagination; @Metric(about = "ops per second for listKeys") - private MutableRate listKeysOpsPerSec; + private MutableGaugeFloat listKeysOpsPerSec; @Metric(about = "ACLs check latency in listKeys") private MutableRate listKeysAclCheckLatencyNs; @@ -229,8 +230,8 @@ public void setListKeysAveragePagination(long averagePagination) { listKeysAveragePagination.add(averagePagination); } - public void setListKeysOpsPerNs(long opsPerNs) { - listKeysOpsPerSec.add(opsPerNs); + public void setListKeysOpsPerSec(float opsPerSec) { + listKeysOpsPerSec.set(opsPerSec); } MutableRate getListKeysAclCheckLatencyNs() { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 2009bed5812a..9a2df850bef7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -329,38 +329,29 @@ public class OmMetadataManagerImpl implements OMMetadataManager, */ public OmMetadataManagerImpl(OzoneConfiguration conf, OzoneManager ozoneManager) throws IOException { - init(conf, ozoneManager); - this.lock = new OzoneManagerLock(conf); - this.omEpoch = OmUtils.getOMEpoch(isRatisEnabled); - this.perfMetrics = null; - start(conf); + this(conf, ozoneManager, null); } public OmMetadataManagerImpl(OzoneConfiguration conf, OzoneManager ozoneManager, OMPerformanceMetrics perfMetrics) throws IOException { - init(conf, ozoneManager); + this.ozoneManager = ozoneManager; + this.perfMetrics = perfMetrics; this.lock = new OzoneManagerLock(conf); + // TODO: This is a temporary check. Once fully implemented, all OM state + // change should go through Ratis - be it standalone (for non-HA) or + // replicated (for HA). + isRatisEnabled = conf.getBoolean( + OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, + OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT); this.omEpoch = OmUtils.getOMEpoch(isRatisEnabled); - this.perfMetrics = perfMetrics; // For test purpose only ignorePipelineinKey = conf.getBoolean( "ozone.om.ignore.pipeline", Boolean.TRUE); start(conf); } - private void init(OzoneConfiguration conf, OzoneManager manager) { - this.ozoneManager = manager; - // TODO: This is a temporary check. Once fully implemented, all OM state - // change should go through Ratis - be it standalone (for non-HA) or - // replicated (for HA). - isRatisEnabled = conf.getBoolean( - OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, - OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT); - // For test purpose only - ignorePipelineinKey = conf.getBoolean( - "ozone.om.ignore.pipeline", Boolean.TRUE); - } + /** * For subclass overriding. */ @@ -1293,9 +1284,9 @@ public ListKeysResult listKeys(String volumeName, String bucketName, averagePagination = cacheKeyMap.size(); } perfMetrics.setListKeysAveragePagination(averagePagination); - long opsPerNs = - averagePagination / (Time.monotonicNowNanos() - startNanos); - perfMetrics.setListKeysOpsPerNs(opsPerNs); + float opsPerSec = + averagePagination / ((Time.monotonicNowNanos() - startNanos) / 1000000000.0f); + perfMetrics.setListKeysOpsPerSec(opsPerSec); perfMetrics.addListKeysReadFromRocksDbLatencyNs(readFromRDbStopNs - readFromRDbStartNs); } // Finally DB entries and cache entries are merged, then return the count diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java index 1073e0323950..6085a78c4ec7 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java @@ -260,8 +260,7 @@ private void addBucketToMetaTable(OmBucketInfo bucketInfo) * @throws IOException if I/O error occurs in setting up data store for the * metadata manager. */ - private OmMetadataManagerImpl createOMMetadataManagerSpy() - throws IOException { + private OmMetadataManagerImpl createOMMetadataManagerSpy() throws IOException { OzoneConfiguration conf = new OzoneConfiguration(); File newFolder = folder.toFile(); if (!newFolder.exists()) { diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java index 784c71a9851d..fabc7f456aec 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java @@ -128,9 +128,9 @@ public void parse(String vol, String buck, String db, OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OMConfigKeys.OZONE_OM_DB_DIRS, db); + OmMetadataManagerImpl metadataManager = new OmMetadataManagerImpl(conf, null); - metadataManager.start(conf); org.apache.hadoop.fs.Path effectivePath = From 1c83093c3dbda50d339c991feb4d1ec776e43f19 Mon Sep 17 00:00:00 2001 From: muskan1012 <55579937+muskan1012@users.noreply.github.com> Date: Mon, 1 Jul 2024 11:38:44 +0530 Subject: [PATCH 24/25] Apply suggestions from code review Co-authored-by: Doroszlai, Attila <6454655+adoroszlai@users.noreply.github.com> --- .../org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 6 +++--- .../hadoop/ozone/om/response/TestCleanupTableInfo.java | 2 +- .../java/org/apache/hadoop/ozone/fsck/ContainerMapper.java | 5 ----- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 9a2df850bef7..daaaeb98e9df 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -343,12 +343,12 @@ public OmMetadataManagerImpl(OzoneConfiguration conf, // change should go through Ratis - be it standalone (for non-HA) or // replicated (for HA). isRatisEnabled = conf.getBoolean( - OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, - OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT); + OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, + OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT); this.omEpoch = OmUtils.getOMEpoch(isRatisEnabled); // For test purpose only ignorePipelineinKey = conf.getBoolean( - "ozone.om.ignore.pipeline", Boolean.TRUE); + "ozone.om.ignore.pipeline", Boolean.TRUE); start(conf); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java index 6085a78c4ec7..1e0c421333d2 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java @@ -260,7 +260,7 @@ private void addBucketToMetaTable(OmBucketInfo bucketInfo) * @throws IOException if I/O error occurs in setting up data store for the * metadata manager. */ - private OmMetadataManagerImpl createOMMetadataManagerSpy() throws IOException { + private OMMetadataManager createOMMetadataManagerSpy() throws IOException { OzoneConfiguration conf = new OzoneConfiguration(); File newFolder = folder.toFile(); if (!newFolder.exists()) { diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java index 4a62b1bedb9a..a7a74c2e3729 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java @@ -79,11 +79,6 @@ public static void main(String[] args) throws IOException { } OmMetadataManagerImpl metadataManager = new OmMetadataManagerImpl(configuration, null); - return getBlockIdDetailsUtils(metadataManager); - } - - private static Map>> getBlockIdDetailsUtils( - OmMetadataManagerImpl metadataManager) throws IOException { try { Table keyTable = metadataManager.getKeyTable(getBucketLayout()); From 8139ae78f4905c8135c7baefc7496437994fefb0 Mon Sep 17 00:00:00 2001 From: muskan1012 Date: Mon, 8 Jul 2024 15:39:03 +0530 Subject: [PATCH 25/25] changed naming convention as per the review comment --- .../apache/hadoop/ozone/om/OMPerformanceMetrics.java | 4 ++-- .../apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java index 74bc5f87eade..a14adc80ddec 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java @@ -226,8 +226,8 @@ public MutableRate getValidateAndUpdateCacheLatencyNs() { return validateAndUpdateCacheLatencyNs; } - public void setListKeysAveragePagination(long averagePagination) { - listKeysAveragePagination.add(averagePagination); + public void setListKeysAveragePagination(long keyCount) { + listKeysAveragePagination.add(keyCount); } public void setListKeysOpsPerSec(float opsPerSec) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index daaaeb98e9df..9f90643a17cc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1277,15 +1277,15 @@ public ListKeysResult listKeys(String volumeName, String bucketName, boolean isTruncated = cacheKeyMap.size() > maxKeys; if (perfMetrics != null) { - long averagePagination; + long keyCount; if (isTruncated) { - averagePagination = maxKeys; + keyCount = maxKeys; } else { - averagePagination = cacheKeyMap.size(); + keyCount = cacheKeyMap.size(); } - perfMetrics.setListKeysAveragePagination(averagePagination); + perfMetrics.setListKeysAveragePagination(keyCount); float opsPerSec = - averagePagination / ((Time.monotonicNowNanos() - startNanos) / 1000000000.0f); + keyCount / ((Time.monotonicNowNanos() - startNanos) / 1000000000.0f); perfMetrics.setListKeysOpsPerSec(opsPerSec); perfMetrics.addListKeysReadFromRocksDbLatencyNs(readFromRDbStopNs - readFromRDbStartNs); }