From b858f85a3b4d3a1d4ceb530f3d1c914ac29126da Mon Sep 17 00:00:00 2001 From: arafat Date: Sat, 20 Jul 2024 02:29:01 +0530 Subject: [PATCH 01/21] HDDS-11205. Implement a search feature for users to locate keys pending Deletion within the OM Deleted Keys Insights section. --- .../hadoop/ozone/recon/ReconConstants.java | 6 +- .../apache/hadoop/ozone/recon/ReconUtils.java | 45 ++ .../recon/api/OMDBInsightSearchEndpoint.java | 157 ++++-- .../api/TestDeletedKeysSearchEndpoint.java | 456 ++++++++++++++++++ 4 files changed, 617 insertions(+), 47 deletions(-) create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java index ed657931e034..0c1a2287b4c4 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java @@ -43,7 +43,6 @@ private ReconConstants() { public static final int DISK_USAGE_TOP_RECORDS_LIMIT = 30; public static final String DEFAULT_OPEN_KEY_INCLUDE_NON_FSO = "false"; public static final String DEFAULT_OPEN_KEY_INCLUDE_FSO = "false"; - public static final String DEFAULT_START_PREFIX = "/"; public static final String DEFAULT_FETCH_COUNT = "1000"; public static final String DEFAULT_KEY_SIZE = "0"; public static final String DEFAULT_BATCH_NUMBER = "1"; @@ -51,8 +50,9 @@ private ReconConstants() { public static final String RECON_QUERY_PREVKEY = "prevKey"; public static final String RECON_OPEN_KEY_INCLUDE_NON_FSO = "includeNonFso"; public static final String RECON_OPEN_KEY_INCLUDE_FSO = "includeFso"; - public static final String RECON_OPEN_KEY_DEFAULT_SEARCH_LIMIT = "1000"; - public static final String RECON_OPEN_KEY_SEARCH_DEFAULT_PREV_KEY = ""; + public static final String RECON_OM_INSIGHTS_DEFAULT_START_PREFIX = "/"; + public static final String RECON_OM_INSIGHTS_DEFAULT_SEARCH_LIMIT = "1000"; + public static final String RECON_OM_INSIGHTS_DEFAULT_SEARCH_PREV_KEY = ""; public static final String RECON_QUERY_FILTER = "missingIn"; public static final String PREV_CONTAINER_ID_DEFAULT_VALUE = "0"; public static final String PREV_DELETED_BLOCKS_TRANSACTION_ID_DEFAULT_VALUE = diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index 5c9f6a5f4e12..9f7c0f6573e7 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -32,6 +32,8 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.List; import java.util.TimeZone; import java.util.Date; @@ -54,6 +56,8 @@ import org.apache.hadoop.hdds.scm.ha.SCMNodeDetails; import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher; import org.apache.hadoop.hdds.utils.HddsServerUtil; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.hdfs.web.URLConnectionFactory; import org.apache.hadoop.io.IOUtils; @@ -596,6 +600,47 @@ public static long convertToEpochMillis(String dateString, String dateFormat, Ti } } + /** + * Common method to retrieve keys from a table based on a search prefix and a limit. + * + * @param table The table to retrieve keys from. + * @param startPrefix The search prefix to match keys against. + * @param limit The maximum number of keys to retrieve. + * @param prevKey The key to start after for the next set of records. + * @return A map of keys and their corresponding OmKeyInfo or RepeatedOmKeyInfo objects. + * @throws IOException If there are problems accessing the table. + */ + public static Map retrieveKeysFromTable( + Table table, String startPrefix, int limit, String prevKey) + throws IOException { + Map matchedKeys = new LinkedHashMap<>(); + try (TableIterator> keyIter = table.iterator()) { + // If a previous key is provided, seek to the previous key and skip it. + if (!prevKey.isEmpty()) { + keyIter.seek(prevKey); + if (keyIter.hasNext()) { + // Skip the previous key + keyIter.next(); + } + } else { + // If no previous key is provided, start from the search prefix. + keyIter.seek(startPrefix); + } + while (keyIter.hasNext() && matchedKeys.size() < limit) { + Table.KeyValue entry = keyIter.next(); + String dbKey = entry.getKey(); + if (!dbKey.startsWith(startPrefix)) { + break; // Exit the loop if the key no longer matches the prefix + } + matchedKeys.put(dbKey, entry.getValue()); + } + } catch (IOException exception) { + log.error("Error retrieving keys from table for path: {}", startPrefix, exception); + throw exception; + } + return matchedKeys; + } + /** * Finds all subdirectories under a parent directory in an FSO bucket. It builds * a list of paths for these subdirectories. These sub-directories are then used diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java index 9cd6fa33d032..27744c5addcd 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java @@ -20,12 +20,11 @@ import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.recon.ReconUtils; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; import org.apache.hadoop.ozone.recon.api.types.KeyEntityInfo; import org.apache.hadoop.ozone.recon.api.types.KeyInsightInfoResponse; @@ -50,13 +49,15 @@ import java.util.ArrayList; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_START_PREFIX; -import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OPEN_KEY_DEFAULT_SEARCH_LIMIT; -import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OPEN_KEY_SEARCH_DEFAULT_PREV_KEY; +import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OM_INSIGHTS_DEFAULT_START_PREFIX; +import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OM_INSIGHTS_DEFAULT_SEARCH_LIMIT; +import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OM_INSIGHTS_DEFAULT_SEARCH_PREV_KEY; import static org.apache.hadoop.ozone.recon.ReconResponseUtils.noMatchedKeysResponse; import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createBadRequestResponse; import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createInternalServerErrorResponse; import static org.apache.hadoop.ozone.recon.ReconUtils.constructObjectPathWithPrefix; +import static org.apache.hadoop.ozone.recon.ReconUtils.retrieveKeysFromTable; +import static org.apache.hadoop.ozone.recon.ReconUtils.gatherSubPaths; import static org.apache.hadoop.ozone.recon.ReconUtils.validateNames; import static org.apache.hadoop.ozone.recon.api.handlers.BucketHandler.getBucketHandler; import static org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.normalizePath; @@ -64,6 +65,11 @@ /** * REST endpoint for search implementation in OM DB Insight. + * + * This class provides endpoints for searching keys in the Ozone Manager database. + * It supports searching for both open and deleted keys across File System Optimized (FSO) + * and Object Store (non-FSO) bucket layouts. The results include matching keys and their + * data sizes. */ @Path("/keys") @Produces(MediaType.APPLICATION_JSON) @@ -88,14 +94,14 @@ public OMDBInsightSearchEndpoint(OzoneStorageContainerManager reconSCM, /** - * Performs a search for open keys in the Ozone Manager (OM) database using a specified search prefix. + * Performs a search for open keys in the Ozone Manager OpenKey and OpenFile table using a specified search prefix. * This endpoint searches across both File System Optimized (FSO) and Object Store (non-FSO) layouts, * compiling a list of keys that match the given prefix along with their data sizes. - *

+ * * The search prefix must start from the bucket level ('/volumeName/bucketName/') or any specific directory * or key level (e.g., '/volA/bucketA/dir1' for everything under 'dir1' inside 'bucketA' of 'volA'). * The search operation matches the prefix against the start of keys' names within the OM DB. - *

+ * * Example Usage: * 1. A startPrefix of "/volA/bucketA/" retrieves every key under bucket 'bucketA' in volume 'volA'. * 2. Specifying "/volA/bucketA/dir1" focuses the search within 'dir1' inside 'bucketA' of 'volA'. @@ -110,11 +116,12 @@ public OMDBInsightSearchEndpoint(OzoneStorageContainerManager reconSCM, @GET @Path("/open/search") public Response searchOpenKeys( - @DefaultValue(DEFAULT_START_PREFIX) @QueryParam("startPrefix") + @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_START_PREFIX) @QueryParam("startPrefix") String startPrefix, - @DefaultValue(RECON_OPEN_KEY_DEFAULT_SEARCH_LIMIT) @QueryParam("limit") + @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_SEARCH_LIMIT) @QueryParam("limit") int limit, - @DefaultValue(RECON_OPEN_KEY_SEARCH_DEFAULT_PREV_KEY) @QueryParam("prevKey") String prevKey) throws IOException { + @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_SEARCH_PREV_KEY) @QueryParam("prevKey") + String prevKey) throws IOException { try { // Ensure startPrefix is not null or empty and starts with '/' @@ -221,7 +228,7 @@ public Map searchOpenKeysInFSO(String startPrefix, subPaths.add(startPrefixObjectPath); // Recursively gather all subpaths - ReconUtils.gatherSubPaths(parentId, subPaths, Long.parseLong(names[0]), Long.parseLong(names[1]), + gatherSubPaths(parentId, subPaths, Long.parseLong(names[0]), Long.parseLong(names[1]), reconNamespaceSummaryManager); // Iterate over the subpaths and retrieve the open files @@ -325,46 +332,108 @@ public String convertToObjectPath(String prevKeyPrefix) throws IOException { return prevKeyPrefix; } - /** - * Common method to retrieve keys from a table based on a search prefix and a limit. + * Performs a search for deleted keys in the Ozone Manager DeletedTable using a specified search prefix. + * This endpoint searches across both File System Optimized (FSO) and Object Store (non-FSO) layouts, + * compiling a list of keys that match the given prefix along with their data sizes. + * + * The search prefix must start from the bucket level ('/volumeName/bucketName/') or any specific directory + * or key level (e.g., '/volA/bucketA/dir1' for everything under 'dir1' inside 'bucketA' of 'volA'). + * The search operation matches the prefix against the start of keys' names within the OM DB DeletedTable. + * + * Example Usage: + * 1. A startPrefix of "/volA/bucketA/" retrieves every key under bucket 'bucketA' in volume 'volA'. + * 2. Specifying "/volA/bucketA/dir1" focuses the search within 'dir1' inside 'bucketA' of 'volA'. * - * @param table The table to retrieve keys from. - * @param startPrefix The search prefix to match keys against. - * @param limit The maximum number of keys to retrieve. + * @param startPrefix The prefix for searching keys, starting from the bucket level or any specific path. + * @param limit Limits the number of returned keys. * @param prevKey The key to start after for the next set of records. - * @return A map of keys and their corresponding OmKeyInfo objects. - * @throws IOException If there are problems accessing the table. + * @return A KeyInsightInfoResponse, containing matching keys and their data sizes. + * @throws IOException On failure to access the OM database or process the operation. */ - private Map retrieveKeysFromTable( - Table table, String startPrefix, int limit, String prevKey) - throws IOException { - Map matchedKeys = new LinkedHashMap<>(); - try (TableIterator> keyIter = table.iterator()) { - // If a previous key is provided, seek to the previous key and skip it. - if (!prevKey.isEmpty()) { - keyIter.seek(prevKey); - if (keyIter.hasNext()) { - // Skip the previous key - keyIter.next(); - } - } else { - // If no previous key is provided, start from the search prefix. - keyIter.seek(startPrefix); + @GET + @Path("/deletePending/search") + public Response searchDeletedKeys( + @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_START_PREFIX) @QueryParam("startPrefix") + String startPrefix, + @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_SEARCH_LIMIT) @QueryParam("limit") + int limit, + @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_SEARCH_PREV_KEY) @QueryParam("prevKey") + String prevKey) throws IOException { + + try { + // Ensure startPrefix is not null or empty and starts with '/' + if (startPrefix == null || startPrefix.length() == 0) { + return createBadRequestResponse( + "Invalid startPrefix: Path must be at the bucket level or deeper."); + } + startPrefix = startPrefix.startsWith("/") ? startPrefix : "/" + startPrefix; + + // Split the path to ensure it's at least at the bucket level + String[] pathComponents = startPrefix.split("/"); + if (pathComponents.length < 3 || pathComponents[2].isEmpty()) { + return createBadRequestResponse( + "Invalid startPrefix: Path must be at the bucket level or deeper."); } - while (keyIter.hasNext() && matchedKeys.size() < limit) { - Table.KeyValue entry = keyIter.next(); - String dbKey = entry.getKey(); - if (!dbKey.startsWith(startPrefix)) { - break; // Exit the loop if the key no longer matches the prefix + + // Ensure the limit is non-negative + limit = Math.max(0, limit); + + // Initialize response object + KeyInsightInfoResponse insightResponse = new KeyInsightInfoResponse(); + long replicatedTotal = 0; + long unreplicatedTotal = 0; + boolean keysFound = false; // Flag to track if any keys are found + String lastKey = null; + + // Search for deleted keys in DeletedTable + Table deletedTable = omMetadataManager.getDeletedTable(); + Map deletedKeys = retrieveKeysFromTable(deletedTable, startPrefix, limit, prevKey); + + for (Map.Entry entry : deletedKeys.entrySet()) { + keysFound = true; + RepeatedOmKeyInfo repeatedOmKeyInfo = entry.getValue(); + + for (OmKeyInfo keyInfo : repeatedOmKeyInfo.getOmKeyInfoList()) { + KeyEntityInfo keyEntityInfo = createKeyEntityInfoFromOmKeyInfo(entry.getKey(), keyInfo); + + // Fetch bucket info and classify key as FSO or Non-FSO + String bucketKey = omMetadataManager.getBucketKey(keyInfo.getVolumeName(), keyInfo.getBucketName()); + OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().getSkipCache(bucketKey); + if (bucketInfo != null) { + if (bucketInfo.getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) { + insightResponse.getFsoKeyInfoList().add(keyEntityInfo); // Add to FSO list + } else { + insightResponse.getNonFSOKeyInfoList().add(keyEntityInfo); // Add to non-FSO list + } + replicatedTotal += keyInfo.getReplicatedSize(); + unreplicatedTotal += keyInfo.getDataSize(); + } } - matchedKeys.put(dbKey, entry.getValue()); + lastKey = entry.getKey(); // Update lastKey } - } catch (IOException exception) { - LOG.error("Error retrieving keys from table for path: {}", startPrefix, exception); - throw exception; + + // If no keys were found, return a response indicating that no keys matched + if (!keysFound) { + return noMatchedKeysResponse(startPrefix); + } + + // Set the aggregated totals in the response + insightResponse.setReplicatedDataSize(replicatedTotal); + insightResponse.setUnreplicatedDataSize(unreplicatedTotal); + insightResponse.setLastKey(lastKey); + + // Return the response with the matched keys and their data sizes + return Response.ok(insightResponse).build(); + } catch (IOException e) { + // Handle IO exceptions and return an internal server error response + return createInternalServerErrorResponse( + "Error searching deleted keys in OM DB: " + e.getMessage()); + } catch (IllegalArgumentException e) { + // Handle illegal argument exceptions and return a bad request response + return createBadRequestResponse( + "Invalid startPrefix: " + e.getMessage()); } - return matchedKeys; } /** diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java new file mode 100644 index 000000000000..f226b39f4271 --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java @@ -0,0 +1,456 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.recon.ReconTestInjector; +import org.apache.hadoop.ozone.recon.api.types.KeyInsightInfoResponse; +import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest; +import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import javax.ws.rs.core.Response; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; +import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * Test class for DeletedKeysSearchEndpoint. + * + * This class tests various scenarios for searching deleted keys within a + * given volume, bucket, and directory structure. The tests include: + * + * 1. Test Root Level Search Restriction: Ensures searching at the root level returns a bad request. + * 2. Test Volume Level Search Restriction: Ensures searching at the volume level returns a bad request. + * 3. Test Bucket Level Search: Verifies search results within different types of buckets, both FSO and OBS. + * 4. Test Directory Level Search: Validates searching inside specific directories. + * 5. Test Key Level Search: Confirms search results for specific keys within buckets, both FSO and OBS. + * 6. Test Key Level Search Under Directory: Verifies searching for keys within nested directories. + * 7. Test Search Under Nested Directory: Checks search results within nested directories. + * 8. Test Limit Search: Tests the limit functionality of the search API. + * 9. Test Search Deleted Keys with Bad Request: Ensures bad requests with invalid params return correct responses. + * 10. Test Last Key in Response: Confirms the presence of the last key in paginated responses. + * 11. Test Search Deleted Keys with Pagination: Verifies paginated search results. + * 12. Test Search in Empty Bucket: Checks the response for searching within an empty bucket. + */ +public class TestDeletedKeysSearchEndpoint extends AbstractReconSqlDBTest { + + @TempDir + private Path temporaryFolder; + private ReconOMMetadataManager reconOMMetadataManager; + private OMDBInsightSearchEndpoint deletedKeysSearchEndpoint; + private OzoneConfiguration ozoneConfiguration; + private static final String ROOT_PATH = "/"; + private OMMetadataManager omMetadataManager; + + @BeforeEach + public void setUp() throws Exception { + ozoneConfiguration = new OzoneConfiguration(); + ozoneConfiguration.setLong(OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD, 100); + omMetadataManager = initializeNewOmMetadataManager( + Files.createDirectory(temporaryFolder.resolve("JunitOmDBDir")).toFile()); + reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, + Files.createDirectory(temporaryFolder.resolve("OmMetataDir")).toFile()); + + ReconTestInjector reconTestInjector = + new ReconTestInjector.Builder(temporaryFolder.toFile()) + .withReconSqlDb() + .withReconOm(reconOMMetadataManager) + .withOmServiceProvider(mock(OzoneManagerServiceProviderImpl.class)) + .addBinding(StorageContainerServiceProvider.class, + mock(StorageContainerServiceProviderImpl.class)) + .addBinding(OzoneStorageContainerManager.class, + mock(OzoneStorageContainerManager.class)) + .addBinding(ReconNamespaceSummaryManager.class, + mock(ReconNamespaceSummaryManager.class)) + .addBinding(OMDBInsightSearchEndpoint.class) + .addBinding(ContainerHealthSchemaManager.class) + .build(); + deletedKeysSearchEndpoint = reconTestInjector.getInstance(OMDBInsightSearchEndpoint.class); + + populateOMDB(); + } + + + private static OMMetadataManager initializeNewOmMetadataManager(File omDbDir) throws IOException { + OzoneConfiguration omConfiguration = new OzoneConfiguration(); + omConfiguration.set(OZONE_OM_DB_DIRS, omDbDir.getAbsolutePath()); + return new OmMetadataManagerImpl(omConfiguration, null); + } + + @Test + public void testRootLevelSearchRestriction() throws IOException { + String rootPath = "/"; + Response response = deletedKeysSearchEndpoint.searchDeletedKeys(rootPath, 20, ""); + assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); + String entity = (String) response.getEntity(); + assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), + "Expected a message indicating the path must be at the bucket level or deeper"); + + rootPath = ""; + response = deletedKeysSearchEndpoint.searchDeletedKeys(rootPath, 20, ""); + assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); + entity = (String) response.getEntity(); + assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), + "Expected a message indicating the path must be at the bucket level or deeper"); + } + + @Test + public void testVolumeLevelSearchRestriction() throws IOException { + String volumePath = "/vola"; + Response response = deletedKeysSearchEndpoint.searchDeletedKeys(volumePath, 20, ""); + assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); + String entity = (String) response.getEntity(); + assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), + "Expected a message indicating the path must be at the bucket level or deeper"); + + volumePath = "/volb"; + response = deletedKeysSearchEndpoint.searchDeletedKeys(volumePath, 20, ""); + assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); + entity = (String) response.getEntity(); + assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), + "Expected a message indicating the path must be at the bucket level or deeper"); + } + + @Test + public void testBucketLevelSearch() throws IOException { + // Search inside FSO bucket + Response response = deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1", 20, ""); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(7, result.getFsoKeyInfoList().size()); + + response = deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1", 2, ""); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(2, result.getFsoKeyInfoList().size()); + + // Search inside OBS bucket + response = deletedKeysSearchEndpoint.searchDeletedKeys("/volc/bucketc1", 20, ""); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(9, result.getNonFSOKeyInfoList().size()); + + response = deletedKeysSearchEndpoint.searchDeletedKeys("/vola/nonexistentbucket", 20, ""); + assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + String entity = (String) response.getEntity(); + assertTrue(entity.contains("No keys matched the search prefix"), + "Expected a message indicating no keys were found"); + } + + @Test + public void testDirectoryLevelSearch() throws IOException { + Response response = deletedKeysSearchEndpoint.searchDeletedKeys("/volc/bucketc1/dirc1", 20, + ""); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(4, result.getNonFSOKeyInfoList().size()); + + response = deletedKeysSearchEndpoint.searchDeletedKeys("/volc/bucketc1/dirc2", 20, + ""); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(5, result.getNonFSOKeyInfoList().size()); + + response = deletedKeysSearchEndpoint.searchDeletedKeys( + "/volb/bucketb1/nonexistentdir", 20, ""); + assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + String entity = (String) response.getEntity(); + assertTrue(entity.contains("No keys matched the search prefix"), + "Expected a message indicating no keys were found"); + } + + @Test + public void testKeyLevelSearch() throws IOException { + // FSO Bucket key-level search + Response response = + deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1/fileb1", 10, + ""); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(1, result.getFsoKeyInfoList().size()); + + response = deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1/fileb2", 10, + ""); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(1, result.getFsoKeyInfoList().size()); + + // Test with non-existent key + response = deletedKeysSearchEndpoint.searchDeletedKeys( + "/volb/bucketb1/nonexistentfile", 1, ""); + assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + String entity = (String) response.getEntity(); + assertTrue(entity.contains("No keys matched the search prefix"), + "Expected a message indicating no keys were found"); + } + + @Test + public void testKeyLevelSearchUnderDirectory() throws IOException { + // FSO Bucket key-level search under directory + Response response = + deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1/dir1/file1", + 10, ""); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = + (KeyInsightInfoResponse) response.getEntity(); + assertEquals(1, result.getFsoKeyInfoList().size()); + + response = deletedKeysSearchEndpoint.searchDeletedKeys( + "/volb/bucketb1/dir1/nonexistentfile", 10, ""); + assertEquals(Response.Status.NOT_FOUND.getStatusCode(), + response.getStatus()); + String entity = (String) response.getEntity(); + assertTrue(entity.contains("No keys matched the search prefix"), + "Expected a message indicating no keys were found"); + } + + @Test + public void testSearchUnderNestedDirectory() throws IOException { + // OBS Bucket nested directory search + Response response = + deletedKeysSearchEndpoint.searchDeletedKeys("/volc/bucketc1/dirc1", 20, ""); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(4, result.getNonFSOKeyInfoList().size()); + + response = deletedKeysSearchEndpoint.searchDeletedKeys( + "/volc/bucketc1/dirc1/dirc11", 20, ""); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(2, result.getNonFSOKeyInfoList().size()); + + response = deletedKeysSearchEndpoint.searchDeletedKeys( + "/volc/bucketc1/dirc1/dirc11/dirc111", 20, ""); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(1, result.getNonFSOKeyInfoList().size()); + + response = deletedKeysSearchEndpoint.searchDeletedKeys( + "/volc/bucketc1/dirc1/dirc11/dirc111/nonexistentfile", 20, ""); + assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + String entity = (String) response.getEntity(); + assertTrue(entity.contains("No keys matched the search prefix"), + "Expected a message indicating no keys were found"); + + response = deletedKeysSearchEndpoint.searchDeletedKeys( + "/volc/bucketc1/dirc1/dirc11/nonexistentfile", 20, ""); + assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + entity = (String) response.getEntity(); + assertTrue(entity.contains("No keys matched the search prefix"), + "Expected a message indicating no keys were found"); + } + + @Test + public void testLimitSearch() throws IOException { + Response response = deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1", 2, ""); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(2, result.getFsoKeyInfoList().size()); + } + + @Test + public void testSearchDeletedKeysWithBadRequest() throws IOException { + int negativeLimit = -1; + Response response = deletedKeysSearchEndpoint.searchDeletedKeys("@323232", negativeLimit, ""); + assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); + String entity = (String) response.getEntity(); + assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), + "Expected a message indicating the path must be at the bucket level or deeper"); + + response = deletedKeysSearchEndpoint.searchDeletedKeys("///", 20, ""); + assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); + entity = (String) response.getEntity(); + assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), + "Expected a message indicating the path must be at the bucket level or deeper"); + } + + @Test + public void testLastKeyInResponse() throws IOException { + Response response = deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1", 20, ""); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(7, result.getFsoKeyInfoList().size()); + assertEquals(result.getFsoKeyInfoList().get(6).getKey(), result.getLastKey(), + "Expected last key to be 'fileb5'"); + } + + @Test + public void testSearchDeletedKeysWithPagination() throws IOException { + String startPrefix = "/volb/bucketb1"; + int limit = 2; + String prevKey = ""; + + Response response = deletedKeysSearchEndpoint.searchDeletedKeys(startPrefix, limit, prevKey); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(2, result.getFsoKeyInfoList().size()); + + prevKey = result.getLastKey(); + assertNotNull(prevKey, "Last key should not be null"); + + response = deletedKeysSearchEndpoint.searchDeletedKeys(startPrefix, limit, prevKey); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(2, result.getFsoKeyInfoList().size()); + + prevKey = result.getLastKey(); + assertNotNull(prevKey, "Last key should not be null"); + + response = deletedKeysSearchEndpoint.searchDeletedKeys(startPrefix, limit, prevKey); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(2, result.getFsoKeyInfoList().size()); + assertEquals(result.getFsoKeyInfoList().get(1).getKey(), + result.getLastKey(), "Expected last key to be empty"); + + prevKey = result.getLastKey(); + assertNotNull(prevKey, "Last key should not be null"); + + response = deletedKeysSearchEndpoint.searchDeletedKeys(startPrefix, limit, prevKey); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(1, result.getFsoKeyInfoList().size()); + assertEquals(result.getFsoKeyInfoList().get(0).getKey(), + result.getLastKey(), "Expected last key to be empty"); + } + + @Test + public void testSearchInEmptyBucket() throws IOException { + Response response = + deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb2", 20, ""); + assertEquals(404, response.getStatus()); + String entity = (String) response.getEntity(); + assertTrue(entity.contains("No keys matched the search prefix"), + "Expected a message indicating no keys were found"); + } + + private void populateOMDB() throws Exception { + // Create FSO bucket + createBucket("volb", "bucketb1", BucketLayout.FILE_SYSTEM_OPTIMIZED); + + // Create OBS bucket + createBucket("volc", "bucketc1", BucketLayout.OBJECT_STORE); + + createDeletedKey("fileb1", "bucketb1", "volb", 1000); + createDeletedKey("fileb2", "bucketb1", "volb", 1000); + createDeletedKey("fileb3", "bucketb1", "volb", 1000); + createDeletedKey("fileb4", "bucketb1", "volb", 1000); + createDeletedKey("fileb5", "bucketb1", "volb", 1000); + + createDeletedKey("dir1/file1", "bucketb1", "volb", 1000); + createDeletedKey("dir1/file2", "bucketb1", "volb", 1000); + + createDeletedKey("dirc1/filec1", "bucketc1", "volc", 1000); + createDeletedKey("dirc1/filec2", "bucketc1", "volc", 1000); + createDeletedKey("dirc2/filec3", "bucketc1", "volc", 1000); + createDeletedKey("dirc2/filec4", "bucketc1", "volc", 1000); + createDeletedKey("dirc2/filec5", "bucketc1", "volc", 1000); + createDeletedKey("dirc2/filgetec6", "bucketc1", "volc", 1000); + createDeletedKey("dirc2/filec7", "bucketc1", "volc", 1000); + + // create nested directories and files in bucketc1 + createDeletedKey("dirc1/dirc11/filec11", "bucketc1", "volc", 1000); + createDeletedKey("dirc1/dirc11/dirc111/filec111", "bucketc1", "volc", 1000); + } + + private void createBucket(String volumeName, String bucketName, BucketLayout bucketLayout) throws Exception { + String bucketKey = reconOMMetadataManager.getBucketKey(volumeName, bucketName); + long bucketId = UUID.randomUUID().getMostSignificantBits() & Long.MAX_VALUE; // Generate positive ID + OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setObjectID(bucketId) + .setBucketLayout(bucketLayout) + .build(); + reconOMMetadataManager.getBucketTable().put(bucketKey, bucketInfo); + } + + private void createDeletedKey(String keyName, String bucketName, + String volumeName, long dataSize) throws IOException { + // Construct the deleted key path + String deletedKey = "/" + volumeName + "/" + bucketName + "/" + keyName + "/" + + UUID.randomUUID().getMostSignificantBits(); + + // Create a list to hold OmKeyInfo objects + List omKeyInfos = new ArrayList<>(); + + // Build OmKeyInfo object + OmKeyInfo omKeyInfo = new OmKeyInfo.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setDataSize(dataSize) + .setReplicationConfig(StandaloneReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.ONE)) + .build(); + + // Add the OmKeyInfo object to the list + omKeyInfos.add(omKeyInfo); + + // Create a RepeatedOmKeyInfo object with the list of OmKeyInfo + RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(omKeyInfos); + + // Write the deleted key information to the OM metadata manager + writeDeletedKeysToOm(reconOMMetadataManager, deletedKey, repeatedOmKeyInfo); + } + + /** + * Writes deleted key information to the Ozone Manager metadata table. + * @param omMetadataManager the Ozone Manager metadata manager + * @param deletedKey the name of the deleted key + * @param repeatedOmKeyInfo the RepeatedOmKeyInfo object containing key information + * @throws IOException if there is an error accessing the metadata table + */ + public static void writeDeletedKeysToOm(OMMetadataManager omMetadataManager, + String deletedKey, + RepeatedOmKeyInfo repeatedOmKeyInfo) throws IOException { + // Put the deleted key information into the deleted table + omMetadataManager.getDeletedTable().put(deletedKey, repeatedOmKeyInfo); + } + +} From f9a96c526c1c6f6440ad294156c471bb682b5c11 Mon Sep 17 00:00:00 2001 From: arafat Date: Mon, 30 Sep 2024 23:50:56 +0530 Subject: [PATCH 02/21] Fixed a bug and improved the tests --- .../recon/api/OMDBInsightSearchEndpoint.java | 35 +++--- .../api/TestDeletedKeysSearchEndpoint.java | 109 +++++++++++------- 2 files changed, 82 insertions(+), 62 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java index 27744c5addcd..ebdb3c105024 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java @@ -354,12 +354,9 @@ public String convertToObjectPath(String prevKeyPrefix) throws IOException { @GET @Path("/deletePending/search") public Response searchDeletedKeys( - @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_START_PREFIX) @QueryParam("startPrefix") - String startPrefix, - @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_SEARCH_LIMIT) @QueryParam("limit") - int limit, - @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_SEARCH_PREV_KEY) @QueryParam("prevKey") - String prevKey) throws IOException { + @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_START_PREFIX) @QueryParam("startPrefix") String startPrefix, + @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_SEARCH_LIMIT) @QueryParam("limit") int limit, + @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_SEARCH_PREV_KEY) @QueryParam("prevKey") String prevKey) throws IOException { try { // Ensure startPrefix is not null or empty and starts with '/' @@ -394,22 +391,16 @@ public Response searchDeletedKeys( keysFound = true; RepeatedOmKeyInfo repeatedOmKeyInfo = entry.getValue(); - for (OmKeyInfo keyInfo : repeatedOmKeyInfo.getOmKeyInfoList()) { - KeyEntityInfo keyEntityInfo = createKeyEntityInfoFromOmKeyInfo(entry.getKey(), keyInfo); - - // Fetch bucket info and classify key as FSO or Non-FSO - String bucketKey = omMetadataManager.getBucketKey(keyInfo.getVolumeName(), keyInfo.getBucketName()); - OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().getSkipCache(bucketKey); - if (bucketInfo != null) { - if (bucketInfo.getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) { - insightResponse.getFsoKeyInfoList().add(keyEntityInfo); // Add to FSO list - } else { - insightResponse.getNonFSOKeyInfoList().add(keyEntityInfo); // Add to non-FSO list - } - replicatedTotal += keyInfo.getReplicatedSize(); - unreplicatedTotal += keyInfo.getDataSize(); - } - } + // We know each RepeatedOmKeyInfo has just one OmKeyInfo object + OmKeyInfo keyInfo = repeatedOmKeyInfo.getOmKeyInfoList().get(0); + KeyEntityInfo keyEntityInfo = createKeyEntityInfoFromOmKeyInfo(entry.getKey(), keyInfo); + + // Add the key directly to the list without classification + insightResponse.getRepeatedOmKeyInfoList().add(repeatedOmKeyInfo); + + replicatedTotal += keyInfo.getReplicatedSize(); + unreplicatedTotal += keyInfo.getDataSize(); + lastKey = entry.getKey(); // Update lastKey } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java index f226b39f4271..f550105925de 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java @@ -162,18 +162,18 @@ public void testBucketLevelSearch() throws IOException { Response response = deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1", 20, ""); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); - assertEquals(7, result.getFsoKeyInfoList().size()); + assertEquals(7, result.getRepeatedOmKeyInfoList().size()); response = deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1", 2, ""); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); - assertEquals(2, result.getFsoKeyInfoList().size()); + assertEquals(2, result.getRepeatedOmKeyInfoList().size()); // Search inside OBS bucket response = deletedKeysSearchEndpoint.searchDeletedKeys("/volc/bucketc1", 20, ""); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); - assertEquals(9, result.getNonFSOKeyInfoList().size()); + assertEquals(9, result.getRepeatedOmKeyInfoList().size()); response = deletedKeysSearchEndpoint.searchDeletedKeys("/vola/nonexistentbucket", 20, ""); assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); @@ -188,13 +188,13 @@ public void testDirectoryLevelSearch() throws IOException { ""); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); - assertEquals(4, result.getNonFSOKeyInfoList().size()); + assertEquals(4, result.getRepeatedOmKeyInfoList().size()); response = deletedKeysSearchEndpoint.searchDeletedKeys("/volc/bucketc1/dirc2", 20, ""); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); - assertEquals(5, result.getNonFSOKeyInfoList().size()); + assertEquals(5, result.getRepeatedOmKeyInfoList().size()); response = deletedKeysSearchEndpoint.searchDeletedKeys( "/volb/bucketb1/nonexistentdir", 20, ""); @@ -212,13 +212,13 @@ public void testKeyLevelSearch() throws IOException { ""); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); - assertEquals(1, result.getFsoKeyInfoList().size()); + assertEquals(1, result.getRepeatedOmKeyInfoList().size()); response = deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1/fileb2", 10, ""); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); - assertEquals(1, result.getFsoKeyInfoList().size()); + assertEquals(1, result.getRepeatedOmKeyInfoList().size()); // Test with non-existent key response = deletedKeysSearchEndpoint.searchDeletedKeys( @@ -238,7 +238,7 @@ public void testKeyLevelSearchUnderDirectory() throws IOException { assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); - assertEquals(1, result.getFsoKeyInfoList().size()); + assertEquals(1, result.getRepeatedOmKeyInfoList().size()); response = deletedKeysSearchEndpoint.searchDeletedKeys( "/volb/bucketb1/dir1/nonexistentfile", 10, ""); @@ -256,19 +256,19 @@ public void testSearchUnderNestedDirectory() throws IOException { deletedKeysSearchEndpoint.searchDeletedKeys("/volc/bucketc1/dirc1", 20, ""); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); - assertEquals(4, result.getNonFSOKeyInfoList().size()); + assertEquals(4, result.getRepeatedOmKeyInfoList().size()); response = deletedKeysSearchEndpoint.searchDeletedKeys( "/volc/bucketc1/dirc1/dirc11", 20, ""); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); - assertEquals(2, result.getNonFSOKeyInfoList().size()); + assertEquals(2, result.getRepeatedOmKeyInfoList().size()); response = deletedKeysSearchEndpoint.searchDeletedKeys( "/volc/bucketc1/dirc1/dirc11/dirc111", 20, ""); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); - assertEquals(1, result.getNonFSOKeyInfoList().size()); + assertEquals(1, result.getRepeatedOmKeyInfoList().size()); response = deletedKeysSearchEndpoint.searchDeletedKeys( "/volc/bucketc1/dirc1/dirc11/dirc111/nonexistentfile", 20, ""); @@ -290,7 +290,7 @@ public void testLimitSearch() throws IOException { Response response = deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1", 2, ""); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); - assertEquals(2, result.getFsoKeyInfoList().size()); + assertEquals(2, result.getRepeatedOmKeyInfoList().size()); } @Test @@ -314,9 +314,15 @@ public void testLastKeyInResponse() throws IOException { Response response = deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1", 20, ""); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); - assertEquals(7, result.getFsoKeyInfoList().size()); - assertEquals(result.getFsoKeyInfoList().get(6).getKey(), result.getLastKey(), - "Expected last key to be 'fileb5'"); + assertEquals(7, result.getRepeatedOmKeyInfoList().size()); + + // Compute the expected last key from the last entry in the result list + String computedLastKey = "/" + result.getRepeatedOmKeyInfoList().get(6).getOmKeyInfoList().get(0).getVolumeName() + "/" + + result.getRepeatedOmKeyInfoList().get(6).getOmKeyInfoList().get(0).getBucketName() + "/" + + result.getRepeatedOmKeyInfoList().get(6).getOmKeyInfoList().get(0).getKeyName() + "/"; + + // Check that the last key in the response starts with the expected value + assertTrue(result.getLastKey().startsWith(computedLastKey)); } @Test @@ -328,7 +334,7 @@ public void testSearchDeletedKeysWithPagination() throws IOException { Response response = deletedKeysSearchEndpoint.searchDeletedKeys(startPrefix, limit, prevKey); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); - assertEquals(2, result.getFsoKeyInfoList().size()); + assertEquals(2, result.getRepeatedOmKeyInfoList().size()); prevKey = result.getLastKey(); assertNotNull(prevKey, "Last key should not be null"); @@ -336,7 +342,7 @@ public void testSearchDeletedKeysWithPagination() throws IOException { response = deletedKeysSearchEndpoint.searchDeletedKeys(startPrefix, limit, prevKey); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); - assertEquals(2, result.getFsoKeyInfoList().size()); + assertEquals(2, result.getRepeatedOmKeyInfoList().size()); prevKey = result.getLastKey(); assertNotNull(prevKey, "Last key should not be null"); @@ -344,9 +350,7 @@ public void testSearchDeletedKeysWithPagination() throws IOException { response = deletedKeysSearchEndpoint.searchDeletedKeys(startPrefix, limit, prevKey); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); - assertEquals(2, result.getFsoKeyInfoList().size()); - assertEquals(result.getFsoKeyInfoList().get(1).getKey(), - result.getLastKey(), "Expected last key to be empty"); + assertEquals(2, result.getRepeatedOmKeyInfoList().size()); prevKey = result.getLastKey(); assertNotNull(prevKey, "Last key should not be null"); @@ -354,9 +358,18 @@ public void testSearchDeletedKeysWithPagination() throws IOException { response = deletedKeysSearchEndpoint.searchDeletedKeys(startPrefix, limit, prevKey); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); - assertEquals(1, result.getFsoKeyInfoList().size()); - assertEquals(result.getFsoKeyInfoList().get(0).getKey(), - result.getLastKey(), "Expected last key to be empty"); + assertEquals(1, result.getRepeatedOmKeyInfoList().size()); + // Compute the expected last key from the last entry in the result list + String computedLastKey = "/" + + result.getRepeatedOmKeyInfoList().get(0).getOmKeyInfoList().get(0) + .getVolumeName() + "/" + + result.getRepeatedOmKeyInfoList().get(0).getOmKeyInfoList().get(0) + .getBucketName() + "/" + + result.getRepeatedOmKeyInfoList().get(0).getOmKeyInfoList().get(0) + .getKeyName() + "/"; + + // Check that the last key in the response starts with the expected value + assertTrue(result.getLastKey().startsWith(computedLastKey)); } @Test @@ -369,12 +382,39 @@ public void testSearchInEmptyBucket() throws IOException { "Expected a message indicating no keys were found"); } + /** + * Populates the OMDB with a set of deleted keys for testing purposes. + * This diagram is for reference: + * * root + * ├── volb (Total Size: 7000KB) + * │ ├── bucketb1 (Total Size: 7000KB) + * │ │ ├── fileb1 (Size: 1000KB) + * │ │ ├── fileb2 (Size: 1000KB) + * │ │ ├── fileb3 (Size: 1000KB) + * │ │ ├── fileb4 (Size: 1000KB) + * │ │ ├── fileb5 (Size: 1000KB) + * │ │ ├── dir1 (Total Size: 2000KB) + * │ │ │ ├── file1 (Size: 1000KB) + * │ │ │ └── file2 (Size: 1000KB) + * ├── volc (Total Size: 9000KB) + * │ ├── bucketc1 (Total Size: 9000KB) + * │ │ ├── dirc1 (Total Size: 4000KB) + * │ │ │ ├── filec1 (Size: 1000KB) + * │ │ │ ├── filec2 (Size: 1000KB) + * │ │ │ ├── dirc11 (Total Size: 2000KB) + * │ │ │ ├── filec11 (Size: 1000KB) + * │ │ │ └── dirc111 (Total Size: 1000KB) + * │ │ │ └── filec111 (Size: 1000KB) + * │ │ ├── dirc2 (Total Size: 5000KB) + * │ │ │ ├── filec3 (Size: 1000KB) + * │ │ │ ├── filec4 (Size: 1000KB) + * │ │ │ ├── filec5 (Size: 1000KB) + * │ │ │ ├── filgetec6 (Size: 1000KB) + * │ │ │ └── filec7 (Size: 1000KB) + * + * @throws Exception if an error occurs while creating deleted keys. + */ private void populateOMDB() throws Exception { - // Create FSO bucket - createBucket("volb", "bucketb1", BucketLayout.FILE_SYSTEM_OPTIMIZED); - - // Create OBS bucket - createBucket("volc", "bucketc1", BucketLayout.OBJECT_STORE); createDeletedKey("fileb1", "bucketb1", "volb", 1000); createDeletedKey("fileb2", "bucketb1", "volb", 1000); @@ -398,18 +438,6 @@ private void populateOMDB() throws Exception { createDeletedKey("dirc1/dirc11/dirc111/filec111", "bucketc1", "volc", 1000); } - private void createBucket(String volumeName, String bucketName, BucketLayout bucketLayout) throws Exception { - String bucketKey = reconOMMetadataManager.getBucketKey(volumeName, bucketName); - long bucketId = UUID.randomUUID().getMostSignificantBits() & Long.MAX_VALUE; // Generate positive ID - OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setObjectID(bucketId) - .setBucketLayout(bucketLayout) - .build(); - reconOMMetadataManager.getBucketTable().put(bucketKey, bucketInfo); - } - private void createDeletedKey(String keyName, String bucketName, String volumeName, long dataSize) throws IOException { // Construct the deleted key path @@ -425,6 +453,7 @@ private void createDeletedKey(String keyName, String bucketName, .setBucketName(bucketName) .setKeyName(keyName) .setDataSize(dataSize) + .setObjectID(UUID.randomUUID().getMostSignificantBits()) .setReplicationConfig(StandaloneReplicationConfig.getInstance( HddsProtos.ReplicationFactor.ONE)) .build(); From 26223d0a184d5449fd42e1e85ab8fd0202fbb350 Mon Sep 17 00:00:00 2001 From: arafat Date: Tue, 1 Oct 2024 19:31:33 +0530 Subject: [PATCH 03/21] Refactored some code --- .../recon/api/OMDBInsightSearchEndpoint.java | 64 +++++++++---------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java index ebdb3c105024..4fdcb7b161d0 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java @@ -124,22 +124,10 @@ public Response searchOpenKeys( String prevKey) throws IOException { try { - // Ensure startPrefix is not null or empty and starts with '/' - if (startPrefix == null || startPrefix.length() == 0) { - return createBadRequestResponse( - "Invalid startPrefix: Path must be at the bucket level or deeper."); + // Validate the request parameters + if (!validateStartPrefixAndLimit(startPrefix, limit)) { + return createBadRequestResponse("Invalid startPrefix: Path must be at the bucket level or deeper."); } - startPrefix = startPrefix.startsWith("/") ? startPrefix : "/" + startPrefix; - - // Split the path to ensure it's at least at the bucket level - String[] pathComponents = startPrefix.split("/"); - if (pathComponents.length < 3 || pathComponents[2].isEmpty()) { - return createBadRequestResponse( - "Invalid startPrefix: Path must be at the bucket level or deeper."); - } - - // Ensure the limit is non-negative - limit = Math.max(0, limit); // Initialize response object KeyInsightInfoResponse insightResponse = new KeyInsightInfoResponse(); @@ -354,27 +342,18 @@ public String convertToObjectPath(String prevKeyPrefix) throws IOException { @GET @Path("/deletePending/search") public Response searchDeletedKeys( - @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_START_PREFIX) @QueryParam("startPrefix") String startPrefix, - @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_SEARCH_LIMIT) @QueryParam("limit") int limit, - @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_SEARCH_PREV_KEY) @QueryParam("prevKey") String prevKey) throws IOException { + @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_START_PREFIX) @QueryParam("startPrefix") + String startPrefix, + @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_SEARCH_LIMIT) @QueryParam("limit") + int limit, + @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_SEARCH_PREV_KEY) @QueryParam("prevKey") + String prevKey) throws IOException { try { - // Ensure startPrefix is not null or empty and starts with '/' - if (startPrefix == null || startPrefix.length() == 0) { - return createBadRequestResponse( - "Invalid startPrefix: Path must be at the bucket level or deeper."); + // Validate the request parameters + if (!validateStartPrefixAndLimit(startPrefix, limit)) { + return createBadRequestResponse("Invalid startPrefix: Path must be at the bucket level or deeper."); } - startPrefix = startPrefix.startsWith("/") ? startPrefix : "/" + startPrefix; - - // Split the path to ensure it's at least at the bucket level - String[] pathComponents = startPrefix.split("/"); - if (pathComponents.length < 3 || pathComponents[2].isEmpty()) { - return createBadRequestResponse( - "Invalid startPrefix: Path must be at the bucket level or deeper."); - } - - // Ensure the limit is non-negative - limit = Math.max(0, limit); // Initialize response object KeyInsightInfoResponse insightResponse = new KeyInsightInfoResponse(); @@ -446,4 +425,23 @@ private KeyEntityInfo createKeyEntityInfoFromOmKeyInfo(String dbKey, return keyEntityInfo; } + private boolean validateStartPrefixAndLimit(String startPrefix, int limit) { + // Ensure startPrefix is not null or empty and starts with '/' + if (startPrefix == null || startPrefix.isEmpty()) { + return false; + } + startPrefix = startPrefix.startsWith("/") ? startPrefix : "/" + startPrefix; + + // Split the path to ensure it's at least at the bucket level + String[] pathComponents = startPrefix.split("/"); + if (pathComponents.length < 3 || pathComponents[2].isEmpty()) { + return false; + } + + // Ensure the limit is non-negative + limit = Math.max(0, limit); + + return true; // Validation passed + } + } From abb9d656d81f45bb71880e37d3aa08d88cdc7726 Mon Sep 17 00:00:00 2001 From: arafat Date: Tue, 1 Oct 2024 19:33:42 +0530 Subject: [PATCH 04/21] Improved the java doc --- .../hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java index 4fdcb7b161d0..a7fcce5d8e19 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java @@ -322,8 +322,8 @@ public String convertToObjectPath(String prevKeyPrefix) throws IOException { /** * Performs a search for deleted keys in the Ozone Manager DeletedTable using a specified search prefix. - * This endpoint searches across both File System Optimized (FSO) and Object Store (non-FSO) layouts, - * compiling a list of keys that match the given prefix along with their data sizes. + * In the DeletedTable both the fso and non-fso keys are stored in a similar format, this endpoint compiles + * a list of keys that match the given prefix along with their data sizes. * * The search prefix must start from the bucket level ('/volumeName/bucketName/') or any specific directory * or key level (e.g., '/volA/bucketA/dir1' for everything under 'dir1' inside 'bucketA' of 'volA'). From abca3c9ed54877912bed4fdb438155d0178fde11 Mon Sep 17 00:00:00 2001 From: arafat Date: Fri, 4 Oct 2024 12:43:45 +0530 Subject: [PATCH 05/21] Fixed checkstyle issues --- .../ozone/recon/api/TestDeletedKeysSearchEndpoint.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java index f550105925de..ec47d989c19c 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java @@ -24,8 +24,6 @@ import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.recon.ReconTestInjector; @@ -317,7 +315,8 @@ public void testLastKeyInResponse() throws IOException { assertEquals(7, result.getRepeatedOmKeyInfoList().size()); // Compute the expected last key from the last entry in the result list - String computedLastKey = "/" + result.getRepeatedOmKeyInfoList().get(6).getOmKeyInfoList().get(0).getVolumeName() + "/" + + String computedLastKey = "/" + + result.getRepeatedOmKeyInfoList().get(6).getOmKeyInfoList().get(0).getVolumeName() + "/" + result.getRepeatedOmKeyInfoList().get(6).getOmKeyInfoList().get(0).getBucketName() + "/" + result.getRepeatedOmKeyInfoList().get(6).getOmKeyInfoList().get(0).getKeyName() + "/"; From 2af1de161117e0bf273fe9573fe413472e2f0564 Mon Sep 17 00:00:00 2001 From: arafat Date: Sun, 6 Oct 2024 12:45:32 +0530 Subject: [PATCH 06/21] Fixed a dead store variable --- .../recon/api/OMDBInsightSearchEndpoint.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java index a7fcce5d8e19..32038c7051e4 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java @@ -125,10 +125,13 @@ public Response searchOpenKeys( try { // Validate the request parameters - if (!validateStartPrefixAndLimit(startPrefix, limit)) { + if (!validateStartPrefixAndLimit(startPrefix)) { return createBadRequestResponse("Invalid startPrefix: Path must be at the bucket level or deeper."); } + // Ensure the limit is non-negative + limit = Math.max(0, limit); + // Initialize response object KeyInsightInfoResponse insightResponse = new KeyInsightInfoResponse(); long replicatedTotal = 0; @@ -351,9 +354,12 @@ public Response searchDeletedKeys( try { // Validate the request parameters - if (!validateStartPrefixAndLimit(startPrefix, limit)) { + if (!validateStartPrefixAndLimit(startPrefix)) { return createBadRequestResponse("Invalid startPrefix: Path must be at the bucket level or deeper."); } + + // Ensure the limit is non-negative + limit = Math.max(0, limit); // Initialize response object KeyInsightInfoResponse insightResponse = new KeyInsightInfoResponse(); @@ -425,7 +431,7 @@ private KeyEntityInfo createKeyEntityInfoFromOmKeyInfo(String dbKey, return keyEntityInfo; } - private boolean validateStartPrefixAndLimit(String startPrefix, int limit) { + private boolean validateStartPrefixAndLimit(String startPrefix) { // Ensure startPrefix is not null or empty and starts with '/' if (startPrefix == null || startPrefix.isEmpty()) { return false; @@ -438,9 +444,6 @@ private boolean validateStartPrefixAndLimit(String startPrefix, int limit) { return false; } - // Ensure the limit is non-negative - limit = Math.max(0, limit); - return true; // Validation passed } From ab9765a5aabe94a2ab7c4c673cefb95ff4a5ca11 Mon Sep 17 00:00:00 2001 From: arafat Date: Thu, 10 Oct 2024 14:15:05 +0530 Subject: [PATCH 07/21] Refactored key search logic to OMDBInsightEndpoint and enhanced with improved pagination and prefix handling. --- .../hadoop/ozone/recon/ReconConstants.java | 4 +- .../ozone/recon/api/OMDBInsightEndpoint.java | 259 ++++++++++++------ .../recon/api/OMDBInsightSearchEndpoint.java | 89 ------ .../api/TestDeletedKeysSearchEndpoint.java | 189 ++++++++----- .../api/TestOMDBInsightSearchEndpoint.java | 6 +- .../recon/api/TestOmDBInsightEndPoint.java | 129 ++++++++- 6 files changed, 433 insertions(+), 243 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java index 0c1a2287b4c4..5768166c9503 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java @@ -48,6 +48,7 @@ private ReconConstants() { public static final String DEFAULT_BATCH_NUMBER = "1"; public static final String RECON_QUERY_BATCH_PARAM = "batchNum"; public static final String RECON_QUERY_PREVKEY = "prevKey"; + public static final String RECON_QUERY_START_PREFIX = "startPrefix"; public static final String RECON_OPEN_KEY_INCLUDE_NON_FSO = "includeNonFso"; public static final String RECON_OPEN_KEY_INCLUDE_FSO = "includeFso"; public static final String RECON_OM_INSIGHTS_DEFAULT_START_PREFIX = "/"; @@ -55,8 +56,7 @@ private ReconConstants() { public static final String RECON_OM_INSIGHTS_DEFAULT_SEARCH_PREV_KEY = ""; public static final String RECON_QUERY_FILTER = "missingIn"; public static final String PREV_CONTAINER_ID_DEFAULT_VALUE = "0"; - public static final String PREV_DELETED_BLOCKS_TRANSACTION_ID_DEFAULT_VALUE = - "0"; + public static final String PREV_DELETED_BLOCKS_TRANSACTION_ID_DEFAULT_VALUE = "0"; // Only include containers that are missing in OM by default public static final String DEFAULT_FILTER_FOR_MISSING_CONTAINERS = "SCM"; public static final String RECON_QUERY_LIMIT = "limit"; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java index 3f95c04fc916..a4e9609c2183 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java @@ -66,19 +66,24 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.OPEN_FILE_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.OPEN_KEY_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_DIR_TABLE; import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_FETCH_COUNT; -import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_KEY_SIZE; -import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_QUERY_LIMIT; -import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_QUERY_PREVKEY; import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_OPEN_KEY_INCLUDE_FSO; import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_OPEN_KEY_INCLUDE_NON_FSO; import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OPEN_KEY_INCLUDE_FSO; import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OPEN_KEY_INCLUDE_NON_FSO; +import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_QUERY_START_PREFIX; +import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_KEY_SIZE; +import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_QUERY_LIMIT; +import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_QUERY_PREVKEY; +import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createBadRequestResponse; +import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createInternalServerErrorResponse; +import static org.apache.hadoop.ozone.recon.ReconResponseUtils.noMatchedKeysResponse; import static org.apache.hadoop.ozone.recon.api.handlers.BucketHandler.getBucketHandler; import static org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.normalizePath; import static org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.parseRequestPath; @@ -211,7 +216,7 @@ public Response getOpenKeyInfo( keyIter = openKeyTable.iterator()) { boolean skipPrevKey = false; String seekKey = prevKey; - if (!skipPrevKeyDone && StringUtils.isNotBlank(prevKey)) { + if (!skipPrevKeyDone && isNotBlank(prevKey)) { skipPrevKey = true; Table.KeyValue seekKeyValue = keyIter.seek(seekKey); @@ -219,7 +224,7 @@ public Response getOpenKeyInfo( // if not, then return empty result // In case of an empty prevKeyPrefix, all the keys are returned if (seekKeyValue == null || - (StringUtils.isNotBlank(prevKey) && + (isNotBlank(prevKey) && !seekKeyValue.getKey().equals(prevKey))) { continue; } @@ -339,62 +344,6 @@ private Long getValueFromId(GlobalStats record) { return record != null ? record.getValue() : 0L; } - private void getPendingForDeletionKeyInfo( - int limit, - String prevKey, - KeyInsightInfoResponse deletedKeyAndDirInsightInfo) { - List repeatedOmKeyInfoList = - deletedKeyAndDirInsightInfo.getRepeatedOmKeyInfoList(); - Table deletedTable = - omMetadataManager.getDeletedTable(); - try ( - TableIterator> - keyIter = deletedTable.iterator()) { - boolean skipPrevKey = false; - String seekKey = prevKey; - String lastKey = ""; - if (StringUtils.isNotBlank(prevKey)) { - skipPrevKey = true; - Table.KeyValue seekKeyValue = - keyIter.seek(seekKey); - // check if RocksDB was able to seek correctly to the given key prefix - // if not, then return empty result - // In case of an empty prevKeyPrefix, all the keys are returned - if (seekKeyValue == null || - (StringUtils.isNotBlank(prevKey) && - !seekKeyValue.getKey().equals(prevKey))) { - return; - } - } - while (keyIter.hasNext()) { - Table.KeyValue kv = keyIter.next(); - String key = kv.getKey(); - lastKey = key; - RepeatedOmKeyInfo repeatedOmKeyInfo = kv.getValue(); - // skip the prev key if prev key is present - if (skipPrevKey && key.equals(prevKey)) { - continue; - } - updateReplicatedAndUnReplicatedTotal(deletedKeyAndDirInsightInfo, - repeatedOmKeyInfo); - repeatedOmKeyInfoList.add(repeatedOmKeyInfo); - if ((repeatedOmKeyInfoList.size()) == limit) { - break; - } - } - deletedKeyAndDirInsightInfo.setLastKey(lastKey); - } catch (IOException ex) { - throw new WebApplicationException(ex, - Response.Status.INTERNAL_SERVER_ERROR); - } catch (IllegalArgumentException e) { - throw new WebApplicationException(e, Response.Status.BAD_REQUEST); - } catch (Exception ex) { - throw new WebApplicationException(ex, - Response.Status.INTERNAL_SERVER_ERROR); - } - } - /** Retrieves the summary of deleted keys. * * This method calculates and returns a summary of deleted keys. @@ -428,6 +377,7 @@ public Response getDeletedKeySummary() { * limit - limits the number of key/files returned. * prevKey - E.g. /vol1/bucket1/key1, this will skip keys till it * seeks correctly to the given prevKey. + * startPrefix - E.g. /vol1/bucket1, this will return keys matching this prefix. * Sample API Response: * { * "lastKey": "vol1/bucket1/key1", @@ -476,17 +426,159 @@ public Response getDeletedKeySummary() { @GET @Path("/deletePending") public Response getDeletedKeyInfo( - @DefaultValue(DEFAULT_FETCH_COUNT) @QueryParam(RECON_QUERY_LIMIT) - int limit, - @DefaultValue(StringUtils.EMPTY) @QueryParam(RECON_QUERY_PREVKEY) - String prevKey) { - KeyInsightInfoResponse - deletedKeyInsightInfo = new KeyInsightInfoResponse(); - getPendingForDeletionKeyInfo(limit, prevKey, - deletedKeyInsightInfo); + @DefaultValue(DEFAULT_FETCH_COUNT) @QueryParam(RECON_QUERY_LIMIT) int limit, + @DefaultValue(StringUtils.EMPTY) @QueryParam(RECON_QUERY_PREVKEY) String prevKey, + @DefaultValue(StringUtils.EMPTY) @QueryParam(RECON_QUERY_START_PREFIX) String startPrefix) { + + // Initialize the response object to hold the key information + KeyInsightInfoResponse deletedKeyInsightInfo = new KeyInsightInfoResponse(); + + // Ensure the limit is non-negative + limit = Math.max(0, limit); + + boolean keysFound = false; + + try { + if (isNotBlank(startPrefix)) { + // Validate and apply prefix-based search + if (!validateStartPrefix(startPrefix)) { + return createBadRequestResponse("Invalid startPrefix: Path must be at the bucket level or deeper."); + } + keysFound = getPendingForDeletionKeyInfo(limit, prevKey, startPrefix, deletedKeyInsightInfo); + } else { + // Retrieve all records without prefix + keysFound = getPendingForDeletionKeyInfo(limit, prevKey, "", deletedKeyInsightInfo); + } + } catch (IllegalArgumentException e) { + LOG.debug("Invalid startPrefix provided: {}", startPrefix, e); + return createBadRequestResponse("Invalid startPrefix: " + e.getMessage()); + } catch (IOException e) { + LOG.debug("I/O error while searching deleted keys in OM DB", e); + return createInternalServerErrorResponse("Error searching deleted keys in OM DB: " + e.getMessage()); + } catch (Exception e) { + LOG.debug("Unexpected error occurred while searching deleted keys", e); + return createInternalServerErrorResponse("Unexpected error: " + e.getMessage()); + } + + if (!keysFound) { + return noMatchedKeysResponse(""); + } + return Response.ok(deletedKeyInsightInfo).build(); } + /** + * Retrieves keys pending deletion based on startPrefix, filtering keys matching the prefix. + * + * @param limit The limit of records to return. + * @param prevKey Pagination key. + * @param startPrefix The search prefix. + * @param deletedKeyInsightInfo The response object to populate. + */ + private boolean getPendingForDeletionKeyInfo( + int limit, String prevKey, String startPrefix, + KeyInsightInfoResponse deletedKeyInsightInfo) throws IOException { + + long replicatedTotal = 0; + long unreplicatedTotal = 0; + boolean keysFound = false; + String lastKey = null; + + // Search for deleted keys in DeletedTable + Table deletedTable = omMetadataManager.getDeletedTable(); + Map deletedKeys = + retrieveKeysFromTable(deletedTable, startPrefix, limit, prevKey); + + // Iterate over the retrieved keys and populate the response + for (Map.Entry entry : deletedKeys.entrySet()) { + keysFound = true; + RepeatedOmKeyInfo repeatedOmKeyInfo = entry.getValue(); + + // We know each RepeatedOmKeyInfo has just one OmKeyInfo object + OmKeyInfo keyInfo = repeatedOmKeyInfo.getOmKeyInfoList().get(0); + KeyEntityInfo keyEntityInfo = createKeyEntityInfoFromOmKeyInfo(entry.getKey(), keyInfo); + + // Add the key directly to the list without classification + deletedKeyInsightInfo.getRepeatedOmKeyInfoList().add(repeatedOmKeyInfo); + + replicatedTotal += keyInfo.getReplicatedSize(); + unreplicatedTotal += keyInfo.getDataSize(); + + lastKey = entry.getKey(); // Update lastKey + } + + // Set the aggregated totals in the response + deletedKeyInsightInfo.setReplicatedDataSize(replicatedTotal); + deletedKeyInsightInfo.setUnreplicatedDataSize(unreplicatedTotal); + deletedKeyInsightInfo.setLastKey(lastKey); + + return keysFound; + } + + /** + * Retrieves keys from the specified table based on pagination and prefix filtering. + * This method handles different scenarios based on the presence of startPrefix and prevKey, + * enabling efficient key retrieval from the table. + * + * The method handles the following cases: + * + * 1. prevKey provided, startPrefix empty: + * - Seeks to prevKey, skips it, and returns subsequent records up to the limit. + * + * 2. prevKey empty, startPrefix empty: + * - Iterates from the beginning of the table, retrieving all records up to the limit. + * + * 3. startPrefix provided, prevKey empty: + * - Seeks to the first key matching startPrefix and returns all matching keys up to the limit. + * + * 4. startPrefix provided, prevKey provided: + * - Seeks to prevKey, skips it, and returns subsequent keys that match startPrefix, up to the limit. + * + * If limit is 0, all matching keys are retrieved. If both startPrefix and prevKey are empty, the method starts + * from the beginning of the table. + */ + public static Map retrieveKeysFromTable( + Table table, String startPrefix, int limit, String prevKey) + throws IOException { + + Map matchedKeys = new LinkedHashMap<>(); + try (TableIterator> keyIter = table.iterator()) { + + // Scenario 1 & 4: prevKey is provided (whether startPrefix is empty or not) + if (!prevKey.isEmpty()) { + keyIter.seek(prevKey); + if (keyIter.hasNext()) { + // Skip the previous key record + keyIter.next(); + } + } else if (!startPrefix.isEmpty()) { + // Scenario 3: startPrefix is provided but prevKey is empty, so seek to startPrefix + keyIter.seek(startPrefix); + } + // Scenario 2: Both startPrefix and prevKey are empty (iterate from the start of the table) + // No seeking needed; just start iterating from the first record in the table + // This is implicit in the following loop, as the iterator will start from the beginning + + // Iterate through the keys while adhering to the limit (if the limit is not zero) + while (keyIter.hasNext() && (limit == 0 || matchedKeys.size() < limit)) { + Table.KeyValue entry = keyIter.next(); + String dbKey = entry.getKey(); + + // Scenario 3 & 4: If startPrefix is provided, ensure the key matches startPrefix + if (!startPrefix.isEmpty() && !dbKey.startsWith(startPrefix)) { + break; // If the key no longer matches the prefix, exit the loop + } + + // Add the valid key-value pair to the results + matchedKeys.put(dbKey, entry.getValue()); + } + } catch (IOException exception) { + LOG.error("Error retrieving keys from table for path: {}", startPrefix, exception); + throw exception; + } + return matchedKeys; + } + /** * Creates a keys summary for deleted keys and updates the provided * keysSummary map. Calculates the total number of deleted keys, replicated @@ -526,7 +618,7 @@ private void getPendingForDeletionDirInfo( boolean skipPrevKey = false; String seekKey = prevKey; String lastKey = ""; - if (StringUtils.isNotBlank(prevKey)) { + if (isNotBlank(prevKey)) { skipPrevKey = true; Table.KeyValue seekKeyValue = keyIter.seek(seekKey); @@ -534,7 +626,7 @@ private void getPendingForDeletionDirInfo( // if not, then return empty result // In case of an empty prevKeyPrefix, all the keys are returned if (seekKeyValue == null || - (StringUtils.isNotBlank(prevKey) && + (isNotBlank(prevKey) && !seekKeyValue.getKey().equals(prevKey))) { return; } @@ -1161,7 +1253,7 @@ private Map retrieveKeysFromTable( try ( TableIterator> keyIter = table.iterator()) { - if (!paramInfo.isSkipPrevKeyDone() && StringUtils.isNotBlank(seekKey)) { + if (!paramInfo.isSkipPrevKeyDone() && isNotBlank(seekKey)) { skipPrevKey = true; Table.KeyValue seekKeyValue = keyIter.seek(seekKey); @@ -1277,17 +1369,18 @@ private void createSummaryForDeletedDirectories( dirSummary.put("totalDeletedDirectories", deletedDirCount); } - private void updateReplicatedAndUnReplicatedTotal( - KeyInsightInfoResponse deletedKeyAndDirInsightInfo, - RepeatedOmKeyInfo repeatedOmKeyInfo) { - repeatedOmKeyInfo.getOmKeyInfoList().forEach(omKeyInfo -> { - deletedKeyAndDirInsightInfo.setUnreplicatedDataSize( - deletedKeyAndDirInsightInfo.getUnreplicatedDataSize() + - omKeyInfo.getDataSize()); - deletedKeyAndDirInsightInfo.setReplicatedDataSize( - deletedKeyAndDirInsightInfo.getReplicatedDataSize() + - omKeyInfo.getReplicatedSize()); - }); + private boolean validateStartPrefix(String startPrefix) { + + // Ensure startPrefix starts with '/' for non-empty values + startPrefix = startPrefix.startsWith("/") ? startPrefix : "/" + startPrefix; + + // Split the path to ensure it's at least at the bucket level (volume/bucket). + String[] pathComponents = startPrefix.split("/"); + if (pathComponents.length < 3 || pathComponents[2].isEmpty()) { + return false; // Invalid if not at bucket level or deeper + } + + return true; } private String createPath(OmKeyInfo omKeyInfo) { diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java index 32038c7051e4..9d267250fd60 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java @@ -323,95 +323,6 @@ public String convertToObjectPath(String prevKeyPrefix) throws IOException { return prevKeyPrefix; } - /** - * Performs a search for deleted keys in the Ozone Manager DeletedTable using a specified search prefix. - * In the DeletedTable both the fso and non-fso keys are stored in a similar format, this endpoint compiles - * a list of keys that match the given prefix along with their data sizes. - * - * The search prefix must start from the bucket level ('/volumeName/bucketName/') or any specific directory - * or key level (e.g., '/volA/bucketA/dir1' for everything under 'dir1' inside 'bucketA' of 'volA'). - * The search operation matches the prefix against the start of keys' names within the OM DB DeletedTable. - * - * Example Usage: - * 1. A startPrefix of "/volA/bucketA/" retrieves every key under bucket 'bucketA' in volume 'volA'. - * 2. Specifying "/volA/bucketA/dir1" focuses the search within 'dir1' inside 'bucketA' of 'volA'. - * - * @param startPrefix The prefix for searching keys, starting from the bucket level or any specific path. - * @param limit Limits the number of returned keys. - * @param prevKey The key to start after for the next set of records. - * @return A KeyInsightInfoResponse, containing matching keys and their data sizes. - * @throws IOException On failure to access the OM database or process the operation. - */ - @GET - @Path("/deletePending/search") - public Response searchDeletedKeys( - @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_START_PREFIX) @QueryParam("startPrefix") - String startPrefix, - @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_SEARCH_LIMIT) @QueryParam("limit") - int limit, - @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_SEARCH_PREV_KEY) @QueryParam("prevKey") - String prevKey) throws IOException { - - try { - // Validate the request parameters - if (!validateStartPrefixAndLimit(startPrefix)) { - return createBadRequestResponse("Invalid startPrefix: Path must be at the bucket level or deeper."); - } - - // Ensure the limit is non-negative - limit = Math.max(0, limit); - - // Initialize response object - KeyInsightInfoResponse insightResponse = new KeyInsightInfoResponse(); - long replicatedTotal = 0; - long unreplicatedTotal = 0; - boolean keysFound = false; // Flag to track if any keys are found - String lastKey = null; - - // Search for deleted keys in DeletedTable - Table deletedTable = omMetadataManager.getDeletedTable(); - Map deletedKeys = retrieveKeysFromTable(deletedTable, startPrefix, limit, prevKey); - - for (Map.Entry entry : deletedKeys.entrySet()) { - keysFound = true; - RepeatedOmKeyInfo repeatedOmKeyInfo = entry.getValue(); - - // We know each RepeatedOmKeyInfo has just one OmKeyInfo object - OmKeyInfo keyInfo = repeatedOmKeyInfo.getOmKeyInfoList().get(0); - KeyEntityInfo keyEntityInfo = createKeyEntityInfoFromOmKeyInfo(entry.getKey(), keyInfo); - - // Add the key directly to the list without classification - insightResponse.getRepeatedOmKeyInfoList().add(repeatedOmKeyInfo); - - replicatedTotal += keyInfo.getReplicatedSize(); - unreplicatedTotal += keyInfo.getDataSize(); - - lastKey = entry.getKey(); // Update lastKey - } - - // If no keys were found, return a response indicating that no keys matched - if (!keysFound) { - return noMatchedKeysResponse(startPrefix); - } - - // Set the aggregated totals in the response - insightResponse.setReplicatedDataSize(replicatedTotal); - insightResponse.setUnreplicatedDataSize(unreplicatedTotal); - insightResponse.setLastKey(lastKey); - - // Return the response with the matched keys and their data sizes - return Response.ok(insightResponse).build(); - } catch (IOException e) { - // Handle IO exceptions and return an internal server error response - return createInternalServerErrorResponse( - "Error searching deleted keys in OM DB: " + e.getMessage()); - } catch (IllegalArgumentException e) { - // Handle illegal argument exceptions and return a bad request response - return createBadRequestResponse( - "Invalid startPrefix: " + e.getMessage()); - } - } - /** * Creates a KeyEntityInfo object from an OmKeyInfo object and the corresponding key. * diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java index ec47d989c19c..4291c90b1aa0 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java @@ -31,7 +31,7 @@ import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest; import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; -import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; @@ -80,7 +80,7 @@ public class TestDeletedKeysSearchEndpoint extends AbstractReconSqlDBTest { @TempDir private Path temporaryFolder; private ReconOMMetadataManager reconOMMetadataManager; - private OMDBInsightSearchEndpoint deletedKeysSearchEndpoint; + private OMDBInsightEndpoint omdbInsightEndpoint; private OzoneConfiguration ozoneConfiguration; private static final String ROOT_PATH = "/"; private OMMetadataManager omMetadataManager; @@ -99,17 +99,15 @@ public void setUp() throws Exception { .withReconSqlDb() .withReconOm(reconOMMetadataManager) .withOmServiceProvider(mock(OzoneManagerServiceProviderImpl.class)) + .addBinding(OzoneStorageContainerManager.class, + ReconStorageContainerManagerFacade.class) + .withContainerDB() .addBinding(StorageContainerServiceProvider.class, mock(StorageContainerServiceProviderImpl.class)) - .addBinding(OzoneStorageContainerManager.class, - mock(OzoneStorageContainerManager.class)) - .addBinding(ReconNamespaceSummaryManager.class, - mock(ReconNamespaceSummaryManager.class)) - .addBinding(OMDBInsightSearchEndpoint.class) + .addBinding(OMDBInsightEndpoint.class) .addBinding(ContainerHealthSchemaManager.class) .build(); - deletedKeysSearchEndpoint = reconTestInjector.getInstance(OMDBInsightSearchEndpoint.class); - + omdbInsightEndpoint = reconTestInjector.getInstance(OMDBInsightEndpoint.class); populateOMDB(); } @@ -123,31 +121,40 @@ private static OMMetadataManager initializeNewOmMetadataManager(File omDbDir) th @Test public void testRootLevelSearchRestriction() throws IOException { String rootPath = "/"; - Response response = deletedKeysSearchEndpoint.searchDeletedKeys(rootPath, 20, ""); + Response response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", rootPath); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), "Expected a message indicating the path must be at the bucket level or deeper"); + } - rootPath = ""; - response = deletedKeysSearchEndpoint.searchDeletedKeys(rootPath, 20, ""); - assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); - entity = (String) response.getEntity(); - assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), - "Expected a message indicating the path must be at the bucket level or deeper"); + @Test + public void testEmptySearchPrefix() throws IOException { + Response response = omdbInsightEndpoint.getDeletedKeyInfo(100, "", ""); + // In this case we get all the keys from the OMDB + assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(16, result.getRepeatedOmKeyInfoList().size()); + + // Set limit to 10 and pass empty search prefix + response = omdbInsightEndpoint.getDeletedKeyInfo(10, "", null); + // In this case we get all the keys from the OMDB + assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(10, result.getRepeatedOmKeyInfoList().size()); } @Test public void testVolumeLevelSearchRestriction() throws IOException { String volumePath = "/vola"; - Response response = deletedKeysSearchEndpoint.searchDeletedKeys(volumePath, 20, ""); + Response response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", volumePath); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), "Expected a message indicating the path must be at the bucket level or deeper"); volumePath = "/volb"; - response = deletedKeysSearchEndpoint.searchDeletedKeys(volumePath, 20, ""); + response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", volumePath); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); entity = (String) response.getEntity(); assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), @@ -157,23 +164,23 @@ public void testVolumeLevelSearchRestriction() throws IOException { @Test public void testBucketLevelSearch() throws IOException { // Search inside FSO bucket - Response response = deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1", 20, ""); + Response response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", "/volb/bucketb1"); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(7, result.getRepeatedOmKeyInfoList().size()); - response = deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1", 2, ""); + response = omdbInsightEndpoint.getDeletedKeyInfo(2, "", "/volb/bucketb1"); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(2, result.getRepeatedOmKeyInfoList().size()); // Search inside OBS bucket - response = deletedKeysSearchEndpoint.searchDeletedKeys("/volc/bucketc1", 20, ""); + response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", "/volc/bucketc1"); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(9, result.getRepeatedOmKeyInfoList().size()); - response = deletedKeysSearchEndpoint.searchDeletedKeys("/vola/nonexistentbucket", 20, ""); + response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", "/vola/nonexistentbucket"); assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), @@ -182,20 +189,17 @@ public void testBucketLevelSearch() throws IOException { @Test public void testDirectoryLevelSearch() throws IOException { - Response response = deletedKeysSearchEndpoint.searchDeletedKeys("/volc/bucketc1/dirc1", 20, - ""); + Response response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", "/volc/bucketc1/dirc1"); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(4, result.getRepeatedOmKeyInfoList().size()); - response = deletedKeysSearchEndpoint.searchDeletedKeys("/volc/bucketc1/dirc2", 20, - ""); + response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", "/volc/bucketc1/dirc2"); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(5, result.getRepeatedOmKeyInfoList().size()); - response = deletedKeysSearchEndpoint.searchDeletedKeys( - "/volb/bucketb1/nonexistentdir", 20, ""); + response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", "/volb/bucketb1/nonexistentdir"); assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), @@ -206,22 +210,22 @@ public void testDirectoryLevelSearch() throws IOException { public void testKeyLevelSearch() throws IOException { // FSO Bucket key-level search Response response = - deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1/fileb1", 10, - ""); + omdbInsightEndpoint.getDeletedKeyInfo(10, "", "/volb/bucketb1/fileb1"); assertEquals(200, response.getStatus()); - KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + KeyInsightInfoResponse result = + (KeyInsightInfoResponse) response.getEntity(); assertEquals(1, result.getRepeatedOmKeyInfoList().size()); - response = deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1/fileb2", 10, - ""); + response = + omdbInsightEndpoint.getDeletedKeyInfo(10, "", "/volb/bucketb1/fileb2"); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(1, result.getRepeatedOmKeyInfoList().size()); // Test with non-existent key - response = deletedKeysSearchEndpoint.searchDeletedKeys( - "/volb/bucketb1/nonexistentfile", 1, ""); - assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + response = omdbInsightEndpoint.getDeletedKeyInfo(1, "", "/volb/bucketb1/nonexistentfile"); + assertEquals(Response.Status.NOT_FOUND.getStatusCode(), + response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); @@ -231,15 +235,13 @@ public void testKeyLevelSearch() throws IOException { public void testKeyLevelSearchUnderDirectory() throws IOException { // FSO Bucket key-level search under directory Response response = - deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1/dir1/file1", - 10, ""); + omdbInsightEndpoint.getDeletedKeyInfo(10, "", "/volb/bucketb1/dir1/file1"); assertEquals(200, response.getStatus()); - KeyInsightInfoResponse result = - (KeyInsightInfoResponse) response.getEntity(); + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(1, result.getRepeatedOmKeyInfoList().size()); - response = deletedKeysSearchEndpoint.searchDeletedKeys( - "/volb/bucketb1/dir1/nonexistentfile", 10, ""); + response = omdbInsightEndpoint.getDeletedKeyInfo(10, "", + "/volb/bucketb1/dir1/nonexistentfile"); assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); @@ -251,32 +253,28 @@ public void testKeyLevelSearchUnderDirectory() throws IOException { public void testSearchUnderNestedDirectory() throws IOException { // OBS Bucket nested directory search Response response = - deletedKeysSearchEndpoint.searchDeletedKeys("/volc/bucketc1/dirc1", 20, ""); + omdbInsightEndpoint.getDeletedKeyInfo(20, "", "/volc/bucketc1/dirc1"); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(4, result.getRepeatedOmKeyInfoList().size()); - response = deletedKeysSearchEndpoint.searchDeletedKeys( - "/volc/bucketc1/dirc1/dirc11", 20, ""); + response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", "/volc/bucketc1/dirc1/dirc11"); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(2, result.getRepeatedOmKeyInfoList().size()); - response = deletedKeysSearchEndpoint.searchDeletedKeys( - "/volc/bucketc1/dirc1/dirc11/dirc111", 20, ""); + response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", "/volc/bucketc1/dirc1/dirc11/dirc111"); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(1, result.getRepeatedOmKeyInfoList().size()); - response = deletedKeysSearchEndpoint.searchDeletedKeys( - "/volc/bucketc1/dirc1/dirc11/dirc111/nonexistentfile", 20, ""); + response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", "/volc/bucketc1/dirc1/dirc11/dirc111/nonexistentfile"); assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); - response = deletedKeysSearchEndpoint.searchDeletedKeys( - "/volc/bucketc1/dirc1/dirc11/nonexistentfile", 20, ""); + response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", "/volc/bucketc1/dirc1/dirc11/nonexistentfile"); assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), @@ -285,7 +283,7 @@ public void testSearchUnderNestedDirectory() throws IOException { @Test public void testLimitSearch() throws IOException { - Response response = deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1", 2, ""); + Response response = omdbInsightEndpoint.getDeletedKeyInfo(2, "", "/volb/bucketb1"); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(2, result.getRepeatedOmKeyInfoList().size()); @@ -294,13 +292,13 @@ public void testLimitSearch() throws IOException { @Test public void testSearchDeletedKeysWithBadRequest() throws IOException { int negativeLimit = -1; - Response response = deletedKeysSearchEndpoint.searchDeletedKeys("@323232", negativeLimit, ""); + Response response = omdbInsightEndpoint.getDeletedKeyInfo(negativeLimit, "", "@323232"); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), "Expected a message indicating the path must be at the bucket level or deeper"); - response = deletedKeysSearchEndpoint.searchDeletedKeys("///", 20, ""); + response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", "///"); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); entity = (String) response.getEntity(); assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), @@ -309,7 +307,7 @@ public void testSearchDeletedKeysWithBadRequest() throws IOException { @Test public void testLastKeyInResponse() throws IOException { - Response response = deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb1", 20, ""); + Response response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", "/volb/bucketb1"); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(7, result.getRepeatedOmKeyInfoList().size()); @@ -330,7 +328,7 @@ public void testSearchDeletedKeysWithPagination() throws IOException { int limit = 2; String prevKey = ""; - Response response = deletedKeysSearchEndpoint.searchDeletedKeys(startPrefix, limit, prevKey); + Response response = omdbInsightEndpoint.getDeletedKeyInfo(limit, prevKey, startPrefix); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(2, result.getRepeatedOmKeyInfoList().size()); @@ -338,7 +336,7 @@ public void testSearchDeletedKeysWithPagination() throws IOException { prevKey = result.getLastKey(); assertNotNull(prevKey, "Last key should not be null"); - response = deletedKeysSearchEndpoint.searchDeletedKeys(startPrefix, limit, prevKey); + response = omdbInsightEndpoint.getDeletedKeyInfo(limit, prevKey, startPrefix); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(2, result.getRepeatedOmKeyInfoList().size()); @@ -346,7 +344,7 @@ public void testSearchDeletedKeysWithPagination() throws IOException { prevKey = result.getLastKey(); assertNotNull(prevKey, "Last key should not be null"); - response = deletedKeysSearchEndpoint.searchDeletedKeys(startPrefix, limit, prevKey); + response = omdbInsightEndpoint.getDeletedKeyInfo(limit, prevKey, startPrefix); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(2, result.getRepeatedOmKeyInfoList().size()); @@ -354,7 +352,7 @@ public void testSearchDeletedKeysWithPagination() throws IOException { prevKey = result.getLastKey(); assertNotNull(prevKey, "Last key should not be null"); - response = deletedKeysSearchEndpoint.searchDeletedKeys(startPrefix, limit, prevKey); + response = omdbInsightEndpoint.getDeletedKeyInfo(limit, prevKey, startPrefix); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(1, result.getRepeatedOmKeyInfoList().size()); @@ -373,14 +371,81 @@ public void testSearchDeletedKeysWithPagination() throws IOException { @Test public void testSearchInEmptyBucket() throws IOException { - Response response = - deletedKeysSearchEndpoint.searchDeletedKeys("/volb/bucketb2", 20, ""); + Response response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", "/volb/bucketb2"); assertEquals(404, response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); } + @Test + public void testPrevKeyProvidedStartPrefixEmpty() throws IOException { + // Case 1: prevKey provided, startPrefix empty + // Seek to the prevKey, skip the first matching record, then return remaining records until limit is reached. + String prevKey = "/volb/bucketb1/fileb3"; // This key exists, will skip it + int limit = 3; + String startPrefix = ""; // Empty startPrefix + + Response response = omdbInsightEndpoint.getDeletedKeyInfo(limit, prevKey, startPrefix); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + + // Assert that we get the next 3 records after skipping the prevKey + assertEquals(3, result.getRepeatedOmKeyInfoList().size()); + assertEquals("fileb4", result.getRepeatedOmKeyInfoList().get(0).getOmKeyInfoList().get(0).getKeyName()); + } + + @Test + public void testPrevKeyEmptyStartPrefixEmpty() throws IOException { + // Case 2: prevKey empty, startPrefix empty + // No need to seek, start from the first record and return records until limit is reached. + String prevKey = ""; // Empty prevKey + int limit = 100; + String startPrefix = ""; // Empty startPrefix + + Response response = omdbInsightEndpoint.getDeletedKeyInfo(limit, prevKey, startPrefix); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + + // Assert that we get all the 16 records currently in the deleted keys table + assertEquals(16, result.getRepeatedOmKeyInfoList().size()); + } + + @Test + public void testPrevKeyEmptyStartPrefixProvided() throws IOException { + // Case 3: prevKey empty, startPrefix provided + // Seek to the startPrefix and return matching records until limit is reached. + String prevKey = ""; // Empty prevKey + int limit = 2; + String startPrefix = "/volb/bucketb1/fileb"; // Seek to startPrefix and match files + + Response response = omdbInsightEndpoint.getDeletedKeyInfo(limit, prevKey, startPrefix); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + + // Assert that we get the first 2 records that match startPrefix + assertEquals(2, result.getRepeatedOmKeyInfoList().size()); + assertEquals("fileb1", result.getRepeatedOmKeyInfoList().get(0).getOmKeyInfoList().get(0).getKeyName()); + } + + @Test + public void testPrevKeyProvidedStartPrefixProvided() throws IOException { + // Case 4: prevKey provided, startPrefix provided + // Seek to the prevKey, skip it, and return remaining records matching startPrefix until limit is reached. + String prevKey = "/volb/bucketb1/fileb2"; // This key exists, will skip it + int limit = 3; + String startPrefix = "/volb/bucketb1"; // Matching prefix + + Response response = omdbInsightEndpoint.getDeletedKeyInfo(limit, prevKey, startPrefix); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + + // Assert that we get the next 2 records that match startPrefix after skipping prevKey having fileb2 + assertEquals(3, result.getRepeatedOmKeyInfoList().size()); + assertEquals("fileb3", result.getRepeatedOmKeyInfoList().get(0).getOmKeyInfoList().get(0).getKeyName()); + } + + /** * Populates the OMDB with a set of deleted keys for testing purposes. * This diagram is for reference: @@ -408,7 +473,7 @@ public void testSearchInEmptyBucket() throws IOException { * │ │ │ ├── filec3 (Size: 1000KB) * │ │ │ ├── filec4 (Size: 1000KB) * │ │ │ ├── filec5 (Size: 1000KB) - * │ │ │ ├── filgetec6 (Size: 1000KB) + * │ │ │ ├── filec6 (Size: 1000KB) * │ │ │ └── filec7 (Size: 1000KB) * * @throws Exception if an error occurs while creating deleted keys. diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOMDBInsightSearchEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOMDBInsightSearchEndpoint.java index ab16f349af27..2dd4ab8fdf25 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOMDBInsightSearchEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOMDBInsightSearchEndpoint.java @@ -97,11 +97,9 @@ public class TestOMDBInsightSearchEndpoint extends AbstractReconSqlDBTest { @BeforeEach public void setUp() throws Exception { ozoneConfiguration = new OzoneConfiguration(); - ozoneConfiguration.setLong(OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD, - 100); + ozoneConfiguration.setLong(OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD, 100); omMetadataManager = initializeNewOmMetadataManager( - Files.createDirectory(temporaryFolder.resolve("JunitOmDBDir")) - .toFile()); + Files.createDirectory(temporaryFolder.resolve("JunitOmDBDir")).toFile()); reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, Files.createDirectory(temporaryFolder.resolve("OmMetataDir")).toFile()); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java index 74c58cd9d38b..a1e8585401d1 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java @@ -62,6 +62,7 @@ import org.junit.jupiter.api.io.TempDir; import javax.ws.rs.core.Response; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.sql.Timestamp; @@ -1212,7 +1213,7 @@ public void testGetDeletedKeyInfoLimitParam() throws Exception { reconOMMetadataManager.getDeletedTable() .put("/sampleVol/bucketOne/key_three", repeatedOmKeyInfo3); - Response deletedKeyInfo = omdbInsightEndpoint.getDeletedKeyInfo(2, ""); + Response deletedKeyInfo = omdbInsightEndpoint.getDeletedKeyInfo(2, "", ""); KeyInsightInfoResponse keyInsightInfoResp = (KeyInsightInfoResponse) deletedKeyInfo.getEntity(); assertNotNull(keyInsightInfoResp); @@ -1244,7 +1245,7 @@ public void testGetDeletedKeyInfoPrevKeyParam() throws Exception { .put("/sampleVol/bucketOne/key_three", repeatedOmKeyInfo3); Response deletedKeyInfo = omdbInsightEndpoint.getDeletedKeyInfo(2, - "/sampleVol/bucketOne/key_one"); + "/sampleVol/bucketOne/key_one", ""); KeyInsightInfoResponse keyInsightInfoResp = (KeyInsightInfoResponse) deletedKeyInfo.getEntity(); assertNotNull(keyInsightInfoResp); @@ -1278,7 +1279,7 @@ public void testGetDeletedKeyInfo() throws Exception { .get("/sampleVol/bucketOne/key_one"); assertEquals("key_one", repeatedOmKeyInfo1.getOmKeyInfoList().get(0).getKeyName()); - Response deletedKeyInfo = omdbInsightEndpoint.getDeletedKeyInfo(-1, ""); + Response deletedKeyInfo = omdbInsightEndpoint.getDeletedKeyInfo(-1, "", ""); KeyInsightInfoResponse keyInsightInfoResp = (KeyInsightInfoResponse) deletedKeyInfo.getEntity(); assertNotNull(keyInsightInfoResp); @@ -1287,6 +1288,128 @@ public void testGetDeletedKeyInfo() throws Exception { .get(0).getKeyName()); } + @Test + public void testGetDeletedKeysWithPrevKeyProvidedAndStartPrefixEmpty() + throws Exception { + // Prepare mock data in the deletedTable. + for (int i = 1; i <= 10; i++) { + OmKeyInfo omKeyInfo = + getOmKeyInfo("sampleVol", "bucketOne", "deleted_key_" + i, true); + reconOMMetadataManager.getDeletedTable() + .put("/sampleVol/bucketOne/deleted_key_" + i, + new RepeatedOmKeyInfo(omKeyInfo)); + } + + // Case 1: prevKey provided, startPrefix empty + Response deletedKeyInfoResponse = omdbInsightEndpoint.getDeletedKeyInfo(5, + "/sampleVol/bucketOne/deleted_key_3", ""); + KeyInsightInfoResponse keyInsightInfoResp = + (KeyInsightInfoResponse) deletedKeyInfoResponse.getEntity(); + + // Validate that the response skips the prevKey and returns subsequent records. + assertNotNull(keyInsightInfoResp); + assertEquals(5, keyInsightInfoResp.getRepeatedOmKeyInfoList().size()); + assertEquals("deleted_key_4", + keyInsightInfoResp.getRepeatedOmKeyInfoList().get(0).getOmKeyInfoList().get(0).getKeyName()); + assertEquals("deleted_key_8", + keyInsightInfoResp.getRepeatedOmKeyInfoList().get(4).getOmKeyInfoList().get(0).getKeyName()); + } + + @Test + public void testGetDeletedKeysWithPrevKeyEmptyAndStartPrefixEmpty() + throws Exception { + // Prepare mock data in the deletedTable. + for (int i = 1; i < 10; i++) { + OmKeyInfo omKeyInfo = + getOmKeyInfo("sampleVol", "bucketOne", "deleted_key_" + i, true); + reconOMMetadataManager.getDeletedTable() + .put("/sampleVol/bucketOne/deleted_key_" + i, new RepeatedOmKeyInfo(omKeyInfo)); + } + + // Case 2: prevKey empty, startPrefix empty + Response deletedKeyInfoResponse = + omdbInsightEndpoint.getDeletedKeyInfo(5, "", ""); + KeyInsightInfoResponse keyInsightInfoResp = + (KeyInsightInfoResponse) deletedKeyInfoResponse.getEntity(); + + // Validate that the response retrieves from the beginning. + assertNotNull(keyInsightInfoResp); + assertEquals(5, keyInsightInfoResp.getRepeatedOmKeyInfoList().size()); + assertEquals("deleted_key_1", + keyInsightInfoResp.getRepeatedOmKeyInfoList().get(0).getOmKeyInfoList().get(0).getKeyName()); + assertEquals("deleted_key_5", + keyInsightInfoResp.getRepeatedOmKeyInfoList().get(4).getOmKeyInfoList().get(0).getKeyName()); + } + + @Test + public void testGetDeletedKeysWithStartPrefixProvidedAndPrevKeyEmpty() + throws Exception { + // Prepare mock data in the deletedTable. + for (int i = 1; i < 5; i++) { + OmKeyInfo omKeyInfo = + getOmKeyInfo("sampleVol", "bucketOne", "deleted_key_" + i, true); + reconOMMetadataManager.getDeletedTable() + .put("/sampleVol/bucketOne/deleted_key_" + i, new RepeatedOmKeyInfo(omKeyInfo)); + } + for (int i = 5; i < 10; i++) { + OmKeyInfo omKeyInfo = + getOmKeyInfo("sampleVol", "bucketTwo", "deleted_key_" + i, true); + reconOMMetadataManager.getDeletedTable() + .put("/sampleVol/bucketTwo/deleted_key_" + i, new RepeatedOmKeyInfo(omKeyInfo)); + } + + // Case 3: startPrefix provided, prevKey empty + Response deletedKeyInfoResponse = + omdbInsightEndpoint.getDeletedKeyInfo(5, "", + "/sampleVol/bucketOne/"); + KeyInsightInfoResponse keyInsightInfoResp = + (KeyInsightInfoResponse) deletedKeyInfoResponse.getEntity(); + + // Validate that the response retrieves starting from the prefix. + assertNotNull(keyInsightInfoResp); + assertEquals(4, keyInsightInfoResp.getRepeatedOmKeyInfoList().size()); + assertEquals("deleted_key_1", + keyInsightInfoResp.getRepeatedOmKeyInfoList().get(0).getOmKeyInfoList().get(0).getKeyName()); + assertEquals("deleted_key_4", + keyInsightInfoResp.getRepeatedOmKeyInfoList().get(3).getOmKeyInfoList().get(0).getKeyName()); + } + + @Test + public void testGetDeletedKeysWithBothPrevKeyAndStartPrefixProvided() + throws IOException { + // Prepare mock data in the deletedTable. + for (int i = 1; i < 10; i++) { + OmKeyInfo omKeyInfo = + getOmKeyInfo("sampleVol", "bucketOne", "deleted_key_" + i, true); + reconOMMetadataManager.getDeletedTable() + .put("/sampleVol/bucketOne/deleted_key_" + i, new RepeatedOmKeyInfo(omKeyInfo)); + } + for (int i = 10; i < 15; i++) { + OmKeyInfo omKeyInfo = + getOmKeyInfo("sampleVol", "bucketTwo", "deleted_key_" + i, true); + reconOMMetadataManager.getDeletedTable() + .put("/sampleVol/bucketTwo/deleted_key_" + i, new RepeatedOmKeyInfo(omKeyInfo)); + } + + // Case 4: startPrefix and prevKey provided + Response deletedKeyInfoResponse = + omdbInsightEndpoint.getDeletedKeyInfo(5, + "/sampleVol/bucketOne/deleted_key_5", + "/sampleVol/bucketOne/"); + + KeyInsightInfoResponse keyInsightInfoResp = + (KeyInsightInfoResponse) deletedKeyInfoResponse.getEntity(); + + // Validate that the response retrieves starting from the prefix and skips the prevKey. + assertNotNull(keyInsightInfoResp); + assertEquals(4, keyInsightInfoResp.getRepeatedOmKeyInfoList().size()); + assertEquals("deleted_key_6", + keyInsightInfoResp.getRepeatedOmKeyInfoList().get(0).getOmKeyInfoList().get(0).getKeyName()); + assertEquals("deleted_key_9", + keyInsightInfoResp.getRepeatedOmKeyInfoList().get(3).getOmKeyInfoList().get(0).getKeyName()); + } + + private OmKeyInfo getOmKeyInfo(String volumeName, String bucketName, String keyName, boolean isFile) { return new OmKeyInfo.Builder() From 53d81262591afdb1068c87b7237a917158c8d160 Mon Sep 17 00:00:00 2001 From: arafat Date: Thu, 10 Oct 2024 14:51:50 +0530 Subject: [PATCH 08/21] Fixed checkstyle issues --- .../apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java index 9d267250fd60..e48c1525195d 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java @@ -24,7 +24,6 @@ import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; import org.apache.hadoop.ozone.recon.api.types.KeyEntityInfo; import org.apache.hadoop.ozone.recon.api.types.KeyInsightInfoResponse; From 28c45365537e3395d1d196769173b5488aff762e Mon Sep 17 00:00:00 2001 From: arafat Date: Thu, 10 Oct 2024 17:43:22 +0530 Subject: [PATCH 09/21] Made sure all the Insight endpoints utilise one method for extracting keys from their respective tables --- .../apache/hadoop/ozone/recon/ReconUtils.java | 53 ++++++++++----- .../ozone/recon/api/OMDBInsightEndpoint.java | 67 +------------------ .../recon/api/OMDBInsightSearchEndpoint.java | 10 +-- 3 files changed, 46 insertions(+), 84 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index 9f7c0f6573e7..a1a0fd6a438f 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -601,37 +601,60 @@ public static long convertToEpochMillis(String dateString, String dateFormat, Ti } /** - * Common method to retrieve keys from a table based on a search prefix and a limit. + * Retrieves keys from the specified table based on pagination and prefix filtering. + * This method handles different scenarios based on the presence of startPrefix and prevKey, + * enabling efficient key retrieval from the table. * - * @param table The table to retrieve keys from. - * @param startPrefix The search prefix to match keys against. - * @param limit The maximum number of keys to retrieve. - * @param prevKey The key to start after for the next set of records. - * @return A map of keys and their corresponding OmKeyInfo or RepeatedOmKeyInfo objects. - * @throws IOException If there are problems accessing the table. + * The method handles the following cases: + * + * 1. prevKey provided, startPrefix empty: + * - Seeks to prevKey, skips it, and returns subsequent records up to the limit. + * + * 2. prevKey empty, startPrefix empty: + * - Iterates from the beginning of the table, retrieving all records up to the limit. + * + * 3. startPrefix provided, prevKey empty: + * - Seeks to the first key matching startPrefix and returns all matching keys up to the limit. + * + * 4. startPrefix provided, prevKey provided: + * - Seeks to prevKey, skips it, and returns subsequent keys that match startPrefix, up to the limit. + * + * If limit is 0, all matching keys are retrieved. If both startPrefix and prevKey are empty, the method starts + * from the beginning of the table. */ - public static Map retrieveKeysFromTable( + public static Map extractKeysFromTable( Table table, String startPrefix, int limit, String prevKey) throws IOException { + Map matchedKeys = new LinkedHashMap<>(); try (TableIterator> keyIter = table.iterator()) { - // If a previous key is provided, seek to the previous key and skip it. + + // Scenario 1 & 4: prevKey is provided (whether startPrefix is empty or not) if (!prevKey.isEmpty()) { keyIter.seek(prevKey); if (keyIter.hasNext()) { - // Skip the previous key + // Skip the previous key record keyIter.next(); } - } else { - // If no previous key is provided, start from the search prefix. + } else if (!startPrefix.isEmpty()) { + // Scenario 3: startPrefix is provided but prevKey is empty, so seek to startPrefix keyIter.seek(startPrefix); } - while (keyIter.hasNext() && matchedKeys.size() < limit) { + // Scenario 2: Both startPrefix and prevKey are empty (iterate from the start of the table) + // No seeking needed; just start iterating from the first record in the table + // This is implicit in the following loop, as the iterator will start from the beginning + + // Iterate through the keys while adhering to the limit (if the limit is not zero) + while (keyIter.hasNext() && (limit == 0 || matchedKeys.size() < limit)) { Table.KeyValue entry = keyIter.next(); String dbKey = entry.getKey(); - if (!dbKey.startsWith(startPrefix)) { - break; // Exit the loop if the key no longer matches the prefix + + // Scenario 3 & 4: If startPrefix is provided, ensure the key matches startPrefix + if (!startPrefix.isEmpty() && !dbKey.startsWith(startPrefix)) { + break; // If the key no longer matches the prefix, exit the loop } + + // Add the valid key-value pair to the results matchedKeys.put(dbKey, entry.getValue()); } } catch (IOException exception) { diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java index a4e9609c2183..1fa810a48ef8 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java @@ -84,6 +84,7 @@ import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createBadRequestResponse; import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createInternalServerErrorResponse; import static org.apache.hadoop.ozone.recon.ReconResponseUtils.noMatchedKeysResponse; +import static org.apache.hadoop.ozone.recon.ReconUtils.extractKeysFromTable; import static org.apache.hadoop.ozone.recon.api.handlers.BucketHandler.getBucketHandler; import static org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.normalizePath; import static org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.parseRequestPath; @@ -487,7 +488,7 @@ private boolean getPendingForDeletionKeyInfo( // Search for deleted keys in DeletedTable Table deletedTable = omMetadataManager.getDeletedTable(); Map deletedKeys = - retrieveKeysFromTable(deletedTable, startPrefix, limit, prevKey); + extractKeysFromTable(deletedTable, startPrefix, limit, prevKey); // Iterate over the retrieved keys and populate the response for (Map.Entry entry : deletedKeys.entrySet()) { @@ -515,70 +516,6 @@ private boolean getPendingForDeletionKeyInfo( return keysFound; } - /** - * Retrieves keys from the specified table based on pagination and prefix filtering. - * This method handles different scenarios based on the presence of startPrefix and prevKey, - * enabling efficient key retrieval from the table. - * - * The method handles the following cases: - * - * 1. prevKey provided, startPrefix empty: - * - Seeks to prevKey, skips it, and returns subsequent records up to the limit. - * - * 2. prevKey empty, startPrefix empty: - * - Iterates from the beginning of the table, retrieving all records up to the limit. - * - * 3. startPrefix provided, prevKey empty: - * - Seeks to the first key matching startPrefix and returns all matching keys up to the limit. - * - * 4. startPrefix provided, prevKey provided: - * - Seeks to prevKey, skips it, and returns subsequent keys that match startPrefix, up to the limit. - * - * If limit is 0, all matching keys are retrieved. If both startPrefix and prevKey are empty, the method starts - * from the beginning of the table. - */ - public static Map retrieveKeysFromTable( - Table table, String startPrefix, int limit, String prevKey) - throws IOException { - - Map matchedKeys = new LinkedHashMap<>(); - try (TableIterator> keyIter = table.iterator()) { - - // Scenario 1 & 4: prevKey is provided (whether startPrefix is empty or not) - if (!prevKey.isEmpty()) { - keyIter.seek(prevKey); - if (keyIter.hasNext()) { - // Skip the previous key record - keyIter.next(); - } - } else if (!startPrefix.isEmpty()) { - // Scenario 3: startPrefix is provided but prevKey is empty, so seek to startPrefix - keyIter.seek(startPrefix); - } - // Scenario 2: Both startPrefix and prevKey are empty (iterate from the start of the table) - // No seeking needed; just start iterating from the first record in the table - // This is implicit in the following loop, as the iterator will start from the beginning - - // Iterate through the keys while adhering to the limit (if the limit is not zero) - while (keyIter.hasNext() && (limit == 0 || matchedKeys.size() < limit)) { - Table.KeyValue entry = keyIter.next(); - String dbKey = entry.getKey(); - - // Scenario 3 & 4: If startPrefix is provided, ensure the key matches startPrefix - if (!startPrefix.isEmpty() && !dbKey.startsWith(startPrefix)) { - break; // If the key no longer matches the prefix, exit the loop - } - - // Add the valid key-value pair to the results - matchedKeys.put(dbKey, entry.getValue()); - } - } catch (IOException exception) { - LOG.error("Error retrieving keys from table for path: {}", startPrefix, exception); - throw exception; - } - return matchedKeys; - } - /** * Creates a keys summary for deleted keys and updates the provided * keysSummary map. Calculates the total number of deleted keys, replicated diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java index e48c1525195d..94a165132193 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java @@ -55,7 +55,7 @@ import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createBadRequestResponse; import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createInternalServerErrorResponse; import static org.apache.hadoop.ozone.recon.ReconUtils.constructObjectPathWithPrefix; -import static org.apache.hadoop.ozone.recon.ReconUtils.retrieveKeysFromTable; +import static org.apache.hadoop.ozone.recon.ReconUtils.extractKeysFromTable; import static org.apache.hadoop.ozone.recon.ReconUtils.gatherSubPaths; import static org.apache.hadoop.ozone.recon.ReconUtils.validateNames; import static org.apache.hadoop.ozone.recon.api.handlers.BucketHandler.getBucketHandler; @@ -142,7 +142,7 @@ public Response searchOpenKeys( Table openKeyTable = omMetadataManager.getOpenKeyTable(BucketLayout.LEGACY); Map obsKeys = - retrieveKeysFromTable(openKeyTable, startPrefix, limit, prevKey); + extractKeysFromTable(openKeyTable, startPrefix, limit, prevKey); for (Map.Entry entry : obsKeys.entrySet()) { keysFound = true; KeyEntityInfo keyEntityInfo = @@ -223,7 +223,8 @@ public Map searchOpenKeysInFSO(String startPrefix, // Iterate over the subpaths and retrieve the open files for (String subPath : subPaths) { - matchedKeys.putAll(retrieveKeysFromTable(openFileTable, subPath, limit - matchedKeys.size(), prevKey)); + matchedKeys.putAll( + extractKeysFromTable(openFileTable, subPath, limit - matchedKeys.size(), prevKey)); if (matchedKeys.size() >= limit) { break; } @@ -232,7 +233,8 @@ public Map searchOpenKeysInFSO(String startPrefix, } // If the search level is at the volume, bucket or key level, directly search the openFileTable - matchedKeys.putAll(retrieveKeysFromTable(openFileTable, startPrefixObjectPath, limit, prevKey)); + matchedKeys.putAll( + extractKeysFromTable(openFileTable, startPrefixObjectPath, limit, prevKey)); return matchedKeys; } From 1df317d3423069699bbd6f39d69e111aae7c7365 Mon Sep 17 00:00:00 2001 From: arafat Date: Thu, 10 Oct 2024 23:36:23 +0530 Subject: [PATCH 10/21] Made final review changes --- .../apache/hadoop/ozone/recon/ReconUtils.java | 21 ++++++++++++++----- .../ozone/recon/api/OMDBInsightEndpoint.java | 19 +++++++---------- .../recon/api/OMDBInsightSearchEndpoint.java | 4 ++-- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index a1a0fd6a438f..1bd4646812f5 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -619,33 +619,44 @@ public static long convertToEpochMillis(String dateString, String dateFormat, Ti * 4. startPrefix provided, prevKey provided: * - Seeks to prevKey, skips it, and returns subsequent keys that match startPrefix, up to the limit. * - * If limit is 0, all matching keys are retrieved. If both startPrefix and prevKey are empty, the method starts - * from the beginning of the table. + * This method also handles the following limit scenarios: + * - If limit == 0 or limit < -1, no records are returned. + * - If limit == -1, all records are returned. + * - For positive limits, it retrieves records up to the specified limit. */ public static Map extractKeysFromTable( Table table, String startPrefix, int limit, String prevKey) throws IOException { Map matchedKeys = new LinkedHashMap<>(); + + // If limit == 0, return an empty result set + if (limit == 0 || limit < -1) { + return matchedKeys; + } + + // If limit == -1, set it to Integer.MAX_VALUE to return all records + int actualLimit = (limit == -1) ? Integer.MAX_VALUE : limit; + try (TableIterator> keyIter = table.iterator()) { // Scenario 1 & 4: prevKey is provided (whether startPrefix is empty or not) if (!prevKey.isEmpty()) { keyIter.seek(prevKey); if (keyIter.hasNext()) { - // Skip the previous key record - keyIter.next(); + keyIter.next(); // Skip the previous key record } } else if (!startPrefix.isEmpty()) { // Scenario 3: startPrefix is provided but prevKey is empty, so seek to startPrefix keyIter.seek(startPrefix); } + // Scenario 2: Both startPrefix and prevKey are empty (iterate from the start of the table) // No seeking needed; just start iterating from the first record in the table // This is implicit in the following loop, as the iterator will start from the beginning // Iterate through the keys while adhering to the limit (if the limit is not zero) - while (keyIter.hasNext() && (limit == 0 || matchedKeys.size() < limit)) { + while (keyIter.hasNext() && matchedKeys.size() < actualLimit) { Table.KeyValue entry = keyIter.next(); String dbKey = entry.getKey(); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java index 1fa810a48ef8..17563e00f3cc 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java @@ -434,22 +434,17 @@ public Response getDeletedKeyInfo( // Initialize the response object to hold the key information KeyInsightInfoResponse deletedKeyInsightInfo = new KeyInsightInfoResponse(); - // Ensure the limit is non-negative - limit = Math.max(0, limit); - boolean keysFound = false; try { - if (isNotBlank(startPrefix)) { - // Validate and apply prefix-based search - if (!validateStartPrefix(startPrefix)) { - return createBadRequestResponse("Invalid startPrefix: Path must be at the bucket level or deeper."); - } - keysFound = getPendingForDeletionKeyInfo(limit, prevKey, startPrefix, deletedKeyInsightInfo); - } else { - // Retrieve all records without prefix - keysFound = getPendingForDeletionKeyInfo(limit, prevKey, "", deletedKeyInsightInfo); + // Validate startPrefix if it's provided + if (isNotBlank(startPrefix) && !validateStartPrefix(startPrefix)) { + return createBadRequestResponse("Invalid startPrefix: Path must be at the bucket level or deeper."); } + + // Perform the search based on the limit, prevKey, and startPrefix + keysFound = getPendingForDeletionKeyInfo(limit, prevKey, startPrefix, deletedKeyInsightInfo); + } catch (IllegalArgumentException e) { LOG.debug("Invalid startPrefix provided: {}", startPrefix, e); return createBadRequestResponse("Invalid startPrefix: " + e.getMessage()); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java index 94a165132193..fedec210572f 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java @@ -124,7 +124,7 @@ public Response searchOpenKeys( try { // Validate the request parameters - if (!validateStartPrefixAndLimit(startPrefix)) { + if (!validateStartPrefix(startPrefix)) { return createBadRequestResponse("Invalid startPrefix: Path must be at the bucket level or deeper."); } @@ -343,7 +343,7 @@ private KeyEntityInfo createKeyEntityInfoFromOmKeyInfo(String dbKey, return keyEntityInfo; } - private boolean validateStartPrefixAndLimit(String startPrefix) { + private boolean validateStartPrefix(String startPrefix) { // Ensure startPrefix is not null or empty and starts with '/' if (startPrefix == null || startPrefix.isEmpty()) { return false; From 5d9d3c2170d10e52b1fcdd89156264842f58c59b Mon Sep 17 00:00:00 2001 From: arafat Date: Fri, 11 Oct 2024 00:59:56 +0530 Subject: [PATCH 11/21] Fixed failing test --- .../hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java index 4291c90b1aa0..c95adaab733a 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java @@ -137,7 +137,7 @@ public void testEmptySearchPrefix() throws IOException { assertEquals(16, result.getRepeatedOmKeyInfoList().size()); // Set limit to 10 and pass empty search prefix - response = omdbInsightEndpoint.getDeletedKeyInfo(10, "", null); + response = omdbInsightEndpoint.getDeletedKeyInfo(10, "", ""); // In this case we get all the keys from the OMDB assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); From 9ec2c327a67d67d67c675b4cc0d6e83edc750e4e Mon Sep 17 00:00:00 2001 From: arafat Date: Fri, 11 Oct 2024 12:08:25 +0530 Subject: [PATCH 12/21] Fixed java doc error --- .../java/org/apache/hadoop/ozone/recon/ReconUtils.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index 1bd4646812f5..ca3bb98943c6 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -623,6 +623,13 @@ public static long convertToEpochMillis(String dateString, String dateFormat, Ti * - If limit == 0 or limit < -1, no records are returned. * - If limit == -1, all records are returned. * - For positive limits, it retrieves records up to the specified limit. + * + * @param table The table to retrieve keys from. + * @param startPrefix The search prefix to match keys against. + * @param limit The maximum number of keys to retrieve. + * @param prevKey The key to start after for the next set of records. + * @return A map of keys and their corresponding OmKeyInfo or RepeatedOmKeyInfo objects. + * @throws IOException If there are problems accessing the table. */ public static Map extractKeysFromTable( Table table, String startPrefix, int limit, String prevKey) From 28b0b006f472558ddc34d9df7f336b9a9d13a2d7 Mon Sep 17 00:00:00 2001 From: arafat Date: Fri, 11 Oct 2024 14:13:57 +0530 Subject: [PATCH 13/21] Fixed possible java doc comment --- .../java/org/apache/hadoop/ozone/recon/ReconUtils.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index a88520bd9ad5..9b78c4e7e7a3 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -620,8 +620,8 @@ public static long convertToEpochMillis(String dateString, String dateFormat, Ti * - Seeks to prevKey, skips it, and returns subsequent keys that match startPrefix, up to the limit. * * This method also handles the following limit scenarios: - * - If limit == 0 or limit < -1, no records are returned. - * - If limit == -1, all records are returned. + * - If limit = 0 or limit < -1, no records are returned. + * - If limit = -1, all records are returned. * - For positive limits, it retrieves records up to the specified limit. * * @param table The table to retrieve keys from. @@ -637,12 +637,12 @@ public static Map extractKeysFromTable( Map matchedKeys = new LinkedHashMap<>(); - // If limit == 0, return an empty result set + // If limit = 0, return an empty result set if (limit == 0 || limit < -1) { return matchedKeys; } - // If limit == -1, set it to Integer.MAX_VALUE to return all records + // If limit = -1, set it to Integer.MAX_VALUE to return all records int actualLimit = (limit == -1) ? Integer.MAX_VALUE : limit; try (TableIterator> keyIter = table.iterator()) { From 16b2c36c94aaa1e1717d954904b98328727f6f95 Mon Sep 17 00:00:00 2001 From: arafat Date: Mon, 14 Oct 2024 12:32:07 +0530 Subject: [PATCH 14/21] Fixed final java doc problem --- .../apache/hadoop/ozone/recon/ReconUtils.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index 9b78c4e7e7a3..222c70da6de3 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -602,33 +602,33 @@ public static long convertToEpochMillis(String dateString, String dateFormat, Ti /** * Retrieves keys from the specified table based on pagination and prefix filtering. - * This method handles different scenarios based on the presence of startPrefix and prevKey, - * enabling efficient key retrieval from the table. + * This method handles different scenarios based on the presence of {@code startPrefix} + * and {@code prevKey}, enabling efficient key retrieval from the table. * * The method handles the following cases: * - * 1. prevKey provided, startPrefix empty: - * - Seeks to prevKey, skips it, and returns subsequent records up to the limit. + * 1. {@code prevKey} provided, {@code startPrefix} empty: + * - Seeks to {@code prevKey}, skips it, and returns subsequent records up to the limit. * - * 2. prevKey empty, startPrefix empty: - * - Iterates from the beginning of the table, retrieving all records up to the limit. + * 2. {@code prevKey} empty, {@code startPrefix} empty: + * - Iterates from the beginning of the table, retrieving all records up to the limit. * - * 3. startPrefix provided, prevKey empty: - * - Seeks to the first key matching startPrefix and returns all matching keys up to the limit. + * 3. {@code startPrefix} provided, {@code prevKey} empty: + * - Seeks to the first key matching {@code startPrefix} and returns all matching keys up to the limit. * - * 4. startPrefix provided, prevKey provided: - * - Seeks to prevKey, skips it, and returns subsequent keys that match startPrefix, up to the limit. + * 4. {@code startPrefix} provided, {@code prevKey} provided: + * - Seeks to {@code prevKey}, skips it, and returns subsequent keys that match {@code startPrefix}, up to the limit. * - * This method also handles the following limit scenarios: - * - If limit = 0 or limit < -1, no records are returned. - * - If limit = -1, all records are returned. - * - For positive limits, it retrieves records up to the specified limit. + * This method also handles the following {@code limit} scenarios: + * - If {@code limit == 0} or {@code limit < -1}, no records are returned. + * - If {@code limit == -1}, all records are returned. + * - For positive {@code limit}, it retrieves records up to the specified {@code limit}. * * @param table The table to retrieve keys from. * @param startPrefix The search prefix to match keys against. * @param limit The maximum number of keys to retrieve. * @param prevKey The key to start after for the next set of records. - * @return A map of keys and their corresponding OmKeyInfo or RepeatedOmKeyInfo objects. + * @return A map of keys and their corresponding {@code OmKeyInfo} or {@code RepeatedOmKeyInfo} objects. * @throws IOException If there are problems accessing the table. */ public static Map extractKeysFromTable( From da82a5119325c71f29f9721b71aa0311578e915a Mon Sep 17 00:00:00 2001 From: arafat Date: Mon, 14 Oct 2024 12:38:49 +0530 Subject: [PATCH 15/21] Fixed checkstyle issue --- .../main/java/org/apache/hadoop/ozone/recon/ReconUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index 222c70da6de3..dbef5c68d583 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -617,7 +617,8 @@ public static long convertToEpochMillis(String dateString, String dateFormat, Ti * - Seeks to the first key matching {@code startPrefix} and returns all matching keys up to the limit. * * 4. {@code startPrefix} provided, {@code prevKey} provided: - * - Seeks to {@code prevKey}, skips it, and returns subsequent keys that match {@code startPrefix}, up to the limit. + * - Seeks to {@code prevKey}, skips it, and returns subsequent keys that match {@code startPrefix}, + * up to the limit. * * This method also handles the following {@code limit} scenarios: * - If {@code limit == 0} or {@code limit < -1}, no records are returned. From 86c9f464b6281f20966aafff0b9035681d81a386 Mon Sep 17 00:00:00 2001 From: arafat Date: Wed, 16 Oct 2024 17:50:21 +0530 Subject: [PATCH 16/21] Final review refactoring --- .../apache/hadoop/ozone/recon/ReconUtils.java | 14 +++++++++++++ .../ozone/recon/api/OMDBInsightEndpoint.java | 21 ++++--------------- .../recon/api/OMDBInsightSearchEndpoint.java | 21 +------------------ 3 files changed, 19 insertions(+), 37 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index dbef5c68d583..9e52a8cec69b 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -600,6 +600,20 @@ public static long convertToEpochMillis(String dateString, String dateFormat, Ti } } + public static boolean validateStartPrefix(String startPrefix) { + + // Ensure startPrefix starts with '/' for non-empty values + startPrefix = startPrefix.startsWith("/") ? startPrefix : "/" + startPrefix; + + // Split the path to ensure it's at least at the bucket level (volume/bucket). + String[] pathComponents = startPrefix.split("/"); + if (pathComponents.length < 3 || pathComponents[2].isEmpty()) { + return false; // Invalid if not at bucket level or deeper + } + + return true; + } + /** * Retrieves keys from the specified table based on pagination and prefix filtering. * This method handles different scenarios based on the presence of {@code startPrefix} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java index 5df2d2e44876..0e09801e3255 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java @@ -85,6 +85,7 @@ import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createInternalServerErrorResponse; import static org.apache.hadoop.ozone.recon.ReconResponseUtils.noMatchedKeysResponse; import static org.apache.hadoop.ozone.recon.ReconUtils.extractKeysFromTable; +import static org.apache.hadoop.ozone.recon.ReconUtils.validateStartPrefix; import static org.apache.hadoop.ozone.recon.api.handlers.BucketHandler.getBucketHandler; import static org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.normalizePath; import static org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.parseRequestPath; @@ -447,13 +448,13 @@ public Response getDeletedKeyInfo( keysFound = getPendingForDeletionKeyInfo(limit, prevKey, startPrefix, deletedKeyInsightInfo); } catch (IllegalArgumentException e) { - LOG.debug("Invalid startPrefix provided: {}", startPrefix, e); + LOG.error("Invalid startPrefix provided: {}", startPrefix, e); return createBadRequestResponse("Invalid startPrefix: " + e.getMessage()); } catch (IOException e) { - LOG.debug("I/O error while searching deleted keys in OM DB", e); + LOG.error("I/O error while searching deleted keys in OM DB", e); return createInternalServerErrorResponse("Error searching deleted keys in OM DB: " + e.getMessage()); } catch (Exception e) { - LOG.debug("Unexpected error occurred while searching deleted keys", e); + LOG.error("Unexpected error occurred while searching deleted keys", e); return createInternalServerErrorResponse("Unexpected error: " + e.getMessage()); } @@ -1303,20 +1304,6 @@ private void createSummaryForDeletedDirectories( dirSummary.put("totalDeletedDirectories", deletedDirCount); } - private boolean validateStartPrefix(String startPrefix) { - - // Ensure startPrefix starts with '/' for non-empty values - startPrefix = startPrefix.startsWith("/") ? startPrefix : "/" + startPrefix; - - // Split the path to ensure it's at least at the bucket level (volume/bucket). - String[] pathComponents = startPrefix.split("/"); - if (pathComponents.length < 3 || pathComponents[2].isEmpty()) { - return false; // Invalid if not at bucket level or deeper - } - - return true; - } - private String createPath(OmKeyInfo omKeyInfo) { return omKeyInfo.getVolumeName() + OM_KEY_PREFIX + omKeyInfo.getBucketName() + OM_KEY_PREFIX + omKeyInfo.getKeyName(); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java index 6df675c0a5bb..099591b54b99 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java @@ -54,10 +54,7 @@ import static org.apache.hadoop.ozone.recon.ReconResponseUtils.noMatchedKeysResponse; import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createBadRequestResponse; import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createInternalServerErrorResponse; -import static org.apache.hadoop.ozone.recon.ReconUtils.constructObjectPathWithPrefix; -import static org.apache.hadoop.ozone.recon.ReconUtils.extractKeysFromTable; -import static org.apache.hadoop.ozone.recon.ReconUtils.gatherSubPaths; -import static org.apache.hadoop.ozone.recon.ReconUtils.validateNames; +import static org.apache.hadoop.ozone.recon.ReconUtils.*; import static org.apache.hadoop.ozone.recon.api.handlers.BucketHandler.getBucketHandler; import static org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.normalizePath; import static org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.parseRequestPath; @@ -346,20 +343,4 @@ private KeyEntityInfo createKeyEntityInfoFromOmKeyInfo(String dbKey, return keyEntityInfo; } - private boolean validateStartPrefix(String startPrefix) { - // Ensure startPrefix is not null or empty and starts with '/' - if (startPrefix == null || startPrefix.isEmpty()) { - return false; - } - startPrefix = startPrefix.startsWith("/") ? startPrefix : "/" + startPrefix; - - // Split the path to ensure it's at least at the bucket level - String[] pathComponents = startPrefix.split("/"); - if (pathComponents.length < 3 || pathComponents[2].isEmpty()) { - return false; - } - - return true; // Validation passed - } - } From 2338c5e09d4341978f4f3bdc03abc610da8c31b9 Mon Sep 17 00:00:00 2001 From: arafat Date: Wed, 16 Oct 2024 17:56:05 +0530 Subject: [PATCH 17/21] Fixed checkstyle issues --- .../hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java index 099591b54b99..fcd73fbe72f2 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java @@ -54,7 +54,11 @@ import static org.apache.hadoop.ozone.recon.ReconResponseUtils.noMatchedKeysResponse; import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createBadRequestResponse; import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createInternalServerErrorResponse; -import static org.apache.hadoop.ozone.recon.ReconUtils.*; +import static org.apache.hadoop.ozone.recon.ReconUtils.validateStartPrefix; +import static org.apache.hadoop.ozone.recon.ReconUtils.constructObjectPathWithPrefix; +import static org.apache.hadoop.ozone.recon.ReconUtils.extractKeysFromTable; +import static org.apache.hadoop.ozone.recon.ReconUtils.gatherSubPaths; +import static org.apache.hadoop.ozone.recon.ReconUtils.validateNames; import static org.apache.hadoop.ozone.recon.api.handlers.BucketHandler.getBucketHandler; import static org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.normalizePath; import static org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.parseRequestPath; From f688dacbef219408811c973fce44ffe006f35bc6 Mon Sep 17 00:00:00 2001 From: arafat Date: Wed, 16 Oct 2024 20:36:30 +0530 Subject: [PATCH 18/21] Removed unnecessary blank check --- .../org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java index 0e09801e3255..154c678a8f93 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java @@ -440,7 +440,7 @@ public Response getDeletedKeyInfo( try { // Validate startPrefix if it's provided - if (isNotBlank(startPrefix) && !validateStartPrefix(startPrefix)) { + if (!validateStartPrefix(startPrefix)) { return createBadRequestResponse("Invalid startPrefix: Path must be at the bucket level or deeper."); } From 96a7f796a1773c777d4f1d6e34933305d27f27d6 Mon Sep 17 00:00:00 2001 From: arafat Date: Wed, 16 Oct 2024 22:17:50 +0530 Subject: [PATCH 19/21] Reverted back the blank check --- .../org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java index 154c678a8f93..0e09801e3255 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java @@ -440,7 +440,7 @@ public Response getDeletedKeyInfo( try { // Validate startPrefix if it's provided - if (!validateStartPrefix(startPrefix)) { + if (isNotBlank(startPrefix) && !validateStartPrefix(startPrefix)) { return createBadRequestResponse("Invalid startPrefix: Path must be at the bucket level or deeper."); } From 005850af5a111b721f0897c24f1f1d855220cd20 Mon Sep 17 00:00:00 2001 From: arafat Date: Fri, 18 Oct 2024 11:18:39 +0530 Subject: [PATCH 20/21] Done with the null check --- .../org/apache/hadoop/ozone/recon/ReconUtils.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index 9e52a8cec69b..d7dd49833b60 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -32,13 +32,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.time.Instant; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.List; -import java.util.TimeZone; -import java.util.Date; -import java.util.Set; -import java.util.ArrayList; +import java.util.*; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -652,6 +646,12 @@ public static Map extractKeysFromTable( Map matchedKeys = new LinkedHashMap<>(); + // Null check for the table to prevent NPE during omMetaManager initialization + if (table == null) { + log.error("Table object is null. omMetaManager might still be initializing."); + return Collections.emptyMap(); + } + // If limit = 0, return an empty result set if (limit == 0 || limit < -1) { return matchedKeys; From 510b30707a4334e64e7053d04bbb4be69abc9925 Mon Sep 17 00:00:00 2001 From: arafat Date: Fri, 18 Oct 2024 13:00:20 +0530 Subject: [PATCH 21/21] Fixed the checkstyle and changed the status code --- .../hadoop/ozone/recon/ReconResponseUtils.java | 2 +- .../apache/hadoop/ozone/recon/ReconUtils.java | 9 ++++++++- .../ozone/recon/api/OMDBInsightEndpoint.java | 2 +- .../api/TestDeletedKeysSearchEndpoint.java | 14 +++++++------- .../api/TestOMDBInsightSearchEndpoint.java | 18 +++++++++--------- 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconResponseUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconResponseUtils.java index 41235ae54280..dc53f195f675 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconResponseUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconResponseUtils.java @@ -44,7 +44,7 @@ public static Response noMatchedKeysResponse(String startPrefix) { String jsonResponse = String.format( "{\"message\": \"No keys matched the search prefix: '%s'.\"}", startPrefix); - return Response.status(Response.Status.NOT_FOUND) + return Response.status(Response.Status.NO_CONTENT) .entity(jsonResponse) .type(MediaType.APPLICATION_JSON) .build(); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index d7dd49833b60..f65e2f30cb8c 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -32,7 +32,14 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.time.Instant; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java index 0e09801e3255..d28275e54758 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java @@ -986,7 +986,7 @@ public Response listKeys(@QueryParam("replicationType") String replicationType, limit, false, ""); Response response = getListKeysResponse(paramInfo); if ((response.getStatus() != Response.Status.OK.getStatusCode()) && - (response.getStatus() != Response.Status.NOT_FOUND.getStatusCode())) { + (response.getStatus() != Response.Status.NO_CONTENT.getStatusCode())) { return response; } if (response.getEntity() instanceof ListKeysResponse) { diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java index c95adaab733a..5f3d0fa12687 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestDeletedKeysSearchEndpoint.java @@ -181,7 +181,7 @@ public void testBucketLevelSearch() throws IOException { assertEquals(9, result.getRepeatedOmKeyInfoList().size()); response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", "/vola/nonexistentbucket"); - assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); @@ -200,7 +200,7 @@ public void testDirectoryLevelSearch() throws IOException { assertEquals(5, result.getRepeatedOmKeyInfoList().size()); response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", "/volb/bucketb1/nonexistentdir"); - assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); @@ -224,7 +224,7 @@ public void testKeyLevelSearch() throws IOException { // Test with non-existent key response = omdbInsightEndpoint.getDeletedKeyInfo(1, "", "/volb/bucketb1/nonexistentfile"); - assertEquals(Response.Status.NOT_FOUND.getStatusCode(), + assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), @@ -242,7 +242,7 @@ public void testKeyLevelSearchUnderDirectory() throws IOException { response = omdbInsightEndpoint.getDeletedKeyInfo(10, "", "/volb/bucketb1/dir1/nonexistentfile"); - assertEquals(Response.Status.NOT_FOUND.getStatusCode(), + assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), @@ -269,13 +269,13 @@ public void testSearchUnderNestedDirectory() throws IOException { assertEquals(1, result.getRepeatedOmKeyInfoList().size()); response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", "/volc/bucketc1/dirc1/dirc11/dirc111/nonexistentfile"); - assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", "/volc/bucketc1/dirc1/dirc11/nonexistentfile"); - assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); @@ -372,7 +372,7 @@ public void testSearchDeletedKeysWithPagination() throws IOException { @Test public void testSearchInEmptyBucket() throws IOException { Response response = omdbInsightEndpoint.getDeletedKeyInfo(20, "", "/volb/bucketb2"); - assertEquals(404, response.getStatus()); + assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOMDBInsightSearchEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOMDBInsightSearchEndpoint.java index 2dd4ab8fdf25..c3c2fe5debed 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOMDBInsightSearchEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOMDBInsightSearchEndpoint.java @@ -219,7 +219,7 @@ public void testBucketLevelSearch() throws IOException { // Test with bucket that does not exist response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/nonexistentbucket", 20, ""); - assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); @@ -262,7 +262,7 @@ public void testDirectoryLevelSearch() throws IOException { // Test with non-existent directory response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/nonexistentdir", 20, ""); - assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); @@ -310,13 +310,13 @@ public void testKeyLevelSearch() throws IOException { // Test with non-existent key response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/nonexistentfile", 1, ""); - assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); response = omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb1/nonexistentfile", 1, ""); - assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); @@ -342,14 +342,14 @@ public void testKeyLevelSearchUnderDirectory() throws IOException { // Test for unknown file in fso bucket response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira1/unknownfile", 10, ""); - assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); // Test for unknown file in fso bucket response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira2/unknownfile", 10, ""); - assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); @@ -400,14 +400,14 @@ public void testSearchUnderNestedDirectory() throws IOException { // Search for a non existant file under each nested directory response = omdbInsightSearchEndpoint.searchOpenKeys( "/vola/bucketa1/dira3/dira31/dira32/dira33/nonexistentfile", 20, ""); - assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); response = omdbInsightSearchEndpoint.searchOpenKeys( "/vola/bucketa1/dira3/dira31/dira32/nonexistentfile", 20, ""); - assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); @@ -505,7 +505,7 @@ public void testSearchOpenKeysWithPagination() throws IOException { public void testSearchInEmptyBucket() throws IOException { // Search in empty bucket bucketb2 Response response = omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb2", 20, ""); - assertEquals(404, response.getStatus()); + assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found");