diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java index 6dbc4746acba..ed657931e034 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java @@ -43,6 +43,7 @@ private ReconConstants() { public static final int DISK_USAGE_TOP_RECORDS_LIMIT = 30; public static final String DEFAULT_OPEN_KEY_INCLUDE_NON_FSO = "false"; public static final String DEFAULT_OPEN_KEY_INCLUDE_FSO = "false"; + public static final String DEFAULT_START_PREFIX = "/"; public static final String DEFAULT_FETCH_COUNT = "1000"; public static final String DEFAULT_KEY_SIZE = "0"; public static final String DEFAULT_BATCH_NUMBER = "1"; @@ -50,6 +51,8 @@ private ReconConstants() { public static final String RECON_QUERY_PREVKEY = "prevKey"; public static final String RECON_OPEN_KEY_INCLUDE_NON_FSO = "includeNonFso"; public static final String RECON_OPEN_KEY_INCLUDE_FSO = "includeFso"; + public static final String RECON_OPEN_KEY_DEFAULT_SEARCH_LIMIT = "1000"; + public static final String RECON_OPEN_KEY_SEARCH_DEFAULT_PREV_KEY = ""; public static final String RECON_QUERY_FILTER = "missingIn"; public static final String PREV_CONTAINER_ID_DEFAULT_VALUE = "0"; public static final String PREV_DELETED_BLOCKS_TRANSACTION_ID_DEFAULT_VALUE = diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index fe920b7098b5..425121675ef7 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -32,10 +32,11 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.time.Instant; -import java.util.ArrayList; -import java.util.Date; import java.util.List; import java.util.TimeZone; +import java.util.Date; +import java.util.Set; +import java.util.ArrayList; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -569,6 +570,54 @@ public static long convertToEpochMillis(String dateString, String dateFormat, Ti } } + /** + * Finds all subdirectories under a parent directory in an FSO bucket. It builds + * a list of paths for these subdirectories. These sub-directories are then used + * to search for open files in the openFileTable. + * + * How it works: + * - Starts from a parent directory identified by parentId. + * - Looks through all child directories of this parent. + * - For each child, it creates a path that starts with volumeID/bucketID/parentId, + * following our openFileTable format. + * - Adds these paths to a list and explores each child further for more subdirectories. + * + * @param parentId The ID of the parent directory from which to start gathering subdirectories. + * @param subPaths The list to which the paths of subdirectories will be added. + * @param volumeID The ID of the volume containing the parent directory. + * @param bucketID The ID of the bucket containing the parent directory. + * @param reconNamespaceSummaryManager The manager used to retrieve NSSummary objects. + * @throws IOException If an I/O error occurs while fetching NSSummary objects. + */ + public static void gatherSubPaths(long parentId, List subPaths, + long volumeID, long bucketID, + ReconNamespaceSummaryManager reconNamespaceSummaryManager) + throws IOException { + // Fetch the NSSummary object for parentId + NSSummary parentSummary = + reconNamespaceSummaryManager.getNSSummary(parentId); + if (parentSummary == null) { + return; + } + + Set childDirIds = parentSummary.getChildDir(); + for (Long childId : childDirIds) { + // Fetch the NSSummary for each child directory + NSSummary childSummary = + reconNamespaceSummaryManager.getNSSummary(childId); + if (childSummary != null) { + String subPath = + ReconUtils.constructObjectPathWithPrefix(volumeID, bucketID, + childId); + // Add to subPaths + subPaths.add(subPath); + // Recurse into this child directory + gatherSubPaths(childId, subPaths, volumeID, bucketID, + reconNamespaceSummaryManager); + } + } + } + /** * Validates volume or bucket names according to specific rules. * diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java index f4aaf50dfc57..3f95c04fc916 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java @@ -61,7 +61,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.TimeZone; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -1061,7 +1060,8 @@ public Map searchKeysInFSO(ParamInfo paramInfo) subPaths.add(startPrefixObjectPath); // Recursively gather all subpaths - gatherSubPaths(parentId, subPaths, Long.parseLong(names[0]), Long.parseLong(names[1])); + ReconUtils.gatherSubPaths(parentId, subPaths, Long.parseLong(names[0]), + Long.parseLong(names[1]), reconNamespaceSummaryManager); // Iterate over the subpaths and retrieve the files for (String subPath : subPaths) { paramInfo.setStartPrefix(subPath); @@ -1082,49 +1082,6 @@ public Map searchKeysInFSO(ParamInfo paramInfo) return matchedKeys; } - /** - * Finds all subdirectories under a parent directory in an FSO bucket. It builds - * a list of paths for these subdirectories. These sub-directories are then used - * to search for files in the fileTable. - *

- * How it works: - * - Starts from a parent directory identified by parentId. - * - Looks through all child directories of this parent. - * - For each child, it creates a path that starts with volumeID/bucketID/parentId, - * following our fileTable format - * - Adds these paths to a list and explores each child further for more subdirectories. - * - * @param parentId The ID of the directory we start exploring from. - * @param subPaths A list where we collect paths to all subdirectories. - * @param volumeID - * @param bucketID - * @throws IOException If there are problems accessing directory information. - */ - private void gatherSubPaths(long parentId, List subPaths, - long volumeID, long bucketID) throws IOException { - // Fetch the NSSummary object for parentId - NSSummary parentSummary = - reconNamespaceSummaryManager.getNSSummary(parentId); - if (parentSummary == null) { - return; - } - - Set childDirIds = parentSummary.getChildDir(); - for (Long childId : childDirIds) { - // Fetch the NSSummary for each child directory - NSSummary childSummary = - reconNamespaceSummaryManager.getNSSummary(childId); - if (childSummary != null) { - String subPath = - ReconUtils.constructObjectPathWithPrefix(volumeID, bucketID, childId); - // Add to subPaths - subPaths.add(subPath); - // Recurse into this child directory - gatherSubPaths(childId, subPaths, volumeID, bucketID); - } - } - } - /** * Converts a startPrefix path into an objectId path for FSO buckets, using IDs. diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java new file mode 100644 index 000000000000..9cd6fa33d032 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java @@ -0,0 +1,389 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.recon.ReconUtils; +import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; +import org.apache.hadoop.ozone.recon.api.types.KeyEntityInfo; +import org.apache.hadoop.ozone.recon.api.types.KeyInsightInfoResponse; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.impl.ReconNamespaceSummaryManagerImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.QueryParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.ArrayList; + +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_START_PREFIX; +import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OPEN_KEY_DEFAULT_SEARCH_LIMIT; +import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OPEN_KEY_SEARCH_DEFAULT_PREV_KEY; +import static org.apache.hadoop.ozone.recon.ReconResponseUtils.noMatchedKeysResponse; +import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createBadRequestResponse; +import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createInternalServerErrorResponse; +import static org.apache.hadoop.ozone.recon.ReconUtils.constructObjectPathWithPrefix; +import static org.apache.hadoop.ozone.recon.ReconUtils.validateNames; +import static org.apache.hadoop.ozone.recon.api.handlers.BucketHandler.getBucketHandler; +import static org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.normalizePath; +import static org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.parseRequestPath; + +/** + * REST endpoint for search implementation in OM DB Insight. + */ +@Path("/keys") +@Produces(MediaType.APPLICATION_JSON) +@AdminOnly +public class OMDBInsightSearchEndpoint { + + private OzoneStorageContainerManager reconSCM; + private final ReconOMMetadataManager omMetadataManager; + private static final Logger LOG = + LoggerFactory.getLogger(OMDBInsightSearchEndpoint.class); + private ReconNamespaceSummaryManagerImpl reconNamespaceSummaryManager; + + + @Inject + public OMDBInsightSearchEndpoint(OzoneStorageContainerManager reconSCM, + ReconOMMetadataManager omMetadataManager, + ReconNamespaceSummaryManagerImpl reconNamespaceSummaryManager) { + this.reconSCM = reconSCM; + this.omMetadataManager = omMetadataManager; + this.reconNamespaceSummaryManager = reconNamespaceSummaryManager; + } + + + /** + * Performs a search for open keys in the Ozone Manager (OM) database using a specified search prefix. + * This endpoint searches across both File System Optimized (FSO) and Object Store (non-FSO) layouts, + * compiling a list of keys that match the given prefix along with their data sizes. + *

+ * The search prefix must start from the bucket level ('/volumeName/bucketName/') or any specific directory + * or key level (e.g., '/volA/bucketA/dir1' for everything under 'dir1' inside 'bucketA' of 'volA'). + * The search operation matches the prefix against the start of keys' names within the OM DB. + *

+ * Example Usage: + * 1. A startPrefix of "/volA/bucketA/" retrieves every key under bucket 'bucketA' in volume 'volA'. + * 2. Specifying "/volA/bucketA/dir1" focuses the search within 'dir1' inside 'bucketA' of 'volA'. + * + * @param startPrefix The prefix for searching keys, starting from the bucket level or any specific path. + * @param limit Limits the number of returned keys. + * @param prevKey The key to start after for the next set of records. + * @return A KeyInsightInfoResponse, containing matching keys and their data sizes. + * @throws IOException On failure to access the OM database or process the operation. + * @throws IllegalArgumentException If the provided startPrefix or other arguments are invalid. + */ + @GET + @Path("/open/search") + public Response searchOpenKeys( + @DefaultValue(DEFAULT_START_PREFIX) @QueryParam("startPrefix") + String startPrefix, + @DefaultValue(RECON_OPEN_KEY_DEFAULT_SEARCH_LIMIT) @QueryParam("limit") + int limit, + @DefaultValue(RECON_OPEN_KEY_SEARCH_DEFAULT_PREV_KEY) @QueryParam("prevKey") String prevKey) throws IOException { + + try { + // Ensure startPrefix is not null or empty and starts with '/' + if (startPrefix == null || startPrefix.length() == 0) { + return createBadRequestResponse( + "Invalid startPrefix: Path must be at the bucket level or deeper."); + } + startPrefix = startPrefix.startsWith("/") ? startPrefix : "/" + startPrefix; + + // Split the path to ensure it's at least at the bucket level + String[] pathComponents = startPrefix.split("/"); + if (pathComponents.length < 3 || pathComponents[2].isEmpty()) { + return createBadRequestResponse( + "Invalid startPrefix: Path must be at the bucket level or deeper."); + } + + // Ensure the limit is non-negative + limit = Math.max(0, limit); + + // Initialize response object + KeyInsightInfoResponse insightResponse = new KeyInsightInfoResponse(); + long replicatedTotal = 0; + long unreplicatedTotal = 0; + boolean keysFound = false; // Flag to track if any keys are found + String lastKey = null; + + // Search for non-fso keys in KeyTable + Table openKeyTable = + omMetadataManager.getOpenKeyTable(BucketLayout.LEGACY); + Map obsKeys = + retrieveKeysFromTable(openKeyTable, startPrefix, limit, prevKey); + for (Map.Entry entry : obsKeys.entrySet()) { + keysFound = true; + KeyEntityInfo keyEntityInfo = + createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue()); + insightResponse.getNonFSOKeyInfoList() + .add(keyEntityInfo); // Add to non-FSO list + replicatedTotal += entry.getValue().getReplicatedSize(); + unreplicatedTotal += entry.getValue().getDataSize(); + lastKey = entry.getKey(); // Update lastKey + } + + // Search for fso keys in FileTable + Map fsoKeys = searchOpenKeysInFSO(startPrefix, limit, prevKey); + for (Map.Entry entry : fsoKeys.entrySet()) { + keysFound = true; + KeyEntityInfo keyEntityInfo = + createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue()); + insightResponse.getFsoKeyInfoList() + .add(keyEntityInfo); // Add to FSO list + replicatedTotal += entry.getValue().getReplicatedSize(); + unreplicatedTotal += entry.getValue().getDataSize(); + lastKey = entry.getKey(); // Update lastKey + } + + // If no keys were found, return a response indicating that no keys matched + if (!keysFound) { + return noMatchedKeysResponse(startPrefix); + } + + // Set the aggregated totals in the response + insightResponse.setReplicatedDataSize(replicatedTotal); + insightResponse.setUnreplicatedDataSize(unreplicatedTotal); + insightResponse.setLastKey(lastKey); + + // Return the response with the matched keys and their data sizes + return Response.ok(insightResponse).build(); + } catch (IOException e) { + // Handle IO exceptions and return an internal server error response + return createInternalServerErrorResponse( + "Error searching open keys in OM DB: " + e.getMessage()); + } catch (IllegalArgumentException e) { + // Handle illegal argument exceptions and return a bad request response + return createBadRequestResponse( + "Invalid startPrefix: " + e.getMessage()); + } + } + + public Map searchOpenKeysInFSO(String startPrefix, + int limit, String prevKey) + throws IOException, IllegalArgumentException { + Map matchedKeys = new LinkedHashMap<>(); + // Convert the search prefix to an object path for FSO buckets + String startPrefixObjectPath = convertToObjectPath(startPrefix); + String[] names = parseRequestPath(startPrefixObjectPath); + Table openFileTable = + omMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED); + + // If names.length <= 2, then the search prefix is at the volume or bucket level hence + // no need to find parent or extract id's or find subpaths as the openFileTable is + // suitable for volume and bucket level search + if (names.length > 2 && startPrefixObjectPath.endsWith(OM_KEY_PREFIX)) { + // Fetch the parent ID to search for + long parentId = Long.parseLong(names[names.length - 1]); + + // Fetch the nameSpaceSummary for the parent ID + NSSummary parentSummary = reconNamespaceSummaryManager.getNSSummary(parentId); + if (parentSummary == null) { + return matchedKeys; + } + List subPaths = new ArrayList<>(); + // Add the initial search prefix object path because it can have both openFiles + // and subdirectories with openFiles + subPaths.add(startPrefixObjectPath); + + // Recursively gather all subpaths + ReconUtils.gatherSubPaths(parentId, subPaths, Long.parseLong(names[0]), Long.parseLong(names[1]), + reconNamespaceSummaryManager); + + // Iterate over the subpaths and retrieve the open files + for (String subPath : subPaths) { + matchedKeys.putAll(retrieveKeysFromTable(openFileTable, subPath, limit - matchedKeys.size(), prevKey)); + if (matchedKeys.size() >= limit) { + break; + } + } + return matchedKeys; + } + + // If the search level is at the volume, bucket or key level, directly search the openFileTable + matchedKeys.putAll(retrieveKeysFromTable(openFileTable, startPrefixObjectPath, limit, prevKey)); + return matchedKeys; + } + + /** + * Converts a key prefix into an object path for FSO buckets, using IDs. + * + * This method transforms a user-provided path (e.g., "volume/bucket/dir1") into + * a database-friendly format ("/volumeID/bucketID/ParentId/") by replacing names + * with their corresponding IDs. It simplifies database queries for FSO bucket operations. + * + * Examples: + * - Input: "volume/bucket/key" -> Output: "/volumeID/bucketID/parentDirID/key" + * - Input: "volume/bucket/dir1" -> Output: "/volumeID/bucketID/dir1ID/" + * - Input: "volume/bucket/dir1/key1" -> Output: "/volumeID/bucketID/dir1ID/key1" + * - Input: "volume/bucket/dir1/dir2" -> Output: "/volumeID/bucketID/dir2ID/" + * + * @param prevKeyPrefix The path to be converted. + * @return The object path as "/volumeID/bucketID/ParentId/" or an empty string if an error occurs. + * @throws IOException If database access fails. + * @throws IllegalArgumentException If the provided path is invalid or cannot be converted. + */ + public String convertToObjectPath(String prevKeyPrefix) throws IOException { + try { + String[] names = parseRequestPath(normalizePath(prevKeyPrefix, BucketLayout.FILE_SYSTEM_OPTIMIZED)); + Table openFileTable = omMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED); + + // Root-Level: Return the original path + if (names.length == 0) { + return prevKeyPrefix; + } + + // Volume-Level: Fetch the volumeID + String volumeName = names[0]; + validateNames(volumeName); + String volumeKey = omMetadataManager.getVolumeKey(volumeName); + long volumeId = omMetadataManager.getVolumeTable().getSkipCache(volumeKey).getObjectID(); + if (names.length == 1) { + return constructObjectPathWithPrefix(volumeId); + } + + // Bucket-Level: Fetch the bucketID + String bucketName = names[1]; + validateNames(bucketName); + String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); + OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().getSkipCache(bucketKey); + long bucketId = bucketInfo.getObjectID(); + if (names.length == 2 || bucketInfo.getBucketLayout() != BucketLayout.FILE_SYSTEM_OPTIMIZED) { + return constructObjectPathWithPrefix(volumeId, bucketId); + } + + // Directory or Key-Level: Check both key and directory + BucketHandler handler = + getBucketHandler(reconNamespaceSummaryManager, omMetadataManager, reconSCM, bucketInfo); + + if (names.length >= 3) { + String lastEntiry = names[names.length - 1]; + + // Check if the directory exists + OmDirectoryInfo dirInfo = handler.getDirInfo(names); + if (dirInfo != null && dirInfo.getName().equals(lastEntiry)) { + return constructObjectPathWithPrefix(volumeId, bucketId, dirInfo.getObjectID()) + OM_KEY_PREFIX; + } + + // Check if the key exists + long dirID = handler.getDirObjectId(names, names.length); + String keyKey = constructObjectPathWithPrefix(volumeId, bucketId, dirID) + + OM_KEY_PREFIX + lastEntiry; + OmKeyInfo keyInfo = openFileTable.getSkipCache(keyKey); + if (keyInfo != null && keyInfo.getFileName().equals(lastEntiry)) { + return constructObjectPathWithPrefix(volumeId, bucketId, + keyInfo.getParentObjectID()) + OM_KEY_PREFIX + lastEntiry; + } + + return prevKeyPrefix; + } + } catch (IllegalArgumentException e) { + LOG.error( + "IllegalArgumentException encountered while converting key prefix to object path: {}", + prevKeyPrefix, e); + throw e; + } catch (RuntimeException e) { + LOG.error( + "RuntimeException encountered while converting key prefix to object path: {}", + prevKeyPrefix, e); + return prevKeyPrefix; + } + return prevKeyPrefix; + } + + + /** + * Common method to retrieve keys from a table based on a search prefix and a limit. + * + * @param table The table to retrieve keys from. + * @param startPrefix The search prefix to match keys against. + * @param limit The maximum number of keys to retrieve. + * @param prevKey The key to start after for the next set of records. + * @return A map of keys and their corresponding OmKeyInfo objects. + * @throws IOException If there are problems accessing the table. + */ + private Map retrieveKeysFromTable( + Table table, String startPrefix, int limit, String prevKey) + throws IOException { + Map matchedKeys = new LinkedHashMap<>(); + try (TableIterator> keyIter = table.iterator()) { + // If a previous key is provided, seek to the previous key and skip it. + if (!prevKey.isEmpty()) { + keyIter.seek(prevKey); + if (keyIter.hasNext()) { + // Skip the previous key + keyIter.next(); + } + } else { + // If no previous key is provided, start from the search prefix. + keyIter.seek(startPrefix); + } + while (keyIter.hasNext() && matchedKeys.size() < limit) { + Table.KeyValue entry = keyIter.next(); + String dbKey = entry.getKey(); + if (!dbKey.startsWith(startPrefix)) { + break; // Exit the loop if the key no longer matches the prefix + } + matchedKeys.put(dbKey, entry.getValue()); + } + } catch (IOException exception) { + LOG.error("Error retrieving keys from table for path: {}", startPrefix, exception); + throw exception; + } + return matchedKeys; + } + + /** + * Creates a KeyEntityInfo object from an OmKeyInfo object and the corresponding key. + * + * @param dbKey The key in the database corresponding to the OmKeyInfo object. + * @param keyInfo The OmKeyInfo object to create the KeyEntityInfo from. + * @return The KeyEntityInfo object created from the OmKeyInfo object and the key. + */ + private KeyEntityInfo createKeyEntityInfoFromOmKeyInfo(String dbKey, + OmKeyInfo keyInfo) { + KeyEntityInfo keyEntityInfo = new KeyEntityInfo(); + keyEntityInfo.setKey(dbKey); // Set the DB key + keyEntityInfo.setPath(keyInfo.getKeyName()); // Assuming path is the same as key name + keyEntityInfo.setInStateSince(keyInfo.getCreationTime()); + keyEntityInfo.setSize(keyInfo.getDataSize()); + keyEntityInfo.setReplicatedSize(keyInfo.getReplicatedSize()); + keyEntityInfo.setReplicationConfig(keyInfo.getReplicationConfig()); + return keyEntityInfo; + } + +} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOMDBInsightSearchEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOMDBInsightSearchEndpoint.java new file mode 100644 index 000000000000..ab16f349af27 --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOMDBInsightSearchEndpoint.java @@ -0,0 +1,796 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; + +import org.apache.hadoop.ozone.recon.api.types.KeyInsightInfoResponse; +import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest; +import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithFSO; +import org.junit.jupiter.api.BeforeEach; + +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeOpenFileToOm; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeOpenKeyToOm; +import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.mock; + +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.recon.ReconTestInjector; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import javax.ws.rs.core.Response; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.UUID; + +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; + +/** + * Test class for OMDBInsightSearchEndpoint. + * + * This class tests various scenarios for searching open keys within a + * given volume, bucket, and directory structure. The tests include: + * + * 1. Test Root Level Search Restriction: Ensures searching at the root level returns a bad request. + * 2. Test Volume Level Search Restriction: Ensures searching at the volume level returns a bad request. + * 3. Test Bucket Level Search: Verifies search results within different types of buckets (FSO, OBS, Legacy). + * 4. Test Directory Level Search: Validates searching inside specific directories. + * 5. Test Key Level Search: Confirms search results for specific keys within buckets. + * 6. Test Key Level Search Under Directory: Verifies searching for keys within nested directories. + * 7. Test Search Under Nested Directory: Checks search results within nested directories under dira3. + * 8. Test Limit Search: Tests the limit functionality of the search API. + * 9. Test Search Open Keys with Bad Request: Ensures bad requests with invalid parameters return appropriate responses. + * 10. Test Last Key in Response: Confirms the presence of the last key in paginated responses. + * 11. Test Search Open Keys with Pagination: Verifies paginated search results. + * 12. Test Search in Empty Bucket: Checks the response for searching within an empty bucket. + */ +public class TestOMDBInsightSearchEndpoint extends AbstractReconSqlDBTest { + + @TempDir + private Path temporaryFolder; + private ReconOMMetadataManager reconOMMetadataManager; + private OMDBInsightSearchEndpoint omdbInsightSearchEndpoint; + private OzoneConfiguration ozoneConfiguration; + private static final String ROOT_PATH = "/"; + private static final String TEST_USER = "TestUser"; + private OMMetadataManager omMetadataManager; + + private ReconNamespaceSummaryManager reconNamespaceSummaryManager; + + @BeforeEach + public void setUp() throws Exception { + ozoneConfiguration = new OzoneConfiguration(); + ozoneConfiguration.setLong(OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD, + 100); + omMetadataManager = initializeNewOmMetadataManager( + Files.createDirectory(temporaryFolder.resolve("JunitOmDBDir")) + .toFile()); + reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, + Files.createDirectory(temporaryFolder.resolve("OmMetataDir")).toFile()); + + ReconTestInjector reconTestInjector = + new ReconTestInjector.Builder(temporaryFolder.toFile()) + .withReconSqlDb() + .withReconOm(reconOMMetadataManager) + .withOmServiceProvider(mock(OzoneManagerServiceProviderImpl.class)) + .addBinding(OzoneStorageContainerManager.class, + ReconStorageContainerManagerFacade.class) + .withContainerDB() + .addBinding(StorageContainerServiceProvider.class, + mock(StorageContainerServiceProviderImpl.class)) + .addBinding(OMDBInsightEndpoint.class) + .addBinding(ContainerHealthSchemaManager.class) + .build(); + reconNamespaceSummaryManager = + reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); + omdbInsightSearchEndpoint = reconTestInjector.getInstance( + OMDBInsightSearchEndpoint.class); + + // populate OM DB and reprocess into Recon RocksDB + populateOMDB(); + NSSummaryTaskWithFSO nSSummaryTaskWithFso = + new NSSummaryTaskWithFSO(reconNamespaceSummaryManager, + reconOMMetadataManager, ozoneConfiguration); + nSSummaryTaskWithFso.reprocessWithFSO(reconOMMetadataManager); + } + + /** + * Create a new OM Metadata manager instance with one user, one vol, and two + * buckets. + * + * @throws IOException ioEx + */ + private static OMMetadataManager initializeNewOmMetadataManager( + File omDbDir) + throws IOException { + OzoneConfiguration omConfiguration = new OzoneConfiguration(); + omConfiguration.set(OZONE_OM_DB_DIRS, + omDbDir.getAbsolutePath()); + OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( + omConfiguration, null); + return omMetadataManager; + } + + @Test + public void testRootLevelSearchRestriction() throws IOException { + // Test with root level path + String rootPath = "/"; + Response response = omdbInsightSearchEndpoint.searchOpenKeys(rootPath, 20, ""); + assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); + String entity = (String) response.getEntity(); + assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), + "Expected a message indicating the path must be at the bucket level or deeper"); + + // Test with root level path without trailing slash + rootPath = ""; + response = omdbInsightSearchEndpoint.searchOpenKeys(rootPath, 20, ""); + assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); + entity = (String) response.getEntity(); + assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), + "Expected a message indicating the path must be at the bucket level or deeper"); + } + + @Test + public void testVolumeLevelSearchRestriction() throws IOException { + // Test with volume level path + String volumePath = "/vola"; + Response response = omdbInsightSearchEndpoint.searchOpenKeys(volumePath, 20, ""); + assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); + String entity = (String) response.getEntity(); + assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), + "Expected a message indicating the path must be at the bucket level or deeper"); + + // Test with another volume level path + volumePath = "/volb"; + response = omdbInsightSearchEndpoint.searchOpenKeys(volumePath, 20, ""); + assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); + entity = (String) response.getEntity(); + assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), + "Expected a message indicating the path must be at the bucket level or deeper"); + } + + @Test + public void testBucketLevelSearch() throws IOException { + // Search inside FSO bucket + Response response = + omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1", 20, ""); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = + (KeyInsightInfoResponse) response.getEntity(); + assertEquals(14, result.getFsoKeyInfoList().size()); + assertEquals(0, result.getNonFSOKeyInfoList().size()); + // Assert Total Size + assertEquals(14000, result.getUnreplicatedDataSize()); + assertEquals(14000 * 3, result.getReplicatedDataSize()); + + // Search inside OBS bucket + response = + omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb1", 20, ""); + assertEquals(200, response.getStatus()); + result = + (KeyInsightInfoResponse) response.getEntity(); + assertEquals(5, result.getNonFSOKeyInfoList().size()); + assertEquals(0, result.getFsoKeyInfoList().size()); + // Assert Total Size + assertEquals(5000, result.getUnreplicatedDataSize()); + assertEquals(5000 * 3, result.getReplicatedDataSize()); + + // Search Inside LEGACY bucket + response = + omdbInsightSearchEndpoint.searchOpenKeys("/volc/bucketc1", 20, ""); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(7, result.getNonFSOKeyInfoList().size()); + + // Test with bucket that does not exist + response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/nonexistentbucket", 20, ""); + assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + String entity = (String) response.getEntity(); + assertTrue(entity.contains("No keys matched the search prefix"), + "Expected a message indicating no keys were found"); + } + + @Test + public void testDirectoryLevelSearch() throws IOException { + Response response = + omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira1", 20, ""); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = + (KeyInsightInfoResponse) response.getEntity(); + assertEquals(1, result.getFsoKeyInfoList().size()); + assertEquals(0, result.getNonFSOKeyInfoList().size()); + // Assert Total Size + assertEquals(1000, result.getUnreplicatedDataSize()); + assertEquals(1000 * 3, result.getReplicatedDataSize()); + + response = + omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira2", 20, ""); + assertEquals(200, response.getStatus()); + result = + (KeyInsightInfoResponse) response.getEntity(); + assertEquals(1, result.getFsoKeyInfoList().size()); + assertEquals(0, result.getNonFSOKeyInfoList().size()); + // Assert Total Size + assertEquals(1000, result.getUnreplicatedDataSize()); + assertEquals(1000 * 3, result.getReplicatedDataSize()); + + response = + omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira3", 20, ""); + assertEquals(200, response.getStatus()); + result = + (KeyInsightInfoResponse) response.getEntity(); + assertEquals(10, result.getFsoKeyInfoList().size()); + assertEquals(0, result.getNonFSOKeyInfoList().size()); + // Assert Total Size + assertEquals(10000, result.getUnreplicatedDataSize()); + assertEquals(10000 * 3, result.getReplicatedDataSize()); + + // Test with non-existent directory + response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/nonexistentdir", 20, ""); + assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + String entity = (String) response.getEntity(); + assertTrue(entity.contains("No keys matched the search prefix"), + "Expected a message indicating no keys were found"); + } + + @Test + public void testKeyLevelSearch() throws IOException { + // FSO Bucket key-level search + Response response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/filea1", 10, ""); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(1, result.getFsoKeyInfoList().size()); + assertEquals(0, result.getNonFSOKeyInfoList().size()); + // Assert Total Size + assertEquals(1000, result.getUnreplicatedDataSize()); + assertEquals(1000 * 3, result.getReplicatedDataSize()); + + response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/filea2", 10, ""); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(1, result.getFsoKeyInfoList().size()); + assertEquals(0, result.getNonFSOKeyInfoList().size()); + // Assert Total Size + assertEquals(1000, result.getUnreplicatedDataSize()); + assertEquals(1000 * 3, result.getReplicatedDataSize()); + + // OBS Bucket key-level search + response = omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb1/fileb1", 10, ""); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(0, result.getFsoKeyInfoList().size()); + assertEquals(1, result.getNonFSOKeyInfoList().size()); + // Assert Total Size + assertEquals(1000, result.getUnreplicatedDataSize()); + assertEquals(1000 * 3, result.getReplicatedDataSize()); + + response = omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb1/fileb2", 10, ""); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(0, result.getFsoKeyInfoList().size()); + assertEquals(1, result.getNonFSOKeyInfoList().size()); + // Assert Total Size + assertEquals(1000, result.getUnreplicatedDataSize()); + assertEquals(1000 * 3, result.getReplicatedDataSize()); + + // Test with non-existent key + response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/nonexistentfile", 1, ""); + assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + String entity = (String) response.getEntity(); + assertTrue(entity.contains("No keys matched the search prefix"), + "Expected a message indicating no keys were found"); + + response = omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb1/nonexistentfile", 1, ""); + assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + entity = (String) response.getEntity(); + assertTrue(entity.contains("No keys matched the search prefix"), + "Expected a message indicating no keys were found"); + } + + // Test searching for keys under a directory + @Test + public void testKeyLevelSearchUnderDirectory() throws IOException { + // FSO Bucket key-level search + Response response = + omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira1/innerfile", 10, ""); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(1, result.getFsoKeyInfoList().size()); + assertEquals(0, result.getNonFSOKeyInfoList().size()); + + response = + omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira2/innerfile", 10, ""); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(1, result.getFsoKeyInfoList().size()); + assertEquals(0, result.getNonFSOKeyInfoList().size()); + + // Test for unknown file in fso bucket + response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira1/unknownfile", 10, ""); + assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + String entity = (String) response.getEntity(); + assertTrue(entity.contains("No keys matched the search prefix"), + "Expected a message indicating no keys were found"); + + // Test for unknown file in fso bucket + response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira2/unknownfile", 10, ""); + assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + entity = (String) response.getEntity(); + assertTrue(entity.contains("No keys matched the search prefix"), + "Expected a message indicating no keys were found"); + } + + + @Test + public void testSearchUnderNestedDirectory() throws IOException { + Response response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira3", 20, + ""); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(10, result.getFsoKeyInfoList().size()); + assertEquals(0, result.getNonFSOKeyInfoList().size()); + + // Search under dira31 + response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira3/dira31", + 20, ""); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(6, result.getFsoKeyInfoList().size()); + assertEquals(0, result.getNonFSOKeyInfoList().size()); + + // Search under dira32 + response = omdbInsightSearchEndpoint.searchOpenKeys( + "/vola/bucketa1/dira3/dira31/dira32", 20, ""); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(3, result.getFsoKeyInfoList().size()); + assertEquals(0, result.getNonFSOKeyInfoList().size()); + + // Search under dira33 + response = omdbInsightSearchEndpoint.searchOpenKeys( + "/vola/bucketa1/dira3/dira31/dira32/dira33", 20, ""); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(1, result.getFsoKeyInfoList().size()); + assertEquals(0, result.getNonFSOKeyInfoList().size()); + + // Search for the exact file under dira33 + response = omdbInsightSearchEndpoint.searchOpenKeys( + "/vola/bucketa1/dira3/dira31/dira32/dira33/file33_1", 20, ""); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(1, result.getFsoKeyInfoList().size()); + assertEquals(0, result.getNonFSOKeyInfoList().size()); + + // Search for a non existant file under each nested directory + response = omdbInsightSearchEndpoint.searchOpenKeys( + "/vola/bucketa1/dira3/dira31/dira32/dira33/nonexistentfile", 20, ""); + assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + String entity = (String) response.getEntity(); + assertTrue(entity.contains("No keys matched the search prefix"), + "Expected a message indicating no keys were found"); + + response = omdbInsightSearchEndpoint.searchOpenKeys( + "/vola/bucketa1/dira3/dira31/dira32/nonexistentfile", 20, ""); + assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); + entity = (String) response.getEntity(); + assertTrue(entity.contains("No keys matched the search prefix"), + "Expected a message indicating no keys were found"); + } + + @Test + public void testLimitSearch() throws IOException { + Response response = + omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1", 2, ""); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = + (KeyInsightInfoResponse) response.getEntity(); + assertEquals(2, result.getFsoKeyInfoList().size()); + assertEquals(0, result.getNonFSOKeyInfoList().size()); + } + + @Test + public void testSearchOpenKeysWithBadRequest() throws IOException { + // Give a negative limit + int negativeLimit = -1; + Response response = omdbInsightSearchEndpoint.searchOpenKeys("@323232", negativeLimit, ""); + + // Then the response should indicate that the request was bad + assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), + response.getStatus(), "Expected a 400 BAD REQUEST status"); + + String entity = (String) response.getEntity(); + assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), + "Expected a message indicating the path must be at the bucket level or deeper"); + + response = omdbInsightSearchEndpoint.searchOpenKeys("///", 20, ""); + assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); + entity = (String) response.getEntity(); + assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), + "Expected a message indicating the path must be at the bucket level or deeper"); + } + + @Test + public void testLastKeyInResponse() throws IOException { + Response response = + omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb1", 20, ""); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = + (KeyInsightInfoResponse) response.getEntity(); + assertEquals(0, result.getFsoKeyInfoList().size()); + assertEquals(5, result.getNonFSOKeyInfoList().size()); + // Assert Total Size + assertEquals(5000, result.getUnreplicatedDataSize()); + assertEquals(5000 * 3, result.getReplicatedDataSize()); + // Assert Last Key + assertEquals(ROOT_PATH + "volb/bucketb1/fileb5", result.getLastKey(), + "Expected last key to be 'fileb5'"); + } + + @Test + public void testSearchOpenKeysWithPagination() throws IOException { + // Set the initial parameters + String startPrefix = "/volb/bucketb1"; + int limit = 2; + String prevKey = ""; + + // Perform the first search request + Response response = omdbInsightSearchEndpoint.searchOpenKeys(startPrefix, limit, prevKey); + assertEquals(200, response.getStatus()); + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(2, result.getNonFSOKeyInfoList().size()); + assertEquals(0, result.getFsoKeyInfoList().size()); + + // Extract the last key from the response + prevKey = result.getLastKey(); + assertNotNull(prevKey, "Last key should not be null"); + + // Perform the second search request using the last key + response = omdbInsightSearchEndpoint.searchOpenKeys(startPrefix, limit, prevKey); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(2, result.getNonFSOKeyInfoList().size()); + assertEquals(0, result.getFsoKeyInfoList().size()); + + // Extract the last key from the response + prevKey = result.getLastKey(); + assertNotNull(prevKey, "Last key should not be null"); + + // Perform the third search request using the last key + response = omdbInsightSearchEndpoint.searchOpenKeys(startPrefix, limit, prevKey); + assertEquals(200, response.getStatus()); + result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(1, result.getNonFSOKeyInfoList().size()); + assertEquals(0, result.getFsoKeyInfoList().size()); + assertEquals(result.getNonFSOKeyInfoList().get(0).getKey(), result.getLastKey(), + "Expected last key to be empty"); + } + + @Test + public void testSearchInEmptyBucket() throws IOException { + // Search in empty bucket bucketb2 + Response response = omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb2", 20, ""); + assertEquals(404, response.getStatus()); + String entity = (String) response.getEntity(); + assertTrue(entity.contains("No keys matched the search prefix"), + "Expected a message indicating no keys were found"); + } + + /** + * Tests the NSSummaryEndpoint for a given volume, bucket, and directory structure. + * The test setup mimics the following filesystem structure with specified sizes: + * + * root (Total Size: 15000KB) + * ├── vola (Total Size: 10000KB) + * │ ├── bucketa1 (FSO) Total Size: 5000KB + * │ │ ├── filea1 (Size: 1000KB) + * │ │ ├── filea2 (Size: 1000KB) + * │ │ ├── dira1 (Total Size: 1000KB) + * │ │ ├── dira2 (Total Size: 1000KB) + * │ │ └── dira3 (Total Size: 1000KB) + * │ │ ├── dira31 (Total Size: 1000KB) + * │ │ ├── dira32 (Total Size: 1000KB) + * │ │ └── dira33 (Total Size: 1000KB) + * │ ├── bucketa2 (FSO) Total Size: 5000KB + * │ │ ├── filea3 (Size: 1000KB) + * │ │ ├── filea4 (Size: 1000KB) + * │ │ ├── dira4 (Total Size: 1000KB) + * │ │ ├── dira5 (Total Size: 1000KB) + * │ │ └── dira6 (Total Size: 1000KB) + * └── volb (Total Size: 5000KB) + * ├── bucketb1 (OBS) Total Size: 5000KB + * │ ├── fileb1 (Size: 1000KB) + * │ ├── fileb2 (Size: 1000KB) + * │ ├── fileb3 (Size: 1000KB) + * │ ├── fileb4 (Size: 1000KB) + * │ └── fileb5 (Size: 1000KB) + * └── bucketb2 (OBS) Total Size: 0KB (Empty Bucket) + * └── volc (Total Size: 7000KB) + * └── bucketc1 (LEGACY) Total Size: 7000KB + * ├── filec1 (Size: 1000KB) + * ├── filec2 (Size: 1000KB) + * ├── filec3 (Size: 1000KB) + * ├── dirc1/ (Total Size: 2000KB) + * └── dirc2/ (Total Size: 2000KB) + * + * @throws Exception + */ + private void populateOMDB() throws Exception { + // Create Volumes + long volaObjectId = createVolume("vola"); + createVolume("volb"); + createVolume("volc"); + + // Create Buckets in vola + long bucketa1ObjectId = + createBucket("vola", "bucketa1", 1000 + 1000 + 1000 + 1000 + 1000, + getFSOBucketLayout()); + long bucketa2ObjectId = + createBucket("vola", "bucketa2", 1000 + 1000 + 1000 + 1000 + 1000, + getFSOBucketLayout()); + + // Create Bucket in volb + createBucket("volb", "bucketb1", 1000 + 1000 + 1000 + 1000 + 1000, + getOBSBucketLayout()); + createBucket("volb", "bucketb2", 0, getOBSBucketLayout()); // Empty Bucket + + // Create Bucket in volc + createBucket("volc", "bucketc1", 7000, + getLegacyBucketLayout()); + + // Create Directories and Files under bucketa1 + long dira1ObjectId = + createDirectory(bucketa1ObjectId, bucketa1ObjectId, volaObjectId, + "dira1"); + long dira2ObjectId = + createDirectory(bucketa1ObjectId, bucketa1ObjectId, volaObjectId, + "dira2"); + long dira3ObjectId = + createDirectory(bucketa1ObjectId, bucketa1ObjectId, volaObjectId, + "dira3"); + + // Create nested directories under dira3 + long dira31ObjectId = + createDirectory(dira3ObjectId, bucketa1ObjectId, volaObjectId, + "dira31"); + long dira32ObjectId = + createDirectory(dira31ObjectId, bucketa1ObjectId, volaObjectId, + "dira32"); + long dira33ObjectId = + createDirectory(dira32ObjectId, bucketa1ObjectId, volaObjectId, + "dira33"); + + // Files directly under bucketa1 + createOpenFile("filea1", "bucketa1", "vola", "filea1", bucketa1ObjectId, + bucketa1ObjectId, volaObjectId, 1000); + createOpenFile("filea2", "bucketa1", "vola", "filea2", bucketa1ObjectId, + bucketa1ObjectId, volaObjectId, 1000); + + // Files under dira3 + createOpenFile("dira3/file3_1", "bucketa1", "vola", "file3_1", + dira3ObjectId, bucketa1ObjectId, volaObjectId, 1000); + createOpenFile("dira3/file3_2", "bucketa1", "vola", "file3_2", + dira3ObjectId, bucketa1ObjectId, volaObjectId, 1000); + createOpenFile("dira3/file3_3", "bucketa1", "vola", "file3_3", + dira3ObjectId, bucketa1ObjectId, volaObjectId, 1000); + createOpenFile("dira3/file3_4", "bucketa1", "vola", "file3_4", + dira3ObjectId, bucketa1ObjectId, volaObjectId, 1000); + + // Files under dira31 + createOpenFile("dira3/dira31/file31_1", "bucketa1", "vola", "file31_1", + dira31ObjectId, bucketa1ObjectId, volaObjectId, 1000); + createOpenFile("dira3/dira31/file31_2", "bucketa1", "vola", "file31_2", + dira31ObjectId, bucketa1ObjectId, volaObjectId, 1000); + createOpenFile("dira3/dira31/file31_3", "bucketa1", "vola", "file31_3", + dira31ObjectId, bucketa1ObjectId, volaObjectId, 1000); + + // Files under dira32 + createOpenFile("dira3/dira31/dira32/file32_1", "bucketa1", "vola", "file32_1", + dira32ObjectId, bucketa1ObjectId, volaObjectId, 1000); + createOpenFile("dira3/dira31/dira32/file32_2", "bucketa1", "vola", "file32_2", + dira32ObjectId, bucketa1ObjectId, volaObjectId, 1000); + + // Files under dira33 + createOpenFile("dira3/dira31/dira32/dira33/file33_1", "bucketa1", "vola", "file33_1", + dira33ObjectId, bucketa1ObjectId, volaObjectId, 1000); + + // Create Directories and Files under bucketa2 + long dira4ObjectId = + createDirectory(bucketa2ObjectId, bucketa2ObjectId, volaObjectId, + "dira4"); + long dira5ObjectId = + createDirectory(bucketa2ObjectId, bucketa2ObjectId, volaObjectId, + "dira5"); + long dira6ObjectId = + createDirectory(bucketa2ObjectId, bucketa2ObjectId, volaObjectId, + "dira6"); + + // Files directly under bucketa2 + createOpenFile("filea3", "bucketa2", "vola", "filea3", bucketa2ObjectId, + bucketa2ObjectId, volaObjectId, 1000); + createOpenFile("filea4", "bucketa2", "vola", "filea4", bucketa2ObjectId, + bucketa2ObjectId, volaObjectId, 1000); + + // Files directly under bucketb1 + createOpenKey("fileb1", "bucketb1", "volb", 1000); + createOpenKey("fileb2", "bucketb1", "volb", 1000); + createOpenKey("fileb3", "bucketb1", "volb", 1000); + createOpenKey("fileb4", "bucketb1", "volb", 1000); + createOpenKey("fileb5", "bucketb1", "volb", 1000); + + // Create Inner files under directories + createOpenFile("dira1/innerfile", "bucketa1", "vola", "innerfile", + dira1ObjectId, bucketa1ObjectId, volaObjectId, 1000); + createOpenFile("dira2/innerfile", "bucketa1", "vola", "innerfile", + dira2ObjectId, bucketa1ObjectId, volaObjectId, 1000); + createOpenFile("dira4/innerfile", "bucketa2", "vola", "innerfile", + dira4ObjectId, bucketa2ObjectId, volaObjectId, 1000); + createOpenFile("dira5/innerfile", "bucketa2", "vola", "innerfile", + dira5ObjectId, bucketa2ObjectId, volaObjectId, 1000); + createOpenFile("dira6/innerfile", "bucketa2", "vola", "innerfile", + dira6ObjectId, bucketa2ObjectId, volaObjectId, 1000); + + // Create Keys and Directories in bucketc1 (LEGACY layout) + createOpenKey("filec1", "bucketc1", "volc", 1000); + createOpenKey("filec2", "bucketc1", "volc", 1000); + createOpenKey("filec3", "bucketc1", "volc", 1000); + createOpenKey("dirc1/", "bucketc1", "volc", 2000); // Directory indicated by trailing slash + createOpenKey("dirc2/", "bucketc1", "volc", 2000); // Directory indicated by trailing slash + createOpenKey("dirc1/innerfile", "bucketc1", "volc", 2000); // File in directory + createOpenKey("dirc2/innerfile", "bucketc1", "volc", 2000); // File in directory + } + + /** + * Create a volume and add it to the Volume Table. + * + * @return volume Object ID + * @throws IOException + */ + private long createVolume(String volumeName) throws Exception { + String volumeKey = reconOMMetadataManager.getVolumeKey(volumeName); + long volumeId = UUID.randomUUID().getMostSignificantBits() & + Long.MAX_VALUE; // Generate positive ID + OmVolumeArgs args = OmVolumeArgs.newBuilder() + .setObjectID(volumeId) + .setVolume(volumeName) + .setAdminName(TEST_USER) + .setOwnerName(TEST_USER) + .build(); + + reconOMMetadataManager.getVolumeTable().put(volumeKey, args); + return volumeId; + } + + /** + * Create a bucket and add it to the Bucket Table. + * + * @return bucket Object ID + * @throws IOException + */ + private long createBucket(String volumeName, String bucketName, long dataSize, + BucketLayout bucketLayout) + throws Exception { + String bucketKey = + reconOMMetadataManager.getBucketKey(volumeName, bucketName); + long bucketId = UUID.randomUUID().getMostSignificantBits() & + Long.MAX_VALUE; // Generate positive ID + OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setObjectID(bucketId) + .setBucketLayout(bucketLayout) + .setUsedBytes(dataSize) + .build(); + + reconOMMetadataManager.getBucketTable().put(bucketKey, bucketInfo); + return bucketId; + } + + /** + * Create a directory and add it to the Directory Table. + * + * @return directory Object ID + * @throws IOException + */ + private long createDirectory(long parentObjectId, + long bucketObjectId, + long volumeObjectId, + String dirName) throws IOException { + long objectId = UUID.randomUUID().getMostSignificantBits() & + Long.MAX_VALUE; // Ensure positive ID + writeDirToOm(reconOMMetadataManager, objectId, parentObjectId, + bucketObjectId, + volumeObjectId, dirName); + return objectId; + } + + /** + * Create a file and add it to the Open File Table. + * + * @return file Object ID + * @throws IOException + */ + @SuppressWarnings("checkstyle:ParameterNumber") + private long createOpenFile(String key, + String bucket, + String volume, + String fileName, + long parentObjectId, + long bucketObjectId, + long volumeObjectId, + long dataSize) throws IOException { + long objectId = UUID.randomUUID().getMostSignificantBits() & + Long.MAX_VALUE; // Ensure positive ID + writeOpenFileToOm(reconOMMetadataManager, key, bucket, volume, fileName, + objectId, parentObjectId, bucketObjectId, volumeObjectId, null, + dataSize); + return objectId; + } + + /** + * Create a key and add it to the Open Key Table. + * + * @return key Object ID + * @throws IOException + */ + private long createOpenKey(String key, + String bucket, + String volume, + long dataSize) throws IOException { + long objectId = UUID.randomUUID().getMostSignificantBits() & + Long.MAX_VALUE; // Ensure positive ID + writeOpenKeyToOm(reconOMMetadataManager, key, bucket, volume, null, + dataSize); + return objectId; + } + + private static BucketLayout getFSOBucketLayout() { + return BucketLayout.FILE_SYSTEM_OPTIMIZED; + } + + private static BucketLayout getOBSBucketLayout() { + return BucketLayout.OBJECT_STORE; + } + + private static BucketLayout getLegacyBucketLayout() { + return BucketLayout.LEGACY; + } + +}