diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index 407694bf859b..12139e177231 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.ScmUtils; import org.apache.hadoop.hdds.scm.ha.SCMNodeDetails; +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher; import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.hdds.utils.db.Table; @@ -78,8 +79,12 @@ import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; import org.apache.hadoop.ozone.recon.api.ServiceNotReadyException; import org.apache.hadoop.ozone.recon.api.types.NSSummary; import org.apache.hadoop.ozone.recon.api.types.DUResponse; @@ -392,6 +397,100 @@ public static StringBuilder constructFullPathPrefix(long initialParentId, String return fullPath; } + /** + * Converts a key prefix into an object path for FSO buckets, using IDs. + * + * This method transforms a user-provided path (e.g., "volume/bucket/dir1") into + * a database-friendly format ("/volumeID/bucketID/ParentId/") by replacing names + * with their corresponding IDs. It simplifies database queries for FSO bucket operations. + * 
+   * {@code
+   * Examples:
+   * - Input: "volume/bucket/key" -> Output: "/volumeID/bucketID/parentDirID/key"
+   * - Input: "volume/bucket/dir1" -> Output: "/volumeID/bucketID/dir1ID/"
+   * - Input: "volume/bucket/dir1/key1" -> Output: "/volumeID/bucketID/dir1ID/key1"
+   * - Input: "volume/bucket/dir1/dir2" -> Output: "/volumeID/bucketID/dir2ID/"
+   * }
+   *
+ * @param prevKeyPrefix The path to be converted. + * @return The object path as "/volumeID/bucketID/ParentId/" or an empty string if an error occurs. + * @throws IOException If database access fails. + * @throws IllegalArgumentException If the provided path is invalid or cannot be converted. + */ + public static String convertToObjectPathForOpenKeySearch(String prevKeyPrefix, + ReconOMMetadataManager omMetadataManager, + ReconNamespaceSummaryManager reconNamespaceSummaryManager, + OzoneStorageContainerManager reconSCM) + throws IOException { + try { + String[] names = EntityHandler.parseRequestPath(EntityHandler.normalizePath( + prevKeyPrefix, BucketLayout.FILE_SYSTEM_OPTIMIZED)); + Table openFileTable = omMetadataManager.getOpenKeyTable( + BucketLayout.FILE_SYSTEM_OPTIMIZED); + + // Root-Level: Return the original path + if (names.length == 0 || names[0].isEmpty()) { + return prevKeyPrefix; + } + + // Volume-Level: Fetch the volumeID + String volumeName = names[0]; + validateNames(volumeName); + String volumeKey = omMetadataManager.getVolumeKey(volumeName); + long volumeId = omMetadataManager.getVolumeTable().getSkipCache(volumeKey).getObjectID(); + if (names.length == 1) { + return constructObjectPathWithPrefix(volumeId); + } + + // Bucket-Level: Fetch the bucketID + String bucketName = names[1]; + validateNames(bucketName); + String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); + OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().getSkipCache(bucketKey); + long bucketId = bucketInfo.getObjectID(); + if (names.length == 2 || bucketInfo.getBucketLayout() != BucketLayout.FILE_SYSTEM_OPTIMIZED) { + return constructObjectPathWithPrefix(volumeId, bucketId); + } + + // Directory or Key-Level: Check both key and directory + BucketHandler handler = + BucketHandler.getBucketHandler(reconNamespaceSummaryManager, omMetadataManager, reconSCM, bucketInfo); + + if (names.length >= 3) { + String lastEntiry = names[names.length - 1]; + + // Check if the directory exists + OmDirectoryInfo dirInfo = handler.getDirInfo(names); + if (dirInfo != null && dirInfo.getName().equals(lastEntiry)) { + return constructObjectPathWithPrefix(volumeId, bucketId, dirInfo.getObjectID()) + OM_KEY_PREFIX; + } + + // Check if the key exists + long dirID = handler.getDirObjectId(names, names.length); + String keyKey = constructObjectPathWithPrefix(volumeId, bucketId, dirID) + + OM_KEY_PREFIX + lastEntiry; + OmKeyInfo keyInfo = openFileTable.getSkipCache(keyKey); + if (keyInfo != null && keyInfo.getFileName().equals(lastEntiry)) { + return constructObjectPathWithPrefix(volumeId, bucketId, + keyInfo.getParentObjectID()) + OM_KEY_PREFIX + lastEntiry; + } + + return prevKeyPrefix; + } + } catch (IllegalArgumentException e) { + log.error( + "IllegalArgumentException encountered while converting key prefix to object path: {}", + prevKeyPrefix, e); + throw e; + } catch (RuntimeException e) { + log.error( + "RuntimeException encountered while converting key prefix to object path: {}", + prevKeyPrefix, e); + return prevKeyPrefix; + } + return prevKeyPrefix; + } + private static void triggerRebuild(ReconNamespaceSummaryManager reconNamespaceSummaryManager, ReconOMMetadataManager omMetadataManager) { ExecutorService executor = Executors.newSingleThreadExecutor(r -> { diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java index d7cb691253da..64da15db4135 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java @@ -56,9 +56,10 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.io.IOException; +import java.util.Collections; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -80,8 +81,6 @@ import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createBadRequestResponse; import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createInternalServerErrorResponse; import static org.apache.hadoop.ozone.recon.ReconResponseUtils.noMatchedKeysResponse; -import static org.apache.hadoop.ozone.recon.ReconUtils.extractKeysFromTable; -import static org.apache.hadoop.ozone.recon.ReconUtils.validateStartPrefix; import static org.apache.hadoop.ozone.recon.api.handlers.BucketHandler.getBucketHandler; import static org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.normalizePath; import static org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.parseRequestPath; @@ -179,102 +178,133 @@ public OMDBInsightEndpoint(OzoneStorageContainerManager reconSCM, @Path("/open") public Response getOpenKeyInfo( @DefaultValue(DEFAULT_FETCH_COUNT) @QueryParam(RECON_QUERY_LIMIT) - int limit, + int limit, @DefaultValue(StringUtils.EMPTY) @QueryParam(RECON_QUERY_PREVKEY) - String prevKey, - @DefaultValue(DEFAULT_OPEN_KEY_INCLUDE_FSO) - @QueryParam(RECON_OPEN_KEY_INCLUDE_FSO) - boolean includeFso, - @DefaultValue(DEFAULT_OPEN_KEY_INCLUDE_NON_FSO) - @QueryParam(RECON_OPEN_KEY_INCLUDE_NON_FSO) - boolean includeNonFso) { + String prevKey, + @DefaultValue(StringUtils.EMPTY) @QueryParam(RECON_QUERY_START_PREFIX) + String startPrefix, + @DefaultValue(DEFAULT_OPEN_KEY_INCLUDE_FSO) @QueryParam(RECON_OPEN_KEY_INCLUDE_FSO) + boolean includeFso, + @DefaultValue(DEFAULT_OPEN_KEY_INCLUDE_NON_FSO) @QueryParam(RECON_OPEN_KEY_INCLUDE_NON_FSO) + boolean includeNonFso) { + KeyInsightInfoResponse openKeyInsightInfo = new KeyInsightInfoResponse(); - List nonFSOKeyInfoList = - openKeyInsightInfo.getNonFSOKeyInfoList(); - - boolean skipPrevKeyDone = false; - boolean isLegacyBucketLayout = true; - boolean recordsFetchedLimitReached = false; - - String lastKey = ""; - List fsoKeyInfoList = openKeyInsightInfo.getFsoKeyInfoList(); - for (BucketLayout layout : Arrays.asList( - BucketLayout.LEGACY, BucketLayout.FILE_SYSTEM_OPTIMIZED)) { - isLegacyBucketLayout = (layout == BucketLayout.LEGACY); - // Skip bucket iteration based on parameters includeFso and includeNonFso - if ((!includeFso && !isLegacyBucketLayout) || - (!includeNonFso && isLegacyBucketLayout)) { - continue; + + try { + long replicatedTotal = 0; + long unreplicatedTotal = 0; + boolean skipPrevKeyDone = false; // Tracks if prevKey was used earlier + boolean keysFound = false; // Flag to track if any keys are found + String lastKey = null; + Map obsKeys = Collections.emptyMap(); + Map fsoKeys = Collections.emptyMap(); + + // Validate startPrefix if it's provided + if (isNotBlank(startPrefix) && !validateStartPrefix(startPrefix)) { + return createBadRequestResponse("Invalid startPrefix: Path must be at the bucket level or deeper."); } - Table openKeyTable = - omMetadataManager.getOpenKeyTable(layout); - try ( - TableIterator> - keyIter = openKeyTable.iterator()) { - boolean skipPrevKey = false; - String seekKey = prevKey; - if (!skipPrevKeyDone && isNotBlank(prevKey)) { - skipPrevKey = true; - Table.KeyValue seekKeyValue = - keyIter.seek(seekKey); - // check if RocksDB was able to seek correctly to the given key prefix - // if not, then return empty result - // In case of an empty prevKeyPrefix, all the keys are returned - if (seekKeyValue == null || - (isNotBlank(prevKey) && - !seekKeyValue.getKey().equals(prevKey))) { - continue; - } + // Use searchOpenKeys logic with adjustments for FSO and Non-FSO filtering + if (includeNonFso) { + // Search for non-FSO keys in KeyTable + Table openKeyTable = omMetadataManager.getOpenKeyTable(BucketLayout.LEGACY); + obsKeys = ReconUtils.extractKeysFromTable(openKeyTable, startPrefix, limit, prevKey); + for (Map.Entry entry : obsKeys.entrySet()) { + keysFound = true; + skipPrevKeyDone = true; // Don't use the prevKey for the file table + KeyEntityInfo keyEntityInfo = createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue()); + openKeyInsightInfo.getNonFSOKeyInfoList().add(keyEntityInfo); // Add to non-FSO list + replicatedTotal += entry.getValue().getReplicatedSize(); + unreplicatedTotal += entry.getValue().getDataSize(); + lastKey = entry.getKey(); // Update lastKey } - while (keyIter.hasNext()) { - Table.KeyValue kv = keyIter.next(); - String key = kv.getKey(); - lastKey = key; - OmKeyInfo omKeyInfo = kv.getValue(); - // skip the prev key if prev key is present - if (skipPrevKey && key.equals(prevKey)) { - skipPrevKeyDone = true; - continue; - } - KeyEntityInfo keyEntityInfo = new KeyEntityInfo(); - keyEntityInfo.setIsKey(omKeyInfo.isFile()); - keyEntityInfo.setKey(key); - keyEntityInfo.setPath(omKeyInfo.getKeyName()); - keyEntityInfo.setInStateSince(omKeyInfo.getCreationTime()); - keyEntityInfo.setSize(omKeyInfo.getDataSize()); - keyEntityInfo.setReplicatedSize(omKeyInfo.getReplicatedSize()); - keyEntityInfo.setReplicationConfig(omKeyInfo.getReplicationConfig()); - openKeyInsightInfo.setUnreplicatedDataSize( - openKeyInsightInfo.getUnreplicatedDataSize() + - keyEntityInfo.getSize()); - openKeyInsightInfo.setReplicatedDataSize( - openKeyInsightInfo.getReplicatedDataSize() + - keyEntityInfo.getReplicatedSize()); - boolean added = - isLegacyBucketLayout ? nonFSOKeyInfoList.add(keyEntityInfo) : - fsoKeyInfoList.add(keyEntityInfo); - if ((nonFSOKeyInfoList.size() + fsoKeyInfoList.size()) == limit) { - recordsFetchedLimitReached = true; - break; - } + } + + if (includeFso) { + // Search for FSO keys in FileTable + // If prevKey was used for non-FSO keys, skip it for FSO keys. + String effectivePrevKey = skipPrevKeyDone ? "" : prevKey; + // If limit = -1 then we need to fetch all keys without limit + int effectiveLimit = limit == -1 ? limit : limit - obsKeys.size(); + fsoKeys = searchOpenKeysInFSO(startPrefix, effectiveLimit, effectivePrevKey); + for (Map.Entry entry : fsoKeys.entrySet()) { + keysFound = true; + KeyEntityInfo keyEntityInfo = createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue()); + openKeyInsightInfo.getFsoKeyInfoList().add(keyEntityInfo); // Add to FSO list + replicatedTotal += entry.getValue().getReplicatedSize(); + unreplicatedTotal += entry.getValue().getDataSize(); + lastKey = entry.getKey(); // Update lastKey } - } catch (IOException ex) { - throw new WebApplicationException(ex, - Response.Status.INTERNAL_SERVER_ERROR); - } catch (IllegalArgumentException e) { - throw new WebApplicationException(e, Response.Status.BAD_REQUEST); - } catch (Exception ex) { - throw new WebApplicationException(ex, - Response.Status.INTERNAL_SERVER_ERROR); } - if (recordsFetchedLimitReached) { - break; + + // If no keys were found, return a response indicating that no keys matched + if (!keysFound) { + return noMatchedKeysResponse(startPrefix); } + + // Set the aggregated totals in the response + openKeyInsightInfo.setReplicatedDataSize(replicatedTotal); + openKeyInsightInfo.setUnreplicatedDataSize(unreplicatedTotal); + openKeyInsightInfo.setLastKey(lastKey); + + // Return the response with the matched keys and their data sizes + return Response.ok(openKeyInsightInfo).build(); + } catch (IOException e) { + // Handle IO exceptions and return an internal server error response + return createInternalServerErrorResponse("Error searching open keys in OM DB: " + e.getMessage()); + } catch (IllegalArgumentException e) { + // Handle illegal argument exceptions and return a bad request response + return createBadRequestResponse("Invalid argument: " + e.getMessage()); } + } - openKeyInsightInfo.setLastKey(lastKey); - return Response.ok(openKeyInsightInfo).build(); + public Map searchOpenKeysInFSO(String startPrefix, + int limit, String prevKey) + throws IOException, IllegalArgumentException { + Map matchedKeys = new LinkedHashMap<>(); + // Convert the search prefix to an object path for FSO buckets + String startPrefixObjectPath = ReconUtils.convertToObjectPathForOpenKeySearch( + startPrefix, omMetadataManager, reconNamespaceSummaryManager, reconSCM); + String[] names = parseRequestPath(startPrefixObjectPath); + Table openFileTable = + omMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED); + + // If names.length <= 2, then the search prefix is at the volume or bucket level hence + // no need to find parent or extract id's or find subpaths as the openFileTable is + // suitable for volume and bucket level search + if (names.length > 2 && startPrefixObjectPath.endsWith(OM_KEY_PREFIX)) { + // Fetch the parent ID to search for + long parentId = Long.parseLong(names[names.length - 1]); + + // Fetch the nameSpaceSummary for the parent ID + NSSummary parentSummary = reconNamespaceSummaryManager.getNSSummary(parentId); + if (parentSummary == null) { + return matchedKeys; + } + List subPaths = new ArrayList<>(); + // Add the initial search prefix object path because it can have both openFiles + // and subdirectories with openFiles + subPaths.add(startPrefixObjectPath); + + // Recursively gather all subpaths + ReconUtils.gatherSubPaths(parentId, subPaths, Long.parseLong(names[0]), Long.parseLong(names[1]), + reconNamespaceSummaryManager); + + // Iterate over the subpaths and retrieve the open files + for (String subPath : subPaths) { + matchedKeys.putAll( + ReconUtils.extractKeysFromTable(openFileTable, subPath, limit - matchedKeys.size(), prevKey)); + if (matchedKeys.size() >= limit) { + break; + } + } + return matchedKeys; + } + + // If the search level is at the volume, bucket or key level, directly search the openFileTable + matchedKeys.putAll( + ReconUtils.extractKeysFromTable(openFileTable, startPrefixObjectPath, limit, prevKey)); + return matchedKeys; } /** @@ -481,7 +511,7 @@ private boolean getPendingForDeletionKeyInfo( // Search for deleted keys in DeletedTable Table deletedTable = omMetadataManager.getDeletedTable(); Map deletedKeys = - extractKeysFromTable(deletedTable, startPrefix, limit, prevKey); + ReconUtils.extractKeysFromTable(deletedTable, startPrefix, limit, prevKey); // Iterate over the retrieved keys and populate the response for (Map.Entry entry : deletedKeys.entrySet()) { @@ -1265,8 +1295,7 @@ private KeyEntityInfo createKeyEntityInfoFromOmKeyInfo(String dbKey, KeyEntityInfo keyEntityInfo = new KeyEntityInfo(); keyEntityInfo.setKey(dbKey); // Set the DB key keyEntityInfo.setIsKey(keyInfo.isFile()); - keyEntityInfo.setPath(ReconUtils.constructFullPath(keyInfo, reconNamespaceSummaryManager, - omMetadataManager)); + keyEntityInfo.setPath(ReconUtils.constructFullPath(keyInfo, reconNamespaceSummaryManager, omMetadataManager)); keyEntityInfo.setSize(keyInfo.getDataSize()); keyEntityInfo.setCreationTime(keyInfo.getCreationTime()); keyEntityInfo.setModificationTime(keyInfo.getModificationTime()); @@ -1284,6 +1313,20 @@ private void createSummaryForDeletedDirectories( dirSummary.put("totalDeletedDirectories", deletedDirCount); } + private boolean validateStartPrefix(String startPrefix) { + + // Ensure startPrefix starts with '/' for non-empty values + startPrefix = startPrefix.startsWith("/") ? startPrefix : "/" + startPrefix; + + // Split the path to ensure it's at least at the bucket level (volume/bucket). + String[] pathComponents = startPrefix.split("/"); + if (pathComponents.length < 3 || pathComponents[2].isEmpty()) { + return false; // Invalid if not at bucket level or deeper + } + + return true; + } + private String createPath(OmKeyInfo omKeyInfo) { return omKeyInfo.getVolumeName() + OM_KEY_PREFIX + omKeyInfo.getBucketName() + OM_KEY_PREFIX + omKeyInfo.getKeyName(); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java deleted file mode 100644 index fcd73fbe72f2..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java +++ /dev/null @@ -1,350 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.recon.api; - -import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; -import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; -import org.apache.hadoop.ozone.recon.api.types.KeyEntityInfo; -import org.apache.hadoop.ozone.recon.api.types.KeyInsightInfoResponse; -import org.apache.hadoop.ozone.recon.api.types.NSSummary; -import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; -import org.apache.hadoop.ozone.recon.spi.impl.ReconNamespaceSummaryManagerImpl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Inject; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.QueryParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.io.IOException; -import java.util.Map; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.ArrayList; - -import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OM_INSIGHTS_DEFAULT_START_PREFIX; -import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OM_INSIGHTS_DEFAULT_SEARCH_LIMIT; -import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OM_INSIGHTS_DEFAULT_SEARCH_PREV_KEY; -import static org.apache.hadoop.ozone.recon.ReconResponseUtils.noMatchedKeysResponse; -import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createBadRequestResponse; -import static org.apache.hadoop.ozone.recon.ReconResponseUtils.createInternalServerErrorResponse; -import static org.apache.hadoop.ozone.recon.ReconUtils.validateStartPrefix; -import static org.apache.hadoop.ozone.recon.ReconUtils.constructObjectPathWithPrefix; -import static org.apache.hadoop.ozone.recon.ReconUtils.extractKeysFromTable; -import static org.apache.hadoop.ozone.recon.ReconUtils.gatherSubPaths; -import static org.apache.hadoop.ozone.recon.ReconUtils.validateNames; -import static org.apache.hadoop.ozone.recon.api.handlers.BucketHandler.getBucketHandler; -import static org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.normalizePath; -import static org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.parseRequestPath; - -/** - * REST endpoint for search implementation in OM DB Insight. - * - * This class provides endpoints for searching keys in the Ozone Manager database. - * It supports searching for both open and deleted keys across File System Optimized (FSO) - * and Object Store (non-FSO) bucket layouts. The results include matching keys and their - * data sizes. - */ -@Path("/keys") -@Produces(MediaType.APPLICATION_JSON) -@AdminOnly -public class OMDBInsightSearchEndpoint { - - private OzoneStorageContainerManager reconSCM; - private final ReconOMMetadataManager omMetadataManager; - private static final Logger LOG = - LoggerFactory.getLogger(OMDBInsightSearchEndpoint.class); - private ReconNamespaceSummaryManagerImpl reconNamespaceSummaryManager; - - - @Inject - public OMDBInsightSearchEndpoint(OzoneStorageContainerManager reconSCM, - ReconOMMetadataManager omMetadataManager, - ReconNamespaceSummaryManagerImpl reconNamespaceSummaryManager) { - this.reconSCM = reconSCM; - this.omMetadataManager = omMetadataManager; - this.reconNamespaceSummaryManager = reconNamespaceSummaryManager; - } - - - /** - * Performs a search for open keys in the Ozone Manager OpenKey and OpenFile table using a specified search prefix. - * This endpoint searches across both File System Optimized (FSO) and Object Store (non-FSO) layouts, - * compiling a list of keys that match the given prefix along with their data sizes. - * - * The search prefix must start from the bucket level ('/volumeName/bucketName/') or any specific directory - * or key level (e.g., '/volA/bucketA/dir1' for everything under 'dir1' inside 'bucketA' of 'volA'). - * The search operation matches the prefix against the start of keys' names within the OM DB. - * - * Example Usage: - * 1. A startPrefix of "/volA/bucketA/" retrieves every key under bucket 'bucketA' in volume 'volA'. - * 2. Specifying "/volA/bucketA/dir1" focuses the search within 'dir1' inside 'bucketA' of 'volA'. - * - * @param startPrefix The prefix for searching keys, starting from the bucket level or any specific path. - * @param limit Limits the number of returned keys. - * @param prevKey The key to start after for the next set of records. - * @return A KeyInsightInfoResponse, containing matching keys and their data sizes. - * @throws IOException On failure to access the OM database or process the operation. - * @throws IllegalArgumentException If the provided startPrefix or other arguments are invalid. - */ - @GET - @Path("/open/search") - public Response searchOpenKeys( - @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_START_PREFIX) @QueryParam("startPrefix") - String startPrefix, - @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_SEARCH_LIMIT) @QueryParam("limit") - int limit, - @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_SEARCH_PREV_KEY) @QueryParam("prevKey") - String prevKey) throws IOException { - - try { - // Validate the request parameters - if (!validateStartPrefix(startPrefix)) { - return createBadRequestResponse("Invalid startPrefix: Path must be at the bucket level or deeper."); - } - - // Ensure the limit is non-negative - limit = Math.max(0, limit); - - // Initialize response object - KeyInsightInfoResponse insightResponse = new KeyInsightInfoResponse(); - long replicatedTotal = 0; - long unreplicatedTotal = 0; - boolean keysFound = false; // Flag to track if any keys are found - String lastKey = null; - - // Search for non-fso keys in KeyTable - Table openKeyTable = - omMetadataManager.getOpenKeyTable(BucketLayout.LEGACY); - Map obsKeys = - extractKeysFromTable(openKeyTable, startPrefix, limit, prevKey); - for (Map.Entry entry : obsKeys.entrySet()) { - keysFound = true; - KeyEntityInfo keyEntityInfo = - createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue()); - insightResponse.getNonFSOKeyInfoList() - .add(keyEntityInfo); // Add to non-FSO list - replicatedTotal += entry.getValue().getReplicatedSize(); - unreplicatedTotal += entry.getValue().getDataSize(); - lastKey = entry.getKey(); // Update lastKey - } - - // Search for fso keys in FileTable - Map fsoKeys = searchOpenKeysInFSO(startPrefix, limit, prevKey); - for (Map.Entry entry : fsoKeys.entrySet()) { - keysFound = true; - KeyEntityInfo keyEntityInfo = - createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue()); - insightResponse.getFsoKeyInfoList() - .add(keyEntityInfo); // Add to FSO list - replicatedTotal += entry.getValue().getReplicatedSize(); - unreplicatedTotal += entry.getValue().getDataSize(); - lastKey = entry.getKey(); // Update lastKey - } - - // If no keys were found, return a response indicating that no keys matched - if (!keysFound) { - return noMatchedKeysResponse(startPrefix); - } - - // Set the aggregated totals in the response - insightResponse.setReplicatedDataSize(replicatedTotal); - insightResponse.setUnreplicatedDataSize(unreplicatedTotal); - insightResponse.setLastKey(lastKey); - - // Return the response with the matched keys and their data sizes - return Response.ok(insightResponse).build(); - } catch (IOException e) { - // Handle IO exceptions and return an internal server error response - return createInternalServerErrorResponse( - "Error searching open keys in OM DB: " + e.getMessage()); - } catch (IllegalArgumentException e) { - // Handle illegal argument exceptions and return a bad request response - return createBadRequestResponse( - "Invalid startPrefix: " + e.getMessage()); - } - } - - public Map searchOpenKeysInFSO(String startPrefix, - int limit, String prevKey) - throws IOException, IllegalArgumentException { - Map matchedKeys = new LinkedHashMap<>(); - // Convert the search prefix to an object path for FSO buckets - String startPrefixObjectPath = convertToObjectPath(startPrefix); - String[] names = parseRequestPath(startPrefixObjectPath); - Table openFileTable = - omMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED); - - // If names.length <= 2, then the search prefix is at the volume or bucket level hence - // no need to find parent or extract id's or find subpaths as the openFileTable is - // suitable for volume and bucket level search - if (names.length > 2 && startPrefixObjectPath.endsWith(OM_KEY_PREFIX)) { - // Fetch the parent ID to search for - long parentId = Long.parseLong(names[names.length - 1]); - - // Fetch the nameSpaceSummary for the parent ID - NSSummary parentSummary = reconNamespaceSummaryManager.getNSSummary(parentId); - if (parentSummary == null) { - return matchedKeys; - } - List subPaths = new ArrayList<>(); - // Add the initial search prefix object path because it can have both openFiles - // and subdirectories with openFiles - subPaths.add(startPrefixObjectPath); - - // Recursively gather all subpaths - gatherSubPaths(parentId, subPaths, Long.parseLong(names[0]), Long.parseLong(names[1]), - reconNamespaceSummaryManager); - - // Iterate over the subpaths and retrieve the open files - for (String subPath : subPaths) { - matchedKeys.putAll( - extractKeysFromTable(openFileTable, subPath, limit - matchedKeys.size(), prevKey)); - if (matchedKeys.size() >= limit) { - break; - } - } - return matchedKeys; - } - - // If the search level is at the volume, bucket or key level, directly search the openFileTable - matchedKeys.putAll( - extractKeysFromTable(openFileTable, startPrefixObjectPath, limit, prevKey)); - return matchedKeys; - } - - /** - * Converts a key prefix into an object path for FSO buckets, using IDs. - * - * This method transforms a user-provided path (e.g., "volume/bucket/dir1") into - * a database-friendly format ("/volumeID/bucketID/ParentId/") by replacing names - * with their corresponding IDs. It simplifies database queries for FSO bucket operations. - * 

-   * {@code
-   * Examples:
-   * - Input: "volume/bucket/key" -> Output: "/volumeID/bucketID/parentDirID/key"
-   * - Input: "volume/bucket/dir1" -> Output: "/volumeID/bucketID/dir1ID/"
-   * - Input: "volume/bucket/dir1/key1" -> Output: "/volumeID/bucketID/dir1ID/key1"
-   * - Input: "volume/bucket/dir1/dir2" -> Output: "/volumeID/bucketID/dir2ID/"
-   * }
-   *
- * @param prevKeyPrefix The path to be converted. - * @return The object path as "/volumeID/bucketID/ParentId/" or an empty string if an error occurs. - * @throws IOException If database access fails. - * @throws IllegalArgumentException If the provided path is invalid or cannot be converted. - */ - public String convertToObjectPath(String prevKeyPrefix) throws IOException { - try { - String[] names = parseRequestPath(normalizePath(prevKeyPrefix, BucketLayout.FILE_SYSTEM_OPTIMIZED)); - Table openFileTable = omMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED); - - // Root-Level: Return the original path - if (names.length == 0) { - return prevKeyPrefix; - } - - // Volume-Level: Fetch the volumeID - String volumeName = names[0]; - validateNames(volumeName); - String volumeKey = omMetadataManager.getVolumeKey(volumeName); - long volumeId = omMetadataManager.getVolumeTable().getSkipCache(volumeKey).getObjectID(); - if (names.length == 1) { - return constructObjectPathWithPrefix(volumeId); - } - - // Bucket-Level: Fetch the bucketID - String bucketName = names[1]; - validateNames(bucketName); - String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); - OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().getSkipCache(bucketKey); - long bucketId = bucketInfo.getObjectID(); - if (names.length == 2 || bucketInfo.getBucketLayout() != BucketLayout.FILE_SYSTEM_OPTIMIZED) { - return constructObjectPathWithPrefix(volumeId, bucketId); - } - - // Directory or Key-Level: Check both key and directory - BucketHandler handler = - getBucketHandler(reconNamespaceSummaryManager, omMetadataManager, reconSCM, bucketInfo); - - if (names.length >= 3) { - String lastEntiry = names[names.length - 1]; - - // Check if the directory exists - OmDirectoryInfo dirInfo = handler.getDirInfo(names); - if (dirInfo != null && dirInfo.getName().equals(lastEntiry)) { - return constructObjectPathWithPrefix(volumeId, bucketId, dirInfo.getObjectID()) + OM_KEY_PREFIX; - } - - // Check if the key exists - long dirID = handler.getDirObjectId(names, names.length); - String keyKey = constructObjectPathWithPrefix(volumeId, bucketId, dirID) + - OM_KEY_PREFIX + lastEntiry; - OmKeyInfo keyInfo = openFileTable.getSkipCache(keyKey); - if (keyInfo != null && keyInfo.getFileName().equals(lastEntiry)) { - return constructObjectPathWithPrefix(volumeId, bucketId, - keyInfo.getParentObjectID()) + OM_KEY_PREFIX + lastEntiry; - } - - return prevKeyPrefix; - } - } catch (IllegalArgumentException e) { - LOG.error( - "IllegalArgumentException encountered while converting key prefix to object path: {}", - prevKeyPrefix, e); - throw e; - } catch (RuntimeException e) { - LOG.error( - "RuntimeException encountered while converting key prefix to object path: {}", - prevKeyPrefix, e); - return prevKeyPrefix; - } - return prevKeyPrefix; - } - - /** - * Creates a KeyEntityInfo object from an OmKeyInfo object and the corresponding key. - * - * @param dbKey The key in the database corresponding to the OmKeyInfo object. - * @param keyInfo The OmKeyInfo object to create the KeyEntityInfo from. - * @return The KeyEntityInfo object created from the OmKeyInfo object and the key. - */ - private KeyEntityInfo createKeyEntityInfoFromOmKeyInfo(String dbKey, - OmKeyInfo keyInfo) { - KeyEntityInfo keyEntityInfo = new KeyEntityInfo(); - keyEntityInfo.setKey(dbKey); // Set the DB key - keyEntityInfo.setIsKey(keyInfo.isFile()); - keyEntityInfo.setPath(keyInfo.getKeyName()); // Assuming path is the same as key name - keyEntityInfo.setInStateSince(keyInfo.getCreationTime()); - keyEntityInfo.setSize(keyInfo.getDataSize()); - keyEntityInfo.setReplicatedSize(keyInfo.getReplicatedSize()); - keyEntityInfo.setReplicationConfig(keyInfo.getReplicationConfig()); - return keyEntityInfo; - } - -} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java index ed4b82fa3e04..61a9711876e7 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java @@ -90,6 +90,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -897,11 +898,11 @@ public void testGetOpenKeyInfo() throws Exception { .get("/sampleVol/bucketOne/key_one"); assertEquals("key_one", omKeyInfo1.getKeyName()); Response openKeyInfoResp = - omdbInsightEndpoint.getOpenKeyInfo(-1, "", true, true); + omdbInsightEndpoint.getOpenKeyInfo(-1, "", "", true, true); KeyInsightInfoResponse keyInsightInfoResp = (KeyInsightInfoResponse) openKeyInfoResp.getEntity(); assertNotNull(keyInsightInfoResp); - assertEquals("key_one", + assertEquals("sampleVol/bucketOne/key_one", keyInsightInfoResp.getNonFSOKeyInfoList().get(0).getPath()); } @@ -1044,7 +1045,7 @@ public void testGetOpenKeyInfoLimitParam() throws Exception { reconOMMetadataManager.getOpenKeyTable(getBucketLayout()) .put("/sampleVol/bucketOne/key_three", omKeyInfo3); Response openKeyInfoResp = - omdbInsightEndpoint.getOpenKeyInfo(2, "", true, true); + omdbInsightEndpoint.getOpenKeyInfo(2, "", "", true, true); KeyInsightInfoResponse keyInsightInfoResp = (KeyInsightInfoResponse) openKeyInfoResp.getEntity(); assertNotNull(keyInsightInfoResp); @@ -1053,10 +1054,10 @@ public void testGetOpenKeyInfoLimitParam() throws Exception { assertEquals(0, keyInsightInfoResp.getFsoKeyInfoList().size()); assertEquals(2, keyInsightInfoResp.getFsoKeyInfoList().size() + keyInsightInfoResp.getNonFSOKeyInfoList().size()); - assertEquals("key_three", + assertEquals("sampleVol/bucketOne/key_three", keyInsightInfoResp.getNonFSOKeyInfoList().get(1).getPath()); - openKeyInfoResp = omdbInsightEndpoint.getOpenKeyInfo(3, "", true, true); + openKeyInfoResp = omdbInsightEndpoint.getOpenKeyInfo(3, "", "", true, true); keyInsightInfoResp = (KeyInsightInfoResponse) openKeyInfoResp.getEntity(); assertNotNull(keyInsightInfoResp); @@ -1065,7 +1066,7 @@ public void testGetOpenKeyInfoLimitParam() throws Exception { assertEquals(1, keyInsightInfoResp.getFsoKeyInfoList().size()); assertEquals(3, keyInsightInfoResp.getFsoKeyInfoList().size() + keyInsightInfoResp.getNonFSOKeyInfoList().size()); - assertEquals("key_three", + assertEquals("sampleVol/bucketOne/key_three", keyInsightInfoResp.getNonFSOKeyInfoList().get(1).getPath()); } @@ -1107,7 +1108,7 @@ public void testGetOpenKeyInfoWithIncludeFsoAndIncludeNonFsoParams() // CASE 1 :- Display only FSO keys in response // includeFsoKeys=true, includeNonFsoKeys=false Response openKeyInfoResp = - omdbInsightEndpoint.getOpenKeyInfo(10, "", true, false); + omdbInsightEndpoint.getOpenKeyInfo(10, "", "", true, false); KeyInsightInfoResponse keyInsightInfoResp = (KeyInsightInfoResponse) openKeyInfoResp.getEntity(); assertNotNull(keyInsightInfoResp); @@ -1119,7 +1120,7 @@ public void testGetOpenKeyInfoWithIncludeFsoAndIncludeNonFsoParams() // CASE 2 :- Display only Non-FSO keys in response // includeFsoKeys=false, includeNonFsoKeys=true openKeyInfoResp = - omdbInsightEndpoint.getOpenKeyInfo(10, "", false, true); + omdbInsightEndpoint.getOpenKeyInfo(10, "", "", false, true); keyInsightInfoResp = (KeyInsightInfoResponse) openKeyInfoResp.getEntity(); assertNotNull(keyInsightInfoResp); assertEquals(0, @@ -1130,7 +1131,7 @@ public void testGetOpenKeyInfoWithIncludeFsoAndIncludeNonFsoParams() // CASE 3 :- Display both FSO and Non-FSO keys in response // includeFsoKeys=true, includeNonFsoKeys=true openKeyInfoResp = - omdbInsightEndpoint.getOpenKeyInfo(10, "", true, true); + omdbInsightEndpoint.getOpenKeyInfo(10, "", "", true, true); keyInsightInfoResp = (KeyInsightInfoResponse) openKeyInfoResp.getEntity(); assertNotNull(keyInsightInfoResp); assertEquals(4, @@ -1141,45 +1142,39 @@ public void testGetOpenKeyInfoWithIncludeFsoAndIncludeNonFsoParams() // CASE 4 :- Don't Display both FSO and Non-FSO keys in response // includeFsoKeys=false, includeNonFsoKeys=false openKeyInfoResp = - omdbInsightEndpoint.getOpenKeyInfo(10, "", false, false); - keyInsightInfoResp = (KeyInsightInfoResponse) openKeyInfoResp.getEntity(); - assertNotNull(keyInsightInfoResp); - assertEquals(0, - keyInsightInfoResp.getFsoKeyInfoList().size()); - assertEquals(0, - keyInsightInfoResp.getNonFSOKeyInfoList().size()); + omdbInsightEndpoint.getOpenKeyInfo(10, "", "", false, false); + assertEquals(204, openKeyInfoResp.getStatus()); + String entity = (String) openKeyInfoResp.getEntity(); + assertTrue(entity.contains("No keys matched the search prefix"), + "Expected a message indicating no keys were found"); } @Test public void testGetOpenKeyInfoPrevKeyParam() throws Exception { OmKeyInfo omKeyInfo1 = - getOmKeyInfo("sampleVol", "bucketOne", "key_one", true); + getOmKeyInfo("sampleVol", "bucketOne", "key_1", true); OmKeyInfo omKeyInfo2 = - getOmKeyInfo("sampleVol", "bucketOne", "key_two", true); + getOmKeyInfo("sampleVol", "bucketOne", "key_2", true); OmKeyInfo omKeyInfo3 = - getOmKeyInfo("sampleVol", "bucketOne", "key_three", true); + getOmKeyInfo("sampleVol", "bucketOne", "key_3", true); reconOMMetadataManager.getOpenKeyTable(getBucketLayout()) - .put("/sampleVol/bucketOne/key_one", omKeyInfo1); + .put("/sampleVol/bucketOne/key_1", omKeyInfo1); reconOMMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED) - .put("/sampleVol/bucketOne/key_two", omKeyInfo2); + .put("/sampleVol/bucketOne/key_2", omKeyInfo2); reconOMMetadataManager.getOpenKeyTable(getBucketLayout()) - .put("/sampleVol/bucketOne/key_three", omKeyInfo3); + .put("/sampleVol/bucketOne/key_3", omKeyInfo3); Response openKeyInfoResp = - omdbInsightEndpoint.getOpenKeyInfo(-1, "/sampleVol/bucketOne/key_one", + omdbInsightEndpoint.getOpenKeyInfo(-1, "/sampleVol/bucketOne/key_1", "", true, true); KeyInsightInfoResponse keyInsightInfoResp = (KeyInsightInfoResponse) openKeyInfoResp.getEntity(); assertNotNull(keyInsightInfoResp); - assertEquals(1, - keyInsightInfoResp.getNonFSOKeyInfoList().size()); + assertEquals(1, keyInsightInfoResp.getNonFSOKeyInfoList().size()); assertEquals(1, keyInsightInfoResp.getFsoKeyInfoList().size()); - assertEquals(2, keyInsightInfoResp.getFsoKeyInfoList().size() + - keyInsightInfoResp.getNonFSOKeyInfoList().size()); - assertEquals("key_three", - keyInsightInfoResp.getNonFSOKeyInfoList().get(0).getPath()); - assertEquals("key_two", - keyInsightInfoResp.getFsoKeyInfoList().get(0).getPath()); + assertEquals(2, keyInsightInfoResp.getFsoKeyInfoList().size() + keyInsightInfoResp.getNonFSOKeyInfoList().size()); + assertEquals("sampleVol/bucketOne/key_3", keyInsightInfoResp.getNonFSOKeyInfoList().get(0).getPath()); + assertEquals("sampleVol/bucketOne/key_2", keyInsightInfoResp.getFsoKeyInfoList().get(0).getPath()); } @Test diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOMDBInsightSearchEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenKeysSearchEndpoint.java similarity index 83% rename from hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOMDBInsightSearchEndpoint.java rename to hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenKeysSearchEndpoint.java index c3c2fe5debed..f55d988cfe01 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOMDBInsightSearchEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenKeysSearchEndpoint.java @@ -81,12 +81,12 @@ * 11. Test Search Open Keys with Pagination: Verifies paginated search results. * 12. Test Search in Empty Bucket: Checks the response for searching within an empty bucket. */ -public class TestOMDBInsightSearchEndpoint extends AbstractReconSqlDBTest { +public class TestOpenKeysSearchEndpoint extends AbstractReconSqlDBTest { @TempDir private Path temporaryFolder; private ReconOMMetadataManager reconOMMetadataManager; - private OMDBInsightSearchEndpoint omdbInsightSearchEndpoint; + private OMDBInsightEndpoint omdbInsightEndpoint; private OzoneConfiguration ozoneConfiguration; private static final String ROOT_PATH = "/"; private static final String TEST_USER = "TestUser"; @@ -116,11 +116,8 @@ public void setUp() throws Exception { .addBinding(OMDBInsightEndpoint.class) .addBinding(ContainerHealthSchemaManager.class) .build(); - reconNamespaceSummaryManager = - reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); - omdbInsightSearchEndpoint = reconTestInjector.getInstance( - OMDBInsightSearchEndpoint.class); - + reconNamespaceSummaryManager = reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); + omdbInsightEndpoint = reconTestInjector.getInstance(OMDBInsightEndpoint.class); // populate OM DB and reprocess into Recon RocksDB populateOMDB(); NSSummaryTaskWithFSO nSSummaryTaskWithFso = @@ -150,26 +147,19 @@ private static OMMetadataManager initializeNewOmMetadataManager( public void testRootLevelSearchRestriction() throws IOException { // Test with root level path String rootPath = "/"; - Response response = omdbInsightSearchEndpoint.searchOpenKeys(rootPath, 20, ""); + Response response = + omdbInsightEndpoint.getOpenKeyInfo(-1, "", rootPath, true, true); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), "Expected a message indicating the path must be at the bucket level or deeper"); - - // Test with root level path without trailing slash - rootPath = ""; - response = omdbInsightSearchEndpoint.searchOpenKeys(rootPath, 20, ""); - assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); - entity = (String) response.getEntity(); - assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), - "Expected a message indicating the path must be at the bucket level or deeper"); } @Test public void testVolumeLevelSearchRestriction() throws IOException { // Test with volume level path String volumePath = "/vola"; - Response response = omdbInsightSearchEndpoint.searchOpenKeys(volumePath, 20, ""); + Response response = omdbInsightEndpoint.getOpenKeyInfo(20, "", volumePath, true, true); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), @@ -177,7 +167,7 @@ public void testVolumeLevelSearchRestriction() throws IOException { // Test with another volume level path volumePath = "/volb"; - response = omdbInsightSearchEndpoint.searchOpenKeys(volumePath, 20, ""); + response = omdbInsightEndpoint.getOpenKeyInfo(20, "", volumePath, true, true); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); entity = (String) response.getEntity(); assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), @@ -188,7 +178,7 @@ public void testVolumeLevelSearchRestriction() throws IOException { public void testBucketLevelSearch() throws IOException { // Search inside FSO bucket Response response = - omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1", 20, ""); + omdbInsightEndpoint.getOpenKeyInfo(20, "", "/vola/bucketa1", true, true); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); @@ -200,7 +190,7 @@ public void testBucketLevelSearch() throws IOException { // Search inside OBS bucket response = - omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb1", 20, ""); + omdbInsightEndpoint.getOpenKeyInfo(20, "", "/volb/bucketb1", true, true); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); @@ -212,13 +202,13 @@ public void testBucketLevelSearch() throws IOException { // Search Inside LEGACY bucket response = - omdbInsightSearchEndpoint.searchOpenKeys("/volc/bucketc1", 20, ""); + omdbInsightEndpoint.getOpenKeyInfo(20, "", "/volc/bucketc1", true, true); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(7, result.getNonFSOKeyInfoList().size()); // Test with bucket that does not exist - response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/nonexistentbucket", 20, ""); + response = omdbInsightEndpoint.getOpenKeyInfo(20, "", "/vola/nonexistentbucket", true, true); assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), @@ -228,7 +218,7 @@ public void testBucketLevelSearch() throws IOException { @Test public void testDirectoryLevelSearch() throws IOException { Response response = - omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira1", 20, ""); + omdbInsightEndpoint.getOpenKeyInfo(20, "", "/vola/bucketa1/dira1", true, true); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); @@ -239,7 +229,7 @@ public void testDirectoryLevelSearch() throws IOException { assertEquals(1000 * 3, result.getReplicatedDataSize()); response = - omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira2", 20, ""); + omdbInsightEndpoint.getOpenKeyInfo(20, "", "/vola/bucketa1/dira2", true, true); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); @@ -250,7 +240,7 @@ public void testDirectoryLevelSearch() throws IOException { assertEquals(1000 * 3, result.getReplicatedDataSize()); response = - omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira3", 20, ""); + omdbInsightEndpoint.getOpenKeyInfo(20, "", "/vola/bucketa1/dira3", true, true); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); @@ -261,7 +251,7 @@ public void testDirectoryLevelSearch() throws IOException { assertEquals(10000 * 3, result.getReplicatedDataSize()); // Test with non-existent directory - response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/nonexistentdir", 20, ""); + response = omdbInsightEndpoint.getOpenKeyInfo(20, "", "/vola/bucketa1/nonexistentdir", true, true); assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), @@ -271,7 +261,7 @@ public void testDirectoryLevelSearch() throws IOException { @Test public void testKeyLevelSearch() throws IOException { // FSO Bucket key-level search - Response response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/filea1", 10, ""); + Response response = omdbInsightEndpoint.getOpenKeyInfo(10, "", "/vola/bucketa1/filea1", true, true); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(1, result.getFsoKeyInfoList().size()); @@ -280,7 +270,7 @@ public void testKeyLevelSearch() throws IOException { assertEquals(1000, result.getUnreplicatedDataSize()); assertEquals(1000 * 3, result.getReplicatedDataSize()); - response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/filea2", 10, ""); + response = omdbInsightEndpoint.getOpenKeyInfo(10, "", "/vola/bucketa1/filea2", true, true); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(1, result.getFsoKeyInfoList().size()); @@ -290,7 +280,8 @@ public void testKeyLevelSearch() throws IOException { assertEquals(1000 * 3, result.getReplicatedDataSize()); // OBS Bucket key-level search - response = omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb1/fileb1", 10, ""); + response = omdbInsightEndpoint + .getOpenKeyInfo(10, "", "/volb/bucketb1/fileb1", true, true); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(0, result.getFsoKeyInfoList().size()); @@ -299,7 +290,8 @@ public void testKeyLevelSearch() throws IOException { assertEquals(1000, result.getUnreplicatedDataSize()); assertEquals(1000 * 3, result.getReplicatedDataSize()); - response = omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb1/fileb2", 10, ""); + response = omdbInsightEndpoint + .getOpenKeyInfo(10, "", "/volb/bucketb1/fileb2", true, true); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(0, result.getFsoKeyInfoList().size()); @@ -309,13 +301,15 @@ public void testKeyLevelSearch() throws IOException { assertEquals(1000 * 3, result.getReplicatedDataSize()); // Test with non-existent key - response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/nonexistentfile", 1, ""); + response = omdbInsightEndpoint + .getOpenKeyInfo(10, "", "/volb/bucketb1/nonexistentfile", true, true); assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); - response = omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb1/nonexistentfile", 1, ""); + response = omdbInsightEndpoint + .getOpenKeyInfo(10, "", "/volb/bucketb1/nonexistentfile", true, true); assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), @@ -326,29 +320,31 @@ public void testKeyLevelSearch() throws IOException { @Test public void testKeyLevelSearchUnderDirectory() throws IOException { // FSO Bucket key-level search - Response response = - omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira1/innerfile", 10, ""); + Response response = omdbInsightEndpoint + .getOpenKeyInfo(10, "", "/vola/bucketa1/dira1/innerfile", true, true); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(1, result.getFsoKeyInfoList().size()); assertEquals(0, result.getNonFSOKeyInfoList().size()); - response = - omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira2/innerfile", 10, ""); + response = omdbInsightEndpoint + .getOpenKeyInfo(10, "", "/vola/bucketa1/dira2/innerfile", true, true); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(1, result.getFsoKeyInfoList().size()); assertEquals(0, result.getNonFSOKeyInfoList().size()); // Test for unknown file in fso bucket - response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira1/unknownfile", 10, ""); + response = omdbInsightEndpoint + .getOpenKeyInfo(10, "", "/vola/bucketa1/dira1/unknownfile", true, true); assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); // Test for unknown file in fso bucket - response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira2/unknownfile", 10, ""); + response = omdbInsightEndpoint + .getOpenKeyInfo(10, "", "/vola/bucketa1/dira2/unknownfile", true, true); assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), @@ -358,55 +354,55 @@ public void testKeyLevelSearchUnderDirectory() throws IOException { @Test public void testSearchUnderNestedDirectory() throws IOException { - Response response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira3", 20, - ""); + Response response = omdbInsightEndpoint + .getOpenKeyInfo(20, "", "/vola/bucketa1/dira3", true, true); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(10, result.getFsoKeyInfoList().size()); assertEquals(0, result.getNonFSOKeyInfoList().size()); // Search under dira31 - response = omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira3/dira31", - 20, ""); + response = omdbInsightEndpoint + .getOpenKeyInfo(20, "", "/vola/bucketa1/dira3/dira31", true, true); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(6, result.getFsoKeyInfoList().size()); assertEquals(0, result.getNonFSOKeyInfoList().size()); // Search under dira32 - response = omdbInsightSearchEndpoint.searchOpenKeys( - "/vola/bucketa1/dira3/dira31/dira32", 20, ""); + response = omdbInsightEndpoint + .getOpenKeyInfo(20, "", "/vola/bucketa1/dira3/dira31/dira32", true, true); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(3, result.getFsoKeyInfoList().size()); assertEquals(0, result.getNonFSOKeyInfoList().size()); // Search under dira33 - response = omdbInsightSearchEndpoint.searchOpenKeys( - "/vola/bucketa1/dira3/dira31/dira32/dira33", 20, ""); + response = omdbInsightEndpoint + .getOpenKeyInfo(20, "", "/vola/bucketa1/dira3/dira31/dira32/dira33", true, true); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(1, result.getFsoKeyInfoList().size()); assertEquals(0, result.getNonFSOKeyInfoList().size()); // Search for the exact file under dira33 - response = omdbInsightSearchEndpoint.searchOpenKeys( - "/vola/bucketa1/dira3/dira31/dira32/dira33/file33_1", 20, ""); + response = omdbInsightEndpoint + .getOpenKeyInfo(20, "", "/vola/bucketa1/dira3/dira31/dira32/dira33/file33_1", true, true); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(1, result.getFsoKeyInfoList().size()); assertEquals(0, result.getNonFSOKeyInfoList().size()); // Search for a non existant file under each nested directory - response = omdbInsightSearchEndpoint.searchOpenKeys( - "/vola/bucketa1/dira3/dira31/dira32/dira33/nonexistentfile", 20, ""); + response = omdbInsightEndpoint + .getOpenKeyInfo(20, "", "/vola/bucketa1/dira3/dira31/dira32/dira33/nonexistentfile", true, true); assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); - response = omdbInsightSearchEndpoint.searchOpenKeys( - "/vola/bucketa1/dira3/dira31/dira32/nonexistentfile", 20, ""); + response = omdbInsightEndpoint + .getOpenKeyInfo(20, "", "/vola/bucketa1/dira3/dira31/dira32/nonexistentfile", true, true); assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), @@ -416,7 +412,7 @@ public void testSearchUnderNestedDirectory() throws IOException { @Test public void testLimitSearch() throws IOException { Response response = - omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1", 2, ""); + omdbInsightEndpoint.getOpenKeyInfo(2, "", "/vola/bucketa1", true, true); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); @@ -428,8 +424,8 @@ public void testLimitSearch() throws IOException { public void testSearchOpenKeysWithBadRequest() throws IOException { // Give a negative limit int negativeLimit = -1; - Response response = omdbInsightSearchEndpoint.searchOpenKeys("@323232", negativeLimit, ""); - + Response response = omdbInsightEndpoint + .getOpenKeyInfo(negativeLimit, "", "@323232", true, true); // Then the response should indicate that the request was bad assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus(), "Expected a 400 BAD REQUEST status"); @@ -438,7 +434,7 @@ public void testSearchOpenKeysWithBadRequest() throws IOException { assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), "Expected a message indicating the path must be at the bucket level or deeper"); - response = omdbInsightSearchEndpoint.searchOpenKeys("///", 20, ""); + response = omdbInsightEndpoint.getOpenKeyInfo(20, "", "///", true, true); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); entity = (String) response.getEntity(); assertTrue(entity.contains("Invalid startPrefix: Path must be at the bucket level or deeper"), @@ -447,8 +443,8 @@ public void testSearchOpenKeysWithBadRequest() throws IOException { @Test public void testLastKeyInResponse() throws IOException { - Response response = - omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb1", 20, ""); + Response response = omdbInsightEndpoint + .getOpenKeyInfo(20, "", "/volb/bucketb1", true, true); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); @@ -470,7 +466,7 @@ public void testSearchOpenKeysWithPagination() throws IOException { String prevKey = ""; // Perform the first search request - Response response = omdbInsightSearchEndpoint.searchOpenKeys(startPrefix, limit, prevKey); + Response response = omdbInsightEndpoint.getOpenKeyInfo(limit, prevKey, startPrefix, true, true); assertEquals(200, response.getStatus()); KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(2, result.getNonFSOKeyInfoList().size()); @@ -481,7 +477,7 @@ public void testSearchOpenKeysWithPagination() throws IOException { assertNotNull(prevKey, "Last key should not be null"); // Perform the second search request using the last key - response = omdbInsightSearchEndpoint.searchOpenKeys(startPrefix, limit, prevKey); + response = omdbInsightEndpoint.getOpenKeyInfo(limit, prevKey, startPrefix, true, true); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(2, result.getNonFSOKeyInfoList().size()); @@ -492,7 +488,7 @@ public void testSearchOpenKeysWithPagination() throws IOException { assertNotNull(prevKey, "Last key should not be null"); // Perform the third search request using the last key - response = omdbInsightSearchEndpoint.searchOpenKeys(startPrefix, limit, prevKey); + response = omdbInsightEndpoint.getOpenKeyInfo(limit, prevKey, startPrefix, true, true); assertEquals(200, response.getStatus()); result = (KeyInsightInfoResponse) response.getEntity(); assertEquals(1, result.getNonFSOKeyInfoList().size()); @@ -504,13 +500,61 @@ public void testSearchOpenKeysWithPagination() throws IOException { @Test public void testSearchInEmptyBucket() throws IOException { // Search in empty bucket bucketb2 - Response response = omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb2", 20, ""); + Response response = omdbInsightEndpoint.getOpenKeyInfo(20, "", "/volb/bucketb2", true, true); assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); String entity = (String) response.getEntity(); assertTrue(entity.contains("No keys matched the search prefix"), "Expected a message indicating no keys were found"); } + @Test + public void testSearchWithPrevKeyOnly() throws IOException { + String prevKey = "/volb/bucketb1/fileb1"; // Key exists in volb/bucketb1 + Response response = omdbInsightEndpoint.getOpenKeyInfo(4, prevKey, "", true, true); + + assertEquals(200, response.getStatus()); + + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(4, result.getNonFSOKeyInfoList().size(), "Expected 4 remaining keys after 'fileb1'"); + assertEquals("/volb/bucketb1/fileb5", result.getLastKey(), "Expected last key to be 'fileb5'"); + } + + @Test + public void testSearchWithEmptyPrevKeyAndStartPrefix() throws IOException { + Response response = omdbInsightEndpoint.getOpenKeyInfo(-1, "", "", true, true); + + assertEquals(200, response.getStatus()); + + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + // Assert all the keys are returned + assertEquals(12, result.getNonFSOKeyInfoList().size(), "Expected all keys to be returned"); + } + + @Test + public void testSearchWithStartPrefixOnly() throws IOException { + String startPrefix = "/volb/bucketb1/"; + Response response = omdbInsightEndpoint.getOpenKeyInfo(10, "", startPrefix, true, true); + + assertEquals(200, response.getStatus()); + + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(5, result.getNonFSOKeyInfoList().size(), "Expected 5 keys starting with 'fileb1'"); + assertEquals("/volb/bucketb1/fileb5", result.getLastKey(), "Expected last key to be 'fileb5'"); + } + + @Test + public void testSearchWithPrevKeyAndStartPrefix() throws IOException { + String startPrefix = "/volb/bucketb1/"; + String prevKey = "/volb/bucketb1/fileb1"; + Response response = omdbInsightEndpoint.getOpenKeyInfo(10, prevKey, startPrefix, true, true); + + assertEquals(200, response.getStatus()); + + KeyInsightInfoResponse result = (KeyInsightInfoResponse) response.getEntity(); + assertEquals(4, result.getNonFSOKeyInfoList().size(), "Expected 4 keys after 'fileb1'"); + assertEquals("/volb/bucketb1/fileb5", result.getLastKey(), "Expected last key to be 'fileb5'"); + } + /** * Tests the NSSummaryEndpoint for a given volume, bucket, and directory structure. * The test setup mimics the following filesystem structure with specified sizes: @@ -566,7 +610,7 @@ private void populateOMDB() throws Exception { // Create Bucket in volb createBucket("volb", "bucketb1", 1000 + 1000 + 1000 + 1000 + 1000, - getOBSBucketLayout()); + getOBSBucketLayout()); createBucket("volb", "bucketb2", 0, getOBSBucketLayout()); // Empty Bucket // Create Bucket in volc