diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogController.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogController.java index 1d4972b9c0a1..dd43119ea6ed 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogController.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogController.java @@ -33,6 +33,7 @@ subcommands = { ContainerInfoCommand.class, ContainerLogParser.class, + DuplicateOpenContainersCommand.class, ListContainers.class }, description = "Tool to parse and store container logs from datanodes into a temporary SQLite database." + diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogParser.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogParser.java index e2d472e2e95f..cac2518b381f 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogParser.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogParser.java @@ -89,6 +89,7 @@ public Void call() throws Exception { parser.processLogEntries(path, cdd, threadCount); cdd.insertLatestContainerLogData(); + cdd.createIndexes(); out().println("Successfully parsed the log files and updated the respective tables"); return null; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/DuplicateOpenContainersCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/DuplicateOpenContainersCommand.java new file mode 100644 index 000000000000..820d5b5d2e41 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/DuplicateOpenContainersCommand.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.debug.logs.container; + +import java.nio.file.Path; +import java.util.concurrent.Callable; +import org.apache.hadoop.ozone.debug.logs.container.utils.ContainerDatanodeDatabase; +import picocli.CommandLine; + +/** + * Subcommand to list containers that have duplicate OPEN states. + */ + +@CommandLine.Command( + name = "duplicate-open", + description = "List all containers which have duplicate open states." + + "Outputs the container ID along with the count of OPEN state entries." +) +public class DuplicateOpenContainersCommand implements Callable { + + @CommandLine.ParentCommand + private ContainerLogController parent; + + @Override + public Void call() throws Exception { + Path dbPath = parent.resolveDbPath(); + if (dbPath == null) { + return null; + } + + ContainerDatanodeDatabase cdd = new ContainerDatanodeDatabase(dbPath.toString()); + cdd.findDuplicateOpenContainer(); + + return null; + } +} + diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java index 6bc2e4b13641..25a5e7c003e3 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java @@ -19,7 +19,6 @@ import java.nio.file.Path; import java.util.concurrent.Callable; -import org.apache.hadoop.hdds.cli.AbstractSubcommand; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.debug.logs.container.utils.ContainerDatanodeDatabase; import org.apache.hadoop.ozone.shell.ListOptions; @@ -34,7 +33,7 @@ name = "list", description = "Finds containers from the database based on the option provided." ) -public class ListContainers extends AbstractSubcommand implements Callable { +public class ListContainers implements Callable { @CommandLine.Option(names = {"--state"}, description = "Life cycle state of the container.", diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java index e05f70e7bb31..1d44605249c8 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java @@ -116,6 +116,34 @@ private void createContainerLogTable() throws SQLException { } } + public void createIndexes() throws SQLException { + try (Connection connection = getConnection(); + Statement stmt = connection.createStatement()) { + createIdxDclContainerStateTime(stmt); + createContainerLogIndex(stmt); + createIdxContainerlogContainerId(stmt); + } catch (SQLException e) { + throw new SQLException("Error while creating index: " + e.getMessage()); + } catch (Exception e) { + throw new RuntimeException("Unexpected error: " + e); + } + } + + private void createIdxDclContainerStateTime(Statement stmt) throws SQLException { + String createIndexSQL = SQLDBConstants.CREATE_DCL_CONTAINER_STATE_TIME_INDEX; + stmt.execute(createIndexSQL); + } + + private void createContainerLogIndex(Statement stmt) throws SQLException { + String createIndexSQL = SQLDBConstants.CREATE_INDEX_LATEST_STATE; + stmt.execute(createIndexSQL); + } + + private void createIdxContainerlogContainerId(Statement stmt) throws SQLException { + String createIndexSQL = SQLDBConstants.CREATE_CONTAINER_ID_INDEX; + stmt.execute(createIndexSQL); + } + /** * Inserts a list of container log entries into the DatanodeContainerLogTable. * @@ -228,11 +256,6 @@ private void dropTable(String tableName, Statement stmt) throws SQLException { stmt.executeUpdate(dropTableSQL); } - private void createContainerLogIndex(Statement stmt) throws SQLException { - String createIndexSQL = SQLDBConstants.CREATE_INDEX_LATEST_STATE; - stmt.execute(createIndexSQL); - } - /** * Lists containers filtered by the specified state and writes their details to stdout * unless redirected to a file explicitly. @@ -255,10 +278,7 @@ public void listContainersByState(String state, Integer limit) throws SQLExcepti String baseQuery = SQLDBConstants.SELECT_LATEST_CONTAINER_LOGS_BY_STATE; String finalQuery = limitProvided ? baseQuery + " LIMIT ?" : baseQuery; - try (Connection connection = getConnection(); - Statement stmt = connection.createStatement()) { - - createContainerLogIndex(stmt); + try (Connection connection = getConnection()) { try (PreparedStatement pstmt = connection.prepareStatement(finalQuery)) { pstmt.setString(1, state); @@ -304,13 +324,6 @@ public void listContainersByState(String state, Integer limit) throws SQLExcepti } } - private void createIdxDclContainerStateTime(Connection conn) throws SQLException { - String sql = SQLDBConstants.CREATE_DCL_CONTAINER_STATE_TIME_INDEX; - try (Statement stmt = conn.createStatement()) { - stmt.execute(sql); - } - } - /** * Displays detailed information about a container based on its ID, including its state, BCSID, * timestamp, message, and index value. It also checks for issues such as UNHEALTHY @@ -323,7 +336,7 @@ private void createIdxDclContainerStateTime(Connection conn) throws SQLException public void showContainerDetails(Long containerID) throws SQLException { try (Connection connection = getConnection()) { - createIdxDclContainerStateTime(connection); + List logEntries = getContainerLogData(containerID, connection); if (logEntries.isEmpty()) { @@ -541,5 +554,60 @@ private List getContainerLogData(Long containerID, Connec return logEntries; } + + public void findDuplicateOpenContainer() throws SQLException { + String sql = SQLDBConstants.SELECT_DISTINCT_CONTAINER_IDS_QUERY; + + try (Connection connection = getConnection()) { + + try (PreparedStatement statement = connection.prepareStatement(sql); + ResultSet resultSet = statement.executeQuery()) { + int count = 0; + + while (resultSet.next()) { + Long containerID = resultSet.getLong("container_id"); + List logEntries = getContainerLogDataForOpenContainers(containerID, connection); + boolean hasIssue = checkForMultipleOpenStates(logEntries); + if (hasIssue) { + int openStateCount = (int) logEntries.stream() + .filter(entry -> "OPEN".equalsIgnoreCase(entry.getState())) + .count(); + count++; + out.println("Container ID: " + containerID + " - OPEN state count: " + openStateCount); + } + } + + out.println("Total containers that might have duplicate OPEN state : " + count); + + } + } catch (SQLException e) { + throw new SQLException("Error while retrieving containers." + e.getMessage(), e); + } catch (Exception e) { + throw new RuntimeException("Unexpected error: " + e); + } + } + + private List getContainerLogDataForOpenContainers(Long containerID, Connection connection) + throws SQLException { + String query = SQLDBConstants.SELECT_CONTAINER_DETAILS_OPEN_STATE; + List logEntries = new ArrayList<>(); + + try (PreparedStatement preparedStatement = connection.prepareStatement(query)) { + preparedStatement.setLong(1, containerID); + try (ResultSet rs = preparedStatement.executeQuery()) { + while (rs.next()) { + DatanodeContainerInfo entry = new DatanodeContainerInfo.Builder() + .setTimestamp(rs.getString("timestamp")) + .setContainerId(rs.getLong("container_id")) + .setDatanodeId(rs.getString("datanode_id")) + .setState(rs.getString("container_state")) + .build(); + logEntries.add(entry); + } + } + } + + return logEntries; + } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/SQLDBConstants.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/SQLDBConstants.java index c58b605a2f64..74822d35e3d3 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/SQLDBConstants.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/SQLDBConstants.java @@ -68,6 +68,13 @@ public final class SQLDBConstants { "WHERE d.container_id = ? ORDER BY d.datanode_id ASC, d.timestamp ASC;"; public static final String CREATE_DCL_CONTAINER_STATE_TIME_INDEX = "CREATE INDEX IF NOT EXISTS " + "idx_dcl_container_state_time ON DatanodeContainerLogTable(container_id, container_state, timestamp);"; + public static final String CREATE_CONTAINER_ID_INDEX = "CREATE INDEX IF NOT EXISTS idx_containerlog_container_id " + + "ON ContainerLogTable(container_id);"; + public static final String SELECT_DISTINCT_CONTAINER_IDS_QUERY = + "SELECT DISTINCT container_id FROM ContainerLogTable"; + public static final String SELECT_CONTAINER_DETAILS_OPEN_STATE = "SELECT d.timestamp, d.container_id, " + + "d.datanode_id, d.container_state FROM DatanodeContainerLogTable d " + + "WHERE d.container_id = ? AND d.container_state = 'OPEN' ORDER BY d.timestamp ASC;"; private SQLDBConstants() { //Never constructed