diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/containerlog/parser/ContainerDatanodeDatabase.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/containerlog/parser/ContainerDatanodeDatabase.java index 19f7402518a..9dcf3fb7187 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/containerlog/parser/ContainerDatanodeDatabase.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/containerlog/parser/ContainerDatanodeDatabase.java @@ -17,8 +17,9 @@ package org.apache.hadoop.ozone.containerlog.parser; -import java.io.FileNotFoundException; -import java.io.InputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; @@ -26,11 +27,6 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.sqlite.SQLiteConfig; /** @@ -38,37 +34,20 @@ * Provides methods for table creation, log data insertion, and index setup. */ public class ContainerDatanodeDatabase { - - private static Map queries; - - static { - loadProperties(); - } - - private static final Logger LOG = - LoggerFactory.getLogger(ContainerDatanodeDatabase.class); - - private static void loadProperties() { - Properties props = new Properties(); - try (InputStream inputStream = ContainerDatanodeDatabase.class.getClassLoader() - .getResourceAsStream(DBConsts.PROPS_FILE)) { - - if (inputStream != null) { - props.load(inputStream); - queries = props.entrySet().stream() - .collect(Collectors.toMap( - e -> e.getKey().toString(), - e -> e.getValue().toString() - )); - } else { - throw new FileNotFoundException("Property file '" + DBConsts.PROPS_FILE + "' not found."); - } - } catch (Exception e) { - LOG.error(e.getMessage()); + + private static String databasePath; + + public static void setDatabasePath(String dbPath) { + if (databasePath == null) { + databasePath = dbPath; } } private static Connection getConnection() throws Exception { + if (databasePath == null) { + throw new IllegalStateException("Database path not set"); + } + Class.forName(DBConsts.DRIVER); SQLiteConfig config = new SQLiteConfig(); @@ -79,11 +58,11 @@ private static Connection getConnection() throws Exception { config.setSynchronous(SQLiteConfig.SynchronousMode.OFF); config.setTempStore(SQLiteConfig.TempStore.MEMORY); - return DriverManager.getConnection(DBConsts.CONNECTION_PREFIX + DBConsts.DATABASE_NAME, config.toProperties()); + return DriverManager.getConnection(DBConsts.CONNECTION_PREFIX + databasePath, config.toProperties()); } public void createDatanodeContainerLogTable() throws SQLException { - String createTableSQL = queries.get("CREATE_DATANODE_CONTAINER_LOG_TABLE"); + String createTableSQL = DBConsts.CREATE_DATANODE_CONTAINER_LOG_TABLE; try (Connection connection = getConnection(); Statement dropStmt = connection.createStatement(); Statement createStmt = connection.createStatement()) { @@ -91,26 +70,26 @@ public void createDatanodeContainerLogTable() throws SQLException { createStmt.execute(createTableSQL); createDatanodeContainerIndex(createStmt); } catch (SQLException e) { - LOG.error("Error while creating the table: {}", e.getMessage()); + System.err.println("Error while creating the table: " + e.getMessage()); throw e; } catch (Exception e) { - LOG.error(e.getMessage()); + System.err.println("Unexpected error: " + e.getMessage()); throw new RuntimeException(e); } } private void createContainerLogTable() throws SQLException { - String createTableSQL = queries.get("CREATE_CONTAINER_LOG_TABLE"); + String createTableSQL = DBConsts.CREATE_CONTAINER_LOG_TABLE; try (Connection connection = getConnection(); Statement dropStmt = connection.createStatement(); Statement createStmt = connection.createStatement()) { dropTable(DBConsts.CONTAINER_LOG_TABLE_NAME, dropStmt); createStmt.execute(createTableSQL); } catch (SQLException e) { - LOG.error("Error while creating the table: {}", e.getMessage()); + System.err.println("Error while creating the table: " + e.getMessage()); throw e; } catch (Exception e) { - LOG.error(e.getMessage()); + System.err.println("Unexpected error: " + e.getMessage()); throw new RuntimeException(e); } } @@ -123,7 +102,7 @@ private void createContainerLogTable() throws SQLException { public synchronized void insertContainerDatanodeData(List transitionList) throws SQLException { - String insertSQL = queries.get("INSERT_DATANODE_CONTAINER_LOG"); + String insertSQL = DBConsts.INSERT_DATANODE_CONTAINER_LOG; long containerId = 0; String datanodeId = null; @@ -159,16 +138,16 @@ public synchronized void insertContainerDatanodeData(List preparedStatement.executeBatch(); } } catch (SQLException e) { - LOG.error("Failed to insert container log for container {} on datanode {}", containerId, datanodeId, e); + System.err.println("Failed to insert container log for container " + containerId + " on datanode " + datanodeId); throw e; } catch (Exception e) { - LOG.error(e.getMessage()); + System.err.println("Unexpected error: " + e.getMessage()); throw new RuntimeException(e); } } private void createDatanodeContainerIndex(Statement stmt) throws SQLException { - String createIndexSQL = queries.get("CREATE_DATANODE_CONTAINER_INDEX"); + String createIndexSQL = DBConsts.CREATE_DATANODE_CONTAINER_INDEX; stmt.execute(createIndexSQL); } @@ -179,8 +158,8 @@ private void createDatanodeContainerIndex(Statement stmt) throws SQLException { public void insertLatestContainerLogData() throws SQLException { createContainerLogTable(); - String selectSQL = queries.get("SELECT_LATEST_CONTAINER_LOG"); - String insertSQL = queries.get("INSERT_CONTAINER_LOG"); + String selectSQL = DBConsts.SELECT_LATEST_CONTAINER_LOG; + String insertSQL = DBConsts.INSERT_CONTAINER_LOG; try (Connection connection = getConnection(); PreparedStatement selectStmt = connection.prepareStatement(selectSQL); @@ -208,8 +187,8 @@ public void insertLatestContainerLogData() throws SQLException { count = 0; } } catch (SQLException e) { - LOG.error("Failed to insert container log entry for container {} on datanode {} ", - containerId, datanodeId, e); + System.err.println("Failed to insert container log entry for container " + containerId + " on datanode " + + datanodeId); throw e; } } @@ -218,18 +197,95 @@ public void insertLatestContainerLogData() throws SQLException { insertStmt.executeBatch(); } } catch (SQLException e) { - LOG.error("Failed to insert container log entry: {}", e.getMessage()); + System.err.println("Failed to insert container log entry: " + e.getMessage()); throw e; } catch (Exception e) { - LOG.error(e.getMessage()); + System.err.println("Unexpected error: " + e.getMessage()); throw new RuntimeException(e); } } private void dropTable(String tableName, Statement stmt) throws SQLException { - String dropTableSQL = queries.get("DROP_TABLE").replace("{table_name}", tableName); + String dropTableSQL = DBConsts.DROP_TABLE.replace("{table_name}", tableName); stmt.executeUpdate(dropTableSQL); } + private void createContainerLogIndex(Statement stmt) throws SQLException { + String createIndexSQL = DBConsts.CREATE_INDEX_LATEST_STATE; + stmt.execute(createIndexSQL); + } + + /** + * Lists containers filtered by the specified state and writes their details to stdout + * unless redirected to a file explicitly. + * The output includes timestamp, datanode ID, container ID, BCSID, error message, and index value, + * written in a human-readable table format to a file or console. + * Behavior based on the {@code limit} parameter: + * If {@code limit} is provided, only up to the specified number of rows are printed. + * If the number of matching containers exceeds the {@code limit}, + * a note is printed indicating more containers exist. + * + * @param state the container state to filter by (e.g., "OPEN", "CLOSED", "QUASI_CLOSED") + * @param limit the maximum number of rows to display; use {@link Integer#MAX_VALUE} to fetch all rows + */ + + public void listContainersByState(String state, Integer limit) throws SQLException { + int count = 0; + + boolean limitProvided = limit != Integer.MAX_VALUE; + + String baseQuery = DBConsts.SELECT_LATEST_CONTAINER_LOGS_BY_STATE; + String finalQuery = limitProvided ? baseQuery + " LIMIT ?" : baseQuery; + + try (Connection connection = getConnection(); + Statement stmt = connection.createStatement()) { + + createContainerLogIndex(stmt); + + try (PreparedStatement pstmt = connection.prepareStatement(finalQuery)) { + pstmt.setString(1, state); + if (limitProvided) { + pstmt.setInt(2, limit + 1); + } + + try (ResultSet rs = pstmt.executeQuery(); + PrintWriter writer = new PrintWriter(new OutputStreamWriter(System.out, + StandardCharsets.UTF_8), true)) { + + writer.printf("%-25s | %-35s | %-15s | %-15s | %-40s | %-12s%n", + "Timestamp", "Datanode ID", "Container ID", "BCSID", "Message", "Index Value"); + writer.println("-------------------------------------------------------------------------------------" + + "---------------------------------------------------------------------------------------"); + + while (rs.next()) { + if (limitProvided && count >= limit) { + writer.println("Note: There might be more containers. Use -all option to list all entries"); + break; + } + String timestamp = rs.getString("timestamp"); + String datanodeId = rs.getString("datanode_id"); + long containerId = rs.getLong("container_id"); + long latestBcsid = rs.getLong("latest_bcsid"); + String errorMessage = rs.getString("error_message"); + int indexValue = rs.getInt("index_value"); + count++; + + writer.printf("%-25s | %-35s | %-15d | %-15d | %-40s | %-12d%n", + timestamp, datanodeId, containerId, latestBcsid, errorMessage, indexValue); + } + + if (count == 0) { + writer.printf("No containers found for state: %s%n", state); + } else { + writer.printf("Number of containers listed: %d%n", count); + } + } + } + } catch (SQLException e) { + throw new SQLException("Error while retrieving containers with state " + state); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/containerlog/parser/ContainerLogFileParser.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/containerlog/parser/ContainerLogFileParser.java index bafce798e5a..c41beb328f4 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/containerlog/parser/ContainerLogFileParser.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/containerlog/parser/ContainerLogFileParser.java @@ -62,7 +62,7 @@ public class ContainerLogFileParser { */ public void processLogEntries(String logDirectoryPath, ContainerDatanodeDatabase dbstore, int threadCount) - throws SQLException { + throws SQLException, IOException, InterruptedException { try (Stream paths = Files.walk(Paths.get(logDirectoryPath))) { List files = paths.filter(Files::isRegularFile).collect(Collectors.toList()); @@ -116,10 +116,6 @@ public void processLogEntries(String logDirectoryPath, ContainerDatanodeDatabase throw new SQLException("Log file processing failed."); } - } catch (IOException | InterruptedException e) { - e.printStackTrace(); - } catch (NumberFormatException e) { - System.err.println("Invalid datanode ID"); } } @@ -133,7 +129,7 @@ public void processLogEntries(String logDirectoryPath, ContainerDatanodeDatabase */ private void processFile(String logFilePath, ContainerDatanodeDatabase dbstore, String datanodeId) - throws SQLException { + throws SQLException, IOException { List batchList = new ArrayList<>(MAX_OBJ_IN_LIST + 100); try (BufferedReader reader = Files.newBufferedReader(Paths.get(logFilePath), StandardCharsets.UTF_8)) { @@ -199,11 +195,7 @@ private void processFile(String logFilePath, ContainerDatanodeDatabase dbstore, batchList.clear(); } } catch (SQLException e) { - throw new SQLException(e.getMessage()); - } catch (Exception e) { - System.err.println( - "Error processing the batch for container: " + id + " at datanode: " + datanodeId); - e.printStackTrace(); + throw e; } } else { System.err.println("Log line does not have all required fields: " + line); @@ -214,8 +206,6 @@ private void processFile(String logFilePath, ContainerDatanodeDatabase dbstore, batchList.clear(); } - } catch (IOException e) { - e.printStackTrace(); } } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/containerlog/parser/DBConsts.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/containerlog/parser/DBConsts.java index d4b3c5a3204..5cef4a335d9 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/containerlog/parser/DBConsts.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/containerlog/parser/DBConsts.java @@ -21,16 +21,49 @@ * Constants used for ContainerDatanodeDatabase. */ public final class DBConsts { - + + public static final String DEFAULT_DB_FILENAME = "container_datanode.db"; public static final String DRIVER = "org.sqlite.JDBC"; public static final String CONNECTION_PREFIX = "jdbc:sqlite:"; - public static final String DATABASE_NAME = "container_datanode.db"; - public static final String PROPS_FILE = "container-log-db-queries.properties"; public static final int CACHE_SIZE = 1000000; public static final int BATCH_SIZE = 2500; public static final String DATANODE_CONTAINER_LOG_TABLE_NAME = "DatanodeContainerLogTable"; public static final String CONTAINER_LOG_TABLE_NAME = "ContainerLogTable"; - + public static final String CREATE_DATANODE_CONTAINER_LOG_TABLE = + "CREATE TABLE IF NOT EXISTS DatanodeContainerLogTable (datanode_id TEXT NOT NULL, " + + "container_id INTEGER NOT NULL, timestamp TEXT NOT NULL, container_state TEXT, bcsid INTEGER, " + + "error_message TEXT, log_level TEXT NOT NULL," + + " index_value INTEGER);"; + public static final String CREATE_CONTAINER_LOG_TABLE = + "CREATE TABLE IF NOT EXISTS ContainerLogTable (datanode_id TEXT NOT NULL, container_id INTEGER NOT NULL," + + " latest_state TEXT, latest_bcsid INTEGER, PRIMARY KEY (datanode_id, container_id));"; + public static final String CREATE_DATANODE_CONTAINER_INDEX = + "CREATE INDEX IF NOT EXISTS idx_datanode_container ON DatanodeContainerLogTable (datanode_id," + + " container_id, timestamp);"; + public static final String INSERT_DATANODE_CONTAINER_LOG = + "INSERT INTO DatanodeContainerLogTable (datanode_id, container_id, timestamp, container_state, bcsid," + + " error_message, log_level, index_value) VALUES (?, ?, ?, ?, ?, ?, ?, ?);"; + public static final String INSERT_CONTAINER_LOG = + "INSERT OR REPLACE INTO ContainerLogTable (datanode_id, container_id, latest_state," + + " latest_bcsid) VALUES (?, ?, ?, ?);"; + public static final String SELECT_LATEST_CONTAINER_LOG = + "SELECT a.datanode_id, a.container_id, a.container_state, a.bcsid, a.timestamp FROM DatanodeContainerLogTable" + + " AS a JOIN (SELECT datanode_id, container_id, MAX(timestamp) as timestamp FROM DatanodeContainerLogTable" + + " GROUP BY datanode_id, container_id) as b ON a.datanode_id = b.datanode_id AND " + + "a.container_id = b.container_id AND a.timestamp=b.timestamp;"; + public static final String DROP_TABLE = "DROP TABLE IF EXISTS {table_name};"; + public static final String CREATE_INDEX_LATEST_STATE = + "CREATE INDEX IF NOT EXISTS idx_container_log_state ON ContainerLogTable(latest_state);"; + public static final String SELECT_LATEST_CONTAINER_LOGS_BY_STATE = + "SELECT cl.datanode_id, cl.container_id, cl.latest_state, cl.latest_bcsid, dcl.error_message, dcl.index_value," + + " dcl.timestamp FROM ContainerLogTable cl LEFT JOIN DatanodeContainerLogTable dcl ON" + + " cl.datanode_id = dcl.datanode_id AND cl.container_id = dcl.container_id AND cl.latest_bcsid = dcl.bcsid " + + "AND cl.latest_state = dcl.container_state WHERE cl.latest_state = ? " + + "AND dcl.timestamp = (SELECT MAX(timestamp) FROM DatanodeContainerLogTable sub_dcl " + + "WHERE sub_dcl.datanode_id = cl.datanode_id AND" + + " sub_dcl.container_id = cl.container_id AND sub_dcl.bcsid = cl.latest_bcsid" + + " AND sub_dcl.container_state = cl.latest_state)"; + private DBConsts() { //Never constructed } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerLogController.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/LogParser.java similarity index 67% rename from hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerLogController.java rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/LogParser.java index 1043d891470..20847dde18b 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerLogController.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/LogParser.java @@ -15,24 +15,25 @@ * limitations under the License. */ -package org.apache.hadoop.ozone.debug.container; +package org.apache.hadoop.ozone.debug.logs; import org.apache.hadoop.hdds.cli.DebugSubcommand; +import org.apache.hadoop.ozone.debug.logs.container.ContainerLogController; import org.kohsuke.MetaInfServices; import picocli.CommandLine; /** - * A controller for managing container log operations like parsing and listing containers. + * Entry point for Ozone debug log parsing and analysis commands. */ @CommandLine.Command( - name = "container", + name = "log", subcommands = { - ContainerLogParser.class + ContainerLogController.class }, - description = "Parse, Store, Retrieve" + description = "This serves as a common entry point for all commands that parse and analyze logs," + + "regardless of their source or type and require logs to be extracted first." ) @MetaInfServices(DebugSubcommand.class) -public class ContainerLogController implements DebugSubcommand { - +public class LogParser implements DebugSubcommand { } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogController.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogController.java new file mode 100644 index 00000000000..4ba220c9d45 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogController.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.debug.logs.container; + +import picocli.CommandLine; + +/** + * A controller for managing container log operations like parsing and listing containers. + */ + +@CommandLine.Command( + name = "container", + subcommands = { + ContainerLogParser.class, + ListContainers.class + }, + description = "Tool to parse and store container logs from datanodes into a temporary SQLite database." + + " Supports querying state transitions of container replicas using various subcommands." +) + +public class ContainerLogController { + @CommandLine.Option(names = {"--db"}, + scope = CommandLine.ScopeType.INHERIT, + description = "Path to the SQLite database file where the parsed information from logs is stored.") + private String dbPath; + + public String getDbPath() { + return dbPath; + } + + public void setDbPath(String dbPath) { + this.dbPath = dbPath; + } +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerLogParser.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogParser.java similarity index 51% rename from hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerLogParser.java rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogParser.java index c9ef86d5dd0..df1288543bf 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerLogParser.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogParser.java @@ -15,15 +15,15 @@ * limitations under the License. */ -package org.apache.hadoop.ozone.debug.container; +package org.apache.hadoop.ozone.debug.logs.container; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.sql.SQLException; import java.util.concurrent.Callable; import org.apache.hadoop.ozone.containerlog.parser.ContainerDatanodeDatabase; import org.apache.hadoop.ozone.containerlog.parser.ContainerLogFileParser; +import org.apache.hadoop.ozone.containerlog.parser.DBConsts; import picocli.CommandLine; /** @@ -31,18 +31,20 @@ */ @CommandLine.Command( - name = "container_log_parse", - description = "parse the container logs" + name = "parse", + description = "Parse container logs and extract key details such as datanode ID, container ID, state, " + + "BCSID, timestamp, log level, index value, and messages (if any)." ) public class ContainerLogParser implements Callable { private static final int DEFAULT_THREAD_COUNT = 10; - @CommandLine.Option(names = {"--parse"}, - description = "path to the dir which contains log files") + @CommandLine.Option(names = {"--path"}, + description = "Path to the folder which contains container log files to be parsed.", + required = true) private String path; @CommandLine.Option(names = {"--thread-count"}, - description = "Thread count for concurrent processing.", + description = "Thread count for concurrent log file processing.", defaultValue = "10") private int threadCount; @@ -57,36 +59,41 @@ public Void call() throws Exception { threadCount = DEFAULT_THREAD_COUNT; } - if (path != null) { - Path logPath = Paths.get(path); - if (!Files.exists(logPath) || !Files.isDirectory(logPath)) { - System.err.println("Invalid path provided: " + path); - return null; - } - - ContainerDatanodeDatabase cdd = new ContainerDatanodeDatabase(); - ContainerLogFileParser parser = new ContainerLogFileParser(); - - try { + Path logPath = Paths.get(path); + if (!Files.exists(logPath) || !Files.isDirectory(logPath)) { + System.err.println("Invalid path provided: " + path); + return null; + } + + Path providedDbPath; + if (parent.getDbPath() == null) { + providedDbPath = Paths.get(System.getProperty("user.dir"), DBConsts.DEFAULT_DB_FILENAME); - cdd.createDatanodeContainerLogTable(); + System.out.println("No database path provided. Creating new database at: " + providedDbPath); + } else { + providedDbPath = Paths.get(parent.getDbPath()); + Path parentDir = providedDbPath.getParent(); - parser.processLogEntries(path, cdd, threadCount); + if (parentDir != null && !Files.exists(parentDir)) { + System.err.println("The parent directory of the provided database path does not exist: " + parentDir); + return null; + } + } + + ContainerDatanodeDatabase.setDatabasePath(providedDbPath.toString()); + ContainerDatanodeDatabase cdd = new ContainerDatanodeDatabase(); + ContainerLogFileParser parser = new ContainerLogFileParser(); - cdd.insertLatestContainerLogData(); - System.out.println("Successfully parsed the log files and updated the respective tables"); + try { + cdd.createDatanodeContainerLogTable(); - } catch (SQLException e) { - System.err.println("Error occurred while processing logs or inserting data into the database: " - + e.getMessage()); - } catch (Exception e) { - System.err.println("An unexpected error occurred: " + e.getMessage()); - } + parser.processLogEntries(path, cdd, threadCount); - } else { - System.out.println("path to logs folder not provided"); + cdd.insertLatestContainerLogData(); + System.out.println("Successfully parsed the log files and updated the respective tables"); + } catch (Exception e) { + throw e; } - return null; } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java new file mode 100644 index 00000000000..7bfa66a742d --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.debug.logs.container; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.Callable; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.containerlog.parser.ContainerDatanodeDatabase; +import org.apache.hadoop.ozone.containerlog.parser.DBConsts; +import org.apache.hadoop.ozone.shell.ListOptions; +import picocli.CommandLine; + + +/** + * List containers based on the parameter given. + */ + +@CommandLine.Command( + name = "list", + description = "Finds containers from the database based on the option provided." +) +public class ListContainers implements Callable { + + @CommandLine.Option(names = {"--state"}, + description = "Life cycle state of the container.", + required = true) + private HddsProtos.LifeCycleState state; + + @CommandLine.Mixin + private ListOptions listOptions; + + @CommandLine.ParentCommand + private ContainerLogController parent; + + @Override + public Void call() throws Exception { + Path providedDbPath; + if (parent.getDbPath() == null) { + providedDbPath = Paths.get(System.getProperty("user.dir"), DBConsts.DEFAULT_DB_FILENAME); + + if (Files.exists(providedDbPath) && Files.isRegularFile(providedDbPath)) { + System.out.println("Using default database file found in current directory: " + providedDbPath); + } else { + System.err.println("No database path provided and default file '" + DBConsts.DEFAULT_DB_FILENAME + "' not " + + "found in current directory. Please provide a valid database path"); + return null; + } + } else { + providedDbPath = Paths.get(parent.getDbPath()); + Path parentDir = providedDbPath.getParent(); + + if (parentDir != null && !Files.exists(parentDir)) { + System.err.println("The parent directory of the provided database path does not exist: " + parentDir); + return null; + } + } + + ContainerDatanodeDatabase.setDatabasePath(providedDbPath.toString()); + + ContainerDatanodeDatabase cdd = new ContainerDatanodeDatabase(); + try { + cdd.listContainersByState(state.name(), listOptions.getLimit()); + } catch (Exception e) { + throw e; + } + + return null; + } +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/package-info.java similarity index 86% rename from hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/package-info.java rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/package-info.java index 9354dfe28b8..a4a7260a41c 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/package-info.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/package-info.java @@ -16,7 +16,8 @@ */ /** - * Provides functionality for managing container log operations, including parsing, processing, and storing log data. + * Provides functionality for managing container log operations, including parsing, processing, + * storing extracted data into DB and analysis. */ -package org.apache.hadoop.ozone.debug.container; +package org.apache.hadoop.ozone.debug.logs.container; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/package-info.java new file mode 100644 index 00000000000..2609b76dfae --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Provides commands for parsing and analyzing extracted logs. + */ + +package org.apache.hadoop.ozone.debug.logs; diff --git a/hadoop-ozone/tools/src/main/resources/container-log-db-queries.properties b/hadoop-ozone/tools/src/main/resources/container-log-db-queries.properties deleted file mode 100644 index 01a40555670..00000000000 --- a/hadoop-ozone/tools/src/main/resources/container-log-db-queries.properties +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -CREATE_DATANODE_CONTAINER_LOG_TABLE=CREATE TABLE IF NOT EXISTS DatanodeContainerLogTable (datanode_id TEXT NOT NULL, container_id INTEGER NOT NULL, timestamp TEXT NOT NULL, container_state TEXT, bcsid INTEGER, error_message TEXT, log_level TEXT NOT NULL, index_value INTEGER); -CREATE_CONTAINER_LOG_TABLE=CREATE TABLE IF NOT EXISTS ContainerLogTable (datanode_id TEXT NOT NULL, container_id INTEGER NOT NULL, latest_state TEXT, latest_bcsid INTEGER, PRIMARY KEY (datanode_id, container_id)); -CREATE_DATANODE_CONTAINER_INDEX=CREATE INDEX IF NOT EXISTS idx_datanode_container ON DatanodeContainerLogTable (datanode_id, container_id, timestamp); -INSERT_DATANODE_CONTAINER_LOG=INSERT INTO DatanodeContainerLogTable (datanode_id, container_id, timestamp, container_state, bcsid, error_message, log_level, index_value) VALUES (?, ?, ?, ?, ?, ?, ?, ?); -INSERT_CONTAINER_LOG=INSERT OR REPLACE INTO ContainerLogTable (datanode_id, container_id, latest_state, latest_bcsid) VALUES (?, ?, ?, ?); -SELECT_LATEST_CONTAINER_LOG=SELECT a.datanode_id, a.container_id, a.container_state, a.bcsid, a.timestamp FROM DatanodeContainerLogTable AS a JOIN (SELECT datanode_id, container_id, MAX(timestamp) as timestamp FROM DatanodeContainerLogTable GROUP BY datanode_id, container_id) as b ON a.datanode_id = b.datanode_id AND a.container_id = b.container_id AND a.timestamp=b.timestamp; -DROP_TABLE=DROP TABLE IF EXISTS {table_name};