diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieCLI.java b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieCLI.java index d2e6f99525c77..0dafdc4fbe22c 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieCLI.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieCLI.java @@ -25,6 +25,9 @@ import org.apache.hudi.common.util.ConsistencyGuardConfig; import org.apache.hudi.common.util.FSUtils; +/** + * This class is responsible to load table metadata and hoodie related configs. + */ public class HoodieCLI { public static Configuration conf; @@ -35,6 +38,9 @@ public class HoodieCLI { public static HoodieTableMetaClient tableMetadata; public static HoodieTableMetaClient syncTableMetadata; + /** + * Enum for CLI state. + */ public enum CLIState { INIT, DATASET, SYNC } diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieHistoryFileNameProvider.java b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieHistoryFileNameProvider.java index eeb2ff2a93ce2..af01f6675d620 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieHistoryFileNameProvider.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieHistoryFileNameProvider.java @@ -23,6 +23,9 @@ import org.springframework.shell.plugin.support.DefaultHistoryFileNameProvider; import org.springframework.stereotype.Component; +/** + * CLI history file provider. + */ @Component @Order(Ordered.HIGHEST_PRECEDENCE) public class HoodieHistoryFileNameProvider extends DefaultHistoryFileNameProvider { diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/HoodiePrintHelper.java b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodiePrintHelper.java index 3cce3014049cf..0e48911fb6352 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/HoodiePrintHelper.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodiePrintHelper.java @@ -25,12 +25,12 @@ import org.apache.hudi.common.util.Option; /** - * Helper class to render table for hoodie-cli + * Helper class to render table for hoodie-cli. */ public class HoodiePrintHelper { /** - * Print header and raw rows + * Print header and raw rows. * * @param header Header * @param rows Raw Rows @@ -41,7 +41,7 @@ public static String print(String[] header, String[][] rows) { } /** - * Serialize Table to printable string + * Serialize Table to printable string. * * @param rowHeader Row Header * @param fieldNameToConverterMap Field Specific Converters @@ -71,7 +71,7 @@ public static String print(TableHeader rowHeader, Map> fieldN } /** - * Main API to add row to the table + * Main API to add row to the table. * * @param row Row */ @@ -82,7 +82,7 @@ public Table add(List row) { } /** - * Add all rows + * Add all rows. * * @param rows Rows to be aded * @return @@ -93,7 +93,7 @@ public Table addAll(List> rows) { } /** - * Add all rows + * Add all rows. * * @param rows Rows to be added * @return @@ -104,7 +104,7 @@ public Table addAllRows(List rows) { } /** - * API to let the table know writing is over and reading is going to start + * API to let the table know writing is over and reading is going to start. */ public Table flip() { this.finishedAdding = true; @@ -113,7 +113,7 @@ public Table flip() { } /** - * Sorting of rows by a specified field + * Sorting of rows by a specified field. * * @return */ diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/TableHeader.java b/hudi-cli/src/main/java/org/apache/hudi/cli/TableHeader.java index e257e36a1609a..8ec392d1abfe3 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/TableHeader.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/TableHeader.java @@ -22,7 +22,7 @@ import java.util.List; /** - * Header for the table to be rendered + * Header for the table to be rendered. */ public class TableHeader { @@ -30,7 +30,7 @@ public class TableHeader { private final List fieldNames = new ArrayList<>(); /** - * Add a field (column) to table + * Add a field (column) to table. * * @param fieldName field Name */ @@ -40,14 +40,14 @@ public TableHeader addTableHeaderField(String fieldName) { } /** - * Get all field names + * Get all field names. */ public List getFieldNames() { return fieldNames; } /** - * Index of the field in the table + * Index of the field in the table. * * @param fieldName Field Name */ @@ -56,14 +56,14 @@ public int indexOf(String fieldName) { } /** - * Lookup field by offset + * Lookup field by offset. */ public String get(int index) { return fieldNames.get(index); } /** - * Get number of fields in the table + * Get number of fields in the table. */ public int getNumFields() { return fieldNames.size(); diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java index 9a0eff16decfa..cafe6412e58bf 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java @@ -45,6 +45,9 @@ import org.springframework.shell.core.annotation.CliOption; import org.springframework.stereotype.Component; +/** + * CLI command to display archived commits and stats if available. + */ @Component public class ArchivedCommitsCommand implements CommandMarker { diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CleansCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CleansCommand.java index 1257b172dd082..b5edeae4ae5ca 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CleansCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CleansCommand.java @@ -39,6 +39,9 @@ import org.springframework.shell.core.annotation.CliOption; import org.springframework.stereotype.Component; +/** + * CLI command to show cleans options. + */ @Component public class CleansCommand implements CommandMarker { diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java index ad948651fb001..4ae8e80b00e18 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java @@ -44,6 +44,9 @@ import org.springframework.shell.core.annotation.CliOption; import org.springframework.stereotype.Component; +/** + * CLI command to display commits options. + */ @Component public class CommitsCommand implements CommandMarker { diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java index 3d10592493f4f..8520912b272c6 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java @@ -59,6 +59,9 @@ import org.springframework.shell.core.annotation.CliOption; import org.springframework.stereotype.Component; +/** + * CLI command to display compaction related options. + */ @Component public class CompactionCommand implements CommandMarker { diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/DatasetsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/DatasetsCommand.java index b6b1a38ef732a..2ecafa93dc46a 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/DatasetsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/DatasetsCommand.java @@ -35,6 +35,9 @@ import org.springframework.shell.core.annotation.CliOption; import org.springframework.stereotype.Component; +/** + * CLI command to display hudi dataset options. + */ @Component public class DatasetsCommand implements CommandMarker { @@ -67,7 +70,7 @@ public String connect( } /** - * Create a Hoodie Table if it does not exist + * Create a Hoodie Table if it does not exist. * * @param path Base Path * @param name Hoodie Table Name @@ -113,7 +116,7 @@ public boolean isDescAvailable() { } /** - * Describes table properties + * Describes table properties. */ @CliCommand(value = "desc", help = "Describle Hoodie Table properties") public String descTable() { diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/FileSystemViewCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/FileSystemViewCommand.java index 6e9bde6ceb168..5d0ef50affa60 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/FileSystemViewCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/FileSystemViewCommand.java @@ -47,6 +47,9 @@ import org.springframework.shell.core.annotation.CliOption; import org.springframework.stereotype.Component; +/** + * CLI command to display file system options. + */ @Component public class FileSystemViewCommand implements CommandMarker { @@ -206,7 +209,7 @@ public String showLatestFileSlices( } /** - * Build File System View + * Build File System View. * * @param globRegex Path Regex * @param maxInstant Max Instants to be used for displaying file-instants diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HDFSParquetImportCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HDFSParquetImportCommand.java index e78b7356f1149..1765840db06f1 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HDFSParquetImportCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HDFSParquetImportCommand.java @@ -33,6 +33,9 @@ import org.springframework.stereotype.Component; import scala.collection.JavaConverters; +/** + * CLI command for importing parquet dataset to hudi dataset. + */ @Component public class HDFSParquetImportCommand implements CommandMarker { diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java index 927909fe534c7..8c0d5264ace53 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java @@ -59,6 +59,9 @@ import scala.Tuple2; import scala.Tuple3; +/** + * CLI command to display log file options. + */ @Component public class HoodieLogFileCommand implements CommandMarker { diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieSyncCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieSyncCommand.java index ecd15b891972c..024a57f2896ae 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieSyncCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieSyncCommand.java @@ -32,6 +32,9 @@ import org.springframework.shell.core.annotation.CliOption; import org.springframework.stereotype.Component; +/** + * CLI command to display sync options. + */ @Component public class HoodieSyncCommand implements CommandMarker { diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java index d0365b544ac78..3d18865953669 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java @@ -34,6 +34,9 @@ import org.springframework.shell.core.annotation.CliOption; import org.springframework.stereotype.Component; +/** + * CLI command to display and trigger repair options. + */ @Component public class RepairsCommand implements CommandMarker { diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RollbacksCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RollbacksCommand.java index 90a716f3cd1d4..0eac7ee6bb70e 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RollbacksCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RollbacksCommand.java @@ -42,6 +42,9 @@ import org.springframework.shell.core.annotation.CliOption; import org.springframework.stereotype.Component; +/** + * CLI command to display rollback options. + */ @Component public class RollbacksCommand implements CommandMarker { @@ -115,7 +118,7 @@ public String showRollback( } /** - * An Active timeline containing only rollbacks + * An Active timeline containing only rollbacks. */ class RollbackTimeline extends HoodieActiveTimeline { diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SavepointsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SavepointsCommand.java index c2430353809ae..248157868b87c 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SavepointsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SavepointsCommand.java @@ -40,6 +40,9 @@ import org.springframework.shell.core.annotation.CliOption; import org.springframework.stereotype.Component; +/** + * CLI command to display savepoint options. + */ @Component public class SavepointsCommand implements CommandMarker { diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java index 0f4cf79680547..34149d1c63917 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java @@ -35,12 +35,15 @@ import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SQLContext; +/** + * This class deals with initializing spark context based on command entered to hudi-cli. + */ public class SparkMain { protected static final Logger LOG = Logger.getLogger(SparkMain.class); /** - * Commands + * Commands. */ enum SparkCommand { ROLLBACK, DEDUPLICATE, ROLLBACK_TO_SAVEPOINT, SAVEPOINT, IMPORT, UPSERT, COMPACT_SCHEDULE, COMPACT_RUN, COMPACT_UNSCHEDULE_PLAN, COMPACT_UNSCHEDULE_FILE, COMPACT_VALIDATE, COMPACT_REPAIR diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/StatsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/StatsCommand.java index 8b7546b2482b9..84eb481b40000 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/StatsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/StatsCommand.java @@ -47,6 +47,9 @@ import org.springframework.shell.core.annotation.CliOption; import org.springframework.stereotype.Component; +/** + * CLI command to displays stats options. + */ @Component public class StatsCommand implements CommandMarker { diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/UtilsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/UtilsCommand.java index 514a3f88aa49f..eebcfbb971197 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/UtilsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/UtilsCommand.java @@ -23,6 +23,9 @@ import org.springframework.shell.core.annotation.CliOption; import org.springframework.stereotype.Component; +/** + * CLI command to display utils. + */ @Component public class UtilsCommand implements CommandMarker { diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/CommitUtil.java b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/CommitUtil.java index 26568e7e58529..e91d20aa32548 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/CommitUtil.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/CommitUtil.java @@ -25,6 +25,9 @@ import org.apache.hudi.common.table.HoodieTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; +/** + * Utilities related to commit operation. + */ public class CommitUtil { public static long countNewRecords(HoodieTableMetaClient target, List commitsToCatchup) throws IOException { diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/HiveUtil.java b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/HiveUtil.java index dfeae269bfc50..8ebb41ca48c89 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/HiveUtil.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/HiveUtil.java @@ -26,6 +26,9 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.joda.time.DateTime; +/** + * Hive connection related utilities. + */ public class HiveUtil { private static final String driverName = "org.apache.hive.jdbc.HiveDriver"; diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/InputStreamConsumer.java b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/InputStreamConsumer.java index ec001544fa600..625793c72ada1 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/InputStreamConsumer.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/InputStreamConsumer.java @@ -24,6 +24,9 @@ import java.io.InputStreamReader; import java.util.logging.Logger; +/** + * This class is responsible to read a Process output. + */ public class InputStreamConsumer extends Thread { protected static final Logger LOG = Logger.getLogger(InputStreamConsumer.class.getName()); diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java index e8d2a08c449c7..3011981318266 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java @@ -29,13 +29,16 @@ import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.launcher.SparkLauncher; +/** + * Utility functions dealing with Spark. + */ public class SparkUtil { public static Logger logger = Logger.getLogger(SparkUtil.class); public static final String DEFUALT_SPARK_MASTER = "yarn-client"; /** - * TODO: Need to fix a bunch of hardcoded stuff here eg: history server, spark distro + * TODO: Need to fix a bunch of hardcoded stuff here eg: history server, spark distro. */ public static SparkLauncher initLauncher(String propertiesFile) throws URISyntaxException { String currentJar = new File(SparkUtil.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath())