From 323b4c31f891e54222abb062c33c2c61a4ce06f1 Mon Sep 17 00:00:00 2001 From: lamber-ken Date: Wed, 4 Dec 2019 20:32:54 +0800 Subject: [PATCH] [HUDI-379] Refactor the codes based on new JavadocStyle code style rule --- .../apache/hudi/CompactionAdminClient.java | 12 ++--- .../org/apache/hudi/HoodieCleanClient.java | 4 +- .../org/apache/hudi/HoodieReadClient.java | 2 +- .../org/apache/hudi/HoodieWriteClient.java | 24 ++++----- .../embedded/EmbeddedTimelineService.java | 4 +- .../apache/hudi/client/utils/ClientUtils.java | 2 +- .../hudi/config/HoodieCompactionConfig.java | 6 +-- .../hudi/config/HoodieHBaseIndexConfig.java | 16 +++--- .../apache/hudi/config/HoodieIndexConfig.java | 2 +- .../hudi/config/HoodieMemoryConfig.java | 2 +- .../hudi/config/HoodieStorageConfig.java | 2 +- .../apache/hudi/config/HoodieWriteConfig.java | 24 ++++----- .../hudi/exception/HoodieAppendException.java | 2 +- .../hudi/exception/HoodieCommitException.java | 2 +- ...ieDependentSystemUnavailableException.java | 2 +- .../hudi/exception/HoodieInsertException.java | 2 +- .../hudi/exception/HoodieUpsertException.java | 2 +- .../hudi/func/BulkInsertMapFunction.java | 2 +- .../func/CopyOnWriteLazyInsertIterable.java | 2 +- .../hudi/func/LazyIterableIterator.java | 2 +- .../org/apache/hudi/index/HoodieIndex.java | 2 +- .../apache/hudi/index/InMemoryHashIndex.java | 2 +- .../hudi/index/bloom/BloomIndexFileInfo.java | 4 +- .../hudi/index/bloom/HoodieBloomIndex.java | 6 +-- .../bloom/HoodieBloomIndexCheckFunction.java | 2 +- .../index/bloom/HoodieGlobalBloomIndex.java | 2 +- ...ntervalTreeBasedGlobalIndexFileFilter.java | 2 +- .../IntervalTreeBasedIndexFileFilter.java | 2 +- .../hudi/index/bloom/KeyLookupResult.java | 2 +- .../apache/hudi/index/bloom/KeyRangeNode.java | 2 +- .../bloom/ListBasedGlobalIndexFileFilter.java | 2 +- .../index/bloom/ListBasedIndexFileFilter.java | 2 +- .../apache/hudi/index/hbase/HBaseIndex.java | 10 ++-- .../hbase/HBaseIndexQPSResourceAllocator.java | 4 +- .../apache/hudi/io/HoodieAppendHandle.java | 2 +- .../org/apache/hudi/io/HoodieCleanHelper.java | 7 +-- .../hudi/io/HoodieCommitArchiveLog.java | 4 +- .../apache/hudi/io/HoodieCreateHandle.java | 6 +-- .../apache/hudi/io/HoodieKeyLookupHandle.java | 2 +- .../org/apache/hudi/io/HoodieMergeHandle.java | 10 ++-- .../apache/hudi/io/HoodieRangeInfoHandle.java | 2 +- .../org/apache/hudi/io/HoodieWriteHandle.java | 8 +-- .../hudi/io/compact/HoodieCompactor.java | 6 +-- .../strategy/BoundedIOCompactionStrategy.java | 2 +- .../compact/strategy/CompactionStrategy.java | 2 +- .../LogFileSizeBasedCompactionStrategy.java | 2 +- .../apache/hudi/metrics/MetricsReporter.java | 4 +- .../hudi/table/HoodieCopyOnWriteTable.java | 24 ++++----- .../hudi/table/HoodieMergeOnReadTable.java | 2 +- .../org/apache/hudi/table/HoodieTable.java | 52 +++++++++---------- .../apache/hudi/table/RollbackExecutor.java | 8 +-- .../apache/hudi/table/RollbackRequest.java | 14 ++--- .../apache/hudi/table/WorkloadProfile.java | 6 +-- .../org/apache/hudi/TestAsyncCompaction.java | 4 +- .../java/org/apache/hudi/TestCleaner.java | 46 ++++++++-------- .../org/apache/hudi/TestClientRollback.java | 8 +-- .../hudi/TestCompactionAdminClient.java | 8 +-- .../org/apache/hudi/TestHoodieClientBase.java | 36 ++++++------- .../TestHoodieClientOnCopyOnWriteStorage.java | 52 +++++++++---------- .../org/apache/hudi/TestHoodieReadClient.java | 20 +++---- .../hudi/common/HoodieClientTestUtils.java | 2 +- .../common/HoodieMergeOnReadTestUtils.java | 2 +- .../hudi/func/TestBoundedInMemoryQueue.java | 2 +- .../index/bloom/TestKeyRangeLookupTree.java | 8 +-- .../apache/hudi/io/TestHoodieMergeHandle.java | 4 +- .../hudi/table/TestMergeOnReadTable.java | 4 +- .../model/EmptyHoodieRecordPayload.java | 2 +- .../common/table/log/TestHoodieLogFormat.java | 2 +- .../uber/hoodie/hadoop/HoodieInputFormat.java | 2 +- .../realtime/HoodieRealtimeInputFormat.java | 2 +- .../hudi/hadoop/HoodieROTablePathFilter.java | 4 +- .../hadoop/RecordReaderValueIterator.java | 4 +- .../hadoop/UseFileSplitsFromInputFormat.java | 2 +- .../hive/HoodieCombineHiveInputFormat.java | 12 ++--- .../AbstractRealtimeRecordReader.java | 6 +-- .../HoodieParquetRealtimeInputFormat.java | 4 +- .../hadoop/realtime/HoodieParquetSerde.java | 2 +- .../realtime/HoodieRealtimeRecordReader.java | 2 +- .../RealtimeUnmergedRecordReader.java | 2 +- .../hadoop/TestRecordReaderValueIterator.java | 2 +- .../org/apache/hudi/hive/HiveSyncTool.java | 2 +- .../apache/hudi/hive/HoodieHiveClient.java | 20 +++---- .../hudi/hive/NonPartitionedExtractor.java | 2 +- .../apache/hudi/hive/SchemaDifference.java | 2 +- .../org/apache/hudi/hive/util/SchemaUtil.java | 16 +++--- .../apache/hudi/hive/TestHiveSyncTool.java | 2 +- .../hudi/hive/util/HiveTestService.java | 2 +- .../java/org/apache/hudi/BaseAvroPayload.java | 6 +-- .../java/org/apache/hudi/DataSourceUtils.java | 4 +- .../apache/hudi/HoodieDataSourceHelpers.java | 4 +- .../java/org/apache/hudi/KeyGenerator.java | 2 +- .../hudi/NonpartitionedKeyGenerator.java | 2 +- hudi-spark/src/test/java/HoodieJavaApp.java | 4 +- .../src/test/java/HoodieJavaStreamingApp.java | 8 +-- .../service/FileSystemViewHandler.java | 12 ++--- .../timeline/service/TimelineService.java | 2 +- .../service/handlers/DataFileHandler.java | 2 +- .../service/handlers/FileSliceHandler.java | 2 +- .../service/handlers/TimelineHandler.java | 2 +- .../TestRemoteHoodieTableFileSystemView.java | 2 +- .../hudi/utilities/HDFSParquetImporter.java | 4 +- .../apache/hudi/utilities/HoodieCleaner.java | 6 +-- .../utilities/HoodieCompactionAdminTool.java | 8 +-- .../apache/hudi/utilities/UtilHelpers.java | 8 +-- .../adhoc/UpgradePayloadFromUberToApache.java | 2 +- .../AbstractDeltaStreamerService.java | 6 +-- .../utilities/deltastreamer/Compactor.java | 2 +- .../utilities/deltastreamer/DeltaSync.java | 43 ++++++++------- .../deltastreamer/HoodieDeltaStreamer.java | 28 +++++----- .../deltastreamer/SchedulerConfGenerator.java | 2 +- .../deltastreamer/SourceFormatAdapter.java | 2 +- .../keygen/TimestampBasedKeyGenerator.java | 2 +- .../schema/FilebasedSchemaProvider.java | 4 +- .../hudi/utilities/schema/SchemaProvider.java | 2 +- .../schema/SchemaRegistryProvider.java | 4 +- .../hudi/utilities/sources/AvroDFSSource.java | 2 +- .../utilities/sources/AvroKafkaSource.java | 2 +- .../utilities/sources/HiveIncrPullSource.java | 2 +- .../utilities/sources/HoodieIncrSource.java | 12 ++--- .../hudi/utilities/sources/JsonDFSSource.java | 2 +- .../utilities/sources/JsonKafkaSource.java | 2 +- .../utilities/sources/ParquetDFSSource.java | 2 +- .../apache/hudi/utilities/sources/Source.java | 2 +- .../sources/helpers/AvroConvertor.java | 6 +-- .../sources/helpers/DFSPathSelector.java | 2 +- .../sources/helpers/IncrSourceHelper.java | 6 +-- .../sources/helpers/KafkaOffsetGen.java | 4 +- .../transform/FlatteningTransformer.java | 4 +- .../transform/IdentityTransformer.java | 2 +- .../transform/SqlQueryBasedTransformer.java | 2 +- .../hudi/utilities/transform/Transformer.java | 4 +- .../utilities/TestHoodieDeltaStreamer.java | 8 +-- .../hudi/utilities/UtilitiesTestBase.java | 4 +- .../hudi/utilities/sources/TestDFSSource.java | 2 +- .../utilities/sources/TestKafkaSource.java | 2 +- .../sources/config/TestSourceConfig.java | 2 +- style/checkstyle.xml | 5 +- 137 files changed, 434 insertions(+), 433 deletions(-) diff --git a/hudi-client/src/main/java/org/apache/hudi/CompactionAdminClient.java b/hudi-client/src/main/java/org/apache/hudi/CompactionAdminClient.java index 6034408513b91..76b66319ef5e5 100644 --- a/hudi-client/src/main/java/org/apache/hudi/CompactionAdminClient.java +++ b/hudi-client/src/main/java/org/apache/hudi/CompactionAdminClient.java @@ -61,7 +61,7 @@ import static org.apache.hudi.common.table.HoodieTimeline.COMPACTION_ACTION; /** - * Client to perform admin operations related to compaction + * Client to perform admin operations related to compaction. */ public class CompactionAdminClient extends AbstractHoodieClient { @@ -214,7 +214,7 @@ public List repairCompaction(String compactionInstant, int paral } /** - * Construction Compaction Plan from compaction instant + * Construction Compaction Plan from compaction instant. */ private static HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, String compactionInstant) throws IOException { @@ -273,7 +273,7 @@ protected static void renameLogFile(HoodieTableMetaClient metaClient, HoodieLogF } /** - * Check if a compaction operation is valid + * Check if a compaction operation is valid. * * @param metaClient Hoodie Table Meta client * @param compactionInstant Compaction Instant @@ -342,7 +342,7 @@ private ValidationOpResult validateCompactionOperation(HoodieTableMetaClient met } /** - * Execute Renaming operation + * Execute Renaming operation. * * @param metaClient HoodieTable MetaClient * @param renameActions List of rename operations @@ -484,7 +484,7 @@ public List> getRenamingActionsForUnschedulin } /** - * Holds Operation result for Renaming + * Holds Operation result for Renaming. */ public static class RenameOpResult extends OperationResult { @@ -505,7 +505,7 @@ public RenameOpResult(Pair op, boolean executed, b } /** - * Holds Operation result for Renaming + * Holds Operation result for Renaming. */ public static class ValidationOpResult extends OperationResult { diff --git a/hudi-client/src/main/java/org/apache/hudi/HoodieCleanClient.java b/hudi-client/src/main/java/org/apache/hudi/HoodieCleanClient.java index e08ecfb61d3bd..c35a0b4f74e94 100644 --- a/hudi-client/src/main/java/org/apache/hudi/HoodieCleanClient.java +++ b/hudi-client/src/main/java/org/apache/hudi/HoodieCleanClient.java @@ -103,7 +103,7 @@ protected HoodieCleanMetadata clean(String startCleanTime) throws HoodieIOExcept } /** - * Creates a Cleaner plan if there are files to be cleaned and stores them in instant file + * Creates a Cleaner plan if there are files to be cleaned and stores them in instant file. * * @param startCleanTime Cleaner Instant Time * @return Cleaner Plan if generated @@ -133,7 +133,7 @@ protected Option scheduleClean(String startCleanTime) { } /** - * Executes the Cleaner plan stored in the instant metadata + * Executes the Cleaner plan stored in the instant metadata. * * @param table Hoodie Table * @param cleanInstantTs Cleaner Instant Timestamp diff --git a/hudi-client/src/main/java/org/apache/hudi/HoodieReadClient.java b/hudi-client/src/main/java/org/apache/hudi/HoodieReadClient.java index 16f79fb3c7d1a..3c4290c89020d 100644 --- a/hudi-client/src/main/java/org/apache/hudi/HoodieReadClient.java +++ b/hudi-client/src/main/java/org/apache/hudi/HoodieReadClient.java @@ -145,7 +145,7 @@ private Option convertToDataFilePath(Option> partit } /** - * Given a bunch of hoodie keys, fetches all the individual records out as a data frame + * Given a bunch of hoodie keys, fetches all the individual records out as a data frame. * * @return a dataframe */ diff --git a/hudi-client/src/main/java/org/apache/hudi/HoodieWriteClient.java b/hudi-client/src/main/java/org/apache/hudi/HoodieWriteClient.java index 3ccba7d0eac8e..4fdcc1f2aa554 100644 --- a/hudi-client/src/main/java/org/apache/hudi/HoodieWriteClient.java +++ b/hudi-client/src/main/java/org/apache/hudi/HoodieWriteClient.java @@ -159,7 +159,7 @@ public JavaRDD> filterExists(JavaRDD> hoodieReco } /** - * Upserts a bunch of new records into the Hoodie table, at the supplied commitTime + * Upserts a bunch of new records into the Hoodie table, at the supplied commitTime. */ public JavaRDD upsert(JavaRDD> records, final String commitTime) { HoodieTable table = getTableAndInitCtx(OperationType.UPSERT); @@ -505,14 +505,14 @@ record -> new Tuple2<>(new Tuple2<>(record.getKey(), Option.ofNullable(record.ge } /** - * Commit changes performed at the given commitTime marker + * Commit changes performed at the given commitTime marker. */ public boolean commit(String commitTime, JavaRDD writeStatuses) { return commit(commitTime, writeStatuses, Option.empty()); } /** - * Commit changes performed at the given commitTime marker + * Commit changes performed at the given commitTime marker. */ public boolean commit(String commitTime, JavaRDD writeStatuses, Option> extraMetadata) { @@ -988,7 +988,7 @@ protected HoodieCleanMetadata clean(String startCleanTime) throws HoodieIOExcept } /** - * Provides a new commit time for a write operation (insert/update) + * Provides a new commit time for a write operation (insert/update). */ public String startCommit() { // NOTE : Need to ensure that rollback is done before a new commit is started @@ -1027,7 +1027,7 @@ private void startCommit(String instantTime) { } /** - * Schedules a new compaction instant + * Schedules a new compaction instant. */ public Option scheduleCompaction(Option> extraMetadata) throws IOException { String instantTime = HoodieActiveTimeline.createNewCommitTime(); @@ -1037,7 +1037,7 @@ public Option scheduleCompaction(Option> extraMetada } /** - * Schedules a new compaction instant with passed-in instant time + * Schedules a new compaction instant with passed-in instant time. * * @param instantTime Compaction Instant Time * @param extraMetadata Extra Metadata to be stored @@ -1074,7 +1074,7 @@ public boolean scheduleCompactionAtInstant(String instantTime, Option deduplicateKeys(JavaRDD keys, int parallelism) { } /** - * Cleanup all inflight commits + * Cleanup all inflight commits. */ private void rollbackInflightCommits() { HoodieTable table = HoodieTable.getHoodieTable(createMetaClient(true), config, jsc); @@ -1197,7 +1197,7 @@ private void setWriteSchemaFromLastInstant(HoodieTableMetaClient metaClient) { */ /** - * Ensures compaction instant is in expected state and performs Compaction for the workload stored in instant-time + * Ensures compaction instant is in expected state and performs Compaction for the workload stored in instant-time. * * @param compactionInstantTime Compaction Instant Time */ @@ -1226,7 +1226,7 @@ private JavaRDD compact(String compactionInstantTime, boolean autoC } /** - * Perform compaction operations as specified in the compaction commit file + * Perform compaction operations as specified in the compaction commit file. * * @param compactionInstant Compacton Instant time * @param activeTimeline Active Timeline @@ -1254,7 +1254,7 @@ private JavaRDD runCompaction(HoodieInstant compactionInstant, Hood } /** - * Commit Compaction and track metrics + * Commit Compaction and track metrics. * * @param compactedStatuses Compaction Write status * @param table Hoodie Table @@ -1404,7 +1404,7 @@ private void updateMetadataAndRollingStats(String actionType, HoodieCommitMetada } /** - * Refers to different operation types + * Refers to different operation types. */ enum OperationType { INSERT, diff --git a/hudi-client/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java b/hudi-client/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java index 54aef32db84c9..d743aa77c6fc2 100644 --- a/hudi-client/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java +++ b/hudi-client/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java @@ -33,7 +33,7 @@ import java.io.IOException; /** - * Timeline Service that runs as part of write client + * Timeline Service that runs as part of write client. */ public class EmbeddedTimelineService { @@ -86,7 +86,7 @@ private void setHostAddrFromSparkConf(SparkConf sparkConf) { } /** - * Retrieves proper view storage configs for remote clients to access this service + * Retrieves proper view storage configs for remote clients to access this service. */ public FileSystemViewStorageConfig getRemoteFileSystemViewConfig() { return FileSystemViewStorageConfig.newBuilder().withStorageType(FileSystemViewStorageType.REMOTE_FIRST) diff --git a/hudi-client/src/main/java/org/apache/hudi/client/utils/ClientUtils.java b/hudi-client/src/main/java/org/apache/hudi/client/utils/ClientUtils.java index fe2998e6913f3..12499928f810b 100644 --- a/hudi-client/src/main/java/org/apache/hudi/client/utils/ClientUtils.java +++ b/hudi-client/src/main/java/org/apache/hudi/client/utils/ClientUtils.java @@ -26,7 +26,7 @@ public class ClientUtils { /** - * Create Consistency Aware MetaClient + * Create Consistency Aware MetaClient. * * @param jsc JavaSparkContext * @param config HoodieWriteConfig diff --git a/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java index 2f1c3e7e97c3b..4ecc87fbc7d6b 100644 --- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java @@ -33,7 +33,7 @@ import java.util.Properties; /** - * Compaction related config + * Compaction related config. */ @Immutable public class HoodieCompactionConfig extends DefaultHoodieConfig { @@ -55,8 +55,8 @@ public class HoodieCompactionConfig extends DefaultHoodieConfig { // By default, treat any file <= 100MB as a small file. public static final String DEFAULT_PARQUET_SMALL_FILE_LIMIT_BYTES = String.valueOf(104857600); /** - * Configs related to specific table types - **/ + * Configs related to specific table types. + */ // Number of inserts, that will be put each partition/bucket for writing public static final String COPY_ON_WRITE_TABLE_INSERT_SPLIT_SIZE = "hoodie.copyonwrite.insert" + ".split.size"; // The rationale to pick the insert parallelism is the following. Writing out 100MB files, diff --git a/hudi-client/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java index 9fd32cb967f9b..73dfabd55b725 100644 --- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java @@ -34,17 +34,17 @@ public class HoodieHBaseIndexConfig extends DefaultHoodieConfig { public static final String HBASE_ZK_ZNODEPARENT = "hoodie.index.hbase.zknode.path"; /** * Note that if HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE_PROP is set to true, this batch size will not be honored for HBase - * Puts + * Puts. */ public static final String HBASE_PUT_BATCH_SIZE_PROP = "hoodie.index.hbase.put.batch.size"; /** - * Property to set which implementation of HBase QPS resource allocator to be used + * Property to set which implementation of HBase QPS resource allocator to be used. */ public static final String HBASE_INDEX_QPS_ALLOCATOR_CLASS = "hoodie.index.hbase.qps.allocator.class"; public static final String DEFAULT_HBASE_INDEX_QPS_ALLOCATOR_CLASS = DefaultHBaseQPSResourceAllocator.class.getName(); /** - * Property to set to enable auto computation of put batch size + * Property to set to enable auto computation of put batch size. */ public static final String HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE_PROP = "hoodie.index.hbase.put.batch.size.autocompute"; public static final String DEFAULT_HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE = "false"; @@ -62,7 +62,7 @@ public class HoodieHBaseIndexConfig extends DefaultHoodieConfig { */ public static String HBASE_MAX_QPS_PER_REGION_SERVER_PROP = "hoodie.index.hbase.max.qps.per.region.server"; /** - * Default batch size, used only for Get, but computed for Put + * Default batch size, used only for Get, but computed for Put. */ public static final int DEFAULT_HBASE_BATCH_SIZE = 100; /** @@ -70,17 +70,17 @@ public class HoodieHBaseIndexConfig extends DefaultHoodieConfig { */ public static final int DEFAULT_HBASE_MAX_QPS_PER_REGION_SERVER = 1000; /** - * Default is 50%, which means a total of 2 jobs can run using HbaseIndex without overwhelming Region Servers + * Default is 50%, which means a total of 2 jobs can run using HbaseIndex without overwhelming Region Servers. */ public static final float DEFAULT_HBASE_QPS_FRACTION = 0.5f; /** - * Property to decide if HBASE_QPS_FRACTION_PROP is dynamically calculated based on volume + * Property to decide if HBASE_QPS_FRACTION_PROP is dynamically calculated based on volume. */ public static final String HOODIE_INDEX_COMPUTE_QPS_DYNAMICALLY = "hoodie.index.hbase.dynamic_qps"; public static final boolean DEFAULT_HOODIE_INDEX_COMPUTE_QPS_DYNAMICALLY = false; /** - * Min and Max for HBASE_QPS_FRACTION_PROP to stabilize skewed volume workloads + * Min and Max for HBASE_QPS_FRACTION_PROP to stabilize skewed volume workloads. */ public static final String HBASE_MIN_QPS_FRACTION_PROP = "hoodie.index.hbase.min.qps.fraction"; public static final String DEFAULT_HBASE_MIN_QPS_FRACTION_PROP = "0.002"; @@ -88,7 +88,7 @@ public class HoodieHBaseIndexConfig extends DefaultHoodieConfig { public static final String HBASE_MAX_QPS_FRACTION_PROP = "hoodie.index.hbase.max.qps.fraction"; public static final String DEFAULT_HBASE_MAX_QPS_FRACTION_PROP = "0.06"; /** - * Hoodie index desired puts operation time in seconds + * Hoodie index desired puts operation time in seconds. */ public static final String HOODIE_INDEX_DESIRED_PUTS_TIME_IN_SECS = "hoodie.index.hbase.desired_puts_time_in_secs"; public static final int DEFAULT_HOODIE_INDEX_DESIRED_PUTS_TIME_IN_SECS = 600; diff --git a/hudi-client/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java index de9895509b3c3..abba272d5f075 100644 --- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java @@ -29,7 +29,7 @@ /** - * Indexing related config + * Indexing related config. */ @Immutable public class HoodieIndexConfig extends DefaultHoodieConfig { diff --git a/hudi-client/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java index 43b28cc373e4e..f19a64c67728b 100644 --- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java @@ -29,7 +29,7 @@ import java.util.Properties; /** - * Memory related config + * Memory related config. */ @Immutable public class HoodieMemoryConfig extends DefaultHoodieConfig { diff --git a/hudi-client/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java index f9c98c7e1abfa..24cf19027047f 100644 --- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java @@ -26,7 +26,7 @@ import java.util.Properties; /** - * Storage related config + * Storage related config. */ @Immutable public class HoodieStorageConfig extends DefaultHoodieConfig { diff --git a/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index 37431163d18e4..3c2563d95abee 100644 --- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -42,7 +42,7 @@ import java.util.Properties; /** - * Class storing configs for the {@link HoodieWriteClient} + * Class storing configs for the {@link HoodieWriteClient}. */ @Immutable public class HoodieWriteConfig extends DefaultHoodieConfig { @@ -115,8 +115,8 @@ public static HoodieWriteConfig.Builder newBuilder() { } /** - * base properties - **/ + * base properties. + */ public String getBasePath() { return props.getProperty(BASE_PATH_PROP); } @@ -210,8 +210,8 @@ public int getMaxConsistencyCheckIntervalMs() { } /** - * compaction properties - **/ + * compaction properties. + */ public HoodieCleaningPolicy getCleanerPolicy() { return HoodieCleaningPolicy.valueOf(props.getProperty(HoodieCompactionConfig.CLEANER_POLICY_PROP)); } @@ -297,8 +297,8 @@ public int getCommitArchivalBatchSize() { } /** - * index properties - **/ + * index properties. + */ public HoodieIndex.IndexType getIndexType() { return HoodieIndex.IndexType.valueOf(props.getProperty(HoodieIndexConfig.INDEX_TYPE_PROP)); } @@ -417,8 +417,8 @@ public StorageLevel getBloomIndexInputStorageLevel() { } /** - * storage properties - **/ + * storage properties. + */ public long getParquetMaxFileSize() { return Long.parseLong(props.getProperty(HoodieStorageConfig.PARQUET_FILE_MAX_BYTES)); } @@ -452,8 +452,8 @@ public double getLogFileToParquetCompressionRatio() { } /** - * metrics properties - **/ + * metrics properties. + */ public boolean isMetricsOn() { return Boolean.parseBoolean(props.getProperty(HoodieMetricsConfig.METRICS_ON)); } @@ -483,7 +483,7 @@ public int getJmxPort() { } /** - * memory configs + * memory configs. */ public Double getMaxMemoryFractionPerPartitionMerge() { return Double.valueOf(props.getProperty(HoodieMemoryConfig.MAX_MEMORY_FRACTION_FOR_MERGE_PROP)); diff --git a/hudi-client/src/main/java/org/apache/hudi/exception/HoodieAppendException.java b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieAppendException.java index e6035699a7ff1..ffe99cceaee41 100644 --- a/hudi-client/src/main/java/org/apache/hudi/exception/HoodieAppendException.java +++ b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieAppendException.java @@ -20,7 +20,7 @@ /** *

- * Exception thrown for any higher level errors when HoodieClient is doing a delta commit + * Exception thrown for any higher level errors when HoodieClient is doing a delta commit. *

*/ public class HoodieAppendException extends HoodieException { diff --git a/hudi-client/src/main/java/org/apache/hudi/exception/HoodieCommitException.java b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieCommitException.java index 3fb15a6340ae0..b26f63ea7dba3 100644 --- a/hudi-client/src/main/java/org/apache/hudi/exception/HoodieCommitException.java +++ b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieCommitException.java @@ -20,7 +20,7 @@ /** *

- * Exception thrown for any higher level errors when HoodieClient is doing a Commit + * Exception thrown for any higher level errors when HoodieClient is doing a Commit. *

*/ public class HoodieCommitException extends HoodieException { diff --git a/hudi-client/src/main/java/org/apache/hudi/exception/HoodieDependentSystemUnavailableException.java b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieDependentSystemUnavailableException.java index 76d679fec07a7..4c83ebc849bc4 100644 --- a/hudi-client/src/main/java/org/apache/hudi/exception/HoodieDependentSystemUnavailableException.java +++ b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieDependentSystemUnavailableException.java @@ -20,7 +20,7 @@ /** *

- * Exception thrown when dependent system is not available + * Exception thrown when dependent system is not available. *

*/ public class HoodieDependentSystemUnavailableException extends HoodieException { diff --git a/hudi-client/src/main/java/org/apache/hudi/exception/HoodieInsertException.java b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieInsertException.java index 37995bf7d1c64..fc014adfea344 100644 --- a/hudi-client/src/main/java/org/apache/hudi/exception/HoodieInsertException.java +++ b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieInsertException.java @@ -20,7 +20,7 @@ /** *

- * Exception thrown for any higher level errors when HoodieClient is doing a bulk insert + * Exception thrown for any higher level errors when HoodieClient is doing a bulk insert. *

*/ public class HoodieInsertException extends HoodieException { diff --git a/hudi-client/src/main/java/org/apache/hudi/exception/HoodieUpsertException.java b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieUpsertException.java index 062ef67f777e0..80427215571fa 100644 --- a/hudi-client/src/main/java/org/apache/hudi/exception/HoodieUpsertException.java +++ b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieUpsertException.java @@ -20,7 +20,7 @@ /** *

- * Exception thrown for any higher level errors when HoodieClient is doing a incremental upsert + * Exception thrown for any higher level errors when HoodieClient is doing a incremental upsert. *

*/ public class HoodieUpsertException extends HoodieException { diff --git a/hudi-client/src/main/java/org/apache/hudi/func/BulkInsertMapFunction.java b/hudi-client/src/main/java/org/apache/hudi/func/BulkInsertMapFunction.java index c40bb2a3a06a3..1386eef958daa 100644 --- a/hudi-client/src/main/java/org/apache/hudi/func/BulkInsertMapFunction.java +++ b/hudi-client/src/main/java/org/apache/hudi/func/BulkInsertMapFunction.java @@ -30,7 +30,7 @@ import java.util.List; /** - * Map function that handles a sorted stream of HoodieRecords + * Map function that handles a sorted stream of HoodieRecords. */ public class BulkInsertMapFunction implements Function2>, Iterator>> { diff --git a/hudi-client/src/main/java/org/apache/hudi/func/CopyOnWriteLazyInsertIterable.java b/hudi-client/src/main/java/org/apache/hudi/func/CopyOnWriteLazyInsertIterable.java index 8e1c191033e88..4d526f4850bcc 100644 --- a/hudi-client/src/main/java/org/apache/hudi/func/CopyOnWriteLazyInsertIterable.java +++ b/hudi-client/src/main/java/org/apache/hudi/func/CopyOnWriteLazyInsertIterable.java @@ -122,7 +122,7 @@ protected CopyOnWriteInsertHandler getInsertHandler() { } /** - * Consumes stream of hoodie records from in-memory queue and writes to one or more create-handles + * Consumes stream of hoodie records from in-memory queue and writes to one or more create-handles. */ protected class CopyOnWriteInsertHandler extends BoundedInMemoryQueueConsumer, List> { diff --git a/hudi-client/src/main/java/org/apache/hudi/func/LazyIterableIterator.java b/hudi-client/src/main/java/org/apache/hudi/func/LazyIterableIterator.java index ec05b85311eeb..fe5bfbc6a3445 100644 --- a/hudi-client/src/main/java/org/apache/hudi/func/LazyIterableIterator.java +++ b/hudi-client/src/main/java/org/apache/hudi/func/LazyIterableIterator.java @@ -43,7 +43,7 @@ public LazyIterableIterator(Iterator in) { } /** - * Called once, before any elements are processed + * Called once, before any elements are processed. */ protected abstract void start(); diff --git a/hudi-client/src/main/java/org/apache/hudi/index/HoodieIndex.java b/hudi-client/src/main/java/org/apache/hudi/index/HoodieIndex.java index 235b3fb56ebec..e4baed3823243 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/HoodieIndex.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/HoodieIndex.java @@ -39,7 +39,7 @@ import java.io.Serializable; /** - * Base class for different types of indexes to determine the mapping from uuid + * Base class for different types of indexes to determine the mapping from uuid. */ public abstract class HoodieIndex implements Serializable { diff --git a/hudi-client/src/main/java/org/apache/hudi/index/InMemoryHashIndex.java b/hudi-client/src/main/java/org/apache/hudi/index/InMemoryHashIndex.java index a8cd6e554435d..ecfced3d10ebd 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/InMemoryHashIndex.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/InMemoryHashIndex.java @@ -99,7 +99,7 @@ public boolean rollbackCommit(String commitTime) { } /** - * Only looks up by recordKey + * Only looks up by recordKey. */ @Override public boolean isGlobal() { diff --git a/hudi-client/src/main/java/org/apache/hudi/index/bloom/BloomIndexFileInfo.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/BloomIndexFileInfo.java index 3a51cb9e3973b..51e1b76520a25 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/bloom/BloomIndexFileInfo.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/BloomIndexFileInfo.java @@ -23,7 +23,7 @@ import java.io.Serializable; /** - * Metadata about a given file group, useful for index lookup + * Metadata about a given file group, useful for index lookup. */ public class BloomIndexFileInfo implements Serializable { @@ -62,7 +62,7 @@ public boolean hasKeyRanges() { } /** - * Does the given key fall within the range (inclusive) + * Does the given key fall within the range (inclusive). */ public boolean isKeyInRange(String recordKey) { return minRecordKey.compareTo(recordKey) <= 0 && maxRecordKey.compareTo(recordKey) >= 0; diff --git a/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java index aec8cc36c70c0..0ffdf1111361d 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java @@ -141,7 +141,7 @@ public JavaPairRDD>> fetchRecordLocation( /** * Lookup the location for each record key and return the pair for all record keys already - * present and drop the record keys if not present + * present and drop the record keys if not present. */ private JavaPairRDD lookupIndex( JavaPairRDD partitionRecordKeyPairRDD, final JavaSparkContext jsc, @@ -167,7 +167,7 @@ private JavaPairRDD lookupIndex( } /** - * Compute the estimated number of bloom filter comparisons to be performed on each file group + * Compute the estimated number of bloom filter comparisons to be performed on each file group. */ private Map computeComparisonsPerFileGroup(final Map recordsPerPartition, final Map> partitionToFileInfo, @@ -278,7 +278,7 @@ public boolean rollbackCommit(String commitTime) { } /** - * This is not global, since we depend on the partitionPath to do the lookup + * This is not global, since we depend on the partitionPath to do the lookup. */ @Override public boolean isGlobal() { diff --git a/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndexCheckFunction.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndexCheckFunction.java index 2a289fee27df3..2881ce006f7b5 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndexCheckFunction.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndexCheckFunction.java @@ -37,7 +37,7 @@ import scala.Tuple2; /** - * Function performing actual checking of RDD partition containing (fileId, hoodieKeys) against the actual files + * Function performing actual checking of RDD partition containing (fileId, hoodieKeys) against the actual files. */ public class HoodieBloomIndexCheckFunction implements Function2>, Iterator>> { diff --git a/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java index ec308c600a867..8db45e035d75f 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java @@ -106,7 +106,7 @@ JavaRDD> explodeRecordRDDWithFileComparisons( /** - * Tagging for global index should only consider the record key + * Tagging for global index should only consider the record key. */ @Override protected JavaRDD> tagLocationBacktoRecords( diff --git a/hudi-client/src/main/java/org/apache/hudi/index/bloom/IntervalTreeBasedGlobalIndexFileFilter.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/IntervalTreeBasedGlobalIndexFileFilter.java index 4e269d745c2d2..f3001ea05b435 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/bloom/IntervalTreeBasedGlobalIndexFileFilter.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/IntervalTreeBasedGlobalIndexFileFilter.java @@ -36,7 +36,7 @@ class IntervalTreeBasedGlobalIndexFileFilter implements IndexFileFilter { private final Set filesWithNoRanges = new HashSet<>(); /** - * Instantiates {@link IntervalTreeBasedGlobalIndexFileFilter} + * Instantiates {@link IntervalTreeBasedGlobalIndexFileFilter}. * * @param partitionToFileIndexInfo Map of partition to List of {@link BloomIndexFileInfo}s */ diff --git a/hudi-client/src/main/java/org/apache/hudi/index/bloom/IntervalTreeBasedIndexFileFilter.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/IntervalTreeBasedIndexFileFilter.java index 9737772b711e0..849cdc6945e14 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/bloom/IntervalTreeBasedIndexFileFilter.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/IntervalTreeBasedIndexFileFilter.java @@ -35,7 +35,7 @@ class IntervalTreeBasedIndexFileFilter implements IndexFileFilter { private final Map> partitionToFilesWithNoRanges = new HashMap<>(); /** - * Instantiates {@link IntervalTreeBasedIndexFileFilter} + * Instantiates {@link IntervalTreeBasedIndexFileFilter}. * * @param partitionToFileIndexInfo Map of partition to List of {@link BloomIndexFileInfo}s */ diff --git a/hudi-client/src/main/java/org/apache/hudi/index/bloom/KeyLookupResult.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/KeyLookupResult.java index 631ff768cb7e3..191cfd360241a 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/bloom/KeyLookupResult.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/KeyLookupResult.java @@ -21,7 +21,7 @@ import java.util.List; /** - * Encapsulates the result from a key lookup + * Encapsulates the result from a key lookup. */ public class KeyLookupResult { diff --git a/hudi-client/src/main/java/org/apache/hudi/index/bloom/KeyRangeNode.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/KeyRangeNode.java index 659498121af6d..df2e97189ac1a 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/bloom/KeyRangeNode.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/KeyRangeNode.java @@ -39,7 +39,7 @@ class KeyRangeNode implements Comparable, Serializable { private KeyRangeNode right = null; /** - * Instantiates a new {@link KeyRangeNode} + * Instantiates a new {@link KeyRangeNode}. * * @param minRecordKey min record key of the index file * @param maxRecordKey max record key of the index file diff --git a/hudi-client/src/main/java/org/apache/hudi/index/bloom/ListBasedGlobalIndexFileFilter.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/ListBasedGlobalIndexFileFilter.java index d5fe4f6d2e34c..1f3e04e2fc4b5 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/bloom/ListBasedGlobalIndexFileFilter.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/ListBasedGlobalIndexFileFilter.java @@ -26,7 +26,7 @@ class ListBasedGlobalIndexFileFilter extends ListBasedIndexFileFilter { /** - * Instantiates {@link ListBasedGlobalIndexFileFilter} + * Instantiates {@link ListBasedGlobalIndexFileFilter}. * * @param partitionToFileIndexInfo Map of partition to List of {@link BloomIndexFileInfo} */ diff --git a/hudi-client/src/main/java/org/apache/hudi/index/bloom/ListBasedIndexFileFilter.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/ListBasedIndexFileFilter.java index 967320998aa57..b9a6ce382f301 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/bloom/ListBasedIndexFileFilter.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/ListBasedIndexFileFilter.java @@ -32,7 +32,7 @@ class ListBasedIndexFileFilter implements IndexFileFilter { final Map> partitionToFileIndexInfo; /** - * Instantiates {@link ListBasedIndexFileFilter} + * Instantiates {@link ListBasedIndexFileFilter}. * * @param partitionToFileIndexInfo Map of partition to List of {@link BloomIndexFileInfo} */ diff --git a/hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndex.java b/hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndex.java index beb5b4a9c85a3..87d555795793f 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndex.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndex.java @@ -67,7 +67,7 @@ import scala.Tuple2; /** - * Hoodie Index implementation backed by HBase + * Hoodie Index implementation backed by HBase. */ public class HBaseIndex extends HoodieIndex { @@ -89,7 +89,7 @@ public class HBaseIndex extends HoodieIndex { private int maxQpsPerRegionServer; /** * multiPutBatchSize will be computed and re-set in updateLocation if - * {@link HoodieIndexConfig.HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE_PROP} is set to true + * {@link HoodieIndexConfig.HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE_PROP} is set to true. */ private Integer multiPutBatchSize; private Integer numRegionServersForTable; @@ -150,7 +150,7 @@ private Connection getHBaseConnection() { /** * Since we are sharing the HbaseConnection across tasks in a JVM, make sure the HbaseConnectio is closed when JVM - * exits + * exits. */ private void addShutDownHook() { Runtime.getRuntime().addShutdownHook(new Thread() { @@ -342,7 +342,7 @@ private Function2, Iterator> updateL } /** - * Helper method to facilitate performing puts and deletes in Hbase + * Helper method to facilitate performing puts and deletes in Hbase. */ private void doPutsAndDeletes(HTable hTable, List puts, List deletes) throws IOException { if (puts.size() > 0) { @@ -500,7 +500,7 @@ public boolean rollbackCommit(String commitTime) { } /** - * Only looks up by recordKey + * Only looks up by recordKey. */ @Override public boolean isGlobal() { diff --git a/hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndexQPSResourceAllocator.java b/hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndexQPSResourceAllocator.java index bccdd5bcc7955..bb876b28b9590 100644 --- a/hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndexQPSResourceAllocator.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndexQPSResourceAllocator.java @@ -21,7 +21,7 @@ import java.io.Serializable; /** - * HBaseIndexQPSResourceAllocator defines methods to manage resource allocation for HBase index operations + * HBaseIndexQPSResourceAllocator defines methods to manage resource allocation for HBase index operations. */ public interface HBaseIndexQPSResourceAllocator extends Serializable { @@ -45,7 +45,7 @@ public interface HBaseIndexQPSResourceAllocator extends Serializable { float acquireQPSResources(final float desiredQPSFraction, final long numPuts); /** - * This method releases the acquired QPS Fraction + * This method releases the acquired QPS Fraction. */ void releaseQPSResources(); } diff --git a/hudi-client/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java index 62c9e6b384beb..712702ac5240f 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java @@ -301,7 +301,7 @@ private void writeToBuffer(HoodieRecord record) { } /** - * Checks if the number of records have reached the set threshold and then flushes the records to disk + * Checks if the number of records have reached the set threshold and then flushes the records to disk. */ private void flushToDiskIfRequired(HoodieRecord record) { // Append if max number of records reached to achieve block size diff --git a/hudi-client/src/main/java/org/apache/hudi/io/HoodieCleanHelper.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieCleanHelper.java index c79b39f26ac6f..eaaf72f47e36d 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieCleanHelper.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieCleanHelper.java @@ -51,7 +51,7 @@ import java.util.stream.Collectors; /** - * Cleaner is responsible for garbage collecting older files in a given partition path, such that + * Cleaner is responsible for garbage collecting older files in a given partition path. Such that *

* 1) It provides sufficient time for existing queries running on older versions, to close *

@@ -83,7 +83,8 @@ public HoodieCleanHelper(HoodieTable hoodieTable, HoodieWriteConfig config) { } /** - * Returns list of partitions where clean operations needs to be performed + * Returns list of partitions where clean operations needs to be performed. + * * @param newInstantToRetain New instant to be retained after this cleanup operation * @return list of partitions to scan for cleaning * @throws IOException when underlying file-system throws this exception @@ -294,7 +295,7 @@ public Option getEarliestCommitToRetain() { } /** - * Determine if file slice needed to be preserved for pending compaction + * Determine if file slice needed to be preserved for pending compaction. * * @param fileSlice File Slice * @return true if file slice needs to be preserved, false otherwise. diff --git a/hudi-client/src/main/java/org/apache/hudi/io/HoodieCommitArchiveLog.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieCommitArchiveLog.java index c69294926c4b5..e7e0cfe96633e 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieCommitArchiveLog.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieCommitArchiveLog.java @@ -64,7 +64,7 @@ import java.util.stream.Stream; /** - * Archiver to bound the growth of .commit files + * Archiver to bound the growth of .commit files. */ public class HoodieCommitArchiveLog { @@ -201,7 +201,7 @@ private boolean deleteArchivedInstants(List archivedInstants) thr } /** - * Remove older instants from auxiliary meta folder + * Remove older instants from auxiliary meta folder. * * @param thresholdInstant Hoodie Instant * @return success if all eligible file deleted successfully diff --git a/hudi-client/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java index 06601b1826dfe..c4769084bc4c9 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java @@ -77,7 +77,7 @@ public HoodieCreateHandle(HoodieWriteConfig config, String commitTime, HoodieTab } /** - * Called by the compactor code path + * Called by the compactor code path. */ public HoodieCreateHandle(HoodieWriteConfig config, String commitTime, HoodieTable hoodieTable, String partitionPath, String fileId, Iterator> recordIterator) { @@ -124,7 +124,7 @@ public void write(HoodieRecord record, Option avroRecord) { } /** - * Writes all records passed + * Writes all records passed. */ public void write() { try { @@ -147,7 +147,7 @@ public WriteStatus getWriteStatus() { } /** - * Performs actions to durably, persist the current changes and returns a WriteStatus object + * Performs actions to durably, persist the current changes and returns a WriteStatus object. */ @Override public WriteStatus close() { diff --git a/hudi-client/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java index 08e9802517656..d2cde542ecc6e 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java @@ -124,7 +124,7 @@ public KeyLookupResult getLookupResult() { } /** - * Encapsulates the result from a key lookup + * Encapsulates the result from a key lookup. */ public static class KeyLookupResult { diff --git a/hudi-client/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java index 800c893e757f4..075be1ae73f3b 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java @@ -77,7 +77,7 @@ public HoodieMergeHandle(HoodieWriteConfig config, String commitTime, HoodieTabl } /** - * Called by compactor code path + * Called by compactor code path. */ public HoodieMergeHandle(HoodieWriteConfig config, String commitTime, HoodieTable hoodieTable, Map> keyToNewRecords, String fileId, HoodieDataFile dataFileToBeMerged) { @@ -108,7 +108,7 @@ public Schema getWriterSchema() { } /** - * Determines whether we can accept the incoming records, into the current file, depending on + * Determines whether we can accept the incoming records, into the current file. Depending on *

* - Whether it belongs to the same partitionPath as existing records - Whether the current file written bytes lt max * file size @@ -139,14 +139,14 @@ public void write(HoodieRecord record, Option avroRecord, Option< } /** - * Rewrite the GenericRecord with the Schema containing the Hoodie Metadata fields + * Rewrite the GenericRecord with the Schema containing the Hoodie Metadata fields. */ protected GenericRecord rewriteRecord(GenericRecord record) { return HoodieAvroUtils.rewriteRecord(record, writerSchema); } /** - * Extract old file path, initialize StorageWriter and WriteStatus + * Extract old file path, initialize StorageWriter and WriteStatus. */ private void init(String fileId, String partitionPath, HoodieDataFile dataFileToBeMerged) { logger.info("partitionPath:" + partitionPath + ", fileId to be merged:" + fileId); @@ -189,7 +189,7 @@ private void init(String fileId, String partitionPath, HoodieDataFile dataFileTo } /** - * Load the new incoming records in a map and return partitionPath + * Load the new incoming records in a map and return partitionPath. */ private String init(String fileId, Iterator> newRecordsItr) { try { diff --git a/hudi-client/src/main/java/org/apache/hudi/io/HoodieRangeInfoHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieRangeInfoHandle.java index ea1d84ea88fa7..3220766574d57 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieRangeInfoHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieRangeInfoHandle.java @@ -28,7 +28,7 @@ import org.apache.hadoop.fs.Path; /** - * Extract range information for a given file slice + * Extract range information for a given file slice. */ public class HoodieRangeInfoHandle extends HoodieReadHandle { diff --git a/hudi-client/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java index 95b994aee7830..c7e6f9a2cafef 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java @@ -90,7 +90,7 @@ public Path makeNewPath(String partitionPath) { } /** - * Creates an empty marker file corresponding to storage writer path + * Creates an empty marker file corresponding to storage writer path. * * @param partitionPath Partition path */ @@ -105,7 +105,7 @@ protected void createMarkerFile(String partitionPath) { } /** - * THe marker path will be /.hoodie/.temp//2019/04/25/filename + * THe marker path will be /.hoodie/.temp//2019/04/25/filename. */ private Path makeNewMarkerPath(String partitionPath) { Path markerRootPath = new Path(hoodieTable.getMetaClient().getMarkerFolderPath(instantTime)); @@ -123,7 +123,7 @@ public Schema getWriterSchema() { } /** - * Determines whether we can accept the incoming records, into the current file, depending on + * Determines whether we can accept the incoming records, into the current file. Depending on *

* - Whether it belongs to the same partitionPath as existing records - Whether the current file written bytes lt max * file size @@ -154,7 +154,7 @@ public void write(HoodieRecord record, Option avroRecord, Option< } /** - * Rewrite the GenericRecord with the Schema containing the Hoodie Metadata fields + * Rewrite the GenericRecord with the Schema containing the Hoodie Metadata fields. */ protected GenericRecord rewriteRecord(GenericRecord record) { return HoodieAvroUtils.rewriteRecord(record, writerSchema); diff --git a/hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieCompactor.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieCompactor.java index b62f8b487100f..afd76911c152e 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieCompactor.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieCompactor.java @@ -32,12 +32,12 @@ import java.util.Set; /** - * A HoodieCompactor runs compaction on a hoodie table + * A HoodieCompactor runs compaction on a hoodie table. */ public interface HoodieCompactor extends Serializable { /** - * Generate a new compaction plan for scheduling + * Generate a new compaction plan for scheduling. * * @param jsc Spark Context * @param hoodieTable Hoodie Table @@ -51,7 +51,7 @@ HoodieCompactionPlan generateCompactionPlan(JavaSparkContext jsc, HoodieTable ho String compactionCommitTime, Set fgIdsInPendingCompactions) throws IOException; /** - * Execute compaction operations and report back status + * Execute compaction operations and report back status. */ JavaRDD compact(JavaSparkContext jsc, HoodieCompactionPlan compactionPlan, HoodieTable hoodieTable, HoodieWriteConfig config, String compactionInstantTime) throws IOException; diff --git a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedIOCompactionStrategy.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedIOCompactionStrategy.java index 0d51df599f9fb..c84df1b820727 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedIOCompactionStrategy.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedIOCompactionStrategy.java @@ -28,7 +28,7 @@ /** * CompactionStrategy which looks at total IO to be done for the compaction (read + write) and limits the list of - * compactions to be under a configured limit on the IO + * compactions to be under a configured limit on the IO. * * @see CompactionStrategy */ diff --git a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/CompactionStrategy.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/CompactionStrategy.java index bdd24970c7c59..79a14b64cab2f 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/CompactionStrategy.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/CompactionStrategy.java @@ -116,7 +116,7 @@ public List orderAndFilter(HoodieWriteConfig writeCon } /** - * Filter the partition paths based on compaction strategy + * Filter the partition paths based on compaction strategy. * * @param writeConfig * @param allPartitionPaths diff --git a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/LogFileSizeBasedCompactionStrategy.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/LogFileSizeBasedCompactionStrategy.java index b7b3863aababd..4c9ee5fd2ee7b 100644 --- a/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/LogFileSizeBasedCompactionStrategy.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/LogFileSizeBasedCompactionStrategy.java @@ -32,7 +32,7 @@ /** * LogFileSizeBasedCompactionStrategy orders the compactions based on the total log files size and limits the - * compactions within a configured IO bound + * compactions within a configured IO bound. * * @see BoundedIOCompactionStrategy * @see CompactionStrategy diff --git a/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporter.java b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporter.java index 45ce852faecaa..de52f35b156c4 100644 --- a/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporter.java +++ b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporter.java @@ -26,12 +26,12 @@ public abstract class MetricsReporter { /** - * Push out metrics at scheduled intervals + * Push out metrics at scheduled intervals. */ public abstract void start(); /** - * Deterministically push out metrics + * Deterministically push out metrics. */ public abstract void report(); diff --git a/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java index b98c5d2a90760..b6c8b6f4f124f 100644 --- a/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java @@ -84,7 +84,7 @@ import scala.Tuple2; /** - * Implementation of a very heavily read-optimized Hoodie Table where + * Implementation of a very heavily read-optimized Hoodie Table where. *

* INSERTS - Produce new files, block aligned to desired size (or) Merge with the smallest existing file, to expand it *

@@ -273,7 +273,7 @@ public Iterator> handleInsertPartition(String commitTime, Inte } /** - * Generates List of files to be cleaned + * Generates List of files to be cleaned. * * @param jsc JavaSparkContext * @return Cleaner Plan @@ -389,7 +389,7 @@ private List generateRollbackRequests(HoodieInstant instantToRo } /** - * Delete Inflight instant if enabled + * Delete Inflight instant if enabled. * * @param deleteInstant Enable Deletion of Inflight instant * @param activeTimeline Hoodie active timeline @@ -414,7 +414,7 @@ enum BucketType { } /** - * Consumer that dequeues records from queue and sends to Merge Handle + * Consumer that dequeues records from queue and sends to Merge Handle. */ private static class UpdateHandler extends BoundedInMemoryQueueConsumer { @@ -474,7 +474,7 @@ private PartitionCleanStat merge(PartitionCleanStat other) { } /** - * Helper class for a small file's location and its actual size on disk + * Helper class for a small file's location and its actual size on disk. */ static class SmallFile implements Serializable { @@ -493,7 +493,7 @@ public String toString() { /** * Helper class for an insert bucket along with the weight [0.0, 0.1] that defines the amount of incoming inserts that - * should be allocated to the bucket + * should be allocated to the bucket. */ class InsertBucket implements Serializable { @@ -512,7 +512,7 @@ public String toString() { } /** - * Helper class for a bucket's type (INSERT and UPDATE) and its file location + * Helper class for a bucket's type (INSERT and UPDATE) and its file location. */ class BucketInfo implements Serializable { @@ -530,16 +530,16 @@ public String toString() { } /** - * Packs incoming records to be upserted, into buckets (1 bucket = 1 RDD partition) + * Packs incoming records to be upserted, into buckets (1 bucket = 1 RDD partition). */ class UpsertPartitioner extends Partitioner { /** - * List of all small files to be corrected + * List of all small files to be corrected. */ List smallFiles = new ArrayList(); /** - * Total number of RDD partitions, is determined by total buckets we want to pack the incoming workload into + * Total number of RDD partitions, is determined by total buckets we want to pack the incoming workload into. */ private int totalBuckets = 0; /** @@ -560,7 +560,7 @@ class UpsertPartitioner extends Partitioner { private HashMap bucketInfoMap; /** - * Rolling stats for files + * Rolling stats for files. */ protected HoodieRollingStatMetadata rollingStatMetadata; protected long averageRecordSize; @@ -672,7 +672,7 @@ private void assignInserts(WorkloadProfile profile) { } /** - * Returns a list of small files in the given partition path + * Returns a list of small files in the given partition path. */ protected List getSmallFiles(String partitionPath) { diff --git a/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java index e488e9f359396..da29d9800a0b0 100644 --- a/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java @@ -201,7 +201,7 @@ public List rollback(JavaSparkContext jsc, String commit, bo /** * Generate all rollback requests that we need to perform for rolling back this action without actually performing - * rolling back + * rolling back. * * @param jsc JavaSparkContext * @param instantToRollback Instant to Rollback diff --git a/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java index 1ea174d46950f..b7f485de3562b 100644 --- a/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java @@ -69,7 +69,7 @@ import java.util.stream.Stream; /** - * Abstract implementation of a HoodieTable + * Abstract implementation of a HoodieTable. */ public abstract class HoodieTable implements Serializable { @@ -111,17 +111,17 @@ public static HoodieTable getHoodieTable(Hood } /** - * Provides a partitioner to perform the upsert operation, based on the workload profile + * Provides a partitioner to perform the upsert operation, based on the workload profile. */ public abstract Partitioner getUpsertPartitioner(WorkloadProfile profile); /** - * Provides a partitioner to perform the insert operation, based on the workload profile + * Provides a partitioner to perform the insert operation, based on the workload profile. */ public abstract Partitioner getInsertPartitioner(WorkloadProfile profile); /** - * Return whether this HoodieTable implementation can benefit from workload profiling + * Return whether this HoodieTable implementation can benefit from workload profiling. */ public abstract boolean isWorkloadProfileNeeded(); @@ -138,84 +138,84 @@ public Configuration getHadoopConf() { } /** - * Get the view of the file system for this table + * Get the view of the file system for this table. */ public TableFileSystemView getFileSystemView() { return new HoodieTableFileSystemView(metaClient, getCompletedCommitsTimeline()); } /** - * Get the read optimized view of the file system for this table + * Get the read optimized view of the file system for this table. */ public TableFileSystemView.ReadOptimizedView getROFileSystemView() { return getViewManager().getFileSystemView(metaClient.getBasePath()); } /** - * Get the real time view of the file system for this table + * Get the real time view of the file system for this table. */ public TableFileSystemView.RealtimeView getRTFileSystemView() { return getViewManager().getFileSystemView(metaClient.getBasePath()); } /** - * Get complete view of the file system for this table with ability to force sync + * Get complete view of the file system for this table with ability to force sync. */ public SyncableFileSystemView getHoodieView() { return getViewManager().getFileSystemView(metaClient.getBasePath()); } /** - * Get only the completed (no-inflights) commit + deltacommit timeline + * Get only the completed (no-inflights) commit + deltacommit timeline. */ public HoodieTimeline getCompletedCommitsTimeline() { return metaClient.getCommitsTimeline().filterCompletedInstants(); } /** - * Get only the completed (no-inflights) commit timeline + * Get only the completed (no-inflights) commit timeline. */ public HoodieTimeline getCompletedCommitTimeline() { return metaClient.getCommitTimeline().filterCompletedInstants(); } /** - * Get only the inflights (no-completed) commit timeline + * Get only the inflights (no-completed) commit timeline. */ public HoodieTimeline getInflightCommitTimeline() { return metaClient.getCommitsTimeline().filterInflightsExcludingCompaction(); } /** - * Get only the completed (no-inflights) clean timeline + * Get only the completed (no-inflights) clean timeline. */ public HoodieTimeline getCompletedCleanTimeline() { return getActiveTimeline().getCleanerTimeline().filterCompletedInstants(); } /** - * Get clean timeline + * Get clean timeline. */ public HoodieTimeline getCleanTimeline() { return getActiveTimeline().getCleanerTimeline(); } /** - * Get only the completed (no-inflights) savepoint timeline + * Get only the completed (no-inflights) savepoint timeline. */ public HoodieTimeline getCompletedSavepointTimeline() { return getActiveTimeline().getSavePointTimeline().filterCompletedInstants(); } /** - * Get the list of savepoints in this table + * Get the list of savepoints in this table. */ public List getSavepoints() { return getCompletedSavepointTimeline().getInstants().map(HoodieInstant::getTimestamp).collect(Collectors.toList()); } /** - * Get the list of data file names savepointed + * Get the list of data file names savepointed. */ public Stream getSavepointedDataFiles(String savepointTime) { if (!getSavepoints().contains(savepointTime)) { @@ -237,26 +237,26 @@ public HoodieActiveTimeline getActiveTimeline() { } /** - * Return the index + * Return the index. */ public HoodieIndex getIndex() { return index; } /** - * Perform the ultimate IO for a given upserted (RDD) partition + * Perform the ultimate IO for a given upserted (RDD) partition. */ public abstract Iterator> handleUpsertPartition(String commitTime, Integer partition, Iterator> recordIterator, Partitioner partitioner); /** - * Perform the ultimate IO for a given inserted (RDD) partition + * Perform the ultimate IO for a given inserted (RDD) partition. */ public abstract Iterator> handleInsertPartition(String commitTime, Integer partition, Iterator> recordIterator, Partitioner partitioner); /** - * Schedule compaction for the instant time + * Schedule compaction for the instant time. * * @param jsc Spark Context * @param instantTime Instant Time for scheduling compaction @@ -265,7 +265,7 @@ public abstract Iterator> handleInsertPartition(String commitT public abstract HoodieCompactionPlan scheduleCompaction(JavaSparkContext jsc, String instantTime); /** - * Run Compaction on the table. Compaction arranges the data so that it is optimized for data access + * Run Compaction on the table. Compaction arranges the data so that it is optimized for data access. * * @param jsc Spark Context * @param compactionInstantTime Instant Time @@ -275,7 +275,7 @@ public abstract JavaRDD compact(JavaSparkContext jsc, String compac HoodieCompactionPlan compactionPlan); /** - * Generates list of files that are eligible for cleaning + * Generates list of files that are eligible for cleaning. * * @param jsc Java Spark Context * @return Cleaner Plan containing list of files to be deleted. @@ -283,7 +283,7 @@ public abstract JavaRDD compact(JavaSparkContext jsc, String compac public abstract HoodieCleanerPlan scheduleClean(JavaSparkContext jsc); /** - * Cleans the files listed in the cleaner plan associated with clean instant + * Cleans the files listed in the cleaner plan associated with clean instant. * * @param jsc Java Spark Context * @param cleanInstant Clean Instant @@ -300,7 +300,7 @@ public abstract List rollback(JavaSparkContext jsc, String c throws IOException; /** - * Finalize the written data onto storage. Perform any final cleanups + * Finalize the written data onto storage. Perform any final cleanups. * * @param jsc Spark Context * @param stats List of HoodieWriteStats @@ -312,7 +312,7 @@ public void finalizeWrite(JavaSparkContext jsc, String instantTs, List performRollback(JavaSparkContext jsc, HoodieInst } /** - * Helper to merge 2 rollback-stats for a given partition + * Helper to merge 2 rollback-stats for a given partition. * * @param stat1 HoodieRollbackStat * @param stat2 HoodieRollbackStat @@ -177,7 +177,7 @@ private HoodieRollbackStat mergeRollbackStat(HoodieRollbackStat stat1, HoodieRol } /** - * Common method used for cleaning out parquet files under a partition path during rollback of a set of commits + * Common method used for cleaning out parquet files under a partition path during rollback of a set of commits. */ private Map deleteCleanedFiles(HoodieTableMetaClient metaClient, HoodieWriteConfig config, Map results, String partitionPath, PathFilter filter) throws IOException { @@ -193,7 +193,7 @@ private Map deleteCleanedFiles(HoodieTableMetaClient metaCl } /** - * Common method used for cleaning out parquet files under a partition path during rollback of a set of commits + * Common method used for cleaning out parquet files under a partition path during rollback of a set of commits. */ private Map deleteCleanedFiles(HoodieTableMetaClient metaClient, HoodieWriteConfig config, Map results, String commit, String partitionPath) throws IOException { diff --git a/hudi-client/src/main/java/org/apache/hudi/table/RollbackRequest.java b/hudi-client/src/main/java/org/apache/hudi/table/RollbackRequest.java index 326f347ad0fa9..57db41842a38c 100644 --- a/hudi-client/src/main/java/org/apache/hudi/table/RollbackRequest.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/RollbackRequest.java @@ -22,39 +22,39 @@ import org.apache.hudi.common.util.Option; /** - * Request for performing one rollback action + * Request for performing one rollback action. */ public class RollbackRequest { /** - * Rollback Action Types + * Rollback Action Types. */ public enum RollbackAction { DELETE_DATA_FILES_ONLY, DELETE_DATA_AND_LOG_FILES, APPEND_ROLLBACK_BLOCK } /** - * Partition path that needs to be rolled-back + * Partition path that needs to be rolled-back. */ private final String partitionPath; /** - * Rollback Instant + * Rollback Instant. */ private final HoodieInstant rollbackInstant; /** - * FileId in case of appending rollback block + * FileId in case of appending rollback block. */ private final Option fileId; /** - * Latest base instant needed for appending rollback block instant + * Latest base instant needed for appending rollback block instant. */ private final Option latestBaseInstant; /** - * Rollback Action + * Rollback Action. */ private final RollbackAction rollbackAction; diff --git a/hudi-client/src/main/java/org/apache/hudi/table/WorkloadProfile.java b/hudi-client/src/main/java/org/apache/hudi/table/WorkloadProfile.java index 31d2e1daea953..029276ad357cb 100644 --- a/hudi-client/src/main/java/org/apache/hudi/table/WorkloadProfile.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/WorkloadProfile.java @@ -33,19 +33,19 @@ import scala.Tuple2; /** - * Information about incoming records for upsert/insert obtained either via sampling or introspecting the data fully + * Information about incoming records for upsert/insert obtained either via sampling or introspecting the data fully. *

* TODO(vc): Think about obtaining this directly from index.tagLocation */ public class WorkloadProfile implements Serializable { /** - * Input workload + * Input workload. */ private final JavaRDD> taggedRecords; /** - * Computed workload profile + * Computed workload profile. */ private final HashMap partitionPathStatMap; diff --git a/hudi-client/src/test/java/org/apache/hudi/TestAsyncCompaction.java b/hudi-client/src/test/java/org/apache/hudi/TestAsyncCompaction.java index 5a91ca1645a25..451a2b11425fb 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestAsyncCompaction.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestAsyncCompaction.java @@ -65,7 +65,7 @@ import static org.junit.Assert.assertTrue; /** - * Test Cases for Async Compaction and Ingestion interaction + * Test Cases for Async Compaction and Ingestion interaction. */ public class TestAsyncCompaction extends TestHoodieClientBase { @@ -400,7 +400,7 @@ public void testInterleavedCompaction() throws Exception { } /** - * HELPER METHODS FOR TESTING + * HELPER METHODS FOR TESTING. **/ private void validateDeltaCommit(String latestDeltaCommit, diff --git a/hudi-client/src/test/java/org/apache/hudi/TestCleaner.java b/hudi-client/src/test/java/org/apache/hudi/TestCleaner.java index 200575abf5fce..57d5fc022b66d 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestCleaner.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestCleaner.java @@ -87,7 +87,7 @@ import static org.junit.Assert.assertTrue; /** - * Test Cleaning related logic + * Test Cleaning related logic. */ public class TestCleaner extends TestHoodieClientBase { @@ -95,7 +95,7 @@ public class TestCleaner extends TestHoodieClientBase { private static Logger logger = LogManager.getLogger(TestHoodieClientBase.class); /** - * Helper method to do first batch of insert for clean by versions/commits tests + * Helper method to do first batch of insert for clean by versions/commits tests. * * @param cfg Hoodie Write Config * @param client Hoodie Client @@ -140,7 +140,7 @@ private String insertFirstBigBatchForClientCleanerTest(HoodieWriteConfig cfg, Ho } /** - * Test Clean-By-Versions using insert/upsert API + * Test Clean-By-Versions using insert/upsert API. */ @Test public void testInsertAndCleanByVersions() throws Exception { @@ -148,7 +148,7 @@ public void testInsertAndCleanByVersions() throws Exception { } /** - * Test Clean-By-Versions using prepped versions of insert/upsert API + * Test Clean-By-Versions using prepped versions of insert/upsert API. */ @Test public void testInsertPreppedAndCleanByVersions() throws Exception { @@ -157,7 +157,7 @@ public void testInsertPreppedAndCleanByVersions() throws Exception { } /** - * Test Clean-By-Versions using bulk-insert/upsert API + * Test Clean-By-Versions using bulk-insert/upsert API. */ @Test public void testBulkInsertAndCleanByVersions() throws Exception { @@ -165,7 +165,7 @@ public void testBulkInsertAndCleanByVersions() throws Exception { } /** - * Test Clean-By-Versions using prepped versions of bulk-insert/upsert API + * Test Clean-By-Versions using prepped versions of bulk-insert/upsert API. */ @Test public void testBulkInsertPreppedAndCleanByVersions() throws Exception { @@ -175,7 +175,7 @@ public void testBulkInsertPreppedAndCleanByVersions() throws Exception { } /** - * Test Helper for Cleaning by versions logic from HoodieWriteClient API perspective + * Test Helper for Cleaning by versions logic from HoodieWriteClient API perspective. * * @param insertFn Insert API to be tested * @param upsertFn Upsert API to be tested @@ -301,7 +301,7 @@ private void testInsertAndCleanByVersions( } /** - * Test Clean-By-Versions using insert/upsert API + * Test Clean-By-Versions using insert/upsert API. */ @Test public void testInsertAndCleanByCommits() throws Exception { @@ -309,7 +309,7 @@ public void testInsertAndCleanByCommits() throws Exception { } /** - * Test Clean-By-Versions using prepped version of insert/upsert API + * Test Clean-By-Versions using prepped version of insert/upsert API. */ @Test public void testInsertPreppedAndCleanByCommits() throws Exception { @@ -317,7 +317,7 @@ public void testInsertPreppedAndCleanByCommits() throws Exception { } /** - * Test Clean-By-Versions using prepped versions of bulk-insert/upsert API + * Test Clean-By-Versions using prepped versions of bulk-insert/upsert API. */ @Test public void testBulkInsertPreppedAndCleanByCommits() throws Exception { @@ -327,7 +327,7 @@ public void testBulkInsertPreppedAndCleanByCommits() throws Exception { } /** - * Test Clean-By-Versions using bulk-insert/upsert API + * Test Clean-By-Versions using bulk-insert/upsert API. */ @Test public void testBulkInsertAndCleanByCommits() throws Exception { @@ -335,7 +335,7 @@ public void testBulkInsertAndCleanByCommits() throws Exception { } /** - * Test Helper for Cleaning by versions logic from HoodieWriteClient API perspective + * Test Helper for Cleaning by versions logic from HoodieWriteClient API perspective. * * @param insertFn Insert API to be tested * @param upsertFn Upsert API to be tested @@ -407,7 +407,7 @@ private void testInsertAndCleanByCommits( } /** - * Helper to run cleaner and collect Clean Stats + * Helper to run cleaner and collect Clean Stats. * * @param config HoodieWriteConfig */ @@ -416,7 +416,7 @@ private List runCleaner(HoodieWriteConfig config) { } /** - * Helper to run cleaner and collect Clean Stats + * Helper to run cleaner and collect Clean Stats. * * @param config HoodieWriteConfig */ @@ -463,7 +463,7 @@ private List runCleaner(HoodieWriteConfig config, boolean simul } /** - * Test HoodieTable.clean() Cleaning by versions logic + * Test HoodieTable.clean() Cleaning by versions logic. */ @Test public void testKeepLatestFileVersions() throws IOException { @@ -555,7 +555,7 @@ public void testKeepLatestFileVersions() throws IOException { } /** - * Test HoodieTable.clean() Cleaning by versions logic for MOR table with Log files + * Test HoodieTable.clean() Cleaning by versions logic for MOR table with Log files. */ @Test public void testKeepLatestFileVersionsMOR() throws IOException { @@ -701,7 +701,7 @@ private void testCleanMetadataPathEquality(HoodieCleanMetadata metadata, Map createMarkerFiles(String commitTime, int numFiles) throws I } /*** - * Helper method to return temporary files count + * Helper method to return temporary files count. * * @return Number of temporary files found * @throws IOException in case of error diff --git a/hudi-client/src/test/java/org/apache/hudi/TestClientRollback.java b/hudi-client/src/test/java/org/apache/hudi/TestClientRollback.java index 28d2351e8598a..45cfbd2e7efdc 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestClientRollback.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestClientRollback.java @@ -47,12 +47,12 @@ import static org.junit.Assert.fail; /** - * Test Cases for rollback of snapshots and commits + * Test Cases for rollback of snapshots and commits. */ public class TestClientRollback extends TestHoodieClientBase { /** - * Test case for rollback-savepoint interaction + * Test case for rollback-savepoint interaction. */ @Test public void testSavepointAndRollback() throws Exception { @@ -165,7 +165,7 @@ public void testSavepointAndRollback() throws Exception { } /** - * Test Cases for effects of rollbacking completed/inflight commits + * Test Cases for effects of rollbacking completed/inflight commits. */ @Test public void testRollbackCommit() throws Exception { @@ -255,7 +255,7 @@ public void testRollbackCommit() throws Exception { } /** - * Test auto-rollback of commits which are in flight + * Test auto-rollback of commits which are in flight. */ @Test public void testAutoRollbackInflightCommit() throws Exception { diff --git a/hudi-client/src/test/java/org/apache/hudi/TestCompactionAdminClient.java b/hudi-client/src/test/java/org/apache/hudi/TestCompactionAdminClient.java index 7d23ecde45f67..1ddc0fd764982 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestCompactionAdminClient.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestCompactionAdminClient.java @@ -163,7 +163,7 @@ private void validateRepair(String ingestionInstant, String compactionInstant, i } /** - * Enssure compaction plan is valid + * Enssure compaction plan is valid. * * @param compactionInstant Compaction Instant */ @@ -206,7 +206,7 @@ private void validateRenameFiles(List> rename } /** - * Validate Unschedule operations + * Validate Unschedule operations. */ private List> validateUnSchedulePlan(CompactionAdminClient client, String ingestionInstant, String compactionInstant, int numEntriesPerInstant, int expNumRenames) throws Exception { @@ -215,7 +215,7 @@ private List> validateUnSchedulePlan(Compacti } /** - * Validate Unschedule operations + * Validate Unschedule operations. */ private List> validateUnSchedulePlan(CompactionAdminClient client, String ingestionInstant, String compactionInstant, int numEntriesPerInstant, int expNumRenames, @@ -287,7 +287,7 @@ private List> validateUnSchedulePlan(Compacti } /** - * Validate Unschedule operations + * Validate Unschedule operations. */ private void validateUnScheduleFileId(CompactionAdminClient client, String ingestionInstant, String compactionInstant, CompactionOperation op, int expNumRenames) throws Exception { diff --git a/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientBase.java b/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientBase.java index 7e99c9ce50b8d..d35a1e5a3aa19 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientBase.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientBase.java @@ -66,7 +66,7 @@ import static org.junit.Assert.assertTrue; /** - * Base Class providing setup/cleanup and utility methods for testing Hoodie Client facing tests + * Base Class providing setup/cleanup and utility methods for testing Hoodie Client facing tests. */ public class TestHoodieClientBase extends HoodieClientTestHarness { @@ -104,7 +104,7 @@ protected HoodieReadClient getHoodieReadClient(String basePath) { } /** - * Get Default HoodieWriteConfig for tests + * Get Default HoodieWriteConfig for tests. * * @return Default Hoodie Write Config for tests */ @@ -114,7 +114,7 @@ protected HoodieWriteConfig getConfig() { /** - * Get Config builder with default configs set + * Get Config builder with default configs set. * * @return Config Builder */ @@ -123,7 +123,7 @@ HoodieWriteConfig.Builder getConfigBuilder() { } /** - * Get Config builder with default configs set + * Get Config builder with default configs set. * * @return Config Builder */ @@ -147,7 +147,7 @@ protected HoodieTable getHoodieTable(HoodieTableMetaClient metaClient, HoodieWri } /** - * Assert no failures in writing hoodie files + * Assert no failures in writing hoodie files. * * @param statuses List of Write Status */ @@ -159,7 +159,7 @@ static void assertNoWriteErrors(List statuses) { } /** - * Ensure presence of partition meta-data at known depth + * Ensure presence of partition meta-data at known depth. * * @param partitionPaths Partition paths to check * @param fs File System @@ -175,7 +175,7 @@ void assertPartitionMetadata(String[] partitionPaths, FileSystem fs) throws IOEx } /** - * Ensure records have location field set + * Ensure records have location field set. * * @param taggedRecords Tagged Records * @param commitTime Commit Timestamp @@ -189,7 +189,7 @@ void checkTaggedRecords(List taggedRecords, String commitTime) { } /** - * Assert that there is no duplicate key at the partition level + * Assert that there is no duplicate key at the partition level. * * @param records List of Hoodie records */ @@ -252,7 +252,7 @@ private Function2, String, Integer> wrapDeleteKeysGenFunctionFor } /** - * Generate wrapper for record generation function for testing Prepped APIs + * Generate wrapper for record generation function for testing Prepped APIs. * * @param isPreppedAPI Flag to indicate if this is for testing prepped-version of APIs * @param writeConfig Hoodie Write Config @@ -269,7 +269,7 @@ Function2, String, Integer> generateWrapRecordsFn(boolean isP } /** - * Generate wrapper for delete key generation function for testing Prepped APIs + * Generate wrapper for delete key generation function for testing Prepped APIs. * * @param isPreppedAPI Flag to indicate if this is for testing prepped-version of APIs * @param writeConfig Hoodie Write Config @@ -286,7 +286,7 @@ Function2, String, Integer> generateWrapDeleteKeysFn(boolean isP } /** - * Helper to insert first batch of records and do regular assertions on the state after successful completion + * Helper to insert first batch of records and do regular assertions on the state after successful completion. * * @param writeConfig Hoodie Write Config * @param client Hoodie Write Client @@ -312,7 +312,7 @@ JavaRDD insertFirstBatch(HoodieWriteConfig writeConfig, HoodieWrite } /** - * Helper to upsert batch of records and do regular assertions on the state after successful completion + * Helper to upsert batch of records and do regular assertions on the state after successful completion. * * @param writeConfig Hoodie Write Config * @param client Hoodie Write Client @@ -344,7 +344,7 @@ JavaRDD updateBatch(HoodieWriteConfig writeConfig, HoodieWriteClien } /** - * Helper to delete batch of keys and do regular assertions on the state after successful completion + * Helper to delete batch of keys and do regular assertions on the state after successful completion. * * @param writeConfig Hoodie Write Config * @param client Hoodie Write Client @@ -374,7 +374,7 @@ JavaRDD deleteBatch(HoodieWriteConfig writeConfig, HoodieWriteClien } /** - * Helper to insert/upsert batch of records and do regular assertions on the state after successful completion + * Helper to insert/upsert batch of records and do regular assertions on the state after successful completion. * * @param client Hoodie Write Client * @param newCommitTime New Commit Timestamp to be used @@ -445,7 +445,7 @@ JavaRDD writeBatch(HoodieWriteClient client, String newCommitTime, } /** - * Helper to delete batch of hoodie keys and do regular assertions on the state after successful completion + * Helper to delete batch of hoodie keys and do regular assertions on the state after successful completion. * * @param client Hoodie Write Client * @param newCommitTime New Commit Timestamp to be used @@ -507,7 +507,7 @@ JavaRDD deleteBatch(HoodieWriteClient client, String newCommitTime, } /** - * Get Cleaner state corresponding to a partition path + * Get Cleaner state corresponding to a partition path. * * @param hoodieCleanStatsTwo List of Clean Stats * @param partitionPath Partition path for filtering @@ -518,7 +518,7 @@ HoodieCleanStat getCleanStat(List hoodieCleanStatsTwo, String p } /** - * Utility to simulate commit touching files in a partition + * Utility to simulate commit touching files in a partition. * * @param files List of file-Ids to be touched * @param partitionPath Partition @@ -532,7 +532,7 @@ void updateAllFilesInPartition(List files, String partitionPath, String } /** - * Helper methods to create new data files in a partition + * Helper methods to create new data files in a partition. * * @param partitionPath Partition * @param commitTime Commit Timestamp diff --git a/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientOnCopyOnWriteStorage.java index 790f1b3224495..7cf2da7c615f8 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientOnCopyOnWriteStorage.java @@ -77,7 +77,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends TestHoodieClientBase { /** - * Test Auto Commit behavior for HoodieWriteClient insert API + * Test Auto Commit behavior for HoodieWriteClient insert API. */ @Test public void testAutoCommitOnInsert() throws Exception { @@ -85,7 +85,7 @@ public void testAutoCommitOnInsert() throws Exception { } /** - * Test Auto Commit behavior for HoodieWriteClient insertPrepped API + * Test Auto Commit behavior for HoodieWriteClient insertPrepped API. */ @Test public void testAutoCommitOnInsertPrepped() throws Exception { @@ -93,7 +93,7 @@ public void testAutoCommitOnInsertPrepped() throws Exception { } /** - * Test Auto Commit behavior for HoodieWriteClient upsert API + * Test Auto Commit behavior for HoodieWriteClient upsert API. */ @Test public void testAutoCommitOnUpsert() throws Exception { @@ -101,7 +101,7 @@ public void testAutoCommitOnUpsert() throws Exception { } /** - * Test Auto Commit behavior for HoodieWriteClient upsert Prepped API + * Test Auto Commit behavior for HoodieWriteClient upsert Prepped API. */ @Test public void testAutoCommitOnUpsertPrepped() throws Exception { @@ -109,7 +109,7 @@ public void testAutoCommitOnUpsertPrepped() throws Exception { } /** - * Test Auto Commit behavior for HoodieWriteClient bulk-insert API + * Test Auto Commit behavior for HoodieWriteClient bulk-insert API. */ @Test public void testAutoCommitOnBulkInsert() throws Exception { @@ -117,7 +117,7 @@ public void testAutoCommitOnBulkInsert() throws Exception { } /** - * Test Auto Commit behavior for HoodieWriteClient bulk-insert prepped API + * Test Auto Commit behavior for HoodieWriteClient bulk-insert prepped API. */ @Test public void testAutoCommitOnBulkInsertPrepped() throws Exception { @@ -126,7 +126,7 @@ public void testAutoCommitOnBulkInsertPrepped() throws Exception { } /** - * Test auto-commit by applying write function + * Test auto-commit by applying write function. * * @param writeFn One of HoodieWriteClient Write API * @throws Exception in case of failure @@ -152,7 +152,7 @@ private void testAutoCommit(Function3, HoodieWriteClient, J } /** - * Test De-duplication behavior for HoodieWriteClient insert API + * Test De-duplication behavior for HoodieWriteClient insert API. */ @Test public void testDeduplicationOnInsert() throws Exception { @@ -160,7 +160,7 @@ public void testDeduplicationOnInsert() throws Exception { } /** - * Test De-duplication behavior for HoodieWriteClient bulk-insert API + * Test De-duplication behavior for HoodieWriteClient bulk-insert API. */ @Test public void testDeduplicationOnBulkInsert() throws Exception { @@ -168,7 +168,7 @@ public void testDeduplicationOnBulkInsert() throws Exception { } /** - * Test De-duplication behavior for HoodieWriteClient upsert API + * Test De-duplication behavior for HoodieWriteClient upsert API. */ @Test public void testDeduplicationOnUpsert() throws Exception { @@ -176,7 +176,7 @@ public void testDeduplicationOnUpsert() throws Exception { } /** - * Test Deduplication Logic for write function + * Test Deduplication Logic for write function. * * @param writeFn One of HoddieWriteClient non-prepped write APIs * @throws Exception in case of failure @@ -224,7 +224,7 @@ private void testDeduplication( } /** - * Build a test Hoodie WriteClient with dummy index to configure isGlobal flag + * Build a test Hoodie WriteClient with dummy index to configure isGlobal flag. * * @param isGlobal Flag to control HoodieIndex.isGlobal * @return Hoodie Write Client @@ -237,7 +237,7 @@ private HoodieWriteClient getWriteClientWithDummyIndex(final boolean isGlobal) t } /** - * Test Upsert API + * Test Upsert API. */ @Test public void testUpserts() throws Exception { @@ -245,7 +245,7 @@ public void testUpserts() throws Exception { } /** - * Test UpsertPrepped API + * Test UpsertPrepped API. */ @Test public void testUpsertsPrepped() throws Exception { @@ -253,7 +253,7 @@ public void testUpsertsPrepped() throws Exception { } /** - * Test one of HoodieWriteClient upsert(Prepped) APIs + * Test one of HoodieWriteClient upsert(Prepped) APIs. * * @param hoodieWriteConfig Write Config * @param writeFn One of Hoodie Write Function API @@ -291,7 +291,7 @@ private void testUpsertsInternal(HoodieWriteConfig hoodieWriteConfig, } /** - * Tesst deletion of records + * Tesst deletion of records. */ @Test public void testDeletes() throws Exception { @@ -318,7 +318,7 @@ public void testDeletes() throws Exception { -1, recordGenFunction, HoodieWriteClient::upsert, true, 200, 200, 1); /** - * Write 2 (deletes+writes) + * Write 2 (deletes+writes). */ String prevCommitTime = newCommitTime; newCommitTime = "004"; @@ -336,7 +336,7 @@ public void testDeletes() throws Exception { } /** - * Test scenario of new file-group getting added during upsert() + * Test scenario of new file-group getting added during upsert(). */ @Test public void testSmallInsertHandlingForUpserts() throws Exception { @@ -448,7 +448,7 @@ public void testSmallInsertHandlingForUpserts() throws Exception { } /** - * Test scenario of new file-group getting added during insert() + * Test scenario of new file-group getting added during insert(). */ @Test public void testSmallInsertHandlingForInserts() throws Exception { @@ -530,7 +530,7 @@ public void testSmallInsertHandlingForInserts() throws Exception { } /** - * Test delete with delete api + * Test delete with delete api. */ @Test public void testDeletesWithDeleteApi() throws Exception { @@ -659,7 +659,7 @@ private void testDeletes(HoodieWriteClient client, List previousRe } /** - * Test delete with delete api + * Test delete with delete api. */ @Test public void testDeletesWithoutInserts() throws Exception { @@ -688,7 +688,7 @@ public void testDeletesWithoutInserts() throws Exception { } /** - * Test to ensure commit metadata points to valid files + * Test to ensure commit metadata points to valid files. */ @Test public void testCommitWritesRelativePaths() throws Exception { @@ -735,7 +735,7 @@ public void testCommitWritesRelativePaths() throws Exception { } /** - * Test to ensure commit metadata points to valid files + * Test to ensure commit metadata points to valid files. */ @Test public void testRollingStatsInMetadata() throws Exception { @@ -810,7 +810,7 @@ public void testRollingStatsInMetadata() throws Exception { } /** - * Tests behavior of committing only when consistency is verified + * Tests behavior of committing only when consistency is verified. */ @Test public void testConsistencyCheckDuringFinalize() throws Exception { @@ -879,14 +879,14 @@ private Pair> testConsistencyCheck(HoodieTableMetaCli } /** - * Build Hoodie Write Config for small data file sizes + * Build Hoodie Write Config for small data file sizes. */ private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize) { return getSmallInsertWriteConfig(insertSplitSize, false); } /** - * Build Hoodie Write Config for small data file sizes + * Build Hoodie Write Config for small data file sizes. */ private HoodieWriteConfig getSmallInsertWriteConfig(int insertSplitSize, boolean useNullSchema) { HoodieWriteConfig.Builder builder = getConfigBuilder(useNullSchema ? NULL_SCHEMA : TRIP_EXAMPLE_SCHEMA); diff --git a/hudi-client/src/test/java/org/apache/hudi/TestHoodieReadClient.java b/hudi-client/src/test/java/org/apache/hudi/TestHoodieReadClient.java index 858e5ff59b080..03b38fba55dd6 100644 --- a/hudi-client/src/test/java/org/apache/hudi/TestHoodieReadClient.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestHoodieReadClient.java @@ -40,7 +40,7 @@ public class TestHoodieReadClient extends TestHoodieClientBase { /** - * Test ReadFilter API after writing new records using HoodieWriteClient.insert + * Test ReadFilter API after writing new records using HoodieWriteClient.insert. */ @Test public void testReadFilterExistAfterInsert() throws Exception { @@ -48,7 +48,7 @@ public void testReadFilterExistAfterInsert() throws Exception { } /** - * Test ReadFilter API after writing new records using HoodieWriteClient.insertPrepped + * Test ReadFilter API after writing new records using HoodieWriteClient.insertPrepped. */ @Test public void testReadFilterExistAfterInsertPrepped() throws Exception { @@ -56,7 +56,7 @@ public void testReadFilterExistAfterInsertPrepped() throws Exception { } /** - * Test ReadFilter API after writing new records using HoodieWriteClient.bulkInsert + * Test ReadFilter API after writing new records using HoodieWriteClient.bulkInsert. */ @Test public void testReadFilterExistAfterBulkInsert() throws Exception { @@ -64,7 +64,7 @@ public void testReadFilterExistAfterBulkInsert() throws Exception { } /** - * Test ReadFilter API after writing new records using HoodieWriteClient.bulkInsertPrepped + * Test ReadFilter API after writing new records using HoodieWriteClient.bulkInsertPrepped. */ @Test public void testReadFilterExistAfterBulkInsertPrepped() throws Exception { @@ -76,7 +76,7 @@ public void testReadFilterExistAfterBulkInsertPrepped() throws Exception { /** * Helper to write new records using one of HoodieWriteClient's write API and use ReadClient to test filterExists() - * API works correctly + * API works correctly. * * @param config Hoodie Write Config * @param writeFn Write Function for writing records @@ -111,7 +111,7 @@ private void testReadFilterExist(HoodieWriteConfig config, } /** - * Test tagLocation API after insert() + * Test tagLocation API after insert(). */ @Test public void testTagLocationAfterInsert() throws Exception { @@ -119,7 +119,7 @@ public void testTagLocationAfterInsert() throws Exception { } /** - * Test tagLocation API after insertPrepped() + * Test tagLocation API after insertPrepped(). */ @Test public void testTagLocationAfterInsertPrepped() throws Exception { @@ -128,7 +128,7 @@ public void testTagLocationAfterInsertPrepped() throws Exception { } /** - * Test tagLocation API after bulk-insert() + * Test tagLocation API after bulk-insert(). */ @Test public void testTagLocationAfterBulkInsert() throws Exception { @@ -137,7 +137,7 @@ public void testTagLocationAfterBulkInsert() throws Exception { } /** - * Test tagLocation API after bulkInsertPrepped() + * Test tagLocation API after bulkInsertPrepped(). */ @Test public void testTagLocationAfterBulkInsertPrepped() throws Exception { @@ -148,7 +148,7 @@ public void testTagLocationAfterBulkInsertPrepped() throws Exception { } /** - * Helper method to test tagLocation after using different HoodieWriteClient write APIS + * Helper method to test tagLocation after using different HoodieWriteClient write APIS. * * @param hoodieWriteConfig Write Config * @param insertFn Hoodie Write Client first Insert API diff --git a/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java b/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java index a7bd7af9ee93a..9c26ffcd2fc89 100644 --- a/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java +++ b/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java @@ -192,7 +192,7 @@ public static Dataset readSince(String basePath, SQLContext sqlContext, Hoo } /** - * Reads the paths under the a hoodie dataset out as a DataFrame + * Reads the paths under the a hoodie dataset out as a DataFrame. */ public static Dataset read(JavaSparkContext jsc, String basePath, SQLContext sqlContext, FileSystem fs, String... paths) { diff --git a/hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java b/hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java index facc964737a5d..e13e3dc749d07 100644 --- a/hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java +++ b/hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java @@ -41,7 +41,7 @@ import java.util.stream.Collectors; /** - * Utility methods to aid in testing MergeOnRead (workaround for HoodieReadClient for MOR) + * Utility methods to aid in testing MergeOnRead (workaround for HoodieReadClient for MOR). */ public class HoodieMergeOnReadTestUtils { diff --git a/hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryQueue.java b/hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryQueue.java index 5331c15255ffd..4e87c5c05524f 100644 --- a/hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryQueue.java +++ b/hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryQueue.java @@ -110,7 +110,7 @@ public void testRecordReading() throws Exception { } /** - * Test to ensure that we are reading all records from queue iterator when we have multiple producers + * Test to ensure that we are reading all records from queue iterator when we have multiple producers. */ @SuppressWarnings("unchecked") @Test(timeout = 60000) diff --git a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestKeyRangeLookupTree.java b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestKeyRangeLookupTree.java index 496e52b30bfc3..a30eab8bf5cd0 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestKeyRangeLookupTree.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestKeyRangeLookupTree.java @@ -31,7 +31,7 @@ import static junit.framework.TestCase.assertTrue; /** - * Tests {@link KeyRangeLookupTree} + * Tests {@link KeyRangeLookupTree}. */ public class TestKeyRangeLookupTree { @@ -59,7 +59,7 @@ public void testFileGroupLookUpOneEntry() { } /** - * Tests for many entries in the tree with same start value and different end values + * Tests for many entries in the tree with same start value and different end values. */ @Test public void testFileGroupLookUpManyEntriesWithSameStartValue() { @@ -78,7 +78,7 @@ public void testFileGroupLookUpManyEntriesWithSameStartValue() { } /** - * Tests for many duplicte entries in the tree + * Tests for many duplicte entries in the tree. */ @Test public void testFileGroupLookUpManyDulicateEntries() { @@ -158,7 +158,7 @@ private void testRangeOfInputs(long start, long end) { } /** - * Updates the expected matches for a given {@link KeyRangeNode} + * Updates the expected matches for a given {@link KeyRangeNode}. * * @param toInsert the {@link KeyRangeNode} to be inserted */ diff --git a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java index 6a46592ea18af..7b8d1ce28c795 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java @@ -318,7 +318,7 @@ private Dataset getRecords() { } /** - * Assert no failures in writing hoodie files + * Assert no failures in writing hoodie files. * * @param statuses List of Write Status */ @@ -340,7 +340,7 @@ HoodieWriteConfig.Builder getConfigBuilder() { } /** - * Overridden so that we can capture and inspect all success records + * Overridden so that we can capture and inspect all success records. */ public static class TestWriteStatus extends WriteStatus { diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java b/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java index b9ff1af9802f6..ff0fb648c2f13 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java @@ -1019,7 +1019,7 @@ public void testInsertsGeneratedIntoLogFilesRollbackAfterCompaction() throws Exc } /** - * Test to ensure rolling stats are correctly written to metadata file + * Test to ensure rolling stats are correctly written to metadata file. */ @Test public void testRollingStatsInMetadata() throws Exception { @@ -1118,7 +1118,7 @@ public void testRollingStatsInMetadata() throws Exception { } /** - * Test to ensure rolling stats are correctly written to the metadata file, identifies small files and corrects them + * Test to ensure rolling stats are correctly written to the metadata file, identifies small files and corrects them. */ @Test public void testRollingStatsWithSmallFileHandling() throws Exception { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/EmptyHoodieRecordPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/EmptyHoodieRecordPayload.java index e12a29ce2b603..783422fc648f2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/EmptyHoodieRecordPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/EmptyHoodieRecordPayload.java @@ -25,7 +25,7 @@ import org.apache.avro.generic.IndexedRecord; /** - * Empty payload used for deletions + * Empty payload used for deletions. */ public class EmptyHoodieRecordPayload implements HoodieRecordPayload { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java b/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java index 2635c3c1813a2..7b1bffb7e348d 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java @@ -310,7 +310,7 @@ public void testMultipleAppend() throws IOException, URISyntaxException, Interru * writer.getCurrentSize(); assertTrue("We just wrote a new block - size2 should be > size1", size2 > size1); * assertEquals("Write should be auto-flushed. The size reported by FileStatus and the writer should match", * size2, fs.getFileStatus(writer.getLogFile().getPath()).getLen()); writer.close(); } - **/ + */ @Test public void testAppendNotSupported() throws IOException, URISyntaxException, InterruptedException { diff --git a/hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieInputFormat.java b/hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieInputFormat.java index b22820e69ee16..49588ebc8f53a 100644 --- a/hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieInputFormat.java +++ b/hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieInputFormat.java @@ -21,7 +21,7 @@ import org.apache.hudi.hadoop.HoodieParquetInputFormat; /** - * Temporary class to allow seamless migration of com.uber.hoodie to org.apache.hudi + * Temporary class to allow seamless migration of com.uber.hoodie to org.apache.hudi . */ public class HoodieInputFormat extends HoodieParquetInputFormat { diff --git a/hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeInputFormat.java b/hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeInputFormat.java index 99d3c3a26e0fb..ec10e169db7ee 100644 --- a/hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeInputFormat.java +++ b/hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeInputFormat.java @@ -21,7 +21,7 @@ import org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat; /** - * Temporary class to allow seamless migration of com.uber.hoodie to org.apache.hudi + * Temporary class to allow seamless migration of com.uber.hoodie to org.apache.hudi . */ public class HoodieRealtimeInputFormat extends HoodieParquetRealtimeInputFormat { diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java index 51a9805dcc4a5..14ca2f4ed4d7d 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java @@ -54,7 +54,7 @@ public class HoodieROTablePathFilter implements PathFilter, Serializable { /** * Its quite common, to have all files from a given partition path be passed into accept(), cache the check for hoodie - * metadata for known partition paths and the latest versions of files + * metadata for known partition paths and the latest versions of files. */ private HashMap> hoodiePathCache; @@ -72,7 +72,7 @@ public HoodieROTablePathFilter() { } /** - * Obtain the path, two levels from provided path + * Obtain the path, two levels from provided path. * * @return said path if available, null otherwise */ diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/RecordReaderValueIterator.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/RecordReaderValueIterator.java index 4201470cad06a..21427cc1c2956 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/RecordReaderValueIterator.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/RecordReaderValueIterator.java @@ -29,7 +29,7 @@ import java.util.NoSuchElementException; /** - * Provides Iterator Interface to iterate value entries read from record reader + * Provides Iterator Interface to iterate value entries read from record reader. * * @param Key Type * @param Value Type @@ -42,7 +42,7 @@ public class RecordReaderValueIterator implements Iterator { private V nextVal = null; /** - * Construct RecordReaderValueIterator + * Construct RecordReaderValueIterator. * * @param reader reader */ diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/UseFileSplitsFromInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/UseFileSplitsFromInputFormat.java index f21e24ef22aef..3e1d60746954a 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/UseFileSplitsFromInputFormat.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/UseFileSplitsFromInputFormat.java @@ -27,7 +27,7 @@ /** * When annotated on a InputFormat, informs the query engines, that they should use the FileSplits provided by the input - * format to execute the queries + * format to execute the queries. */ @Inherited @Documented diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineHiveInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineHiveInputFormat.java index 6512d94701ce7..1bf0da1e2e1a5 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineHiveInputFormat.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineHiveInputFormat.java @@ -486,7 +486,7 @@ private InputSplit[] getCombineSplits(JobConf job, int numSplits, Map getNonCombinablePathIndices(JobConf job, Path[] paths, int numThreads) @@ -602,8 +602,8 @@ private void processPaths(JobConf job, CombineFileInputFormatShim combine, List< } /** - * MOD - Just added this for visibility - **/ + * MOD - Just added this for visibility. + */ Path[] getInputPaths(JobConf job) throws IOException { Path[] dirs = FileInputFormat.getInputPaths(job); if (dirs.length == 0) { @@ -786,7 +786,7 @@ public String toString() { } /** - * This is a marker interface that is used to identify the formats where combine split generation is not applicable + * This is a marker interface that is used to identify the formats where combine split generation is not applicable. */ public interface AvoidSplitCombination { @@ -794,8 +794,8 @@ public interface AvoidSplitCombination { } /** - * **MOD** this is the implementation of CombineFileInputFormat which is a copy of - * org.apache.hadoop.hive.shims.HadoopShimsSecure.CombineFileInputFormatShim with changes in listStatus + * **MOD** This is the implementation of CombineFileInputFormat which is a copy of + * org.apache.hadoop.hive.shims.HadoopShimsSecure.CombineFileInputFormatShim with changes in listStatus. */ public static class HoodieCombineFileInputFormatShim extends CombineFileInputFormat implements org.apache.hadoop.hive.shims.HadoopShims.CombineFileInputFormatShim { diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java index 68bf517815f34..a15ed76e665b4 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java @@ -130,7 +130,7 @@ private static MessageType readSchema(Configuration conf, Path parquetFilePath) } /** - * Prints a JSON representation of the ArrayWritable for easier debuggability + * Prints a JSON representation of the ArrayWritable for easier debuggability. */ protected static String arrayWritableToString(ArrayWritable writable) { if (writable == null) { @@ -197,7 +197,7 @@ private static List orderFields(String fieldNameCsv, String fieldOrderCs } /** - * Generate a reader schema off the provided writeSchema, to just project out the provided columns + * Generate a reader schema off the provided writeSchema, to just project out the provided columns. */ public static Schema generateProjectionSchema(Schema writeSchema, Map schemaFieldsMap, List fieldNames) { @@ -234,7 +234,7 @@ public static Map getNameToFieldMap(Schema schema) { } /** - * Convert the projected read from delta record into an array writable + * Convert the projected read from delta record into an array writable. */ public static Writable avroToArrayWritable(Object value, Schema schema) { diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java index 6d467288c2a6a..ef03c0d5988a7 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java @@ -60,7 +60,7 @@ import java.util.stream.Stream; /** - * Input Format, that provides a real-time view of data in a Hoodie dataset + * Input Format, that provides a real-time view of data in a Hoodie dataset. */ @UseFileSplitsFromInputFormat public class HoodieParquetRealtimeInputFormat extends HoodieParquetInputFormat implements Configurable { @@ -160,7 +160,7 @@ public FileStatus[] listStatus(JobConf job) throws IOException { } /** - * Add a field to the existing fields projected + * Add a field to the existing fields projected. */ private static Configuration addProjectionField(Configuration conf, String fieldName, int fieldIndex) { String readColNames = conf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR, ""); diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetSerde.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetSerde.java index 72195f686bf2f..012150678b548 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetSerde.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetSerde.java @@ -21,7 +21,7 @@ import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; /** - * Simply extends ParquetHiveSerDe + * Simply extends ParquetHiveSerDe. */ public class HoodieParquetSerde extends ParquetHiveSerDe { diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeRecordReader.java index e15631641a7b2..8af2f0851e2fb 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeRecordReader.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeRecordReader.java @@ -52,7 +52,7 @@ public static boolean canSkipMerging(JobConf jobConf) { } /** - * Construct record reader based on job configuration + * Construct record reader based on job configuration. * * @param split File Split * @param jobConf Job Configuration diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeUnmergedRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeUnmergedRecordReader.java index cd6f41dfe2eab..f1a8eb87e2f23 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeUnmergedRecordReader.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeUnmergedRecordReader.java @@ -60,7 +60,7 @@ class RealtimeUnmergedRecordReader extends AbstractRealtimeRecordReader /** * Construct a Unmerged record reader that parallely consumes both parquet and log records and buffers for upstream - * clients to consume + * clients to consume. * * @param split File split * @param job Job Configuration diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestRecordReaderValueIterator.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestRecordReaderValueIterator.java index 2dfd036557c5f..36799d8d9c794 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestRecordReaderValueIterator.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestRecordReaderValueIterator.java @@ -49,7 +49,7 @@ public void testValueIterator() { } /** - * Simple replay record reader for unit-testing + * Simple replay record reader for unit-testing. */ private static class TestRecordReader implements RecordReader { diff --git a/hudi-hive/src/main/java/org/apache/hudi/hive/HiveSyncTool.java b/hudi-hive/src/main/java/org/apache/hudi/hive/HiveSyncTool.java index ebf1c0f69eb24..5acb3d6548677 100644 --- a/hudi-hive/src/main/java/org/apache/hudi/hive/HiveSyncTool.java +++ b/hudi-hive/src/main/java/org/apache/hudi/hive/HiveSyncTool.java @@ -160,7 +160,7 @@ private void syncSchema(boolean tableExists, boolean isRealTime, MessageType sch /** * Syncs the list of storage parititions passed in (checks if the partition is in hive, if not adds it or if the - * partition path does not match, it updates the partition path) + * partition path does not match, it updates the partition path). */ private void syncPartitions(List writtenPartitionsSince) { try { diff --git a/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java b/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java index dd0a4606e91dc..1dffee5fbdc1b 100644 --- a/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java +++ b/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java @@ -132,7 +132,7 @@ public HoodieTimeline getActiveTimeline() { } /** - * Add the (NEW) partitons to the table + * Add the (NEW) partitons to the table. */ void addPartitionsToTable(List partitionsToAdd) { if (partitionsToAdd.isEmpty()) { @@ -145,7 +145,7 @@ void addPartitionsToTable(List partitionsToAdd) { } /** - * Partition path has changed - update the path for te following partitions + * Partition path has changed - update the path for te following partitions. */ void updatePartitionsToTable(List changedPartitions) { if (changedPartitions.isEmpty()) { @@ -172,7 +172,7 @@ private String constructAddPartitions(List partitions) { } /** - * Generate Hive Partition from partition values + * Generate Hive Partition from partition values. * * @param partition Partition path * @return @@ -241,7 +241,7 @@ List getPartitionEvents(List tablePartitions, List scanTablePartitions() throws TException { return client.listPartitions(syncConfig.databaseName, syncConfig.tableName, (short) -1); @@ -274,7 +274,7 @@ void createTable(MessageType storageSchema, String inputFormatClass, String outp } /** - * Get the table schema + * Get the table schema. */ public Map getTableSchema() { if (syncConfig.useJdbc) { @@ -428,7 +428,7 @@ private MessageType readSchemaFromLastCompaction(Option lastCompa } /** - * Read the schema from the log file on path + * Read the schema from the log file on path. */ @SuppressWarnings("OptionalUsedAsFieldOrParameterType") private MessageType readSchemaFromLogFile(Option lastCompactionCommitOpt, Path path) @@ -443,7 +443,7 @@ private MessageType readSchemaFromLogFile(Option lastCompactionCo } /** - * Read the parquet schema from a parquet File + * Read the parquet schema from a parquet File. */ private MessageType readSchemaFromDataFile(Path parquetFilePath) throws IOException { LOG.info("Reading schema from " + parquetFilePath); @@ -468,7 +468,7 @@ public boolean doesTableExist() { } /** - * Execute a update in hive metastore with this SQL + * Execute a update in hive metastore with this SQL. * * @param s SQL to execute */ @@ -490,7 +490,7 @@ public void updateHiveSQL(String s) { } /** - * Execute a update in hive using Hive Driver + * Execute a update in hive using Hive Driver. * * @param sql SQL statement to execute */ @@ -663,7 +663,7 @@ void updateLastCommitTimeSynced() { } /** - * Partition Event captures any partition that needs to be added or updated + * Partition Event captures any partition that needs to be added or updated. */ static class PartitionEvent { diff --git a/hudi-hive/src/main/java/org/apache/hudi/hive/NonPartitionedExtractor.java b/hudi-hive/src/main/java/org/apache/hudi/hive/NonPartitionedExtractor.java index e122bbd82fb70..dc6243910e132 100644 --- a/hudi-hive/src/main/java/org/apache/hudi/hive/NonPartitionedExtractor.java +++ b/hudi-hive/src/main/java/org/apache/hudi/hive/NonPartitionedExtractor.java @@ -22,7 +22,7 @@ import java.util.List; /** - * Extractor for Non-partitioned hive tables + * Extractor for Non-partitioned hive tables. */ public class NonPartitionedExtractor implements PartitionValueExtractor { diff --git a/hudi-hive/src/main/java/org/apache/hudi/hive/SchemaDifference.java b/hudi-hive/src/main/java/org/apache/hudi/hive/SchemaDifference.java index 752c09946bbd4..21152cea5f5ee 100644 --- a/hudi-hive/src/main/java/org/apache/hudi/hive/SchemaDifference.java +++ b/hudi-hive/src/main/java/org/apache/hudi/hive/SchemaDifference.java @@ -29,7 +29,7 @@ import java.util.Map; /** - * Represents the schema difference between the storage schema and hive table schema + * Represents the schema difference between the storage schema and hive table schema. */ public class SchemaDifference { diff --git a/hudi-hive/src/main/java/org/apache/hudi/hive/util/SchemaUtil.java b/hudi-hive/src/main/java/org/apache/hudi/hive/util/SchemaUtil.java index 95cc55086c271..a67d2a88d1021 100644 --- a/hudi-hive/src/main/java/org/apache/hudi/hive/util/SchemaUtil.java +++ b/hudi-hive/src/main/java/org/apache/hudi/hive/util/SchemaUtil.java @@ -49,14 +49,14 @@ import java.util.stream.Collectors; /** - * Schema Utilities + * Schema Utilities. */ public class SchemaUtil { private static final Logger LOG = LogManager.getLogger(SchemaUtil.class); /** - * Get the schema difference between the storage schema and hive table schema + * Get the schema difference between the storage schema and hive table schema. */ public static SchemaDifference getSchemaDifference(MessageType storageSchema, Map tableSchema, List partitionKeys) { @@ -135,7 +135,7 @@ private static boolean isFieldExistsInSchema(Map newTableSchema, } /** - * Returns equivalent Hive table schema read from a parquet file + * Returns equivalent Hive table schema read from a parquet file. * * @param messageType : Parquet Schema * @return : Hive Table schema read from parquet file MAP[String,String] @@ -158,7 +158,7 @@ public static Map convertParquetSchemaToHiveSchema(MessageType m } /** - * Convert one field data type of parquet schema into an equivalent Hive schema + * Convert one field data type of parquet schema into an equivalent Hive schema. * * @param parquetType : Single paruet field * @return : Equivalent sHive schema @@ -272,7 +272,7 @@ public String convertBINARY(PrimitiveType.PrimitiveTypeName primitiveTypeName) { } /** - * Return a 'struct' Hive schema from a list of Parquet fields + * Return a 'struct' Hive schema from a list of Parquet fields. * * @param parquetFields : list of parquet fields * @return : Equivalent 'struct' Hive schema @@ -324,14 +324,14 @@ private static String removeSurroundingTick(String result) { } /** - * Create a 'Map' schema from Parquet map field + * Create a 'Map' schema from Parquet map field. */ private static String createHiveMap(String keyType, String valueType) { return "MAP< " + keyType + ", " + valueType + ">"; } /** - * Create an Array Hive schema from equivalent parquet list type + * Create an Array Hive schema from equivalent parquet list type. */ private static String createHiveArray(Type elementType, String elementName) { StringBuilder array = new StringBuilder(); @@ -425,7 +425,7 @@ private static String getPartitionKeyType(Map hiveSchema, String } /** - * Read the schema from the log file on path + * Read the schema from the log file on path. * * @return */ diff --git a/hudi-hive/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java b/hudi-hive/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java index 826653127c31f..2276c477fe74e 100644 --- a/hudi-hive/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java +++ b/hudi-hive/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java @@ -74,7 +74,7 @@ public void teardown() throws IOException, InterruptedException { } /** - * Testing converting array types to Hive field declaration strings, according to the Parquet-113 spec: + * Testing converting array types to Hive field declaration strings. According to the Parquet-113 spec: * https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists */ @Test diff --git a/hudi-hive/src/test/java/org/apache/hudi/hive/util/HiveTestService.java b/hudi-hive/src/test/java/org/apache/hudi/hive/util/HiveTestService.java index b83eddcb77e3b..c9a711ca5ed69 100644 --- a/hudi-hive/src/test/java/org/apache/hudi/hive/util/HiveTestService.java +++ b/hudi-hive/src/test/java/org/apache/hudi/hive/util/HiveTestService.java @@ -64,7 +64,7 @@ public class HiveTestService { private static final int CONNECTION_TIMEOUT = 30000; /** - * Configuration settings + * Configuration settings. */ private Configuration hadoopConf; private String workDir; diff --git a/hudi-spark/src/main/java/org/apache/hudi/BaseAvroPayload.java b/hudi-spark/src/main/java/org/apache/hudi/BaseAvroPayload.java index 126836281027b..b13356050f729 100644 --- a/hudi-spark/src/main/java/org/apache/hudi/BaseAvroPayload.java +++ b/hudi-spark/src/main/java/org/apache/hudi/BaseAvroPayload.java @@ -28,17 +28,17 @@ import java.io.Serializable; /** - * Base class for all AVRO record based payloads, that can be ordered based on a field + * Base class for all AVRO record based payloads, that can be ordered based on a field. */ public abstract class BaseAvroPayload implements Serializable { /** - * Avro data extracted from the source converted to bytes + * Avro data extracted from the source converted to bytes. */ protected final byte[] recordBytes; /** - * For purposes of preCombining + * For purposes of preCombining. */ protected final Comparable orderingVal; diff --git a/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java b/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java index e7cfcc8fdddf6..2a8551a563cdb 100644 --- a/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java +++ b/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java @@ -49,7 +49,7 @@ import java.util.stream.Collectors; /** - * Utilities used throughout the data source + * Utilities used throughout the data source. */ public class DataSourceUtils { @@ -120,7 +120,7 @@ public static KeyGenerator createKeyGenerator(TypedProperties props) throws IOEx } /** - * Create a partition value extractor class via reflection, passing in any configs needed + * Create a partition value extractor class via reflection, passing in any configs needed. */ public static PartitionValueExtractor createPartitionExtractor(String partitionExtractorClass) { try { diff --git a/hudi-spark/src/main/java/org/apache/hudi/HoodieDataSourceHelpers.java b/hudi-spark/src/main/java/org/apache/hudi/HoodieDataSourceHelpers.java index f9df30ad2be1a..362cdf035ab2b 100644 --- a/hudi-spark/src/main/java/org/apache/hudi/HoodieDataSourceHelpers.java +++ b/hudi-spark/src/main/java/org/apache/hudi/HoodieDataSourceHelpers.java @@ -31,7 +31,7 @@ import java.util.stream.Collectors; /** - * List of helpers to aid, construction of instanttime for read and write operations using datasource + * List of helpers to aid, construction of instanttime for read and write operations using datasource. */ public class HoodieDataSourceHelpers { @@ -53,7 +53,7 @@ public static List listCommitsSince(FileSystem fs, String basePath, Stri } /** - * Returns the last successful write operation's instant time + * Returns the last successful write operation's instant time. */ public static String latestCommit(FileSystem fs, String basePath) { HoodieTimeline timeline = allCompletedCommitsCompactions(fs, basePath); diff --git a/hudi-spark/src/main/java/org/apache/hudi/KeyGenerator.java b/hudi-spark/src/main/java/org/apache/hudi/KeyGenerator.java index 4b8084b9f088c..17b5e0b396bd8 100644 --- a/hudi-spark/src/main/java/org/apache/hudi/KeyGenerator.java +++ b/hudi-spark/src/main/java/org/apache/hudi/KeyGenerator.java @@ -26,7 +26,7 @@ import java.io.Serializable; /** - * Abstract class to extend for plugging in extraction of {@link HoodieKey} from an Avro record + * Abstract class to extend for plugging in extraction of {@link HoodieKey} from an Avro record. */ public abstract class KeyGenerator implements Serializable { diff --git a/hudi-spark/src/main/java/org/apache/hudi/NonpartitionedKeyGenerator.java b/hudi-spark/src/main/java/org/apache/hudi/NonpartitionedKeyGenerator.java index 8c0a664d6b59f..35829a1f4fe3f 100644 --- a/hudi-spark/src/main/java/org/apache/hudi/NonpartitionedKeyGenerator.java +++ b/hudi-spark/src/main/java/org/apache/hudi/NonpartitionedKeyGenerator.java @@ -25,7 +25,7 @@ import org.apache.avro.generic.GenericRecord; /** - * Simple Key generator for unpartitioned Hive Tables + * Simple Key generator for unpartitioned Hive Tables. */ public class NonpartitionedKeyGenerator extends SimpleKeyGenerator { diff --git a/hudi-spark/src/test/java/HoodieJavaApp.java b/hudi-spark/src/test/java/HoodieJavaApp.java index 389c015542443..5370e75de480d 100644 --- a/hudi-spark/src/test/java/HoodieJavaApp.java +++ b/hudi-spark/src/test/java/HoodieJavaApp.java @@ -45,7 +45,7 @@ import java.util.List; /** - * Sample program that writes & reads hoodie datasets via the Spark datasource + * Sample program that writes & reads hoodie datasets via the Spark datasource. */ public class HoodieJavaApp { @@ -232,7 +232,7 @@ public void run() throws Exception { } /** - * Setup configs for syncing to hive + * Setup configs for syncing to hive. */ private DataFrameWriter updateHiveSyncConfig(DataFrameWriter writer) { if (enableHiveSync) { diff --git a/hudi-spark/src/test/java/HoodieJavaStreamingApp.java b/hudi-spark/src/test/java/HoodieJavaStreamingApp.java index c448abbc73981..694ae88bb73ff 100644 --- a/hudi-spark/src/test/java/HoodieJavaStreamingApp.java +++ b/hudi-spark/src/test/java/HoodieJavaStreamingApp.java @@ -45,7 +45,7 @@ import java.util.concurrent.Future; /** - * Sample program that writes & reads hoodie datasets via the Spark datasource streaming + * Sample program that writes & reads hoodie datasets via the Spark datasource streaming. */ public class HoodieJavaStreamingApp { @@ -165,7 +165,7 @@ public void run() throws Exception { } /** - * Adding data to the streaming source and showing results over time + * Adding data to the streaming source and showing results over time. * * @param spark * @param fs @@ -215,7 +215,7 @@ public void show(SparkSession spark, FileSystem fs, Dataset inputDF1, Datas } /** - * Hoodie spark streaming job + * Hoodie spark streaming job. * * @param streamingInput * @throws Exception @@ -236,7 +236,7 @@ public void stream(Dataset streamingInput) throws Exception { } /** - * Setup configs for syncing to hive + * Setup configs for syncing to hive. * * @param writer * @return diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/FileSystemViewHandler.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/FileSystemViewHandler.java index fcc053fc85b43..56cfe23628a5f 100644 --- a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/FileSystemViewHandler.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/FileSystemViewHandler.java @@ -49,7 +49,7 @@ import java.util.stream.Collectors; /** - * Main REST Handler class that handles local view staleness and delegates calls to slice/data-file/timeline handlers + * Main REST Handler class that handles local view staleness and delegates calls to slice/data-file/timeline handlers. */ public class FileSystemViewHandler { @@ -79,7 +79,7 @@ public void register() { } /** - * Determines if local view of dataset's timeline is behind that of client's view + * Determines if local view of dataset's timeline is behind that of client's view. */ private boolean isLocalViewBehind(Context ctx) { String basePath = ctx.queryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM); @@ -108,7 +108,7 @@ private boolean isLocalViewBehind(Context ctx) { } /** - * Syncs data-set view if local view is behind + * Syncs data-set view if local view is behind. */ private boolean syncIfLocalViewBehind(Context ctx) { if (isLocalViewBehind(ctx)) { @@ -141,7 +141,7 @@ private void writeValueAsString(Context ctx, Object obj) throws JsonProcessingEx } /** - * Register Timeline API calls + * Register Timeline API calls. */ private void registerTimelineAPI() { app.get(RemoteHoodieTableFileSystemView.LAST_INSTANT, new ViewHandler(ctx -> { @@ -158,7 +158,7 @@ private void registerTimelineAPI() { } /** - * Register Data-Files API calls + * Register Data-Files API calls. */ private void registerDataFilesAPI() { app.get(RemoteHoodieTableFileSystemView.LATEST_PARTITION_DATA_FILES_URL, new ViewHandler(ctx -> { @@ -215,7 +215,7 @@ private void registerDataFilesAPI() { } /** - * Register File Slices API calls + * Register File Slices API calls. */ private void registerFileSlicesAPI() { app.get(RemoteHoodieTableFileSystemView.LATEST_PARTITION_SLICES_URL, new ViewHandler(ctx -> { diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java index 16ff2f7a9ce1f..e8ac49d3f5ab6 100644 --- a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java @@ -36,7 +36,7 @@ import java.io.Serializable; /** - * A stand alone timeline service exposing File-System View interfaces to clients + * A stand alone timeline service exposing File-System View interfaces to clients. */ public class TimelineService { diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/DataFileHandler.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/DataFileHandler.java index 2b92bbf681b15..d18c20ba8eab4 100644 --- a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/DataFileHandler.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/DataFileHandler.java @@ -30,7 +30,7 @@ import java.util.stream.Collectors; /** - * REST Handler servicing data-file requests + * REST Handler servicing data-file requests. */ public class DataFileHandler extends Handler { diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/FileSliceHandler.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/FileSliceHandler.java index e58f83556ad7a..7cca939cf8331 100644 --- a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/FileSliceHandler.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/FileSliceHandler.java @@ -32,7 +32,7 @@ import java.util.stream.Collectors; /** - * REST Handler servicing file-slice requests + * REST Handler servicing file-slice requests. */ public class FileSliceHandler extends Handler { diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/TimelineHandler.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/TimelineHandler.java index faa81a4d06402..75f53a0fb4f7b 100644 --- a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/TimelineHandler.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/TimelineHandler.java @@ -30,7 +30,7 @@ import java.util.List; /** - * REST Handler servicing timeline requests + * REST Handler servicing timeline requests. */ public class TimelineHandler extends Handler { diff --git a/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/table/view/TestRemoteHoodieTableFileSystemView.java b/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/table/view/TestRemoteHoodieTableFileSystemView.java index d69a8281700ed..5f7267634b59c 100644 --- a/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/table/view/TestRemoteHoodieTableFileSystemView.java +++ b/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/table/view/TestRemoteHoodieTableFileSystemView.java @@ -32,7 +32,7 @@ import org.apache.log4j.Logger; /** - * Bring up a remote Timeline Server and run all test-cases of TestHoodieTableFileSystemView against it + * Bring up a remote Timeline Server and run all test-cases of TestHoodieTableFileSystemView against it. */ public class TestRemoteHoodieTableFileSystemView extends TestHoodieTableFileSystemView { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HDFSParquetImporter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HDFSParquetImporter.java index 62d2aa79b7ef5..069c8bbca43dd 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HDFSParquetImporter.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HDFSParquetImporter.java @@ -62,7 +62,7 @@ import scala.Tuple2; /** - * Loads data from Parquet Sources + * Loads data from Parquet Sources. */ public class HDFSParquetImporter implements Serializable { @@ -190,7 +190,7 @@ protected JavaRDD> buildHoodieRecordsForImport } /** - * Imports records to Hoodie dataset + * Imports records to Hoodie dataset. * * @param client Hoodie Client * @param instantTime Instant Time diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java index 27c32206304dd..6afcc7a7d127b 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java @@ -41,17 +41,17 @@ public class HoodieCleaner { private static volatile Logger log = LogManager.getLogger(HoodieCleaner.class); /** - * Config for Cleaner + * Config for Cleaner. */ private final Config cfg; /** - * Filesystem used + * Filesystem used. */ private transient FileSystem fs; /** - * Spark context + * Spark context. */ private transient JavaSparkContext jssc; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactionAdminTool.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactionAdminTool.java index d42a45dfc329a..45b904db6b7ac 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactionAdminTool.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactionAdminTool.java @@ -56,7 +56,7 @@ public static void main(String[] args) throws Exception { } /** - * Executes one of compaction admin operations + * Executes one of compaction admin operations. */ public void run(JavaSparkContext jsc) throws Exception { HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), cfg.basePath); @@ -118,7 +118,7 @@ private void serializeOperationResult(FileSystem fs, T result) throws Except } /** - * Print Operation Result + * Print Operation Result. * * @param initialLine Initial Line * @param result Result @@ -131,14 +131,14 @@ private void printOperationResult(String initialLine, List result) { } /** - * Operation Types + * Operation Types. */ public enum Operation { VALIDATE, UNSCHEDULE_PLAN, UNSCHEDULE_FILE, REPAIR } /** - * Admin Configuration Options + * Admin Configuration Options. */ public static class Config implements Serializable { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java index 1f33ca6194502..663ce0fa97aaf 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java @@ -55,7 +55,7 @@ import java.util.Map; /** - * Bunch of helper methods + * Bunch of helper methods. */ public class UtilHelpers { private static Logger logger = LogManager.getLogger(UtilHelpers.class); @@ -115,7 +115,7 @@ public static TypedProperties buildProperties(List props) { } /** - * Parse Schema from file + * Parse Schema from file. * * @param fs File System * @param schemaFile Schema File @@ -167,7 +167,7 @@ public static JavaSparkContext buildSparkContext(String appName, String defaultM } /** - * Build Spark Context for ingestion/compaction + * Build Spark Context for ingestion/compaction. * * @return */ @@ -178,7 +178,7 @@ public static JavaSparkContext buildSparkContext(String appName, String sparkMas } /** - * Build Hoodie write client + * Build Hoodie write client. * * @param jsc Java Spark Context * @param basePath Base Path diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/adhoc/UpgradePayloadFromUberToApache.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/adhoc/UpgradePayloadFromUberToApache.java index 6793d94eee293..bfbcf71f0a99a 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/adhoc/UpgradePayloadFromUberToApache.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/adhoc/UpgradePayloadFromUberToApache.java @@ -41,7 +41,7 @@ /** * This is an one-time use class meant for migrating the configuration for "hoodie.compaction.payload.class" in - * .hoodie/hoodie.properties from com.uber.hoodie to org.apache.hudi It takes in a file containing base-paths for a set + * .hoodie/hoodie.properties from com.uber.hoodie to org.apache.hudi . It takes in a file containing base-paths for a set * of hudi datasets and does the migration */ public class UpgradePayloadFromUberToApache implements Serializable { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/AbstractDeltaStreamerService.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/AbstractDeltaStreamerService.java index bd4d8a269ffcb..b6f5306afab54 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/AbstractDeltaStreamerService.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/AbstractDeltaStreamerService.java @@ -32,7 +32,7 @@ import java.util.function.Function; /** - * Base Class for running delta-sync/compaction in separate thread and controlling their life-cyle + * Base Class for running delta-sync/compaction in separate thread and controlling their life-cyle. */ public abstract class AbstractDeltaStreamerService implements Serializable { @@ -116,14 +116,14 @@ public void start(Function onShutdownCallback) { } /** - * Service implementation + * Service implementation. * * @return */ protected abstract Pair startService(); /** - * A monitor thread is started which would trigger a callback if the service is shutdown + * A monitor thread is started which would trigger a callback if the service is shutdown. * * @param onShutdownCallback */ diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/Compactor.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/Compactor.java index 3285ba70136c7..140081aeda611 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/Compactor.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/Compactor.java @@ -33,7 +33,7 @@ import java.io.Serializable; /** - * Run one round of compaction + * Run one round of compaction. */ public class Compactor implements Serializable { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java index 532ec697f7e63..91a9bc60f5923 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java @@ -78,9 +78,8 @@ import static org.apache.hudi.utilities.schema.RowBasedSchemaProvider.HOODIE_RECORD_NAMESPACE; import static org.apache.hudi.utilities.schema.RowBasedSchemaProvider.HOODIE_RECORD_STRUCT_NAME; - /** - * Sync's one batch of data to hoodie dataset + * Sync's one batch of data to hoodie dataset. */ public class DeltaSync implements Serializable { @@ -89,12 +88,12 @@ public class DeltaSync implements Serializable { public static String CHECKPOINT_RESET_KEY = "deltastreamer.checkpoint.reset_key"; /** - * Delta Sync Config + * Delta Sync Config. */ private final HoodieDeltaStreamer.Config cfg; /** - * Source to pull deltas from + * Source to pull deltas from. */ private transient SourceFormatAdapter formatAdapter; @@ -104,32 +103,32 @@ public class DeltaSync implements Serializable { private transient SchemaProvider schemaProvider; /** - * Allows transforming source to target dataset before writing + * Allows transforming source to target dataset before writing. */ private transient Transformer transformer; /** - * Extract the key for the target dataset + * Extract the key for the target dataset. */ private KeyGenerator keyGenerator; /** - * Filesystem used + * Filesystem used. */ private transient FileSystem fs; /** - * Spark context + * Spark context. */ private transient JavaSparkContext jssc; /** - * Spark Session + * Spark Session. */ private transient SparkSession sparkSession; /** - * Hive Config + * Hive Config. */ private transient HiveConf hiveConf; @@ -139,22 +138,22 @@ public class DeltaSync implements Serializable { private final TypedProperties props; /** - * Callback when write client is instantiated + * Callback when write client is instantiated. */ private transient Function onInitializingHoodieWriteClient; /** - * Timeline with completed commits + * Timeline with completed commits. */ private transient Option commitTimelineOpt; /** - * Write Client + * Write Client. */ private transient HoodieWriteClient writeClient; /** - * Table Type + * Table Type. */ private final HoodieTableType tableType; @@ -190,7 +189,7 @@ public DeltaSync(HoodieDeltaStreamer.Config cfg, SparkSession sparkSession, Sche } /** - * Refresh Timeline + * Refresh Timeline. */ private void refreshTimeline() throws IOException { if (fs.exists(new Path(cfg.targetBasePath))) { @@ -204,7 +203,7 @@ private void refreshTimeline() throws IOException { } /** - * Run one round of delta sync and return new compaction instant if one got scheduled + * Run one round of delta sync and return new compaction instant if one got scheduled. */ public Option syncOnce() throws Exception { Option scheduledCompaction = Option.empty(); @@ -236,7 +235,7 @@ public Option syncOnce() throws Exception { } /** - * Read from Upstream Source and apply transformation if needed + * Read from Upstream Source and apply transformation if needed. */ private Pair>> readFromSource( Option commitTimelineOpt) throws Exception { @@ -321,7 +320,7 @@ private Pair>> readFromSource } /** - * Perform Hoodie Write. Run Cleaner, schedule compaction and syncs to hive if needed + * Perform Hoodie Write. Run Cleaner, schedule compaction and syncs to hive if needed. * * @param records Input Records * @param checkpointStr Checkpoint String @@ -434,7 +433,7 @@ private String startCommit() { } /** - * Sync to Hive + * Sync to Hive. */ private void syncHive() throws ClassNotFoundException { if (cfg.enableHiveSync) { @@ -462,7 +461,7 @@ public void setupWriteClient() { } /** - * Helper to construct Write Client config + * Helper to construct Write Client config. * * @param schemaProvider Schema Provider */ @@ -491,7 +490,7 @@ private HoodieWriteConfig getHoodieClientConfig(SchemaProvider schemaProvider) { } /** - * Register Avro Schemas + * Register Avro Schemas. * * @param schemaProvider Schema Provider */ @@ -510,7 +509,7 @@ private void registerAvroSchemas(SchemaProvider schemaProvider) { } /** - * Close all resources + * Close all resources. */ public void close() { if (null != writeClient) { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java index 9893f0d87740e..baef2ea02e8b4 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java @@ -107,7 +107,7 @@ private static HiveConf getDefaultHiveConf(Configuration cfg) { } /** - * Main method to start syncing + * Main method to start syncing. * * @throws Exception */ @@ -306,7 +306,7 @@ public static void main(String[] args) throws Exception { public static class DeltaSyncService extends AbstractDeltaStreamerService { /** - * Delta Sync Config + * Delta Sync Config. */ private final HoodieDeltaStreamer.Config cfg; @@ -316,12 +316,12 @@ public static class DeltaSyncService extends AbstractDeltaStreamerService { private transient SchemaProvider schemaProvider; /** - * Spark Session + * Spark Session. */ private transient SparkSession sparkSession; /** - * Spark context + * Spark context. */ private transient JavaSparkContext jssc; @@ -331,17 +331,17 @@ public static class DeltaSyncService extends AbstractDeltaStreamerService { TypedProperties props; /** - * Async Compactor Service + * Async Compactor Service. */ private AsyncCompactService asyncCompactService; /** - * Table Type + * Table Type. */ private final HoodieTableType tableType; /** - * Delta Sync + * Delta Sync. */ private transient DeltaSync deltaSync; @@ -419,7 +419,7 @@ protected Pair startService() { } /** - * Shutdown compactor as DeltaSync is shutdown + * Shutdown compactor as DeltaSync is shutdown. */ private void shutdownCompactor(boolean error) { log.info("Delta Sync shutdown. Error ?" + error); @@ -430,7 +430,7 @@ private void shutdownCompactor(boolean error) { } /** - * Callback to initialize write client and start compaction service if required + * Callback to initialize write client and start compaction service if required. * * @param writeClient HoodieWriteClient * @return @@ -458,7 +458,7 @@ protected Boolean onInitializingWriteClient(HoodieWriteClient writeClient) { } /** - * Close all resources + * Close all resources. */ public void close() { if (null != deltaSync) { @@ -507,14 +507,14 @@ public AsyncCompactService(JavaSparkContext jssc, HoodieWriteClient client) { } /** - * Enqueues new Pending compaction + * Enqueues new Pending compaction. */ public void enqueuePendingCompaction(HoodieInstant instant) { pendingCompactions.add(instant); } /** - * Wait till outstanding pending compactions reduces to the passed in value + * Wait till outstanding pending compactions reduces to the passed in value. * * @param numPendingCompactions Maximum pending compactions allowed * @throws InterruptedException @@ -531,7 +531,7 @@ public void waitTillPendingCompactionsReducesTo(int numPendingCompactions) throw } /** - * Fetch Next pending compaction if available + * Fetch Next pending compaction if available. * * @return * @throws InterruptedException @@ -552,7 +552,7 @@ private HoodieInstant fetchNextCompactionInstant() throws InterruptedException { } /** - * Start Compaction Service + * Start Compaction Service. */ protected Pair startService() { ExecutorService executor = Executors.newFixedThreadPool(maxConcurrentCompaction); diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SchedulerConfGenerator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SchedulerConfGenerator.java index 66d2c475a1ecc..1e754ba203261 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SchedulerConfGenerator.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SchedulerConfGenerator.java @@ -60,7 +60,7 @@ private static String generateConfig(Integer deltaSyncWeight, Integer compaction } /** - * Helper to set Spark Scheduling Configs dynamically + * Helper to set Spark Scheduling Configs dynamically. * * @param cfg Config */ diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SourceFormatAdapter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SourceFormatAdapter.java index b41efcde289f6..65779e0311902 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SourceFormatAdapter.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SourceFormatAdapter.java @@ -39,7 +39,7 @@ import static org.apache.hudi.utilities.schema.RowBasedSchemaProvider.HOODIE_RECORD_STRUCT_NAME; /** - * Adapts data-format provided by the source to the data-format required by the client (DeltaStreamer) + * Adapts data-format provided by the source to the data-format required by the client (DeltaStreamer). */ public final class SourceFormatAdapter { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/keygen/TimestampBasedKeyGenerator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/keygen/TimestampBasedKeyGenerator.java index 04c6ecef23ac6..140c7097d4e96 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/keygen/TimestampBasedKeyGenerator.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/keygen/TimestampBasedKeyGenerator.java @@ -52,7 +52,7 @@ enum TimestampType implements Serializable { private final String outputDateFormat; /** - * Supported configs + * Supported configs. */ static class Config { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/FilebasedSchemaProvider.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/FilebasedSchemaProvider.java index 5776984c82efc..219948a46466d 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/FilebasedSchemaProvider.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/FilebasedSchemaProvider.java @@ -32,12 +32,12 @@ import java.util.Collections; /** - * A simple schema provider, that reads off files on DFS + * A simple schema provider, that reads off files on DFS. */ public class FilebasedSchemaProvider extends SchemaProvider { /** - * Configs supported + * Configs supported. */ public static class Config { private static final String SOURCE_SCHEMA_FILE_PROP = "hoodie.deltastreamer.schemaprovider" + ".source.schema.file"; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaProvider.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaProvider.java index 8378383c5ef0e..1a93751179e58 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaProvider.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaProvider.java @@ -26,7 +26,7 @@ import java.io.Serializable; /** - * Class to provide schema for reading data and also writing into a Hoodie table + * Class to provide schema for reading data and also writing into a Hoodie table. */ public abstract class SchemaProvider implements Serializable { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaRegistryProvider.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaRegistryProvider.java index d03c6daac4373..263ddce1d628a 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaRegistryProvider.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaRegistryProvider.java @@ -32,14 +32,14 @@ import java.util.Collections; /** - * Obtains latest schema from the Confluent/Kafka schema-registry + * Obtains latest schema from the Confluent/Kafka schema-registry. * * https://github.com/confluentinc/schema-registry */ public class SchemaRegistryProvider extends SchemaProvider { /** - * Configs supported + * Configs supported. */ public static class Config { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/AvroDFSSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/AvroDFSSource.java index 4cd8f075d9d8c..e791970907792 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/AvroDFSSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/AvroDFSSource.java @@ -34,7 +34,7 @@ import org.apache.spark.sql.SparkSession; /** - * DFS Source that reads avro data + * DFS Source that reads avro data. */ public class AvroDFSSource extends AvroSource { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/AvroKafkaSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/AvroKafkaSource.java index 9588a814b6a85..da7d016cb4dd4 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/AvroKafkaSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/AvroKafkaSource.java @@ -36,7 +36,7 @@ import org.apache.spark.streaming.kafka.OffsetRange; /** - * Reads avro serialized Kafka data, based on the confluent schema-registry + * Reads avro serialized Kafka data, based on the confluent schema-registry. */ public class AvroKafkaSource extends AvroSource { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HiveIncrPullSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HiveIncrPullSource.java index 4e4d603402d0c..2b54b452ee113 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HiveIncrPullSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HiveIncrPullSource.java @@ -66,7 +66,7 @@ public class HiveIncrPullSource extends AvroSource { private final String incrPullRootPath; /** - * Configs supported + * Configs supported. */ static class Config { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java index 3edb296911705..67de9c2a8a249 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java @@ -40,25 +40,25 @@ public class HoodieIncrSource extends RowSource { protected static class Config { /** - * {@value #HOODIE_SRC_BASE_PATH} is the base-path for the source Hoodie table + * {@value #HOODIE_SRC_BASE_PATH} is the base-path for the source Hoodie table. */ private static final String HOODIE_SRC_BASE_PATH = "hoodie.deltastreamer.source.hoodieincr.path"; /** - * {@value #NUM_INSTANTS_PER_FETCH} allows the max number of instants whose changes can be incrementally fetched + * {@value #NUM_INSTANTS_PER_FETCH} allows the max number of instants whose changes can be incrementally fetched. */ private static final String NUM_INSTANTS_PER_FETCH = "hoodie.deltastreamer.source.hoodieincr.num_instants"; private static final Integer DEFAULT_NUM_INSTANTS_PER_FETCH = 1; /** * {@value #HOODIE_SRC_PARTITION_FIELDS} specifies partition fields that needs to be added to source table after - * parsing _hoodie_partition_path + * parsing _hoodie_partition_path. */ private static final String HOODIE_SRC_PARTITION_FIELDS = "hoodie.deltastreamer.source.hoodieincr.partition.fields"; /** * {@value #HOODIE_SRC_PARTITION_EXTRACTORCLASS} PartitionValueExtractor class to extract partition fields from - * _hoodie_partition_path + * _hoodie_partition_path. */ private static final String HOODIE_SRC_PARTITION_EXTRACTORCLASS = "hoodie.deltastreamer.source.hoodieincr.partition.extractor.class"; @@ -90,7 +90,7 @@ public Pair>, String> fetchNextBatch(Option lastCkpt * props.getStringList(Config.HOODIE_SRC_PARTITION_FIELDS, ",", new ArrayList<>()); PartitionValueExtractor * extractor = DataSourceUtils.createPartitionExtractor(props.getString( Config.HOODIE_SRC_PARTITION_EXTRACTORCLASS, * Config.DEFAULT_HOODIE_SRC_PARTITION_EXTRACTORCLASS)); - **/ + */ String srcPath = props.getString(Config.HOODIE_SRC_BASE_PATH); int numInstantsPerFetch = props.getInteger(Config.NUM_INSTANTS_PER_FETCH, Config.DEFAULT_NUM_INSTANTS_PER_FETCH); boolean readLatestOnMissingCkpt = props.getBoolean(Config.READ_LATEST_INSTANT_ON_MISSING_CKPT, @@ -136,7 +136,7 @@ public Pair>, String> fetchNextBatch(Option lastCkpt * RowFactory.create(rowObjs.toArray()); } return row; }, RowEncoder.apply(newSchema)); * * log.info("Validated Source Schema :" + validated.schema()); - **/ + */ // Remove Hoodie meta columns except partition path from input source final Dataset src = source.drop(HoodieRecord.HOODIE_META_COLUMNS.stream() diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonDFSSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonDFSSource.java index ed9b82ae295fd..37f67e47bfc5b 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonDFSSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonDFSSource.java @@ -29,7 +29,7 @@ import org.apache.spark.sql.SparkSession; /** - * DFS Source that reads json data + * DFS Source that reads json data. */ public class JsonDFSSource extends JsonSource { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonKafkaSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonKafkaSource.java index ba68ac92dae2e..0da89f90c101a 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonKafkaSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonKafkaSource.java @@ -34,7 +34,7 @@ import org.apache.spark.streaming.kafka.OffsetRange; /** - * Read json kafka data + * Read json kafka data. */ public class JsonKafkaSource extends JsonSource { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/ParquetDFSSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/ParquetDFSSource.java index 3695fcee0620b..9f4eab1c8aa24 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/ParquetDFSSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/ParquetDFSSource.java @@ -32,7 +32,7 @@ import org.apache.spark.sql.SparkSession; /** - * DFS Source that reads parquet data + * DFS Source that reads parquet data. */ public class ParquetDFSSource extends ParquetSource { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/Source.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/Source.java index 48e3bd74136eb..c724c991c7aa3 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/Source.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/Source.java @@ -63,7 +63,7 @@ protected Source(TypedProperties props, JavaSparkContext sparkContext, SparkSess protected abstract InputBatch fetchNewData(Option lastCkptStr, long sourceLimit); /** - * Main API called by Hoodie Delta Streamer to fetch records + * Main API called by Hoodie Delta Streamer to fetch records. * * @param lastCkptStr Last Checkpoint * @param sourceLimit Source Limit diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/AvroConvertor.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/AvroConvertor.java index 97bb9372bb6ec..9dd2c6a653005 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/AvroConvertor.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/AvroConvertor.java @@ -35,20 +35,20 @@ public class AvroConvertor implements Serializable { /** - * To be lazily inited on executors + * To be lazily inited on executors. */ private transient Schema schema; private final String schemaStr; /** - * To be lazily inited on executors + * To be lazily inited on executors. */ private transient MercifulJsonConverter jsonConverter; /** - * To be lazily inited on executors + * To be lazily inited on executors. */ private transient Injection recordInjection; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DFSPathSelector.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DFSPathSelector.java index 2bc0466c29ce2..ba4a4054232ca 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DFSPathSelector.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DFSPathSelector.java @@ -43,7 +43,7 @@ public class DFSPathSelector { /** - * Configs supported + * Configs supported. */ static class Config { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/IncrSourceHelper.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/IncrSourceHelper.java index c6430eafffb66..19a85eb233596 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/IncrSourceHelper.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/IncrSourceHelper.java @@ -31,7 +31,7 @@ public class IncrSourceHelper { /** - * Get a timestamp which is the next value in a descending sequence + * Get a timestamp which is the next value in a descending sequence. * * @param timestamp Timestamp */ @@ -43,7 +43,7 @@ private static String getStrictlyLowerTimestamp(String timestamp) { } /** - * Find begin and end instants to be set for the next fetch + * Find begin and end instants to be set for the next fetch. * * @param jssc Java Spark Context * @param srcBasePath Base path of Hudi source table @@ -77,7 +77,7 @@ public static Pair calculateBeginAndEndInstants(JavaSparkContext } /** - * Validate instant time seen in the incoming row + * Validate instant time seen in the incoming row. * * @param row Input Row * @param instantTime Hoodie Instant time of the row diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KafkaOffsetGen.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KafkaOffsetGen.java index 278a35246daeb..a4e82ddd3738c 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KafkaOffsetGen.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KafkaOffsetGen.java @@ -47,7 +47,7 @@ import scala.util.Either; /** - * Source to read data from Kafka, incrementally + * Source to read data from Kafka, incrementally. */ public class KafkaOffsetGen { @@ -162,7 +162,7 @@ public static java.util.Map toJavaMap(Map m) { } /** - * Kafka reset offset strategies + * Kafka reset offset strategies. */ enum KafkaResetOffsetStrategies { LARGEST, SMALLEST diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/FlatteningTransformer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/FlatteningTransformer.java index 7c41e8dc1ce09..8bfa7bf4f0c06 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/FlatteningTransformer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/FlatteningTransformer.java @@ -39,7 +39,9 @@ public class FlatteningTransformer implements Transformer { private static final String TMP_TABLE = "HUDI_SRC_TMP_TABLE_"; private static volatile Logger log = LogManager.getLogger(SqlQueryBasedTransformer.class); - /** Configs supported */ + /** + * Configs supported. + */ @Override public Dataset apply(JavaSparkContext jsc, SparkSession sparkSession, Dataset rowDataset, TypedProperties properties) { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/IdentityTransformer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/IdentityTransformer.java index f74291f8d15e6..31f0ce63a0045 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/IdentityTransformer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/IdentityTransformer.java @@ -26,7 +26,7 @@ import org.apache.spark.sql.SparkSession; /** - * Identity transformer + * Identity transformer. */ public class IdentityTransformer implements Transformer { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlQueryBasedTransformer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlQueryBasedTransformer.java index ff563a13d2b7a..d7ec911f75033 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlQueryBasedTransformer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlQueryBasedTransformer.java @@ -42,7 +42,7 @@ public class SqlQueryBasedTransformer implements Transformer { private static final String TMP_TABLE = "HOODIE_SRC_TMP_TABLE_"; /** - * Configs supported + * Configs supported. */ static class Config { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/Transformer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/Transformer.java index f97b3024baae8..46857e61d8f7a 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/Transformer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/Transformer.java @@ -26,12 +26,12 @@ import org.apache.spark.sql.SparkSession; /** - * Transform source to target dataset before writing + * Transform source to target dataset before writing. */ public interface Transformer { /** - * Transform source RDD to target RDD + * Transform source RDD to target RDD. * * @param jsc JavaSparkContext * @param sparkSession Spark Session diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java index d2be9133c557a..f4e39ae937168 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java @@ -560,12 +560,12 @@ public void testDistributedTestDataSource() throws Exception { } /** - * UDF to calculate Haversine distance + * UDF to calculate Haversine distance. */ public static class DistanceUDF implements UDF4 { /** - * Returns some random number as distance between the points + * Returns some random number as distance between the points. * * @param lat1 Latitiude of source * @param lat2 Latitude of destination @@ -580,7 +580,7 @@ public Double call(Double lat1, Double lat2, Double lon1, Double lon2) { } /** - * Adds a new field "haversine_distance" to the row + * Adds a new field "haversine_distance" to the row. */ public static class TripsWithDistanceTransformer implements Transformer { @@ -601,7 +601,7 @@ public TestGenerator(TypedProperties props) { } /** - * Return empty dataset + * Return empty dataset. */ public static class DropAllTransformer implements Transformer { diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/UtilitiesTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/UtilitiesTestBase.java index cfe679850b633..753f947744e64 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/UtilitiesTestBase.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/UtilitiesTestBase.java @@ -119,7 +119,7 @@ public void teardown() throws Exception { } /** - * Helper to get hive sync config + * Helper to get hive sync config. * * @param basePath * @param tableName @@ -140,7 +140,7 @@ protected static HiveSyncConfig getHiveSyncConfig(String basePath, String tableN } /** - * Initialize Hive DB + * Initialize Hive DB. * * @throws IOException */ diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestDFSSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestDFSSource.java index f8b4869a4c392..369e3852f9913 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestDFSSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestDFSSource.java @@ -48,7 +48,7 @@ import static org.junit.Assert.assertTrue; /** - * Basic tests against all subclasses of {@link JsonDFSSource} and {@link ParquetDFSSource} + * Basic tests against all subclasses of {@link JsonDFSSource} and {@link ParquetDFSSource}. */ public class TestDFSSource extends UtilitiesTestBase { diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKafkaSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKafkaSource.java index c6066212e1708..d5eb6c3ed745b 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKafkaSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKafkaSource.java @@ -48,7 +48,7 @@ import static org.junit.Assert.assertEquals; /** - * Tests against {@link AvroKafkaSource} + * Tests against {@link AvroKafkaSource}. */ public class TestKafkaSource extends UtilitiesTestBase { diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/config/TestSourceConfig.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/config/TestSourceConfig.java index 217c6155aa345..f9e15984043eb 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/config/TestSourceConfig.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/config/TestSourceConfig.java @@ -19,7 +19,7 @@ package org.apache.hudi.utilities.sources.config; /** - * Configurations for Test Data Sources + * Configurations for Test Data Sources. */ public class TestSourceConfig { diff --git a/style/checkstyle.xml b/style/checkstyle.xml index c201cfa4c9a81..10ef5fc40d2fb 100644 --- a/style/checkstyle.xml +++ b/style/checkstyle.xml @@ -271,9 +271,8 @@ - - - + +