diff --git a/hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java b/hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java index 5b78b05bc8473..681b05f2508c0 100644 --- a/hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java +++ b/hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java @@ -18,8 +18,6 @@ package org.apache.hudi; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.engine.HoodieEngineContext; @@ -41,6 +39,9 @@ import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.hadoop.CachingPath; import org.apache.hudi.metadata.HoodieTableMetadata; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -453,6 +454,10 @@ private static FileStatus[] flatMap(Collection arrays) { return arrays.stream().flatMap(Arrays::stream).toArray(FileStatus[]::new); } + /** + * Partition path information containing the relative partition path + * and values of partition columns. + */ public static final class PartitionPath { final String path; @@ -480,6 +485,9 @@ public int hashCode() { } } + /** + * APIs for caching {@link FileStatus}. + */ protected interface FileStatusCache { Option get(Path path); diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaUtils.java b/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaUtils.java index 6a87cc3c29907..4c97db275f15f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaUtils.java @@ -27,6 +27,9 @@ import static org.apache.hudi.common.util.ValidationUtils.checkState; +/** + * Utils for Avro Schema. + */ public class AvroSchemaUtils { private AvroSchemaUtils() {} diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java index ef2de67347fff..077d30104b8a6 100644 --- a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java @@ -53,7 +53,6 @@ import org.apache.avro.io.JsonDecoder; import org.apache.avro.io.JsonEncoder; import org.apache.avro.specific.SpecificRecordBase; - import org.apache.hadoop.util.VersionUtil; import java.io.ByteArrayInputStream; @@ -71,12 +70,12 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Deque; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Deque; -import java.util.LinkedList; import java.util.Set; import java.util.TimeZone; import java.util.stream.Collectors; diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java index e87364fb90970..2908cdda5c076 100644 --- a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java +++ b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java @@ -18,10 +18,11 @@ package org.apache.hudi.avro; -import org.apache.avro.Schema; import org.apache.hudi.common.bloom.BloomFilter; import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.common.util.Option; + +import org.apache.avro.Schema; import org.apache.parquet.avro.AvroWriteSupport; import org.apache.parquet.hadoop.api.WriteSupport; import org.apache.parquet.schema.MessageType; diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java b/hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java index 15335193414ae..2207961c0ba43 100644 --- a/hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java +++ b/hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java @@ -18,14 +18,15 @@ package org.apache.hudi.avro; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; + import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.avro.Schema; import org.apache.avro.Schema.Type; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; -import org.apache.hudi.common.util.collection.Pair; -import org.apache.hudi.exception.HoodieException; -import org.apache.hudi.exception.HoodieIOException; import java.io.IOException; import java.io.Serializable; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigClassProperty.java b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigClassProperty.java index f5d72a6bece71..f0a341eb8244d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigClassProperty.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigClassProperty.java @@ -23,17 +23,18 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; -@Retention(RetentionPolicy.RUNTIME) -@Target(ElementType.TYPE) - /** * Annotation for superclasses of {@link HoodieConfig} that includes the - * human-readable name of the config class, the config group ({@link ConfigGroupName}) + * human-readable name of the config class, the config group ({@link ConfigGroups}) * it belongs to (e.g., spark/ flink/ write) * and the description of the config class. */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) public @interface ConfigClassProperty { String name(); + ConfigGroups.Names groupName(); + String description(); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java index fef00389d8c54..0488401f10abf 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java @@ -25,6 +25,9 @@ * This class maintains the human readable name and description of each config group. */ public class ConfigGroups { + /** + * Config group names. + */ public enum Names { SPARK_DATASOURCE("Spark Datasource Configs"), FLINK_SQL("Flink Sql Configs"), diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieCommonConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieCommonConfig.java index 00ff7e5683307..63e8896760b76 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieCommonConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieCommonConfig.java @@ -26,6 +26,9 @@ import java.util.Locale; import java.util.Properties; +/** + * Hudi configs used across engines. + */ @ConfigClassProperty(name = "Common Configurations", groupName = ConfigGroups.Names.WRITE_CLIENT, description = "The following set of configurations are common across Hudi.") @@ -45,8 +48,8 @@ public class HoodieCommonConfig extends HoodieConfig { .key("hoodie.datasource.write.reconcile.schema") .defaultValue(false) .withDocumentation("When a new batch of write has records with old schema, but latest table schema got " - + "evolved, this config will upgrade the records to leverage latest table schema(default values will be " - + "injected to missing fields). If not, the write batch would fail."); + + "evolved, this config will upgrade the records to leverage latest table schema(default values will be " + + "injected to missing fields). If not, the write batch would fail."); public static final ConfigProperty SPILLABLE_DISK_MAP_TYPE = ConfigProperty .key("hoodie.common.spillable.diskmap.type") @@ -76,6 +79,9 @@ public static HoodieCommonConfig.Builder newBuilder() { return new HoodieCommonConfig.Builder(); } + /** + * Builder for {@link HoodieCommonConfig}. + */ public static class Builder { private final HoodieCommonConfig commonConfig = new HoodieCommonConfig(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java index 793d79e256aff..9979be38cfc94 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java @@ -329,6 +329,9 @@ public boolean getUseLogRecordReaderScanV2() { return getBoolean(USE_LOG_RECORD_READER_SCAN_V2); } + /** + * Builder for {@link HoodieMetadataConfig}. + */ public static class Builder { private EngineType engineType = EngineType.SPARK; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetastoreConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetastoreConfig.java index 36e2798a4d32a..d17c7e3486c04 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetastoreConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetastoreConfig.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.config; import javax.annotation.concurrent.Immutable; + import java.util.Properties; /** @@ -72,6 +73,9 @@ public int getConnectionRetryDelay() { return getIntOrDefault(METASTORE_CONNECTION_RETRY_DELAY); } + /** + * Builder for {@link HoodieMetastoreConfig}. + */ public static class Builder { private final HoodieMetastoreConfig config = new HoodieMetastoreConfig(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/data/HoodieBaseListData.java b/hudi-common/src/main/java/org/apache/hudi/common/data/HoodieBaseListData.java index 398762dc63070..7bc276b36e67a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/data/HoodieBaseListData.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/data/HoodieBaseListData.java @@ -25,6 +25,11 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +/** + * Data representation of either a stream or a list of objects with Type T. + * + * @param Object value type. + */ public abstract class HoodieBaseListData { protected final Either, List> data; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieLocalEngineContext.java b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieLocalEngineContext.java index 5d7d193dc6b8f..9781c694dc246 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieLocalEngineContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieLocalEngineContext.java @@ -18,8 +18,6 @@ package org.apache.hudi.common.engine; -import org.apache.hadoop.conf.Configuration; - import org.apache.hudi.common.config.SerializableConfiguration; import org.apache.hudi.common.data.HoodieAccumulator; import org.apache.hudi.common.data.HoodieAtomicLongAccumulator; @@ -31,10 +29,11 @@ import org.apache.hudi.common.function.SerializablePairFlatMapFunction; import org.apache.hudi.common.function.SerializablePairFunction; import org.apache.hudi.common.util.Option; - import org.apache.hudi.common.util.collection.ImmutablePair; import org.apache.hudi.common.util.collection.Pair; +import org.apache.hadoop.conf.Configuration; + import java.util.Collections; import java.util.Iterator; import java.util.List; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/engine/LocalTaskContextSupplier.java b/hudi-common/src/main/java/org/apache/hudi/common/engine/LocalTaskContextSupplier.java index 0c7ae20e1d5e8..b0decb8696f7e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/engine/LocalTaskContextSupplier.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/engine/LocalTaskContextSupplier.java @@ -22,6 +22,9 @@ import java.util.function.Supplier; +/** + * Supplier of task context using local Java engine. + */ public final class LocalTaskContextSupplier extends TaskContextSupplier { @Override public Supplier getPartitionIdSupplier() { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/BoundedFsDataInputStream.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/BoundedFsDataInputStream.java index 27315f85e62c9..0f2e5909610a4 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/BoundedFsDataInputStream.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/BoundedFsDataInputStream.java @@ -23,6 +23,9 @@ import java.io.IOException; import java.io.InputStream; +/** + * Implementation of {@link FSDataInputStream} with bound check based on file size. + */ public class BoundedFsDataInputStream extends FSDataInputStream { private FileSystem fs; private Path file; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java index 21b21256fa482..b7ab3e6cf913c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java @@ -845,6 +845,12 @@ public static List getFileStatusAtLevel( return result; } + /** + * Serializable function interface. + * + * @param Input value type. + * @param Output value type. + */ public interface SerializableFunction extends Function, Serializable { } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieRetryWrapperFileSystem.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieRetryWrapperFileSystem.java index 075f811a42ea7..ed32eeee636c2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieRetryWrapperFileSystem.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieRetryWrapperFileSystem.java @@ -18,6 +18,8 @@ package org.apache.hudi.common.fs; +import org.apache.hudi.common.util.RetryHelper; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; @@ -31,13 +33,15 @@ import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Progressable; -import org.apache.hudi.common.util.RetryHelper; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.util.EnumSet; +/** + * Extending {@link FileSystem} implementation of I/O operations with retries. + */ public class HoodieRetryWrapperFileSystem extends FileSystem { private FileSystem fileSystem; @@ -139,7 +143,7 @@ public FSDataOutputStream create(Path f, FsPermission permission, EnumSet fileSystem.create(f, permission, flags, bufferSize, replication, - blockSize, progress, checksumOpt)).start(); + blockSize, progress, checksumOpt)).start(); } @Override diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieWrapperFileSystem.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieWrapperFileSystem.java index fba24097fb6fb..14d3d31abb0c1 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieWrapperFileSystem.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieWrapperFileSystem.java @@ -73,6 +73,9 @@ public class HoodieWrapperFileSystem extends FileSystem { private static final String TMP_PATH_POSTFIX = ".tmp"; + /** + * Names for metrics. + */ protected enum MetricName { create, rename, delete, listStatus, mkdirs, getFileStatus, globStatus, listFiles, read, write } @@ -91,6 +94,11 @@ public static void setMetricsRegistry(Registry registry, Registry registryMeta) private URI uri; private ConsistencyGuard consistencyGuard = new NoOpConsistencyGuard(); + /** + * Checked function interface. + * + * @param Type of return value. + */ @FunctionalInterface public interface CheckedFunction { R get() throws IOException; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/function/SerializableSupplier.java b/hudi-common/src/main/java/org/apache/hudi/common/function/SerializableSupplier.java index 0500955f94dc8..d8cae0548f7e1 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/function/SerializableSupplier.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/function/SerializableSupplier.java @@ -20,6 +20,11 @@ import java.io.Serializable; +/** + * Serializable supplier interface. + * + * @param Type of return value. + */ @FunctionalInterface public interface SerializableSupplier extends Serializable { T get(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/metrics/Registry.java b/hudi-common/src/main/java/org/apache/hudi/common/metrics/Registry.java index bf5ee7e7b71c2..2dc52b4dff94d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/metrics/Registry.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/metrics/Registry.java @@ -18,14 +18,13 @@ package org.apache.hudi.common.metrics; +import org.apache.hudi.common.util.ReflectionUtils; + import java.io.Serializable; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import org.apache.hudi.common.util.ReflectionUtils; - - /** * Interface which defines a lightweight Metrics Registry to track Hudi events. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/AWSDmsAvroPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/AWSDmsAvroPayload.java index fe044e0b431f1..451008257a535 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/AWSDmsAvroPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/AWSDmsAvroPayload.java @@ -18,10 +18,11 @@ package org.apache.hudi.common.model; +import org.apache.hudi.common.util.Option; + import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; -import org.apache.hudi.common.util.Option; import java.io.IOException; import java.util.Properties; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/BaseFile.java b/hudi-common/src/main/java/org/apache/hudi/common/model/BaseFile.java index fe9837e6c693b..7c4c89777957c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/BaseFile.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/BaseFile.java @@ -18,9 +18,10 @@ package org.apache.hudi.common.model; +import org.apache.hudi.hadoop.CachingPath; + import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; -import org.apache.hudi.hadoop.CachingPath; import java.io.Serializable; import java.util.Objects; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapBaseFileMapping.java b/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapBaseFileMapping.java index af375038c5104..349a953557724 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapBaseFileMapping.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapBaseFileMapping.java @@ -18,10 +18,11 @@ package org.apache.hudi.common.model; -import java.io.Serializable; import org.apache.hudi.avro.model.HoodieFileStatus; import org.apache.hudi.common.bootstrap.FileStatusUtils; +import java.io.Serializable; + /** * POJO storing (partitionPath, hoodieFileId) -> external base file path. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapFileMapping.java b/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapFileMapping.java index 428d41104e3a3..e816a7385e2df 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapFileMapping.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapFileMapping.java @@ -18,9 +18,10 @@ package org.apache.hudi.common.model; +import org.apache.hudi.avro.model.HoodieFileStatus; + import java.io.Serializable; import java.util.Objects; -import org.apache.hudi.avro.model.HoodieFileStatus; /** * Value stored in the bootstrap index. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/CleanFileInfo.java b/hudi-common/src/main/java/org/apache/hudi/common/model/CleanFileInfo.java index dd6db7aa32512..22939a2aee7d0 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/CleanFileInfo.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/CleanFileInfo.java @@ -22,6 +22,9 @@ import java.io.Serializable; +/** + * File info for clean action. + */ public class CleanFileInfo implements Serializable { private final String filePath; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/ConsistentHashingNode.java b/hudi-common/src/main/java/org/apache/hudi/common/model/ConsistentHashingNode.java index 66eb8deab2af8..8c931685ac2c7 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/ConsistentHashingNode.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/ConsistentHashingNode.java @@ -86,6 +86,9 @@ public String toString() { return sb.toString(); } + /** + * Node tag. + */ public enum NodeTag { /** * Standard node. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroRecord.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroRecord.java index 9a9bbb2b7427f..210c42def8d81 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroRecord.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroRecord.java @@ -19,6 +19,11 @@ package org.apache.hudi.common.model; +/** + * Implementation of {@link HoodieRecord} using Avro payload. + * + * @param payload implementation. + */ public class HoodieAvroRecord extends HoodieRecord { public HoodieAvroRecord(HoodieKey key, T data) { super(key, data); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieColumnRangeMetadata.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieColumnRangeMetadata.java index 1974544e63278..8afe08a37c863 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieColumnRangeMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieColumnRangeMetadata.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.model; import javax.annotation.Nullable; + import java.io.Serializable; import java.util.Objects; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java index 8568e9926a4bc..b1014885c815e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java @@ -22,15 +22,14 @@ import org.apache.hudi.common.util.JsonUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieException; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; -import org.apache.hudi.exception.HoodieException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieDeltaWriteStat.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieDeltaWriteStat.java index b3ae5b8ceced4..4fee7cdcb6eaa 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieDeltaWriteStat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieDeltaWriteStat.java @@ -18,9 +18,10 @@ package org.apache.hudi.common.model; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.apache.hudi.common.util.Option; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java index d4ad2cae1fe18..d4be8b50680c5 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java @@ -18,10 +18,11 @@ package org.apache.hudi.common.model; +import org.apache.hudi.common.fs.FSUtils; + import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hudi.common.fs.FSUtils; import java.io.IOException; import java.io.Serializable; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieMetadataWrapper.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieMetadataWrapper.java index ecc18a7dfaddd..e5b1c17a99eb1 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieMetadataWrapper.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieMetadataWrapper.java @@ -20,6 +20,9 @@ import org.apache.hudi.avro.model.HoodieArchivedMetaEntry; +/** + * Wrapper for Hudi metadata. + */ public class HoodieMetadataWrapper { private HoodieArchivedMetaEntry avroMetadataFromTimeline; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieWriteStat.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieWriteStat.java index 6c1efc1561210..095c1b38387c0 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieWriteStat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieWriteStat.java @@ -18,9 +18,10 @@ package org.apache.hudi.common.model; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.util.JsonUtils; +import org.apache.hadoop.fs.Path; + import javax.annotation.Nullable; import java.io.Serializable; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/OverwriteNonDefaultsWithLatestAvroPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/OverwriteNonDefaultsWithLatestAvroPayload.java index 5ee2b58373c51..3a7fc8d642c81 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/OverwriteNonDefaultsWithLatestAvroPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/OverwriteNonDefaultsWithLatestAvroPayload.java @@ -18,13 +18,13 @@ package org.apache.hudi.common.model; +import org.apache.hudi.common.util.Option; + import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecordBuilder; import org.apache.avro.generic.IndexedRecord; -import org.apache.hudi.common.util.Option; - import java.io.IOException; import java.util.List; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/OverwriteWithLatestAvroPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/OverwriteWithLatestAvroPayload.java index d8469ed5a148e..6bd945f389d09 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/OverwriteWithLatestAvroPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/OverwriteWithLatestAvroPayload.java @@ -18,10 +18,10 @@ package org.apache.hudi.common.model; -import org.apache.avro.JsonProperties; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.common.util.Option; +import org.apache.avro.JsonProperties; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/PartialUpdateAvroPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/PartialUpdateAvroPayload.java index daa40acc76404..217cf6cc88dee 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/PartialUpdateAvroPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/PartialUpdateAvroPayload.java @@ -18,17 +18,17 @@ package org.apache.hudi.common.model; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.GenericRecordBuilder; -import org.apache.avro.generic.IndexedRecord; - import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.GenericRecordBuilder; +import org.apache.avro.generic.IndexedRecord; + import java.io.IOException; import java.util.List; import java.util.Properties; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/RewriteAvroPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/RewriteAvroPayload.java index d5c19b9116bbc..bfeaf8770eea4 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/RewriteAvroPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/RewriteAvroPayload.java @@ -18,10 +18,11 @@ package org.apache.hudi.common.model; +import org.apache.hudi.common.util.Option; + import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; -import org.apache.hudi.common.util.Option; import java.io.IOException; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java index ac3608fc003dc..bad9b81629ec7 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java @@ -54,11 +54,11 @@ import java.io.IOException; import java.time.Instant; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; -import java.util.HashSet; import java.util.function.BiConsumer; import java.util.stream.Collectors; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java index 4fbf7c53b7175..17e1c29ceee81 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java @@ -734,6 +734,9 @@ public static PropertyBuilder withPropertyBuilder() { return new PropertyBuilder(); } + /** + * Builder for {@link Properties}. + */ public static class PropertyBuilder { private HoodieTableType tableType; @@ -929,9 +932,9 @@ public PropertyBuilder set(Map props) { public PropertyBuilder fromMetaClient(HoodieTableMetaClient metaClient) { return setTableType(metaClient.getTableType()) - .setTableName(metaClient.getTableConfig().getTableName()) - .setArchiveLogFolder(metaClient.getArchivePath()) - .setPayloadClassName(metaClient.getTableConfig().getPayloadClass()); + .setTableName(metaClient.getTableConfig().getTableName()) + .setArchiveLogFolder(metaClient.getArchivePath()) + .setPayloadClassName(metaClient.getTableConfig().getPayloadClass()); } public PropertyBuilder fromProperties(Properties properties) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCExtractor.java b/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCExtractor.java index f4cb5fec4ae9e..9e2fe4e09e799 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCExtractor.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCExtractor.java @@ -19,13 +19,13 @@ package org.apache.hudi.common.table.cdc; import org.apache.hudi.common.fs.FSUtils; +import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieFileGroupId; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieReplaceCommitMetadata; import org.apache.hudi.common.model.HoodieWriteStat; -import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.log.InstantRange; @@ -50,16 +50,16 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Locale; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static org.apache.hudi.common.table.cdc.HoodieCDCInferCase.BASE_FILE_INSERT; import static org.apache.hudi.common.table.cdc.HoodieCDCInferCase.AS_IS; -import static org.apache.hudi.common.table.cdc.HoodieCDCInferCase.LOG_FILE; import static org.apache.hudi.common.table.cdc.HoodieCDCInferCase.BASE_FILE_DELETE; +import static org.apache.hudi.common.table.cdc.HoodieCDCInferCase.BASE_FILE_INSERT; +import static org.apache.hudi.common.table.cdc.HoodieCDCInferCase.LOG_FILE; import static org.apache.hudi.common.table.cdc.HoodieCDCInferCase.REPLACE_COMMIT; import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION; import static org.apache.hudi.common.table.timeline.HoodieTimeline.DELTA_COMMIT_ACTION; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java index 0c8d8b3f6cfeb..0787324b82cab 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java @@ -18,8 +18,8 @@ package org.apache.hudi.common.table.log; -import org.apache.hudi.common.model.DeleteRecord; import org.apache.hudi.avro.HoodieAvroUtils; +import org.apache.hudi.common.model.DeleteRecord; import org.apache.hudi.common.model.HoodieAvroRecord; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodiePayloadProps; @@ -36,22 +36,22 @@ import org.apache.hudi.common.table.log.block.HoodieParquetDataBlock; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.util.ClosableIterator; +import org.apache.hudi.common.util.InternalSchemaCache; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.SpillableMapUtils; -import org.apache.hudi.common.util.InternalSchemaCache; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.internal.schema.InternalSchema; +import org.apache.hudi.internal.schema.action.InternalSchemaMerger; +import org.apache.hudi.internal.schema.convert.AvroInternalSchemaConverter; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hudi.internal.schema.InternalSchema; -import org.apache.hudi.internal.schema.action.InternalSchemaMerger; -import org.apache.hudi.internal.schema.convert.AvroInternalSchemaConverter; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -64,8 +64,8 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Properties; import java.util.Map; +import java.util.Properties; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; @@ -786,6 +786,9 @@ protected Properties getPayloadProps() { return payloadProps; } + /** + * Key specification with a list of column names. + */ protected static class KeySpec { private final List keys; private final boolean fullKey; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieCDCLogRecordIterator.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieCDCLogRecordIterator.java index 3d22a68a0af65..230dae73de99a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieCDCLogRecordIterator.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieCDCLogRecordIterator.java @@ -25,13 +25,15 @@ import org.apache.avro.Schema; import org.apache.avro.generic.IndexedRecord; - import org.apache.hadoop.fs.FileSystem; import java.io.IOException; import java.util.Arrays; import java.util.Iterator; +/** + * Record iterator for Hudi logs in CDC format. + */ public class HoodieCDCLogRecordIterator implements ClosableIterator { private final FileSystem fs; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java index c48107e392515..b8e4ee9940564 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java @@ -21,10 +21,10 @@ import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.internal.schema.InternalSchema; import org.apache.avro.Schema; import org.apache.hadoop.fs.FileSystem; -import org.apache.hudi.internal.schema.InternalSchema; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java index 708015b1c16f2..5fda4bb7def7f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java @@ -18,9 +18,6 @@ package org.apache.hudi.common.table.log; -import org.apache.avro.Schema; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.config.HoodieCommonConfig; import org.apache.hudi.common.model.DeleteRecord; import org.apache.hudi.common.model.HoodieAvroRecord; @@ -39,6 +36,10 @@ import org.apache.hudi.common.util.collection.ExternalSpillableMap; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.internal.schema.InternalSchema; + +import org.apache.avro.Schema; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java index b0d127c562991..863fa8162b1a6 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java @@ -23,10 +23,10 @@ import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.table.cdc.HoodieCDCUtils; import org.apache.hudi.common.util.Option; +import org.apache.hudi.internal.schema.InternalSchema; import org.apache.avro.Schema; import org.apache.hadoop.fs.FileSystem; -import org.apache.hudi.internal.schema.InternalSchema; import java.util.List; import java.util.stream.Collectors; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java index 9e74d14c048f2..a840195d5581b 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java @@ -18,6 +18,13 @@ package org.apache.hudi.common.table.log.block; +import org.apache.hudi.common.fs.SizeAwareDataInputStream; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.util.ClosableIterator; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.internal.schema.InternalSchema; + import org.apache.avro.Schema; import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericDatumWriter; @@ -29,14 +36,9 @@ import org.apache.avro.io.Encoder; import org.apache.avro.io.EncoderFactory; import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hudi.common.fs.SizeAwareDataInputStream; -import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.util.ClosableIterator; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.exception.HoodieIOException; -import org.apache.hudi.internal.schema.InternalSchema; import javax.annotation.Nonnull; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCDCDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCDCDataBlock.java index cc5663262c1ce..1a29e60a95770 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCDCDataBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCDCDataBlock.java @@ -18,13 +18,12 @@ package org.apache.hudi.common.table.log.block; +import org.apache.hudi.common.util.Option; + import org.apache.avro.Schema; import org.apache.avro.generic.IndexedRecord; - import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hudi.common.util.Option; - import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java index c83b3bc82d56c..018dc8bac2e1e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java @@ -21,11 +21,11 @@ import org.apache.hudi.common.util.ClosableIterator; import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.internal.schema.InternalSchema; import org.apache.avro.Schema; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hudi.internal.schema.InternalSchema; import java.io.IOException; import java.util.HashSet; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieHFileDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieHFileDataBlock.java index d923c592708c8..c14ef60c2e98f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieHFileDataBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieHFileDataBlock.java @@ -18,6 +18,17 @@ package org.apache.hudi.common.table.log.block; +import org.apache.hudi.avro.HoodieAvroUtils; +import org.apache.hudi.common.fs.inline.InLineFSUtils; +import org.apache.hudi.common.fs.inline.InLineFileSystem; +import org.apache.hudi.common.util.ClosableIterator; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; +import org.apache.hudi.common.util.ValidationUtils; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.io.storage.HoodieHBaseKVComparator; +import org.apache.hudi.io.storage.HoodieHFileReader; + import org.apache.avro.Schema; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.conf.Configuration; @@ -30,16 +41,6 @@ import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; -import org.apache.hudi.avro.HoodieAvroUtils; -import org.apache.hudi.common.fs.inline.InLineFSUtils; -import org.apache.hudi.common.fs.inline.InLineFileSystem; -import org.apache.hudi.common.util.ClosableIterator; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.StringUtils; -import org.apache.hudi.common.util.ValidationUtils; -import org.apache.hudi.exception.HoodieIOException; -import org.apache.hudi.io.storage.HoodieHBaseKVComparator; -import org.apache.hudi.io.storage.HoodieHFileReader; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java index f8307333d257c..b85febb9a0f54 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java @@ -18,7 +18,6 @@ package org.apache.hudi.common.table.log.block; -import org.apache.hadoop.conf.Configuration; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner; import org.apache.hudi.common.util.Option; @@ -26,6 +25,7 @@ import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import javax.annotation.Nonnull; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieParquetDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieParquetDataBlock.java index eaad70fb73475..2c0034c4d8bab 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieParquetDataBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieParquetDataBlock.java @@ -18,12 +18,6 @@ package org.apache.hudi.common.table.log.block; -import org.apache.avro.Schema; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.HoodieAvroWriteSupport; import org.apache.hudi.common.fs.inline.InLineFSUtils; import org.apache.hudi.common.fs.inline.InLineFileSystem; @@ -32,6 +26,13 @@ import org.apache.hudi.common.util.ParquetReaderIterator; import org.apache.hudi.io.storage.HoodieParquetConfig; import org.apache.hudi.io.storage.HoodieParquetStreamWriter; + +import org.apache.avro.Schema; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; import org.apache.parquet.avro.AvroParquetReader; import org.apache.parquet.avro.AvroReadSupport; import org.apache.parquet.avro.AvroSchemaConverter; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstantTimeGenerator.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstantTimeGenerator.java index de18c4fbaba6c..e839e73669e90 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstantTimeGenerator.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstantTimeGenerator.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.table.timeline; import org.apache.hudi.common.model.HoodieTimelineTimeZone; + import java.text.ParseException; import java.time.LocalDateTime; import java.time.ZoneId; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineMetadataUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineMetadataUtils.java index b50846b8780bf..c0550fef6fe08 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineMetadataUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineMetadataUtils.java @@ -59,6 +59,9 @@ import java.util.Map; import java.util.stream.Collectors; +/** + * Utils for Hudi timeline metadata. + */ public class TimelineMetadataUtils { private static final Integer DEFAULT_VERSION = 1; @@ -74,7 +77,7 @@ public static HoodieRestoreMetadata convertRestoreMetadata(String startRestoreTi } public static HoodieRollbackMetadata convertRollbackMetadata(String startRollbackTime, Option durationInMs, - List instants, List rollbackStats) { + List instants, List rollbackStats) { Map partitionMetadataBuilder = new HashMap<>(); int totalDeleted = 0; for (HoodieRollbackStat stat : rollbackStats) { @@ -87,20 +90,20 @@ public static HoodieRollbackMetadata convertRollbackMetadata(String startRollbac } return new HoodieRollbackMetadata(startRollbackTime, durationInMs.orElseGet(() -> -1L), totalDeleted, - instants.stream().map(HoodieInstant::getTimestamp).collect(Collectors.toList()), - Collections.unmodifiableMap(partitionMetadataBuilder), DEFAULT_VERSION, - instants.stream().map(instant -> new HoodieInstantInfo(instant.getTimestamp(), instant.getAction())).collect(Collectors.toList())); + instants.stream().map(HoodieInstant::getTimestamp).collect(Collectors.toList()), + Collections.unmodifiableMap(partitionMetadataBuilder), DEFAULT_VERSION, + instants.stream().map(instant -> new HoodieInstantInfo(instant.getTimestamp(), instant.getAction())).collect(Collectors.toList())); } public static HoodieSavepointMetadata convertSavepointMetadata(String user, String comment, - Map> latestFiles) { + Map> latestFiles) { Map partitionMetadataBuilder = new HashMap<>(); for (Map.Entry> stat : latestFiles.entrySet()) { HoodieSavepointPartitionMetadata metadata = new HoodieSavepointPartitionMetadata(stat.getKey(), stat.getValue()); partitionMetadataBuilder.put(stat.getKey(), metadata); } return new HoodieSavepointMetadata(user, System.currentTimeMillis(), comment, - Collections.unmodifiableMap(partitionMetadataBuilder), DEFAULT_VERSION); + Collections.unmodifiableMap(partitionMetadataBuilder), DEFAULT_VERSION); } public static Option serializeCompactionPlan(HoodieCompactionPlan compactionWorkload) throws IOException { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/ClusteringOpDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/ClusteringOpDTO.java index e87226c990098..ba69dd9192286 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/ClusteringOpDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/ClusteringOpDTO.java @@ -18,12 +18,13 @@ package org.apache.hudi.common.table.timeline.dto; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.hudi.common.model.HoodieFileGroupId; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.collection.Pair; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + /** * The data transfer object of clustering. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanMetadataMigrator.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanMetadataMigrator.java index adb5cc00a2e79..a656b0abb4e09 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanMetadataMigrator.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanMetadataMigrator.java @@ -24,6 +24,9 @@ import java.util.Arrays; +/** + * Migrator for clean metadata. + */ public class CleanMetadataMigrator extends MetadataMigrator { public CleanMetadataMigrator(HoodieTableMetaClient metaClient) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanMetadataV1MigrationHandler.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanMetadataV1MigrationHandler.java index ca1c5506e5e39..1f7b5792eb09f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanMetadataV1MigrationHandler.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanMetadataV1MigrationHandler.java @@ -31,6 +31,9 @@ import java.util.Map; import java.util.stream.Collectors; +/** + * Migration handler for clean metadata in version 1. + */ public class CleanMetadataV1MigrationHandler extends AbstractMigratorBase { public static final Integer VERSION = 1; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanMetadataV2MigrationHandler.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanMetadataV2MigrationHandler.java index 5eb47d2647619..d811047cf6f5f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanMetadataV2MigrationHandler.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanMetadataV2MigrationHandler.java @@ -31,6 +31,9 @@ import java.util.Map; import java.util.stream.Collectors; +/** + * Migration handler for clean metadata in version 2. + */ public class CleanMetadataV2MigrationHandler extends AbstractMigratorBase { public static final Integer VERSION = 2; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanPlanV1MigrationHandler.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanPlanV1MigrationHandler.java index 5c8d9b8fb3e26..844376cbbfd64 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanPlanV1MigrationHandler.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanPlanV1MigrationHandler.java @@ -18,8 +18,6 @@ package org.apache.hudi.common.table.timeline.versioning.clean; -import java.util.ArrayList; -import java.util.HashMap; import org.apache.hudi.avro.model.HoodieCleanerPlan; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.versioning.AbstractMigratorBase; @@ -27,10 +25,15 @@ import org.apache.hadoop.fs.Path; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +/** + * Migration handler for clean plan in version 1. + */ public class CleanPlanV1MigrationHandler extends AbstractMigratorBase { public static final Integer VERSION = 1; @@ -47,18 +50,18 @@ public Integer getManagedVersion() { @Override public HoodieCleanerPlan upgradeFrom(HoodieCleanerPlan plan) { throw new IllegalArgumentException( - "This is the lowest version. Plan cannot be any lower version"); + "This is the lowest version. Plan cannot be any lower version"); } @Override public HoodieCleanerPlan downgradeFrom(HoodieCleanerPlan plan) { if (metaClient.getTableConfig().getBootstrapBasePath().isPresent()) { throw new IllegalArgumentException( - "This version do not support METADATA_ONLY bootstrapped tables. Failed to downgrade."); + "This version do not support METADATA_ONLY bootstrapped tables. Failed to downgrade."); } Map> filesPerPartition = plan.getFilePathsToBeDeletedPerPartition().entrySet().stream() .map(e -> Pair.of(e.getKey(), e.getValue().stream().map(v -> new Path(v.getFilePath()).getName()) - .collect(Collectors.toList()))).collect(Collectors.toMap(Pair::getKey, Pair::getValue)); + .collect(Collectors.toList()))).collect(Collectors.toMap(Pair::getKey, Pair::getValue)); return new HoodieCleanerPlan(plan.getEarliestInstantToRetain(), plan.getLastCompletedCommitTimestamp(), plan.getPolicy(), filesPerPartition, VERSION, new HashMap<>(), new ArrayList<>()); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanPlanV2MigrationHandler.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanPlanV2MigrationHandler.java index c17af4020a3ca..aacdd26aeda5b 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanPlanV2MigrationHandler.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanPlanV2MigrationHandler.java @@ -33,6 +33,9 @@ import java.util.Map; import java.util.stream.Collectors; +/** + * Migration handler for clean plan in version 2. + */ public class CleanPlanV2MigrationHandler extends AbstractMigratorBase { public static final Integer VERSION = 2; @@ -50,9 +53,9 @@ public Integer getManagedVersion() { public HoodieCleanerPlan upgradeFrom(HoodieCleanerPlan plan) { Map> filePathsPerPartition = plan.getFilesToBeDeletedPerPartition().entrySet().stream().map(e -> Pair.of(e.getKey(), e.getValue().stream() - .map(v -> new HoodieCleanFileInfo( - new Path(FSUtils.getPartitionPath(metaClient.getBasePath(), e.getKey()), v).toString(), false)) - .collect(Collectors.toList()))).collect(Collectors.toMap(Pair::getKey, Pair::getValue)); + .map(v -> new HoodieCleanFileInfo( + new Path(FSUtils.getPartitionPath(metaClient.getBasePath(), e.getKey()), v).toString(), false)) + .collect(Collectors.toList()))).collect(Collectors.toMap(Pair::getKey, Pair::getValue)); return new HoodieCleanerPlan(plan.getEarliestInstantToRetain(), plan.getLastCompletedCommitTimestamp(), plan.getPolicy(), new HashMap<>(), VERSION, filePathsPerPartition, new ArrayList<>()); } @@ -60,6 +63,6 @@ public HoodieCleanerPlan upgradeFrom(HoodieCleanerPlan plan) { @Override public HoodieCleanerPlan downgradeFrom(HoodieCleanerPlan input) { throw new IllegalArgumentException( - "This is the current highest version. Plan cannot be any higher version"); + "This is the current highest version. Plan cannot be any higher version"); } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTablePreCommitFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTablePreCommitFileSystemView.java index 7401617a6abb6..d8d210e9c3d58 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTablePreCommitFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTablePreCommitFileSystemView.java @@ -18,11 +18,12 @@ package org.apache.hudi.common.table.view; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hadoop.fs.Path; + import java.util.Collections; import java.util.List; import java.util.Map; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java index 4c2aa79f27c6d..ea49cfb54a82b 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java @@ -18,8 +18,6 @@ package org.apache.hudi.common.table.view; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.avro.model.HoodieRestoreMetadata; @@ -44,6 +42,9 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieException; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java index 2389f267818ec..005b6d1fe9118 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java @@ -18,8 +18,6 @@ package org.apache.hudi.common.table.view; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.model.BootstrapBaseFileMapping; import org.apache.hudi.common.model.CompactionOperation; import org.apache.hudi.common.model.FileSlice; @@ -35,6 +33,9 @@ import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.common.util.collection.RocksDBDAO; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java index 26d36d9c97bc0..3bfa8dd04bd07 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java @@ -18,8 +18,6 @@ package org.apache.hudi.common.table.view; -import org.apache.hadoop.fs.FileStatus; - import org.apache.hudi.common.config.HoodieCommonConfig; import org.apache.hudi.common.model.BootstrapBaseFileMapping; import org.apache.hudi.common.model.CompactionOperation; @@ -32,6 +30,8 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.ExternalSpillableMap; import org.apache.hudi.common.util.collection.Pair; + +import org.apache.hadoop.fs.FileStatus; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/AvroOrcUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/AvroOrcUtils.java index 607b1fb3bd4d7..e5e4791fe569f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/AvroOrcUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/AvroOrcUtils.java @@ -18,24 +18,14 @@ package org.apache.hudi.common.util; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.sql.Timestamp; -import java.util.Base64; -import java.util.Date; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; +import org.apache.hudi.exception.HoodieIOException; + import org.apache.avro.Conversions; import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; import org.apache.avro.Schema.Field; import org.apache.avro.generic.GenericData; -import java.nio.charset.StandardCharsets; -import org.apache.avro.Schema; import org.apache.avro.generic.GenericData.StringType; import org.apache.avro.util.Utf8; import org.apache.hadoop.hive.common.type.HiveDecimal; @@ -50,9 +40,21 @@ import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector; import org.apache.hadoop.hive.serde2.io.DateWritable; -import org.apache.hudi.exception.HoodieIOException; import org.apache.orc.TypeDescription; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + import static org.apache.avro.JsonProperties.NULL_VALUE; import static org.apache.hudi.common.util.BinaryUtil.toBytes; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/Base64CodecUtil.java b/hudi-common/src/main/java/org/apache/hudi/common/util/Base64CodecUtil.java index 97e9133cfa51c..d40659de6ff52 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/Base64CodecUtil.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/Base64CodecUtil.java @@ -21,6 +21,9 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; +/** + * Utils for Base64 encoding and decoding. + */ public final class Base64CodecUtil { /** diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/BaseFileUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/BaseFileUtils.java index badb5e37a70f3..b7bfddc68724e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/BaseFileUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/BaseFileUtils.java @@ -39,6 +39,9 @@ import java.util.Map; import java.util.Set; +/** + * Utils for Hudi base file. + */ public abstract class BaseFileUtils { public static BaseFileUtils getInstance(String path) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/BinaryUtil.java b/hudi-common/src/main/java/org/apache/hudi/common/util/BinaryUtil.java index 9d8f6c8e90cf3..502ce85f4e82b 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/BinaryUtil.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/BinaryUtil.java @@ -22,6 +22,9 @@ import java.nio.charset.Charset; import java.util.zip.CRC32; +/** + * Utils for Java byte array. + */ public class BinaryUtil { /** diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/CleanerUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/CleanerUtils.java index 513c4fa29ed25..704dacc6073d8 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/CleanerUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/CleanerUtils.java @@ -18,7 +18,6 @@ package org.apache.hudi.common.util; -import java.util.stream.Collectors; import org.apache.hudi.avro.model.HoodieCleanFileInfo; import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieCleanPartitionMetadata; @@ -42,9 +41,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION; +/** + * Utils for clean action. + */ public class CleanerUtils { private static final Logger LOG = LogManager.getLogger(CleanerUtils.class); @@ -83,7 +86,7 @@ public static HoodieCleanMetadata convertCleanMetadata(String startCleanTime, } return new HoodieCleanMetadata(startCleanTime, durationInMs.orElseGet(() -> -1L), totalDeleted, earliestCommitToRetain, - lastCompletedCommitTimestamp, partitionMetadataMap, CLEAN_METADATA_VERSION_2, partitionBootstrapMetadataMap); + lastCompletedCommitTimestamp, partitionMetadataMap, CLEAN_METADATA_VERSION_2, partitionBootstrapMetadataMap); } /** diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/ClusteringUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ClusteringUtils.java index 9d741a03f82ec..b415b3bb64dfa 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/ClusteringUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ClusteringUtils.java @@ -36,6 +36,7 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; + import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/CollectionUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/CollectionUtils.java index 2df785eec2180..e3a84e7d307ac 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/CollectionUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/CollectionUtils.java @@ -42,6 +42,9 @@ import static org.apache.hudi.common.util.ValidationUtils.checkArgument; +/** + * Utils for Java Collection. + */ public class CollectionUtils { private static final Properties EMPTY_PROPERTIES = new Properties(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/CustomizedThreadFactory.java b/hudi-common/src/main/java/org/apache/hudi/common/util/CustomizedThreadFactory.java index 2af7c078f7be5..1b77cac9c223e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/CustomizedThreadFactory.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/CustomizedThreadFactory.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.util; import javax.annotation.Nonnull; + import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicLong; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/DateTimeUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/DateTimeUtils.java index cf90eff8d6185..9dde7727806c2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/DateTimeUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/DateTimeUtils.java @@ -35,6 +35,9 @@ import java.util.Objects; import java.util.stream.Collectors; +/** + * Utils for Hudi instant time. + */ public class DateTimeUtils { private static final Map LABEL_TO_UNIT_MAP = Collections.unmodifiableMap(initMap()); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/Either.java b/hudi-common/src/main/java/org/apache/hudi/common/util/Either.java index fb624c6075349..9eefed5ce8bc1 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/Either.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/Either.java @@ -65,8 +65,15 @@ public static Either left(L left) { return new EitherLeft<>(left); } + /** + * Utility that holds R value only. + * + * @param Left value type. + * @param Right value type. + */ public static class EitherRight extends Either { private final R value; + private EitherRight(@Nonnull R right) { this.value = right; } @@ -78,8 +85,15 @@ protected R getValue() { } } + /** + * Utility that holds L value only. + * + * @param Left value type. + * @param Right value type. + */ public static class EitherLeft extends Either { private final L value; + private EitherLeft(@Nonnull L value) { this.value = value; } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/FutureUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/FutureUtils.java index b0029917eebdf..d9c1dee6e8173 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/FutureUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/FutureUtils.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.util; import javax.annotation.Nonnull; + import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/InternalSchemaCache.java b/hudi-common/src/main/java/org/apache/hudi/common/util/InternalSchemaCache.java index 1d2786197780c..a848fd42e2eff 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/InternalSchemaCache.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/InternalSchemaCache.java @@ -18,9 +18,6 @@ package org.apache.hudi.common.util; -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; - import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.table.HoodieTableMetaClient; @@ -33,6 +30,8 @@ import org.apache.hudi.internal.schema.utils.InternalSchemaUtils; import org.apache.hudi.internal.schema.utils.SerDeHelper; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; @@ -47,6 +46,11 @@ import java.util.TreeMap; import java.util.stream.Collectors; +/** + * An internal cache implementation for managing different version of schemas. + * This is a Global cache; all threads in one container/executor share the same cache. + * A map of (tablePath, HistorySchemas) is maintained. + */ public class InternalSchemaCache { private static final Logger LOG = LogManager.getLogger(InternalSchemaCache.class); // Use segment lock to reduce competition. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/JsonUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/JsonUtils.java index 4dce10f454530..7fc1b6fa23016 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/JsonUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/JsonUtils.java @@ -19,16 +19,21 @@ package org.apache.hudi.common.util; +import org.apache.hudi.exception.HoodieIOException; + import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.hudi.exception.HoodieIOException; +/** + * Utils for JSON serialization and deserialization. + */ public class JsonUtils { private static final ObjectMapper MAPPER = new ObjectMapper(); + static { MAPPER.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); // We need to exclude custom getters, setters and creators which can use member fields diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java index c39f6fd74f424..57c9dfd0f816a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java @@ -22,6 +22,9 @@ import java.util.Map; import java.util.Objects; +/** + * Utils for Java Map. + */ public class MapUtils { public static boolean isNullOrEmpty(Map m) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/OrcReaderIterator.java b/hudi-common/src/main/java/org/apache/hudi/common/util/OrcReaderIterator.java index 727bccb5e2de6..f19a1daf74c4a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/OrcReaderIterator.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/OrcReaderIterator.java @@ -18,17 +18,17 @@ package org.apache.hudi.common.util; -import java.util.List; +import org.apache.hudi.exception.HoodieIOException; + import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericData.Record; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hudi.exception.HoodieIOException; - import org.apache.orc.RecordReader; import org.apache.orc.TypeDescription; import java.io.IOException; +import java.util.List; /** * This class wraps a ORC reader and provides an iterator based api to read from an ORC file. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java index 8c41f7f3e58d9..2431c6605a0f2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java @@ -32,14 +32,14 @@ import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.orc.OrcFile; import org.apache.orc.OrcProto.UserMetadataItem; import org.apache.orc.Reader; import org.apache.orc.Reader.Options; import org.apache.orc.RecordReader; import org.apache.orc.TypeDescription; -import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/RateLimiter.java b/hudi-common/src/main/java/org/apache/hudi/common/util/RateLimiter.java index 4915e454af215..c52bbbc6236c3 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/RateLimiter.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/RateLimiter.java @@ -21,12 +21,16 @@ import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import javax.annotation.concurrent.ThreadSafe; + import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; -import javax.annotation.concurrent.ThreadSafe; +/** + * Thread-safe rate limiter implementation. + */ @ThreadSafe public class RateLimiter { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/RetryHelper.java b/hudi-common/src/main/java/org/apache/hudi/common/util/RetryHelper.java index 2e82b548f0da7..d713a343b67fe 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/RetryHelper.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/RetryHelper.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.util; import org.apache.hudi.exception.HoodieException; + import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -30,6 +31,11 @@ import java.util.Random; import java.util.stream.Collectors; +/** + * Retry Helper implementation. + * + * @param Type of return value for checked function. + */ public class RetryHelper implements Serializable { private static final Logger LOG = LogManager.getLogger(RetryHelper.class); private transient CheckedFunction func; @@ -48,9 +54,9 @@ public RetryHelper(long maxRetryIntervalMs, int maxRetryNumbers, long initialRet } else { try { this.retryExceptionsClasses = Arrays.stream(retryExceptions.split(",")) - .map(exception -> (Exception) ReflectionUtils.loadClass(exception, "")) - .map(Exception::getClass) - .collect(Collectors.toList()); + .map(exception -> (Exception) ReflectionUtils.loadClass(exception, "")) + .map(Exception::getClass) + .collect(Collectors.toList()); } catch (HoodieException e) { LOG.error("Exception while loading retry exceptions classes '" + retryExceptions + "'.", e); this.retryExceptionsClasses = new ArrayList<>(); @@ -136,6 +142,11 @@ private long getWaitTimeExp(int retryCount) { return (long) Math.pow(2, retryCount) * initialIntervalTime + random.nextInt(100); } + /** + * Checked function interface. + * + * @param Type of return value. + */ @FunctionalInterface public interface CheckedFunction extends Serializable { T get() throws IOException; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/TablePathUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/TablePathUtils.java index 9d279d5328ccc..937019db8d00d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/TablePathUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/TablePathUtils.java @@ -17,17 +17,21 @@ package org.apache.hudi.common.util; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.model.HoodiePartitionMetadata; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.exception.HoodieException; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import java.io.IOException; +/** + * Utils for table path. + */ public class TablePathUtils { private static final Logger LOG = LogManager.getLogger(TablePathUtils.class); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/ThreadUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ThreadUtils.java index aef791aa87cac..a61040c5ef51b 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/ThreadUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ThreadUtils.java @@ -21,6 +21,9 @@ import java.util.Arrays; import java.util.List; +/** + * Utils for Java threading. + */ public class ThreadUtils { /** diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/TypeUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/TypeUtils.java index 87ce471baa4d7..c4695c9c11f4a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/TypeUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/TypeUtils.java @@ -18,11 +18,15 @@ package org.apache.hudi.common.util; import javax.annotation.Nonnull; + import java.util.Arrays; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; +/** + * Utils for Java type cast. + */ public final class TypeUtils { private TypeUtils() {} diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/ValidationUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ValidationUtils.java index 0f7c43e497f19..ad8c53c821372 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/ValidationUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ValidationUtils.java @@ -18,7 +18,7 @@ package org.apache.hudi.common.util; -/* +/** * Simple utility to test validation conditions (to replace Guava's PreConditions) */ public class ValidationUtils { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/hash/HashID.java b/hudi-common/src/main/java/org/apache/hudi/common/util/hash/HashID.java index ccb29dfbb580d..eeaeb4df5bfe7 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/hash/HashID.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/hash/HashID.java @@ -19,11 +19,12 @@ package org.apache.hudi.common.util.hash; +import org.apache.hudi.exception.HoodieIOException; + import net.jpountz.xxhash.XXHash32; import net.jpountz.xxhash.XXHash64; import net.jpountz.xxhash.XXHashFactory; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hudi.exception.HoodieIOException; import java.io.Serializable; import java.nio.charset.StandardCharsets; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/io/ByteBufferBackedInputStream.java b/hudi-common/src/main/java/org/apache/hudi/common/util/io/ByteBufferBackedInputStream.java index 0f96d1011a3f0..b8a2ea2f213ff 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/io/ByteBufferBackedInputStream.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/io/ByteBufferBackedInputStream.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.util.io; import javax.annotation.Nonnull; + import java.io.InputStream; import java.nio.ByteBuffer; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification32bit.java b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification32bit.java index e76c67e39276b..9e75a1e06b1c6 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification32bit.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification32bit.java @@ -18,6 +18,10 @@ package org.apache.hudi.common.util.jvm; +/** + * Implementation of {@link MemoryLayoutSpecification} based on + * Hot Spot Memory Layout Specification on 32-bit. + */ public class HotSpotMemoryLayoutSpecification32bit implements MemoryLayoutSpecification { @Override public int getArrayHeaderSize() { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bit.java b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bit.java index 4f4e2a4b6cabc..c75472fc3160b 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bit.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bit.java @@ -18,6 +18,10 @@ package org.apache.hudi.common.util.jvm; +/** + * Implementation of {@link MemoryLayoutSpecification} based on + * Hot Spot Memory Layout Specification on 64-bit. + */ public class HotSpotMemoryLayoutSpecification64bit implements MemoryLayoutSpecification { @Override public int getArrayHeaderSize() { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bitCompressed.java b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bitCompressed.java index 60ad8be288991..62229a87e17bb 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bitCompressed.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bitCompressed.java @@ -18,6 +18,10 @@ package org.apache.hudi.common.util.jvm; +/** + * Implementation of {@link MemoryLayoutSpecification} based on + * Hot Spot Memory Layout Specification on 64-bit compressed. + */ public class HotSpotMemoryLayoutSpecification64bitCompressed implements MemoryLayoutSpecification { @Override public int getArrayHeaderSize() { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification32bit.java b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification32bit.java index 1e66e0cecf122..aa1486c9f089d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification32bit.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification32bit.java @@ -18,6 +18,10 @@ package org.apache.hudi.common.util.jvm; +/** + * Implementation of {@link MemoryLayoutSpecification} based on + * OpenJ9 Memory Layout Specification on 32-bit. + */ public class OpenJ9MemoryLayoutSpecification32bit implements MemoryLayoutSpecification { @Override public int getArrayHeaderSize() { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bit.java b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bit.java index d89d05ae475b2..4565b380e67b9 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bit.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bit.java @@ -18,6 +18,10 @@ package org.apache.hudi.common.util.jvm; +/** + * Implementation of {@link MemoryLayoutSpecification} based on + * OpenJ9 Memory Layout Specification on 64-bit. + */ public class OpenJ9MemoryLayoutSpecification64bit implements MemoryLayoutSpecification { @Override public int getArrayHeaderSize() { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bitCompressed.java b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bitCompressed.java index 05a457525a361..d1680d0775b32 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bitCompressed.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bitCompressed.java @@ -18,6 +18,10 @@ package org.apache.hudi.common.util.jvm; +/** + * Implementation of {@link MemoryLayoutSpecification} based on + * OpenJ9 Memory Layout Specification on 64-bit compressed. + */ public class OpenJ9MemoryLayoutSpecification64bitCompressed implements MemoryLayoutSpecification { @Override public int getArrayHeaderSize() { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryExecutor.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryExecutor.java index ce5898c7c3101..85b5bf98be388 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryExecutor.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryExecutor.java @@ -23,8 +23,8 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.SizeEstimator; import org.apache.hudi.exception.HoodieException; - import org.apache.hudi.exception.HoodieIOException; + import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorExecutor.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorExecutor.java index 7ea5de07c0dca..5bf92bdd37ca1 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorExecutor.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorExecutor.java @@ -20,8 +20,8 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieException; - import org.apache.hudi.exception.HoodieIOException; + import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorMessageQueue.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorMessageQueue.java index eccd881af1431..de0833f29d1a8 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorMessageQueue.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorMessageQueue.java @@ -18,6 +18,9 @@ package org.apache.hudi.common.util.queue; +import org.apache.hudi.common.util.CustomizedThreadFactory; +import org.apache.hudi.common.util.Option; + import com.lmax.disruptor.EventFactory; import com.lmax.disruptor.EventHandler; import com.lmax.disruptor.EventTranslator; @@ -25,8 +28,6 @@ import com.lmax.disruptor.WaitStrategy; import com.lmax.disruptor.dsl.Disruptor; import com.lmax.disruptor.dsl.ProducerType; -import org.apache.hudi.common.util.CustomizedThreadFactory; -import org.apache.hudi.common.util.Option; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -34,6 +35,12 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; +/** + * Implementation of {@link HoodieMessageQueue} based on Disruptor Queue. + * + * @param Input type. + * @param Transformed output type. + */ public class DisruptorMessageQueue implements HoodieMessageQueue { private static final Logger LOG = LogManager.getLogger(DisruptorMessageQueue.class); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorWaitStrategyType.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorWaitStrategyType.java index 1a8e86835d0cf..bdc2011b6e5bd 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorWaitStrategyType.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorWaitStrategyType.java @@ -24,6 +24,9 @@ import java.util.Arrays; import java.util.List; +/** + * Enum for the type of waiting strategy in Disruptor Queue. + */ public enum DisruptorWaitStrategyType { /** diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/HoodieExecutorBase.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/HoodieExecutorBase.java index 8dc07e35e4901..4930100fde497 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/HoodieExecutorBase.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/HoodieExecutorBase.java @@ -22,6 +22,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.exception.HoodieException; + import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/HoodieMessageQueue.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/HoodieMessageQueue.java index ae226f8adb0a3..33098b7d02155 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/HoodieMessageQueue.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/HoodieMessageQueue.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.util.queue; import org.apache.hudi.common.util.Option; + import java.io.Closeable; /** diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/WaitStrategyFactory.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/WaitStrategyFactory.java index 8137d2a1360e9..dc3c4474810b9 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/WaitStrategyFactory.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/WaitStrategyFactory.java @@ -18,16 +18,20 @@ package org.apache.hudi.common.util.queue; -import static org.apache.hudi.common.util.queue.DisruptorWaitStrategyType.BLOCKING_WAIT; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieException; import com.lmax.disruptor.BlockingWaitStrategy; import com.lmax.disruptor.BusySpinWaitStrategy; import com.lmax.disruptor.SleepingWaitStrategy; import com.lmax.disruptor.WaitStrategy; import com.lmax.disruptor.YieldingWaitStrategy; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.exception.HoodieException; +import static org.apache.hudi.common.util.queue.DisruptorWaitStrategyType.BLOCKING_WAIT; + +/** + * Factory for creating wait strategy for a queue. + */ public class WaitStrategyFactory { public static final String DEFAULT_STRATEGY = BLOCKING_WAIT.name(); diff --git a/hudi-common/src/main/java/org/apache/hudi/exception/HoodieCompactException.java b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieCompactException.java index 0d51706bbec5b..5c1e30775fc09 100644 --- a/hudi-common/src/main/java/org/apache/hudi/exception/HoodieCompactException.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieCompactException.java @@ -18,6 +18,9 @@ package org.apache.hudi.exception; +/** + * Exception for Hudi compaction. + */ public class HoodieCompactException extends HoodieException { public HoodieCompactException(String msg) { diff --git a/hudi-common/src/main/java/org/apache/hudi/exception/HoodieDebeziumAvroPayloadException.java b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieDebeziumAvroPayloadException.java index a1c83477dc3b9..0123597a458bf 100644 --- a/hudi-common/src/main/java/org/apache/hudi/exception/HoodieDebeziumAvroPayloadException.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieDebeziumAvroPayloadException.java @@ -21,6 +21,9 @@ import java.io.IOException; +/** + * Exception for HoodieDebeziumAvroPayload. + */ public class HoodieDebeziumAvroPayloadException extends IOException { public HoodieDebeziumAvroPayloadException(String msg) { diff --git a/hudi-common/src/main/java/org/apache/hudi/exception/HoodieSecondaryIndexException.java b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieSecondaryIndexException.java index 361416c9090a5..d17d51f2e7c3f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/exception/HoodieSecondaryIndexException.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieSecondaryIndexException.java @@ -19,6 +19,9 @@ package org.apache.hudi.exception; +/** + * Exception for Hudi secondary index. + */ public class HoodieSecondaryIndexException extends HoodieException { public HoodieSecondaryIndexException(String message) { super(message); diff --git a/hudi-common/src/main/java/org/apache/hudi/hadoop/CachingPath.java b/hudi-common/src/main/java/org/apache/hudi/hadoop/CachingPath.java index 2a71bcd77c0ad..698eabcd7967b 100644 --- a/hudi-common/src/main/java/org/apache/hudi/hadoop/CachingPath.java +++ b/hudi-common/src/main/java/org/apache/hudi/hadoop/CachingPath.java @@ -18,10 +18,12 @@ package org.apache.hudi.hadoop; -import org.apache.hadoop.fs.Path; import org.apache.hudi.exception.HoodieException; +import org.apache.hadoop.fs.Path; + import javax.annotation.concurrent.ThreadSafe; + import java.net.URI; import java.net.URISyntaxException; diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/Type.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/Type.java index 5fec01a822477..d51ab37719aff 100644 --- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/Type.java +++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/Type.java @@ -29,10 +29,14 @@ * to do add support for localTime if avro version is updated */ public interface Type extends Serializable { + /** + * Enums for type names. + */ enum TypeID { RECORD, ARRAY, MAP, FIXED, STRING, BINARY, INT, LONG, FLOAT, DOUBLE, DATE, BOOLEAN, TIME, TIMESTAMP, DECIMAL, UUID; private String name; + TypeID() { this.name = this.name().toLowerCase(Locale.ROOT); } @@ -56,6 +60,9 @@ default boolean isNestedType() { return false; } + /** + * The type of a schema for primitive fields. + */ abstract class PrimitiveType implements Type { @Override public boolean isNestedType() { @@ -84,6 +91,9 @@ public int hashCode() { } } + /** + * The type of a schema for nested fields. + */ abstract class NestedType implements Type { @Override diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/Types.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/Types.java index fff10a700f618..b0f4701af7384 100644 --- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/Types.java +++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/Types.java @@ -18,8 +18,8 @@ package org.apache.hudi.internal.schema; -import org.apache.hudi.internal.schema.Type.PrimitiveType; import org.apache.hudi.internal.schema.Type.NestedType; +import org.apache.hudi.internal.schema.Type.PrimitiveType; import java.io.Serializable; import java.util.Arrays; @@ -30,10 +30,16 @@ import java.util.Objects; import java.util.stream.Collectors; +/** + * Types supported in schema evolution. + */ public class Types { private Types() { } + /** + * Boolean primitive type. + */ public static class BooleanType extends PrimitiveType { private static final BooleanType INSTANCE = new BooleanType(); @@ -52,6 +58,9 @@ public String toString() { } } + /** + * Integer primitive type. + */ public static class IntType extends PrimitiveType { private static final IntType INSTANCE = new IntType(); @@ -70,6 +79,9 @@ public String toString() { } } + /** + * Long primitive type. + */ public static class LongType extends PrimitiveType { private static final LongType INSTANCE = new LongType(); @@ -88,6 +100,9 @@ public String toString() { } } + /** + * Float primitive type. + */ public static class FloatType extends PrimitiveType { private static final FloatType INSTANCE = new FloatType(); @@ -106,6 +121,9 @@ public String toString() { } } + /** + * Double primitive type. + */ public static class DoubleType extends PrimitiveType { private static final DoubleType INSTANCE = new DoubleType(); @@ -124,6 +142,9 @@ public String toString() { } } + /** + * Date primitive type. + */ public static class DateType extends PrimitiveType { private static final DateType INSTANCE = new DateType(); @@ -142,6 +163,9 @@ public String toString() { } } + /** + * Time primitive type. + */ public static class TimeType extends PrimitiveType { private static final TimeType INSTANCE = new TimeType(); @@ -163,6 +187,9 @@ public String toString() { } } + /** + * Time primitive type. + */ public static class TimestampType extends PrimitiveType { private static final TimestampType INSTANCE = new TimestampType(); @@ -184,6 +211,9 @@ public String toString() { } } + /** + * String primitive type. + */ public static class StringType extends PrimitiveType { private static final StringType INSTANCE = new StringType(); @@ -202,6 +232,9 @@ public String toString() { } } + /** + * Binary primitive type. + */ public static class BinaryType extends PrimitiveType { private static final BinaryType INSTANCE = new BinaryType(); @@ -220,6 +253,9 @@ public String toString() { } } + /** + * Fixed primitive type. + */ public static class FixedType extends PrimitiveType { public static FixedType getFixed(int size) { return new FixedType(size); @@ -263,6 +299,9 @@ public int hashCode() { } } + /** + * Decimal primitive type. + */ public static class DecimalType extends PrimitiveType { public static DecimalType get(int precision, int scale) { return new DecimalType(precision, scale); @@ -345,6 +384,9 @@ public int hashCode() { } } + /** + * UUID primitive type. + */ public static class UUIDType extends PrimitiveType { private static final UUIDType INSTANCE = new UUIDType(); @@ -456,6 +498,9 @@ public int hashCode() { } } + /** + * Record nested type. + */ public static class RecordType extends NestedType { public static RecordType get(List fields) { @@ -541,9 +586,12 @@ public int hashCode() { } } + /** + * Array nested type. + */ public static class ArrayType extends NestedType { public static ArrayType get(int elementId, boolean isOptional, Type elementType) { - return new ArrayType(Field.get(elementId, isOptional,"element", elementType)); + return new ArrayType(Field.get(elementId, isOptional, "element", elementType)); } private final Field elementField; @@ -612,6 +660,9 @@ public int hashCode() { } } + /** + * Map nested type. + */ public static class MapType extends NestedType { public static MapType get(int keyId, int valueId, Type keyType, Type valueType) { diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/action/TableChange.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/action/TableChange.java index 7594f94732a90..2fe7a52e616ec 100644 --- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/action/TableChange.java +++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/action/TableChange.java @@ -37,7 +37,9 @@ * to do support partition changes */ public interface TableChange { - /* The action Type of schema change. */ + /** + * The action Type of schema change. + */ enum ColumnChangeID { ADD, UPDATE, DELETE, PROPERTY_CHANGE, REPLACE; private String name; @@ -74,6 +76,9 @@ default boolean withPositionChange() { return false; } + /** + * Information of base column changes + */ abstract class BaseColumnChange implements TableChange { protected final InternalSchema internalSchema; protected final Map id2parent; diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/action/TableChanges.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/action/TableChanges.java index 4e0adc27895af..3142f67e8fb29 100644 --- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/action/TableChanges.java +++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/action/TableChanges.java @@ -35,6 +35,9 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +/** + * Operations of schema changes supported in schema evolution + */ public class TableChanges { /** Deal with update columns changes for table. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/convert/AvroInternalSchemaConverter.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/convert/AvroInternalSchemaConverter.java index d941b27328aba..9ae0b4cbbf334 100644 --- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/convert/AvroInternalSchemaConverter.java +++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/convert/AvroInternalSchemaConverter.java @@ -18,15 +18,16 @@ package org.apache.hudi.internal.schema.convert; -import org.apache.avro.JsonProperties; -import org.apache.avro.LogicalType; -import org.apache.avro.LogicalTypes; -import org.apache.avro.Schema; import org.apache.hudi.internal.schema.HoodieSchemaException; import org.apache.hudi.internal.schema.InternalSchema; import org.apache.hudi.internal.schema.Type; import org.apache.hudi.internal.schema.Types; +import org.apache.avro.JsonProperties; +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; + import java.util.ArrayList; import java.util.Deque; import java.util.HashMap; diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java index 6cca0728a8312..81814a3e6fcfb 100644 --- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java +++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java @@ -18,10 +18,6 @@ package org.apache.hudi.internal.schema.io; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; @@ -34,6 +30,10 @@ import org.apache.hudi.internal.schema.utils.InternalSchemaUtils; import org.apache.hudi.internal.schema.utils.SerDeHelper; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -47,6 +47,9 @@ import static org.apache.hudi.common.table.timeline.HoodieTimeline.SCHEMA_COMMIT_ACTION; +/** + * {@link AbstractInternalSchemaStorageManager} implementation based on the schema files. + */ public class FileBasedInternalSchemaStorageManager extends AbstractInternalSchemaStorageManager { private static final Logger LOG = LogManager.getLogger(FileBasedInternalSchemaStorageManager.class); diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java index 413a3c4df1bc3..e2b33915853be 100644 --- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java @@ -18,12 +18,13 @@ package org.apache.hudi.internal.schema.utils; -import org.apache.avro.Schema; import org.apache.hudi.internal.schema.InternalSchema; import org.apache.hudi.internal.schema.Types; import org.apache.hudi.internal.schema.action.TableChanges; import org.apache.hudi.internal.schema.convert.AvroInternalSchemaConverter; +import org.apache.avro.Schema; + import java.util.ArrayList; import java.util.List; import java.util.TreeMap; diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/InternalSchemaUtils.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/InternalSchemaUtils.java index c799c236d0db0..45ec0649b2a72 100644 --- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/InternalSchemaUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/InternalSchemaUtils.java @@ -30,9 +30,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.TreeMap; -import java.util.SortedMap; import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; import java.util.stream.Collectors; /** diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/SerDeHelper.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/SerDeHelper.java index aebda533e324a..0456111e1418f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/SerDeHelper.java +++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/SerDeHelper.java @@ -18,11 +18,6 @@ package org.apache.hudi.internal.schema.utils; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; - import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; @@ -30,6 +25,11 @@ import org.apache.hudi.internal.schema.Type; import org.apache.hudi.internal.schema.Types; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + import java.io.IOException; import java.io.StringWriter; import java.util.ArrayList; @@ -41,6 +41,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +/** + * Utils of serialization and deserialization. + */ public class SerDeHelper { private SerDeHelper() { diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/visitor/NameToIDVisitor.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/visitor/NameToIDVisitor.java index 4960f434eeb23..60737c5b1b12d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/visitor/NameToIDVisitor.java +++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/visitor/NameToIDVisitor.java @@ -18,8 +18,6 @@ package org.apache.hudi.internal.schema.visitor; -import static org.apache.hudi.internal.schema.utils.InternalSchemaUtils.createFullName; - import org.apache.hudi.internal.schema.InternalSchema; import org.apache.hudi.internal.schema.Type; import org.apache.hudi.internal.schema.Types; @@ -30,6 +28,8 @@ import java.util.List; import java.util.Map; +import static org.apache.hudi.internal.schema.utils.InternalSchemaUtils.createFullName; + /** * Schema visitor to produce name -> id map for internalSchema. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileReader.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileReader.java index 6490425c42b75..36fafe3e9c8dd 100644 --- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileReader.java @@ -18,17 +18,23 @@ package org.apache.hudi.io.storage; -import org.apache.avro.Schema; -import org.apache.avro.generic.IndexedRecord; import org.apache.hudi.common.bloom.BloomFilter; import org.apache.hudi.common.util.ClosableIterator; import org.apache.hudi.common.util.Option; +import org.apache.avro.Schema; +import org.apache.avro.generic.IndexedRecord; + import java.io.IOException; import java.io.UnsupportedEncodingException; import java.util.List; import java.util.Set; +/** + * Hudi base file reader interface. + * + * @param Record implementation that permits field access by integer index. + */ public interface HoodieFileReader extends AutoCloseable { String[] readMinMaxRecordKeys(); diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileReaderFactory.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileReaderFactory.java index 1c7663b371e6e..3542956df3d28 100644 --- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileReaderFactory.java +++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileReaderFactory.java @@ -27,10 +27,13 @@ import java.io.IOException; +import static org.apache.hudi.common.model.HoodieFileFormat.HFILE; import static org.apache.hudi.common.model.HoodieFileFormat.ORC; import static org.apache.hudi.common.model.HoodieFileFormat.PARQUET; -import static org.apache.hudi.common.model.HoodieFileFormat.HFILE; +/** + * Factory methods to create Hudi file reader. + */ public class HoodieFileReaderFactory { public static HoodieFileReader getFileReader(Configuration conf, Path path) throws IOException { diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieHFileReader.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieHFileReader.java index 3e5b3ff6acba0..997d625aa2778 100644 --- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieHFileReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieHFileReader.java @@ -18,6 +18,17 @@ package org.apache.hudi.io.storage; +import org.apache.hudi.avro.HoodieAvroUtils; +import org.apache.hudi.common.bloom.BloomFilter; +import org.apache.hudi.common.bloom.BloomFilterFactory; +import org.apache.hudi.common.fs.FSUtils; +import org.apache.hudi.common.util.ClosableIterator; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.io.ByteBufferBackedInputStream; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.util.Lazy; + import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; @@ -33,16 +44,6 @@ import org.apache.hadoop.hbase.io.hfile.HFileInfo; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.nio.ByteBuff; -import org.apache.hudi.avro.HoodieAvroUtils; -import org.apache.hudi.common.bloom.BloomFilter; -import org.apache.hudi.common.bloom.BloomFilterFactory; -import org.apache.hudi.common.fs.FSUtils; -import org.apache.hudi.common.util.ClosableIterator; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.io.ByteBufferBackedInputStream; -import org.apache.hudi.exception.HoodieException; -import org.apache.hudi.exception.HoodieIOException; -import org.apache.hudi.util.Lazy; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieOrcReader.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieOrcReader.java index 5431bf3782af2..66a47eb53d434 100644 --- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieOrcReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieOrcReader.java @@ -18,10 +18,6 @@ package org.apache.hudi.io.storage; -import org.apache.avro.Schema; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.bloom.BloomFilter; import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.util.AvroOrcUtils; @@ -29,6 +25,11 @@ import org.apache.hudi.common.util.ClosableIterator; import org.apache.hudi.common.util.OrcReaderIterator; import org.apache.hudi.exception.HoodieIOException; + +import org.apache.avro.Schema; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.orc.OrcFile; import org.apache.orc.Reader; import org.apache.orc.Reader.Options; @@ -38,6 +39,11 @@ import java.io.IOException; import java.util.Set; +/** + * {@link HoodieFileReader} implementation for ORC format. + * + * @param Record implementation that permits field access by integer index. + */ public class HoodieOrcReader implements HoodieFileReader { private Path path; private Configuration conf; diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetReader.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetReader.java index 804e4354c749e..3df04a1dbba74 100644 --- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetReader.java @@ -18,15 +18,16 @@ package org.apache.hudi.io.storage; -import org.apache.avro.Schema; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.bloom.BloomFilter; import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.util.BaseFileUtils; import org.apache.hudi.common.util.ClosableIterator; import org.apache.hudi.common.util.ParquetReaderIterator; + +import org.apache.avro.Schema; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.parquet.avro.AvroParquetReader; import org.apache.parquet.avro.AvroReadSupport; import org.apache.parquet.hadoop.ParquetReader; @@ -36,8 +37,13 @@ import java.util.List; import java.util.Set; +/** + * {@link HoodieFileReader} implementation for parquet format. + * + * @param Record implementation that permits field access by integer index. + */ public class HoodieParquetReader implements HoodieFileReader { - + private final Path path; private final Configuration conf; private final BaseFileUtils parquetUtils; diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetStreamWriter.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetStreamWriter.java index c8f78c3501158..01460859e02fd 100644 --- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetStreamWriter.java +++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetStreamWriter.java @@ -18,12 +18,13 @@ package org.apache.hudi.io.storage; +import org.apache.hudi.avro.HoodieAvroWriteSupport; +import org.apache.hudi.parquet.io.OutputStreamBackedOutputFile; + import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; -import org.apache.hudi.avro.HoodieAvroWriteSupport; -import org.apache.hudi.parquet.io.OutputStreamBackedOutputFile; import org.apache.parquet.hadoop.ParquetFileWriter; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.api.WriteSupport; @@ -31,7 +32,13 @@ import java.io.IOException; -// TODO(HUDI-3035) unify w/ HoodieParquetWriter +/** + * Hudi log block writer for parquet format. + *

+ * TODO(HUDI-3035) unify w/ HoodieParquetWriter. + * + * @param Record implementation that permits field access by integer index. + */ public class HoodieParquetStreamWriter implements AutoCloseable { private final ParquetWriter writer; diff --git a/hudi-common/src/main/java/org/apache/hudi/keygen/BaseKeyGenerator.java b/hudi-common/src/main/java/org/apache/hudi/keygen/BaseKeyGenerator.java index a09101dedfbeb..d0baa90391960 100644 --- a/hudi-common/src/main/java/org/apache/hudi/keygen/BaseKeyGenerator.java +++ b/hudi-common/src/main/java/org/apache/hudi/keygen/BaseKeyGenerator.java @@ -17,14 +17,19 @@ package org.apache.hudi.keygen; -import org.apache.avro.generic.GenericRecord; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.exception.HoodieKeyException; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; +import org.apache.avro.generic.GenericRecord; + import java.util.List; +/** + * Base abstract class to extend for {@link KeyGenerator} with default logic of taking + * partitioning and timestamp configs. + */ public abstract class BaseKeyGenerator extends KeyGenerator { protected List recordKeyFields; diff --git a/hudi-common/src/main/java/org/apache/hudi/keygen/KeyGeneratorInterface.java b/hudi-common/src/main/java/org/apache/hudi/keygen/KeyGeneratorInterface.java index fc8fee241dbd7..32cf67e93c5da 100644 --- a/hudi-common/src/main/java/org/apache/hudi/keygen/KeyGeneratorInterface.java +++ b/hudi-common/src/main/java/org/apache/hudi/keygen/KeyGeneratorInterface.java @@ -18,9 +18,10 @@ package org.apache.hudi.keygen; -import org.apache.avro.generic.GenericRecord; import org.apache.hudi.common.model.HoodieKey; +import org.apache.avro.generic.GenericRecord; + import java.io.Serializable; import java.util.List; diff --git a/hudi-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java b/hudi-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java index a2be94a453e10..99d40439b7c61 100644 --- a/hudi-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java +++ b/hudi-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java @@ -23,6 +23,9 @@ import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; +/** + * Key generator configs. + */ @ConfigClassProperty(name = "Key Generator Options", groupName = ConfigGroups.Names.WRITE_CLIENT, description = "Hudi maintains keys (record key + partition path) " diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java index e8b0aa52b72d6..1034aa3125532 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java @@ -66,6 +66,9 @@ import java.util.function.Function; import java.util.stream.Collectors; +/** + * Abstract class for implementing common table metadata operations. + */ public abstract class BaseTableMetadata implements HoodieTableMetadata { private static final Logger LOG = LogManager.getLogger(BaseTableMetadata.class); diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java index 001f3d53b3118..2dcd6cf10d7c8 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java @@ -47,6 +47,9 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.stream.Collectors; +/** + * Implementation of {@link HoodieTableMetadata} based file-system-backed table metadata. + */ public class FileSystemBackedTableMetadata implements HoodieTableMetadata { private static final int DEFAULT_LISTING_PARALLELISM = 1500; diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMergedLogRecordReader.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMergedLogRecordReader.java index afcd726a0bc2c..eb7edcd2812d1 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMergedLogRecordReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMergedLogRecordReader.java @@ -18,10 +18,6 @@ package org.apache.hudi.metadata; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.fs.FileSystem; import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.model.HoodieAvroRecord; import org.apache.hudi.common.model.HoodieRecord; @@ -33,6 +29,11 @@ import org.apache.hudi.common.util.collection.ExternalSpillableMap; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.internal.schema.InternalSchema; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.fs.FileSystem; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java index fe8612c42e802..0bccf02be5342 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java @@ -36,6 +36,9 @@ import java.util.Map; import java.util.stream.Collectors; +/** + * Metrics for metadata. + */ public class HoodieMetadataMetrics implements Serializable { // Metric names diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java index 8205ca8e8f740..08128539bc9e8 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java @@ -18,16 +18,6 @@ package org.apache.hudi.metadata; -import org.apache.avro.Conversions; -import org.apache.avro.LogicalTypes; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.avro.util.Utf8; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.model.BooleanWrapper; import org.apache.hudi.avro.model.BytesWrapper; import org.apache.hudi.avro.model.DateWrapper; @@ -57,6 +47,17 @@ import org.apache.hudi.io.storage.HoodieHFileReader; import org.apache.hudi.util.Lazy; +import org.apache.avro.Conversions; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.avro.util.Utf8; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + import java.io.IOException; import java.math.BigDecimal; import java.nio.ByteBuffer; diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java index 286ef50eec8dd..711a08a48952a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java @@ -18,8 +18,6 @@ package org.apache.hudi.metadata; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.model.HoodieMetadataColumnStats; import org.apache.hudi.common.bloom.BloomFilter; import org.apache.hudi.common.config.HoodieMetadataConfig; @@ -32,6 +30,9 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieMetadataException; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; + import java.io.IOException; import java.io.Serializable; import java.util.Collection; diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java b/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java index ddb76ca2579ea..98cbcba311742 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java @@ -21,6 +21,9 @@ import java.util.Arrays; import java.util.List; +/** + * Partition types for metadata table. + */ public enum MetadataPartitionType { FILES(HoodieTableMetadataUtil.PARTITION_NAME_FILES, "files-"), COLUMN_STATS(HoodieTableMetadataUtil.PARTITION_NAME_COLUMN_STATS, "col-stats-"), diff --git a/hudi-common/src/main/java/org/apache/hudi/parquet/io/ByteBufferBackedInputFile.java b/hudi-common/src/main/java/org/apache/hudi/parquet/io/ByteBufferBackedInputFile.java index 40454d306ac78..8da622befea47 100644 --- a/hudi-common/src/main/java/org/apache/hudi/parquet/io/ByteBufferBackedInputFile.java +++ b/hudi-common/src/main/java/org/apache/hudi/parquet/io/ByteBufferBackedInputFile.java @@ -19,6 +19,7 @@ package org.apache.hudi.parquet.io; import org.apache.hudi.common.util.io.ByteBufferBackedInputStream; + import org.apache.parquet.io.DelegatingSeekableInputStream; import org.apache.parquet.io.InputFile; import org.apache.parquet.io.SeekableInputStream; diff --git a/hudi-common/src/main/java/org/apache/hudi/parquet/io/OutputStreamBackedOutputFile.java b/hudi-common/src/main/java/org/apache/hudi/parquet/io/OutputStreamBackedOutputFile.java index 48c2c82e7b422..a3e599a4abeb7 100644 --- a/hudi-common/src/main/java/org/apache/hudi/parquet/io/OutputStreamBackedOutputFile.java +++ b/hudi-common/src/main/java/org/apache/hudi/parquet/io/OutputStreamBackedOutputFile.java @@ -23,6 +23,7 @@ import org.apache.parquet.io.PositionOutputStream; import javax.annotation.Nonnull; + import java.io.IOException; /** diff --git a/hudi-common/src/main/java/org/apache/hudi/secondary/index/HoodieSecondaryIndex.java b/hudi-common/src/main/java/org/apache/hudi/secondary/index/HoodieSecondaryIndex.java index 8b50d9268eb82..4a93242474ba1 100644 --- a/hudi-common/src/main/java/org/apache/hudi/secondary/index/HoodieSecondaryIndex.java +++ b/hudi-common/src/main/java/org/apache/hudi/secondary/index/HoodieSecondaryIndex.java @@ -25,6 +25,9 @@ import java.util.LinkedHashMap; import java.util.Map; +/** + * Represents a single secondary index. + */ public class HoodieSecondaryIndex { private String indexName; private SecondaryIndexType indexType; @@ -91,6 +94,9 @@ public String toString() { + '}'; } + /** + * Builder for {@link HoodieSecondaryIndex}. + */ public static class Builder { private String indexName; private SecondaryIndexType indexType; @@ -122,6 +128,9 @@ public HoodieSecondaryIndex build() { } } + /** + * Comparator for {@link HoodieSecondaryIndex}. + */ public static class HoodieIndexCompactor implements Comparator { @Override public int compare(HoodieSecondaryIndex o1, HoodieSecondaryIndex o2) { diff --git a/hudi-common/src/main/java/org/apache/hudi/secondary/index/SecondaryIndexManager.java b/hudi-common/src/main/java/org/apache/hudi/secondary/index/SecondaryIndexManager.java index 61bd7d76219b7..fbb65bc321041 100644 --- a/hudi-common/src/main/java/org/apache/hudi/secondary/index/SecondaryIndexManager.java +++ b/hudi-common/src/main/java/org/apache/hudi/secondary/index/SecondaryIndexManager.java @@ -41,6 +41,9 @@ import static org.apache.hudi.secondary.index.SecondaryIndexUtils.getSecondaryIndexes; +/** + * Manages secondary index. + */ public class SecondaryIndexManager { private static final Logger LOG = LoggerFactory.getLogger(SecondaryIndexManager.class); diff --git a/hudi-common/src/main/java/org/apache/hudi/secondary/index/SecondaryIndexType.java b/hudi-common/src/main/java/org/apache/hudi/secondary/index/SecondaryIndexType.java index 108d2effce077..15378a3e95433 100644 --- a/hudi-common/src/main/java/org/apache/hudi/secondary/index/SecondaryIndexType.java +++ b/hudi-common/src/main/java/org/apache/hudi/secondary/index/SecondaryIndexType.java @@ -23,6 +23,9 @@ import java.util.Arrays; +/** + * Type of secondary index. + */ public enum SecondaryIndexType { LUCENE((byte) 1); diff --git a/hudi-common/src/main/java/org/apache/hudi/secondary/index/SecondaryIndexUtils.java b/hudi-common/src/main/java/org/apache/hudi/secondary/index/SecondaryIndexUtils.java index 40b2ee9124180..d134a457caf38 100644 --- a/hudi-common/src/main/java/org/apache/hudi/secondary/index/SecondaryIndexUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/secondary/index/SecondaryIndexUtils.java @@ -33,6 +33,9 @@ import java.util.List; +/** + * Utils for secondary index. + */ public class SecondaryIndexUtils { /** diff --git a/hudi-common/src/main/java/org/apache/hudi/util/Lazy.java b/hudi-common/src/main/java/org/apache/hudi/util/Lazy.java index 1a843430b7d9c..72e205c400f4d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/util/Lazy.java +++ b/hudi-common/src/main/java/org/apache/hudi/util/Lazy.java @@ -19,6 +19,7 @@ package org.apache.hudi.util; import javax.annotation.concurrent.ThreadSafe; + import java.util.function.Supplier; /** diff --git a/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java b/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java index 483c49b1f50bc..896843b58f28c 100644 --- a/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java @@ -28,9 +28,8 @@ import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; - -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.math.BigDecimal; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/TestRegistry.java b/hudi-common/src/test/java/org/apache/hudi/common/TestRegistry.java index dec708f0af132..fa79b71a3dc51 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/TestRegistry.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/TestRegistry.java @@ -19,6 +19,7 @@ package org.apache.hudi.common; import org.apache.hudi.common.metrics.Registry; + import org.junit.jupiter.api.Test; import java.util.HashMap; @@ -27,6 +28,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +/** + * Tests metrics {@link Registry}. + */ public class TestRegistry { @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/config/TestConfigProperty.java b/hudi-common/src/test/java/org/apache/hudi/common/config/TestConfigProperty.java index 6cbb9bd48e496..091209c56bc5e 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/config/TestConfigProperty.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/config/TestConfigProperty.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.config; import org.apache.hudi.common.util.Option; + import org.junit.jupiter.api.Test; import java.util.Properties; @@ -27,6 +28,9 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +/** + * Tests {@link ConfigProperty}. + */ public class TestConfigProperty extends HoodieConfig { public static ConfigProperty FAKE_STRING_CONFIG = ConfigProperty diff --git a/hudi-common/src/test/java/org/apache/hudi/common/data/TestHoodieListDataPairData.java b/hudi-common/src/test/java/org/apache/hudi/common/data/TestHoodieListDataPairData.java index bb65909230da0..9a20fe9bdb2b6 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/data/TestHoodieListDataPairData.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/data/TestHoodieListDataPairData.java @@ -22,6 +22,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.ImmutablePair; import org.apache.hudi.common.util.collection.Pair; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -42,6 +43,9 @@ import static org.apache.hudi.common.util.CollectionUtils.createImmutableMap; import static org.junit.jupiter.api.Assertions.assertEquals; +/** + * Tests {@link HoodieListPairData}. + */ public class TestHoodieListDataPairData { private static final String KEY1 = "key1"; @@ -80,7 +84,7 @@ public void testKeys() { @Test public void testValues() { assertHoodieDataEquals(Arrays.asList( - STRING_VALUE1, STRING_VALUE2, STRING_VALUE3, STRING_VALUE4, STRING_VALUE5, STRING_VALUE6), + STRING_VALUE1, STRING_VALUE2, STRING_VALUE3, STRING_VALUE4, STRING_VALUE5, STRING_VALUE6), testHoodiePairData.values()); } @@ -103,7 +107,7 @@ public void testCountByKey() { @Test public void testMap() { assertHoodieDataEquals(Arrays.asList( - "key1,value1", "key1,value2", "key2,value3", "key2,value4", "key3,value5", "key4,value6"), + "key1,value1", "key1,value2", "key2,value3", "key2,value4", "key3,value5", "key4,value6"), testHoodiePairData.map(pair -> pair.getKey() + "," + pair.getValue())); } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtilsWithRetryWrapperEnable.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtilsWithRetryWrapperEnable.java index 0b849ebec8185..aa976cf11274c 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtilsWithRetryWrapperEnable.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtilsWithRetryWrapperEnable.java @@ -28,7 +28,6 @@ import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Progressable; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/InLineFSUtilsTest.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/InLineFSUtilsTest.java index 7d704c91126d6..896ebe2f44978 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/InLineFSUtilsTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/InLineFSUtilsTest.java @@ -18,8 +18,9 @@ package org.apache.hudi.common.fs.inline; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.testutils.FileSystemTestUtils; + +import org.apache.hadoop.fs.Path; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestAWSDmsAvroPayload.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestAWSDmsAvroPayload.java index 07bc1d6f43e1f..2d0fd5b094e62 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestAWSDmsAvroPayload.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestAWSDmsAvroPayload.java @@ -18,11 +18,12 @@ package org.apache.hudi.common.model; +import org.apache.hudi.common.util.Option; + import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; -import org.apache.hudi.common.util.Option; import org.junit.jupiter.api.Test; import java.util.Properties; @@ -31,6 +32,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +/** + * Tests {@link AWSDmsAvroPayload}. + */ public class TestAWSDmsAvroPayload { private static final String AVRO_SCHEMA_STRING = "{\"type\": \"record\"," diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieConsistentHashingMetadata.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieConsistentHashingMetadata.java index 8aa2e65561c59..40a90c3551269 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieConsistentHashingMetadata.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieConsistentHashingMetadata.java @@ -21,6 +21,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +/** + * Tests {@link HoodieConsistentHashingMetadata}. + */ public class TestHoodieConsistentHashingMetadata { @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieFileGroup.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieFileGroup.java index 91a2019f10b7c..a7cdf22f8020f 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieFileGroup.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieFileGroup.java @@ -32,6 +32,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +/** + * Tests {@link HoodieFileGroup}. + */ public class TestHoodieFileGroup { @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodiePartitionMetadata.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodiePartitionMetadata.java index 84ac3351df813..af6e6f5a390c0 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodiePartitionMetadata.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodiePartitionMetadata.java @@ -39,6 +39,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +/** + * Tests {@link HoodiePartitionMetadata}. + */ public class TestHoodiePartitionMetadata extends HoodieCommonTestHarness { FileSystem fs; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieReplaceCommitMetadata.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieReplaceCommitMetadata.java index f2c0c1c043bf8..2a7486cbd8c3b 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieReplaceCommitMetadata.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieReplaceCommitMetadata.java @@ -29,6 +29,9 @@ import static org.apache.hudi.common.model.TestHoodieCommitMetadata.verifyMetadataFieldNames; +/** + * Tests {@link HoodieReplaceCommitMetadata}. + */ public class TestHoodieReplaceCommitMetadata { private static final List EXPECTED_FIELD_NAMES = Arrays.asList( diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieWriteStat.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieWriteStat.java index 631c7cd41a385..e8a7205f769e9 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieWriteStat.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieWriteStat.java @@ -20,6 +20,7 @@ import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; + import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.Test; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestOverwriteNonDefaultsWithLatestAvroPayload.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestOverwriteNonDefaultsWithLatestAvroPayload.java index 4b7e4bda0b36c..97b0ac5c108cd 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestOverwriteNonDefaultsWithLatestAvroPayload.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestOverwriteNonDefaultsWithLatestAvroPayload.java @@ -18,12 +18,13 @@ package org.apache.hudi.common.model; +import org.apache.hudi.avro.HoodieAvroUtils; + import org.apache.avro.JsonProperties; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; -import org.apache.hudi.avro.HoodieAvroUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestPartialUpdateAvroPayload.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestPartialUpdateAvroPayload.java index b64b289abfb6e..2d5bb511981ee 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestPartialUpdateAvroPayload.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestPartialUpdateAvroPayload.java @@ -18,12 +18,11 @@ package org.apache.hudi.common.model; +import org.apache.hudi.common.util.Option; + import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; - -import org.apache.hudi.common.util.Option; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -36,7 +35,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; - /** * Unit tests {@link TestPartialUpdateAvroPayload}. */ diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/debezium/TestMySqlDebeziumAvroPayload.java b/hudi-common/src/test/java/org/apache/hudi/common/model/debezium/TestMySqlDebeziumAvroPayload.java index f9e922622d6c1..55badd4b4a3d2 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/debezium/TestMySqlDebeziumAvroPayload.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/debezium/TestMySqlDebeziumAvroPayload.java @@ -38,6 +38,9 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +/** + * Tests {@link MySqlDebeziumAvroPayload}. + */ public class TestMySqlDebeziumAvroPayload { private static final String KEY_FIELD_NAME = "Key"; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/debezium/TestPostgresDebeziumAvroPayload.java b/hudi-common/src/test/java/org/apache/hudi/common/model/debezium/TestPostgresDebeziumAvroPayload.java index 78599afc1fe16..7cb8dea9fdbd4 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/debezium/TestPostgresDebeziumAvroPayload.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/debezium/TestPostgresDebeziumAvroPayload.java @@ -43,6 +43,9 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +/** + * Tests {@link PostgresDebeziumAvroPayload}. + */ public class TestPostgresDebeziumAvroPayload { private static final String KEY_FIELD_NAME = "Key"; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/properties/TestOrderedProperties.java b/hudi-common/src/test/java/org/apache/hudi/common/properties/TestOrderedProperties.java index c75df04a2a633..edbe3f4383824 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/properties/TestOrderedProperties.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/properties/TestOrderedProperties.java @@ -27,6 +27,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +/** + * Tests {@link OrderedProperties}. + */ public class TestOrderedProperties { @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/properties/TestTypedProperties.java b/hudi-common/src/test/java/org/apache/hudi/common/properties/TestTypedProperties.java index 954b53651af5c..e184da7eee5f6 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/properties/TestTypedProperties.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/properties/TestTypedProperties.java @@ -28,6 +28,9 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +/** + * Tests {@link TypedProperties}. + */ public class TestTypedProperties { @Test public void testGetString() { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java b/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java index 7c9345b0e6082..e17c3c4d792f6 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java @@ -42,6 +42,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +/** + * Tests {@link HoodieTableConfig}. + */ public class TestHoodieTableConfig extends HoodieCommonTestHarness { private FileSystem fs; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java b/hudi-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java index 5d949431e4937..3ac42b9d3b7c6 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java @@ -30,6 +30,9 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +/** + * Tests {@link TableSchemaResolver}. + */ public class TestTableSchemaResolver { @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java index 094c607fb9be7..e66a13001735c 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java @@ -37,6 +37,7 @@ import org.apache.hudi.common.testutils.HoodieCommonTestHarness; import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.common.util.Option; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -55,6 +56,9 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +/** + * Tests {@link TimelineUtils}. + */ public class TestTimelineUtils extends HoodieCommonTestHarness { @BeforeEach @@ -80,7 +84,7 @@ public void testGetPartitionsWithReplaceCommits() throws IOException { activeTimeline.createNewInstant(instant1); // create replace metadata only with replaced file Ids (no new files created) activeTimeline.saveAsComplete(instant1, - Option.of(getReplaceCommitMetadata(basePath, ts1, replacePartition, 2, + Option.of(getReplaceCommitMetadata(basePath, ts1, replacePartition, 2, newFilePartition, 0, Collections.emptyMap(), WriteOperationType.CLUSTER))); metaClient.reloadActiveTimeline(); @@ -222,7 +226,7 @@ public void testGetExtraMetadata() throws Exception { // verify modified partitions included cleaned data verifyExtraMetadataLatestValue(extraMetadataKey, extraMetadataValue1, false); assertFalse(TimelineUtils.getExtraMetadataFromLatest(metaClient, "unknownKey").isPresent()); - + // verify adding clustering commit doesn't change behavior of getExtraMetadataFromLatest String ts2 = "2"; HoodieInstant instant2 = new HoodieInstant(true, HoodieTimeline.REPLACE_COMMIT_ACTION, ts2); @@ -233,7 +237,7 @@ public void testGetExtraMetadata() throws Exception { Option.of(getReplaceCommitMetadata(basePath, ts2, "p2", 0, "p2", 3, extraMetadata, WriteOperationType.CLUSTER))); metaClient.reloadActiveTimeline(); - + verifyExtraMetadataLatestValue(extraMetadataKey, extraMetadataValue1, false); verifyExtraMetadataLatestValue(extraMetadataKey, newValueForMetadata, true); assertFalse(TimelineUtils.getExtraMetadataFromLatest(metaClient, "unknownKey").isPresent()); @@ -246,10 +250,10 @@ public void testGetExtraMetadata() throws Exception { assertTrue(extraMetadataEntries.get("2").isPresent()); assertEquals(newValueForMetadata, extraMetadataEntries.get("2").get()); } - + private void verifyExtraMetadataLatestValue(String extraMetadataKey, String expected, boolean includeClustering) { final Option extraLatestValue; - if (includeClustering) { + if (includeClustering) { extraLatestValue = TimelineUtils.getExtraMetadataFromLatestIncludeClustering(metaClient, extraMetadataKey); } else { extraLatestValue = TimelineUtils.getExtraMetadataFromLatest(metaClient, extraMetadataKey); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestTimelineLayout.java b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestTimelineLayout.java index 41cf4abf7baed..4a50238849959 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestTimelineLayout.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestTimelineLayout.java @@ -30,6 +30,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +/** + * Tests {@link TimelineLayout}. + */ public class TestTimelineLayout { @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFSViewWithClustering.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFSViewWithClustering.java index 58bb670922e16..8edcadc383cc5 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFSViewWithClustering.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFSViewWithClustering.java @@ -51,6 +51,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +/** + * Tests {@link TestHoodieTableFSViewWithClustering}. + */ public class TestHoodieTableFSViewWithClustering extends HoodieCommonTestHarness { private static final String TEST_WRITE_TOKEN = "1-0-1"; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java index 07cd694b3307b..e51c17613fdef 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java @@ -50,9 +50,9 @@ import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; import org.apache.hadoop.fs.Path; -import org.apache.hudi.exception.HoodieIOException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.junit.jupiter.api.AfterEach; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java index c428c248b878f..b297d320c7a6b 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java @@ -58,6 +58,9 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +/** + * Tests {@link PriorityBasedFileSystemView}. + */ @ExtendWith(MockitoExtension.class) public class TestPriorityBasedFileSystemView { @@ -537,7 +540,7 @@ public void testGetAllFileGroups() { public void testGetPendingCompactionOperations() { Stream> actual; Stream> expected = Collections.singleton( - (Pair) new ImmutablePair<>("test", new CompactionOperation())) + (Pair) new ImmutablePair<>("test", new CompactionOperation())) .stream(); when(primary.getPendingCompactionOperations()).thenReturn(expected); @@ -566,7 +569,7 @@ public void testGetPendingCompactionOperations() { public void testGetPendingLogCompactionOperations() { Stream> actual; Stream> expected = Collections.singleton( - (Pair) new ImmutablePair<>("test", new CompactionOperation())) + (Pair) new ImmutablePair<>("test", new CompactionOperation())) .stream(); when(primary.getPendingLogCompactionOperations()).thenReturn(expected); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/CheckedFunction.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/CheckedFunction.java index b0b3588c626a6..fab5c76836141 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/CheckedFunction.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/CheckedFunction.java @@ -19,6 +19,12 @@ package org.apache.hudi.common.testutils; +/** + * Checked function for testing. + * + * @param Passed-in value type. + * @param Return value type. + */ @FunctionalInterface public interface CheckedFunction { R apply(T t) throws Exception; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/ClusteringTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/ClusteringTestUtils.java index b142fe90b4cfa..0d263b300274f 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/ClusteringTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/ClusteringTestUtils.java @@ -33,6 +33,9 @@ import static org.apache.hudi.common.testutils.FileCreateUtils.createBaseFile; import static org.apache.hudi.common.testutils.HoodieTestUtils.DEFAULT_PARTITION_PATHS; +/** + * Utils for clustering in testing. + */ public class ClusteringTestUtils { public static HoodieClusteringPlan createClusteringPlan(HoodieTableMetaClient metaClient, String instantTime, String fileId) { @@ -42,10 +45,10 @@ public static HoodieClusteringPlan createClusteringPlan(HoodieTableMetaClient me createBaseFile(basePath, partition, instantTime, fileId, 1); FileSlice slice = new FileSlice(partition, instantTime, fileId); slice.setBaseFile(new CompactionTestUtils.DummyHoodieBaseFile(Paths.get(basePath, partition, - baseFileName(instantTime, fileId)).toString())); + baseFileName(instantTime, fileId)).toString())); List[] fileSliceGroups = new List[] {Collections.singletonList(slice)}; HoodieClusteringPlan clusteringPlan = ClusteringUtils.createClusteringPlan("strategy", new HashMap<>(), - fileSliceGroups, Collections.emptyMap()); + fileSliceGroups, Collections.emptyMap()); return clusteringPlan; } catch (Exception e) { throw new HoodieException(e.getMessage(), e); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java index 8be78a3a96927..9a2dba04681a5 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java @@ -68,6 +68,9 @@ import static org.apache.hudi.common.table.timeline.TimelineMetadataUtils.serializeRollbackMetadata; import static org.apache.hudi.common.table.timeline.TimelineMetadataUtils.serializeRollbackPlan; +/** + * Utils for creating dummy Hudi files in testing. + */ public class FileCreateUtils { private static final Logger LOG = LogManager.getLogger(FileCreateUtils.class); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileSystemTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileSystemTestUtils.java index 95188bb0b68d9..9178742b0ae7a 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileSystemTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileSystemTestUtils.java @@ -18,15 +18,15 @@ package org.apache.hudi.common.testutils; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; -import org.apache.hadoop.fs.RemoteIterator; import org.apache.hudi.common.fs.inline.InLineFSUtils; import org.apache.hudi.common.fs.inline.InLineFileSystem; import org.apache.hudi.common.fs.inline.InMemoryFileSystem; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import java.io.File; import java.io.IOException; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HadoopMapRedUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HadoopMapRedUtils.java index a06039b5fba35..0649ec3ffb745 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HadoopMapRedUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HadoopMapRedUtils.java @@ -18,13 +18,17 @@ package org.apache.hudi.common.testutils; +import org.apache.hudi.common.util.Option; + import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.Reporter; -import org.apache.hudi.common.util.Option; import java.util.concurrent.ConcurrentHashMap; +/** + * Utils for Hadoop MapRed in testing. + */ public class HadoopMapRedUtils { /** diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java index 8614060126dfa..e1aa0c7a01d74 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java @@ -882,6 +882,9 @@ public int getNumExistingKeys(String schemaStr) { return numKeysBySchema.getOrDefault(schemaStr, 0); } + /** + * Object containing the key and partition path for testing. + */ public static class KeyPartition implements Serializable { public HoodieKey key; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java index c2531d47c1c8b..dd6287c650203 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java @@ -119,6 +119,9 @@ import static org.apache.hudi.common.util.CommitUtils.getCommitActionType; import static org.apache.hudi.common.util.StringUtils.EMPTY_STRING; +/** + * Test Hoodie Table for testing only. + */ public class HoodieTestTable { public static final String PHONY_TABLE_SCHEMA = @@ -1079,7 +1082,7 @@ private static HoodieTestTableState getTestTableStateWithPartitionFileInfo(Write } public static List generateHoodieWriteStatForPartition(Map>> partitionToFileIdMap, - String commitTime, boolean bootstrap) { + String commitTime, boolean bootstrap) { List writeStats = new ArrayList<>(); for (Map.Entry>> entry : partitionToFileIdMap.entrySet()) { String partition = entry.getKey(); @@ -1124,6 +1127,9 @@ private static List generateHoodieWriteStatForPartitionLogFiles return writeStats; } + /** + * Exception for {@link HoodieTestTable}. + */ public static class HoodieTestTableException extends RuntimeException { public HoodieTestTableException(Throwable t) { super(t); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/NetworkTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/NetworkTestUtils.java index 1f99b0eaf440a..52545946abbb6 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/NetworkTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/NetworkTestUtils.java @@ -23,6 +23,9 @@ import java.io.IOException; import java.net.ServerSocket; +/** + * Utils for networking setup in testing. + */ public class NetworkTestUtils { public static int nextFreePort() { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestAvroOrcUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestAvroOrcUtils.java index 692aa1ed14e19..9f1f0333a364b 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestAvroOrcUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestAvroOrcUtils.java @@ -18,20 +18,23 @@ package org.apache.hudi.common.util; -import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.AVRO_SCHEMA; +import org.apache.hudi.common.testutils.HoodieCommonTestHarness; -import java.util.Arrays; -import java.util.List; import org.apache.avro.Schema; -import org.apache.hudi.common.testutils.HoodieCommonTestHarness; import org.apache.orc.TypeDescription; - import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import java.util.Arrays; +import java.util.List; + +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.AVRO_SCHEMA; import static org.junit.jupiter.api.Assertions.assertEquals; +/** + * Tests {@link AvroOrcUtils}. + */ public class TestAvroOrcUtils extends HoodieCommonTestHarness { public static List testCreateOrcSchemaArgs() { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestBase64CodecUtil.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestBase64CodecUtil.java index 9129a8c032825..8cee7a24541ba 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestBase64CodecUtil.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestBase64CodecUtil.java @@ -25,6 +25,9 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; +/** + * Tests {@link Base64CodecUtil}. + */ public class TestBase64CodecUtil { @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestBinaryUtil.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestBinaryUtil.java index 1efe5a06865d8..21123eea24aa4 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestBinaryUtil.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestBinaryUtil.java @@ -27,6 +27,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +/** + * Tests {@link BinaryUtil}. + */ public class TestBinaryUtil { @Test @@ -112,6 +115,7 @@ public void testFloatConvert() { private class ConvertResultWrapper { T index; byte[] result; + public ConvertResultWrapper(T index, byte[] result) { this.index = index; this.result = result; @@ -121,6 +125,7 @@ public ConvertResultWrapper(T index, byte[] result) { private class OrginValueWrapper { T index; T originValue; + public OrginValueWrapper(T index, T originValue) { this.index = index; this.originValue = originValue; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestClusteringUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestClusteringUtils.java index a5d45d1184f9b..0948cb615be79 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestClusteringUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestClusteringUtils.java @@ -30,6 +30,7 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.TimelineMetadataUtils; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCommitUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCommitUtils.java index c55e34acfa877..6d0b2738b3cb0 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCommitUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCommitUtils.java @@ -23,6 +23,7 @@ import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.timeline.HoodieTimeline; + import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -35,6 +36,9 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +/** + * Tests {@link CommitUtils}. + */ public class TestCommitUtils { @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCustomizedThreadFactory.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCustomizedThreadFactory.java index 797643a2c994f..36d2918548c49 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCustomizedThreadFactory.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCustomizedThreadFactory.java @@ -20,12 +20,16 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; + import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.locks.LockSupport; +/** + * Tests {@link CustomizedThreadFactory}. + */ public class TestCustomizedThreadFactory { @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java index 73d10d73d2f73..df3efbd91f9e0 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java @@ -18,16 +18,16 @@ package org.apache.hudi.common.util; -import org.apache.hadoop.conf.Configuration; import org.apache.hudi.common.config.DFSPropertiesConfiguration; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.testutils.minicluster.HdfsTestService; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hudi.exception.HoodieIOException; import org.junit.Rule; import org.junit.contrib.java.lang.system.EnvironmentVariables; import org.junit.jupiter.api.AfterAll; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestDateTimeUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestDateTimeUtils.java index 996c8ba6cc1ac..0b886ffcca19d 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestDateTimeUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestDateTimeUtils.java @@ -28,6 +28,9 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertThrows; +/** + * Tests {@link DateTimeUtils}. + */ public class TestDateTimeUtils { @ParameterizedTest diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestObjectSizeCalculator.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestObjectSizeCalculator.java index 625e30198404a..5a2a407bc4b07 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestObjectSizeCalculator.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestObjectSizeCalculator.java @@ -32,6 +32,9 @@ import static org.apache.hudi.common.util.ObjectSizeCalculator.getObjectSize; import static org.junit.jupiter.api.Assertions.assertEquals; +/** + * Tests {@link ObjectSizeCalculator}. + */ public class TestObjectSizeCalculator { @Test @@ -92,6 +95,9 @@ class PayloadClass implements Serializable { private HoodieRecord record; } + /** + * Test class for object size estimation. + */ class Person { private String name; @@ -100,6 +106,9 @@ public Person(String name) { } } + /** + * Test enum for object size estimation. + */ public enum DayOfWeek { MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestOrcReaderIterator.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestOrcReaderIterator.java index 0f36f1877dec4..5801f7074f331 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestOrcReaderIterator.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestOrcReaderIterator.java @@ -22,15 +22,15 @@ import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.orc.CompressionKind; import org.apache.orc.OrcFile; import org.apache.orc.Reader; import org.apache.orc.RecordReader; import org.apache.orc.TypeDescription; import org.apache.orc.Writer; -import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -42,6 +42,9 @@ import static org.apache.hudi.common.testutils.SchemaTestUtil.getSchemaFromResource; import static org.junit.jupiter.api.Assertions.assertEquals; +/** + * Tests {@link OrcReaderIterator}. + */ public class TestOrcReaderIterator { private final Path filePath = new Path(System.getProperty("java.io.tmpdir") + "/f1_1-0-1_000.orc"); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetReaderIterator.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetReaderIterator.java index 37fead4928b43..eab1908cbaeb8 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetReaderIterator.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetReaderIterator.java @@ -34,6 +34,9 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +/** + * Tests {@link ParquetReaderIterator}. + */ public class TestParquetReaderIterator { @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestRatelimiter.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestRatelimiter.java index c712543c7280d..3c27138bf2d51 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestRatelimiter.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestRatelimiter.java @@ -18,12 +18,16 @@ package org.apache.hudi.common.util; -import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; +import java.util.concurrent.TimeUnit; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +/** + * Tests {@link RateLimiter}. + */ public class TestRatelimiter { @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializableSchema.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializableSchema.java index 03421a3005f04..731e88353ba3e 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializableSchema.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializableSchema.java @@ -18,10 +18,11 @@ package org.apache.hudi.common.util; -import org.apache.avro.Schema; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.common.config.SerializableSchema; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; + +import org.apache.avro.Schema; import org.junit.jupiter.api.Test; import java.io.ByteArrayInputStream; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializationUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializationUtils.java index f2714aaf9a268..c2e992223f4e2 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializationUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializationUtils.java @@ -18,10 +18,11 @@ package org.apache.hudi.common.util; -import org.apache.avro.util.Utf8; import org.apache.hudi.common.model.DeleteRecord; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.table.log.block.HoodieDeleteBlock; + +import org.apache.avro.util.Utf8; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -92,6 +93,9 @@ private void verifyObject(T expectedValue) throws IOException { } } + /** + * A class for non-serializable. + */ private static class NonSerializableClass { private String id; private String name; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestStringUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestStringUtils.java index bbaca74434dca..d319d2a1c57b8 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestStringUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestStringUtils.java @@ -30,6 +30,9 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +/** + * Tests {@link StringUtils}. + */ public class TestStringUtils { private static final String[] STRINGS = {"This", "is", "a", "test"}; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java index eae1cdce8399b..e72ea4bdf9dae 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java @@ -39,6 +39,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +/** + * Tests {@link TablePathUtils}. + */ public final class TestTablePathUtils { private static final String BASE_FILE_EXTENSION = HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension(); @@ -74,10 +77,10 @@ private void setup(Option partitionMetafileFormat) throws IOEx assertTrue(new File(partitionPathURI2).mkdirs()); HoodiePartitionMetadata partitionMetadata1 = new HoodiePartitionMetadata(fs, Instant.now().toString(), tablePath, - partitionPath1, partitionMetafileFormat); + partitionPath1, partitionMetafileFormat); partitionMetadata1.trySave(1); HoodiePartitionMetadata partitionMetadata2 = new HoodiePartitionMetadata(fs, Instant.now().toString(), tablePath, - partitionPath2, partitionMetafileFormat); + partitionPath2, partitionMetafileFormat); partitionMetadata2.trySave(2); // Create files diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDBDAO.java b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDBDAO.java index 0c2cf3ffe1f1d..cde265fdbd7a0 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDBDAO.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDBDAO.java @@ -238,6 +238,9 @@ public void testWithSerializableKey() { assertFalse(new File(rocksDBBasePath).exists()); } + /** + * Payload key object. + */ public static class PayloadKey implements Serializable { private String key; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/hash/TestHashID.java b/hudi-common/src/test/java/org/apache/hudi/common/util/hash/TestHashID.java index de0424f42580a..3bf316cc4c18a 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/hash/TestHashID.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/hash/TestHashID.java @@ -24,6 +24,7 @@ import org.junit.jupiter.params.provider.EnumSource; import javax.xml.bind.DatatypeConverter; + import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; @@ -33,6 +34,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +/** + * Tests {@link HashID}. + */ public class TestHashID { /** diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/io/TestByteBufferBackedInputStream.java b/hudi-common/src/test/java/org/apache/hudi/common/util/io/TestByteBufferBackedInputStream.java index 87bd2eea2ebe5..56291648daa9e 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/io/TestByteBufferBackedInputStream.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/io/TestByteBufferBackedInputStream.java @@ -26,6 +26,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +/** + * Tests {@link ByteBufferBackedInputStream}. + */ public class TestByteBufferBackedInputStream { @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/internal/schema/TestSerDeHelper.java b/hudi-common/src/test/java/org/apache/hudi/internal/schema/TestSerDeHelper.java index 5a337004812ca..c3704556f7ae6 100644 --- a/hudi-common/src/test/java/org/apache/hudi/internal/schema/TestSerDeHelper.java +++ b/hudi-common/src/test/java/org/apache/hudi/internal/schema/TestSerDeHelper.java @@ -20,14 +20,18 @@ import org.apache.hudi.internal.schema.utils.InternalSchemaUtils; import org.apache.hudi.internal.schema.utils.SerDeHelper; -import org.junit.jupiter.api.Test; + import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.TreeMap; +/** + * Tests {@link SerDeHelper}. + */ public class TestSerDeHelper { @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/internal/schema/action/TestMergeSchema.java b/hudi-common/src/test/java/org/apache/hudi/internal/schema/action/TestMergeSchema.java index 3118e143870e3..7d257270170ff 100644 --- a/hudi-common/src/test/java/org/apache/hudi/internal/schema/action/TestMergeSchema.java +++ b/hudi-common/src/test/java/org/apache/hudi/internal/schema/action/TestMergeSchema.java @@ -20,13 +20,16 @@ import org.apache.hudi.internal.schema.InternalSchema; import org.apache.hudi.internal.schema.Types; - import org.apache.hudi.internal.schema.utils.SchemaChangeUtils; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.Arrays; +/** + * Tests {@link InternalSchemaMerger}. + */ public class TestMergeSchema { @Test @@ -54,7 +57,7 @@ public void testPrimitiveMerge() { updateChange.updateColumnType("col2", Types.LongType.get()) .updateColumnComment("col2", "alter col2 comments") .renameColumn("col2", "colx").addPositionChange("col2", - "col4", "after"); + "col4", "after"); InternalSchema updateSchema = SchemaChangeUtils.applyTableChanges2Schema(newDeleteSchema, updateChange); // add col1 again diff --git a/hudi-common/src/test/java/org/apache/hudi/internal/schema/action/TestTableChanges.java b/hudi-common/src/test/java/org/apache/hudi/internal/schema/action/TestTableChanges.java index f8f5a4dc0293a..7c8233d6daf2e 100644 --- a/hudi-common/src/test/java/org/apache/hudi/internal/schema/action/TestTableChanges.java +++ b/hudi-common/src/test/java/org/apache/hudi/internal/schema/action/TestTableChanges.java @@ -22,15 +22,18 @@ import org.apache.hudi.internal.schema.InternalSchema; import org.apache.hudi.internal.schema.Type; import org.apache.hudi.internal.schema.Types; - import org.apache.hudi.internal.schema.Types.StringType; import org.apache.hudi.internal.schema.action.TableChange.ColumnPositionChange.ColumnPositionType; import org.apache.hudi.internal.schema.utils.SchemaChangeUtils; -import org.junit.jupiter.api.Test; + import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.Arrays; +/** + * Tests schema changes with {@link TableChanges}. + */ public class TestTableChanges { @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/internal/schema/io/TestFileBasedInternalSchemaStorageManager.java b/hudi-common/src/test/java/org/apache/hudi/internal/schema/io/TestFileBasedInternalSchemaStorageManager.java index c9bf80633a6ea..b3edb9725bbdb 100644 --- a/hudi-common/src/test/java/org/apache/hudi/internal/schema/io/TestFileBasedInternalSchemaStorageManager.java +++ b/hudi-common/src/test/java/org/apache/hudi/internal/schema/io/TestFileBasedInternalSchemaStorageManager.java @@ -26,6 +26,7 @@ import org.apache.hudi.internal.schema.InternalSchema; import org.apache.hudi.internal.schema.Types; import org.apache.hudi.internal.schema.utils.SerDeHelper; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java b/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java index 6126c479c6154..515dd978fde90 100644 --- a/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java @@ -18,12 +18,6 @@ package org.apache.hudi.internal.schema.utils; -import org.apache.avro.JsonProperties; -import org.apache.avro.LogicalTypes; -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericRecord; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.internal.schema.InternalSchema; import org.apache.hudi.internal.schema.InternalSchemaBuilder; @@ -31,8 +25,15 @@ import org.apache.hudi.internal.schema.Types; import org.apache.hudi.internal.schema.action.TableChanges; import org.apache.hudi.internal.schema.convert.AvroInternalSchemaConverter; -import org.junit.jupiter.api.Test; + +import org.apache.avro.JsonProperties; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.math.BigDecimal; import java.nio.ByteBuffer; @@ -44,6 +45,9 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +/** + * Tests {@link AvroSchemaEvolutionUtils}. + */ public class TestAvroSchemaEvolutionUtils { String schemaStr = "{\"type\":\"record\",\"name\":\"newTableName\",\"fields\":[{\"name\":\"id\",\"type\":\"int\"},{\"name\":\"data\"," diff --git a/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestInternalSchemaUtils.java b/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestInternalSchemaUtils.java index 27482f4c5019d..844dc147b4c6f 100644 --- a/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestInternalSchemaUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestInternalSchemaUtils.java @@ -21,8 +21,9 @@ import org.apache.hudi.internal.schema.InternalSchema; import org.apache.hudi.internal.schema.InternalSchemaBuilder; import org.apache.hudi.internal.schema.Types; -import org.junit.jupiter.api.Test; + import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -33,6 +34,9 @@ import java.util.List; import java.util.Map; +/** + * Tests {@link InternalSchemaUtils}. + */ public class TestInternalSchemaUtils { @Test public void testPruneSchema() { diff --git a/hudi-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java b/hudi-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java index 6656ece519f29..db7608cea965e 100644 --- a/hudi-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java +++ b/hudi-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java @@ -18,12 +18,13 @@ package org.apache.hudi.metadata; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.config.SerializableConfiguration; import org.apache.hudi.common.engine.HoodieLocalEngineContext; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; import org.apache.hudi.common.testutils.HoodieTestTable; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -37,6 +38,9 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +/** + * Tests {@link FileSystemBackedTableMetadata}. + */ public class TestFileSystemBackedTableMetadata extends HoodieCommonTestHarness { private static final String DEFAULT_PARTITION = ""; diff --git a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java index 7b4d432b3f80e..12baa02d8649e 100644 --- a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java +++ b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java @@ -18,12 +18,13 @@ package org.apache.hudi.metadata; -import org.apache.avro.generic.IndexedRecord; import org.apache.hudi.common.model.HoodieColumnRangeMetadata; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; + +import org.apache.avro.generic.IndexedRecord; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -34,6 +35,9 @@ import static org.apache.hudi.common.util.CollectionUtils.createImmutableMap; import static org.junit.jupiter.api.Assertions.assertEquals; +/** + * Tests {@link HoodieMetadataPayload}. + */ public class TestHoodieMetadataPayload extends HoodieCommonTestHarness { @Test