Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
import java.util.Map;

/**
* Converts Json record to Avro Generic Record
* Converts Json record to Avro Generic Record.
*/
public class MercifulJsonConverter {

Expand All @@ -61,21 +61,21 @@ private static Map<Schema.Type, JsonToAvroFieldProcessor> getFieldTypeProcessors
}

/**
* Uses a default objectMapper to deserialize a json string
* Uses a default objectMapper to deserialize a json string.
*/
public MercifulJsonConverter() {
this(new ObjectMapper());
}

/**
* Allows a configured ObjectMapper to be passed for converting json records to avro record
* Allows a configured ObjectMapper to be passed for converting json records to avro record.
*/
public MercifulJsonConverter(ObjectMapper mapper) {
this.mapper = mapper;
}

/**
* Converts json to Avro generic record
* Converts json to Avro generic record.
*
* @param json Json record
* @param schema Schema
Expand Down Expand Up @@ -133,7 +133,7 @@ private static Object convertJsonToAvroField(Object value, String name, Schema s
}

/**
* Base Class for converting json to avro fields
* Base Class for converting json to avro fields.
*/
private abstract static class JsonToAvroFieldProcessor implements Serializable {

Expand Down Expand Up @@ -311,7 +311,7 @@ public Pair<Boolean, Object> convert(Object value, String name, Schema schema)
}

/**
* Exception Class for any schema conversion issue
* Exception Class for any schema conversion issue.
*/
public static class HoodieJsonToAvroConversionException extends HoodieException {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import java.util.List;

/**
* Collects stats about a single partition clean operation
* Collects stats about a single partition clean operation.
*/
public class HoodieCleanStat implements Serializable {

Expand Down Expand Up @@ -80,6 +80,9 @@ public static HoodieCleanStat.Builder newBuilder() {
return new Builder();
}

/**
* A builder used to build {@link HoodieCleanStat}.
*/
public static class Builder {

private HoodieCleaningPolicy policy;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@
import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;

/**
* Hoodie json payload.
*/
public class HoodieJsonPayload implements HoodieRecordPayload<HoodieJsonPayload> {

private byte[] jsonDataCompressed;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import java.util.stream.Collectors;

/**
* Collects stats about a single partition clean operation
* Collects stats about a single partition clean operation.
*/
public class HoodieRollbackStat implements Serializable {

Expand Down Expand Up @@ -66,6 +66,9 @@ public static HoodieRollbackStat.Builder newBuilder() {
return new Builder();
}

/**
* A builder used to build {@link HoodieRollbackStat}.
*/
public static class Builder {

private List<String> successDeleteFiles;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@
import java.io.ObjectOutputStream;
import java.io.Serializable;

/**
* A wrapped configuration which can be serialized.
*/
public class SerializableConfiguration implements Serializable {

private transient Configuration configuration;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@

package org.apache.hudi.common.model;

/**
* The supported action types.
*/
public enum ActionType {
commit, savepoint, compaction, clean, rollback
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,7 @@

/**
* Encapsulates all the needed information about a compaction and make a decision whether this compaction is effective
* or not
*
* or not.
*/
public class CompactionOperation implements Serializable {

Expand Down Expand Up @@ -118,7 +117,7 @@ public Option<HoodieDataFile> getBaseFile(String basePath, String partitionPath)
}

/**
* Convert Avro generated Compaction operation to POJO for Spark RDD operation
* Convert Avro generated Compaction operation to POJO for Spark RDD operation.
*
* @param operation Hoodie Compaction Operation
* @return
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,22 +27,22 @@

/**
* Within a file group, a slice is a combination of data file written at a commit time and list of log files, containing
* changes to the data file from that commit time
* changes to the data file from that commit time.
*/
public class FileSlice implements Serializable {

/**
* File Group Id of the Slice
* File Group Id of the Slice.
*/
private HoodieFileGroupId fileGroupId;

/**
* Point in the timeline, at which the slice was created
* Point in the timeline, at which the slice was created.
*/
private String baseInstantTime;

/**
* data file, with the compacted data, for this slice
* data file, with the compacted data, for this slice.
*/
private HoodieDataFile dataFile;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;

/**
* The hoodie archived log file.
*/
public class HoodieArchivedLogFile extends HoodieLogFile {

public static final String ARCHIVE_EXTENSION = ".archive";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@

package org.apache.hudi.common.model;

/**
* Hoodie cleaning policies.
*/
public enum HoodieCleaningPolicy {
KEEP_LATEST_FILE_VERSIONS, KEEP_LATEST_COMMITS
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@
import java.io.Serializable;
import java.util.Objects;

/**
* Hoodie data file.
*/
public class HoodieDataFile implements Serializable {

private transient FileStatus fileStatus;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@

package org.apache.hudi.common.model;

/**
* Hoodie file format.
*/
public enum HoodieFileFormat {
PARQUET(".parquet"), HOODIE_LOG(".log");

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
import java.util.stream.Stream;

/**
* A set of data/base files + set of log files, that make up an unit for all operations
* A set of data/base files + set of log files, that make up an unit for all operations.
*/
public class HoodieFileGroup implements Serializable {

Expand All @@ -39,7 +39,7 @@ public static Comparator<String> getReverseCommitTimeComparator() {
}

/**
* file group id
* file group id.
*/
private final HoodieFileGroupId fileGroupId;

Expand All @@ -49,12 +49,12 @@ public static Comparator<String> getReverseCommitTimeComparator() {
private final TreeMap<String, FileSlice> fileSlices;

/**
* Timeline, based on which all getter work
* Timeline, based on which all getter work.
*/
private final HoodieTimeline timeline;

/**
* The last completed instant, that acts as a high watermark for all getters
* The last completed instant, that acts as a high watermark for all getters.
*/
private final Option<HoodieInstant> lastInstant;

Expand All @@ -71,7 +71,7 @@ public HoodieFileGroup(HoodieFileGroupId fileGroupId, HoodieTimeline timeline) {

/**
* Potentially add a new file-slice by adding base-instant time A file-slice without any data-file and log-files can
* exist (if a compaction just got requested)
* exist (if a compaction just got requested).
*/
public void addNewFileSliceAtInstant(String baseInstantTime) {
if (!fileSlices.containsKey(baseInstantTime)) {
Expand All @@ -80,7 +80,7 @@ public void addNewFileSliceAtInstant(String baseInstantTime) {
}

/**
* Add a new datafile into the file group
* Add a new datafile into the file group.
*/
public void addDataFile(HoodieDataFile dataFile) {
if (!fileSlices.containsKey(dataFile.getCommitTime())) {
Expand All @@ -90,7 +90,7 @@ public void addDataFile(HoodieDataFile dataFile) {
}

/**
* Add a new log file into the group
* Add a new log file into the group.
*/
public void addLogFile(HoodieLogFile logFile) {
if (!fileSlices.containsKey(logFile.getBaseCommitTime())) {
Expand All @@ -109,7 +109,7 @@ public HoodieFileGroupId getFileGroupId() {

/**
* A FileSlice is considered committed, if one of the following is true - There is a committed data file - There are
* some log files, that are based off a commit or delta commit
* some log files, that are based off a commit or delta commit.
*/
private boolean isFileSliceCommitted(FileSlice slice) {
String maxCommitTime = lastInstant.get().getTimestamp();
Expand All @@ -119,14 +119,14 @@ private boolean isFileSliceCommitted(FileSlice slice) {
}

/**
* Get all the the file slices including in-flight ones as seen in underlying file-system
* Get all the the file slices including in-flight ones as seen in underlying file-system.
*/
public Stream<FileSlice> getAllFileSlicesIncludingInflight() {
return fileSlices.entrySet().stream().map(Map.Entry::getValue);
}

/**
* Get latest file slices including in-flight ones
* Get latest file slices including in-flight ones.
*/
public Option<FileSlice> getLatestFileSlicesIncludingInflight() {
return Option.fromJavaOptional(getAllFileSlicesIncludingInflight().findFirst());
Expand All @@ -143,7 +143,7 @@ public Stream<FileSlice> getAllFileSlices() {
}

/**
* Gets the latest slice - this can contain either
* Gets the latest slice - this can contain either.
* <p>
* - just the log files without data file - (or) data file with 0 or more log files
*/
Expand All @@ -153,22 +153,22 @@ public Option<FileSlice> getLatestFileSlice() {
}

/**
* Gets the latest data file
* Gets the latest data file.
*/
public Option<HoodieDataFile> getLatestDataFile() {
return Option.fromJavaOptional(getAllDataFiles().findFirst());
}

/**
* Obtain the latest file slice, upto a commitTime i.e <= maxCommitTime
* Obtain the latest file slice, upto a commitTime i.e <= maxCommitTime.
*/
public Option<FileSlice> getLatestFileSliceBeforeOrOn(String maxCommitTime) {
return Option.fromJavaOptional(getAllFileSlices().filter(slice -> HoodieTimeline
.compareTimestamps(slice.getBaseInstantTime(), maxCommitTime, HoodieTimeline.LESSER_OR_EQUAL)).findFirst());
}

/**
* Obtain the latest file slice, upto a commitTime i.e < maxInstantTime
* Obtain the latest file slice, upto a commitTime i.e < maxInstantTime.
*
* @param maxInstantTime Max Instant Time
* @return
Expand All @@ -185,7 +185,7 @@ public Option<FileSlice> getLatestFileSliceInRange(List<String> commitRange) {
}

/**
* Stream of committed data files, sorted reverse commit time
* Stream of committed data files, sorted reverse commit time.
*/
public Stream<HoodieDataFile> getAllDataFiles() {
return getAllFileSlices().filter(slice -> slice.getDataFile().isPresent()).map(slice -> slice.getDataFile().get());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import java.util.Objects;

/**
* Unique ID to identify a file-group in a data-set
* Unique ID to identify a file-group in a data-set.
*/
public class HoodieFileGroupId implements Serializable {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@
* HoodieKey consists of
* <p>
* - recordKey : a recordKey that acts as primary key for a record - partitionPath : path to the partition that contains
* the record
* the record.
* - partitionPath : the partition path of a record.
*/
public class HoodieKey implements Serializable {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ public static Comparator<HoodieLogFile> getReverseLogFileComparator() {
}

/**
* Comparator to order log-files
* Comparator to order log-files.
*/
public static class LogFileComparator implements Comparator<HoodieLogFile>, Serializable {

Expand Down
Loading