Skip to content
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 52 additions & 5 deletions hudi-client/src/main/java/org/apache/hudi/HoodieWriteClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -108,14 +108,17 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
private transient Timer.Context indexTimer = null;

/**
*
* @param jsc Java Spark Context

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if possible add a line about instantiation and what this constructor is used for compared to other constructors.

* @param clientConfig instance of HoodieWriteConfig
*/
public HoodieWriteClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig) throws Exception {
public HoodieWriteClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig) {
this(jsc, clientConfig, false);
}

/**
*
* @param jsc Java Spark Context
Comment thread
hddong marked this conversation as resolved.
* @param clientConfig instance of HoodieWriteConfig
* @param rollbackPending whether need to cleanup pending commits
*/
public HoodieWriteClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig, boolean rollbackPending) {
this(jsc, clientConfig, rollbackPending, HoodieIndex.createIndex(clientConfig, jsc));
Expand All @@ -135,6 +138,12 @@ public HoodieWriteClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig, b
this.cleanClient = new HoodieCleanClient<>(jsc, config, metrics, timelineService);
}

/**
* Register hudi classes with Kryo serialization
*
* @param conf instance of SparkConf
* @return SparkConf
*/
public static SparkConf registerClasses(SparkConf conf) {
conf.registerKryoClasses(new Class[]{HoodieWriteConfig.class, HoodieRecord.class, HoodieKey.class});
return conf;
Expand All @@ -158,6 +167,10 @@ public JavaRDD<HoodieRecord<T>> filterExists(JavaRDD<HoodieRecord<T>> hoodieReco

/**
* Upserts a bunch of new records into the Hoodie table, at the supplied commitTime.
*
Comment thread
hddong marked this conversation as resolved.
* @param records hoodieRecords to upsert

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor: RDD of hoodieRecords to upsert. or feel free to leave it as is.

* @param commitTime Commit Time handle
* @return JavaRDD[WriteStatus] - RDD of WriteStatus to inspect errors and counts
*/
public JavaRDD<WriteStatus> upsert(JavaRDD<HoodieRecord<T>> records, final String commitTime) {
HoodieTable<T> table = getTableAndInitCtx(OperationType.UPSERT);
Expand Down Expand Up @@ -509,13 +522,21 @@ record -> new Tuple2<>(new Tuple2<>(record.getKey(), Option.ofNullable(record.ge

/**
* Commit changes performed at the given commitTime marker.
*
* @param commitTime Commit Time handle

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

just instant time of the commit . . Commit Time Handle does not sound accurate to me.

* @param writeStatuses RDD of WriteStatus to inspect errors and counts
* @return true if commit successfully

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor. {@code true} if commit is successful. {@code false} otherwise.

*/
public boolean commit(String commitTime, JavaRDD<WriteStatus> writeStatuses) {
return commit(commitTime, writeStatuses, Option.empty());
}

/**
* Commit changes performed at the given commitTime marker.
* @param commitTime Commit Time handle
* @param writeStatuses RDD of WriteStatus to inspect errors and counts
Comment thread
hddong marked this conversation as resolved.
Outdated
* @param extraMetadata Extra Metadata to be stored
* @return true if commit successfully

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same comment as above

*/
public boolean commit(String commitTime, JavaRDD<WriteStatus> writeStatuses,
Option<Map<String, String>> extraMetadata) {
Expand Down Expand Up @@ -775,7 +796,10 @@ public boolean rollbackToSavepoint(String savepointTime) {
/**
* Rollback the (inflight/committed) record changes with the given commit time. Three steps: (1) Atomically unpublish
* this commit (2) clean indexing data, (3) clean new generated parquet files. (4) Finally delete .commit or .inflight
* file,
* file.
*
* @param commitTime Commit Time handle
* @return true if commit was rollback to successfully

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same comment as above

*/
public boolean rollback(final String commitTime) throws HoodieRollbackException {
rollbackInternal(commitTime);
Expand All @@ -784,9 +808,11 @@ public boolean rollback(final String commitTime) throws HoodieRollbackException

/**
* NOTE : This action requires all writers (ingest and compact) to a dataset to be stopped before proceeding. Revert
* the (inflight/committed) record changes for all commits after the provided @param. Three steps: (1) Atomically
* the (inflight/committed) record changes for all commits after the provided @param. Four steps: (1) Atomically
* unpublish this commit (2) clean indexing data, (3) clean new generated parquet/log files and/or append rollback to
* existing log files. (4) Finally delete .commit, .inflight, .compaction.inflight or .compaction.requested file
*
* @param instantTime Instant time

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor: instant time to which restoration is requested

*/
public void restoreToInstant(final String instantTime) throws HoodieRollbackException {

Expand Down Expand Up @@ -1017,6 +1043,11 @@ public String startCommit() {
return commitTime;
}

/**
* Provides a new commit time for a write operation (insert/update).

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor: include delete as well.

*
* @param instantTime Instant time to be generated
*/
public void startCommitWithTime(String instantTime) {
// NOTE : Need to ensure that rollback is done before a new commit is started
if (rollbackPending) {
Expand Down Expand Up @@ -1044,6 +1075,8 @@ private void startCommit(String instantTime) {

/**
* Schedules a new compaction instant.
*
* @param extraMetadata Extra Metadata to be stored
*/
public Option<String> scheduleCompaction(Option<Map<String, String>> extraMetadata) throws IOException {
String instantTime = HoodieActiveTimeline.createNewInstantTime();
Expand Down Expand Up @@ -1093,13 +1126,18 @@ public boolean scheduleCompactionAtInstant(String instantTime, Option<Map<String
* Performs Compaction for the workload stored in instant-time.
*
* @param compactionInstantTime Compaction Instant Time
* @return RDD of WriteStatus
*/
public JavaRDD<WriteStatus> compact(String compactionInstantTime) throws IOException {
return compact(compactionInstantTime, config.shouldAutoCommit());
}

/**
* Commit a compaction operation. Allow passing additional meta-data to be stored in commit instant file.
*
* @param compactionInstantTime Compaction Instant Time
* @param writeStatuses RDD of WriteStatus to inspect errors and counts
* @param extraMetadata Extra Metadata to be stored
*/
public void commitCompaction(String compactionInstantTime, JavaRDD<WriteStatus> writeStatuses,
Option<Map<String, String>> extraMetadata) throws IOException {
Expand All @@ -1124,6 +1162,10 @@ public void commitCompaction(String compactionInstantTime, JavaRDD<WriteStatus>

/**
* Deduplicate Hoodie records, using the given deduplication funciton.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

typo funciton -> function

*
* @param records hoodieRecords to deduplicate
* @param parallelism partitions of RDD
Comment thread
hddong marked this conversation as resolved.
Outdated
* @return RDD of HoodieRecord already be deduplicated
*/
JavaRDD<HoodieRecord<T>> deduplicateRecords(JavaRDD<HoodieRecord<T>> records, int parallelism) {
boolean isIndexingGlobal = index.isGlobal();
Expand All @@ -1144,6 +1186,10 @@ JavaRDD<HoodieRecord<T>> deduplicateRecords(JavaRDD<HoodieRecord<T>> records, in

/**
* Deduplicate Hoodie records, using the given deduplication funciton.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto

*
* @param keys RDD of HoodieKey to deduplicate
* @param parallelism partitions of RDD
Comment thread
hddong marked this conversation as resolved.
Outdated
* @return RDD of HoodieKey already be deduplicated
*/
JavaRDD<HoodieKey> deduplicateKeys(JavaRDD<HoodieKey> keys, int parallelism) {
boolean isIndexingGlobal = index.isGlobal();
Expand Down Expand Up @@ -1216,6 +1262,7 @@ private void setWriteSchemaFromLastInstant(HoodieTableMetaClient metaClient) {
* Ensures compaction instant is in expected state and performs Compaction for the workload stored in instant-time.
*
* @param compactionInstantTime Compaction Instant Time
* @return RDD of Write Status
*/
private JavaRDD<WriteStatus> compact(String compactionInstantTime, boolean autoCommit) throws IOException {
// Create a Hoodie table which encapsulated the commits and files visible
Expand Down