-
Notifications
You must be signed in to change notification settings - Fork 2.5k
[HUDI-331]Fix java docs for all public apis in HoodieWriteClient #1111
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
e450df2
7179fd9
ba996e1
0191290
cc98dbf
73eeafe
e01a7a0
33d33c4
1f11fa0
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -108,14 +108,17 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo | |
| private transient Timer.Context indexTimer = null; | ||
|
|
||
| /** | ||
| * | ||
| * @param jsc Java Spark Context | ||
| * @param clientConfig instance of HoodieWriteConfig | ||
| */ | ||
| public HoodieWriteClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig) throws Exception { | ||
| public HoodieWriteClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig) { | ||
| this(jsc, clientConfig, false); | ||
| } | ||
|
|
||
| /** | ||
| * | ||
| * @param jsc Java Spark Context | ||
|
hddong marked this conversation as resolved.
|
||
| * @param clientConfig instance of HoodieWriteConfig | ||
| * @param rollbackPending whether need to cleanup pending commits | ||
| */ | ||
| public HoodieWriteClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig, boolean rollbackPending) { | ||
| this(jsc, clientConfig, rollbackPending, HoodieIndex.createIndex(clientConfig, jsc)); | ||
|
|
@@ -135,6 +138,12 @@ public HoodieWriteClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig, b | |
| this.cleanClient = new HoodieCleanClient<>(jsc, config, metrics, timelineService); | ||
| } | ||
|
|
||
| /** | ||
| * Register hudi classes with Kryo serialization | ||
| * | ||
| * @param conf instance of SparkConf | ||
| * @return SparkConf | ||
| */ | ||
| public static SparkConf registerClasses(SparkConf conf) { | ||
| conf.registerKryoClasses(new Class[]{HoodieWriteConfig.class, HoodieRecord.class, HoodieKey.class}); | ||
| return conf; | ||
|
|
@@ -158,6 +167,10 @@ public JavaRDD<HoodieRecord<T>> filterExists(JavaRDD<HoodieRecord<T>> hoodieReco | |
|
|
||
| /** | ||
| * Upserts a bunch of new records into the Hoodie table, at the supplied commitTime. | ||
| * | ||
|
hddong marked this conversation as resolved.
|
||
| * @param records hoodieRecords to upsert | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. minor: RDD of hoodieRecords to upsert. or feel free to leave it as is. |
||
| * @param commitTime Commit Time handle | ||
| * @return JavaRDD[WriteStatus] - RDD of WriteStatus to inspect errors and counts | ||
| */ | ||
| public JavaRDD<WriteStatus> upsert(JavaRDD<HoodieRecord<T>> records, final String commitTime) { | ||
| HoodieTable<T> table = getTableAndInitCtx(OperationType.UPSERT); | ||
|
|
@@ -509,13 +522,21 @@ record -> new Tuple2<>(new Tuple2<>(record.getKey(), Option.ofNullable(record.ge | |
|
|
||
| /** | ||
| * Commit changes performed at the given commitTime marker. | ||
| * | ||
| * @param commitTime Commit Time handle | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. just |
||
| * @param writeStatuses RDD of WriteStatus to inspect errors and counts | ||
| * @return true if commit successfully | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
| */ | ||
| public boolean commit(String commitTime, JavaRDD<WriteStatus> writeStatuses) { | ||
| return commit(commitTime, writeStatuses, Option.empty()); | ||
| } | ||
|
|
||
| /** | ||
| * Commit changes performed at the given commitTime marker. | ||
| * @param commitTime Commit Time handle | ||
| * @param writeStatuses RDD of WriteStatus to inspect errors and counts | ||
|
hddong marked this conversation as resolved.
Outdated
|
||
| * @param extraMetadata Extra Metadata to be stored | ||
| * @return true if commit successfully | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. same comment as above |
||
| */ | ||
| public boolean commit(String commitTime, JavaRDD<WriteStatus> writeStatuses, | ||
| Option<Map<String, String>> extraMetadata) { | ||
|
|
@@ -775,7 +796,10 @@ public boolean rollbackToSavepoint(String savepointTime) { | |
| /** | ||
| * Rollback the (inflight/committed) record changes with the given commit time. Three steps: (1) Atomically unpublish | ||
| * this commit (2) clean indexing data, (3) clean new generated parquet files. (4) Finally delete .commit or .inflight | ||
| * file, | ||
| * file. | ||
| * | ||
| * @param commitTime Commit Time handle | ||
| * @return true if commit was rollback to successfully | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. same comment as above |
||
| */ | ||
| public boolean rollback(final String commitTime) throws HoodieRollbackException { | ||
| rollbackInternal(commitTime); | ||
|
|
@@ -784,9 +808,11 @@ public boolean rollback(final String commitTime) throws HoodieRollbackException | |
|
|
||
| /** | ||
| * NOTE : This action requires all writers (ingest and compact) to a dataset to be stopped before proceeding. Revert | ||
| * the (inflight/committed) record changes for all commits after the provided @param. Three steps: (1) Atomically | ||
| * the (inflight/committed) record changes for all commits after the provided @param. Four steps: (1) Atomically | ||
| * unpublish this commit (2) clean indexing data, (3) clean new generated parquet/log files and/or append rollback to | ||
| * existing log files. (4) Finally delete .commit, .inflight, .compaction.inflight or .compaction.requested file | ||
| * | ||
| * @param instantTime Instant time | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. minor: instant time to which restoration is requested |
||
| */ | ||
| public void restoreToInstant(final String instantTime) throws HoodieRollbackException { | ||
|
|
||
|
|
@@ -1017,6 +1043,11 @@ public String startCommit() { | |
| return commitTime; | ||
| } | ||
|
|
||
| /** | ||
| * Provides a new commit time for a write operation (insert/update). | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. minor: include delete as well. |
||
| * | ||
| * @param instantTime Instant time to be generated | ||
| */ | ||
| public void startCommitWithTime(String instantTime) { | ||
| // NOTE : Need to ensure that rollback is done before a new commit is started | ||
| if (rollbackPending) { | ||
|
|
@@ -1044,6 +1075,8 @@ private void startCommit(String instantTime) { | |
|
|
||
| /** | ||
| * Schedules a new compaction instant. | ||
| * | ||
| * @param extraMetadata Extra Metadata to be stored | ||
| */ | ||
| public Option<String> scheduleCompaction(Option<Map<String, String>> extraMetadata) throws IOException { | ||
| String instantTime = HoodieActiveTimeline.createNewInstantTime(); | ||
|
|
@@ -1093,13 +1126,18 @@ public boolean scheduleCompactionAtInstant(String instantTime, Option<Map<String | |
| * Performs Compaction for the workload stored in instant-time. | ||
| * | ||
| * @param compactionInstantTime Compaction Instant Time | ||
| * @return RDD of WriteStatus | ||
| */ | ||
| public JavaRDD<WriteStatus> compact(String compactionInstantTime) throws IOException { | ||
| return compact(compactionInstantTime, config.shouldAutoCommit()); | ||
| } | ||
|
|
||
| /** | ||
| * Commit a compaction operation. Allow passing additional meta-data to be stored in commit instant file. | ||
| * | ||
| * @param compactionInstantTime Compaction Instant Time | ||
| * @param writeStatuses RDD of WriteStatus to inspect errors and counts | ||
| * @param extraMetadata Extra Metadata to be stored | ||
| */ | ||
| public void commitCompaction(String compactionInstantTime, JavaRDD<WriteStatus> writeStatuses, | ||
| Option<Map<String, String>> extraMetadata) throws IOException { | ||
|
|
@@ -1124,6 +1162,10 @@ public void commitCompaction(String compactionInstantTime, JavaRDD<WriteStatus> | |
|
|
||
| /** | ||
| * Deduplicate Hoodie records, using the given deduplication funciton. | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. typo funciton -> function |
||
| * | ||
| * @param records hoodieRecords to deduplicate | ||
| * @param parallelism partitions of RDD | ||
|
hddong marked this conversation as resolved.
Outdated
|
||
| * @return RDD of HoodieRecord already be deduplicated | ||
| */ | ||
| JavaRDD<HoodieRecord<T>> deduplicateRecords(JavaRDD<HoodieRecord<T>> records, int parallelism) { | ||
| boolean isIndexingGlobal = index.isGlobal(); | ||
|
|
@@ -1144,6 +1186,10 @@ JavaRDD<HoodieRecord<T>> deduplicateRecords(JavaRDD<HoodieRecord<T>> records, in | |
|
|
||
| /** | ||
| * Deduplicate Hoodie records, using the given deduplication funciton. | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ditto |
||
| * | ||
| * @param keys RDD of HoodieKey to deduplicate | ||
| * @param parallelism partitions of RDD | ||
|
hddong marked this conversation as resolved.
Outdated
|
||
| * @return RDD of HoodieKey already be deduplicated | ||
| */ | ||
| JavaRDD<HoodieKey> deduplicateKeys(JavaRDD<HoodieKey> keys, int parallelism) { | ||
| boolean isIndexingGlobal = index.isGlobal(); | ||
|
|
@@ -1216,6 +1262,7 @@ private void setWriteSchemaFromLastInstant(HoodieTableMetaClient metaClient) { | |
| * Ensures compaction instant is in expected state and performs Compaction for the workload stored in instant-time. | ||
| * | ||
| * @param compactionInstantTime Compaction Instant Time | ||
| * @return RDD of Write Status | ||
| */ | ||
| private JavaRDD<WriteStatus> compact(String compactionInstantTime, boolean autoCommit) throws IOException { | ||
| // Create a Hoodie table which encapsulated the commits and files visible | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
if possible add a line about instantiation and what this constructor is used for compared to other constructors.