From 0c435a9e68878fd2a8239fd6f2efc281e27e82b4 Mon Sep 17 00:00:00 2001 From: Rahil Chertara Date: Wed, 27 Jul 2022 12:11:06 -0700 Subject: [PATCH] [HUDI-4490] Make AWSDmsAvroPayload class backwards compatible --- .../hudi/common/model/AWSDmsAvroPayload.java | 3 +- .../common/model}/TestAWSDmsAvroPayload.java | 7 +-- .../hudi/payload/AWSDmsAvroPayload.java | 48 +++++++++++++++++++ 3 files changed, 51 insertions(+), 7 deletions(-) rename {hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/payload => hudi-common/src/test/java/org/apache/hudi/common/model}/TestAWSDmsAvroPayload.java (96%) create mode 100644 hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/payload/AWSDmsAvroPayload.java diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/AWSDmsAvroPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/AWSDmsAvroPayload.java index c515338ee64b4..20a20fb62999b 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/AWSDmsAvroPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/AWSDmsAvroPayload.java @@ -18,11 +18,10 @@ package org.apache.hudi.common.model; -import org.apache.hudi.common.util.Option; - import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.util.Option; import java.io.IOException; import java.util.Properties; diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/payload/TestAWSDmsAvroPayload.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestAWSDmsAvroPayload.java similarity index 96% rename from hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/payload/TestAWSDmsAvroPayload.java rename to hudi-common/src/test/java/org/apache/hudi/common/model/TestAWSDmsAvroPayload.java index 7373553dfdb50..5ba537269ea19 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/payload/TestAWSDmsAvroPayload.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestAWSDmsAvroPayload.java @@ -16,16 +16,13 @@ * limitations under the License. */ -package org.apache.hudi.payload; - -import org.apache.hudi.common.model.AWSDmsAvroPayload; -import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload; -import org.apache.hudi.common.util.Option; +package org.apache.hudi.common.model; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.util.Option; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertFalse; diff --git a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/payload/AWSDmsAvroPayload.java b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/payload/AWSDmsAvroPayload.java new file mode 100644 index 0000000000000..1411d4f4796c0 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/payload/AWSDmsAvroPayload.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.payload; + +import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.common.util.Option; + +/** + * Provides support for seamlessly applying changes captured via Amazon Database Migration Service onto S3. + * + * Typically, we get the following pattern of full change records corresponding to DML against the + * source database + * + * - Full load records with no `Op` field + * - For inserts against the source table, records contain full after image with `Op=I` + * - For updates against the source table, records contain full after image with `Op=U` + * - For deletes against the source table, records contain full before image with `Op=D` + * + * This payload implementation will issue matching insert, delete, updates against the hudi table + * + */ +@Deprecated +public class AWSDmsAvroPayload extends org.apache.hudi.common.model.AWSDmsAvroPayload { + + public AWSDmsAvroPayload(GenericRecord record, Comparable orderingVal) { + super(record, orderingVal); + } + + public AWSDmsAvroPayload(Option record) { + super(record); + } +}