diff --git a/docker/demo/config/test-suite/cow-spark-long-running.yaml b/docker/demo/config/test-suite/cow-spark-long-running.yaml index 795a4a5f60709..00fea43f4578e 100644 --- a/docker/demo/config/test-suite/cow-spark-long-running.yaml +++ b/docker/demo/config/test-suite/cow-spark-long-running.yaml @@ -38,7 +38,7 @@ dag_content: first_delete: config: num_partitions_delete: 50 - num_records_delete: 8000 + num_records_delete: 4000 type: SparkDeleteNode deps: first_upsert second_validate: diff --git a/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-hive.yaml b/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-hive.yaml index 09dd6168c985e..76172203866b0 100644 --- a/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-hive.yaml +++ b/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-hive.yaml @@ -60,7 +60,7 @@ dag_content: first_delete: config: num_partitions_delete: 50 - num_records_delete: 8000 + num_records_delete: 4000 type: DeleteNode deps: first_upsert second_hive_sync: diff --git a/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-metadata.yaml b/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-metadata.yaml index b2ab525b1af65..57c8d010080a0 100644 --- a/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-metadata.yaml +++ b/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-metadata.yaml @@ -54,7 +54,7 @@ dag_content: first_delete: config: num_partitions_delete: 50 - num_records_delete: 8000 + num_records_delete: 4000 type: DeleteNode deps: first_upsert second_validate: diff --git a/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions.yaml b/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions.yaml index b8f2b686066c3..a29152bb45431 100644 --- a/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions.yaml +++ b/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions.yaml @@ -54,7 +54,7 @@ dag_content: first_delete: config: num_partitions_delete: 50 - num_records_delete: 8000 + num_records_delete: 4000 type: DeleteNode deps: first_upsert second_validate: diff --git a/docker/demo/config/test-suite/simple-clustering-hive.yaml b/docker/demo/config/test-suite/simple-clustering-hive.yaml index e1f79bfe93c0f..1127bd02b93e8 100644 --- a/docker/demo/config/test-suite/simple-clustering-hive.yaml +++ b/docker/demo/config/test-suite/simple-clustering-hive.yaml @@ -44,7 +44,7 @@ dag_content: first_delete: config: num_partitions_delete: 1 - num_records_delete: 9000 + num_records_delete: 3000 type: DeleteNode deps: third_insert first_hive_sync: diff --git a/docker/demo/config/test-suite/simple-clustering.yaml b/docker/demo/config/test-suite/simple-clustering.yaml index 01849bb6436ea..4ede6394cf752 100644 --- a/docker/demo/config/test-suite/simple-clustering.yaml +++ b/docker/demo/config/test-suite/simple-clustering.yaml @@ -44,7 +44,7 @@ dag_content: first_delete: config: num_partitions_delete: 1 - num_records_delete: 9000 + num_records_delete: 3000 type: DeleteNode deps: third_insert first_validate: diff --git a/docker/demo/config/test-suite/spark-long-running.yaml b/docker/demo/config/test-suite/spark-long-running.yaml new file mode 100644 index 0000000000000..00fea43f4578e --- /dev/null +++ b/docker/demo/config/test-suite/spark-long-running.yaml @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +dag_name: cow-spark-deltastreamer-long-running-multi-partitions.yaml +dag_rounds: 30 +dag_intermittent_delay_mins: 0 +dag_content: + first_insert: + config: + record_size: 200 + num_partitions_insert: 50 + repeat_count: 1 + num_records_insert: 10000 + type: SparkInsertNode + deps: none + first_upsert: + config: + record_size: 200 + num_partitions_insert: 50 + num_records_insert: 300 + repeat_count: 1 + num_records_upsert: 3000 + num_partitions_upsert: 50 + type: SparkUpsertNode + deps: first_insert + first_delete: + config: + num_partitions_delete: 50 + num_records_delete: 4000 + type: SparkDeleteNode + deps: first_upsert + second_validate: + config: + validate_once_every_itr : 5 + validate_hive: false + delete_input_data: true + type: ValidateDatasetNode + deps: first_delete + last_validate: + config: + execute_itr_count: 30 + type: ValidateAsyncOperations + deps: second_validate diff --git a/docker/demo/config/test-suite/spark-simple.yaml b/docker/demo/config/test-suite/spark-simple.yaml new file mode 100644 index 0000000000000..192adcf377dc0 --- /dev/null +++ b/docker/demo/config/test-suite/spark-simple.yaml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +dag_name: cow-spark-simple.yaml +dag_rounds: 1 +dag_intermittent_delay_mins: 1 +dag_content: + first_insert: + config: + record_size: 1000 + num_partitions_insert: 1 + repeat_count: 1 + num_records_insert: 100 + type: SparkInsertNode + deps: none + first_validate: + config: + validate_hive: false + type: ValidateDatasetNode + deps: first_insert + first_upsert: + config: + record_size: 1000 + num_partitions_insert: 1 + num_records_insert: 50 + repeat_count: 1 + num_records_upsert: 100 + num_partitions_upsert: 1 + type: SparkUpsertNode + deps: first_validate + first_delete: + config: + num_partitions_delete: 1 + num_records_delete: 30 + type: SparkDeleteNode + deps: first_upsert + second_validate: + config: + validate_hive: false + delete_input_data: false + type: ValidateDatasetNode + deps: first_delete \ No newline at end of file diff --git a/docker/demo/config/test-suite/test-aggressive-clean-archival.properties b/docker/demo/config/test-suite/test-aggressive-clean-archival.properties index dcbbfb31c9936..159c1f233185c 100644 --- a/docker/demo/config/test-suite/test-aggressive-clean-archival.properties +++ b/docker/demo/config/test-suite/test-aggressive-clean-archival.properties @@ -22,9 +22,9 @@ hoodie.insert.shuffle.parallelism=100 hoodie.upsert.shuffle.parallelism=100 hoodie.bulkinsert.shuffle.parallelism=100 -hoodie.cleaner.commits.retained=5 -hoodie.keep.min.commits=9 -hoodie.keep.max.commits=10 +hoodie.cleaner.commits.retained=8 +hoodie.keep.min.commits=12 +hoodie.keep.max.commits=14 hoodie.deltastreamer.source.test.num_partitions=100 hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false diff --git a/docker/demo/config/test-suite/test-clustering-aggressive-clean-archival.properties b/docker/demo/config/test-suite/test-clustering-aggressive-clean-archival.properties index abddd77ba327a..d079536f95363 100644 --- a/docker/demo/config/test-suite/test-clustering-aggressive-clean-archival.properties +++ b/docker/demo/config/test-suite/test-clustering-aggressive-clean-archival.properties @@ -22,9 +22,9 @@ hoodie.insert.shuffle.parallelism=100 hoodie.upsert.shuffle.parallelism=100 hoodie.bulkinsert.shuffle.parallelism=100 -hoodie.cleaner.commits.retained=5 -hoodie.keep.min.commits=9 -hoodie.keep.max.commits=10 +hoodie.cleaner.commits.retained=8 +hoodie.keep.min.commits=12 +hoodie.keep.max.commits=14 hoodie.deltastreamer.source.test.num_partitions=100 hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false diff --git a/docker/demo/config/test-suite/test-clustering-metadata-aggressive-clean-archival.properties b/docker/demo/config/test-suite/test-clustering-metadata-aggressive-clean-archival.properties index 931b1e3a09668..23b95f430408d 100644 --- a/docker/demo/config/test-suite/test-clustering-metadata-aggressive-clean-archival.properties +++ b/docker/demo/config/test-suite/test-clustering-metadata-aggressive-clean-archival.properties @@ -22,9 +22,9 @@ hoodie.insert.shuffle.parallelism=100 hoodie.upsert.shuffle.parallelism=100 hoodie.bulkinsert.shuffle.parallelism=100 -hoodie.cleaner.commits.retained=5 -hoodie.keep.min.commits=9 -hoodie.keep.max.commits=10 +hoodie.cleaner.commits.retained=8 +hoodie.keep.min.commits=12 +hoodie.keep.max.commits=14 hoodie.metadata.enable=true diff --git a/docker/demo/config/test-suite/test-metadata-aggressive-clean-archival.properties b/docker/demo/config/test-suite/test-metadata-aggressive-clean-archival.properties index 8935ffb4264be..160da83004f44 100644 --- a/docker/demo/config/test-suite/test-metadata-aggressive-clean-archival.properties +++ b/docker/demo/config/test-suite/test-metadata-aggressive-clean-archival.properties @@ -22,9 +22,9 @@ hoodie.insert.shuffle.parallelism=100 hoodie.upsert.shuffle.parallelism=100 hoodie.bulkinsert.shuffle.parallelism=100 -hoodie.cleaner.commits.retained=5 -hoodie.keep.min.commits=9 -hoodie.keep.max.commits=10 +hoodie.cleaner.commits.retained=8 +hoodie.keep.min.commits=12 +hoodie.keep.max.commits=14 hoodie.metadata.enable=true