diff --git a/docker/demo/config/test-suite/cow-spark-long-running.yaml b/docker/demo/config/test-suite/cow-spark-long-running.yaml index 8229ba749502e..c25b95c8da457 100644 --- a/docker/demo/config/test-suite/cow-spark-long-running.yaml +++ b/docker/demo/config/test-suite/cow-spark-long-running.yaml @@ -25,17 +25,11 @@ dag_content: num_records_insert: 10000 type: SparkInsertNode deps: none - first_hive_sync: - config: - queue_name: "adhoc" - engine: "mr" - type: HiveSyncNode - deps: first_insert first_validate: config: validate_hive: false type: ValidateDatasetNode - deps: first_hive_sync + deps: first_insert first_upsert: config: record_size: 200 @@ -52,18 +46,12 @@ dag_content: num_records_delete: 8000 type: SparkDeleteNode deps: first_upsert - second_hive_sync: - config: - queue_name: "adhoc" - engine: "mr" - type: HiveSyncNode - deps: first_delete second_validate: config: validate_hive: false delete_input_data: true type: ValidateDatasetNode - deps: second_hive_sync + deps: first_delete last_validate: config: execute_itr_count: 30 diff --git a/docker/demo/config/test-suite/cow-spark-simple.yaml b/docker/demo/config/test-suite/cow-spark-simple.yaml index 0859c63200203..192adcf377dc0 100644 --- a/docker/demo/config/test-suite/cow-spark-simple.yaml +++ b/docker/demo/config/test-suite/cow-spark-simple.yaml @@ -25,17 +25,11 @@ dag_content: num_records_insert: 100 type: SparkInsertNode deps: none - first_hive_sync: - config: - queue_name: "adhoc" - engine: "mr" - type: HiveSyncNode - deps: first_insert first_validate: config: validate_hive: false type: ValidateDatasetNode - deps: first_hive_sync + deps: first_insert first_upsert: config: record_size: 1000 @@ -52,15 +46,9 @@ dag_content: num_records_delete: 30 type: SparkDeleteNode deps: first_upsert - second_hive_sync: - config: - queue_name: "adhoc" - engine: "mr" - type: HiveSyncNode - deps: first_delete second_validate: config: validate_hive: false delete_input_data: false type: ValidateDatasetNode - deps: second_hive_sync \ No newline at end of file + deps: first_delete \ No newline at end of file diff --git a/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-metadata.yaml b/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-metadata.yaml index 817552224e8f3..0212fdf43c512 100644 --- a/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-metadata.yaml +++ b/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-metadata.yaml @@ -57,18 +57,12 @@ dag_content: num_records_delete: 8000 type: DeleteNode deps: first_upsert - second_hive_sync: - config: - queue_name: "adhoc" - engine: "mr" - type: HiveSyncNode - deps: first_delete second_validate: config: validate_hive: false delete_input_data: true type: ValidateDatasetNode - deps: second_hive_sync + deps: first_delete last_validate: config: execute_itr_count: 30 diff --git a/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions.yaml b/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions.yaml index ba490a8c194dd..d7b11194736e9 100644 --- a/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions.yaml +++ b/docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions.yaml @@ -57,18 +57,12 @@ dag_content: num_records_delete: 8000 type: DeleteNode deps: first_upsert - second_hive_sync: - config: - queue_name: "adhoc" - engine: "mr" - type: HiveSyncNode - deps: first_delete second_validate: config: validate_hive: false delete_input_data: true type: ValidateDatasetNode - deps: second_hive_sync + deps: first_delete last_validate: config: execute_itr_count: 50 diff --git a/docker/demo/config/test-suite/deltastreamer-medium-full-dataset-validation.yaml b/docker/demo/config/test-suite/deltastreamer-medium-full-dataset-validation.yaml index aa83b2780127c..77898640ea144 100644 --- a/docker/demo/config/test-suite/deltastreamer-medium-full-dataset-validation.yaml +++ b/docker/demo/config/test-suite/deltastreamer-medium-full-dataset-validation.yaml @@ -60,18 +60,12 @@ dag_content: num_records_delete: 8000 type: DeleteNode deps: first_upsert - second_hive_sync: - config: - queue_name: "adhoc" - engine: "mr" - type: HiveSyncNode - deps: first_delete second_validate: config: validate_hive: false delete_input_data: false type: ValidateDatasetNode - deps: second_hive_sync + deps: first_delete last_validate: config: execute_itr_count: 20 diff --git a/docker/demo/config/test-suite/detlastreamer-long-running-example.yaml b/docker/demo/config/test-suite/detlastreamer-long-running-example.yaml index 28578eb9b687e..4b2ee7ad13cc4 100644 --- a/docker/demo/config/test-suite/detlastreamer-long-running-example.yaml +++ b/docker/demo/config/test-suite/detlastreamer-long-running-example.yaml @@ -41,17 +41,11 @@ dag_content: num_records_insert: 300 deps: second_insert type: InsertNode - first_hive_sync: - config: - queue_name: "adhoc" - engine: "mr" - type: HiveSyncNode - deps: third_insert first_validate: config: validate_hive: false type: ValidateDatasetNode - deps: first_hive_sync + deps: third_insert first_upsert: config: record_size: 1000 @@ -68,22 +62,14 @@ dag_content: num_records_delete: 8000 type: DeleteNode deps: first_upsert - second_hive_sync: - config: - queue_name: "adhoc" - engine: "mr" - type: HiveSyncNode - deps: first_delete second_validate: config: validate_hive: false delete_input_data: true type: ValidateDatasetNode - deps: second_hive_sync + deps: first_delete last_validate: config: execute_itr_count: 50 - validate_clean: true - validate_archival: true type: ValidateAsyncOperations deps: second_validate diff --git a/docker/demo/config/test-suite/insert-overwrite-table.yaml b/docker/demo/config/test-suite/insert-overwrite-table.yaml index 8b5a26e4683b7..1a58abdcc4789 100644 --- a/docker/demo/config/test-suite/insert-overwrite-table.yaml +++ b/docker/demo/config/test-suite/insert-overwrite-table.yaml @@ -53,19 +53,13 @@ dag_content: num_partitions_upsert: 10 type: SparkUpsertNode deps: second_insert - first_hive_sync: - config: - queue_name: "adhoc" - engine: "mr" - type: HiveSyncNode - deps: second_upsert first_insert_overwrite_table: config: record_size: 1000 repeat_count: 10 num_records_insert: 10 type: SparkInsertOverwriteTableNode - deps: first_hive_sync + deps: second_upsert delete_all_input_except_last: config: delete_input_data_except_latest: true @@ -89,16 +83,10 @@ dag_content: num_partitions_upsert: 10 type: SparkUpsertNode deps: third_insert - second_hive_sync: - config: - queue_name: "adhoc" - engine: "mr" - type: HiveSyncNode - deps: third_upsert second_validate: config: validate_full_data : true validate_hive: false delete_input_data: false type: ValidateDatasetNode - deps: second_hive_sync + deps: third_upsert diff --git a/docker/demo/config/test-suite/insert-overwrite.yaml b/docker/demo/config/test-suite/insert-overwrite.yaml index f2299c50c08f3..dc185d5938f6d 100644 --- a/docker/demo/config/test-suite/insert-overwrite.yaml +++ b/docker/demo/config/test-suite/insert-overwrite.yaml @@ -54,12 +54,6 @@ dag_content: num_partitions_upsert: 10 type: SparkUpsertNode deps: second_insert - first_hive_sync: - config: - queue_name: "adhoc" - engine: "mr" - type: HiveSyncNode - deps: second_upsert first_insert_overwrite: config: record_size: 1000 @@ -67,7 +61,7 @@ dag_content: repeat_count: 1 num_records_insert: 10 type: SparkInsertOverwriteNode - deps: first_hive_sync + deps: second_upsert delete_all_input_except_last: config: delete_input_data_except_latest: true @@ -91,16 +85,10 @@ dag_content: num_partitions_upsert: 10 type: SparkUpsertNode deps: third_insert - second_hive_sync: - config: - queue_name: "adhoc" - engine: "mr" - type: HiveSyncNode - deps: third_upsert second_validate: config: validate_full_data : true validate_hive: false delete_input_data: false type: ValidateDatasetNode - deps: second_hive_sync + deps: third_upsert diff --git a/docker/demo/config/test-suite/simple-clustering.yaml b/docker/demo/config/test-suite/simple-clustering.yaml index 7389ee3ebc34b..01849bb6436ea 100644 --- a/docker/demo/config/test-suite/simple-clustering.yaml +++ b/docker/demo/config/test-suite/simple-clustering.yaml @@ -47,30 +47,18 @@ dag_content: num_records_delete: 9000 type: DeleteNode deps: third_insert - first_hive_sync: - config: - queue_name: "adhoc" - engine: "mr" - type: HiveSyncNode - deps: first_delete first_validate: config: validate_hive: false type: ValidateDatasetNode - deps: first_hive_sync + deps: first_delete first_cluster: config: execute_itr_count: 25 type: ClusteringNode deps: first_validate - second_hive_sync: - config: - queue_name: "adhoc" - engine: "mr" - type: HiveSyncNode - deps: first_cluster second_validate: config: validate_hive: false type: ValidateDatasetNode - deps: second_hive_sync + deps: first_cluster diff --git a/docker/demo/config/test-suite/simple-deltastreamer.yaml b/docker/demo/config/test-suite/simple-deltastreamer.yaml index f49a41baf8541..11b7f17d34a4d 100644 --- a/docker/demo/config/test-suite/simple-deltastreamer.yaml +++ b/docker/demo/config/test-suite/simple-deltastreamer.yaml @@ -41,17 +41,11 @@ dag_content: num_records_insert: 300 deps: second_insert type: InsertNode - first_hive_sync: - config: - queue_name: "adhoc" - engine: "mr" - type: HiveSyncNode - deps: third_insert first_validate: config: validate_hive: false type: ValidateDatasetNode - deps: first_hive_sync + deps: third_insert first_upsert: config: record_size: 1000 @@ -68,15 +62,9 @@ dag_content: num_records_delete: 2000 type: DeleteNode deps: first_upsert - second_hive_sync: - config: - queue_name: "adhoc" - engine: "mr" - type: HiveSyncNode - deps: first_delete second_validate: config: validate_hive: false delete_input_data: true type: ValidateDatasetNode - deps: second_hive_sync + deps: first_delete diff --git a/docker/demo/config/test-suite/spark-clustering.yaml b/docker/demo/config/test-suite/spark-clustering.yaml index e8e722ca77c7c..8da4f953983b8 100644 --- a/docker/demo/config/test-suite/spark-clustering.yaml +++ b/docker/demo/config/test-suite/spark-clustering.yaml @@ -59,15 +59,9 @@ dag_content: num_records_delete: 16000 type: SparkDeleteNode deps: second_upsert - second_hive_sync: - config: - queue_name: "adhoc" - engine: "mr" - type: HiveSyncNode - deps: first_delete second_validate: config: validate_hive: false delete_input_data: false type: ValidateDatasetNode - deps: second_hive_sync \ No newline at end of file + deps: first_delete \ No newline at end of file