From e45a2b36733acc7bff16dc02ec872c1363946c6b Mon Sep 17 00:00:00 2001 From: luoyajun10 Date: Wed, 1 Jun 2022 14:54:05 +0800 Subject: [PATCH 1/2] [HUDI-3863] Add UT for drop partition column in deltastreamer testsuite --- .../functional/TestHoodieDeltaStreamer.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java index ad94ada59b2fd..ae38968187793 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java @@ -2142,6 +2142,28 @@ public void testFetchingCheckpointFromPreviousCommits() throws IOException { .getCommitsTimeline()).get().getMetadata(CHECKPOINT_KEY), "def"); } + @Test + public void testDropPartitionColumns() throws Exception { + String tableBasePath = dfsBasePath + "/test_drop_partition_columns" + testNum++; + // ingest data with dropping partition columns enabled + HoodieDeltaStreamer.Config cfg = TestHelpers.makeConfig(tableBasePath, WriteOperationType.UPSERT); + cfg.configs.add(String.format("%s=%s", HoodieTableConfig.DROP_PARTITION_COLUMNS.key(), "true")); + HoodieDeltaStreamer ds = new HoodieDeltaStreamer(cfg, jsc); + ds.sync(); + // assert ingest successful + TestHelpers.assertAtLeastNCommits(1, tableBasePath, dfs); + + TableSchemaResolver tableSchemaResolver = new TableSchemaResolver( + HoodieTableMetaClient.builder().setBasePath(tableBasePath).setConf(dfs.getConf()).build()); + // get schema from data file written in the latest commit + Schema tableSchema = tableSchemaResolver.getTableAvroSchemaFromDataFile(); + assertNotNull(tableSchema); + + List tableFields = tableSchema.getFields().stream().map(Schema.Field::name).collect(Collectors.toList()); + // now assert that the partition column is not in the target schema + assertFalse(tableFields.contains("partition_path")); + } + class TestDeltaSync extends DeltaSync { public TestDeltaSync(HoodieDeltaStreamer.Config cfg, SparkSession sparkSession, SchemaProvider schemaProvider, TypedProperties props, From 1c746fcd6e407508fe5e8ac2d8cef55c3ab8bb76 Mon Sep 17 00:00:00 2001 From: luoyajun10 Date: Sat, 11 Jun 2022 12:28:04 +0800 Subject: [PATCH 2/2] trigger ci