From 78a0b1bdc2788cc688236935410f1d039fdc998e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Osipiuk?= Date: Fri, 13 Jun 2025 14:18:23 +0200 Subject: [PATCH 1/2] Unflake deltalake.TestCloseIdleWriters --- .../io/trino/plugin/deltalake/TestCloseIdleWriters.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestCloseIdleWriters.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestCloseIdleWriters.java index c9eb57f7699c..bfcd9e8136ba 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestCloseIdleWriters.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestCloseIdleWriters.java @@ -77,9 +77,9 @@ WHEN shipmode IN ('AIR', 'FOB', 'SHIP', 'TRUCK') THEN 0 WHEN shipmode IN ('MAIL', 'RAIL', 'REG AIR') THEN 1 ELSE 2 END AS shipmodeVal - FROM tpch.tiny.lineitem + FROM tpch."sf0.1".lineitem ORDER BY shipmode - LIMIT 60174 + LIMIT 600571 """.formatted(tableName); // Disable all kind of scaling and set idle writer threshold to 10MB @@ -92,7 +92,7 @@ WHEN shipmode IN ('MAIL', 'RAIL', 'REG AIR') THEN 1 .setSystemProperty(IDLE_WRITER_MIN_DATA_SIZE_THRESHOLD, "0.1MB") .build(), createTableSql, - 60174); + 600571); long files = (long) computeScalar("SELECT count(DISTINCT \"$path\") FROM " + tableName); // There should more than 2 files since we triggered close idle writers. assertThat(files).isGreaterThan(2); From a2a87aa81c52cee9acce9f312d06d67fc60fb27c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Osipiuk?= Date: Fri, 13 Jun 2025 14:28:34 +0200 Subject: [PATCH 2/2] Clean up deltalake.TestCloseIdleWriters --- .../deltalake/TestCloseIdleWriters.java | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestCloseIdleWriters.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestCloseIdleWriters.java index bfcd9e8136ba..9f82a1682ce5 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestCloseIdleWriters.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestCloseIdleWriters.java @@ -60,8 +60,15 @@ protected QueryRunner createQueryRunner() @Test public void testCloseIdleWriters() { - String tableName = "task_close_idle_writers_" + randomNameSuffix(); + String sourceTable = "tpch.\"sf0.1\".lineitem"; + String targetTable = "task_close_idle_writers_" + randomNameSuffix(); try { + String zeroShipModes = "'AIR', 'FOB', 'SHIP', 'TRUCK'"; + String oneShipModes = "'MAIL', 'RAIL', 'REG AIR'"; + String bothShipModes = zeroShipModes + ", " + oneShipModes; + + long expectedCount = (long) computeScalar("SELECT count (*) FROM %s WHERE shipmode IN (%s)".formatted(sourceTable, bothShipModes)); + // Create a table with two partitions (0 and 1). Using the order by trick we will write the partitions in // this order 0, 1, and then again 0. This way we are sure that during partition 1 write there will // be an idle writer for partition 0. Additionally, during second partition 0 write, there will be an idle @@ -73,16 +80,16 @@ public void testCloseIdleWriters() discount, tax, returnflag, linestatus, commitdate, receiptdate, shipinstruct, comment, shipdate, CASE - WHEN shipmode IN ('AIR', 'FOB', 'SHIP', 'TRUCK') THEN 0 - WHEN shipmode IN ('MAIL', 'RAIL', 'REG AIR') THEN 1 - ELSE 2 + WHEN shipmode IN (%s) THEN 0 + WHEN shipmode IN (%s) THEN 1 END AS shipmodeVal - FROM tpch."sf0.1".lineitem + FROM %s + WHERE shipmode IN (%s) ORDER BY shipmode - LIMIT 600571 - """.formatted(tableName); + LIMIT %s + """.formatted(targetTable, zeroShipModes, oneShipModes, sourceTable, bothShipModes, expectedCount); - // Disable all kind of scaling and set idle writer threshold to 10MB + // Disable all kind of scaling and set low idle writer threshold assertUpdate( Session.builder(getSession()) .setSystemProperty(SCALE_WRITERS, "false") @@ -92,13 +99,13 @@ WHEN shipmode IN ('MAIL', 'RAIL', 'REG AIR') THEN 1 .setSystemProperty(IDLE_WRITER_MIN_DATA_SIZE_THRESHOLD, "0.1MB") .build(), createTableSql, - 600571); - long files = (long) computeScalar("SELECT count(DISTINCT \"$path\") FROM " + tableName); + expectedCount); + long files = (long) computeScalar("SELECT count(DISTINCT \"$path\") FROM " + targetTable); // There should more than 2 files since we triggered close idle writers. assertThat(files).isGreaterThan(2); } finally { - assertUpdate("DROP TABLE IF EXISTS " + tableName); + assertUpdate("DROP TABLE IF EXISTS " + targetTable); } } }