Test Iceberg cost-based plans with small files on TPC-DS

testing/trino-tests/src/test/java/io/trino/sql/planner/BaseCostBasedPlanTest.java

@@ -88,12 +88,19 @@ public abstract class BaseCostBasedPlanTest
     private final String schemaName;
     private final Optional<String> fileFormatName;
     private final boolean partitioned;
+    protected boolean smallFiles;
 
     public BaseCostBasedPlanTest(String schemaName, Optional<String> fileFormatName, boolean partitioned)
+    {
+        this(schemaName, fileFormatName, partitioned, false);
+    }
+
+    public BaseCostBasedPlanTest(String schemaName, Optional<String> fileFormatName, boolean partitioned, boolean smallFiles)
     {
         this.schemaName = requireNonNull(schemaName, "schemaName is null");
         this.fileFormatName = requireNonNull(fileFormatName, "fileFormatName is null");
         this.partitioned = partitioned;
+        this.smallFiles = smallFiles;
     }
 
     @Override
@@ -141,7 +148,7 @@ private String getQueryPlanResourcePath(String queryResourcePath)
         Path queryPath = Paths.get(queryResourcePath);
         String connectorName = getQueryRunner().getCatalogManager().getCatalog(CATALOG_NAME).orElseThrow().getConnectorName().toString();
         Path directory = queryPath.getParent();
-        directory = directory.resolve(connectorName);
+        directory = directory.resolve(connectorName + (smallFiles ? "_small_files" : ""));
         if (fileFormatName.isPresent()) {
             directory = directory.resolve(fileFormatName.get());
         }

testing/trino-tests/src/test/java/io/trino/sql/planner/BaseIcebergCostBasedPlanTest.java

@@ -94,6 +94,11 @@ protected BaseIcebergCostBasedPlanTest(String schemaName, String fileFormatName,
         super(schemaName, Optional.of(fileFormatName), partitioned);
     }
 
+    protected BaseIcebergCostBasedPlanTest(String schemaName, String fileFormatName, boolean partitioned, boolean smallFiles)
+    {
+        super(schemaName, Optional.of(fileFormatName), partitioned, smallFiles);
+    }
+
     @Override
     protected ConnectorFactory createConnectorFactory()
     {

testing/trino-tests/src/test/java/io/trino/sql/planner/TestIcebergSmallFilesParquetTpcdsCostBasedPlan.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.trino.sql.planner;
+
+import java.util.stream.Stream;
+
+/**
+ * This class tests cost-based optimization rules related to joins. It contains unmodified TPC-DS queries.
+ * This class is using Iceberg connector unpartitioned TPC-DS tables.
+ */
+public class TestIcebergSmallFilesParquetTpcdsCostBasedPlan
+        extends BaseIcebergCostBasedPlanTest
+{
+    public TestIcebergSmallFilesParquetTpcdsCostBasedPlan()
+    {
+        super("tpcds_50mb_files_sf1000_parquet", "parquet", false, true);
+    }
+
+    @Override
+    protected void doPrepareTables()
+    {
+        io.trino.tpcds.Table.getBaseTables().forEach(table -> {
+            if (table == io.trino.tpcds.Table.DBGEN_VERSION) {
+                return;
+            }
+            populateTableFromResource(
+                    table.getName(),
+                    "iceberg_small_files/tpcds/sf1000/parquet/unpartitioned/" + table.getName(),
+                    "iceberg-50MB-files-tpcds-sf1000-PARQUET/" + table.getName());
+        });
+    }
+
+    @Override
+    protected Stream<String> getQueryResourcePaths()
+    {
+        return TPCDS_SQL_FILES.stream();
+    }
+
+    public static void main(String[] args)
+    {
+        new TestIcebergSmallFilesParquetTpcdsCostBasedPlan().generate();
+    }
+}

testing/trino-tests/src/test/resources/iceberg_small_files/tpcds/sf1000/parquet/unpartitioned/call_center/metadata/00000-dbd37a20-4eed-4e77-ad02-4c936c7d8d6a.metadata.json

@@ -0,0 +1,212 @@
+{
+  "format-version" : 2,
+  "table-uuid" : "20e2bcf3-7de2-4f0b-99b6-99fb87c0fd0c",
+  "location" : "s3://starburst-benchmarks-data/iceberg-50MB-files-tpcds-sf1000-PARQUET/call_center",
+  "last-sequence-number" : 1,
+  "last-updated-ms" : 1678866105146,
+  "last-column-id" : 30,
+  "current-schema-id" : 0,
+  "schemas" : [ {
+    "type" : "struct",
+    "schema-id" : 0,
+    "fields" : [ {
+      "id" : 1,
+      "name" : "cc_call_center_sk",
+      "required" : false,
+      "type" : "long"
+    }, {
+      "id" : 2,
+      "name" : "cc_call_center_id",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 3,
+      "name" : "cc_rec_start_date",
+      "required" : false,
+      "type" : "date"
+    }, {
+      "id" : 4,
+      "name" : "cc_rec_end_date",
+      "required" : false,
+      "type" : "date"
+    }, {
+      "id" : 5,
+      "name" : "cc_closed_date_sk",
+      "required" : false,
+      "type" : "int"
+    }, {
+      "id" : 6,
+      "name" : "cc_open_date_sk",
+      "required" : false,
+      "type" : "int"
+    }, {
+      "id" : 7,
+      "name" : "cc_name",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 8,
+      "name" : "cc_class",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 9,
+      "name" : "cc_employees",
+      "required" : false,
+      "type" : "int"
+    }, {
+      "id" : 10,
+      "name" : "cc_sq_ft",
+      "required" : false,
+      "type" : "int"
+    }, {
+      "id" : 11,
+      "name" : "cc_hours",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 12,
+      "name" : "cc_manager",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 13,
+      "name" : "cc_mkt_id",
+      "required" : false,
+      "type" : "int"
+    }, {
+      "id" : 14,
+      "name" : "cc_mkt_class",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 15,
+      "name" : "cc_mkt_desc",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 16,
+      "name" : "cc_market_manager",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 17,
+      "name" : "cc_division",
+      "required" : false,
+      "type" : "int"
+    }, {
+      "id" : 18,
+      "name" : "cc_division_name",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 19,
+      "name" : "cc_company_name",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 20,
+      "name" : "cc_street_number",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 21,
+      "name" : "cc_street_name",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 22,
+      "name" : "cc_street_type",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 23,
+      "name" : "cc_suite_number",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 24,
+      "name" : "cc_city",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 25,
+      "name" : "cc_county",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 26,
+      "name" : "cc_state",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 27,
+      "name" : "cc_zip",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 28,
+      "name" : "cc_country",
+      "required" : false,
+      "type" : "string"
+    }, {
+      "id" : 29,
+      "name" : "cc_gmt_offset",
+      "required" : false,
+      "type" : "decimal(5, 2)"
+    }, {
+      "id" : 30,
+      "name" : "cc_tax_percentage",
+      "required" : false,
+      "type" : "decimal(5, 2)"
+    } ]
+  } ],
+  "default-spec-id" : 0,
+  "partition-specs" : [ {
+    "spec-id" : 0,
+    "fields" : [ ]
+  } ],
+  "last-partition-id" : 999,
+  "default-sort-order-id" : 0,
+  "sort-orders" : [ {
+    "order-id" : 0,
+    "fields" : [ ]
+  } ],
+  "properties" : {
+    "write.format.default" : "PARQUET"
+  },
+  "current-snapshot-id" : 180463549666864477,
+  "refs" : {
+    "main" : {
+      "snapshot-id" : 180463549666864477,
+      "type" : "branch"
+    }
+  },
+  "snapshots" : [ {
+    "sequence-number" : 1,
+    "snapshot-id" : 180463549666864477,
+    "timestamp-ms" : 1678866105146,
+    "summary" : {
+      "operation" : "append",
+      "trino_query_id" : "20230315_074143_00039_9f4mz",
+      "added-data-files" : "1",
+      "added-records" : "42",
+      "added-files-size" : "10246",
+      "changed-partition-count" : "1",
+      "total-records" : "42",
+      "total-files-size" : "10246",
+      "total-data-files" : "1",
+      "total-delete-files" : "0",
+      "total-position-deletes" : "0",
+      "total-equality-deletes" : "0"
+    },
+    "manifest-list" : "s3://starburst-benchmarks-data/iceberg-50MB-files-tpcds-sf1000-PARQUET/call_center/metadata/snap-180463549666864477-1-f8bcd002-b25e-42d3-9a20-680286272ed5.avro",
+    "schema-id" : 0
+  } ],
+  "statistics" : [ ],
+  "snapshot-log" : [ {
+    "timestamp-ms" : 1678866105146,
+    "snapshot-id" : 180463549666864477
+  } ],
+  "metadata-log" : [ ]
+}

testing/trino-tests/src/test/resources/iceberg_small_files/tpcds/sf1000/parquet/unpartitioned/call_center/metadata/f8bcd002-b25e-42d3-9a20-680286272ed5-m0.avro

@@ -0,0 +1,6 @@
+Obj
schema�{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"cc_call_center_sk","required":false,"type":"long"},{"id":2,"name":"cc_call_center_id","required":false,"type":"string"},{"id":3,"name":"cc_rec_start_date","required":false,"type":"date"},{"id":4,"name":"cc_rec_end_date","required":false,"type":"date"},{"id":5,"name":"cc_closed_date_sk","required":false,"type":"int"},{"id":6,"name":"cc_open_date_sk","required":false,"type":"int"},{"id":7,"name":"cc_name","required":false,"type":"string"},{"id":8,"name":"cc_class","required":false,"type":"string"},{"id":9,"name":"cc_employees","required":false,"type":"int"},{"id":10,"name":"cc_sq_ft","required":false,"type":"int"},{"id":11,"name":"cc_hours","required":false,"type":"string"},{"id":12,"name":"cc_manager","required":false,"type":"string"},{"id":13,"name":"cc_mkt_id","required":false,"type":"int"},{"id":14,"name":"cc_mkt_class","required":false,"type":"string"},{"id":15,"name":"cc_mkt_desc","required":false,"type":"string"},{"id":16,"name":"cc_market_manager","required":false,"type":"string"},{"id":17,"name":"cc_division","required":false,"type":"int"},{"id":18,"name":"cc_division_name","required":false,"type":"string"},{"id":19,"name":"cc_company_name","required":false,"type":"string"},{"id":20,"name":"cc_street_number","required":false,"type":"string"},{"id":21,"name":"cc_street_name","required":false,"type":"string"},{"id":22,"name":"cc_street_type","required":false,"type":"string"},{"id":23,"name":"cc_suite_number","required":false,"type":"string"},{"id":24,"name":"cc_city","required":false,"type":"string"},{"id":25,"name":"cc_county","required":false,"type":"string"},{"id":26,"name":"cc_state","required":false,"type":"string"},{"id":27,"name":"cc_zip","required":false,"type":"string"},{"id":28,"name":"cc_country","required":false,"type":"string"},{"id":29,"name":"cc_gmt_offset","required":false,"type":"decimal(5, 2)"},{"id":30,"name":"cc_tax_percentage","required":false,"type":"decimal(5, 2)"}]}avro.schema�5{"type":"record","name":"manifest_entry","fields":[{"name":"status","type":"int","field-id":0},{"name":"snapshot_id","type":["null","long"],"default":null,"field-id":1},{"name":"sequence_number","type":["null","long"],"default":null,"field-id":3},{"name":"file_sequence_number","type":["null","long"],"default":null,"field-id":4},{"name":"data_file","type":{"type":"record","name":"r2","fields":[{"name":"content","type":"int","doc":"Contents of the file: 0=data, 1=position deletes, 2=equality deletes","field-id":134},{"name":"file_path","type":"string","doc":"Location URI with FS scheme","field-id":100},{"name":"file_format","type":"string","doc":"File format name: avro, orc, or parquet","field-id":101},{"name":"partition","type":{"type":"record","name":"r102","fields":[]},"doc":"Partition data tuple, schema based on the partition spec","field-id":102},{"name":"record_count","type":"long","doc":"Number of records in the file","field-id":103},{"name":"file_size_in_bytes","type":"long","doc":"Total file size in bytes","field-id":104},{"name":"column_sizes","type":["null",{"type":"array","items":{"type":"record","name":"k117_v118","fields":[{"name":"key","type":"int","field-id":117},{"name":"value","type":"long","field-id":118}]},"logicalType":"map"}],"doc":"Map of column id to total size on disk","default":null,"field-id":108},{"name":"value_counts","type":["null",{"type":"array","items":{"type":"record","name":"k119_v120","fields":[{"name":"key","type":"int","field-id":119},{"name":"value","type":"long","field-id":120}]},"logicalType":"map"}],"doc":"Map of column id to total count, including null and NaN","default":null,"field-id":109},{"name":"null_value_counts","type":["null",{"type":"array","items":{"type":"record","name":"k121_v122","fields":[{"name":"key","type":"int","field-id":121},{"name":"value","type":"long","field-id":122}]},"logicalType":"map"}],"doc":"Map of column id to null value count","default":null,"field-id":110},{"name":"nan_value_counts","type":["null",{"type":"array","items":{"type":"record","name":"k138_v139","fields":[{"name":"key","type":"int","field-id":138},{"name":"value","type":"long","field-id":139}]},"logicalType":"map"}],"doc":"Map of column id to number of NaN values in the column","default":null,"field-id":137},{"name":"lower_bounds","type":["null",{"type":"array","items":{"type":"record","name":"k126_v127","fields":[{"name":"key","type":"int","field-id":126},{"name":"value","type":"bytes","field-id":127}]},"logicalType":"map"}],"doc":"Map of column id to lower bound","default":null,"field-id":125},{"name":"upper_bounds","type":["null",{"type":"array","items":{"type":"record","name":"k129_v130","fields":[{"name":"key","type":"int","field-id":129},{"name":"value","type":"bytes","field-id":130}]},"logicalType":"map"}],"doc":"Map of column id to upper bound","default":null,"field-id":128},{"name":"key_metadata","type":["null","bytes"],"doc":"Encryption key metadata blob","default":null,"field-id":131},{"name":"split_offsets","type":["null",{"type":"array","items":"long","element-id":133}],"doc":"Splittable offsets","default":null,"field-id":132},{"name":"equality_ids","type":["null",{"type":"array","items":"int","element-id":136}],"doc":"Equality comparison field IDs","default":null,"field-id":135},{"name":"sort_order_id","type":["null","int"],"doc":"Sort order ID","default":null,"field-id":140}]},"field-id":2}]}avro.codecdeflateformat-version2"partition-spec-id0iceberg.schema�+{"type":"struct","schema-id":0,"fields":[{"id":0,"name":"status","required":true,"type":"int"},{"id":1,"name":"snapshot_id","required":false,"type":"long"},{"id":3,"name":"sequence_number","required":false,"type":"long"},{"id":4,"name":"file_sequence_number","required":false,"type":"long"},{"id":2,"name":"data_file","required":true,"type":{"type":"struct","fields":[{"id":134,"name":"content","required":true,"type":"int","doc":"Contents of the file: 0=data, 1=position deletes, 2=equality deletes"},{"id":100,"name":"file_path","required":true,"type":"string","doc":"Location URI with FS scheme"},{"id":101,"name":"file_format","required":true,"type":"string","doc":"File format name: avro, orc, or parquet"},{"id":102,"name":"partition","required":true,"type":{"type":"struct","fields":[]},"doc":"Partition data tuple, schema based on the partition spec"},{"id":103,"name":"record_count","required":true,"type":"long","doc":"Number of records in the file"},{"id":104,"name":"file_size_in_bytes","required":true,"type":"long","doc":"Total file size in bytes"},{"id":108,"name":"column_sizes","required":false,"type":{"type":"map","key-id":117,"key":"int","value-id":118,"value":"long","value-required":true},"doc":"Map of column id to total size on disk"},{"id":109,"name":"value_counts","required":false,"type":{"type":"map","key-id":119,"key":"int","value-id":120,"value":"long","value-required":true},"doc":"Map of column id to total count, including null and NaN"},{"id":110,"name":"null_value_counts","required":false,"type":{"type":"map","key-id":121,"key":"int","value-id":122,"value":"long","value-required":true},"doc":"Map of column id to null value count"},{"id":137,"name":"nan_value_counts","required":false,"type":{"type":"map","key-id":138,"key":"int","value-id":139,"value":"long","value-required":true},"doc":"Map of column id to number of NaN values in the column"},{"id":125,"name":"lower_bounds","required":false,"type":{"type":"map","key-id":126,"key":"int","value-id":127,"value":"binary","value-required":true},"doc":"Map of column id to lower bound"},{"id":128,"name":"upper_bounds","required":false,"type":{"type":"map","key-id":129,"key":"int","value-id":130,"value":"binary","value-required":true},"doc":"Map of column id to upper bound"},{"id":131,"name":"key_metadata","required":false,"type":"binary","doc":"Encryption key metadata blob"},{"id":132,"name":"split_offsets","required":false,"type":{"type":"list","element-id":133,"element":"long","element-required":true},"doc":"Splittable offsets"},{"id":135,"name":"equality_ids","required":false,"type":{"type":"list","element-id":136,"element":"int","element-required":true},"doc":"Equality comparison field IDs"},{"id":140,"name":"sort_order_id","required":false,"type":"int","doc":"Sort order ID"}]}}]}partition-spec[]contentdata+#���!VKbJ�2��i�e�M�#E�_״mo�M�!�B1�B:�tz��!��A���2��:�i�t*3���X��!{�E<�y�ŏ�� �AO��Ȟ��A�=
+
+^D���z���P����~<���������מ���z=�21Y
+�����s&^������|�ő�Ev��Ysi�'�T�r�'��{���=G{!��Ð')���!΀�[���w��x��3w~�f.!q|��o���mo���->t.z���tO�xe�S�}�#�ћH��3��B�]��F�o�;Z�ˍ��F�g��Q�����P��i�˓�w��-Լ�Zol�o�{�{�wt�s��
+�B����L��@��:5�Is4O-Z�EZ�eZ�UZ�u��&m�&m�6��.%ԡ.R���fr�����`A
�P�2T�
+5��Mh@ZІt��.���
��48�b��2�?�0͟�y�%�X�G��H"V��L���S��ʝ+�^�k��`�R	�)O�~��rQ53��$Jq„X�bե�1�8JB���8\���L� �y���5L6�ys�[��k�}��N>���%�Z��(�+H�h;��s���2IW����a��ɖW����)�OY��H��Wp�����!|jg>�3?����h���B�ǧ<-��\�bќ�}�P|{��W�V�B�J����U�4��s�6v�V8�x>���j�#�fJ������A����LȌD�l'"jf�e��߶�"�ю�,�8 �}U��]R�~�H.�5����8��q��x���|�
�+#���!VKbJ�2��i