diff --git a/core/src/main/java/org/apache/iceberg/PartitionsTable.java b/core/src/main/java/org/apache/iceberg/PartitionsTable.java index 4e2288a3322b..5a289d756c23 100644 --- a/core/src/main/java/org/apache/iceberg/PartitionsTable.java +++ b/core/src/main/java/org/apache/iceberg/PartitionsTable.java @@ -45,9 +45,11 @@ public class PartitionsTable extends BaseMetadataTable { this.schema = new Schema( Types.NestedField.required(1, "partition", Partitioning.partitionType(table)), - Types.NestedField.required(2, "record_count", Types.LongType.get()), - Types.NestedField.required(3, "file_count", Types.IntegerType.get()), - Types.NestedField.required(4, "spec_id", Types.IntegerType.get())); + Types.NestedField.required(4, "spec_id", Types.IntegerType.get()), + Types.NestedField.required( + 2, "record_count", Types.LongType.get(), "Count of records in data files"), + Types.NestedField.required( + 3, "file_count", Types.IntegerType.get(), "Count of data files")); } @Override @@ -77,7 +79,7 @@ private DataTask task(StaticTableScan scan) { schema(), scan.schema(), partitions, - root -> StaticDataTask.Row.of(root.recordCount, root.fileCount)); + root -> StaticDataTask.Row.of(root.dataRecordCount, root.dataFileCount)); } else { return StaticDataTask.of( io().newInputFile(table().operations().current().metadataFileLocation()), @@ -90,7 +92,7 @@ private DataTask task(StaticTableScan scan) { private static StaticDataTask.Row convertPartition(Partition partition) { return StaticDataTask.Row.of( - partition.key, partition.recordCount, partition.fileCount, partition.specId); + partition.key, partition.specId, partition.dataRecordCount, partition.dataFileCount); } private static Iterable partitions(Table table, StaticTableScan scan) { @@ -220,20 +222,20 @@ Iterable all() { static class Partition { private final StructLike key; - private long recordCount; - private int fileCount; private int specId; + private long dataRecordCount; + private int dataFileCount; Partition(StructLike key) { this.key = key; - this.recordCount = 0; - this.fileCount = 0; this.specId = 0; + this.dataRecordCount = 0; + this.dataFileCount = 0; } void update(DataFile file) { - this.recordCount += file.recordCount(); - this.fileCount += 1; + this.dataRecordCount += file.recordCount(); + this.dataFileCount += 1; this.specId = file.specId(); } } diff --git a/core/src/test/java/org/apache/iceberg/TestMetadataTableScans.java b/core/src/test/java/org/apache/iceberg/TestMetadataTableScans.java index 1ba141f7b632..c45da54f03e6 100644 --- a/core/src/test/java/org/apache/iceberg/TestMetadataTableScans.java +++ b/core/src/test/java/org/apache/iceberg/TestMetadataTableScans.java @@ -332,7 +332,8 @@ public void testPartitionsTableScanWithProjection() { Table partitionsTable = new PartitionsTable(table); Types.StructType expected = - new Schema(required(3, "file_count", Types.IntegerType.get())).asStruct(); + new Schema(required(3, "file_count", Types.IntegerType.get(), "Count of data files")) + .asStruct(); TableScan scanWithProjection = partitionsTable.newScan().select("file_count"); Assert.assertEquals(expected, scanWithProjection.schema().asStruct()); diff --git a/spark/v2.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java b/spark/v2.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java index 07d77dc99b8c..bf7e6e0960a0 100644 --- a/spark/v2.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java +++ b/spark/v2.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java @@ -1234,8 +1234,8 @@ public void testUnpartitionedPartitionsTable() { Types.StructType expectedSchema = Types.StructType.of( - required(2, "record_count", Types.LongType.get()), - required(3, "file_count", Types.IntegerType.get())); + required(2, "record_count", Types.LongType.get(), "Count of records in data files"), + required(3, "file_count", Types.IntegerType.get(), "Count of data files")); Table partitionsTable = loadTable(tableIdentifier, "partitions"); diff --git a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java index c51470f19c93..634b53c3d9b3 100644 --- a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java +++ b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java @@ -1232,8 +1232,8 @@ public void testUnpartitionedPartitionsTable() { Types.StructType expectedSchema = Types.StructType.of( - required(2, "record_count", Types.LongType.get()), - required(3, "file_count", Types.IntegerType.get())); + required(2, "record_count", Types.LongType.get(), "Count of records in data files"), + required(3, "file_count", Types.IntegerType.get(), "Count of data files")); Table partitionsTable = loadTable(tableIdentifier, "partitions"); diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java index 361011dd1ce7..4f3e96d0e1cd 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java @@ -1249,8 +1249,8 @@ public void testUnpartitionedPartitionsTable() { Types.StructType expectedSchema = Types.StructType.of( - required(2, "record_count", Types.LongType.get()), - required(3, "file_count", Types.IntegerType.get())); + required(2, "record_count", Types.LongType.get(), "Count of records in data files"), + required(3, "file_count", Types.IntegerType.get(), "Count of data files")); Table partitionsTable = loadTable(tableIdentifier, "partitions"); diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java index 0f6ae3f20d77..480fe556ad71 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java @@ -1252,8 +1252,8 @@ public void testUnpartitionedPartitionsTable() { Types.StructType expectedSchema = Types.StructType.of( - required(2, "record_count", Types.LongType.get()), - required(3, "file_count", Types.IntegerType.get())); + required(2, "record_count", Types.LongType.get(), "Count of records in data files"), + required(3, "file_count", Types.IntegerType.get(), "Count of data files")); Table partitionsTable = loadTable(tableIdentifier, "partitions");