Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion core/src/main/java/org/apache/iceberg/AllManifestsTable.java
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ public class AllManifestsTable extends BaseMetadataTable {
Types.NestedField.required(11, "contains_nan", Types.BooleanType.get()),
Types.NestedField.optional(12, "lower_bound", Types.StringType.get()),
Types.NestedField.optional(13, "upper_bound", Types.StringType.get())
)))
))),
Types.NestedField.optional(14, "content", Types.IntegerType.get(), "Contents of the manifest: 0=data, 1=deletes")
);

AllManifestsTable(TableOperations ops, Table table) {
Expand Down
6 changes: 4 additions & 2 deletions core/src/main/java/org/apache/iceberg/ManifestsTable.java
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ public class ManifestsTable extends BaseMetadataTable {
Types.NestedField.required(11, "contains_nan", Types.BooleanType.get()),
Types.NestedField.optional(12, "lower_bound", Types.StringType.get()),
Types.NestedField.optional(13, "upper_bound", Types.StringType.get())
)))
))),
Types.NestedField.required(14, "content", Types.IntegerType.get(), "Contents of the manifest: 0=data, 1=deletes")
);

private final PartitionSpec spec;
Expand Down Expand Up @@ -94,7 +95,8 @@ static StaticDataTask.Row manifestFileToRow(PartitionSpec spec, ManifestFile man
manifest.addedFilesCount(),
manifest.existingFilesCount(),
manifest.deletedFilesCount(),
partitionSummariesToRows(spec, manifest.partitions())
partitionSummariesToRows(spec, manifest.partitions()),
manifest.content().id()
);
}

Expand Down
63 changes: 63 additions & 0 deletions core/src/test/java/org/apache/iceberg/TestManifestReader.java
Original file line number Diff line number Diff line change
Expand Up @@ -145,4 +145,67 @@ public void testDeleteFilePositions() throws IOException {
}
}
}

@Test
public void testReadDataManifestTable() throws IOException {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think that these tests should be in this class. This suite tests the manifest reader, not the manifest metadata tables. Why not put these tests in TestIcebergSourceTablesBase?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this test has covered the use cases.

PartitionSpec spec = PartitionSpec.builderFor(table.schema())
.bucket("id", 8)
.bucket("data", 16)
.build();
table.ops().commit(table.ops().current(), table.ops().current().updatePartitionSpec(spec));

ManifestFile manifestFile = writeManifest(1000L, FILE_A);
StaticDataTask.Row actual = ManifestsTable.manifestFileToRow(spec, manifestFile);

Assert.assertEquals(manifestFile.path(), actual.get(0, String.class));
Assert.assertEquals(manifestFile.length(), (long) actual.get(1, Long.class));
Assert.assertEquals(manifestFile.partitionSpecId(), (int) actual.get(2, Integer.class));
Assert.assertEquals(1000L, (long) actual.get(3, Long.class));
Assert.assertEquals(1, (int) actual.get(4, Integer.class));
Assert.assertEquals(0, (int) actual.get(5, Integer.class));
Assert.assertEquals(0, (int) actual.get(6, Integer.class));

List<StaticDataTask.Row> summary = actual.get(7, List.class);
Assert.assertEquals(2, summary.size());
Assert.assertEquals(false, summary.get(0).get(0, Boolean.class));
Assert.assertEquals(false, summary.get(0).get(1, Boolean.class));
Assert.assertEquals("0", summary.get(0).get(2, String.class));
Assert.assertEquals("0", summary.get(0).get(3, String.class));
Assert.assertEquals(true, summary.get(1).get(0, Boolean.class));
Assert.assertEquals(false, summary.get(1).get(1, Boolean.class));
Assert.assertEquals("null", summary.get(1).get(2, String.class));
Assert.assertEquals("null", summary.get(1).get(3, String.class));

Assert.assertEquals(ManifestContent.DATA.id(), (int) actual.get(8, Integer.class));
}

@Test
public void testReadDeletedManifestTable() throws IOException {
Assume.assumeTrue("Delete files only work for format version 2", formatVersion == 2);
PartitionSpec spec = PartitionSpec.builderFor(table.schema())
.bucket("id", 8)
.bucket("data", 16)
.build();
table.ops().commit(table.ops().current(), table.ops().current().updatePartitionSpec(spec));

ManifestFile manifestFile = writeDeleteManifest(formatVersion, 1000L, FILE_A_DELETES);
StaticDataTask.Row actual = ManifestsTable.manifestFileToRow(spec, manifestFile);

Assert.assertEquals(manifestFile.path(), actual.get(0, String.class));
Assert.assertEquals(manifestFile.length(), (long) actual.get(1, Long.class));
Assert.assertEquals(manifestFile.partitionSpecId(), (int) actual.get(2, Integer.class));
Assert.assertEquals(1000L, (long) actual.get(3, Long.class));
Assert.assertEquals(1, (int) actual.get(4, Integer.class));
Assert.assertEquals(0, (int) actual.get(5, Integer.class));
Assert.assertEquals(0, (int) actual.get(6, Integer.class));

List<StaticDataTask.Row> summary = actual.get(7, List.class);
Assert.assertEquals(1, summary.size());
Assert.assertEquals(false, summary.get(0).get(0, Boolean.class));
Assert.assertEquals(false, summary.get(0).get(1, Boolean.class));
Assert.assertEquals("0", summary.get(0).get(2, String.class));
Assert.assertEquals("0", summary.get(0).get(3, String.class));

Assert.assertEquals(ManifestContent.DELETES.id(), (int) actual.get(8, Integer.class));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -717,6 +717,7 @@ public void testManifestsTable() {
.set("upper_bound", "1")
.build()
))
.set("content", manifest.content().id())
.build()
);

Expand Down Expand Up @@ -772,6 +773,7 @@ public void testAllManifestsTable() {
.set("upper_bound", "1")
.build()
))
.set("content", manifest.content().id())
.build()
));

Expand Down