Skip to content

feat(plugin-iceberg): Add Iceberg metadata table $metadata_log_entries #24302

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
agrawalreetika merged 1 commit into from
Dec 16, 2025
Merged

feat(plugin-iceberg): Add Iceberg metadata table $metadata_log_entries #24302

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions presto-docs/src/main/sphinx/connector/iceberg.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -936,6 +936,22 @@ Details about Iceberg references including branches and tags. For more informati
testBranch | BRANCH | 3374797416068698476 | NULL | NULL | NULL
testTag | TAG | 4686954189838128572 | 10 | NULL | NULL

``$metadata_log_entries`` Table
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Provides metadata log entries for the table.

.. code-block:: sql

SELECT * FROM "region$metadata_log_entries";

.. code-block:: text

timestamp | file | latest_snapshot_id | latest_schema_id | latest_sequence_number
-------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+---------------------+------------------+------------------------
2024-12-28 23:41:30.451 Asia/Kolkata | hdfs://localhost:9000/user/hive/warehouse/iceberg_schema.db/region1/metadata/00000-395385ba-3b69-47a7-9c5b-61d056de55c6.metadata.json | 5983271822201743253 | 0 | 1
2024-12-28 23:42:42.207 Asia/Kolkata | hdfs://localhost:9000/user/hive/warehouse/iceberg_schema.db/region1/metadata/00001-61151efc-0e01-4a47-a5e6-7b72749cc4a8.metadata.json | 5841566266546816471 | 0 | 2
2024-12-28 23:42:47.591 Asia/Kolkata | hdfs://localhost:9000/user/hive/warehouse/iceberg_schema.db/region1/metadata/00002-d4a9c326-5053-4a26-9082-d9fbf1d6cd14.metadata.json | 6894018661156805064 | 0 | 3

Presto C++ Support
^^^^^^^^^^^^^^^^^^

Expand Down
2 changes: 2 additions & 0 deletions presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -487,6 +487,8 @@ protected Optional<SystemTable> getIcebergSystemTable(SchemaTableName tableName,
return Optional.of(new PropertiesTable(systemTableName, table));
case REFS:
return Optional.of(new RefsTable(systemTableName, table));
case METADATA_LOG_ENTRIES:
return Optional.of(new MetadataLogTable(systemTableName, table));
}
return Optional.empty();
}
Expand Down
3 changes: 2 additions & 1 deletion presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableName.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import static com.facebook.presto.iceberg.IcebergTableType.FILES;
import static com.facebook.presto.iceberg.IcebergTableType.HISTORY;
import static com.facebook.presto.iceberg.IcebergTableType.MANIFESTS;
import static com.facebook.presto.iceberg.IcebergTableType.METADATA_LOG_ENTRIES;
import static com.facebook.presto.iceberg.IcebergTableType.PARTITIONS;
import static com.facebook.presto.iceberg.IcebergTableType.PROPERTIES;
import static com.facebook.presto.iceberg.IcebergTableType.REFS;
Expand All @@ -51,7 +52,7 @@ public class IcebergTableName

private final Optional<Long> changelogEndSnapshot;

private static final Set<IcebergTableType> SYSTEM_TABLES = Sets.immutableEnumSet(FILES, MANIFESTS, PARTITIONS, HISTORY, SNAPSHOTS, PROPERTIES, REFS);
private static final Set<IcebergTableType> SYSTEM_TABLES = Sets.immutableEnumSet(FILES, MANIFESTS, PARTITIONS, HISTORY, SNAPSHOTS, PROPERTIES, REFS, METADATA_LOG_ENTRIES);

@JsonCreator
public IcebergTableName(
Expand Down
1 change: 1 addition & 0 deletions presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ public enum IcebergTableType
PARTITIONS(true),
FILES(true),
REFS(true),
METADATA_LOG_ENTRIES(true),
PROPERTIES(true),
CHANGELOG(true),
EQUALITY_DELETES(true),
Expand Down
145 changes: 145 additions & 0 deletions presto-iceberg/src/main/java/com/facebook/presto/iceberg/MetadataLogTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.iceberg;

import com.facebook.presto.common.predicate.TupleDomain;
import com.facebook.presto.common.type.Type;
import com.facebook.presto.spi.ColumnMetadata;
import com.facebook.presto.spi.ConnectorSession;
import com.facebook.presto.spi.ConnectorTableMetadata;
import com.facebook.presto.spi.InMemoryRecordSet;
import com.facebook.presto.spi.RecordCursor;
import com.facebook.presto.spi.SchemaTableName;
import com.facebook.presto.spi.SystemTable;
import com.facebook.presto.spi.connector.ConnectorTransactionHandle;
import com.google.common.collect.ImmutableList;
import org.apache.iceberg.BaseTable;
import org.apache.iceberg.Snapshot;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableMetadata;
import org.apache.iceberg.TableMetadata.MetadataLogEntry;

import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.stream.Collectors;

import static com.facebook.presto.common.type.BigintType.BIGINT;
import static com.facebook.presto.common.type.DateTimeEncoding.packDateTimeWithZone;
import static com.facebook.presto.common.type.IntegerType.INTEGER;
import static com.facebook.presto.common.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE;
import static com.facebook.presto.common.type.VarcharType.VARCHAR;
import static java.util.Objects.requireNonNull;
import static org.apache.iceberg.util.SnapshotUtil.snapshotIdAsOfTime;

public class MetadataLogTable
implements SystemTable
{
private final ConnectorTableMetadata tableMetadata;
private final Table icebergTable;

private static final List<ColumnMetadata> COLUMN_DEFINITIONS = ImmutableList.<ColumnMetadata>builder()
.add(ColumnMetadata.builder().setName("timestamp").setType(TIMESTAMP_WITH_TIME_ZONE).build())
.add(ColumnMetadata.builder().setName("file").setType(VARCHAR).build())
.add(ColumnMetadata.builder().setName("latest_snapshot_id").setType(BIGINT).build())
.add(ColumnMetadata.builder().setName("latest_schema_id").setType(INTEGER).build())
.add(ColumnMetadata.builder().setName("latest_sequence_number").setType(BIGINT).build())
.build();

private static final List<Type> COLUMN_TYPES = COLUMN_DEFINITIONS.stream()
.map(ColumnMetadata::getType)
.collect(Collectors.toList());

public MetadataLogTable(SchemaTableName tableName, Table icebergTable)
{
tableMetadata = new ConnectorTableMetadata(requireNonNull(tableName, "tableName is null"), COLUMN_DEFINITIONS);
this.icebergTable = requireNonNull(icebergTable, "icebergTable is null");
}

@Override
public Distribution getDistribution()
{
return Distribution.SINGLE_COORDINATOR;
}

@Override
public ConnectorTableMetadata getTableMetadata()
{
return tableMetadata;
}

@Override
public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain<Integer> constraint)
{
Iterable<List<?>> rowIterable = () -> new Iterator<List<?>>()
{
private final Iterator<MetadataLogEntry> metadataLogEntriesIterator = ((BaseTable) icebergTable).operations().current().previousFiles().iterator();
private boolean addedLatestEntry;

@Override
public boolean hasNext()
{
return metadataLogEntriesIterator.hasNext() || !addedLatestEntry;
}

@Override
public List<?> next()
{
if (metadataLogEntriesIterator.hasNext()) {
return processMetadataLogEntries(session, metadataLogEntriesIterator.next());
}
if (!addedLatestEntry) {
addedLatestEntry = true;
TableMetadata currentMetadata = ((BaseTable) icebergTable).operations().current();
return buildLatestMetadataRow(session, currentMetadata);
}
throw new NoSuchElementException();
}
};
return new InMemoryRecordSet(COLUMN_TYPES, rowIterable).cursor();
}

private List<?> processMetadataLogEntries(ConnectorSession session, MetadataLogEntry metadataLogEntry)
{
Long snapshotId = null;
Snapshot snapshot = null;
try {
snapshotId = snapshotIdAsOfTime(icebergTable, metadataLogEntry.timestampMillis());
snapshot = icebergTable.snapshot(snapshotId);
}
catch (IllegalArgumentException ignored) {
// Implies this metadata file was created during table creation
}
return addRow(session, metadataLogEntry.timestampMillis(), metadataLogEntry.file(), snapshotId, snapshot);
}

private List<?> buildLatestMetadataRow(ConnectorSession session, TableMetadata metadata)
{
Snapshot latestSnapshot = icebergTable.currentSnapshot();
Long latestSnapshotId = (latestSnapshot != null) ? latestSnapshot.snapshotId() : null;

return addRow(session, metadata.lastUpdatedMillis(), metadata.metadataFileLocation(), latestSnapshotId, latestSnapshot);
}

private List<?> addRow(ConnectorSession session, long timestampMillis, String fileLocation, Long snapshotId, Snapshot snapshot)
{
return Arrays.asList(
packDateTimeWithZone(timestampMillis, session.getSqlFunctionProperties().getTimeZoneKey()),
fileLocation,
snapshotId,
snapshot != null ? snapshot.schemaId() : null,
snapshot != null ? snapshot.sequenceNumber() : null);
}
}
75 changes: 75 additions & 0 deletions presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,11 @@
import java.lang.reflect.Field;
import java.net.URI;
import java.nio.ByteBuffer;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
Expand Down Expand Up @@ -145,9 +148,11 @@
import static com.facebook.presto.common.type.IntegerType.INTEGER;
import static com.facebook.presto.common.type.RealType.REAL;
import static com.facebook.presto.common.type.TimeZoneKey.UTC_KEY;
import static com.facebook.presto.common.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE;
import static com.facebook.presto.common.type.VarcharType.VARCHAR;
import static com.facebook.presto.hive.BaseHiveColumnHandle.ColumnType.SYNTHESIZED;
import static com.facebook.presto.hive.HiveCommonSessionProperties.PARQUET_BATCH_READ_OPTIMIZATION_ENABLED;
import static com.facebook.presto.iceberg.CatalogType.HADOOP;
import static com.facebook.presto.iceberg.FileContent.EQUALITY_DELETES;
import static com.facebook.presto.iceberg.FileContent.POSITION_DELETES;
import static com.facebook.presto.iceberg.FileFormat.ORC;
Expand Down Expand Up @@ -186,6 +191,7 @@
import static org.apache.iceberg.types.Type.TypeID.TIME;
import static org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_1_0;
import static org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_2_0;
import static org.assertj.core.api.Assertions.assertThat;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertNotNull;
Expand Down Expand Up @@ -2205,6 +2211,75 @@ public void testRefsTable()
assertQuery("SELECT * FROM test_table_references FOR SYSTEM_VERSION AS OF 'testTag' where id1=1", "VALUES(1, NULL)");
}

@Test
public void testMetadataLogTable()
{
try {
assertUpdate("CREATE TABLE test_table_metadatalog (id1 BIGINT, id2 BIGINT)");
assertQuery("SELECT count(*) FROM \"test_table_metadatalog$metadata_log_entries\"", "VALUES 1");
//metadata file created at table creation
assertQuery("SELECT latest_snapshot_id FROM \"test_table_metadatalog$metadata_log_entries\"", "VALUES NULL");

assertUpdate("INSERT INTO test_table_metadatalog VALUES (0, 00), (1, 10), (2, 20)", 3);
Table icebergTable = loadTable("test_table_metadatalog");
Snapshot latestSnapshot = icebergTable.currentSnapshot();
assertQuery("SELECT count(*) FROM \"test_table_metadatalog$metadata_log_entries\"", "VALUES 2");
assertQuery("SELECT latest_snapshot_id FROM \"test_table_metadatalog$metadata_log_entries\" order by timestamp DESC limit 1", "values " + latestSnapshot.snapshotId());
}
finally {
assertUpdate("DROP TABLE IF EXISTS test_table_metadatalog");
}
}
Comment on lines +2214 to +2232
Copy link
Copy Markdown
Member

@hantangwangd hantangwangd Jan 1, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it convenient to add some test cases considering different timezone and legacyTimestamp, and verify the output column timestamp?

Copy link
Copy Markdown
Member Author

@agrawalreetika agrawalreetika Jan 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@hantangwangd Could you please provide me some example around which type of testcases would fit in here considering different timezone?
I just looked at other metadata tables with timestamp column, but couldn't find any example around same.

Copy link
Copy Markdown
Member

@hantangwangd hantangwangd Jan 4, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Refer to Iceberg's test case, I think we can add some tests similar with the following code:

Session session = sessionWithTimezone(zoneId);
assertUpdate(session, "CREATE TABLE test_table_metadatalog (id1 BIGINT, id2 BIGINT)");
assertQuery(session, "SELECT count(*) FROM \"test_table_metadatalog$metadata_log_entries\"", "VALUES 1");
Table icebergTable = loadTable("test_table_metadatalog");
TableMetadata tableMetadata = ((HasTableOperations) icebergTable).operations().current();
ZonedDateTime zonedDateTime1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(tableMetadata.lastUpdatedMillis()), ZoneId.of(zoneId));
String metadataFileLocation1 = "file:" + tableMetadata.metadataFileLocation();

assertUpdate(session, "INSERT INTO test_table_metadatalog VALUES (0, 00), (1, 10), (2, 20)", 3);
tableMetadata = ((HasTableOperations) icebergTable).operations().refresh();
ZonedDateTime zonedDateTime2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(tableMetadata.lastUpdatedMillis()), ZoneId.of(zoneId));
String metadataFileLocation2 = "file:" + tableMetadata.metadataFileLocation();
Snapshot latestSnapshot = tableMetadata.currentSnapshot();

MaterializedResult result = getQueryRunner().execute(session, "SELECT * FROM \"test_table_metadatalog$metadata_log_entries\"");
assertThat(result).hasSize(2);
assertThat(result)
        .anySatisfy(row -> assertThat(row)
                .isEqualTo(new MaterializedRow(MaterializedResult.DEFAULT_PRECISION, zonedDateTime1, metadataFileLocation1, null, null, null)))
        .anySatisfy(row -> assertThat(row)
                .isEqualTo(new MaterializedRow(MaterializedResult.DEFAULT_PRECISION, zonedDateTime2, metadataFileLocation2, latestSnapshot.snapshotId(), latestSnapshot.schemaId(), latestSnapshot.sequenceNumber())));

And test it under different zoneIds.


@DataProvider(name = "timezoneId")
public Object[][] getTimezonesId()
{
return new Object[][]{{"UTC"}, {"America/Los_Angeles"}, {"Asia/Shanghai"}, {"Asia/Kolkata"}, {"America/Bahia_Banderas"}, {"Europe/Brussels"}};
}

@Test(dataProvider = "timezoneId")
public void testMetadataLogTableWithTimeZoneId(String zoneId)
{
try {
Session sessionForTimeZone = Session.builder(getSession())
.setTimeZoneKey(TimeZoneKey.getTimeZoneKey(zoneId)).build();

assertUpdate(sessionForTimeZone, "CREATE TABLE test_table_metadatalog_tz_id (id1 BIGINT, id2 BIGINT)");
assertQuery(sessionForTimeZone, "SELECT count(*) FROM \"test_table_metadatalog_tz_id$metadata_log_entries\"", "VALUES 1");
assertQuery(sessionForTimeZone, "SELECT latest_snapshot_id FROM \"test_table_metadatalog_tz_id$metadata_log_entries\"", "VALUES NULL");
Table icebergTable = loadTable("test_table_metadatalog_tz_id");
TableMetadata tableMetadata = ((BaseTable) icebergTable).operations().current();
ZonedDateTime zonedDateTime1 = Instant.ofEpochMilli(tableMetadata.lastUpdatedMillis())
.atZone(ZoneId.of(zoneId));
//metadata file created at table creation
String metadataFileLocation1 = tableMetadata.metadataFileLocation();

assertUpdate("INSERT INTO test_table_metadatalog_tz_id VALUES (0, 00), (1, 10), (2, 20)", 3);
icebergTable = loadTable("test_table_metadatalog_tz_id");
tableMetadata = ((BaseTable) icebergTable).operations().current();
ZonedDateTime zonedDateTime2 = Instant.ofEpochMilli(tableMetadata.lastUpdatedMillis())
.atZone(ZoneId.of(zoneId));
//metadata file created after table insertion
String metadataFileLocation2 = tableMetadata.metadataFileLocation();

Snapshot latestSnapshot = icebergTable.currentSnapshot();
assertQuery("SELECT count(*) FROM \"test_table_metadatalog_tz_id$metadata_log_entries\"", "VALUES 2");
assertQuery("SELECT latest_snapshot_id FROM \"test_table_metadatalog_tz_id$metadata_log_entries\" order by timestamp DESC limit 1", "values " + latestSnapshot.snapshotId());

MaterializedResult actual = getQueryRunner().execute(sessionForTimeZone, "SELECT * FROM \"test_table_metadatalog_tz_id$metadata_log_entries\"");
assertThat(actual).hasSize(2);
MaterializedResult expected = resultBuilder(getSession(), TIMESTAMP_WITH_TIME_ZONE, VARCHAR, BIGINT, INTEGER, BIGINT)
.row(zonedDateTime1, metadataFileLocation1, null, null, null)
.row(zonedDateTime2, metadataFileLocation2, latestSnapshot.snapshotId(), latestSnapshot.schemaId(), latestSnapshot.sequenceNumber())
.build();

assertEquals(actual, expected);
}
finally {
assertUpdate("DROP TABLE IF EXISTS test_table_metadatalog_tz_id");
}
}

@Test
public void testAllIcebergType()
{
Expand Down
14 changes: 14 additions & 0 deletions presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergSystemTables.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,20 @@ public void testRefsTable()
assertQuerySucceeds("SELECT * FROM test_schema.\"test_table_multilevel_partitions$refs\"");
}

@Test
public void testMetadataLogTable()
{
assertQuery("SHOW COLUMNS FROM test_schema.\"test_table$metadata_log_entries\"",
"VALUES ('timestamp', 'timestamp with time zone', '', '', null, null, null)," +
"('file', 'varchar', '', '', null, null, 2147483647)," +
"('latest_snapshot_id', 'bigint', '', '', 19, null, null)," +
"('latest_schema_id', 'integer', '', '', 10, null, null)," +
"('latest_sequence_number', 'bigint', '', '', 19, null, null)");
assertQuerySucceeds("SELECT * FROM test_schema.\"test_table$metadata_log_entries\"");

assertQuerySucceeds("SELECT * FROM test_schema.\"test_table_multilevel_partitions$metadata_log_entries\"");
}

@Test
public void testSessionPropertiesInManuallyStartedTransaction()
{
Expand Down
Loading