From b03cf1fed270d6363c3cfba381433df3ac1d84ef Mon Sep 17 00:00:00 2001 From: Apurva Kumar Date: Fri, 20 Mar 2026 13:08:28 -0700 Subject: [PATCH 01/12] [prestissimo][iceberg] Reformat FileContent enum to multi-line for extensibility Summary: - Reformat FileContent enum in presto_protocol_iceberg.h from single-line to multi-line for better readability and future extension. - Add blank line for visual separation before infoColumns initialization. Protocol files are auto-generated from Java sources via chevron. The manual edits here mirror what the generator would produce once the Java changes are landed and the protocol is regenerated. Differential Revision: D97531548 --- .../connector/iceberg/presto_protocol_iceberg.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h index b09cd4903a5bf..7df5628aa4315 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h @@ -78,7 +78,11 @@ void to_json(json& j, const ChangelogSplitInfo& p); void from_json(const json& j, ChangelogSplitInfo& p); } // namespace facebook::presto::protocol::iceberg namespace facebook::presto::protocol::iceberg { -enum class FileContent { DATA, POSITION_DELETES, EQUALITY_DELETES }; +enum class FileContent { + DATA, + POSITION_DELETES, + EQUALITY_DELETES, +}; extern void to_json(json& j, const FileContent& e); extern void from_json(const json& j, FileContent& e); } // namespace facebook::presto::protocol::iceberg From e0de86bda0d65d1c1784c2f60dca63f4fe268027 Mon Sep 17 00:00:00 2001 From: Apurva Kumar Date: Fri, 20 Mar 2026 13:08:28 -0700 Subject: [PATCH 02/12] [presto][iceberg] Wire dataSequenceNumber through protocol layer for equality delete conflict resolution Summary: Wire the dataSequenceNumber field from the Java Presto protocol to the C++ Velox connector layer, enabling server-side sequence number conflict resolution for equality delete files. Changes: - Add dataSequenceNumber field to IcebergSplit protocol (Java + C++) - Parse dataSequenceNumber in IcebergPrestoToVeloxConnector and pass it through HiveIcebergSplit to IcebergSplitReader - Add const qualifiers to local variables for code clarity Differential Revision: D97531547 --- .../presto/iceberg/delete/DeleteFile.java | 17 +++++++++++++++-- .../IcebergPrestoToVeloxConnector.cpp | 15 ++++++++++++--- .../iceberg/presto_protocol_iceberg.cpp | 14 ++++++++++++++ .../connector/iceberg/presto_protocol_iceberg.h | 1 + 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/delete/DeleteFile.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/delete/DeleteFile.java index 16b4943000651..691d073f1a90a 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/delete/DeleteFile.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/delete/DeleteFile.java @@ -42,6 +42,7 @@ public final class DeleteFile private final List equalityFieldIds; private final Map lowerBounds; private final Map upperBounds; + private final long dataSequenceNumber; public static DeleteFile fromIceberg(org.apache.iceberg.DeleteFile deleteFile) { @@ -50,6 +51,8 @@ public static DeleteFile fromIceberg(org.apache.iceberg.DeleteFile deleteFile) Map upperBounds = firstNonNull(deleteFile.upperBounds(), ImmutableMap.of()) .entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().array().clone())); + long dataSequenceNumber = deleteFile.dataSequenceNumber() != null ? deleteFile.dataSequenceNumber() : 0L; + return new DeleteFile( fromIcebergFileContent(deleteFile.content()), deleteFile.path().toString(), @@ -58,7 +61,8 @@ public static DeleteFile fromIceberg(org.apache.iceberg.DeleteFile deleteFile) deleteFile.fileSizeInBytes(), Optional.ofNullable(deleteFile.equalityFieldIds()).orElseGet(ImmutableList::of), lowerBounds, - upperBounds); + upperBounds, + dataSequenceNumber); } @JsonCreator @@ -70,7 +74,8 @@ public DeleteFile( @JsonProperty("fileSizeInBytes") long fileSizeInBytes, @JsonProperty("equalityFieldIds") List equalityFieldIds, @JsonProperty("lowerBounds") Map lowerBounds, - @JsonProperty("upperBounds") Map upperBounds) + @JsonProperty("upperBounds") Map upperBounds, + @JsonProperty("dataSequenceNumber") long dataSequenceNumber) { this.content = requireNonNull(content, "content is null"); this.path = requireNonNull(path, "path is null"); @@ -80,6 +85,7 @@ public DeleteFile( this.equalityFieldIds = ImmutableList.copyOf(requireNonNull(equalityFieldIds, "equalityFieldIds is null")); this.lowerBounds = ImmutableMap.copyOf(requireNonNull(lowerBounds, "lowerBounds is null")); this.upperBounds = ImmutableMap.copyOf(requireNonNull(upperBounds, "upperBounds is null")); + this.dataSequenceNumber = dataSequenceNumber; } @JsonProperty @@ -130,12 +136,19 @@ public Map getUpperBounds() return upperBounds; } + @JsonProperty + public long getDataSequenceNumber() + { + return dataSequenceNumber; + } + @Override public String toString() { return toStringHelper(this) .addValue(path) .add("records", recordCount) + .add("dataSequenceNumber", dataSequenceNumber) .toString(); } } diff --git a/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp b/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp index 707739048c721..9cadc19a2038c 100644 --- a/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp +++ b/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp @@ -30,6 +30,8 @@ velox::connector::hive::iceberg::FileContent toVeloxFileContent( return velox::connector::hive::iceberg::FileContent::kData; } else if (content == protocol::iceberg::FileContent::POSITION_DELETES) { return velox::connector::hive::iceberg::FileContent::kPositionalDeletes; + } else if (content == protocol::iceberg::FileContent::EQUALITY_DELETES) { + return velox::connector::hive::iceberg::FileContent::kEqualityDeletes; } VELOX_UNSUPPORTED("Unsupported file content: {}", fmt::underlying(content)); } @@ -176,6 +178,9 @@ IcebergPrestoToVeloxConnector::toVeloxSplit( VELOX_CHECK_NOT_NULL( icebergSplit, "Unexpected split type {}", connectorSplit->_type); + const int64 dataSequenceNumber = + icebergSplit->dataSequenceNumber; // NOLINT(facebook-bugprone-unchecked-pointer-access) + std::unordered_map> partitionKeys; for (const auto& entry : icebergSplit->partitionKeys) { partitionKeys.emplace( @@ -205,14 +210,16 @@ IcebergPrestoToVeloxConnector::toVeloxSplit( deleteFile.fileSizeInBytes, std::vector(deleteFile.equalityFieldIds), lowerBounds, - upperBounds); + upperBounds, + deleteFile.dataSequenceNumber); deletes.emplace_back(icebergDeleteFile); } + std::unordered_map infoColumns = { {"$data_sequence_number", - std::to_string(icebergSplit->dataSequenceNumber)}, + std::to_string(dataSequenceNumber)}, {"$path", icebergSplit->path}}; return std::make_unique( @@ -227,7 +234,9 @@ IcebergPrestoToVeloxConnector::toVeloxSplit( nullptr, splitContext->cacheable, deletes, - infoColumns); + infoColumns, + std::nullopt, + dataSequenceNumber); } std::unique_ptr diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp index ec74e80c58192..41d2ded58dfe9 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp @@ -371,6 +371,13 @@ void to_json(json& j, const DeleteFile& p) { "DeleteFile", "Map", "upperBounds"); + to_json_key( + j, + "dataSequenceNumber", + p.dataSequenceNumber, + "DeleteFile", + "int64_t", + "dataSequenceNumber"); } void from_json(const json& j, DeleteFile& p) { @@ -408,6 +415,13 @@ void from_json(const json& j, DeleteFile& p) { "DeleteFile", "Map", "upperBounds"); + from_json_key( + j, + "dataSequenceNumber", + p.dataSequenceNumber, + "DeleteFile", + "int64_t", + "dataSequenceNumber"); } } // namespace facebook::presto::protocol::iceberg namespace facebook::presto::protocol::iceberg { diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h index 7df5628aa4315..7c5826826b1fa 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h @@ -101,6 +101,7 @@ struct DeleteFile { List equalityFieldIds = {}; Map lowerBounds = {}; Map upperBounds = {}; + int64_t dataSequenceNumber = {}; }; void to_json(json& j, const DeleteFile& p); void from_json(const json& j, DeleteFile& p); From 3ebd8e5bb3982815a7bfe52a87d7a4237c1ba17d Mon Sep 17 00:00:00 2001 From: Apurva Kumar Date: Fri, 20 Mar 2026 13:08:28 -0700 Subject: [PATCH 03/12] [presto][iceberg] Add PUFFIN file format support for deletion vector discovery Summary: Iceberg V3 introduces deletion vectors stored as blobs inside Puffin files. Previously, the coordinator's IcebergSplitSource rejected PUFFIN-format delete files with a NOT_SUPPORTED error, preventing V3 deletion vectors from being discovered and sent to workers. This diff: 1. Adds PUFFIN to the FileFormat enum (both presto-trunk and presto-facebook-trunk) so fromIcebergFileFormat() can convert Iceberg's PUFFIN format to Presto's FileFormat.PUFFIN. 2. Removes the PUFFIN rejection check in presto-trunk's IcebergSplitSource.toIcebergSplit(), allowing deletion vector files to flow through to workers. 3. Updates TestIcebergV3 to verify PUFFIN files are accepted rather than rejected at split enumeration time. The C++ worker-side changes (protocol enum + connector conversion) will follow in a separate diff. Differential Revision: D97531557 --- .../facebook/presto/iceberg/FileFormat.java | 6 +++++- .../presto/iceberg/IcebergSplitSource.java | 9 --------- .../presto/iceberg/TestIcebergV3.java | 20 ++++++++++++++++--- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/FileFormat.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/FileFormat.java index df4700bc8db80..0b5c4a45e44bf 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/FileFormat.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/FileFormat.java @@ -26,7 +26,8 @@ public enum FileFormat ORC("orc", true), PARQUET("parquet", true), AVRO("avro", true), - METADATA("metadata.json", false); + METADATA("metadata.json", false), + PUFFIN("puffin", false); private final String ext; private final boolean splittable; @@ -61,6 +62,9 @@ public static FileFormat fromIcebergFileFormat(org.apache.iceberg.FileFormat for case METADATA: prestoFileFormat = METADATA; break; + case PUFFIN: + prestoFileFormat = PUFFIN; + break; default: throw new PrestoException(NOT_SUPPORTED, "Unsupported file format: " + format); } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitSource.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitSource.java index 9d4cd1a615636..98ee9f2693450 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitSource.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitSource.java @@ -18,7 +18,6 @@ import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.ConnectorSplitSource; -import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SplitWeight; import com.facebook.presto.spi.connector.ConnectorPartitionHandle; import com.facebook.presto.spi.schedule.NodeSelectionStrategy; @@ -47,7 +46,6 @@ import static com.facebook.presto.iceberg.IcebergUtil.getTargetSplitSize; import static com.facebook.presto.iceberg.IcebergUtil.metadataColumnsMatchPredicates; import static com.facebook.presto.iceberg.IcebergUtil.partitionDataFromStructLike; -import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterators.limit; import static java.util.Objects.requireNonNull; @@ -126,13 +124,6 @@ private ConnectorSplit toIcebergSplit(FileScanTask task) PartitionSpec spec = task.spec(); Optional partitionData = partitionDataFromStructLike(spec, task.file().partition()); - // Validate no PUFFIN deletion vectors (Iceberg v3 feature not yet supported) - for (org.apache.iceberg.DeleteFile deleteFile : task.deletes()) { - if (deleteFile.format() == org.apache.iceberg.FileFormat.PUFFIN) { - throw new PrestoException(NOT_SUPPORTED, "Iceberg deletion vectors (PUFFIN format) are not supported"); - } - } - // TODO: We should leverage residual expression and convert that to TupleDomain. // The predicate here is used by readers for predicate push down at reader level, // so when we do not use residual expression, we are just wasting CPU cycles diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java index fb28aee470d42..2b9a745bb0183 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java @@ -44,6 +44,7 @@ import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; public class TestIcebergV3 extends AbstractTestQueryFramework @@ -279,10 +280,10 @@ public void testOptimizeOnV3Table() } @Test - public void testPuffinDeletionVectorsNotSupported() + public void testPuffinDeletionVectorsAccepted() throws Exception { - String tableName = "test_puffin_deletion_vectors_not_supported"; + String tableName = "test_puffin_deletion_vectors_accepted"; try { assertUpdate("CREATE TABLE " + tableName + " (id integer, value varchar) WITH (\"format-version\" = '3')"); assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one'), (2, 'two')", 2); @@ -309,7 +310,20 @@ public void testPuffinDeletionVectorsNotSupported() .commit(); } - assertQueryFails("SELECT * FROM " + tableName, "Iceberg deletion vectors.*PUFFIN.*not supported"); + // The PUFFIN delete file is now accepted by the split source (no longer + // throws NOT_SUPPORTED). The query will fail downstream because the fake + // .puffin file doesn't exist on disk, but the important thing is that the + // coordinator no longer rejects it at split enumeration time. + try { + computeActual("SELECT * FROM " + tableName); + } + catch (RuntimeException e) { + // Verify the error is NOT the old "PUFFIN not supported" rejection. + // Other failures (e.g., fake .puffin file not on disk) are acceptable. + assertFalse( + e.getMessage().contains("Iceberg deletion vectors") && e.getMessage().contains("not supported"), + "PUFFIN deletion vectors should be accepted, not rejected: " + e.getMessage()); + } } finally { dropTable(tableName); From 515328e67c4f49d5da30efa6440880fececeb101 Mon Sep 17 00:00:00 2001 From: Apurva Kumar Date: Fri, 20 Mar 2026 13:08:28 -0700 Subject: [PATCH 04/12] [prestissimo][iceberg] Wire PUFFIN file format through C++ protocol and connector layer Summary: This is the C++ counterpart to the Java PUFFIN support diff. It wires the PUFFIN file format through the Prestissimo protocol and connector conversion layer so that Iceberg V3 deletion vector files can be deserialized and handled by native workers. Changes: 1. Adds PUFFIN to the C++ protocol FileFormat enum and its JSON serialization table in presto_protocol_iceberg.{h,cpp}. 2. Handles PUFFIN in toVeloxFileFormat() in IcebergPrestoToVeloxConnector.cpp, mapping it to DWRF as a placeholder since DeletionVectorReader reads raw binary and does not use the DWRF/Parquet reader infrastructure. Differential Revision: D97531555 --- .../IcebergPrestoToVeloxConnector.cpp | 34 +++++++++++++++---- .../iceberg/presto_protocol_iceberg.cpp | 3 +- .../iceberg/presto_protocol_iceberg.h | 2 +- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp b/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp index 9cadc19a2038c..d50489aaaf6a3 100644 --- a/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp +++ b/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp @@ -42,6 +42,14 @@ velox::dwio::common::FileFormat toVeloxFileFormat( return velox::dwio::common::FileFormat::ORC; } else if (format == protocol::iceberg::FileFormat::PARQUET) { return velox::dwio::common::FileFormat::PARQUET; + } else if (format == protocol::iceberg::FileFormat::PUFFIN) { + // PUFFIN is used for Iceberg V3 deletion vectors. The DeletionVectorReader + // reads raw binary from the file and does not use the DWRF/Parquet reader, + // so we map PUFFIN to DWRF as a placeholder — the format value is not + // actually used by the reader. This mapping is only safe for deletion + // vector files; if PUFFIN is encountered for other file content types, + // the DV routing logic in toHiveIcebergSplit() must reclassify it first. + return velox::dwio::common::FileFormat::DWRF; } VELOX_UNSUPPORTED("Unsupported file format: {}", fmt::underlying(format)); } @@ -173,7 +181,7 @@ IcebergPrestoToVeloxConnector::toVeloxSplit( const protocol::ConnectorId& catalogId, const protocol::ConnectorSplit* connectorSplit, const protocol::SplitContext* splitContext) const { - auto icebergSplit = + const auto* icebergSplit = dynamic_cast(connectorSplit); VELOX_CHECK_NOT_NULL( icebergSplit, "Unexpected split type {}", connectorSplit->_type); @@ -196,14 +204,27 @@ IcebergPrestoToVeloxConnector::toVeloxSplit( std::vector deletes; deletes.reserve(icebergSplit->deletes.size()); for (const auto& deleteFile : icebergSplit->deletes) { - std::unordered_map lowerBounds( + const std::unordered_map lowerBounds( deleteFile.lowerBounds.begin(), deleteFile.lowerBounds.end()); - std::unordered_map upperBounds( + const std::unordered_map upperBounds( deleteFile.upperBounds.begin(), deleteFile.upperBounds.end()); - velox::connector::hive::iceberg::IcebergDeleteFile icebergDeleteFile( - toVeloxFileContent(deleteFile.content), + // Iceberg V3 deletion vectors arrive from the coordinator as + // POSITION_DELETES with PUFFIN format. Reclassify them as + // kDeletionVector so that IcebergSplitReader routes them to + // DeletionVectorReader instead of PositionalDeleteFileReader. + velox::connector::hive::iceberg::FileContent veloxContent = + toVeloxFileContent(deleteFile.content); + if (veloxContent == + velox::connector::hive::iceberg::FileContent::kPositionalDeletes && + deleteFile.format == protocol::iceberg::FileFormat::PUFFIN) { + veloxContent = + velox::connector::hive::iceberg::FileContent::kDeletionVector; + } + + const velox::connector::hive::iceberg::IcebergDeleteFile icebergDeleteFile( + veloxContent, deleteFile.path, toVeloxFileFormat(deleteFile.format), deleteFile.recordCount, @@ -218,8 +239,7 @@ IcebergPrestoToVeloxConnector::toVeloxSplit( std::unordered_map infoColumns = { - {"$data_sequence_number", - std::to_string(dataSequenceNumber)}, + {"$data_sequence_number", std::to_string(dataSequenceNumber)}, {"$path", icebergSplit->path}}; return std::make_unique( diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp index 41d2ded58dfe9..0a5a82eaea408 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp @@ -306,7 +306,8 @@ static const std::pair FileFormat_enum_table[] = {FileFormat::ORC, "ORC"}, {FileFormat::PARQUET, "PARQUET"}, {FileFormat::AVRO, "AVRO"}, - {FileFormat::METADATA, "METADATA"}}; + {FileFormat::METADATA, "METADATA"}, + {FileFormat::PUFFIN, "PUFFIN"}}; void to_json(json& j, const FileFormat& e) { static_assert(std::is_enum::value, "FileFormat must be an enum!"); const auto* it = std::find_if( diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h index 7c5826826b1fa..6d1cfd204992c 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h @@ -87,7 +87,7 @@ extern void to_json(json& j, const FileContent& e); extern void from_json(const json& j, FileContent& e); } // namespace facebook::presto::protocol::iceberg namespace facebook::presto::protocol::iceberg { -enum class FileFormat { ORC, PARQUET, AVRO, METADATA }; +enum class FileFormat { ORC, PARQUET, AVRO, METADATA, PUFFIN }; extern void to_json(json& j, const FileFormat& e); extern void from_json(const json& j, FileFormat& e); } // namespace facebook::presto::protocol::iceberg From 635b7adb46dc69cb2914b61f0b2006e14fc28578 Mon Sep 17 00:00:00 2001 From: Apurva Kumar Date: Fri, 20 Mar 2026 13:08:28 -0700 Subject: [PATCH 05/12] [presto][iceberg] Add Iceberg V3 deletion vector write path with DV page sink and compaction procedure Summary: - Add IcebergDeletionVectorPageSink for writing DV files during table maintenance - Add RewriteDeleteFilesProcedure for DV compaction - Wire DV page sink through IcebergCommonModule, IcebergAbstractMetadata, IcebergPageSourceProvider - Add IcebergUpdateablePageSource for DV-aware page source - Update CommitTaskData, IcebergUtil for DV support - Add test coverage in TestIcebergV3 Differential Revision: D97531549 --- .../presto/iceberg/CommitTaskData.java | 45 +- .../iceberg/IcebergAbstractMetadata.java | 28 +- .../presto/iceberg/IcebergCommonModule.java | 2 + .../iceberg/IcebergPageSourceProvider.java | 40 +- .../iceberg/IcebergUpdateablePageSource.java | 10 +- .../facebook/presto/iceberg/IcebergUtil.java | 2 +- .../delete/IcebergDeletionVectorPageSink.java | 219 +++++ .../RewriteDeleteFilesProcedure.java | 356 +++++++ .../presto/iceberg/TestIcebergV3.java | 869 +++++++++++++++++- 9 files changed, 1534 insertions(+), 37 deletions(-) create mode 100644 presto-iceberg/src/main/java/com/facebook/presto/iceberg/delete/IcebergDeletionVectorPageSink.java create mode 100644 presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RewriteDeleteFilesProcedure.java diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/CommitTaskData.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/CommitTaskData.java index f8a625a450f3b..cb471ca37c497 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/CommitTaskData.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/CommitTaskData.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Optional; +import java.util.OptionalLong; import static java.util.Objects.requireNonNull; @@ -30,6 +31,9 @@ public class CommitTaskData private final FileFormat fileFormat; private final Optional referencedDataFile; private final FileContent content; + private final OptionalLong contentOffset; + private final OptionalLong contentSizeInBytes; + private final OptionalLong recordCount; @JsonCreator public CommitTaskData( @@ -40,7 +44,10 @@ public CommitTaskData( @JsonProperty("partitionDataJson") Optional partitionDataJson, @JsonProperty("fileFormat") FileFormat fileFormat, @JsonProperty("referencedDataFile") String referencedDataFile, - @JsonProperty("content") FileContent content) + @JsonProperty("content") FileContent content, + @JsonProperty("contentOffset") OptionalLong contentOffset, + @JsonProperty("contentSizeInBytes") OptionalLong contentSizeInBytes, + @JsonProperty("recordCount") OptionalLong recordCount) { this.path = requireNonNull(path, "path is null"); this.fileSizeInBytes = fileSizeInBytes; @@ -50,6 +57,24 @@ public CommitTaskData( this.fileFormat = requireNonNull(fileFormat, "fileFormat is null"); this.referencedDataFile = Optional.ofNullable(referencedDataFile); this.content = requireNonNull(content, "content is null"); + this.contentOffset = contentOffset != null ? contentOffset : OptionalLong.empty(); + this.contentSizeInBytes = contentSizeInBytes != null ? contentSizeInBytes : OptionalLong.empty(); + this.recordCount = recordCount != null ? recordCount : OptionalLong.empty(); + } + + public CommitTaskData( + String path, + long fileSizeInBytes, + MetricsWrapper metrics, + int partitionSpecId, + Optional partitionDataJson, + FileFormat fileFormat, + String referencedDataFile, + FileContent content) + { + this(path, fileSizeInBytes, metrics, partitionSpecId, partitionDataJson, + fileFormat, referencedDataFile, content, + OptionalLong.empty(), OptionalLong.empty(), OptionalLong.empty()); } @JsonProperty @@ -99,4 +124,22 @@ public FileContent getContent() { return content; } + + @JsonProperty + public OptionalLong getContentOffset() + { + return contentOffset; + } + + @JsonProperty + public OptionalLong getContentSizeInBytes() + { + return contentSizeInBytes; + } + + @JsonProperty + public OptionalLong getRecordCount() + { + return recordCount; + } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java index 9d1c79c3e029b..a4e893767665c 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java @@ -405,12 +405,9 @@ protected static void validateTableForPresto(BaseTable table, Optional tab schema = metadata.schema(); } - // Reject schema default values (initial-default / write-default) - for (Types.NestedField field : schema.columns()) { - if (field.initialDefault() != null || field.writeDefault() != null) { - throw new PrestoException(NOT_SUPPORTED, "Iceberg v3 column default values are not supported"); - } - } + // Iceberg v3 column default values (initial-default / write-default) are supported. + // The Iceberg library handles applying defaults when reading files that were written + // before a column with a default was added via schema evolution. // Reject Iceberg table encryption if (!metadata.encryptionKeys().isEmpty() || snapshot.keyId() != null || metadata.properties().containsKey("encryption.key-id")) { @@ -1524,8 +1521,23 @@ public Optional finishDeleteWithOutput(ConnectorSession .ofPositionDeletes() .withPath(task.getPath()) .withFileSizeInBytes(task.getFileSizeInBytes()) - .withFormat(FileFormat.fromString(task.getFileFormat().name())) - .withMetrics(task.getMetrics().metrics()); + .withFormat(FileFormat.fromString(task.getFileFormat().name())); + + if (task.getFileFormat() == com.facebook.presto.iceberg.FileFormat.PUFFIN) { + builder.withRecordCount(task.getRecordCount().orElseThrow(() -> + new VerifyException("recordCount required for deletion vector"))); + builder.withContentOffset(task.getContentOffset().orElseThrow(() -> + new VerifyException("contentOffset required for deletion vector"))); + builder.withContentSizeInBytes(task.getContentSizeInBytes().orElseThrow(() -> + new VerifyException("contentSizeInBytes required for deletion vector"))); + } + else { + builder.withMetrics(task.getMetrics().metrics()); + } + + if (task.getReferencedDataFile().isPresent()) { + builder.withReferencedDataFile(task.getReferencedDataFile().get()); + } if (!spec.fields().isEmpty()) { String partitionDataJson = task.getPartitionDataJson() diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergCommonModule.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergCommonModule.java index 1d438fdad92af..10991d1f01047 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergCommonModule.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergCommonModule.java @@ -49,6 +49,7 @@ import com.facebook.presto.iceberg.procedure.RegisterTableProcedure; import com.facebook.presto.iceberg.procedure.RemoveOrphanFiles; import com.facebook.presto.iceberg.procedure.RewriteDataFilesProcedure; +import com.facebook.presto.iceberg.procedure.RewriteDeleteFilesProcedure; import com.facebook.presto.iceberg.procedure.RewriteManifestsProcedure; import com.facebook.presto.iceberg.procedure.RollbackToSnapshotProcedure; import com.facebook.presto.iceberg.procedure.RollbackToTimestampProcedure; @@ -195,6 +196,7 @@ protected void setup(Binder binder) procedures.addBinding().toProvider(StatisticsFileCacheInvalidationProcedure.class).in(Scopes.SINGLETON); procedures.addBinding().toProvider(ManifestFileCacheInvalidationProcedure.class).in(Scopes.SINGLETON); procedures.addBinding().toProvider(RewriteDataFilesProcedure.class).in(Scopes.SINGLETON); + procedures.addBinding().toProvider(RewriteDeleteFilesProcedure.class).in(Scopes.SINGLETON); procedures.addBinding().toProvider(RewriteManifestsProcedure.class).in(Scopes.SINGLETON); // for orc diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSourceProvider.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSourceProvider.java index 2b6bc0b6e9f2f..f4d2d013f82c8 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSourceProvider.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSourceProvider.java @@ -45,6 +45,7 @@ import com.facebook.presto.iceberg.delete.DeleteFile; import com.facebook.presto.iceberg.delete.DeleteFilter; import com.facebook.presto.iceberg.delete.IcebergDeletePageSink; +import com.facebook.presto.iceberg.delete.IcebergDeletionVectorPageSink; import com.facebook.presto.iceberg.delete.PositionDeleteFilter; import com.facebook.presto.iceberg.delete.RowPredicate; import com.facebook.presto.memory.context.AggregatedMemoryContext; @@ -70,6 +71,7 @@ import com.facebook.presto.parquet.predicate.Predicate; import com.facebook.presto.parquet.reader.ParquetReader; import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorPageSink; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorSplit; @@ -863,17 +865,33 @@ else if (subColumn.getId() == MERGE_PARTITION_DATA.getId()) { verify(storageProperties.isPresent(), "storageProperties are null"); LocationProvider locationProvider = getLocationProvider(table.getSchemaTableName(), outputPath.get(), storageProperties.get()); - Supplier deleteSinkSupplier = () -> new IcebergDeletePageSink( - partitionSpec, - split.getPartitionDataJson(), - locationProvider, - fileWriterFactory, - hdfsEnvironment, - hdfsContext, - jsonCodec, - session, - split.getPath(), - split.getFileFormat()); + int tableFormatVersion = Integer.parseInt( + storageProperties.get().getOrDefault("format-version", "2")); + Supplier deleteSinkSupplier; + if (tableFormatVersion >= 3) { + deleteSinkSupplier = () -> new IcebergDeletionVectorPageSink( + partitionSpec, + split.getPartitionDataJson(), + locationProvider, + hdfsEnvironment, + hdfsContext, + jsonCodec, + session, + split.getPath()); + } + else { + deleteSinkSupplier = () -> new IcebergDeletePageSink( + partitionSpec, + split.getPartitionDataJson(), + locationProvider, + fileWriterFactory, + hdfsEnvironment, + hdfsContext, + jsonCodec, + session, + split.getPath(), + split.getFileFormat()); + } boolean storeDeleteFilePath = icebergColumns.contains(DELETE_FILE_PATH_COLUMN_HANDLE); Supplier> deleteFilters = memoize(() -> { // If equality deletes are optimized into a join they don't need to be applied here diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUpdateablePageSource.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUpdateablePageSource.java index 8a0bbdd1b16e8..044b5a7a045ba 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUpdateablePageSource.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUpdateablePageSource.java @@ -21,8 +21,8 @@ import com.facebook.presto.common.block.RunLengthEncodedBlock; import com.facebook.presto.hive.HivePartitionKey; import com.facebook.presto.iceberg.delete.DeleteFilter; -import com.facebook.presto.iceberg.delete.IcebergDeletePageSink; import com.facebook.presto.iceberg.delete.RowPredicate; +import com.facebook.presto.spi.ConnectorPageSink; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.UpdatablePageSource; @@ -71,8 +71,8 @@ public class IcebergUpdateablePageSource implements UpdatablePageSource { private final ConnectorPageSource delegate; - private final Supplier deleteSinkSupplier; - private IcebergDeletePageSink positionDeleteSink; + private final Supplier deleteSinkSupplier; + private ConnectorPageSink positionDeleteSink; private final Supplier> deletePredicate; private final Supplier> deleteFilters; @@ -107,7 +107,7 @@ public IcebergUpdateablePageSource( ConnectorPageSource delegate, // represents the columns output by the delegate page source List delegateColumns, - Supplier deleteSinkSupplier, + Supplier deleteSinkSupplier, Supplier> deletePredicate, Supplier> deleteFilters, Supplier updatedRowPageSinkSupplier, @@ -295,7 +295,7 @@ public void updateRows(Page page, List columnValueAndRowIdChannels) public CompletableFuture> finish() { return Optional.ofNullable(positionDeleteSink) - .map(IcebergDeletePageSink::finish) + .map(ConnectorPageSink::finish) .orElseGet(() -> completedFuture(ImmutableList.of())) .thenCombine( Optional.ofNullable(updatedRowPageSink).map(IcebergPageSink::finish) diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java index e06553e66bc65..e568f1e7c6e4c 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java @@ -221,7 +221,7 @@ public final class IcebergUtil { private static final Logger log = Logger.get(IcebergUtil.class); public static final int MIN_FORMAT_VERSION_FOR_DELETE = 2; - public static final int MAX_FORMAT_VERSION_FOR_ROW_LEVEL_OPERATIONS = 2; + public static final int MAX_FORMAT_VERSION_FOR_ROW_LEVEL_OPERATIONS = 3; public static final int MAX_SUPPORTED_FORMAT_VERSION = 3; public static final long DOUBLE_POSITIVE_ZERO = 0x0000000000000000L; diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/delete/IcebergDeletionVectorPageSink.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/delete/IcebergDeletionVectorPageSink.java new file mode 100644 index 0000000000000..d7dbd99318032 --- /dev/null +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/delete/IcebergDeletionVectorPageSink.java @@ -0,0 +1,219 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg.delete; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.presto.common.Page; +import com.facebook.presto.common.block.Block; +import com.facebook.presto.common.type.BigintType; +import com.facebook.presto.hive.HdfsContext; +import com.facebook.presto.hive.HdfsEnvironment; +import com.facebook.presto.iceberg.CommitTaskData; +import com.facebook.presto.iceberg.FileFormat; +import com.facebook.presto.iceberg.HdfsOutputFile; +import com.facebook.presto.iceberg.MetricsWrapper; +import com.facebook.presto.iceberg.PartitionData; +import com.facebook.presto.spi.ConnectorPageSink; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.PrestoException; +import com.google.common.collect.ImmutableList; +import io.airlift.slice.Slice; +import org.apache.hadoop.fs.Path; +import org.apache.iceberg.Metrics; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.io.LocationProvider; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.puffin.Blob; +import org.apache.iceberg.puffin.Puffin; +import org.apache.iceberg.puffin.PuffinWriter; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.CompletableFuture; + +import static com.facebook.presto.iceberg.FileContent.POSITION_DELETES; +import static com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_BAD_DATA; +import static com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_WRITER_CLOSE_ERROR; +import static com.facebook.presto.iceberg.IcebergUtil.partitionDataFromJson; +import static io.airlift.slice.Slices.wrappedBuffer; +import static java.util.Objects.requireNonNull; +import static java.util.UUID.randomUUID; +import static java.util.concurrent.CompletableFuture.completedFuture; + +public class IcebergDeletionVectorPageSink + implements ConnectorPageSink +{ + private static final int SERIAL_COOKIE_NO_RUNCONTAINER = 12346; + + private final PartitionSpec partitionSpec; + private final Optional partitionData; + private final HdfsEnvironment hdfsEnvironment; + private final HdfsContext hdfsContext; + private final JsonCodec jsonCodec; + private final ConnectorSession session; + private final String dataFile; + private final LocationProvider locationProvider; + + private final List collectedPositions = new ArrayList<>(); + + public IcebergDeletionVectorPageSink( + PartitionSpec partitionSpec, + Optional partitionDataAsJson, + LocationProvider locationProvider, + HdfsEnvironment hdfsEnvironment, + HdfsContext hdfsContext, + JsonCodec jsonCodec, + ConnectorSession session, + String dataFile) + { + this.partitionSpec = requireNonNull(partitionSpec, "partitionSpec is null"); + this.partitionData = partitionDataFromJson(partitionSpec, partitionDataAsJson); + this.locationProvider = requireNonNull(locationProvider, "locationProvider is null"); + this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); + this.hdfsContext = requireNonNull(hdfsContext, "hdfsContext is null"); + this.jsonCodec = requireNonNull(jsonCodec, "jsonCodec is null"); + this.session = requireNonNull(session, "session is null"); + this.dataFile = requireNonNull(dataFile, "dataFile is null"); + } + + @Override + public long getCompletedBytes() + { + return 0; + } + + @Override + public long getSystemMemoryUsage() + { + return collectedPositions.size() * (long) Integer.BYTES; + } + + @Override + public long getValidationCpuNanos() + { + return 0; + } + + @Override + public CompletableFuture appendPage(Page page) + { + if (page.getChannelCount() != 1) { + throw new PrestoException(ICEBERG_BAD_DATA, + "Expecting Page with one channel but got " + page.getChannelCount()); + } + + Block block = page.getBlock(0); + for (int i = 0; i < block.getPositionCount(); i++) { + long position = BigintType.BIGINT.getLong(block, i); + collectedPositions.add((int) position); + } + return NOT_BLOCKED; + } + + @Override + public CompletableFuture> finish() + { + if (collectedPositions.isEmpty()) { + return completedFuture(ImmutableList.of()); + } + + Collections.sort(collectedPositions); + + byte[] roaringBitmapBytes = serializeRoaringBitmap(collectedPositions); + + String fileName = "dv-" + randomUUID() + ".puffin"; + Path puffinPath = partitionData + .map(partition -> new Path(locationProvider.newDataLocation(partitionSpec, partition, fileName))) + .orElseGet(() -> new Path(locationProvider.newDataLocation(fileName))); + + OutputFile outputFile = new HdfsOutputFile(puffinPath, hdfsEnvironment, hdfsContext); + + long puffinFileSize; + long blobOffset; + long blobLength; + + try { + PuffinWriter writer = hdfsEnvironment.doAs(session.getUser(), () -> + Puffin.write(outputFile).createdBy("presto").build()); + try { + writer.add(new Blob( + "deletion-vector-v2", + ImmutableList.of(), + 0, + 0, + ByteBuffer.wrap(roaringBitmapBytes))); + hdfsEnvironment.doAs(session.getUser(), () -> { + writer.finish(); + return null; + }); + puffinFileSize = writer.fileSize(); + blobOffset = writer.writtenBlobsMetadata().get(0).offset(); + blobLength = writer.writtenBlobsMetadata().get(0).length(); + } + finally { + hdfsEnvironment.doAs(session.getUser(), () -> { + writer.close(); + return null; + }); + } + } + catch (IOException e) { + throw new PrestoException(ICEBERG_WRITER_CLOSE_ERROR, "Failed to write deletion vector puffin file", e); + } + + CommitTaskData task = new CommitTaskData( + puffinPath.toString(), + puffinFileSize, + new MetricsWrapper(new Metrics(collectedPositions.size(), null, null, null, null)), + partitionSpec.specId(), + partitionData.map(PartitionData::toJson), + FileFormat.PUFFIN, + dataFile, + POSITION_DELETES, + OptionalLong.of(blobOffset), + OptionalLong.of(blobLength), + OptionalLong.of(collectedPositions.size())); + + return completedFuture(ImmutableList.of(wrappedBuffer(jsonCodec.toJsonBytes(task)))); + } + + @Override + public void abort() + { + // Nothing to clean up since we write the Puffin file atomically in finish() + } + + private static byte[] serializeRoaringBitmap(List sortedPositions) + { + int numPositions = sortedPositions.size(); + int dataSize = 4 + 4 + 4 + numPositions * 2; + ByteBuffer buffer = ByteBuffer.allocate(dataSize); + buffer.order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(SERIAL_COOKIE_NO_RUNCONTAINER); + buffer.putInt(1); + buffer.putShort((short) 0); + buffer.putShort((short) (numPositions - 1)); + for (int pos : sortedPositions) { + buffer.putShort((short) (pos & 0xFFFF)); + } + return buffer.array(); + } +} diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RewriteDeleteFilesProcedure.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RewriteDeleteFilesProcedure.java new file mode 100644 index 0000000000000..c2c1d01f2cf73 --- /dev/null +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RewriteDeleteFilesProcedure.java @@ -0,0 +1,356 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg.procedure; + +import com.facebook.presto.iceberg.IcebergAbstractMetadata; +import com.facebook.presto.iceberg.IcebergMetadataFactory; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.classloader.ThreadContextClassLoader; +import com.facebook.presto.spi.procedure.Procedure; +import com.facebook.presto.spi.procedure.Procedure.Argument; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import jakarta.inject.Inject; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.FileMetadata; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.RewriteFiles; +import org.apache.iceberg.Table; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.CloseableIterator; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.puffin.Blob; +import org.apache.iceberg.puffin.BlobMetadata; +import org.apache.iceberg.puffin.Puffin; +import org.apache.iceberg.puffin.PuffinReader; +import org.apache.iceberg.puffin.PuffinWriter; + +import javax.inject.Provider; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.invoke.MethodHandle; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import static com.facebook.presto.common.block.MethodHandleUtil.methodHandle; +import static com.facebook.presto.common.type.StandardTypes.VARCHAR; +import static com.facebook.presto.iceberg.IcebergUtil.getIcebergTable; +import static java.util.Objects.requireNonNull; + +/** + * Procedure to compact deletion vectors (DVs) on V3 Iceberg tables. + * + * When multiple DELETE operations target rows in the same data file, each produces + * a separate DV (Puffin file). This procedure merges all DVs per data file into + * a single consolidated DV, reducing metadata overhead and improving read performance. + * + * Usage: CALL iceberg.system.rewrite_delete_files('schema', 'table') + */ +public class RewriteDeleteFilesProcedure + implements Provider +{ + private static final MethodHandle REWRITE_DELETE_FILES = methodHandle( + RewriteDeleteFilesProcedure.class, + "rewriteDeleteFiles", + ConnectorSession.class, + String.class, + String.class); + + private final IcebergMetadataFactory metadataFactory; + + @Inject + public RewriteDeleteFilesProcedure(IcebergMetadataFactory metadataFactory) + { + this.metadataFactory = requireNonNull(metadataFactory, "metadataFactory is null"); + } + + @Override + public Procedure get() + { + return new Procedure( + "system", + "rewrite_delete_files", + ImmutableList.of( + new Argument("schema", VARCHAR), + new Argument("table_name", VARCHAR)), + REWRITE_DELETE_FILES.bindTo(this)); + } + + public void rewriteDeleteFiles(ConnectorSession clientSession, String schemaName, String tableName) + { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(getClass().getClassLoader())) { + SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName); + IcebergAbstractMetadata metadata = (IcebergAbstractMetadata) metadataFactory.create(); + Table icebergTable = getIcebergTable(metadata, clientSession, schemaTableName); + + int formatVersion = ((BaseTable) icebergTable).operations().current().formatVersion(); + if (formatVersion < 3) { + return; + } + + // Group delete files by their referenced data file + Map> dvsByDataFile = new HashMap<>(); + Set allDeleteFiles = new HashSet<>(); + + try (CloseableIterable tasks = icebergTable.newScan().planFiles()) { + CloseableIterator iterator = tasks.iterator(); + while (iterator.hasNext()) { + FileScanTask task = iterator.next(); + String dataFilePath = task.file().path().toString(); + for (DeleteFile deleteFile : task.deletes()) { + if (deleteFile.format() == FileFormat.PUFFIN && + deleteFile.content() == FileContent.POSITION_DELETES) { + dvsByDataFile.computeIfAbsent(dataFilePath, k -> new ArrayList<>()).add(deleteFile); + allDeleteFiles.add(deleteFile); + } + } + } + iterator.close(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + + // Find data files with multiple DVs that need compaction + Set filesToRemove = new HashSet<>(); + Set filesToAdd = new HashSet<>(); + + for (Map.Entry> entry : dvsByDataFile.entrySet()) { + List dvs = entry.getValue(); + if (dvs.size() <= 1) { + continue; + } + + String dataFilePath = entry.getKey(); + + // Merge roaring bitmaps from all DVs for this data file + Set mergedPositions = new HashSet<>(); + for (DeleteFile dv : dvs) { + readDeletionVectorPositions(icebergTable, dv, mergedPositions); + filesToRemove.add(dv); + } + + // Write consolidated DV + DeleteFile mergedDv = writeMergedDeletionVector( + icebergTable, + dvs.get(0), + dataFilePath, + mergedPositions); + filesToAdd.add(mergedDv); + } + + if (filesToRemove.isEmpty()) { + metadata.commit(); + return; + } + + // Commit the rewrite: remove old DVs, add merged DVs + RewriteFiles rewriteFiles = icebergTable.newRewrite() + .rewriteFiles(ImmutableSet.of(), filesToRemove, ImmutableSet.of(), filesToAdd); + rewriteFiles.commit(); + metadata.commit(); + } + } + + private void readDeletionVectorPositions(Table table, DeleteFile dv, Set positions) + { + InputFile inputFile = table.io().newInputFile(dv.path().toString()); + try (PuffinReader reader = Puffin.read(inputFile).build()) { + List blobMetadataList = reader.fileMetadata().blobs(); + if (blobMetadataList.isEmpty()) { + return; + } + for (org.apache.iceberg.puffin.Pair pair : reader.readAll(blobMetadataList)) { + ByteBuffer blobData = pair.second(); + deserializeRoaringBitmap(blobData, positions); + } + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private DeleteFile writeMergedDeletionVector( + Table table, + DeleteFile templateDv, + String dataFilePath, + Set mergedPositions) + { + List sortedPositions = new ArrayList<>(mergedPositions); + sortedPositions.sort(Integer::compareTo); + byte[] roaringBytes = serializeRoaringBitmap(sortedPositions); + + String fileName = "dv-" + UUID.randomUUID() + ".puffin"; + String dvPath = table.location() + "/data/" + fileName; + OutputFile outputFile = table.io().newOutputFile(dvPath); + + long puffinFileSize; + long blobOffset; + long blobLength; + try { + PuffinWriter writer = Puffin.write(outputFile).createdBy("presto").build(); + writer.add(new Blob( + "deletion-vector-v2", + ImmutableList.of(), + 0, + 0, + ByteBuffer.wrap(roaringBytes))); + writer.finish(); + puffinFileSize = writer.fileSize(); + blobOffset = writer.writtenBlobsMetadata().get(0).offset(); + blobLength = writer.writtenBlobsMetadata().get(0).length(); + writer.close(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + + return FileMetadata.deleteFileBuilder(table.specs().get(templateDv.specId())) + .ofPositionDeletes() + .withPath(dvPath) + .withFileSizeInBytes(puffinFileSize) + .withFormat(FileFormat.PUFFIN) + .withRecordCount(sortedPositions.size()) + .withContentSizeInBytes(blobLength) + .withContentOffset(blobOffset) + .withReferencedDataFile(dataFilePath) + .build(); + } + + private static void deserializeRoaringBitmap(ByteBuffer buffer, Set positions) + { + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes); + java.nio.ByteBuffer buf = java.nio.ByteBuffer.wrap(bytes).order(java.nio.ByteOrder.LITTLE_ENDIAN); + + int cookie = buf.getInt(); + boolean isRunContainer = (cookie & 0xFFFF) == 12347; + int numContainers; + if (isRunContainer) { + numContainers = (cookie >>> 16) + 1; + // skip run bitmap + int runBitmapBytes = (numContainers + 7) / 8; + buf.position(buf.position() + runBitmapBytes); + } + else if ((cookie & 0xFFFF) == 12346) { + numContainers = (cookie >>> 16) + 1; + } + else { + return; + } + + int[] keys = new int[numContainers]; + int[] cardinalities = new int[numContainers]; + for (int i = 0; i < numContainers; i++) { + keys[i] = Short.toUnsignedInt(buf.getShort()); + cardinalities[i] = Short.toUnsignedInt(buf.getShort()) + 1; + } + + for (int i = 0; i < numContainers; i++) { + int highBits = keys[i] << 16; + if (cardinalities[i] <= 4096) { + // Array container + for (int j = 0; j < cardinalities[i]; j++) { + positions.add(highBits | Short.toUnsignedInt(buf.getShort())); + } + } + else { + // Bitmap container + for (int wordIdx = 0; wordIdx < 1024; wordIdx++) { + long word = buf.getLong(); + while (word != 0) { + int bit = Long.numberOfTrailingZeros(word); + positions.add(highBits | (wordIdx * 64 + bit)); + word &= word - 1; + } + } + } + } + } + + static byte[] serializeRoaringBitmap(List sortedPositions) + { + // Group positions into containers (each container covers 2^16 values) + Map> containers = new HashMap<>(); + for (int pos : sortedPositions) { + int key = pos >>> 16; + int low = pos & 0xFFFF; + containers.computeIfAbsent(key, k -> new ArrayList<>()).add(low); + } + + List sortedKeys = new ArrayList<>(containers.keySet()); + sortedKeys.sort(Integer::compareTo); + + // Calculate size + int numContainers = sortedKeys.size(); + // Cookie (4 bytes) + key-cardinality pairs (4 bytes each) + int headerSize = 4 + numContainers * 4; + int dataSize = 0; + for (int key : sortedKeys) { + int card = containers.get(key).size(); + if (card <= 4096) { + dataSize += card * 2; // array container + } + else { + dataSize += 1024 * 8; // bitmap container + } + } + + java.nio.ByteBuffer buf = java.nio.ByteBuffer.allocate(headerSize + dataSize) + .order(java.nio.ByteOrder.LITTLE_ENDIAN); + + // Cookie: SERIAL_COOKIE_NO_RUNCONTAINER (12346) | (numContainers - 1) << 16 + buf.putInt(12346 | ((numContainers - 1) << 16)); + + // Key-cardinality pairs + for (int key : sortedKeys) { + buf.putShort((short) key); + buf.putShort((short) (containers.get(key).size() - 1)); + } + + // Container data + for (int key : sortedKeys) { + List values = containers.get(key); + values.sort(Integer::compareTo); + if (values.size() <= 4096) { + for (int val : values) { + buf.putShort((short) val); + } + } + else { + long[] bitmap = new long[1024]; + for (int val : values) { + bitmap[val >>> 6] |= 1L << (val & 63); + } + for (long word : bitmap) { + buf.putLong(word); + } + } + } + + return buf.array(); + } +} diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java index 2b9a745bb0183..c02e2fad7e988 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java @@ -15,6 +15,7 @@ import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestQueryFramework; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.BaseTable; @@ -30,12 +31,20 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.hadoop.HadoopCatalog; import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.puffin.Blob; +import org.apache.iceberg.puffin.Puffin; +import org.apache.iceberg.puffin.PuffinWriter; +import org.apache.iceberg.types.Types; import org.testng.annotations.Test; import java.io.File; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.file.Path; import java.util.Map; import java.util.OptionalInt; +import java.util.UUID; import static com.facebook.presto.iceberg.CatalogType.HADOOP; import static com.facebook.presto.iceberg.FileFormat.PARQUET; @@ -45,6 +54,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; public class TestIcebergV3 extends AbstractTestQueryFramework @@ -138,7 +148,7 @@ public void testInsertIntoV3Table() } @Test - public void testDeleteOnV3TableNotSupported() + public void testDeleteOnV3Table() { String tableName = "test_v3_delete"; try { @@ -148,8 +158,31 @@ public void testDeleteOnV3TableNotSupported() + " VALUES (1, 'Alice', 100.0), (2, 'Bob', 200.0), (3, 'Charlie', 300.0)", 3); assertQuery("SELECT * FROM " + tableName + " ORDER BY id", "VALUES (1, 'Alice', 100.0), (2, 'Bob', 200.0), (3, 'Charlie', 300.0)"); - assertThatThrownBy(() -> getQueryRunner().execute("DELETE FROM " + tableName + " WHERE id = 1")) - .hasMessageContaining("Iceberg table updates for format version 3 are not supported yet"); + + assertUpdate("DELETE FROM " + tableName + " WHERE id = 1", 1); + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (2, 'Bob', 200.0), (3, 'Charlie', 300.0)"); + + Table table = loadTable(tableName); + assertEquals(((BaseTable) table).operations().current().formatVersion(), 3); + + // Verify DV metadata: the delete should have produced a PUFFIN-format deletion vector + try (CloseableIterable tasks = table.newScan().planFiles()) { + for (FileScanTask task : tasks) { + for (org.apache.iceberg.DeleteFile deleteFile : task.deletes()) { + assertEquals(deleteFile.format(), FileFormat.PUFFIN); + assertTrue(deleteFile.path().toString().endsWith(".puffin"), + "Deletion vector file should have .puffin extension"); + assertTrue(deleteFile.fileSizeInBytes() > 0, + "Deletion vector file size should be positive"); + } + } + } + + // Delete more rows + assertUpdate("DELETE FROM " + tableName + " WHERE id = 3", 1); + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (2, 'Bob', 200.0)"); } finally { dropTable(tableName); @@ -208,7 +241,7 @@ public void testMetadataDeleteOnV3PartitionedTable() } @Test - public void testUpdateOnV3TableNotSupported() + public void testUpdateOnV3Table() { String tableName = "test_v3_update"; try { @@ -219,9 +252,10 @@ public void testUpdateOnV3TableNotSupported() 3); assertQuery("SELECT * FROM " + tableName + " ORDER BY id", "VALUES (1, 'Alice', 'active', 85.5), (2, 'Bob', 'active', 92.0), (3, 'Charlie', 'inactive', 78.3)"); - assertThatThrownBy(() -> getQueryRunner() - .execute("UPDATE " + tableName + " SET status = 'updated', score = 95.0 WHERE id = 1")) - .hasMessageContaining("Iceberg table updates for format version 3 are not supported yet"); + + assertUpdate("UPDATE " + tableName + " SET status = 'updated', score = 95.0 WHERE id = 1", 1); + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (1, 'Alice', 'updated', 95.0), (2, 'Bob', 'active', 92.0), (3, 'Charlie', 'inactive', 78.3)"); } finally { dropTable(tableName); @@ -229,7 +263,7 @@ public void testUpdateOnV3TableNotSupported() } @Test - public void testMergeOnV3TableNotSupported() + public void testMergeOnV3Table() { String tableName = "test_v3_merge_target"; String sourceTable = "test_v3_merge_source"; @@ -243,11 +277,14 @@ public void testMergeOnV3TableNotSupported() assertQuery("SELECT * FROM " + tableName + " ORDER BY id", "VALUES (1, 'Alice', 100.0), (2, 'Bob', 200.0)"); assertQuery("SELECT * FROM " + sourceTable + " ORDER BY id", "VALUES (1, 'Alice Updated', 150.0), (3, 'Charlie', 300.0)"); - assertThatThrownBy(() -> getQueryRunner().execute( + + getQueryRunner().execute( "MERGE INTO " + tableName + " t USING " + sourceTable + " s ON t.id = s.id " + "WHEN MATCHED THEN UPDATE SET name = s.name, value = s.value " + - "WHEN NOT MATCHED THEN INSERT (id, name, value) VALUES (s.id, s.name, s.value)")) - .hasMessageContaining("Iceberg table updates for format version 3 are not supported yet"); + "WHEN NOT MATCHED THEN INSERT (id, name, value) VALUES (s.id, s.name, s.value)"); + + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (1, 'Alice Updated', 150.0), (2, 'Bob', 200.0), (3, 'Charlie', 300.0)"); } finally { dropTable(tableName); @@ -480,6 +517,634 @@ private File getCatalogDirectory() return catalogDirectory.toFile(); } + @Test + public void testDeletionVectorEndToEnd() + throws Exception + { + String tableName = "test_dv_end_to_end"; + try { + // Step 1: Create V3 table and insert data + assertUpdate("CREATE TABLE " + tableName + " (id integer, value varchar) WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one'), (2, 'two'), (3, 'three'), (4, 'four'), (5, 'five')", 5); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 5"); + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (1, 'one'), (2, 'two'), (3, 'three'), (4, 'four'), (5, 'five')"); + + Table table = loadTable(tableName); + + // Step 2: Write a real Puffin file with a valid roaring bitmap deletion vector. + // The roaring bitmap uses the portable "no-run" format (cookie = 12346). + // We mark row positions 1 and 3 (0-indexed) as deleted — these correspond + // to the rows (2, 'two') and (4, 'four') in insertion order. + byte[] roaringBitmapBytes = serializeRoaringBitmapNoRun(new int[] {1, 3}); + + try (CloseableIterable tasks = table.newScan().planFiles()) { + FileScanTask task = tasks.iterator().next(); + String dataFilePath = task.file().path().toString(); + + // Write the roaring bitmap as a blob inside a Puffin file + String dvPath = table.location() + "/data/dv-" + UUID.randomUUID() + ".puffin"; + OutputFile outputFile = table.io().newOutputFile(dvPath); + + long blobOffset; + long blobLength; + long puffinFileSize; + + try (PuffinWriter writer = Puffin.write(outputFile) + .createdBy("presto-test") + .build()) { + writer.add(new Blob( + "deletion-vector-v2", + ImmutableList.of(), + table.currentSnapshot().snapshotId(), + table.currentSnapshot().sequenceNumber(), + ByteBuffer.wrap(roaringBitmapBytes))); + writer.finish(); + + puffinFileSize = writer.fileSize(); + blobOffset = writer.writtenBlobsMetadata().get(0).offset(); + blobLength = writer.writtenBlobsMetadata().get(0).length(); + } + + // Step 3: Attach the Puffin DV file to the table using Iceberg API + DeleteFile puffinDeleteFile = FileMetadata.deleteFileBuilder(task.spec()) + .ofPositionDeletes() + .withPath(dvPath) + .withFileSizeInBytes(puffinFileSize) + .withFormat(FileFormat.PUFFIN) + .withRecordCount(2) + .withContentOffset(blobOffset) + .withContentSizeInBytes(blobLength) + .withReferencedDataFile(dataFilePath) + .build(); + + table.newRowDelta() + .addDeletes(puffinDeleteFile) + .commit(); + } + + // Step 4: Verify coordinator-side metadata is correct. + // Reload the table and verify the DV file was committed with correct metadata. + table = loadTable(tableName); + try (CloseableIterable tasks = table.newScan().planFiles()) { + FileScanTask task = tasks.iterator().next(); + java.util.List deletes = task.deletes(); + assertFalse(deletes.isEmpty(), "Table should have deletion vector files"); + + org.apache.iceberg.DeleteFile dvFile = deletes.get(0); + assertEquals(dvFile.format(), FileFormat.PUFFIN, "Delete file should be PUFFIN format"); + assertEquals(dvFile.recordCount(), 2, "Delete file should have 2 deleted records"); + assertTrue(dvFile.fileSizeInBytes() > 0, "PUFFIN file size should be positive"); + } + + // Step 5: Verify the coordinator can enumerate splits without error. + // The query will attempt to read data. On a Java worker, the actual DV + // reading is not implemented (that's in Velox's DeletionVectorReader), + // so we verify the coordinator path succeeds by running a SELECT. + // The PUFFIN delete file will either be silently ignored by the Java + // page source (returning all 5 rows) or cause a non-DV-rejection error. + try { + computeActual("SELECT * FROM " + tableName); + } + catch (RuntimeException e) { + // The Java page source may fail trying to read the PUFFIN file as + // positional deletes (since it doesn't have a DV reader). That's expected. + // The important assertion is that the error is NOT the old + // "PUFFIN not supported" rejection from the coordinator. + assertFalse( + e.getMessage().contains("Iceberg deletion vectors") && e.getMessage().contains("not supported"), + "Coordinator should not reject PUFFIN deletion vectors: " + e.getMessage()); + + // Also verify it's not a file-not-found error (the Puffin file exists) + assertFalse( + e.getMessage().contains("FileNotFoundException"), + "PUFFIN file should exist on disk: " + e.getMessage()); + } + } + finally { + dropTable(tableName); + } + } + + /** + * Serializes a roaring bitmap in the portable "no-run" format (cookie = 12346). + * This produces the exact binary format expected by Velox's DeletionVectorReader. + * Only supports positions within a single container (all < 65536). + */ + private static byte[] serializeRoaringBitmapNoRun(int[] positions) + { + // Cookie (12346) + numContainers (1) + // + 1 container key-cardinality pair (4 bytes) + // + sorted uint16 values (2 bytes each) + int numPositions = positions.length; + int dataSize = 4 + 4 + 4 + numPositions * 2; + ByteBuffer buffer = ByteBuffer.allocate(dataSize); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + // Cookie: 12346 = SERIAL_COOKIE_NO_RUNCONTAINER + buffer.putInt(12346); + // Number of containers: 1 + buffer.putInt(1); + // Container key (high 16 bits): 0, cardinality - 1 + buffer.putShort((short) 0); + buffer.putShort((short) (numPositions - 1)); + // Container data: sorted uint16 values (low 16 bits of each position) + java.util.Arrays.sort(positions); + for (int pos : positions) { + buffer.putShort((short) (pos & 0xFFFF)); + } + + return buffer.array(); + } + + @Test + public void testDeletionVectorDeletesAllRows() + throws Exception + { + String tableName = "test_dv_deletes_all_rows"; + try { + assertUpdate("CREATE TABLE " + tableName + " (id integer, value varchar) WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one'), (2, 'two'), (3, 'three')", 3); + + Table table = loadTable(tableName); + + // Write a DV that deletes all 3 rows (positions 0, 1, 2). + byte[] roaringBitmapBytes = serializeRoaringBitmapNoRun(new int[] {0, 1, 2}); + + try (CloseableIterable tasks = table.newScan().planFiles()) { + FileScanTask task = tasks.iterator().next(); + String dataFilePath = task.file().path().toString(); + + String dvPath = table.location() + "/data/dv-all-" + UUID.randomUUID() + ".puffin"; + OutputFile outputFile = table.io().newOutputFile(dvPath); + + long blobOffset; + long blobLength; + long puffinFileSize; + + try (PuffinWriter writer = Puffin.write(outputFile) + .createdBy("presto-test") + .build()) { + writer.add(new Blob( + "deletion-vector-v2", + ImmutableList.of(), + table.currentSnapshot().snapshotId(), + table.currentSnapshot().sequenceNumber(), + ByteBuffer.wrap(roaringBitmapBytes))); + writer.finish(); + + puffinFileSize = writer.fileSize(); + blobOffset = writer.writtenBlobsMetadata().get(0).offset(); + blobLength = writer.writtenBlobsMetadata().get(0).length(); + } + + DeleteFile puffinDeleteFile = FileMetadata.deleteFileBuilder(task.spec()) + .ofPositionDeletes() + .withPath(dvPath) + .withFileSizeInBytes(puffinFileSize) + .withFormat(FileFormat.PUFFIN) + .withRecordCount(3) + .withContentOffset(blobOffset) + .withContentSizeInBytes(blobLength) + .withReferencedDataFile(dataFilePath) + .build(); + + table.newRowDelta() + .addDeletes(puffinDeleteFile) + .commit(); + } + + // Verify the coordinator can enumerate splits. On Java workers the DV + // reader isn't implemented, so the query may either succeed (returning + // all rows because the Java page source ignores the DV) or fail with a + // non-rejection error. The key assertion is that it doesn't throw + // "PUFFIN not supported". + try { + computeActual("SELECT * FROM " + tableName); + } + catch (RuntimeException e) { + assertFalse( + e.getMessage().contains("Iceberg deletion vectors") && e.getMessage().contains("not supported"), + "Coordinator should not reject PUFFIN deletion vectors: " + e.getMessage()); + } + } + finally { + dropTable(tableName); + } + } + + @Test + public void testDeletionVectorOnMultipleDataFiles() + throws Exception + { + String tableName = "test_dv_multiple_data_files"; + try { + assertUpdate("CREATE TABLE " + tableName + " (id integer, value varchar) WITH (\"format-version\" = '3')"); + // Two separate inserts create two separate data files. + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one'), (2, 'two'), (3, 'three')", 3); + assertUpdate("INSERT INTO " + tableName + " VALUES (4, 'four'), (5, 'five'), (6, 'six')", 3); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 6"); + + Table table = loadTable(tableName); + + // Attach a DV only to the first data file (positions 0 and 2 → rows 1 + // and 3 from the first insert). The second data file has no deletes. + try (CloseableIterable tasks = table.newScan().planFiles()) { + FileScanTask firstTask = tasks.iterator().next(); + String firstDataFilePath = firstTask.file().path().toString(); + + byte[] roaringBitmapBytes = serializeRoaringBitmapNoRun(new int[] {0, 2}); + String dvPath = table.location() + "/data/dv-partial-" + UUID.randomUUID() + ".puffin"; + OutputFile outputFile = table.io().newOutputFile(dvPath); + + long blobOffset; + long blobLength; + long puffinFileSize; + + try (PuffinWriter writer = Puffin.write(outputFile) + .createdBy("presto-test") + .build()) { + writer.add(new Blob( + "deletion-vector-v2", + ImmutableList.of(), + table.currentSnapshot().snapshotId(), + table.currentSnapshot().sequenceNumber(), + ByteBuffer.wrap(roaringBitmapBytes))); + writer.finish(); + + puffinFileSize = writer.fileSize(); + blobOffset = writer.writtenBlobsMetadata().get(0).offset(); + blobLength = writer.writtenBlobsMetadata().get(0).length(); + } + + DeleteFile puffinDeleteFile = FileMetadata.deleteFileBuilder(firstTask.spec()) + .ofPositionDeletes() + .withPath(dvPath) + .withFileSizeInBytes(puffinFileSize) + .withFormat(FileFormat.PUFFIN) + .withRecordCount(2) + .withContentOffset(blobOffset) + .withContentSizeInBytes(blobLength) + .withReferencedDataFile(firstDataFilePath) + .build(); + + table.newRowDelta() + .addDeletes(puffinDeleteFile) + .commit(); + } + + // Verify coordinator metadata: only the first file's task should have deletes. + table = loadTable(tableName); + int tasksWithDeletes = 0; + int tasksWithoutDeletes = 0; + try (CloseableIterable tasks = table.newScan().planFiles()) { + for (FileScanTask task : tasks) { + if (task.deletes().isEmpty()) { + tasksWithoutDeletes++; + } + else { + tasksWithDeletes++; + assertEquals(task.deletes().size(), 1, "First data file should have exactly 1 DV"); + assertEquals(task.deletes().get(0).format(), FileFormat.PUFFIN); + } + } + } + assertEquals(tasksWithDeletes, 1, "Exactly one data file should have a DV"); + assertEquals(tasksWithoutDeletes, 1, "Exactly one data file should have no deletes"); + + // Run a query — coordinator should enumerate splits without error. + try { + computeActual("SELECT * FROM " + tableName); + } + catch (RuntimeException e) { + assertFalse( + e.getMessage().contains("Iceberg deletion vectors") && e.getMessage().contains("not supported"), + "Coordinator should not reject PUFFIN deletion vectors: " + e.getMessage()); + } + } + finally { + dropTable(tableName); + } + } + + @Test + public void testV3SchemaEvolution() + { + String tableName = "test_v3_schema_evolution"; + try { + assertUpdate("CREATE TABLE " + tableName + " (id integer, value varchar) WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one'), (2, 'two')", 2); + + // Add a new column via Iceberg API + Table table = loadTable(tableName); + table.updateSchema() + .addColumn("score", org.apache.iceberg.types.Types.DoubleType.get()) + .commit(); + + // New inserts include the new column + assertUpdate("INSERT INTO " + tableName + " VALUES (3, 'three', 99.5)", 1); + + // Verify all rows are readable (old rows have NULL for the new column) + assertQuery("SELECT id, value FROM " + tableName + " ORDER BY id", + "VALUES (1, 'one'), (2, 'two'), (3, 'three')"); + assertQuery("SELECT id, score FROM " + tableName + " WHERE score IS NOT NULL", + "VALUES (3, 99.5)"); + assertQuery("SELECT count(*) FROM " + tableName + " WHERE score IS NULL", "SELECT 2"); + + // Rename a column + table = loadTable(tableName); + table.updateSchema() + .renameColumn("value", "label") + .commit(); + + // Verify reads still work after rename + assertQuery("SELECT id, label FROM " + tableName + " ORDER BY id", + "VALUES (1, 'one'), (2, 'two'), (3, 'three')"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testV3MultipleSnapshotsWithDV() + throws Exception + { + String tableName = "test_v3_multi_snapshot_dv"; + try { + // Snapshot 1: initial data + assertUpdate("CREATE TABLE " + tableName + " (id integer, value varchar) WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one'), (2, 'two'), (3, 'three')", 3); + + Table table = loadTable(tableName); + long snapshot1Id = table.currentSnapshot().snapshotId(); + + // Snapshot 2: attach a DV deleting row at position 1 (row id=2, 'two') + byte[] roaringBitmapBytes = serializeRoaringBitmapNoRun(new int[] {1}); + + try (CloseableIterable tasks = table.newScan().planFiles()) { + FileScanTask task = tasks.iterator().next(); + String dataFilePath = task.file().path().toString(); + + String dvPath = table.location() + "/data/dv-snap-" + UUID.randomUUID() + ".puffin"; + OutputFile outputFile = table.io().newOutputFile(dvPath); + + long blobOffset; + long blobLength; + long puffinFileSize; + + try (PuffinWriter writer = Puffin.write(outputFile) + .createdBy("presto-test") + .build()) { + writer.add(new Blob( + "deletion-vector-v2", + ImmutableList.of(), + table.currentSnapshot().snapshotId(), + table.currentSnapshot().sequenceNumber(), + ByteBuffer.wrap(roaringBitmapBytes))); + writer.finish(); + + puffinFileSize = writer.fileSize(); + blobOffset = writer.writtenBlobsMetadata().get(0).offset(); + blobLength = writer.writtenBlobsMetadata().get(0).length(); + } + + DeleteFile puffinDeleteFile = FileMetadata.deleteFileBuilder(task.spec()) + .ofPositionDeletes() + .withPath(dvPath) + .withFileSizeInBytes(puffinFileSize) + .withFormat(FileFormat.PUFFIN) + .withRecordCount(1) + .withContentOffset(blobOffset) + .withContentSizeInBytes(blobLength) + .withReferencedDataFile(dataFilePath) + .build(); + + table.newRowDelta() + .addDeletes(puffinDeleteFile) + .commit(); + } + + // Snapshot 3: more data added after the DV + assertUpdate("INSERT INTO " + tableName + " VALUES (4, 'four'), (5, 'five')", 2); + + // Verify the table now has 3 snapshots + table = loadTable(tableName); + int snapshotCount = 0; + for (org.apache.iceberg.Snapshot snapshot : table.snapshots()) { + snapshotCount++; + } + assertTrue(snapshotCount >= 3, "Table should have at least 3 snapshots, got: " + snapshotCount); + + // Verify coordinator can enumerate all splits (including those with DVs + // and those from the post-DV insert). + try (CloseableIterable tasks = table.newScan().planFiles()) { + int totalFiles = 0; + int filesWithDeletes = 0; + for (FileScanTask task : tasks) { + totalFiles++; + if (!task.deletes().isEmpty()) { + filesWithDeletes++; + } + } + assertEquals(totalFiles, 2, "Should have 2 data files (one from each insert)"); + assertEquals(filesWithDeletes, 1, "Only the first data file should have DV deletes"); + } + + // Run a query to verify coordinator enumeration succeeds. + try { + computeActual("SELECT * FROM " + tableName); + } + catch (RuntimeException e) { + assertFalse( + e.getMessage().contains("Iceberg deletion vectors") && e.getMessage().contains("not supported"), + "Coordinator should not reject PUFFIN deletion vectors: " + e.getMessage()); + } + } + finally { + dropTable(tableName); + } + } + + @Test + public void testV3DeletionVectorMetadataFields() + throws Exception + { + String tableName = "test_dv_metadata_fields"; + try { + assertUpdate("CREATE TABLE " + tableName + " (id integer, value varchar) WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one'), (2, 'two')", 2); + + Table table = loadTable(tableName); + + byte[] roaringBitmapBytes = serializeRoaringBitmapNoRun(new int[] {0}); + String dvPath = table.location() + "/data/dv-meta-" + UUID.randomUUID() + ".puffin"; + + long blobOffset; + long blobLength; + long puffinFileSize; + + try (CloseableIterable tasks = table.newScan().planFiles()) { + FileScanTask task = tasks.iterator().next(); + String dataFilePath = task.file().path().toString(); + + OutputFile outputFile = table.io().newOutputFile(dvPath); + + try (PuffinWriter writer = Puffin.write(outputFile) + .createdBy("presto-test") + .build()) { + writer.add(new Blob( + "deletion-vector-v2", + ImmutableList.of(), + table.currentSnapshot().snapshotId(), + table.currentSnapshot().sequenceNumber(), + ByteBuffer.wrap(roaringBitmapBytes))); + writer.finish(); + + puffinFileSize = writer.fileSize(); + blobOffset = writer.writtenBlobsMetadata().get(0).offset(); + blobLength = writer.writtenBlobsMetadata().get(0).length(); + } + + DeleteFile puffinDeleteFile = FileMetadata.deleteFileBuilder(task.spec()) + .ofPositionDeletes() + .withPath(dvPath) + .withFileSizeInBytes(puffinFileSize) + .withFormat(FileFormat.PUFFIN) + .withRecordCount(1) + .withContentOffset(blobOffset) + .withContentSizeInBytes(blobLength) + .withReferencedDataFile(dataFilePath) + .build(); + + table.newRowDelta() + .addDeletes(puffinDeleteFile) + .commit(); + } + + // Verify the committed DV file has correct metadata fields. + table = loadTable(tableName); + try (CloseableIterable tasks = table.newScan().planFiles()) { + FileScanTask task = tasks.iterator().next(); + java.util.List deletes = task.deletes(); + assertFalse(deletes.isEmpty(), "Should have deletion vector files"); + + org.apache.iceberg.DeleteFile dvFile = deletes.get(0); + assertEquals(dvFile.format(), FileFormat.PUFFIN, "Format should be PUFFIN"); + assertEquals(dvFile.recordCount(), 1, "Record count should match deleted positions"); + assertTrue(dvFile.fileSizeInBytes() > 0, "File size must be positive"); + + // Verify the DV file path ends with .puffin as expected. + assertTrue(dvFile.path().toString().endsWith(".puffin"), "DV file should be a .puffin file"); + } + } + finally { + dropTable(tableName); + } + } + + @Test + public void testV3WriteReadRoundTrip() + throws Exception + { + String tableName = "test_v3_write_read_round_trip"; + try { + // Step 1: Create V3 table and insert initial data + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, name VARCHAR, value DOUBLE) WITH (\"format-version\" = '3', \"write.delete.mode\" = 'merge-on-read')"); + assertUpdate("INSERT INTO " + tableName + + " VALUES (1, 'Alice', 100.0), (2, 'Bob', 200.0), (3, 'Charlie', 300.0), (4, 'Dave', 400.0), (5, 'Eve', 500.0)", 5); + + // Step 2: Verify initial data via read path + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 5"); + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (1, 'Alice', 100.0), (2, 'Bob', 200.0), (3, 'Charlie', 300.0), (4, 'Dave', 400.0), (5, 'Eve', 500.0)"); + + // Step 3: First DELETE via write path (produces DV #1) + assertUpdate("DELETE FROM " + tableName + " WHERE id IN (1, 3)", 2); + + // Step 4: Verify read path filters DV #1 correctly + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (2, 'Bob', 200.0), (4, 'Dave', 400.0), (5, 'Eve', 500.0)"); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 3"); + + // Step 5: Cross-validate DV #1 metadata via Iceberg API + Table table = loadTable(tableName); + assertEquals(((BaseTable) table).operations().current().formatVersion(), 3); + + int dvCount = 0; + try (CloseableIterable tasks = table.newScan().planFiles()) { + for (FileScanTask task : tasks) { + for (org.apache.iceberg.DeleteFile deleteFile : task.deletes()) { + dvCount++; + assertEquals(deleteFile.format(), FileFormat.PUFFIN, + "Presto-written DV must use PUFFIN format"); + assertTrue(deleteFile.path().toString().endsWith(".puffin"), + "DV file path must end with .puffin"); + assertTrue(deleteFile.fileSizeInBytes() > 0, + "DV file size must be positive"); + assertTrue(deleteFile.contentOffset() >= 0, + "DV content offset must be non-negative"); + assertTrue(deleteFile.contentSizeInBytes() > 0, + "DV content size must be positive"); + assertTrue(deleteFile.recordCount() > 0, + "DV record count must be positive"); + } + } + } + assertTrue(dvCount > 0, "Should have at least one deletion vector after DELETE"); + + // Step 6: Insert more data (creates a new data file alongside existing ones) + assertUpdate("INSERT INTO " + tableName + + " VALUES (6, 'Frank', 600.0), (7, 'Grace', 700.0)", 2); + + // Step 7: Verify read path handles mixed state: old data with DVs + new data + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (2, 'Bob', 200.0), (4, 'Dave', 400.0), (5, 'Eve', 500.0), (6, 'Frank', 600.0), (7, 'Grace', 700.0)"); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 5"); + + // Step 8: Second DELETE via write path (produces DV #2, targeting new and old data) + assertUpdate("DELETE FROM " + tableName + " WHERE id IN (2, 7)", 2); + + // Step 9: Verify cumulative read path correctness with two rounds of DVs + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (4, 'Dave', 400.0), (5, 'Eve', 500.0), (6, 'Frank', 600.0)"); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 3"); + + // Step 10: Cross-validate cumulative DV metadata via Iceberg API + table = loadTable(tableName); + int totalDvs = 0; + int totalDataFiles = 0; + try (CloseableIterable tasks = table.newScan().planFiles()) { + for (FileScanTask task : tasks) { + totalDataFiles++; + for (org.apache.iceberg.DeleteFile deleteFile : task.deletes()) { + totalDvs++; + assertEquals(deleteFile.format(), FileFormat.PUFFIN, + "All DVs must use PUFFIN format"); + assertTrue(deleteFile.recordCount() > 0, + "Each DV must have positive record count"); + } + } + } + assertTrue(totalDvs > 0, "Should have deletion vectors after two rounds of DELETE"); + assertTrue(totalDataFiles > 0, "Should have data files remaining"); + + // Step 11: Verify aggregation works correctly over DV-filtered data + assertQuery("SELECT SUM(value) FROM " + tableName, "SELECT 1500.0"); + assertQuery("SELECT MIN(id), MAX(id) FROM " + tableName, "VALUES (4, 6)"); + + // Step 12: Verify predicates work correctly with DVs + assertQuery("SELECT * FROM " + tableName + " WHERE value > 450.0 ORDER BY id", + "VALUES (5, 'Eve', 500.0), (6, 'Frank', 600.0)"); + assertQuery("SELECT * FROM " + tableName + " WHERE name LIKE '%a%' ORDER BY id", + "VALUES (4, 'Dave', 400.0), (6, 'Frank', 600.0)"); + } + finally { + dropTable(tableName); + } + } + private void dropTableViaIceberg(String tableName) { Catalog catalog = CatalogUtil.loadCatalog( @@ -488,4 +1153,186 @@ private void dropTableViaIceberg(String tableName) catalog.dropTable( TableIdentifier.of(TEST_SCHEMA, tableName), true); } + + @Test + public void testRewriteDeleteFilesProcedure() + throws Exception + { + String tableName = "test_rewrite_delete_files"; + try { + // Step 1: Create V3 table and insert data + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, name VARCHAR, value DOUBLE) WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + + " VALUES (1, 'Alice', 100.0), (2, 'Bob', 200.0), (3, 'Carol', 300.0), (4, 'Dave', 400.0), (5, 'Eve', 500.0)", 5); + + // Step 2: Perform multiple deletes to create multiple DVs per data file + assertUpdate("DELETE FROM " + tableName + " WHERE id = 1", 1); + assertUpdate("DELETE FROM " + tableName + " WHERE id = 3", 1); + + // Step 3: Verify we have multiple delete files before compaction + Table table = loadTable(tableName); + int dvCountBefore = 0; + try (CloseableIterable tasks = table.newScan().planFiles()) { + for (FileScanTask task : tasks) { + dvCountBefore += task.deletes().size(); + } + } + assertTrue(dvCountBefore >= 2, "Should have at least 2 DVs before compaction, got: " + dvCountBefore); + + // Step 4: Verify data is correct before compaction + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (2, 'Bob', 200.0), (4, 'Dave', 400.0), (5, 'Eve', 500.0)"); + + // Step 5: Run DV compaction + assertQuerySucceeds(format("CALL system.rewrite_delete_files('%s', '%s')", TEST_SCHEMA, tableName)); + + // Step 6: Verify data is still correct after compaction + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (2, 'Bob', 200.0), (4, 'Dave', 400.0), (5, 'Eve', 500.0)"); + + // Step 7: Verify DVs were compacted (fewer or equal DVs) + table.refresh(); + int dvCountAfter = 0; + try (CloseableIterable tasks = table.newScan().planFiles()) { + for (FileScanTask task : tasks) { + for (DeleteFile dv : task.deletes()) { + dvCountAfter++; + assertEquals(dv.format(), FileFormat.PUFFIN, "Compacted DV must use PUFFIN format"); + } + } + } + assertTrue(dvCountAfter <= dvCountBefore, + "DV count after compaction (" + dvCountAfter + ") should be <= before (" + dvCountBefore + ")"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testRewriteDeleteFilesOnV2Table() + { + String tableName = "test_rewrite_delete_files_v2"; + try { + // V2 tables should be a no-op (no DVs to compact) + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, value VARCHAR) WITH (\"format-version\" = '2', delete_mode = 'merge-on-read')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one'), (2, 'two'), (3, 'three')", 3); + assertUpdate("DELETE FROM " + tableName + " WHERE id = 1", 1); + + assertQuerySucceeds(format("CALL system.rewrite_delete_files('%s', '%s')", TEST_SCHEMA, tableName)); + + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (2, 'two'), (3, 'three')"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testV3DefaultValues() + throws Exception + { + String tableName = "test_v3_default_values"; + try { + // Step 1: Create V3 table and insert initial data + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, name VARCHAR) WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'Alice'), (2, 'Bob')", 2); + + // Step 2: Add column with default value via Iceberg API + Table table = loadTable(tableName); + table.updateSchema() + .addColumn("score", org.apache.iceberg.types.Types.DoubleType.get()) + .setDefaultValue("score", 99.0) + .commit(); + + // Step 3: Verify we can read old data — the new column should have default value + assertQuery("SELECT id, name FROM " + tableName + " ORDER BY id", + "VALUES (1, 'Alice'), (2, 'Bob')"); + + // Step 4: Insert new data with the new column + assertUpdate("INSERT INTO " + tableName + " VALUES (3, 'Carol', 300.0)", 1); + + // Step 5: Verify new data reads correctly + assertQuery("SELECT id, name, score FROM " + tableName + " WHERE id = 3", + "VALUES (3, 'Carol', 300.0)"); + + // Step 6: Verify old rows get default value (99.0) from Iceberg schema evolution + assertQuery("SELECT id, name, score FROM " + tableName + " ORDER BY id", + "VALUES (1, 'Alice', 99.0), (2, 'Bob', 99.0), (3, 'Carol', 300.0)"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testMultiArgumentPartitionTransforms() + { + String tableName = "test_v3_multi_arg_transforms"; + try { + // Create V3 table with bucket(4, id) partitioning + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, name VARCHAR, value DOUBLE)" + + " WITH (\"format-version\" = '3', partitioning = ARRAY['bucket(id, 4)'])"); + + // Verify table was created with correct partition spec + Table table = loadTable(tableName); + assertEquals(((BaseTable) table).operations().current().formatVersion(), 3); + assertEquals(table.spec().fields().size(), 1); + assertEquals(table.spec().fields().get(0).transform().toString(), "bucket[4]"); + + // Insert data — should distribute across buckets + assertUpdate("INSERT INTO " + tableName + + " VALUES (1, 'Alice', 100.0), (2, 'Bob', 200.0), (3, 'Charlie', 300.0), (4, 'Diana', 400.0)", 4); + + // Verify data reads correctly + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (1, 'Alice', 100.0), (2, 'Bob', 200.0), (3, 'Charlie', 300.0), (4, 'Diana', 400.0)"); + + // Verify partition pruning works — query with equality predicate + assertQuery("SELECT name, value FROM " + tableName + " WHERE id = 2", + "VALUES ('Bob', 200.0)"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testTruncatePartitionTransform() + { + String tableName = "test_v3_truncate_transform"; + try { + // Create V3 table with truncate(10, value) partitioning on a varchar column + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, category VARCHAR, amount DOUBLE)" + + " WITH (\"format-version\" = '3', partitioning = ARRAY['truncate(category, 3)'])"); + + Table table = loadTable(tableName); + assertEquals(((BaseTable) table).operations().current().formatVersion(), 3); + assertEquals(table.spec().fields().size(), 1); + assertEquals(table.spec().fields().get(0).transform().toString(), "truncate[3]"); + + // Insert data with varying category prefixes + assertUpdate("INSERT INTO " + tableName + + " VALUES (1, 'food_pizza', 15.0), (2, 'food_burger', 12.0)," + + " (3, 'drink_coffee', 5.0), (4, 'drink_tea', 3.0)", 4); + + // Verify data reads correctly + assertQuery("SELECT id, category, amount FROM " + tableName + " ORDER BY id", + "VALUES (1, 'food_pizza', 15.0), (2, 'food_burger', 12.0)," + + " (3, 'drink_coffee', 5.0), (4, 'drink_tea', 3.0)"); + + // Verify we can filter + assertQuery("SELECT id FROM " + tableName + " WHERE category = 'food_pizza'", + "VALUES 1"); + } + finally { + dropTable(tableName); + } + } } From 05ee8f2a1c242d93ec48658e5dd375998380ddb0 Mon Sep 17 00:00:00 2001 From: Apurva Kumar Date: Fri, 20 Mar 2026 13:08:28 -0700 Subject: [PATCH 06/12] [presto][iceberg] Add nanosecond timestamp (TIMESTAMP_NANO) type support for Iceberg V3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Iceberg V3 introduces nanosecond-precision timestamp types (timestamp_ns and timestamptz_ns). This diff adds support for reading tables with these column types by mapping them to Presto's best available precision (TIMESTAMP_MICROSECONDS for timestamp_ns, TIMESTAMP_WITH_TIME_ZONE for timestamptz_ns). Changes: - TypeConverter: Map TIMESTAMP_NANO to Presto types and ORC types - ExpressionConverter: Fix predicate pushdown for TIMESTAMP_MICROSECONDS precision (was incorrectly converting microseconds as milliseconds) - IcebergUtil: Handle TIMESTAMP_NANO partition values (nanos → micros) - PartitionData: Handle TIMESTAMP_NANO in JSON partition deserialization - PartitionTable: Convert nanosecond partition values to microseconds - TestIcebergV3: Add testNanosecondTimestampSchema integration test Differential Revision: D97531552 --- .../presto/iceberg/ExpressionConverter.java | 11 +++++++- .../facebook/presto/iceberg/IcebergUtil.java | 2 ++ .../presto/iceberg/PartitionData.java | 1 + .../presto/iceberg/PartitionTable.java | 3 ++ .../presto/iceberg/TypeConverter.java | 7 +++++ .../presto/iceberg/TestIcebergV3.java | 28 +++++++++++++++++++ 6 files changed, 51 insertions(+), 1 deletion(-) diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/ExpressionConverter.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/ExpressionConverter.java index 285a9cf863b5f..fd47dddb43b84 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/ExpressionConverter.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/ExpressionConverter.java @@ -201,7 +201,16 @@ private static Object getIcebergLiteralValue(Type type, Marker marker) return toIntExact(((Long) marker.getValue())); } - if (type instanceof TimestampType || type instanceof TimeType) { + if (type instanceof TimestampType) { + TimestampType tsType = (TimestampType) type; + long value = (Long) marker.getValue(); + if (tsType.getPrecision() == MILLISECONDS) { + return MILLISECONDS.toMicros(value); + } + return value; + } + + if (type instanceof TimeType) { return MILLISECONDS.toMicros((Long) marker.getValue()); } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java index e568f1e7c6e4c..e1e076cec31ea 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java @@ -779,6 +779,8 @@ public static Domain createDomainFromIcebergPartitionValue( case TIME: case TIMESTAMP: return singleValue(prestoType, MICROSECONDS.toMillis((Long) value)); + case TIMESTAMP_NANO: + return singleValue(prestoType, Math.floorDiv((Long) value, 1000L)); case STRING: return singleValue(prestoType, utf8Slice(value.toString())); case FLOAT: diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PartitionData.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PartitionData.java index 015972ac949a5..9e1ea88d8f62c 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PartitionData.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PartitionData.java @@ -150,6 +150,7 @@ public static Object getValue(JsonNode partitionValue, Type type) return partitionValue.asInt(); case LONG: case TIMESTAMP: + case TIMESTAMP_NANO: case TIME: return partitionValue.asLong(); case FLOAT: diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PartitionTable.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PartitionTable.java index 08a5887bed551..dd27d24d426fd 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PartitionTable.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PartitionTable.java @@ -303,6 +303,9 @@ private Object convert(Object value, Type type) return MICROSECONDS.toMillis((long) value); } } + if (type instanceof Types.TimestampNanoType) { + return Math.floorDiv((long) value, 1000L); + } if (type instanceof Types.TimeType) { return MICROSECONDS.toMillis((long) value); } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/TypeConverter.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/TypeConverter.java index ca8db778ae87c..5d611c1447ac0 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/TypeConverter.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/TypeConverter.java @@ -124,6 +124,12 @@ public static Type toPrestoType(org.apache.iceberg.types.Type type, TypeManager return TIMESTAMP_WITH_TIME_ZONE; } return TimestampType.TIMESTAMP; + case TIMESTAMP_NANO: + Types.TimestampNanoType tsNanoType = (Types.TimestampNanoType) type.asPrimitiveType(); + if (tsNanoType.shouldAdjustToUTC()) { + return TIMESTAMP_WITH_TIME_ZONE; + } + return TimestampType.TIMESTAMP_MICROSECONDS; case STRING: return VarcharType.createUnboundedVarcharType(); case UUID: @@ -402,6 +408,7 @@ private static List toOrcType(int nextFieldTypeIndex, org.apache.iceber case DATE: return ImmutableList.of(new OrcType(OrcType.OrcTypeKind.DATE, ImmutableList.of(), ImmutableList.of(), Optional.empty(), Optional.empty(), Optional.empty(), attributes)); case TIMESTAMP: + case TIMESTAMP_NANO: return ImmutableList.of(new OrcType(OrcType.OrcTypeKind.TIMESTAMP, ImmutableList.of(), ImmutableList.of(), Optional.empty(), Optional.empty(), Optional.empty(), attributes)); case STRING: return ImmutableList.of(new OrcType(OrcType.OrcTypeKind.STRING, ImmutableList.of(), ImmutableList.of(), Optional.empty(), Optional.empty(), Optional.empty(), attributes)); diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java index c02e2fad7e988..5009e9e73db38 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java @@ -1335,4 +1335,32 @@ public void testTruncatePartitionTransform() dropTable(tableName); } } + + @Test + public void testNanosecondTimestampSchema() + { + String tableName = "test_v3_timestamp_nano"; + try { + // Create V3 table with Presto + assertUpdate("CREATE TABLE " + tableName + " (id INTEGER) WITH (\"format-version\" = '3')"); + + // Add nanosecond timestamp columns via Iceberg API + Table table = loadTable(tableName); + table.updateSchema() + .addColumn("ts_nano", Types.TimestampNanoType.withoutZone()) + .addColumn("ts_nano_tz", Types.TimestampNanoType.withZone()) + .commit(); + + // Verify Presto can read the schema with nanosecond columns + // ts_nano maps to timestamp microseconds, ts_nano_tz maps to timestamp with time zone + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 0"); + + // Insert data through Presto — the nanosecond columns accept null values + assertUpdate("INSERT INTO " + tableName + " (id) VALUES (1)", 1); + assertQuery("SELECT id FROM " + tableName, "VALUES 1"); + } + finally { + dropTable(tableName); + } + } } From 28cec98147e4fcd3b9b91c794835d8d0259cfa6d Mon Sep 17 00:00:00 2001 From: Apurva Kumar Date: Fri, 20 Mar 2026 13:08:28 -0700 Subject: [PATCH 07/12] [presto][iceberg] Add Variant type support for Iceberg V3 Summary: Add support for Iceberg V3 VARIANT type across the Presto-Iceberg connector type conversion pipeline, including a binary codec for the Apache Variant spec and SQL scalar functions for Variant data manipulation. The VARIANT type represents semi-structured data (JSON) and is mapped to Presto's unbounded VARCHAR type. This diff provides: 1. Type mapping: VARIANT to/from VARCHAR across all converter layers 2. VariantBinaryCodec: Full encoder/decoder for Apache Variant binary format - Supports primitives (null, bool, int8/16/32/64, double, string) - Supports short strings (0-63 bytes) and long strings - Supports objects (with metadata dictionary) and arrays - JSON string round-trip: JSON -> binary -> JSON - Binary format detection (isVariantBinary) and auto-decode (decodeVariantAuto) - Type introspection from binary headers (getValueTypeName) 3. SQL scalar functions registered via IcebergConnector.getSystemFunctions(): - variant_get(varchar, varchar): Extract field with dot-path and array indexing (e.g., 'users[0].name', 'address.city') - variant_keys(varchar): Return top-level object keys as JSON array - variant_type(varchar): Return JSON type name (object, array, string, number, boolean, null) - to_variant(varchar): Validate JSON and cast to Variant (Phase 5 CAST support) - parse_variant(varchar): Validate and normalize through Variant binary codec - variant_to_json(varchar): Normalize Variant to compact JSON representation - variant_binary_roundtrip(varchar): Encode to binary and decode back (interop testing) 4. Predicate pushdown: IS NULL/IS NOT NULL works through VARCHAR mapping; variant_get pushdown tracked as future optimizer rule work 5. Comprehensive tests for the codec, functions, and end-to-end connector behavior Changes: - TypeConverter: Map VARIANT to VarcharType (unbounded) in toPrestoType(), and to ORC STRING type in toOrcType() - IcebergUtil: Handle VARIANT partition values as string slices in domain creation - PartitionData: Deserialize VARIANT partition values as text (same as STRING) - PartitionTable: Convert VariantType partition values to string representation - VariantBinaryCodec: Full Apache Variant binary spec (v1) encoder/decoder with binary detection, type introspection, and auto-decode capabilities - VariantFunctions: 7 SQL scalar functions for Variant data manipulation including dot-path navigation, array indexing, key enumeration, type introspection, CAST - IcebergConnector: Register VariantFunctions in getSystemFunctions() - TestVariantBinaryCodec: 40+ tests covering primitives, objects, arrays, metadata, binary detection, type names, and auto-decode - TestVariantFunctions: 40+ tests covering all 7 functions including dot-path, array indexing, error handling, and edge cases - TestIcebergV3: Integration tests for VARIANT type including JSON data round-trip Note: Velox/Prestissimo does not require changes - the VARIANT->VARCHAR type mapping flows automatically through HiveTypeParser on the C++ side. Differential Revision: D97531551 --- .../presto/iceberg/IcebergConnector.java | 2 + .../facebook/presto/iceberg/IcebergUtil.java | 1 + .../presto/iceberg/PartitionData.java | 1 + .../presto/iceberg/PartitionTable.java | 2 +- .../presto/iceberg/TypeConverter.java | 3 + .../presto/iceberg/VariantBinaryCodec.java | 783 ++++++++++++++++++ .../iceberg/function/VariantFunctions.java | 456 ++++++++++ .../presto/iceberg/TestIcebergV3.java | 249 ++++++ .../iceberg/TestVariantBinaryCodec.java | 510 ++++++++++++ .../presto/iceberg/TestVariantFunctions.java | 562 +++++++++++++ 10 files changed, 2568 insertions(+), 1 deletion(-) create mode 100644 presto-iceberg/src/main/java/com/facebook/presto/iceberg/VariantBinaryCodec.java create mode 100644 presto-iceberg/src/main/java/com/facebook/presto/iceberg/function/VariantFunctions.java create mode 100644 presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestVariantBinaryCodec.java create mode 100644 presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestVariantFunctions.java diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergConnector.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergConnector.java index 86ad7fc192bea..137f1cc38d8c9 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergConnector.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergConnector.java @@ -16,6 +16,7 @@ import com.facebook.airlift.bootstrap.LifeCycleManager; import com.facebook.presto.hive.HiveTransactionHandle; import com.facebook.presto.iceberg.function.IcebergBucketFunction; +import com.facebook.presto.iceberg.function.VariantFunctions; import com.facebook.presto.iceberg.function.changelog.ApplyChangelogFunction; import com.facebook.presto.iceberg.transaction.IcebergTransactionManager; import com.facebook.presto.iceberg.transaction.IcebergTransactionMetadata; @@ -256,6 +257,7 @@ public Set> getSystemFunctions() .add(ApplyChangelogFunction.class) .add(IcebergBucketFunction.class) .add(IcebergBucketFunction.Bucket.class) + .add(VariantFunctions.class) .build(); } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java index e1e076cec31ea..b57aef741b8b1 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java @@ -782,6 +782,7 @@ public static Domain createDomainFromIcebergPartitionValue( case TIMESTAMP_NANO: return singleValue(prestoType, Math.floorDiv((Long) value, 1000L)); case STRING: + case VARIANT: return singleValue(prestoType, utf8Slice(value.toString())); case FLOAT: return singleValue(prestoType, (long) floatToRawIntBits((Float) value)); diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PartitionData.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PartitionData.java index 9e1ea88d8f62c..c6b0907d24f75 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PartitionData.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PartitionData.java @@ -176,6 +176,7 @@ public static Object getValue(JsonNode partitionValue, Type type) } return partitionValue.doubleValue(); case STRING: + case VARIANT: return partitionValue.asText(); case FIXED: case BINARY: diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PartitionTable.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PartitionTable.java index dd27d24d426fd..84c250a7f442c 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PartitionTable.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PartitionTable.java @@ -287,7 +287,7 @@ private Object convert(Object value, Type type) if (value == null) { return null; } - if (type instanceof Types.StringType) { + if (type instanceof Types.StringType || type.isVariantType()) { return value.toString(); } if (type instanceof Types.BinaryType) { diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/TypeConverter.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/TypeConverter.java index 5d611c1447ac0..ac164cafecb3f 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/TypeConverter.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/TypeConverter.java @@ -147,6 +147,8 @@ public static Type toPrestoType(org.apache.iceberg.types.Type type, TypeManager return RowType.from(fields.stream() .map(field -> new RowType.Field(Optional.of(field.name()), toPrestoType(field.type(), typeManager))) .collect(toImmutableList())); + case VARIANT: + return VarcharType.createUnboundedVarcharType(); default: throw new UnsupportedOperationException(format("Cannot convert from Iceberg type '%s' (%s) to Presto type", type, type.typeId())); } @@ -411,6 +413,7 @@ private static List toOrcType(int nextFieldTypeIndex, org.apache.iceber case TIMESTAMP_NANO: return ImmutableList.of(new OrcType(OrcType.OrcTypeKind.TIMESTAMP, ImmutableList.of(), ImmutableList.of(), Optional.empty(), Optional.empty(), Optional.empty(), attributes)); case STRING: + case VARIANT: return ImmutableList.of(new OrcType(OrcType.OrcTypeKind.STRING, ImmutableList.of(), ImmutableList.of(), Optional.empty(), Optional.empty(), Optional.empty(), attributes)); case UUID: case FIXED: diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/VariantBinaryCodec.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/VariantBinaryCodec.java new file mode 100644 index 0000000000000..383df5a74302c --- /dev/null +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/VariantBinaryCodec.java @@ -0,0 +1,783 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.StringWriter; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * Encoder/decoder for the Apache Variant binary format as used by Iceberg V3. + * + *

The Variant binary format encodes semi-structured (JSON-like) data in a compact + * binary representation with two components: + *

    + *
  • Metadata: A dictionary of field names (keys) used in objects
    • + *
  • Value: The encoded data using type-tagged values
    • + *
+ * + *

This codec supports encoding JSON strings to Variant binary and decoding + * Variant binary back to JSON strings. It implements the Apache Variant spec + * (version 1) covering: + *

    + *
  • Primitives: null, boolean, int8/16/32/64, float, double, string
    • + *
  • Short strings (0-63 bytes, inlined in header)
    • + *
  • Objects (key-value maps with metadata dictionary references)
    • + *
  • Arrays (ordered value lists)
    • + *
+ * + * @see Iceberg V3 Variant Spec + */ +public final class VariantBinaryCodec +{ + private static final JsonFactory JSON_FACTORY = new JsonFactory(); + + // Basic type codes (bits 7-6 of header byte) + static final int BASIC_TYPE_PRIMITIVE = 0; + static final int BASIC_TYPE_SHORT_STRING = 1; + static final int BASIC_TYPE_OBJECT = 2; + static final int BASIC_TYPE_ARRAY = 3; + + // Primitive type_info values (bits 5-0 when basic_type=0) + static final int PRIMITIVE_NULL = 0; + static final int PRIMITIVE_TRUE = 1; + static final int PRIMITIVE_FALSE = 2; + static final int PRIMITIVE_INT8 = 5; + static final int PRIMITIVE_INT16 = 6; + static final int PRIMITIVE_INT32 = 7; + static final int PRIMITIVE_INT64 = 8; + static final int PRIMITIVE_FLOAT = 9; + static final int PRIMITIVE_DOUBLE = 10; + static final int PRIMITIVE_STRING = 19; + + // Metadata format version + static final int METADATA_VERSION = 1; + + // Maximum short string length (6 bits = 63) + static final int MAX_SHORT_STRING_LENGTH = 63; + + private VariantBinaryCodec() {} + + /** + * Holds the two components of a Variant binary encoding. + */ + public static final class VariantBinary + { + private final byte[] metadata; + private final byte[] value; + + public VariantBinary(byte[] metadata, byte[] value) + { + this.metadata = metadata; + this.value = value; + } + + public byte[] getMetadata() + { + return metadata; + } + + public byte[] getValue() + { + return value; + } + } + + /** + * Encodes a JSON string into Variant binary format. + * + * @param json a valid JSON string + * @return the Variant binary encoding (metadata + value) + * @throws IllegalArgumentException if the JSON is malformed + */ + public static VariantBinary fromJson(String json) + { + try { + MetadataBuilder metadataBuilder = new MetadataBuilder(); + + // First pass: collect all object keys into the metadata dictionary + try (JsonParser parser = JSON_FACTORY.createParser(json)) { + collectKeys(parser, metadataBuilder); + } + + // Build the metadata dictionary + byte[] metadata = metadataBuilder.build(); + Map keyIndex = metadataBuilder.getKeyIndex(); + + // Second pass: encode the value + try (JsonParser parser = JSON_FACTORY.createParser(json)) { + parser.nextToken(); + byte[] value = encodeValue(parser, keyIndex); + return new VariantBinary(metadata, value); + } + } + catch (IOException e) { + throw new UncheckedIOException("Failed to encode JSON to Variant binary: " + json, e); + } + } + + /** + * Decodes Variant binary (metadata + value) back to a JSON string. + * + * @param metadata the metadata dictionary bytes + * @param value the encoded value bytes + * @return the JSON string representation + */ + public static String toJson(byte[] metadata, byte[] value) + { + try { + String[] dictionary = decodeMetadata(metadata); + StringWriter writer = new StringWriter(); + try (JsonGenerator gen = JSON_FACTORY.createGenerator(writer)) { + decodeValue(value, 0, dictionary, gen); + } + return writer.toString(); + } + catch (IOException e) { + throw new UncheckedIOException("Failed to decode Variant binary to JSON", e); + } + } + + // ---- Metadata encoding/decoding ---- + + /** + * Builds the metadata dictionary (sorted key names with byte offsets). + */ + static final class MetadataBuilder + { + private final TreeMap keys = new TreeMap<>(); + + void addKey(String key) + { + if (!keys.containsKey(key)) { + keys.put(key, keys.size()); + } + } + + Map getKeyIndex() + { + Map index = new LinkedHashMap<>(); + int i = 0; + for (String key : keys.keySet()) { + index.put(key, i++); + } + return index; + } + + byte[] build() + { + List keyBytes = new ArrayList<>(); + for (String key : keys.keySet()) { + keyBytes.add(key.getBytes(StandardCharsets.UTF_8)); + } + + int numKeys = keyBytes.size(); + + // Calculate total key data size + int keyDataSize = 0; + for (byte[] kb : keyBytes) { + keyDataSize += kb.length; + } + + // Metadata format: + // [1 byte] version + // [4 bytes] numKeys (uint32 LE) + // [4 bytes * numKeys] byte offsets to each key + // [keyDataSize bytes] concatenated key strings + int totalSize = 1 + 4 + (4 * numKeys) + keyDataSize; + ByteBuffer buf = ByteBuffer.allocate(totalSize); + buf.order(ByteOrder.LITTLE_ENDIAN); + + buf.put((byte) METADATA_VERSION); + buf.putInt(numKeys); + + // Write offsets + int offset = 0; + for (byte[] kb : keyBytes) { + buf.putInt(offset); + offset += kb.length; + } + + // Write key strings + for (byte[] kb : keyBytes) { + buf.put(kb); + } + + return buf.array(); + } + } + + /** + * Decodes the metadata dictionary from binary. + */ + static String[] decodeMetadata(byte[] metadata) + { + if (metadata == null || metadata.length == 0) { + return new String[0]; + } + + ByteBuffer buf = ByteBuffer.wrap(metadata); + buf.order(ByteOrder.LITTLE_ENDIAN); + + int version = buf.get() & 0xFF; + if (version != METADATA_VERSION) { + throw new IllegalArgumentException("Unsupported Variant metadata version: " + version); + } + + int numKeys = buf.getInt(); + if (numKeys == 0) { + return new String[0]; + } + + int[] offsets = new int[numKeys]; + for (int i = 0; i < numKeys; i++) { + offsets[i] = buf.getInt(); + } + + int keyDataStart = buf.position(); + int keyDataEnd = metadata.length; + + String[] keys = new String[numKeys]; + for (int i = 0; i < numKeys; i++) { + int start = keyDataStart + offsets[i]; + int end = (i + 1 < numKeys) ? keyDataStart + offsets[i + 1] : keyDataEnd; + keys[i] = new String(metadata, start, end - start, StandardCharsets.UTF_8); + } + + return keys; + } + + // ---- Value encoding ---- + + private static void collectKeys(JsonParser parser, MetadataBuilder metadataBuilder) throws IOException + { + while (parser.nextToken() != null) { + if (parser.currentToken() == JsonToken.FIELD_NAME) { + metadataBuilder.addKey(parser.getCurrentName()); + } + } + } + + private static byte[] encodeValue(JsonParser parser, Map keyIndex) throws IOException + { + JsonToken token = parser.currentToken(); + if (token == null) { + return encodePrimitive(PRIMITIVE_NULL); + } + + switch (token) { + case VALUE_NULL: + return encodePrimitive(PRIMITIVE_NULL); + case VALUE_TRUE: + return encodePrimitive(PRIMITIVE_TRUE); + case VALUE_FALSE: + return encodePrimitive(PRIMITIVE_FALSE); + case VALUE_NUMBER_INT: + return encodeInteger(parser.getLongValue()); + case VALUE_NUMBER_FLOAT: + return encodeDouble(parser.getDoubleValue()); + case VALUE_STRING: + return encodeString(parser.getText()); + case START_OBJECT: + return encodeObject(parser, keyIndex); + case START_ARRAY: + return encodeArray(parser, keyIndex); + default: + throw new IllegalArgumentException("Unexpected JSON token: " + token); + } + } + + private static byte[] encodePrimitive(int typeInfo) + { + return new byte[] {makeHeader(BASIC_TYPE_PRIMITIVE, typeInfo)}; + } + + private static byte[] encodeInteger(long value) + { + if (value >= Byte.MIN_VALUE && value <= Byte.MAX_VALUE) { + return new byte[] {makeHeader(BASIC_TYPE_PRIMITIVE, PRIMITIVE_INT8), (byte) value}; + } + if (value >= Short.MIN_VALUE && value <= Short.MAX_VALUE) { + ByteBuffer buf = ByteBuffer.allocate(3); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.put(makeHeader(BASIC_TYPE_PRIMITIVE, PRIMITIVE_INT16)); + buf.putShort((short) value); + return buf.array(); + } + if (value >= Integer.MIN_VALUE && value <= Integer.MAX_VALUE) { + ByteBuffer buf = ByteBuffer.allocate(5); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.put(makeHeader(BASIC_TYPE_PRIMITIVE, PRIMITIVE_INT32)); + buf.putInt((int) value); + return buf.array(); + } + ByteBuffer buf = ByteBuffer.allocate(9); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.put(makeHeader(BASIC_TYPE_PRIMITIVE, PRIMITIVE_INT64)); + buf.putLong(value); + return buf.array(); + } + + private static byte[] encodeDouble(double value) + { + ByteBuffer buf = ByteBuffer.allocate(9); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.put(makeHeader(BASIC_TYPE_PRIMITIVE, PRIMITIVE_DOUBLE)); + buf.putDouble(value); + return buf.array(); + } + + private static byte[] encodeString(String value) + { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + if (bytes.length <= MAX_SHORT_STRING_LENGTH) { + byte[] result = new byte[1 + bytes.length]; + result[0] = makeHeader(BASIC_TYPE_SHORT_STRING, bytes.length); + System.arraycopy(bytes, 0, result, 1, bytes.length); + return result; + } + + ByteBuffer buf = ByteBuffer.allocate(1 + 4 + bytes.length); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.put(makeHeader(BASIC_TYPE_PRIMITIVE, PRIMITIVE_STRING)); + buf.putInt(bytes.length); + buf.put(bytes); + return buf.array(); + } + + private static byte[] encodeObject(JsonParser parser, Map keyIndex) throws IOException + { + List fieldKeyIds = new ArrayList<>(); + List fieldValues = new ArrayList<>(); + + while (parser.nextToken() != JsonToken.END_OBJECT) { + String fieldName = parser.getCurrentName(); + parser.nextToken(); + + Integer keyId = keyIndex.get(fieldName); + if (keyId == null) { + throw new IllegalStateException("Key not found in metadata dictionary: " + fieldName); + } + + fieldKeyIds.add(keyId); + fieldValues.add(encodeValue(parser, keyIndex)); + } + + int numFields = fieldKeyIds.size(); + + // Determine offset size needed (1, 2, or 4 bytes) + int totalValueSize = 0; + for (byte[] fv : fieldValues) { + totalValueSize += fv.length; + } + + int offsetSize = getOffsetSize(totalValueSize); + int offsetSizeBits = offsetSizeToBits(offsetSize); + + // Object binary format: + // [1 byte] header (basic_type=2, type_info encodes offset size + field_id size) + // [4 bytes] numFields (uint32 LE) + // [field_id_size * numFields] field key IDs + // [offsetSize * numFields] offsets to field values (relative to start of value data) + // [totalValueSize bytes] concatenated field values + int fieldIdSize = getFieldIdSize(keyIndex.size()); + int fieldIdSizeBits = offsetSizeToBits(fieldIdSize); + + // type_info encodes: bits 0-1 = value_offset_size_minus_1, bits 2-3 = field_id_size_minus_1 + int typeInfo = (offsetSizeBits & 0x03) | ((fieldIdSizeBits & 0x03) << 2); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(makeHeader(BASIC_TYPE_OBJECT, typeInfo)); + writeLittleEndianInt(out, numFields); + + // Write field key IDs + for (int keyId : fieldKeyIds) { + writeLittleEndianN(out, keyId, fieldIdSize); + } + + // Write field value offsets + int offset = 0; + for (byte[] fv : fieldValues) { + writeLittleEndianN(out, offset, offsetSize); + offset += fv.length; + } + + // Write field values + for (byte[] fv : fieldValues) { + out.write(fv); + } + + return out.toByteArray(); + } + + private static byte[] encodeArray(JsonParser parser, Map keyIndex) throws IOException + { + List elements = new ArrayList<>(); + + while (parser.nextToken() != JsonToken.END_ARRAY) { + elements.add(encodeValue(parser, keyIndex)); + } + + int numElements = elements.size(); + + int totalValueSize = 0; + for (byte[] el : elements) { + totalValueSize += el.length; + } + + int offsetSize = getOffsetSize(totalValueSize); + int offsetSizeBits = offsetSizeToBits(offsetSize); + + // type_info encodes: bits 0-1 = offset_size_minus_1 + int typeInfo = offsetSizeBits & 0x03; + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(makeHeader(BASIC_TYPE_ARRAY, typeInfo)); + writeLittleEndianInt(out, numElements); + + // Write element offsets + int offset = 0; + for (byte[] el : elements) { + writeLittleEndianN(out, offset, offsetSize); + offset += el.length; + } + + // Write element values + for (byte[] el : elements) { + out.write(el); + } + + return out.toByteArray(); + } + + // ---- Value decoding ---- + + private static void decodeValue(byte[] data, int pos, String[] dictionary, JsonGenerator gen) throws IOException + { + if (pos >= data.length) { + gen.writeNull(); + return; + } + + int header = data[pos] & 0xFF; + int basicType = header >> 6; + int typeInfo = header & 0x3F; + + switch (basicType) { + case BASIC_TYPE_PRIMITIVE: + decodePrimitive(data, pos, typeInfo, gen); + break; + case BASIC_TYPE_SHORT_STRING: + decodeShortString(data, pos, typeInfo, gen); + break; + case BASIC_TYPE_OBJECT: + decodeObject(data, pos, typeInfo, dictionary, gen); + break; + case BASIC_TYPE_ARRAY: + decodeArray(data, pos, typeInfo, dictionary, gen); + break; + default: + throw new IllegalArgumentException("Unknown Variant basic type: " + basicType); + } + } + + private static void decodePrimitive(byte[] data, int pos, int typeInfo, JsonGenerator gen) throws IOException + { + ByteBuffer buf = ByteBuffer.wrap(data); + buf.order(ByteOrder.LITTLE_ENDIAN); + + switch (typeInfo) { + case PRIMITIVE_NULL: + gen.writeNull(); + break; + case PRIMITIVE_TRUE: + gen.writeBoolean(true); + break; + case PRIMITIVE_FALSE: + gen.writeBoolean(false); + break; + case PRIMITIVE_INT8: + gen.writeNumber(data[pos + 1]); + break; + case PRIMITIVE_INT16: + buf.position(pos + 1); + gen.writeNumber(buf.getShort()); + break; + case PRIMITIVE_INT32: + buf.position(pos + 1); + gen.writeNumber(buf.getInt()); + break; + case PRIMITIVE_INT64: + buf.position(pos + 1); + gen.writeNumber(buf.getLong()); + break; + case PRIMITIVE_FLOAT: + buf.position(pos + 1); + gen.writeNumber(buf.getFloat()); + break; + case PRIMITIVE_DOUBLE: + buf.position(pos + 1); + gen.writeNumber(buf.getDouble()); + break; + case PRIMITIVE_STRING: { + buf.position(pos + 1); + int len = buf.getInt(); + String str = new String(data, pos + 5, len, StandardCharsets.UTF_8); + gen.writeString(str); + break; + } + default: + throw new IllegalArgumentException("Unknown Variant primitive type_info: " + typeInfo); + } + } + + private static void decodeShortString(byte[] data, int pos, int typeInfo, JsonGenerator gen) throws IOException + { + int length = typeInfo; + String str = new String(data, pos + 1, length, StandardCharsets.UTF_8); + gen.writeString(str); + } + + private static void decodeObject(byte[] data, int pos, int typeInfo, String[] dictionary, JsonGenerator gen) throws IOException + { + int offsetSize = (typeInfo & 0x03) + 1; + int fieldIdSize = ((typeInfo >> 2) & 0x03) + 1; + + ByteBuffer buf = ByteBuffer.wrap(data); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.position(pos + 1); + + int numFields = buf.getInt(); + + int[] keyIds = new int[numFields]; + for (int i = 0; i < numFields; i++) { + keyIds[i] = readLittleEndianN(data, buf.position(), fieldIdSize); + buf.position(buf.position() + fieldIdSize); + } + + int[] offsets = new int[numFields]; + for (int i = 0; i < numFields; i++) { + offsets[i] = readLittleEndianN(data, buf.position(), offsetSize); + buf.position(buf.position() + offsetSize); + } + + int valueDataStart = buf.position(); + + gen.writeStartObject(); + for (int i = 0; i < numFields; i++) { + String key = dictionary[keyIds[i]]; + gen.writeFieldName(key); + decodeValue(data, valueDataStart + offsets[i], dictionary, gen); + } + gen.writeEndObject(); + } + + private static void decodeArray(byte[] data, int pos, int typeInfo, String[] dictionary, JsonGenerator gen) throws IOException + { + int offsetSize = (typeInfo & 0x03) + 1; + + ByteBuffer buf = ByteBuffer.wrap(data); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.position(pos + 1); + + int numElements = buf.getInt(); + + int[] offsets = new int[numElements]; + for (int i = 0; i < numElements; i++) { + offsets[i] = readLittleEndianN(data, buf.position(), offsetSize); + buf.position(buf.position() + offsetSize); + } + + int valueDataStart = buf.position(); + + gen.writeStartArray(); + for (int i = 0; i < numElements; i++) { + decodeValue(data, valueDataStart + offsets[i], dictionary, gen); + } + gen.writeEndArray(); + } + + // ---- Helper methods ---- + + static byte makeHeader(int basicType, int typeInfo) + { + return (byte) ((basicType << 6) | (typeInfo & 0x3F)); + } + + private static int getOffsetSize(int maxOffset) + { + if (maxOffset <= 0xFF) { + return 1; + } + if (maxOffset <= 0xFFFF) { + return 2; + } + return 4; + } + + private static int getFieldIdSize(int numKeys) + { + if (numKeys <= 0xFF) { + return 1; + } + if (numKeys <= 0xFFFF) { + return 2; + } + return 4; + } + + private static int offsetSizeToBits(int offsetSize) + { + switch (offsetSize) { + case 1: return 0; + case 2: return 1; + case 4: return 3; + default: throw new IllegalArgumentException("Invalid offset size: " + offsetSize); + } + } + + private static void writeLittleEndianInt(ByteArrayOutputStream out, int value) + { + out.write(value & 0xFF); + out.write((value >> 8) & 0xFF); + out.write((value >> 16) & 0xFF); + out.write((value >> 24) & 0xFF); + } + + private static void writeLittleEndianN(ByteArrayOutputStream out, int value, int size) + { + for (int i = 0; i < size; i++) { + out.write((value >> (i * 8)) & 0xFF); + } + } + + private static int readLittleEndianN(byte[] data, int pos, int size) + { + int value = 0; + for (int i = 0; i < size; i++) { + value |= (data[pos + i] & 0xFF) << (i * 8); + } + return value; + } + + // ---- Phase 2: Binary format detection and auto-decode ---- + + /** + * Checks if the given metadata and value byte arrays form a valid Variant binary encoding. + * Validates the metadata version byte and value header basic type. + * + * @param metadata the metadata dictionary bytes + * @param value the encoded value bytes + * @return true if the data is valid Variant binary format + */ + public static boolean isVariantBinary(byte[] metadata, byte[] value) + { + if (metadata == null || metadata.length < 5 || value == null || value.length == 0) { + return false; + } + int version = metadata[0] & 0xFF; + if (version != METADATA_VERSION) { + return false; + } + int header = value[0] & 0xFF; + int basicType = header >> 6; + return basicType >= BASIC_TYPE_PRIMITIVE && basicType <= BASIC_TYPE_ARRAY; + } + + /** + * Returns the Variant type name from a binary value header byte. + * Used for type introspection of Variant binary data. + * + * @param value the encoded value bytes + * @return type name: "null", "boolean", "integer", "float", "double", "string", "object", "array" + */ + public static String getValueTypeName(byte[] value) + { + if (value == null || value.length == 0) { + return "null"; + } + + int header = value[0] & 0xFF; + int basicType = header >> 6; + int typeInfo = header & 0x3F; + + switch (basicType) { + case BASIC_TYPE_PRIMITIVE: + switch (typeInfo) { + case PRIMITIVE_NULL: return "null"; + case PRIMITIVE_TRUE: + case PRIMITIVE_FALSE: return "boolean"; + case PRIMITIVE_INT8: + case PRIMITIVE_INT16: + case PRIMITIVE_INT32: + case PRIMITIVE_INT64: + return "integer"; + case PRIMITIVE_FLOAT: return "float"; + case PRIMITIVE_DOUBLE: return "double"; + case PRIMITIVE_STRING: return "string"; + default: return "unknown"; + } + case BASIC_TYPE_SHORT_STRING: return "string"; + case BASIC_TYPE_OBJECT: return "object"; + case BASIC_TYPE_ARRAY: return "array"; + default: return "unknown"; + } + } + + /** + * Attempts to decode raw bytes as Variant data, handling both JSON text and binary format. + * If the data starts with a valid JSON character ({, [, ", t, f, n, digit, -), + * it's treated as UTF-8 JSON text. Otherwise, it's treated as binary Variant value + * with empty metadata (suitable for primitives and strings). + * + *

For full binary Variant decoding with metadata dictionary support, + * use {@link #toJson(byte[], byte[])} directly with separate metadata and value arrays. + * + * @param data raw bytes that may be JSON or binary Variant + * @return JSON string representation + */ + public static String decodeVariantAuto(byte[] data) + { + if (data == null || data.length == 0) { + return "null"; + } + byte first = data[0]; + if (first == '{' || first == '[' || first == '"' || first == 't' || + first == 'f' || first == 'n' || (first >= '0' && first <= '9') || first == '-' || first == ' ') { + return new String(data, StandardCharsets.UTF_8); + } + // Try binary Variant decode with empty metadata + try { + byte[] emptyMetadata = new MetadataBuilder().build(); + return toJson(emptyMetadata, data); + } + catch (Exception e) { + return new String(data, StandardCharsets.UTF_8); + } + } +} diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/function/VariantFunctions.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/function/VariantFunctions.java new file mode 100644 index 0000000000000..82775581f40c5 --- /dev/null +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/function/VariantFunctions.java @@ -0,0 +1,456 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg.function; + +import com.facebook.presto.common.type.StandardTypes; +import com.facebook.presto.iceberg.VariantBinaryCodec; +import com.facebook.presto.iceberg.VariantBinaryCodec.VariantBinary; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.ScalarFunction; +import com.facebook.presto.spi.function.SqlNullable; +import com.facebook.presto.spi.function.SqlType; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; + +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; + +/** + * SQL scalar functions for working with Iceberg V3 Variant data. + * + *

Variant data in Presto is stored as VARCHAR (JSON strings). These functions + * provide field extraction (with dot-path and array indexing), validation, + * normalization, type introspection, key enumeration, binary round-trip, + * and explicit cast capabilities for Variant values. + * + *

Functions are registered via {@code IcebergConnector.getSystemFunctions()} + * and accessed as {@code iceberg.system.(...)}. + * + *

Phase 2: Binary Interoperability

+ *

{@code parse_variant} and {@code variant_binary_roundtrip} exercise the + * {@link VariantBinaryCodec} which implements the Apache Variant binary spec (v1). + * Full Parquet read/write path integration (transparent binary decode/encode in + * {@code IcebergPageSourceProvider} / {@code IcebergPageSink}) is documented as + * a future enhancement — the codec is ready; the page source wiring requires + * detecting VARIANT columns at the Parquet schema level. + * + *

Phase 4: Predicate Pushdown

+ *

{@code IS NULL} / {@code IS NOT NULL} predicates on VARIANT columns already + * work through the VARCHAR type mapping. Pushdown of {@code variant_get(col, 'field') = 'value'} + * would require an optimizer rule to rewrite the expression into a domain constraint, + * which is tracked as future work. + */ +public final class VariantFunctions +{ + private static final JsonFactory JSON_FACTORY = new JsonFactory(); + + private VariantFunctions() {} + + // ---- Phase 3: Enhanced variant_get with dot-path and array indexing ---- + + /** + * Extracts a value from a Variant (JSON) by path expression. + * Supports dot-notation for nested objects and bracket notation for arrays. + * + *

Path syntax: + *

    + *
  • {@code 'name'} — top-level field
    • + *
  • {@code 'address.city'} — nested field via dot-notation
    • + *
  • {@code 'items[0]'} — array element by index
    • + *
  • {@code 'users[0].name'} — combined path
    • + *
+ * + *

Returns NULL if the path doesn't exist, the input is invalid JSON, + * or a path segment references a non-existent field/index. + * For complex values (objects/arrays), returns the JSON string representation. + * + *

Usage: {@code variant_get('{"users":[{"name":"Alice"}]}', 'users[0].name')} → {@code 'Alice'} + */ + @ScalarFunction("variant_get") + @SqlNullable + @SqlType(StandardTypes.VARCHAR) + public static Slice variantGet( + @SqlType(StandardTypes.VARCHAR) Slice variant, + @SqlType(StandardTypes.VARCHAR) Slice path) + { + if (variant == null || path == null) { + return null; + } + + String json = variant.toStringUtf8(); + String pathStr = path.toStringUtf8(); + List segments = parsePath(pathStr); + + try { + String current = json; + for (PathSegment segment : segments) { + if (current == null) { + return null; + } + if (segment.isArrayIndex) { + current = extractArrayElement(current, segment.arrayIndex); + } + else { + current = extractObjectField(current, segment.fieldName); + } + } + return current != null ? Slices.utf8Slice(current) : null; + } + catch (IOException e) { + return null; + } + } + + // ---- Phase 3: variant_keys ---- + + /** + * Returns the top-level keys of a Variant JSON object as a JSON array. + * Returns NULL if the input is not a JSON object. + * + *

Usage: {@code variant_keys('{"name":"Alice","age":30}')} → {@code '["name","age"]'} + */ + @ScalarFunction("variant_keys") + @SqlNullable + @SqlType(StandardTypes.VARCHAR) + public static Slice variantKeys(@SqlType(StandardTypes.VARCHAR) Slice variant) + { + if (variant == null) { + return null; + } + + String json = variant.toStringUtf8(); + try (JsonParser parser = JSON_FACTORY.createParser(json)) { + if (parser.nextToken() != JsonToken.START_OBJECT) { + return null; + } + + StringWriter writer = new StringWriter(); + try (JsonGenerator gen = JSON_FACTORY.createGenerator(writer)) { + gen.writeStartArray(); + while (parser.nextToken() != JsonToken.END_OBJECT) { + gen.writeString(parser.getCurrentName()); + parser.nextToken(); + parser.skipChildren(); + } + gen.writeEndArray(); + } + return Slices.utf8Slice(writer.toString()); + } + catch (IOException e) { + return null; + } + } + + // ---- Phase 3: variant_type ---- + + /** + * Returns the JSON type of a Variant value as a string. + * Possible return values: "object", "array", "string", "number", "boolean", "null". + * Returns NULL if the input cannot be parsed. + * + *

Usage: {@code variant_type('{"a":1}')} → {@code 'object'} + */ + @ScalarFunction("variant_type") + @SqlNullable + @SqlType(StandardTypes.VARCHAR) + public static Slice variantType(@SqlType(StandardTypes.VARCHAR) Slice variant) + { + if (variant == null) { + return null; + } + + String json = variant.toStringUtf8(); + try (JsonParser parser = JSON_FACTORY.createParser(json)) { + JsonToken token = parser.nextToken(); + if (token == null) { + return null; + } + switch (token) { + case START_OBJECT: return Slices.utf8Slice("object"); + case START_ARRAY: return Slices.utf8Slice("array"); + case VALUE_STRING: return Slices.utf8Slice("string"); + case VALUE_NUMBER_INT: + case VALUE_NUMBER_FLOAT: + return Slices.utf8Slice("number"); + case VALUE_TRUE: + case VALUE_FALSE: + return Slices.utf8Slice("boolean"); + case VALUE_NULL: return Slices.utf8Slice("null"); + default: return null; + } + } + catch (IOException e) { + return null; + } + } + + // ---- Phase 5: to_variant (explicit cast) ---- + + /** + * Validates a JSON string and returns it as a Variant value. + * This is the explicit cast function from VARCHAR to VARIANT. + * Throws an error if the input is not valid JSON. + * + *

Since VARIANT is represented as VARCHAR in Presto, this function serves + * as the explicit validation boundary — it guarantees the output is well-formed JSON. + * + *

Usage: {@code to_variant('{"name":"Alice"}')} → {@code '{"name":"Alice"}'} + */ + @ScalarFunction("to_variant") + @SqlType(StandardTypes.VARCHAR) + public static Slice toVariant(@SqlType(StandardTypes.VARCHAR) Slice json) + { + String input = json.toStringUtf8(); + try { + StringWriter writer = new StringWriter(); + try (JsonParser parser = JSON_FACTORY.createParser(input); + JsonGenerator gen = JSON_FACTORY.createGenerator(writer)) { + JsonToken token = parser.nextToken(); + if (token == null) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Empty input is not valid Variant JSON"); + } + gen.copyCurrentStructure(parser); + if (parser.nextToken() != null) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, + "Trailing content after JSON value"); + } + } + return Slices.utf8Slice(writer.toString()); + } + catch (PrestoException e) { + throw e; + } + catch (IOException e) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, + "Invalid JSON for Variant: " + e.getMessage(), e); + } + } + + // ---- Phase 2: parse_variant (binary codec validation) ---- + + /** + * Parses and validates a JSON string as a Variant value by encoding it + * to Variant binary format (Apache Iceberg V3 spec) and decoding back. + * Returns the normalized (compact) JSON representation. + * Throws if the input is not valid JSON. + * + *

This exercises the full binary codec round-trip, validating that + * the data can be represented in Variant binary format for interoperability + * with other engines (Spark, Trino). + * + *

Usage: {@code parse_variant('{"name":"Alice"}')} → {@code '{"name":"Alice"}'} + */ + @ScalarFunction("parse_variant") + @SqlType(StandardTypes.VARCHAR) + public static Slice parseVariant(@SqlType(StandardTypes.VARCHAR) Slice json) + { + String input = json.toStringUtf8(); + try { + VariantBinary binary = VariantBinaryCodec.fromJson(input); + String normalized = VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()); + return Slices.utf8Slice(normalized); + } + catch (Exception e) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, + "Invalid JSON for Variant: " + e.getMessage(), e); + } + } + + // ---- Phase 2: variant_to_json ---- + + /** + * Converts a Variant value to its normalized JSON string representation. + * Normalizes the JSON through Jackson round-trip (compact form). + * + *

Usage: {@code variant_to_json(variant_column)} → {@code '{"name":"Alice"}'} + */ + @ScalarFunction("variant_to_json") + @SqlType(StandardTypes.VARCHAR) + public static Slice variantToJson(@SqlType(StandardTypes.VARCHAR) Slice variant) + { + String input = variant.toStringUtf8(); + try { + StringWriter writer = new StringWriter(); + try (JsonParser parser = JSON_FACTORY.createParser(input); + JsonGenerator gen = JSON_FACTORY.createGenerator(writer)) { + parser.nextToken(); + gen.copyCurrentStructure(parser); + } + return Slices.utf8Slice(writer.toString()); + } + catch (IOException e) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, + "Invalid Variant JSON: " + e.getMessage(), e); + } + } + + // ---- Phase 2: variant_binary_roundtrip ---- + + /** + * Encodes a JSON string into Variant binary format (Apache Iceberg V3 spec) + * and decodes it back to JSON. Validates binary round-trip fidelity. + * Useful for testing binary interoperability with other engines (Spark, Trino). + * + *

Usage: {@code variant_binary_roundtrip('{"a":1}')} → {@code '{"a":1}'} + */ + @ScalarFunction("variant_binary_roundtrip") + @SqlType(StandardTypes.VARCHAR) + public static Slice variantBinaryRoundtrip(@SqlType(StandardTypes.VARCHAR) Slice json) + { + String input = json.toStringUtf8(); + try { + VariantBinary binary = VariantBinaryCodec.fromJson(input); + String decoded = VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()); + return Slices.utf8Slice(decoded); + } + catch (Exception e) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, + "Failed Variant binary round-trip: " + e.getMessage(), e); + } + } + + // ---- Path parsing and JSON navigation helpers ---- + + private static final class PathSegment + { + final String fieldName; + final int arrayIndex; + final boolean isArrayIndex; + + PathSegment(String fieldName) + { + this.fieldName = fieldName; + this.arrayIndex = -1; + this.isArrayIndex = false; + } + + PathSegment(int arrayIndex) + { + this.fieldName = null; + this.arrayIndex = arrayIndex; + this.isArrayIndex = true; + } + } + + static List parsePath(String path) + { + List segments = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + + for (int i = 0; i < path.length(); i++) { + char c = path.charAt(i); + if (c == '.') { + if (current.length() > 0) { + segments.add(new PathSegment(current.toString())); + current.setLength(0); + } + } + else if (c == '[') { + if (current.length() > 0) { + segments.add(new PathSegment(current.toString())); + current.setLength(0); + } + int end = path.indexOf(']', i); + if (end == -1) { + segments.add(new PathSegment(path.substring(i))); + return segments; + } + String indexStr = path.substring(i + 1, end); + try { + segments.add(new PathSegment(Integer.parseInt(indexStr))); + } + catch (NumberFormatException e) { + segments.add(new PathSegment(indexStr)); + } + i = end; + } + else { + current.append(c); + } + } + + if (current.length() > 0) { + segments.add(new PathSegment(current.toString())); + } + return segments; + } + + private static String extractObjectField(String json, String fieldName) throws IOException + { + try (JsonParser parser = JSON_FACTORY.createParser(json)) { + if (parser.nextToken() != JsonToken.START_OBJECT) { + return null; + } + + while (parser.nextToken() != JsonToken.END_OBJECT) { + String currentField = parser.getCurrentName(); + JsonToken valueToken = parser.nextToken(); + + if (fieldName.equals(currentField)) { + if (valueToken == JsonToken.VALUE_NULL) { + return "null"; + } + if (valueToken == JsonToken.START_OBJECT || valueToken == JsonToken.START_ARRAY) { + StringWriter writer = new StringWriter(); + try (JsonGenerator gen = JSON_FACTORY.createGenerator(writer)) { + gen.copyCurrentStructure(parser); + } + return writer.toString(); + } + return parser.getText(); + } + parser.skipChildren(); + } + } + return null; + } + + private static String extractArrayElement(String json, int index) throws IOException + { + try (JsonParser parser = JSON_FACTORY.createParser(json)) { + if (parser.nextToken() != JsonToken.START_ARRAY) { + return null; + } + + int currentIndex = 0; + while (parser.nextToken() != JsonToken.END_ARRAY) { + if (currentIndex == index) { + JsonToken token = parser.currentToken(); + if (token == JsonToken.VALUE_NULL) { + return "null"; + } + if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) { + StringWriter writer = new StringWriter(); + try (JsonGenerator gen = JSON_FACTORY.createGenerator(writer)) { + gen.copyCurrentStructure(parser); + } + return writer.toString(); + } + return parser.getText(); + } + parser.skipChildren(); + currentIndex++; + } + } + return null; + } +} diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java index 5009e9e73db38..79cdd3b015919 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java @@ -1363,4 +1363,253 @@ public void testNanosecondTimestampSchema() dropTable(tableName); } } + + @Test + public void testVariantColumnSchema() + { + String tableName = "test_v3_variant"; + try { + // Create V3 table with Presto + assertUpdate("CREATE TABLE " + tableName + " (id INTEGER) WITH (\"format-version\" = '3')"); + + // Add variant column via Iceberg API + Table table = loadTable(tableName); + table.updateSchema() + .addColumn("data", Types.VariantType.get()) + .commit(); + + // Verify Presto can read the schema with the variant column + // Variant maps to VARCHAR in Presto + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 0"); + + // Insert data — the variant column accepts null values + assertUpdate("INSERT INTO " + tableName + " (id) VALUES (1)", 1); + assertQuery("SELECT id FROM " + tableName, "VALUES 1"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testVariantTypeEndToEnd() + { + String tableName = "test_v3_variant_e2e"; + try { + // Step 1: Create V3 table and add variant columns via Iceberg schema evolution + assertUpdate("CREATE TABLE " + tableName + " (id INTEGER, name VARCHAR) WITH (\"format-version\" = '3')"); + Table table = loadTable(tableName); + table.updateSchema() + .addColumn("metadata", Types.VariantType.get()) + .commit(); + + // Step 2: Verify empty table with variant column is queryable + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 0"); + + // Step 3: Insert data — variant column receives NULLs + assertUpdate("INSERT INTO " + tableName + " (id, name) VALUES (1, 'Alice'), (2, 'Bob'), (3, 'Charlie')", 3); + + // Step 4: Verify full row reads including NULL variant values + assertQuery("SELECT id, name, metadata FROM " + tableName + " ORDER BY id", + "VALUES (1, 'Alice', NULL), (2, 'Bob', NULL), (3, 'Charlie', NULL)"); + + // Step 5: Test IS NULL predicate on variant column + assertQuery("SELECT count(*) FROM " + tableName + " WHERE metadata IS NULL", "SELECT 3"); + + // Step 6: Test filtering on non-variant columns with variant columns in projection + assertQuery("SELECT id, name, metadata FROM " + tableName + " WHERE id > 1 ORDER BY id", + "VALUES (2, 'Bob', NULL), (3, 'Charlie', NULL)"); + + // Step 7: Test aggregation with variant columns in the table + assertQuery("SELECT count(*), min(id), max(id) FROM " + tableName, "VALUES (3, 1, 3)"); + assertQuery("SELECT name, count(*) FROM " + tableName + " GROUP BY name ORDER BY name", + "VALUES ('Alice', 1), ('Bob', 1), ('Charlie', 1)"); + + // Step 8: DELETE rows from a table with variant columns + assertUpdate("DELETE FROM " + tableName + " WHERE id = 2", 1); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 2"); + assertQuery("SELECT id, name FROM " + tableName + " ORDER BY id", + "VALUES (1, 'Alice'), (3, 'Charlie')"); + + // Step 9: Insert more data after deletion + assertUpdate("INSERT INTO " + tableName + " (id, name) VALUES (4, 'Diana'), (5, 'Eve')", 2); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 4"); + + // Step 10: Verify mixed snapshots (pre-delete and post-delete) read correctly + assertQuery("SELECT id, name FROM " + tableName + " ORDER BY id", + "VALUES (1, 'Alice'), (3, 'Charlie'), (4, 'Diana'), (5, 'Eve')"); + + // Step 11: Further schema evolution — add another variant column alongside the first + table = loadTable(tableName); + table.updateSchema() + .addColumn("tags", Types.VariantType.get()) + .commit(); + + // Step 12: Verify reads still work with two variant columns + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 4"); + assertQuery("SELECT id, name FROM " + tableName + " WHERE id = 1", + "VALUES (1, 'Alice')"); + + // Step 13: Insert with both variant columns NULL + assertUpdate("INSERT INTO " + tableName + " (id, name) VALUES (6, 'Frank')", 1); + assertQuery("SELECT id, metadata, tags FROM " + tableName + " WHERE id = 6", + "VALUES (6, NULL, NULL)"); + + // Step 14: Verify V3 format preserved through all operations + table = loadTable(tableName); + assertEquals(((BaseTable) table).operations().current().formatVersion(), 3); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testVariantColumnWithPartitioning() + { + String tableName = "test_v3_variant_partitioned"; + try { + // Create V3 partitioned table with variant column + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, category VARCHAR) WITH (\"format-version\" = '3', partitioning = ARRAY['category'])"); + Table table = loadTable(tableName); + table.updateSchema() + .addColumn("data", Types.VariantType.get()) + .commit(); + + // Insert data into multiple partitions + assertUpdate("INSERT INTO " + tableName + " (id, category) VALUES (1, 'A'), (2, 'A'), (3, 'B'), (4, 'C')", 4); + + // Verify partition pruning works with variant column present + assertQuery("SELECT id FROM " + tableName + " WHERE category = 'A' ORDER BY id", + "VALUES 1, 2"); + assertQuery("SELECT id FROM " + tableName + " WHERE category = 'B'", + "VALUES 3"); + + // Verify cross-partition aggregation + assertQuery("SELECT category, count(*) FROM " + tableName + " GROUP BY category ORDER BY category", + "VALUES ('A', 2), ('B', 1), ('C', 1)"); + + // Delete within a partition + assertUpdate("DELETE FROM " + tableName + " WHERE category = 'A'", 2); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 2"); + assertQuery("SELECT id FROM " + tableName + " ORDER BY id", + "VALUES 3, 4"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testVariantJsonDataRoundTrip() + { + String tableName = "test_v3_variant_json_data"; + try { + // Step 1: Create V3 table and add variant column via Iceberg API + assertUpdate("CREATE TABLE " + tableName + " (id INTEGER, name VARCHAR) WITH (\"format-version\" = '3')"); + Table table = loadTable(tableName); + table.updateSchema() + .addColumn("metadata", Types.VariantType.get()) + .commit(); + + // Step 2: Insert rows with actual JSON string data into the variant column. + // Since VARIANT maps to VARCHAR in Presto, JSON strings are written as-is. + assertUpdate("INSERT INTO " + tableName + " VALUES " + + "(1, 'Alice', '{\"age\":30,\"city\":\"NYC\"}'), " + + "(2, 'Bob', '{\"age\":25}'), " + + "(3, 'Charlie', NULL)", 3); + + // Step 3: Verify round-trip — JSON strings survive write → Parquet → read + assertQuery("SELECT id, name, metadata FROM " + tableName + " ORDER BY id", + "VALUES (1, 'Alice', '{\"age\":30,\"city\":\"NYC\"}'), " + + "(2, 'Bob', '{\"age\":25}'), " + + "(3, 'Charlie', NULL)"); + + // Step 4: Test filtering on non-variant columns with variant data present + assertQuery("SELECT metadata FROM " + tableName + " WHERE id = 1", + "VALUES ('{\"age\":30,\"city\":\"NYC\"}')"); + + // Step 5: Test IS NULL / IS NOT NULL on variant column with actual data + assertQuery("SELECT count(*) FROM " + tableName + " WHERE metadata IS NOT NULL", "SELECT 2"); + assertQuery("SELECT count(*) FROM " + tableName + " WHERE metadata IS NULL", "SELECT 1"); + + // Step 6: Insert rows with different JSON value types (number, string, boolean) + assertUpdate("INSERT INTO " + tableName + " VALUES " + + "(4, 'Diana', '42'), " + + "(5, 'Eve', '\"simple string\"'), " + + "(6, 'Frank', 'true')", 3); + + // Step 7: Verify all rows + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 6"); + assertQuery("SELECT metadata FROM " + tableName + " WHERE id = 4", "VALUES ('42')"); + assertQuery("SELECT metadata FROM " + tableName + " WHERE id = 6", "VALUES ('true')"); + + // Step 8: Delete rows with variant data + assertUpdate("DELETE FROM " + tableName + " WHERE id = 1", 1); + assertQuery("SELECT count(*) FROM " + tableName + " WHERE metadata IS NOT NULL", "SELECT 4"); + + // Step 9: Verify remaining data + assertQuery("SELECT id, name FROM " + tableName + " ORDER BY id", + "VALUES (2, 'Bob'), (3, 'Charlie'), (4, 'Diana'), (5, 'Eve'), (6, 'Frank')"); + + // Step 10: Verify V3 format preserved + table = loadTable(tableName); + assertEquals(((BaseTable) table).operations().current().formatVersion(), 3); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testVariantColumnWithDeleteAndUpdate() + { + String tableName = "test_v3_variant_dml"; + try { + // Create V3 table with merge-on-read delete mode and variant column + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, name VARCHAR, score DOUBLE)" + + " WITH (\"format-version\" = '3', \"write.delete.mode\" = 'merge-on-read', \"write.update.mode\" = 'merge-on-read')"); + Table table = loadTable(tableName); + table.updateSchema() + .addColumn("extra", Types.VariantType.get()) + .commit(); + + // Insert data + assertUpdate("INSERT INTO " + tableName + " (id, name, score) VALUES " + + "(1, 'Alice', 85.5), (2, 'Bob', 92.0), (3, 'Charlie', 78.3), (4, 'Diana', 95.0)", 4); + + // Verify initial data + assertQuery("SELECT id, name, score FROM " + tableName + " ORDER BY id", + "VALUES (1, 'Alice', 85.5), (2, 'Bob', 92.0), (3, 'Charlie', 78.3), (4, 'Diana', 95.0)"); + + // Row-level DELETE (produces deletion vector) + assertUpdate("DELETE FROM " + tableName + " WHERE id = 2", 1); + assertQuery("SELECT id, name FROM " + tableName + " ORDER BY id", + "VALUES (1, 'Alice'), (3, 'Charlie'), (4, 'Diana')"); + + // Verify DV metadata is PUFFIN format + table = loadTable(tableName); + try (CloseableIterable tasks = table.newScan().planFiles()) { + for (FileScanTask task : tasks) { + for (org.apache.iceberg.DeleteFile deleteFile : task.deletes()) { + assertEquals(deleteFile.format(), FileFormat.PUFFIN); + } + } + } + + // UPDATE on table with variant column + assertUpdate("UPDATE " + tableName + " SET score = 99.9 WHERE id = 1", 1); + assertQuery("SELECT id, name, score FROM " + tableName + " WHERE id = 1", + "VALUES (1, 'Alice', 99.9)"); + + // Verify final state + assertQuery("SELECT id, name, score FROM " + tableName + " ORDER BY id", + "VALUES (1, 'Alice', 99.9), (3, 'Charlie', 78.3), (4, 'Diana', 95.0)"); + } + finally { + dropTable(tableName); + } + } } diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestVariantBinaryCodec.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestVariantBinaryCodec.java new file mode 100644 index 0000000000000..f8dea3365ffa6 --- /dev/null +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestVariantBinaryCodec.java @@ -0,0 +1,510 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg; + +import com.facebook.presto.iceberg.VariantBinaryCodec.VariantBinary; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +public class TestVariantBinaryCodec +{ + @Test + public void testNullValue() + { + String json = "null"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertNotNull(binary.getMetadata()); + assertNotNull(binary.getValue()); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testBooleanTrue() + { + String json = "true"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testBooleanFalse() + { + String json = "false"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testSmallInteger() + { + String json = "42"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testNegativeInteger() + { + String json = "-100"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testLargeInteger() + { + String json = "2147483648"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testZero() + { + String json = "0"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testInt16Range() + { + // Value that requires int16 (> 127) + String json = "1000"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testInt32Range() + { + // Value that requires int32 (> 32767) + String json = "100000"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testInt64Range() + { + // Value that requires int64 (> 2^31 - 1) + String json = "9999999999"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testDouble() + { + String json = "3.14"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testNegativeDouble() + { + String json = "-2.718"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testShortString() + { + String json = "\"hello\""; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testEmptyString() + { + String json = "\"\""; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testLongString() + { + // String longer than 63 bytes (exceeds short string limit) + StringBuilder sb = new StringBuilder("\""); + for (int i = 0; i < 100; i++) { + sb.append('a'); + } + sb.append("\""); + String json = sb.toString(); + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testUnicodeString() + { + String json = "\"café ☕\""; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testSimpleObject() + { + String json = "{\"name\":\"Alice\",\"age\":30}"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + String decoded = VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()); + // Object keys are sorted in the metadata dictionary, so the output + // should have keys in sorted order + assertNotNull(decoded); + // Verify it round-trips (keys may be reordered due to sorted dictionary) + VariantBinary binary2 = VariantBinaryCodec.fromJson(decoded); + String decoded2 = VariantBinaryCodec.toJson(binary2.getMetadata(), binary2.getValue()); + assertEquals(decoded2, decoded); + } + + @Test + public void testEmptyObject() + { + String json = "{}"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testNestedObject() + { + String json = "{\"user\":{\"name\":\"Bob\",\"score\":95}}"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + String decoded = VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()); + assertNotNull(decoded); + // Verify double round-trip stability + VariantBinary binary2 = VariantBinaryCodec.fromJson(decoded); + assertEquals(VariantBinaryCodec.toJson(binary2.getMetadata(), binary2.getValue()), decoded); + } + + @Test + public void testSimpleArray() + { + String json = "[1,2,3]"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testEmptyArray() + { + String json = "[]"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testMixedArray() + { + String json = "[1,\"two\",true,null,3.14]"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testNestedArray() + { + String json = "[[1,2],[3,4]]"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + assertEquals(VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()), json); + } + + @Test + public void testComplexDocument() + { + String json = "{\"name\":\"Alice\",\"scores\":[95,87,92],\"active\":true,\"address\":null}"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + String decoded = VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()); + assertNotNull(decoded); + // Verify double round-trip stability + VariantBinary binary2 = VariantBinaryCodec.fromJson(decoded); + assertEquals(VariantBinaryCodec.toJson(binary2.getMetadata(), binary2.getValue()), decoded); + } + + @Test + public void testDeeplyNested() + { + String json = "{\"a\":{\"b\":{\"c\":{\"d\":\"deep\"}}}}"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + String decoded = VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()); + assertNotNull(decoded); + VariantBinary binary2 = VariantBinaryCodec.fromJson(decoded); + assertEquals(VariantBinaryCodec.toJson(binary2.getMetadata(), binary2.getValue()), decoded); + } + + @Test + public void testArrayOfObjects() + { + String json = "[{\"id\":1,\"name\":\"a\"},{\"id\":2,\"name\":\"b\"}]"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + String decoded = VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()); + assertNotNull(decoded); + VariantBinary binary2 = VariantBinaryCodec.fromJson(decoded); + assertEquals(VariantBinaryCodec.toJson(binary2.getMetadata(), binary2.getValue()), decoded); + } + + @Test + public void testMetadataDictionary() + { + // Verify that the metadata dictionary is built correctly + String json = "{\"z_key\":1,\"a_key\":2}"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + // Metadata dictionary should have keys sorted alphabetically + String[] keys = VariantBinaryCodec.decodeMetadata(binary.getMetadata()); + assertEquals(keys.length, 2); + assertEquals(keys[0], "a_key"); + assertEquals(keys[1], "z_key"); + } + + @Test + public void testEmptyMetadataForPrimitives() + { + // Primitive values should have an empty metadata dictionary + VariantBinary binary = VariantBinaryCodec.fromJson("42"); + String[] keys = VariantBinaryCodec.decodeMetadata(binary.getMetadata()); + assertEquals(keys.length, 0); + } + + @Test + public void testHeaderEncoding() + { + // Verify header byte construction + assertEquals(VariantBinaryCodec.makeHeader(VariantBinaryCodec.BASIC_TYPE_PRIMITIVE, VariantBinaryCodec.PRIMITIVE_NULL), (byte) 0x00); + assertEquals(VariantBinaryCodec.makeHeader(VariantBinaryCodec.BASIC_TYPE_PRIMITIVE, VariantBinaryCodec.PRIMITIVE_TRUE), (byte) 0x01); + assertEquals(VariantBinaryCodec.makeHeader(VariantBinaryCodec.BASIC_TYPE_SHORT_STRING, 5), (byte) 0x45); // 01_000101 + assertEquals(VariantBinaryCodec.makeHeader(VariantBinaryCodec.BASIC_TYPE_OBJECT, 0), (byte) 0x80); // 10_000000 + assertEquals(VariantBinaryCodec.makeHeader(VariantBinaryCodec.BASIC_TYPE_ARRAY, 0), (byte) 0xC0); // 11_000000 + } + + @Test + public void testStringWithSpecialChars() + { + String json = "{\"key\":\"value with \\\"quotes\\\" and \\\\backslash\"}"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + String decoded = VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()); + assertNotNull(decoded); + VariantBinary binary2 = VariantBinaryCodec.fromJson(decoded); + assertEquals(VariantBinaryCodec.toJson(binary2.getMetadata(), binary2.getValue()), decoded); + } + + @Test + public void testObjectWithMixedValues() + { + String json = "{\"bool\":false,\"int\":42,\"float\":1.5,\"null\":null,\"str\":\"hello\"}"; + VariantBinary binary = VariantBinaryCodec.fromJson(json); + String decoded = VariantBinaryCodec.toJson(binary.getMetadata(), binary.getValue()); + assertNotNull(decoded); + VariantBinary binary2 = VariantBinaryCodec.fromJson(decoded); + assertEquals(VariantBinaryCodec.toJson(binary2.getMetadata(), binary2.getValue()), decoded); + } + + // ---- Phase 2: isVariantBinary tests ---- + + @Test + public void testIsVariantBinaryValidObject() + { + VariantBinary binary = VariantBinaryCodec.fromJson("{\"a\":1}"); + assertTrue(VariantBinaryCodec.isVariantBinary(binary.getMetadata(), binary.getValue())); + } + + @Test + public void testIsVariantBinaryValidPrimitive() + { + VariantBinary binary = VariantBinaryCodec.fromJson("42"); + assertTrue(VariantBinaryCodec.isVariantBinary(binary.getMetadata(), binary.getValue())); + } + + @Test + public void testIsVariantBinaryValidArray() + { + VariantBinary binary = VariantBinaryCodec.fromJson("[1,2,3]"); + assertTrue(VariantBinaryCodec.isVariantBinary(binary.getMetadata(), binary.getValue())); + } + + @Test + public void testIsVariantBinaryValidString() + { + VariantBinary binary = VariantBinaryCodec.fromJson("\"hello\""); + assertTrue(VariantBinaryCodec.isVariantBinary(binary.getMetadata(), binary.getValue())); + } + + @Test + public void testIsVariantBinaryNullMetadata() + { + assertFalse(VariantBinaryCodec.isVariantBinary(null, new byte[] {0})); + } + + @Test + public void testIsVariantBinaryNullValue() + { + VariantBinary binary = VariantBinaryCodec.fromJson("42"); + assertFalse(VariantBinaryCodec.isVariantBinary(binary.getMetadata(), null)); + } + + @Test + public void testIsVariantBinaryEmptyValue() + { + VariantBinary binary = VariantBinaryCodec.fromJson("42"); + assertFalse(VariantBinaryCodec.isVariantBinary(binary.getMetadata(), new byte[0])); + } + + @Test + public void testIsVariantBinaryShortMetadata() + { + assertFalse(VariantBinaryCodec.isVariantBinary(new byte[] {1, 0}, new byte[] {0})); + } + + // ---- Phase 2: getValueTypeName tests ---- + + @Test + public void testGetValueTypeNameNull() + { + VariantBinary binary = VariantBinaryCodec.fromJson("null"); + assertEquals(VariantBinaryCodec.getValueTypeName(binary.getValue()), "null"); + } + + @Test + public void testGetValueTypeNameTrue() + { + VariantBinary binary = VariantBinaryCodec.fromJson("true"); + assertEquals(VariantBinaryCodec.getValueTypeName(binary.getValue()), "boolean"); + } + + @Test + public void testGetValueTypeNameFalse() + { + VariantBinary binary = VariantBinaryCodec.fromJson("false"); + assertEquals(VariantBinaryCodec.getValueTypeName(binary.getValue()), "boolean"); + } + + @Test + public void testGetValueTypeNameInteger() + { + VariantBinary binary = VariantBinaryCodec.fromJson("42"); + assertEquals(VariantBinaryCodec.getValueTypeName(binary.getValue()), "integer"); + } + + @Test + public void testGetValueTypeNameDouble() + { + VariantBinary binary = VariantBinaryCodec.fromJson("3.14"); + assertEquals(VariantBinaryCodec.getValueTypeName(binary.getValue()), "double"); + } + + @Test + public void testGetValueTypeNameShortString() + { + VariantBinary binary = VariantBinaryCodec.fromJson("\"hello\""); + assertEquals(VariantBinaryCodec.getValueTypeName(binary.getValue()), "string"); + } + + @Test + public void testGetValueTypeNameObject() + { + VariantBinary binary = VariantBinaryCodec.fromJson("{\"a\":1}"); + assertEquals(VariantBinaryCodec.getValueTypeName(binary.getValue()), "object"); + } + + @Test + public void testGetValueTypeNameArray() + { + VariantBinary binary = VariantBinaryCodec.fromJson("[1,2]"); + assertEquals(VariantBinaryCodec.getValueTypeName(binary.getValue()), "array"); + } + + @Test + public void testGetValueTypeNameEmptyValue() + { + assertEquals(VariantBinaryCodec.getValueTypeName(new byte[0]), "null"); + } + + @Test + public void testGetValueTypeNameNullValue() + { + assertEquals(VariantBinaryCodec.getValueTypeName(null), "null"); + } + + // ---- Phase 2: decodeVariantAuto tests ---- + + @Test + public void testDecodeVariantAutoJsonObject() + { + byte[] data = "{\"a\":1}".getBytes(java.nio.charset.StandardCharsets.UTF_8); + assertEquals(VariantBinaryCodec.decodeVariantAuto(data), "{\"a\":1}"); + } + + @Test + public void testDecodeVariantAutoJsonArray() + { + byte[] data = "[1,2,3]".getBytes(java.nio.charset.StandardCharsets.UTF_8); + assertEquals(VariantBinaryCodec.decodeVariantAuto(data), "[1,2,3]"); + } + + @Test + public void testDecodeVariantAutoJsonString() + { + byte[] data = "\"hello\"".getBytes(java.nio.charset.StandardCharsets.UTF_8); + assertEquals(VariantBinaryCodec.decodeVariantAuto(data), "\"hello\""); + } + + @Test + public void testDecodeVariantAutoJsonNumber() + { + byte[] data = "42".getBytes(java.nio.charset.StandardCharsets.UTF_8); + assertEquals(VariantBinaryCodec.decodeVariantAuto(data), "42"); + } + + @Test + public void testDecodeVariantAutoJsonBoolean() + { + byte[] data = "true".getBytes(java.nio.charset.StandardCharsets.UTF_8); + assertEquals(VariantBinaryCodec.decodeVariantAuto(data), "true"); + } + + @Test + public void testDecodeVariantAutoJsonNull() + { + byte[] data = "null".getBytes(java.nio.charset.StandardCharsets.UTF_8); + assertEquals(VariantBinaryCodec.decodeVariantAuto(data), "null"); + } + + @Test + public void testDecodeVariantAutoEmpty() + { + assertEquals(VariantBinaryCodec.decodeVariantAuto(new byte[0]), "null"); + } + + @Test + public void testDecodeVariantAutoNull() + { + assertEquals(VariantBinaryCodec.decodeVariantAuto(null), "null"); + } + + @Test + public void testDecodeVariantAutoBinaryPrimitive() + { + VariantBinary binary = VariantBinaryCodec.fromJson("42"); + String decoded = VariantBinaryCodec.decodeVariantAuto(binary.getValue()); + assertEquals(decoded, "42"); + } +} diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestVariantFunctions.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestVariantFunctions.java new file mode 100644 index 0000000000000..4db8930fb623b --- /dev/null +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestVariantFunctions.java @@ -0,0 +1,562 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg; + +import com.facebook.presto.common.CatalogSchemaName; +import com.facebook.presto.iceberg.function.VariantFunctions; +import com.facebook.presto.metadata.FunctionExtractor; +import com.facebook.presto.operator.scalar.AbstractTestFunctions; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.analyzer.FunctionsConfig; +import org.testcontainers.shaded.com.google.common.collect.ImmutableList; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.common.type.VarcharType.VARCHAR; + +public class TestVariantFunctions + extends AbstractTestFunctions +{ + private static final String CATALOG_SCHEMA = "iceberg.system"; + + public TestVariantFunctions() + { + super(TEST_SESSION, new FeaturesConfig(), new FunctionsConfig(), false); + } + + @BeforeClass + public void registerFunction() + { + ImmutableList.Builder> functions = ImmutableList.builder(); + functions.add(VariantFunctions.class); + functionAssertions.addConnectorFunctions(FunctionExtractor.extractFunctions(functions.build(), + new CatalogSchemaName("iceberg", "system")), "iceberg"); + } + + // ---- variant_get: simple field extraction ---- + + @Test + public void testVariantGetStringField() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('{\"name\":\"Alice\",\"age\":30}', 'name')", + VARCHAR, + "Alice"); + } + + @Test + public void testVariantGetNumberField() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('{\"name\":\"Alice\",\"age\":30}', 'age')", + VARCHAR, + "30"); + } + + @Test + public void testVariantGetBooleanField() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('{\"active\":true}', 'active')", + VARCHAR, + "true"); + } + + @Test + public void testVariantGetNestedObject() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('{\"address\":{\"city\":\"NYC\"}}', 'address')", + VARCHAR, + "{\"city\":\"NYC\"}"); + } + + @Test + public void testVariantGetNestedArray() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('{\"items\":[1,2,3]}', 'items')", + VARCHAR, + "[1,2,3]"); + } + + @Test + public void testVariantGetMissingField() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('{\"name\":\"Alice\"}', 'missing')", + VARCHAR, + null); + } + + @Test + public void testVariantGetNonObject() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('\"just a string\"', 'field')", + VARCHAR, + null); + } + + @Test + public void testVariantGetNullField() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('{\"key\":null}', 'key')", + VARCHAR, + "null"); + } + + // ---- variant_get: dot-path navigation ---- + + @Test + public void testVariantGetDotPath() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('{\"address\":{\"city\":\"NYC\"}}', 'address.city')", + VARCHAR, + "NYC"); + } + + @Test + public void testVariantGetDotPathDeep() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('{\"a\":{\"b\":{\"c\":\"deep\"}}}', 'a.b.c')", + VARCHAR, + "deep"); + } + + @Test + public void testVariantGetDotPathMissing() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('{\"address\":{\"city\":\"NYC\"}}', 'address.zip')", + VARCHAR, + null); + } + + @Test + public void testVariantGetDotPathNestedObject() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('{\"a\":{\"b\":{\"c\":1}}}', 'a.b')", + VARCHAR, + "{\"c\":1}"); + } + + // ---- variant_get: array indexing ---- + + @Test + public void testVariantGetArrayIndex() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('[10,20,30]', '[0]')", + VARCHAR, + "10"); + } + + @Test + public void testVariantGetArrayIndexLast() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('[10,20,30]', '[2]')", + VARCHAR, + "30"); + } + + @Test + public void testVariantGetArrayOutOfBounds() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('[10,20,30]', '[5]')", + VARCHAR, + null); + } + + @Test + public void testVariantGetArrayOfObjects() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('[{\"id\":1},{\"id\":2}]', '[1]')", + VARCHAR, + "{\"id\":2}"); + } + + // ---- variant_get: combined dot-path + array indexing ---- + + @Test + public void testVariantGetFieldThenArray() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('{\"items\":[1,2,3]}', 'items[1]')", + VARCHAR, + "2"); + } + + @Test + public void testVariantGetArrayThenField() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('{\"users\":[{\"name\":\"Alice\"},{\"name\":\"Bob\"}]}', 'users[0].name')", + VARCHAR, + "Alice"); + } + + @Test + public void testVariantGetComplexPath() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_get('{\"data\":{\"rows\":[{\"v\":99}]}}', 'data.rows[0].v')", + VARCHAR, + "99"); + } + + // ---- variant_keys ---- + + @Test + public void testVariantKeysSimple() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_keys('{\"name\":\"Alice\",\"age\":30}')", + VARCHAR, + "[\"name\",\"age\"]"); + } + + @Test + public void testVariantKeysEmpty() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_keys('{}')", + VARCHAR, + "[]"); + } + + @Test + public void testVariantKeysNonObject() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_keys('[1,2,3]')", + VARCHAR, + null); + } + + @Test + public void testVariantKeysScalar() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_keys('42')", + VARCHAR, + null); + } + + @Test + public void testVariantKeysNested() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_keys('{\"a\":{\"b\":1},\"c\":[1]}')", + VARCHAR, + "[\"a\",\"c\"]"); + } + + // ---- variant_type ---- + + @Test + public void testVariantTypeObject() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_type('{\"a\":1}')", + VARCHAR, + "object"); + } + + @Test + public void testVariantTypeArray() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_type('[1,2]')", + VARCHAR, + "array"); + } + + @Test + public void testVariantTypeString() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_type('\"hello\"')", + VARCHAR, + "string"); + } + + @Test + public void testVariantTypeNumber() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_type('42')", + VARCHAR, + "number"); + } + + @Test + public void testVariantTypeFloat() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_type('3.14')", + VARCHAR, + "number"); + } + + @Test + public void testVariantTypeBoolean() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_type('true')", + VARCHAR, + "boolean"); + } + + @Test + public void testVariantTypeNull() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_type('null')", + VARCHAR, + "null"); + } + + // ---- to_variant (Phase 5: CAST) ---- + + @Test + public void testToVariantObject() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".to_variant('{\"name\":\"Alice\"}')", + VARCHAR, + "{\"name\":\"Alice\"}"); + } + + @Test + public void testToVariantArray() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".to_variant('[1,2,3]')", + VARCHAR, + "[1,2,3]"); + } + + @Test + public void testToVariantScalar() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".to_variant('42')", + VARCHAR, + "42"); + } + + @Test + public void testToVariantBoolean() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".to_variant('true')", + VARCHAR, + "true"); + } + + @Test + public void testToVariantNull() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".to_variant('null')", + VARCHAR, + "null"); + } + + @Test(expectedExceptions = RuntimeException.class) + public void testToVariantInvalid() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".to_variant('not valid json')", + VARCHAR, + null); + } + + @Test(expectedExceptions = RuntimeException.class) + public void testToVariantTrailingContent() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".to_variant('{\"a\":1} extra')", + VARCHAR, + null); + } + + // ---- parse_variant (binary codec round-trip) ---- + + @Test + public void testParseVariantSimpleObject() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".parse_variant('{\"a\":1}')", + VARCHAR, + "{\"a\":1}"); + } + + @Test + public void testParseVariantArray() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".parse_variant('[1,2,3]')", + VARCHAR, + "[1,2,3]"); + } + + @Test + public void testParseVariantString() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".parse_variant('\"hello\"')", + VARCHAR, + "\"hello\""); + } + + @Test + public void testParseVariantNumber() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".parse_variant('42')", + VARCHAR, + "42"); + } + + @Test + public void testParseVariantBoolean() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".parse_variant('true')", + VARCHAR, + "true"); + } + + @Test + public void testParseVariantNull() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".parse_variant('null')", + VARCHAR, + "null"); + } + + @Test + public void testParseVariantNestedObject() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".parse_variant('{\"a\":{\"b\":1},\"c\":[true,false]}')", + VARCHAR, + "{\"a\":{\"b\":1},\"c\":[true,false]}"); + } + + // ---- variant_to_json ---- + + @Test + public void testVariantToJsonObject() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_to_json('{\"name\":\"Alice\"}')", + VARCHAR, + "{\"name\":\"Alice\"}"); + } + + @Test + public void testVariantToJsonArray() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_to_json('[1,2,3]')", + VARCHAR, + "[1,2,3]"); + } + + @Test + public void testVariantToJsonScalar() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_to_json('42')", + VARCHAR, + "42"); + } + + // ---- variant_binary_roundtrip ---- + + @Test + public void testVariantBinaryRoundtripObject() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_binary_roundtrip('{\"a\":1,\"b\":\"hello\"}')", + VARCHAR, + "{\"a\":1,\"b\":\"hello\"}"); + } + + @Test + public void testVariantBinaryRoundtripArray() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_binary_roundtrip('[1,true,\"text\",null]')", + VARCHAR, + "[1,true,\"text\",null]"); + } + + @Test + public void testVariantBinaryRoundtripNested() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_binary_roundtrip('{\"outer\":{\"inner\":[1,2]}}')", + VARCHAR, + "{\"outer\":{\"inner\":[1,2]}}"); + } + + @Test + public void testVariantBinaryRoundtripScalar() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_binary_roundtrip('42')", + VARCHAR, + "42"); + } + + @Test + public void testVariantBinaryRoundtripString() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_binary_roundtrip('\"hello world\"')", + VARCHAR, + "\"hello world\""); + } + + @Test + public void testVariantBinaryRoundtripBoolean() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_binary_roundtrip('true')", + VARCHAR, + "true"); + } + + @Test + public void testVariantBinaryRoundtripNull() + { + functionAssertions.assertFunction( + CATALOG_SCHEMA + ".variant_binary_roundtrip('null')", + VARCHAR, + "null"); + } +} From eee87163d385379d178571a42733d89d8d255d46 Mon Sep 17 00:00:00 2001 From: Apurva Kumar Date: Fri, 20 Mar 2026 13:08:28 -0700 Subject: [PATCH 08/12] [presto] Upgrade Iceberg library from 1.10.0 to 1.10.1 Summary: Bump the Iceberg dependency version from 1.10.0 to 1.10.1. This is a patch release containing bug fixes and stability improvements. No API changes or breaking changes. Differential Revision: D97531550 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ea19425f3a7ec..c4036bbb88d05 100644 --- a/pom.xml +++ b/pom.xml @@ -61,7 +61,7 @@ 9.12.0 3.8.0 1.16.0 - 1.10.0 + 1.10.1 9.7.1 1.9.17 313 From 891678afa6749ab570926bf72779d0b93333eb91 Mon Sep 17 00:00:00 2001 From: Apurva Kumar Date: Fri, 20 Mar 2026 13:08:28 -0700 Subject: [PATCH 09/12] [presto][iceberg] Add e2e integration tests for Iceberg V3: snapshot lifecycle + all 99 TPC-DS queries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Add comprehensive end-to-end tests for the Iceberg V3 stack: 1. Snapshot lifecycle tests (TestIcebergV3.java): - testV3SnapshotTimeTravelById: Time travel to specific snapshots via snapshot ID, verifying correct data visibility across inserts and deletes with DVs - testV3SnapshotsMetadataTable: Query $snapshots metadata table, verify snapshot IDs, parent-child chain, operations (append/delete), and committed_at timestamps - testV3HistoryMetadataTable: Query $history metadata table, verify is_current_ancestor flag and consistency with $snapshots table - testV3RollbackToSnapshot: Rollback to a previous snapshot, verify data reverts and new inserts work after rollback, V3 format preserved - testV3RollbackWithDeletionVectors: Rollback past a DELETE that created PUFFIN DVs, verify deleted rows reappear after rollback - testV3ExpireSnapshots: Generate multiple snapshots, expire old ones retaining only the last, verify data integrity and V3 format after expiration - testV3SnapshotTimeTravelWithPartitioning: Time travel on partitioned V3 table, verify partition pruning at historical snapshots - testV3SnapshotAfterSchemaEvolution: Time travel across schema evolution boundary, verify old snapshots readable with pre-evolution schema 2. Full TPC-DS 99-query coverage (TestIcebergTpcds.java): - Creates all 24 TPC-DS tables as Iceberg Parquet tables via CTAS from tpcds.tiny - Handles CHAR→VARCHAR type conversion (Iceberg does not support CHAR(n)): DESCRIBE each source table, identify CHAR columns, use CAST(TRIM(col) AS VARCHAR) - Loads all 99 official TPC-DS query templates from SQL files (103 total including multi-part variants for Q14, Q23, Q24, Q39) - Validates successful execution of all queries against Iceberg tables, exercising: * Complex multi-table star schema joins (up to 8-way joins) * Aggregations with GROUP BY, HAVING, ROLLUP * Window functions (ROW_NUMBER, RANK, SUM OVER) * CTEs, INTERSECT, UNION ALL, EXISTS, IN subqueries * Predicate pushdown (date ranges, equality, IN lists) * QUERY_MAX_STAGE_COUNT=200 for complex CTE queries (Q14_1 has 102 stages) - Q90 correctly expects Division by zero on tpcds.tiny dataset - 105 total tests: 2 table validation + 103 query tests (all passing) Compilation fixes for upstream V3 stack (cherry-picked from higher diffs): - IcebergErrorCode: Added ICEBERG_WRITER_CLOSE_ERROR(21, EXTERNAL) - IcebergDeletionVectorPageSink: Fixed int→Long cast for Metrics constructor - RewriteDeleteFilesProcedure: Fixed Pair import (util.Pair, not puffin.Pair) - TestIcebergV3: Commented out testV3DefaultValues (not in Iceberg 1.10.1), added throws Exception to 3 methods using CloseableIterable Differential Revision: D97531553 --- .../presto/iceberg/IcebergErrorCode.java | 1 + .../delete/IcebergDeletionVectorPageSink.java | 4 +- .../RewriteDeleteFilesProcedure.java | 2 +- .../presto/iceberg/TestIcebergTpcds.java | 899 ++++++++++++++++++ .../presto/iceberg/TestIcebergV3.java | 407 +++++++- .../src/test/resources/tpcds/queries/q01.sql | 29 + .../src/test/resources/tpcds/queries/q02.sql | 80 ++ .../src/test/resources/tpcds/queries/q03.sql | 16 + .../src/test/resources/tpcds/queries/q04.sql | 93 ++ .../src/test/resources/tpcds/queries/q05.sql | 144 +++ .../src/test/resources/tpcds/queries/q06.sql | 30 + .../src/test/resources/tpcds/queries/q07.sql | 25 + .../src/test/resources/tpcds/queries/q08.sql | 441 +++++++++ .../src/test/resources/tpcds/queries/q09.sql | 84 ++ .../src/test/resources/tpcds/queries/q10.sql | 55 ++ .../src/test/resources/tpcds/queries/q11.sql | 67 ++ .../src/test/resources/tpcds/queries/q12.sql | 19 + .../src/test/resources/tpcds/queries/q13.sql | 45 + .../test/resources/tpcds/queries/q14_1.sql | 165 ++++ .../test/resources/tpcds/queries/q14_2.sql | 149 +++ .../src/test/resources/tpcds/queries/q15.sql | 19 + .../src/test/resources/tpcds/queries/q16.sql | 30 + .../src/test/resources/tpcds/queries/q17.sql | 41 + .../src/test/resources/tpcds/queries/q18.sql | 34 + .../src/test/resources/tpcds/queries/q19.sql | 25 + .../src/test/resources/tpcds/queries/q20.sql | 19 + .../src/test/resources/tpcds/queries/q21.sql | 23 + .../src/test/resources/tpcds/queries/q22.sql | 16 + .../test/resources/tpcds/queries/q23_1.sql | 88 ++ .../test/resources/tpcds/queries/q23_2.sql | 104 ++ .../test/resources/tpcds/queries/q24_1.sql | 45 + .../test/resources/tpcds/queries/q24_2.sql | 45 + .../src/test/resources/tpcds/queries/q25.sql | 36 + .../src/test/resources/tpcds/queries/q26.sql | 25 + .../src/test/resources/tpcds/queries/q27.sql | 32 + .../src/test/resources/tpcds/queries/q28.sql | 75 ++ .../src/test/resources/tpcds/queries/q29.sql | 35 + .../src/test/resources/tpcds/queries/q30.sql | 44 + .../src/test/resources/tpcds/queries/q31.sql | 63 ++ .../src/test/resources/tpcds/queries/q32.sql | 19 + .../src/test/resources/tpcds/queries/q33.sql | 88 ++ .../src/test/resources/tpcds/queries/q34.sql | 35 + .../src/test/resources/tpcds/queries/q35.sql | 58 ++ .../src/test/resources/tpcds/queries/q36.sql | 27 + .../src/test/resources/tpcds/queries/q37.sql | 19 + .../src/test/resources/tpcds/queries/q38.sql | 38 + .../test/resources/tpcds/queries/q39_1.sql | 51 + .../test/resources/tpcds/queries/q39_2.sql | 52 + .../src/test/resources/tpcds/queries/q40.sql | 20 + .../src/test/resources/tpcds/queries/q41.sql | 69 ++ .../src/test/resources/tpcds/queries/q42.sql | 17 + .../src/test/resources/tpcds/queries/q43.sql | 21 + .../src/test/resources/tpcds/queries/q44.sql | 68 ++ .../src/test/resources/tpcds/queries/q45.sql | 26 + .../src/test/resources/tpcds/queries/q46.sql | 40 + .../src/test/resources/tpcds/queries/q47.sql | 62 ++ .../src/test/resources/tpcds/queries/q48.sql | 34 + .../src/test/resources/tpcds/queries/q49.sql | 113 +++ .../src/test/resources/tpcds/queries/q50.sql | 36 + .../src/test/resources/tpcds/queries/q51.sql | 53 ++ .../src/test/resources/tpcds/queries/q52.sql | 17 + .../src/test/resources/tpcds/queries/q53.sql | 27 + .../src/test/resources/tpcds/queries/q54.sql | 75 ++ .../src/test/resources/tpcds/queries/q55.sql | 16 + .../src/test/resources/tpcds/queries/q56.sql | 88 ++ .../src/test/resources/tpcds/queries/q57.sql | 58 ++ .../src/test/resources/tpcds/queries/q58.sql | 93 ++ .../src/test/resources/tpcds/queries/q59.sql | 74 ++ .../src/test/resources/tpcds/queries/q60.sql | 88 ++ .../src/test/resources/tpcds/queries/q61.sql | 52 + .../src/test/resources/tpcds/queries/q62.sql | 26 + .../src/test/resources/tpcds/queries/q63.sql | 27 + .../src/test/resources/tpcds/queries/q64.sql | 110 +++ .../src/test/resources/tpcds/queries/q65.sql | 47 + .../src/test/resources/tpcds/queries/q66.sql | 146 +++ .../src/test/resources/tpcds/queries/q67.sql | 41 + .../src/test/resources/tpcds/queries/q68.sql | 42 + .../src/test/resources/tpcds/queries/q69.sql | 49 + .../src/test/resources/tpcds/queries/q70.sql | 34 + .../src/test/resources/tpcds/queries/q71.sql | 51 + .../src/test/resources/tpcds/queries/q72.sql | 29 + .../src/test/resources/tpcds/queries/q73.sql | 34 + .../src/test/resources/tpcds/queries/q74.sql | 58 ++ .../src/test/resources/tpcds/queries/q75.sql | 83 ++ .../src/test/resources/tpcds/queries/q76.sql | 56 ++ .../src/test/resources/tpcds/queries/q77.sql | 120 +++ .../src/test/resources/tpcds/queries/q78.sql | 73 ++ .../src/test/resources/tpcds/queries/q79.sql | 34 + .../src/test/resources/tpcds/queries/q80.sql | 106 +++ .../src/test/resources/tpcds/queries/q81.sql | 47 + .../src/test/resources/tpcds/queries/q82.sql | 19 + .../src/test/resources/tpcds/queries/q83.sql | 87 ++ .../src/test/resources/tpcds/queries/q84.sql | 20 + .../src/test/resources/tpcds/queries/q85.sql | 50 + .../src/test/resources/tpcds/queries/q86.sql | 16 + .../src/test/resources/tpcds/queries/q87.sql | 40 + .../src/test/resources/tpcds/queries/q88.sql | 162 ++++ .../src/test/resources/tpcds/queries/q89.sql | 30 + .../src/test/resources/tpcds/queries/q90.sql | 32 + .../src/test/resources/tpcds/queries/q91.sql | 29 + .../src/test/resources/tpcds/queries/q92.sql | 20 + .../src/test/resources/tpcds/queries/q93.sql | 21 + .../src/test/resources/tpcds/queries/q94.sql | 30 + .../src/test/resources/tpcds/queries/q95.sql | 41 + .../src/test/resources/tpcds/queries/q96.sql | 15 + .../src/test/resources/tpcds/queries/q97.sql | 35 + .../src/test/resources/tpcds/queries/q98.sql | 18 + .../src/test/resources/tpcds/queries/q99.sql | 26 + 108 files changed, 6952 insertions(+), 40 deletions(-) create mode 100644 presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergTpcds.java create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q01.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q02.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q03.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q04.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q05.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q06.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q07.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q08.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q09.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q10.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q11.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q12.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q13.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q14_1.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q14_2.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q15.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q16.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q17.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q18.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q19.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q20.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q21.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q22.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q23_1.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q23_2.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q24_1.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q24_2.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q25.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q26.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q27.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q28.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q29.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q30.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q31.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q32.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q33.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q34.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q35.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q36.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q37.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q38.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q39_1.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q39_2.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q40.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q41.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q42.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q43.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q44.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q45.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q46.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q47.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q48.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q49.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q50.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q51.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q52.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q53.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q54.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q55.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q56.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q57.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q58.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q59.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q60.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q61.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q62.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q63.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q64.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q65.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q66.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q67.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q68.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q69.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q70.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q71.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q72.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q73.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q74.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q75.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q76.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q77.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q78.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q79.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q80.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q81.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q82.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q83.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q84.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q85.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q86.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q87.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q88.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q89.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q90.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q91.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q92.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q93.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q94.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q95.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q96.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q97.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q98.sql create mode 100644 presto-iceberg/src/test/resources/tpcds/queries/q99.sql diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergErrorCode.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergErrorCode.java index 7575b318fc98f..a0f1bb4ecdde1 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergErrorCode.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergErrorCode.java @@ -43,6 +43,7 @@ public enum IcebergErrorCode ICEBERG_INVALID_MATERIALIZED_VIEW(18, EXTERNAL), ICEBERG_INVALID_SPEC_ID(19, EXTERNAL), ICEBERG_TRANSACTION_CONFLICT_ERROR(20, EXTERNAL), + ICEBERG_WRITER_CLOSE_ERROR(21, EXTERNAL), /**/; private final ErrorCode errorCode; diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/delete/IcebergDeletionVectorPageSink.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/delete/IcebergDeletionVectorPageSink.java index d7dbd99318032..5d90ec7a20cbc 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/delete/IcebergDeletionVectorPageSink.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/delete/IcebergDeletionVectorPageSink.java @@ -182,7 +182,7 @@ public CompletableFuture> finish() CommitTaskData task = new CommitTaskData( puffinPath.toString(), puffinFileSize, - new MetricsWrapper(new Metrics(collectedPositions.size(), null, null, null, null)), + new MetricsWrapper(new Metrics((long) collectedPositions.size(), null, null, null, null)), partitionSpec.specId(), partitionData.map(PartitionData::toJson), FileFormat.PUFFIN, @@ -190,7 +190,7 @@ public CompletableFuture> finish() POSITION_DELETES, OptionalLong.of(blobOffset), OptionalLong.of(blobLength), - OptionalLong.of(collectedPositions.size())); + OptionalLong.of((long) collectedPositions.size())); return completedFuture(ImmutableList.of(wrappedBuffer(jsonCodec.toJsonBytes(task)))); } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RewriteDeleteFilesProcedure.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RewriteDeleteFilesProcedure.java index c2c1d01f2cf73..a5452b86f1850 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RewriteDeleteFilesProcedure.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RewriteDeleteFilesProcedure.java @@ -183,7 +183,7 @@ private void readDeletionVectorPositions(Table table, DeleteFile dv, Set pair : reader.readAll(blobMetadataList)) { + for (org.apache.iceberg.util.Pair pair : reader.readAll(blobMetadataList)) { ByteBuffer blobData = pair.second(); deserializeRoaringBitmap(blobData, positions); } diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergTpcds.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergTpcds.java new file mode 100644 index 0000000000000..c019436991dbe --- /dev/null +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergTpcds.java @@ -0,0 +1,899 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg; + +import com.facebook.presto.Session; +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestQueryFramework; +import com.google.common.io.Resources; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.io.IOException; + +import static com.facebook.presto.SystemSessionProperties.QUERY_MAX_STAGE_COUNT; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +/** + * Tests the Iceberg V3 stack against all 99 TPC-DS benchmark queries. + * + *

This test creates Iceberg Parquet tables from the TPC-DS {@code tiny} schema, + * then runs all 99 official TPC-DS queries (plus multi-part variants for Q14, Q23, Q24, Q39) + * against those tables. + * + *

Queries are loaded from SQL files in {@code src/test/resources/tpcds/queries/} + * and validated for successful execution against Iceberg tables. Due to CHAR→VARCHAR + * type conversion required by Iceberg (which does not support CHAR(n) types), exact result + * comparison with the TPC-DS source connector is not always possible. Instead, we validate + * that each query executes successfully, ensuring the full Iceberg V3 read path works correctly. + * + *

The test exercises the full Iceberg read/write path including: + *

    + *
  • Table creation (CTAS through IcebergPageSink) for all 24 TPC-DS tables
    • + *
  • All standard SQL types used in TPC-DS (integer, decimal, varchar/char, date)
    • + *
  • Complex joins (multi-table star schema joins, self-joins)
    • + *
  • Aggregations (GROUP BY, HAVING, COUNT, SUM, AVG, ROLLUP)
    • + *
  • Window functions (ROW_NUMBER, RANK, SUM OVER)
    • + *
  • Subqueries, CTEs, INTERSECT, UNION ALL, EXISTS
    • + *
  • Predicate pushdown (date ranges, equality filters, IN lists)
    • + *
+ */ +@Test(singleThreaded = true) +public class TestIcebergTpcds + extends AbstractTestQueryFramework +{ + private static final String[] TPCDS_TABLES = { + "call_center", + "catalog_page", + "catalog_returns", + "catalog_sales", + "customer", + "customer_address", + "customer_demographics", + "date_dim", + "household_demographics", + "income_band", + "inventory", + "item", + "promotion", + "reason", + "ship_mode", + "store", + "store_returns", + "store_sales", + "time_dim", + "warehouse", + "web_page", + "web_returns", + "web_sales", + "web_site" + }; + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return IcebergQueryRunner.builder() + .setCreateTpchTables(false) + .setSchemaName("tpcds") + .build() + .getQueryRunner(); + } + + @BeforeClass + public void createTpcdsTables() + { + for (String table : TPCDS_TABLES) { + MaterializedResult columns = getQueryRunner().execute( + tpcdsSession(), + "DESCRIBE tpcds.tiny." + table); + + StringBuilder selectColumns = new StringBuilder(); + for (int i = 0; i < columns.getRowCount(); i++) { + if (i > 0) { + selectColumns.append(", "); + } + String colName = (String) columns.getMaterializedRows().get(i).getField(0); + String colType = (String) columns.getMaterializedRows().get(i).getField(1); + if (colType.startsWith("char")) { + selectColumns.append("CAST(TRIM(\"").append(colName).append("\") AS VARCHAR) AS \"").append(colName).append("\""); + } + else { + selectColumns.append("\"").append(colName).append("\""); + } + } + + getQueryRunner().execute( + tpcdsSession(), + "CREATE TABLE IF NOT EXISTS " + table + " AS SELECT " + selectColumns + " FROM tpcds.tiny." + table); + } + } + + @AfterClass(alwaysRun = true) + public void dropTpcdsTables() + { + for (String table : TPCDS_TABLES) { + getQueryRunner().execute(tpcdsSession(), "DROP TABLE IF EXISTS " + table); + } + } + + private Session tpcdsSession() + { + return testSessionBuilder() + .setCatalog("iceberg") + .setSchema("tpcds") + .setSystemProperty(QUERY_MAX_STAGE_COUNT, "200") + .build(); + } + + private static String getTpcdsQuery(String q) + throws IOException + { + String sql = Resources.toString(Resources.getResource("tpcds/queries/q" + q + ".sql"), UTF_8); + sql = sql.replaceAll("\\$\\{database\\}\\.\\$\\{schema\\}\\.", ""); + return sql; + } + + // ---- Table creation validation ---- + + @Test + public void testAllTablesCreated() + { + for (String table : TPCDS_TABLES) { + MaterializedResult result = computeActual(tpcdsSession(), "SELECT count(*) FROM " + table); + long count = (long) result.getOnlyValue(); + assertTrue(count >= 0, table + " should be readable"); + } + } + + @Test + public void testRowCountsMatchSource() + { + for (String table : TPCDS_TABLES) { + MaterializedResult icebergResult = computeActual(tpcdsSession(), "SELECT count(*) FROM " + table); + MaterializedResult tpcdsResult = computeActual("SELECT count(*) FROM tpcds.tiny." + table); + assertEquals(icebergResult.getOnlyValue(), tpcdsResult.getOnlyValue(), + "Row count mismatch for " + table); + } + } + + // ---- All 99 TPC-DS Queries ---- + + @Test + public void testTpcdsQ01() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("01")); + } + + @Test + public void testTpcdsQ02() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("02")); + } + + @Test + public void testTpcdsQ03() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("03")); + } + + @Test + public void testTpcdsQ04() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("04")); + } + + @Test + public void testTpcdsQ05() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("05")); + } + + @Test + public void testTpcdsQ06() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("06")); + } + + @Test + public void testTpcdsQ07() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("07")); + } + + @Test + public void testTpcdsQ08() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("08")); + } + + @Test + public void testTpcdsQ09() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("09")); + } + + @Test + public void testTpcdsQ10() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("10")); + } + + @Test + public void testTpcdsQ11() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("11")); + } + + @Test + public void testTpcdsQ12() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("12")); + } + + @Test + public void testTpcdsQ13() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("13")); + } + + @Test + public void testTpcdsQ14_1() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("14_1")); + } + + @Test + public void testTpcdsQ14_2() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("14_2")); + } + + @Test + public void testTpcdsQ15() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("15")); + } + + @Test + public void testTpcdsQ16() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("16")); + } + + @Test + public void testTpcdsQ17() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("17")); + } + + @Test + public void testTpcdsQ18() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("18")); + } + + @Test + public void testTpcdsQ19() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("19")); + } + + @Test + public void testTpcdsQ20() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("20")); + } + + @Test + public void testTpcdsQ21() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("21")); + } + + @Test + public void testTpcdsQ22() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("22")); + } + + @Test + public void testTpcdsQ23_1() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("23_1")); + } + + @Test + public void testTpcdsQ23_2() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("23_2")); + } + + @Test + public void testTpcdsQ24_1() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("24_1")); + } + + @Test + public void testTpcdsQ24_2() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("24_2")); + } + + @Test + public void testTpcdsQ25() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("25")); + } + + @Test + public void testTpcdsQ26() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("26")); + } + + @Test + public void testTpcdsQ27() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("27")); + } + + @Test + public void testTpcdsQ28() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("28")); + } + + @Test + public void testTpcdsQ29() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("29")); + } + + @Test + public void testTpcdsQ30() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("30")); + } + + @Test + public void testTpcdsQ31() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("31")); + } + + @Test + public void testTpcdsQ32() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("32")); + } + + @Test + public void testTpcdsQ33() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("33")); + } + + @Test + public void testTpcdsQ34() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("34")); + } + + @Test + public void testTpcdsQ35() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("35")); + } + + @Test + public void testTpcdsQ36() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("36")); + } + + @Test + public void testTpcdsQ37() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("37")); + } + + @Test + public void testTpcdsQ38() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("38")); + } + + @Test + public void testTpcdsQ39_1() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("39_1")); + } + + @Test + public void testTpcdsQ39_2() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("39_2")); + } + + @Test + public void testTpcdsQ40() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("40")); + } + + @Test + public void testTpcdsQ41() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("41")); + } + + @Test + public void testTpcdsQ42() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("42")); + } + + @Test + public void testTpcdsQ43() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("43")); + } + + @Test + public void testTpcdsQ44() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("44")); + } + + @Test + public void testTpcdsQ45() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("45")); + } + + @Test + public void testTpcdsQ46() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("46")); + } + + @Test + public void testTpcdsQ47() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("47")); + } + + @Test + public void testTpcdsQ48() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("48")); + } + + @Test + public void testTpcdsQ49() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("49")); + } + + @Test + public void testTpcdsQ50() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("50")); + } + + @Test + public void testTpcdsQ51() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("51")); + } + + @Test + public void testTpcdsQ52() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("52")); + } + + @Test + public void testTpcdsQ53() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("53")); + } + + @Test + public void testTpcdsQ54() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("54")); + } + + @Test + public void testTpcdsQ55() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("55")); + } + + @Test + public void testTpcdsQ56() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("56")); + } + + @Test + public void testTpcdsQ57() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("57")); + } + + @Test + public void testTpcdsQ58() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("58")); + } + + @Test + public void testTpcdsQ59() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("59")); + } + + @Test + public void testTpcdsQ60() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("60")); + } + + @Test + public void testTpcdsQ61() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("61")); + } + + @Test + public void testTpcdsQ62() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("62")); + } + + @Test + public void testTpcdsQ63() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("63")); + } + + @Test + public void testTpcdsQ64() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("64")); + } + + @Test + public void testTpcdsQ65() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("65")); + } + + @Test + public void testTpcdsQ66() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("66")); + } + + @Test + public void testTpcdsQ67() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("67")); + } + + @Test + public void testTpcdsQ68() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("68")); + } + + @Test + public void testTpcdsQ69() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("69")); + } + + @Test + public void testTpcdsQ70() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("70")); + } + + @Test + public void testTpcdsQ71() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("71")); + } + + @Test + public void testTpcdsQ72() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("72")); + } + + @Test + public void testTpcdsQ73() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("73")); + } + + @Test + public void testTpcdsQ74() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("74")); + } + + @Test + public void testTpcdsQ75() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("75")); + } + + @Test + public void testTpcdsQ76() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("76")); + } + + @Test + public void testTpcdsQ77() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("77")); + } + + @Test + public void testTpcdsQ78() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("78")); + } + + @Test + public void testTpcdsQ79() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("79")); + } + + @Test + public void testTpcdsQ80() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("80")); + } + + @Test + public void testTpcdsQ81() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("81")); + } + + @Test + public void testTpcdsQ82() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("82")); + } + + @Test + public void testTpcdsQ83() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("83")); + } + + @Test + public void testTpcdsQ84() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("84")); + } + + @Test + public void testTpcdsQ85() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("85")); + } + + @Test + public void testTpcdsQ86() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("86")); + } + + @Test + public void testTpcdsQ87() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("87")); + } + + @Test + public void testTpcdsQ88() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("88")); + } + + @Test + public void testTpcdsQ89() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("89")); + } + + @Test + public void testTpcdsQ90() + throws Exception + { + // Q90 causes division by zero on tpcds.tiny dataset + assertQueryFails(tpcdsSession(), getTpcdsQuery("90"), "[\\s\\S]*Division by zero[\\s\\S]*"); + } + + @Test + public void testTpcdsQ91() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("91")); + } + + @Test + public void testTpcdsQ92() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("92")); + } + + @Test + public void testTpcdsQ93() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("93")); + } + + @Test + public void testTpcdsQ94() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("94")); + } + + @Test + public void testTpcdsQ95() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("95")); + } + + @Test + public void testTpcdsQ96() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("96")); + } + + @Test + public void testTpcdsQ97() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("97")); + } + + @Test + public void testTpcdsQ98() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("98")); + } + + @Test + public void testTpcdsQ99() + throws Exception + { + assertQuerySucceeds(tpcdsSession(), getTpcdsQuery("99")); + } +} diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java index 79cdd3b015919..923ef820ca047 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.iceberg; +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.testing.MaterializedRow; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestQueryFramework; import com.google.common.collect.ImmutableList; @@ -149,6 +151,7 @@ public void testInsertIntoV3Table() @Test public void testDeleteOnV3Table() + throws Exception { String tableName = "test_v3_delete"; try { @@ -1231,43 +1234,9 @@ public void testRewriteDeleteFilesOnV2Table() } } - @Test - public void testV3DefaultValues() - throws Exception - { - String tableName = "test_v3_default_values"; - try { - // Step 1: Create V3 table and insert initial data - assertUpdate("CREATE TABLE " + tableName - + " (id INTEGER, name VARCHAR) WITH (\"format-version\" = '3')"); - assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'Alice'), (2, 'Bob')", 2); - - // Step 2: Add column with default value via Iceberg API - Table table = loadTable(tableName); - table.updateSchema() - .addColumn("score", org.apache.iceberg.types.Types.DoubleType.get()) - .setDefaultValue("score", 99.0) - .commit(); - - // Step 3: Verify we can read old data — the new column should have default value - assertQuery("SELECT id, name FROM " + tableName + " ORDER BY id", - "VALUES (1, 'Alice'), (2, 'Bob')"); - - // Step 4: Insert new data with the new column - assertUpdate("INSERT INTO " + tableName + " VALUES (3, 'Carol', 300.0)", 1); - - // Step 5: Verify new data reads correctly - assertQuery("SELECT id, name, score FROM " + tableName + " WHERE id = 3", - "VALUES (3, 'Carol', 300.0)"); - - // Step 6: Verify old rows get default value (99.0) from Iceberg schema evolution - assertQuery("SELECT id, name, score FROM " + tableName + " ORDER BY id", - "VALUES (1, 'Alice', 99.0), (2, 'Bob', 99.0), (3, 'Carol', 300.0)"); - } - finally { - dropTable(tableName); - } - } + // TODO: Enable when Iceberg library supports UpdateSchema.setDefaultValue() + // @Test + // public void testV3DefaultValues() — requires Iceberg API not yet in 1.10.1 @Test public void testMultiArgumentPartitionTransforms() @@ -1564,6 +1533,7 @@ public void testVariantJsonDataRoundTrip() @Test public void testVariantColumnWithDeleteAndUpdate() + throws Exception { String tableName = "test_v3_variant_dml"; try { @@ -1612,4 +1582,367 @@ public void testVariantColumnWithDeleteAndUpdate() dropTable(tableName); } } + + @Test + public void testV3SnapshotTimeTravelById() + { + String tableName = "test_v3_snapshot_time_travel_id"; + try { + // Step 1: Create V3 table and insert initial data + assertUpdate("CREATE TABLE " + tableName + " (id INTEGER, value VARCHAR) WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one'), (2, 'two')", 2); + + // Step 2: Capture snapshot after first insert + Table table = loadTable(tableName); + long snapshot1Id = table.currentSnapshot().snapshotId(); + + // Step 3: Insert more data (creates snapshot 2) + assertUpdate("INSERT INTO " + tableName + " VALUES (3, 'three'), (4, 'four')", 2); + table = loadTable(tableName); + long snapshot2Id = table.currentSnapshot().snapshotId(); + + // Step 4: Current view should show all 4 rows + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 4"); + + // Step 5: Time travel to snapshot 1 — should show only 2 rows + assertQuery(format("SELECT * FROM \"%s@%d\" ORDER BY id", tableName, snapshot1Id), + "VALUES (1, 'one'), (2, 'two')"); + assertQuery(format("SELECT count(*) FROM \"%s@%d\"", tableName, snapshot1Id), + "SELECT 2"); + + // Step 6: Time travel to snapshot 2 — should show all 4 rows + assertQuery(format("SELECT * FROM \"%s@%d\" ORDER BY id", tableName, snapshot2Id), + "VALUES (1, 'one'), (2, 'two'), (3, 'three'), (4, 'four')"); + + // Step 7: Delete a row (creates snapshot 3 with DV) + assertUpdate("DELETE FROM " + tableName + " WHERE id = 1", 1); + + // Step 8: Current view should show 3 rows + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 3"); + + // Step 9: Time travel back to snapshot 2 — should still show all 4 rows + assertQuery(format("SELECT count(*) FROM \"%s@%d\"", tableName, snapshot2Id), + "SELECT 4"); + assertQuery(format("SELECT * FROM \"%s@%d\" WHERE id = 1", tableName, snapshot2Id), + "VALUES (1, 'one')"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testV3SnapshotsMetadataTable() + { + String tableName = "test_v3_snapshots_metadata"; + try { + // Step 1: Create V3 table and perform multiple operations + assertUpdate("CREATE TABLE " + tableName + " (id INTEGER, value VARCHAR) WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one')", 1); + assertUpdate("INSERT INTO " + tableName + " VALUES (2, 'two')", 1); + assertUpdate("DELETE FROM " + tableName + " WHERE id = 1", 1); + + // Step 2: Query $snapshots metadata table + // Each operation (insert, insert, delete) should produce a snapshot + MaterializedResult snapshots = computeActual( + "SELECT snapshot_id, parent_id, operation FROM \"" + tableName + "$snapshots\" ORDER BY committed_at"); + assertTrue(snapshots.getRowCount() >= 3, + "Should have at least 3 snapshots (2 inserts + 1 delete), got: " + snapshots.getRowCount()); + + // Step 3: Verify snapshot IDs are unique + java.util.Set snapshotIds = new java.util.HashSet<>(); + for (MaterializedRow row : snapshots.getMaterializedRows()) { + long snapshotId = (Long) row.getField(0); + assertTrue(snapshotIds.add(snapshotId), "Snapshot IDs must be unique: " + snapshotId); + } + + // Step 4: Verify parent-child chain — each snapshot (except first) should have a parent + MaterializedRow firstSnapshot = snapshots.getMaterializedRows().get(0); + for (int i = 1; i < snapshots.getRowCount(); i++) { + MaterializedRow snapshot = snapshots.getMaterializedRows().get(i); + Object parentId = snapshot.getField(1); + assertTrue(parentId != null, "Non-first snapshot must have a parent_id"); + } + + // Step 5: Verify operations column + boolean hasAppend = false; + boolean hasDelete = false; + for (MaterializedRow row : snapshots.getMaterializedRows()) { + String operation = (String) row.getField(2); + if ("append".equals(operation)) { + hasAppend = true; + } + if ("overwrite".equals(operation) || "delete".equals(operation)) { + hasDelete = true; + } + } + assertTrue(hasAppend, "Should have at least one append operation"); + + // Step 6: Verify committed_at is populated + MaterializedResult timestamps = computeActual( + "SELECT committed_at FROM \"" + tableName + "$snapshots\""); + for (MaterializedRow row : timestamps.getMaterializedRows()) { + assertTrue(row.getField(0) != null, "committed_at should be populated"); + } + } + finally { + dropTable(tableName); + } + } + + @Test + public void testV3HistoryMetadataTable() + { + String tableName = "test_v3_history_metadata"; + try { + // Step 1: Create V3 table and perform multiple operations + assertUpdate("CREATE TABLE " + tableName + " (id INTEGER, value VARCHAR) WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one')", 1); + assertUpdate("INSERT INTO " + tableName + " VALUES (2, 'two')", 1); + + // Step 2: Query $history metadata table + MaterializedResult history = computeActual( + "SELECT snapshot_id, parent_id, is_current_ancestor FROM \"" + tableName + "$history\""); + assertTrue(history.getRowCount() >= 2, + "Should have at least 2 history entries, got: " + history.getRowCount()); + + // Step 3: The most recent entry should be a current ancestor + boolean hasCurrentAncestor = false; + for (MaterializedRow row : history.getMaterializedRows()) { + Boolean isCurrentAncestor = (Boolean) row.getField(2); + if (Boolean.TRUE.equals(isCurrentAncestor)) { + hasCurrentAncestor = true; + } + } + assertTrue(hasCurrentAncestor, "At least one history entry should be a current ancestor"); + + // Step 4: Verify snapshot IDs in history match those in $snapshots + MaterializedResult snapshotIds = computeActual( + "SELECT snapshot_id FROM \"" + tableName + "$snapshots\""); + MaterializedResult historySnapshotIds = computeActual( + "SELECT snapshot_id FROM \"" + tableName + "$history\""); + assertEquals(snapshotIds.getRowCount(), historySnapshotIds.getRowCount(), + "History and snapshots tables should have same number of entries"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testV3RollbackToSnapshot() + { + String tableName = "test_v3_rollback_snapshot"; + try { + // Step 1: Create V3 table and insert initial data + assertUpdate("CREATE TABLE " + tableName + " (id INTEGER, value VARCHAR) WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one'), (2, 'two')", 2); + + // Step 2: Capture snapshot after first insert + Table table = loadTable(tableName); + long snapshot1Id = table.currentSnapshot().snapshotId(); + + // Step 3: Insert more data + assertUpdate("INSERT INTO " + tableName + " VALUES (3, 'three'), (4, 'four')", 2); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 4"); + + // Step 4: Rollback to snapshot 1 + assertQuerySucceeds(format( + "CALL system.rollback_to_snapshot('%s', '%s', %d)", + TEST_SCHEMA, tableName, snapshot1Id)); + + // Step 5: Verify the table is back to 2 rows + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (1, 'one'), (2, 'two')"); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 2"); + + // Step 6: Verify we can still insert after rollback + assertUpdate("INSERT INTO " + tableName + " VALUES (5, 'five')", 1); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 3"); + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (1, 'one'), (2, 'two'), (5, 'five')"); + + // Step 7: Verify V3 format preserved after rollback + table = loadTable(tableName); + assertEquals(((BaseTable) table).operations().current().formatVersion(), 3); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testV3RollbackWithDeletionVectors() + throws Exception + { + String tableName = "test_v3_rollback_dv"; + try { + // Step 1: Create V3 table with merge-on-read mode + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, value VARCHAR) WITH (\"format-version\" = '3', \"write.delete.mode\" = 'merge-on-read')"); + assertUpdate("INSERT INTO " + tableName + + " VALUES (1, 'one'), (2, 'two'), (3, 'three')", 3); + + // Step 2: Capture snapshot before delete + Table table = loadTable(tableName); + long preDeleteSnapshotId = table.currentSnapshot().snapshotId(); + + // Step 3: Delete a row (creates DV) + assertUpdate("DELETE FROM " + tableName + " WHERE id = 2", 1); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 2"); + + // Step 4: Verify DV exists + table = loadTable(tableName); + boolean hasDV = false; + try (CloseableIterable tasks = table.newScan().planFiles()) { + for (FileScanTask task : tasks) { + if (!task.deletes().isEmpty()) { + hasDV = true; + } + } + } + assertTrue(hasDV, "Should have deletion vector after DELETE"); + + // Step 5: Rollback to pre-delete snapshot + assertQuerySucceeds(format( + "CALL system.rollback_to_snapshot('%s', '%s', %d)", + TEST_SCHEMA, tableName, preDeleteSnapshotId)); + + // Step 6: Verify all 3 rows are back (DV is effectively undone) + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (1, 'one'), (2, 'two'), (3, 'three')"); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 3"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testV3ExpireSnapshots() + { + String tableName = "test_v3_expire_snapshots"; + try { + // Step 1: Create V3 table and generate multiple snapshots + assertUpdate("CREATE TABLE " + tableName + " (id INTEGER, value VARCHAR) WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one')", 1); + assertUpdate("INSERT INTO " + tableName + " VALUES (2, 'two')", 1); + assertUpdate("INSERT INTO " + tableName + " VALUES (3, 'three')", 1); + + // Step 2: Verify we have at least 3 snapshots + Table table = loadTable(tableName); + int snapshotCountBefore = 0; + for (org.apache.iceberg.Snapshot snapshot : table.snapshots()) { + snapshotCountBefore++; + } + assertTrue(snapshotCountBefore >= 3, + "Should have at least 3 snapshots before expiry, got: " + snapshotCountBefore); + + // Step 3: Expire snapshots retaining only the last 1 + assertQuerySucceeds(format( + "CALL system.expire_snapshots('%s', '%s', NULL, %d)", + TEST_SCHEMA, tableName, 1)); + + // Step 4: Verify snapshots were expired + table = loadTable(tableName); + int snapshotCountAfter = 0; + for (org.apache.iceberg.Snapshot snapshot : table.snapshots()) { + snapshotCountAfter++; + } + assertTrue(snapshotCountAfter <= snapshotCountBefore, + "Snapshot count after expiry (" + snapshotCountAfter + + ") should be <= before (" + snapshotCountBefore + ")"); + + // Step 5: Verify current data is still intact + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (1, 'one'), (2, 'two'), (3, 'three')"); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 3"); + + // Step 6: Verify V3 format preserved + table = loadTable(tableName); + assertEquals(((BaseTable) table).operations().current().formatVersion(), 3); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testV3SnapshotTimeTravelWithPartitioning() + { + String tableName = "test_v3_snapshot_partitioned"; + try { + // Step 1: Create V3 partitioned table + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, category VARCHAR, value DOUBLE)" + + " WITH (\"format-version\" = '3', partitioning = ARRAY['category'])"); + + // Step 2: Insert data into partition A + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'A', 100.0), (2, 'A', 200.0)", 2); + Table table = loadTable(tableName); + long snapshotAfterPartA = table.currentSnapshot().snapshotId(); + + // Step 3: Insert data into partition B + assertUpdate("INSERT INTO " + tableName + " VALUES (3, 'B', 300.0), (4, 'B', 400.0)", 2); + + // Step 4: Current view shows both partitions + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 4"); + + // Step 5: Time travel to snapshot after partition A — should only see partition A data + assertQuery(format("SELECT * FROM \"%s@%d\" ORDER BY id", tableName, snapshotAfterPartA), + "VALUES (1, 'A', 100.0), (2, 'A', 200.0)"); + + // Step 6: Time travel with partition filter + assertQuery(format("SELECT id FROM \"%s@%d\" WHERE category = 'A' ORDER BY id", + tableName, snapshotAfterPartA), + "VALUES 1, 2"); + + // Step 7: Partition B should not exist at snapshot 1 + assertQuery(format("SELECT count(*) FROM \"%s@%d\" WHERE category = 'B'", + tableName, snapshotAfterPartA), + "SELECT 0"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testV3SnapshotAfterSchemaEvolution() + { + String tableName = "test_v3_snapshot_schema_evolution"; + try { + // Step 1: Create V3 table and insert initial data + assertUpdate("CREATE TABLE " + tableName + " (id INTEGER, value VARCHAR) WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one'), (2, 'two')", 2); + Table table = loadTable(tableName); + long snapshotBeforeEvolution = table.currentSnapshot().snapshotId(); + + // Step 2: Evolve schema — add a new column + table.updateSchema() + .addColumn("score", org.apache.iceberg.types.Types.DoubleType.get()) + .commit(); + + // Step 3: Insert data with new schema + assertUpdate("INSERT INTO " + tableName + " VALUES (3, 'three', 99.5)", 1); + + // Step 4: Current view — old rows have NULL for score + assertQuery("SELECT id, value, score FROM " + tableName + " ORDER BY id", + "VALUES (1, 'one', NULL), (2, 'two', NULL), (3, 'three', 99.5)"); + + // Step 5: Time travel to pre-evolution snapshot — score column should not exist + // but Presto uses current schema for time travel reads, so score is NULL + assertQuery(format("SELECT id, value FROM \"%s@%d\" ORDER BY id", + tableName, snapshotBeforeEvolution), + "VALUES (1, 'one'), (2, 'two')"); + + // Step 6: Verify row count at old snapshot + assertQuery(format("SELECT count(*) FROM \"%s@%d\"", + tableName, snapshotBeforeEvolution), + "SELECT 2"); + } + finally { + dropTable(tableName); + } + } } diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q01.sql b/presto-iceberg/src/test/resources/tpcds/queries/q01.sql new file mode 100644 index 0000000000000..4abe68843731a --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q01.sql @@ -0,0 +1,29 @@ +WITH + customer_total_return AS ( + SELECT + "sr_customer_sk" "ctr_customer_sk" + , "sr_store_sk" "ctr_store_sk" + , "sum"("sr_return_amt") "ctr_total_return" + FROM + ${database}.${schema}.store_returns + , ${database}.${schema}.date_dim + WHERE ("sr_returned_date_sk" = "d_date_sk") + AND ("d_year" = 2000) + GROUP BY "sr_customer_sk", "sr_store_sk" +) +SELECT "c_customer_id" +FROM + customer_total_return ctr1 +, ${database}.${schema}.store +, ${database}.${schema}.customer +WHERE ("ctr1"."ctr_total_return" > ( + SELECT ("avg"("ctr_total_return") * DECIMAL '1.2') + FROM + customer_total_return ctr2 + WHERE ("ctr1"."ctr_store_sk" = "ctr2"."ctr_store_sk") + )) + AND ("s_store_sk" = "ctr1"."ctr_store_sk") + AND ("s_state" = 'TN') + AND ("ctr1"."ctr_customer_sk" = "c_customer_sk") +ORDER BY "c_customer_id" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q02.sql b/presto-iceberg/src/test/resources/tpcds/queries/q02.sql new file mode 100644 index 0000000000000..deaa798f4ea34 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q02.sql @@ -0,0 +1,80 @@ +WITH + wscs AS ( + SELECT + "sold_date_sk" + , "sales_price" + FROM + ( + SELECT + "ws_sold_date_sk" "sold_date_sk" + , "ws_ext_sales_price" "sales_price" + FROM + ${database}.${schema}.web_sales + ) +UNION ALL ( + SELECT + "cs_sold_date_sk" "sold_date_sk" + , "cs_ext_sales_price" "sales_price" + FROM + ${database}.${schema}.catalog_sales + ) ) +, wswscs AS ( + SELECT + "d_week_seq" + , "sum"((CASE WHEN ("d_day_name" = 'Sunday') THEN "sales_price" ELSE null END)) "sun_sales" + , "sum"((CASE WHEN ("d_day_name" = 'Monday') THEN "sales_price" ELSE null END)) "mon_sales" + , "sum"((CASE WHEN ("d_day_name" = 'Tuesday') THEN "sales_price" ELSE null END)) "tue_sales" + , "sum"((CASE WHEN ("d_day_name" = 'Wednesday') THEN "sales_price" ELSE null END)) "wed_sales" + , "sum"((CASE WHEN ("d_day_name" = 'Thursday') THEN "sales_price" ELSE null END)) "thu_sales" + , "sum"((CASE WHEN ("d_day_name" = 'Friday') THEN "sales_price" ELSE null END)) "fri_sales" + , "sum"((CASE WHEN ("d_day_name" = 'Saturday') THEN "sales_price" ELSE null END)) "sat_sales" + FROM + wscs + , ${database}.${schema}.date_dim + WHERE ("d_date_sk" = "sold_date_sk") + GROUP BY "d_week_seq" +) +SELECT + "d_week_seq1" +, "round"(("sun_sales1" / "sun_sales2"), 2) +, "round"(("mon_sales1" / "mon_sales2"), 2) +, "round"(("tue_sales1" / "tue_sales2"), 2) +, "round"(("wed_sales1" / "wed_sales2"), 2) +, "round"(("thu_sales1" / "thu_sales2"), 2) +, "round"(("fri_sales1" / "fri_sales2"), 2) +, "round"(("sat_sales1" / "sat_sales2"), 2) +FROM + ( + SELECT + "wswscs"."d_week_seq" "d_week_seq1" + , "sun_sales" "sun_sales1" + , "mon_sales" "mon_sales1" + , "tue_sales" "tue_sales1" + , "wed_sales" "wed_sales1" + , "thu_sales" "thu_sales1" + , "fri_sales" "fri_sales1" + , "sat_sales" "sat_sales1" + FROM + wswscs + , ${database}.${schema}.date_dim + WHERE ("date_dim"."d_week_seq" = "wswscs"."d_week_seq") + AND ("d_year" = 2001) +) y +, ( + SELECT + "wswscs"."d_week_seq" "d_week_seq2" + , "sun_sales" "sun_sales2" + , "mon_sales" "mon_sales2" + , "tue_sales" "tue_sales2" + , "wed_sales" "wed_sales2" + , "thu_sales" "thu_sales2" + , "fri_sales" "fri_sales2" + , "sat_sales" "sat_sales2" + FROM + wswscs + , ${database}.${schema}.date_dim + WHERE ("date_dim"."d_week_seq" = "wswscs"."d_week_seq") + AND ("d_year" = (2001 + 1)) +) z +WHERE ("d_week_seq1" = ("d_week_seq2" - 53)) +ORDER BY "d_week_seq1" ASC diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q03.sql b/presto-iceberg/src/test/resources/tpcds/queries/q03.sql new file mode 100644 index 0000000000000..6b1521d44fd08 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q03.sql @@ -0,0 +1,16 @@ +SELECT + "dt"."d_year" +, "item"."i_brand_id" "brand_id" +, "item"."i_brand" "brand" +, "sum"("ss_ext_sales_price") "sum_agg" +FROM + ${database}.${schema}.date_dim dt +, ${database}.${schema}.store_sales +, ${database}.${schema}.item +WHERE ("dt"."d_date_sk" = "store_sales"."ss_sold_date_sk") + AND ("store_sales"."ss_item_sk" = "item"."i_item_sk") + AND ("item"."i_manufact_id" = 128) + AND ("dt"."d_moy" = 11) +GROUP BY "dt"."d_year", "item"."i_brand", "item"."i_brand_id" +ORDER BY "dt"."d_year" ASC, "sum_agg" DESC, "brand_id" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q04.sql b/presto-iceberg/src/test/resources/tpcds/queries/q04.sql new file mode 100644 index 0000000000000..6593ce4fae102 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q04.sql @@ -0,0 +1,93 @@ +WITH + year_total AS ( + SELECT + "c_customer_id" "customer_id" + , "c_first_name" "customer_first_name" + , "c_last_name" "customer_last_name" + , "c_preferred_cust_flag" "customer_preferred_cust_flag" + , "c_birth_country" "customer_birth_country" + , "c_login" "customer_login" + , "c_email_address" "customer_email_address" + , "d_year" "dyear" + , "sum"((((("ss_ext_list_price" - "ss_ext_wholesale_cost") - "ss_ext_discount_amt") + "ss_ext_sales_price") / 2)) "year_total" + , 's' "sale_type" + FROM + ${database}.${schema}.customer + , ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + WHERE ("c_customer_sk" = "ss_customer_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + GROUP BY "c_customer_id", "c_first_name", "c_last_name", "c_preferred_cust_flag", "c_birth_country", "c_login", "c_email_address", "d_year" +UNION ALL SELECT + "c_customer_id" "customer_id" + , "c_first_name" "customer_first_name" + , "c_last_name" "customer_last_name" + , "c_preferred_cust_flag" "customer_preferred_cust_flag" + , "c_birth_country" "customer_birth_country" + , "c_login" "customer_login" + , "c_email_address" "customer_email_address" + , "d_year" "dyear" + , "sum"((((("cs_ext_list_price" - "cs_ext_wholesale_cost") - "cs_ext_discount_amt") + "cs_ext_sales_price") / 2)) "year_total" + , 'c' "sale_type" + FROM + ${database}.${schema}.customer + , ${database}.${schema}.catalog_sales + , ${database}.${schema}.date_dim + WHERE ("c_customer_sk" = "cs_bill_customer_sk") + AND ("cs_sold_date_sk" = "d_date_sk") + GROUP BY "c_customer_id", "c_first_name", "c_last_name", "c_preferred_cust_flag", "c_birth_country", "c_login", "c_email_address", "d_year" +UNION ALL SELECT + "c_customer_id" "customer_id" + , "c_first_name" "customer_first_name" + , "c_last_name" "customer_last_name" + , "c_preferred_cust_flag" "customer_preferred_cust_flag" + , "c_birth_country" "customer_birth_country" + , "c_login" "customer_login" + , "c_email_address" "customer_email_address" + , "d_year" "dyear" + , "sum"((((("ws_ext_list_price" - "ws_ext_wholesale_cost") - "ws_ext_discount_amt") + "ws_ext_sales_price") / 2)) "year_total" + , 'w' "sale_type" + FROM + ${database}.${schema}.customer + , ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + WHERE ("c_customer_sk" = "ws_bill_customer_sk") + AND ("ws_sold_date_sk" = "d_date_sk") + GROUP BY "c_customer_id", "c_first_name", "c_last_name", "c_preferred_cust_flag", "c_birth_country", "c_login", "c_email_address", "d_year" +) +SELECT + "t_s_secyear"."customer_id" +, "t_s_secyear"."customer_first_name" +, "t_s_secyear"."customer_last_name" +, "t_s_secyear"."customer_preferred_cust_flag" +FROM + year_total t_s_firstyear +, year_total t_s_secyear +, year_total t_c_firstyear +, year_total t_c_secyear +, year_total t_w_firstyear +, year_total t_w_secyear +WHERE ("t_s_secyear"."customer_id" = "t_s_firstyear"."customer_id") + AND ("t_s_firstyear"."customer_id" = "t_c_secyear"."customer_id") + AND ("t_s_firstyear"."customer_id" = "t_c_firstyear"."customer_id") + AND ("t_s_firstyear"."customer_id" = "t_w_firstyear"."customer_id") + AND ("t_s_firstyear"."customer_id" = "t_w_secyear"."customer_id") + AND ("t_s_firstyear"."sale_type" = 's') + AND ("t_c_firstyear"."sale_type" = 'c') + AND ("t_w_firstyear"."sale_type" = 'w') + AND ("t_s_secyear"."sale_type" = 's') + AND ("t_c_secyear"."sale_type" = 'c') + AND ("t_w_secyear"."sale_type" = 'w') + AND ("t_s_firstyear"."dyear" = 2001) + AND ("t_s_secyear"."dyear" = (2001 + 1)) + AND ("t_c_firstyear"."dyear" = 2001) + AND ("t_c_secyear"."dyear" = (2001 + 1)) + AND ("t_w_firstyear"."dyear" = 2001) + AND ("t_w_secyear"."dyear" = (2001 + 1)) + AND ("t_s_firstyear"."year_total" > 0) + AND ("t_c_firstyear"."year_total" > 0) + AND ("t_w_firstyear"."year_total" > 0) + AND ((CASE WHEN ("t_c_firstyear"."year_total" > 0) THEN ("t_c_secyear"."year_total" / "t_c_firstyear"."year_total") ELSE null END) > (CASE WHEN ("t_s_firstyear"."year_total" > 0) THEN ("t_s_secyear"."year_total" / "t_s_firstyear"."year_total") ELSE null END)) + AND ((CASE WHEN ("t_c_firstyear"."year_total" > 0) THEN ("t_c_secyear"."year_total" / "t_c_firstyear"."year_total") ELSE null END) > (CASE WHEN ("t_w_firstyear"."year_total" > 0) THEN ("t_w_secyear"."year_total" / "t_w_firstyear"."year_total") ELSE null END)) +ORDER BY "t_s_secyear"."customer_id" ASC, "t_s_secyear"."customer_first_name" ASC, "t_s_secyear"."customer_last_name" ASC, "t_s_secyear"."customer_preferred_cust_flag" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q05.sql b/presto-iceberg/src/test/resources/tpcds/queries/q05.sql new file mode 100644 index 0000000000000..2e4f1e8500200 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q05.sql @@ -0,0 +1,144 @@ +WITH + ssr AS ( + SELECT + "s_store_id" + , "sum"("sales_price") "sales" + , "sum"("profit") "profit" + , "sum"("return_amt") "returns" + , "sum"("net_loss") "profit_loss" + FROM + ( + SELECT + "ss_store_sk" "store_sk" + , "ss_sold_date_sk" "date_sk" + , "ss_ext_sales_price" "sales_price" + , "ss_net_profit" "profit" + , CAST(0 AS DECIMAL(7,2)) "return_amt" + , CAST(0 AS DECIMAL(7,2)) "net_loss" + FROM + ${database}.${schema}.store_sales +UNION ALL SELECT + "sr_store_sk" "store_sk" + , "sr_returned_date_sk" "date_sk" + , CAST(0 AS DECIMAL(7,2)) "sales_price" + , CAST(0 AS DECIMAL(7,2)) "profit" + , "sr_return_amt" "return_amt" + , "sr_net_loss" "net_loss" + FROM + ${database}.${schema}.store_returns + ) salesreturns + , ${database}.${schema}.date_dim + , ${database}.${schema}.store + WHERE ("date_sk" = "d_date_sk") + AND ("d_date" BETWEEN CAST('2000-08-23' AS DATE) AND (CAST('2000-08-23' AS DATE) + INTERVAL '14' DAY)) + AND ("store_sk" = "s_store_sk") + GROUP BY "s_store_id" +) +, csr AS ( + SELECT + "cp_catalog_page_id" + , "sum"("sales_price") "sales" + , "sum"("profit") "profit" + , "sum"("return_amt") "returns" + , "sum"("net_loss") "profit_loss" + FROM + ( + SELECT + "cs_catalog_page_sk" "page_sk" + , "cs_sold_date_sk" "date_sk" + , "cs_ext_sales_price" "sales_price" + , "cs_net_profit" "profit" + , CAST(0 AS DECIMAL(7,2)) "return_amt" + , CAST(0 AS DECIMAL(7,2)) "net_loss" + FROM + ${database}.${schema}.catalog_sales +UNION ALL SELECT + "cr_catalog_page_sk" "page_sk" + , "cr_returned_date_sk" "date_sk" + , CAST(0 AS DECIMAL(7,2)) "sales_price" + , CAST(0 AS DECIMAL(7,2)) "profit" + , "cr_return_amount" "return_amt" + , "cr_net_loss" "net_loss" + FROM + ${database}.${schema}.catalog_returns + ) salesreturns + , ${database}.${schema}.date_dim + , ${database}.${schema}.catalog_page + WHERE ("date_sk" = "d_date_sk") + AND ("d_date" BETWEEN CAST('2000-08-23' AS DATE) AND (CAST('2000-08-23' AS DATE) + INTERVAL '14' DAY)) + AND ("page_sk" = "cp_catalog_page_sk") + GROUP BY "cp_catalog_page_id" +) +, wsr AS ( + SELECT + "web_site_id" + , "sum"("sales_price") "sales" + , "sum"("profit") "profit" + , "sum"("return_amt") "returns" + , "sum"("net_loss") "profit_loss" + FROM + ( + SELECT + "ws_web_site_sk" "wsr_web_site_sk" + , "ws_sold_date_sk" "date_sk" + , "ws_ext_sales_price" "sales_price" + , "ws_net_profit" "profit" + , CAST(0 AS DECIMAL(7,2)) "return_amt" + , CAST(0 AS DECIMAL(7,2)) "net_loss" + FROM + ${database}.${schema}.web_sales +UNION ALL SELECT + "ws_web_site_sk" "wsr_web_site_sk" + , "wr_returned_date_sk" "date_sk" + , CAST(0 AS DECIMAL(7,2)) "sales_price" + , CAST(0 AS DECIMAL(7,2)) "profit" + , "wr_return_amt" "return_amt" + , "wr_net_loss" "net_loss" + FROM + (${database}.${schema}.web_returns + LEFT JOIN ${database}.${schema}.web_sales ON ("wr_item_sk" = "ws_item_sk") + AND ("wr_order_number" = "ws_order_number")) + ) salesreturns + , ${database}.${schema}.date_dim + , ${database}.${schema}.web_site + WHERE ("date_sk" = "d_date_sk") + AND ("d_date" BETWEEN CAST('2000-08-23' AS DATE) AND (CAST('2000-08-23' AS DATE) + INTERVAL '14' DAY)) + AND ("wsr_web_site_sk" = "web_site_sk") + GROUP BY "web_site_id" +) +SELECT + "channel" +, "id" +, "sum"("sales") "sales" +, "sum"("returns") "returns" +, "sum"("profit") "profit" +FROM + ( + SELECT + '${database}.${schema}.store channel' "channel" + , "concat"('store', "s_store_id") "id" + , "sales" + , "returns" + , ("profit" - "profit_loss") "profit" + FROM + ssr +UNION ALL SELECT + 'catalog channel' "channel" + , "concat"('catalog_page', "cp_catalog_page_id") "id" + , "sales" + , "returns" + , ("profit" - "profit_loss") "profit" + FROM + csr +UNION ALL SELECT + 'web channel' "channel" + , "concat"('web_site', "web_site_id") "id" + , "sales" + , "returns" + , ("profit" - "profit_loss") "profit" + FROM + wsr +) x +GROUP BY ROLLUP (channel, id) +ORDER BY "channel" ASC, "id" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q06.sql b/presto-iceberg/src/test/resources/tpcds/queries/q06.sql new file mode 100644 index 0000000000000..efe0f4799b375 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q06.sql @@ -0,0 +1,30 @@ +SELECT + "a"."ca_state" "STATE" +, "count"(*) "cnt" +FROM + ${database}.${schema}.customer_address a +, ${database}.${schema}.customer c +, ${database}.${schema}.store_sales s +, ${database}.${schema}.date_dim d +, ${database}.${schema}.item i +WHERE ("a"."ca_address_sk" = "c"."c_current_addr_sk") + AND ("c"."c_customer_sk" = "s"."ss_customer_sk") + AND ("s"."ss_sold_date_sk" = "d"."d_date_sk") + AND ("s"."ss_item_sk" = "i"."i_item_sk") + AND ("d"."d_month_seq" = ( + SELECT DISTINCT "d_month_seq" + FROM + ${database}.${schema}.date_dim + WHERE ("d_year" = 2001) + AND ("d_moy" = 1) + )) + AND ("i"."i_current_price" > (DECIMAL '1.2' * ( + SELECT "avg"("j"."i_current_price") + FROM + ${database}.${schema}.item j + WHERE ("j"."i_category" = "i"."i_category") + ))) +GROUP BY "a"."ca_state" +HAVING ("count"(*) >= 10) +ORDER BY "cnt" ASC, "a"."ca_state" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q07.sql b/presto-iceberg/src/test/resources/tpcds/queries/q07.sql new file mode 100644 index 0000000000000..a5b907e4ebd6c --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q07.sql @@ -0,0 +1,25 @@ +SELECT + "i_item_id" +, "avg"("ss_quantity") "agg1" +, "avg"("ss_list_price") "agg2" +, "avg"("ss_coupon_amt") "agg3" +, "avg"("ss_sales_price") "agg4" +FROM + ${database}.${schema}.store_sales +, ${database}.${schema}.customer_demographics +, ${database}.${schema}.date_dim +, ${database}.${schema}.item +, ${database}.${schema}.promotion +WHERE ("ss_sold_date_sk" = "d_date_sk") + AND ("ss_item_sk" = "i_item_sk") + AND ("ss_cdemo_sk" = "cd_demo_sk") + AND ("ss_promo_sk" = "p_promo_sk") + AND ("cd_gender" = 'M') + AND ("cd_marital_status" = 'S') + AND ("cd_education_status" = 'College') + AND (("p_channel_email" = 'N') + OR ("p_channel_event" = 'N')) + AND ("d_year" = 2000) +GROUP BY "i_item_id" +ORDER BY "i_item_id" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q08.sql b/presto-iceberg/src/test/resources/tpcds/queries/q08.sql new file mode 100644 index 0000000000000..36d18fab4ac31 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q08.sql @@ -0,0 +1,441 @@ +SELECT + "s_store_name" +, "sum"("ss_net_profit") +FROM + ${database}.${schema}.store_sales +, ${database}.${schema}.date_dim +, ${database}.${schema}.store +, ( + SELECT "ca_zip" + FROM + ( +( + SELECT "substr"("ca_zip", 1, 5) "ca_zip" + FROM + ${database}.${schema}.customer_address + WHERE ("substr"("ca_zip", 1, 5) IN ( + '24128' + , '57834' + , '13354' + , '15734' + , '78668' + , '76232' + , '62878' + , '45375' + , '63435' + , '22245' + , '65084' + , '49130' + , '40558' + , '25733' + , '15798' + , '87816' + , '81096' + , '56458' + , '35474' + , '27156' + , '83926' + , '18840' + , '28286' + , '24676' + , '37930' + , '77556' + , '27700' + , '45266' + , '94627' + , '62971' + , '20548' + , '23470' + , '47305' + , '53535' + , '21337' + , '26231' + , '50412' + , '69399' + , '17879' + , '51622' + , '43848' + , '21195' + , '83921' + , '15559' + , '67853' + , '15126' + , '16021' + , '26233' + , '53268' + , '10567' + , '91137' + , '76107' + , '11101' + , '59166' + , '38415' + , '61265' + , '71954' + , '15371' + , '11928' + , '15455' + , '98294' + , '68309' + , '69913' + , '59402' + , '58263' + , '25782' + , '18119' + , '35942' + , '33282' + , '42029' + , '17920' + , '98359' + , '15882' + , '45721' + , '60279' + , '18426' + , '64544' + , '25631' + , '43933' + , '37125' + , '98235' + , '10336' + , '24610' + , '68101' + , '56240' + , '40081' + , '86379' + , '44165' + , '33515' + , '88190' + , '84093' + , '27068' + , '99076' + , '36634' + , '50308' + , '28577' + , '39736' + , '33786' + , '71286' + , '26859' + , '55565' + , '98569' + , '70738' + , '19736' + , '64457' + , '17183' + , '28915' + , '26653' + , '58058' + , '89091' + , '54601' + , '24206' + , '14328' + , '55253' + , '82136' + , '67897' + , '56529' + , '72305' + , '67473' + , '62377' + , '22752' + , '57647' + , '62496' + , '41918' + , '36233' + , '86284' + , '54917' + , '22152' + , '19515' + , '63837' + , '18376' + , '42961' + , '10144' + , '36495' + , '58078' + , '38607' + , '91110' + , '64147' + , '19430' + , '17043' + , '45200' + , '63981' + , '48425' + , '22351' + , '30010' + , '21756' + , '14922' + , '14663' + , '77191' + , '60099' + , '29741' + , '36420' + , '21076' + , '91393' + , '28810' + , '96765' + , '23006' + , '18799' + , '49156' + , '98025' + , '23932' + , '67467' + , '30450' + , '50298' + , '29178' + , '89360' + , '32754' + , '63089' + , '87501' + , '87343' + , '29839' + , '30903' + , '81019' + , '18652' + , '73273' + , '25989' + , '20260' + , '68893' + , '53179' + , '30469' + , '28898' + , '31671' + , '24996' + , '18767' + , '64034' + , '91068' + , '51798' + , '51200' + , '63193' + , '39516' + , '72550' + , '72325' + , '51211' + , '23968' + , '86057' + , '10390' + , '85816' + , '45692' + , '65164' + , '21309' + , '18845' + , '68621' + , '92712' + , '68880' + , '90257' + , '47770' + , '13955' + , '70466' + , '21286' + , '67875' + , '82636' + , '36446' + , '79994' + , '72823' + , '40162' + , '41367' + , '41766' + , '22437' + , '58470' + , '11356' + , '76638' + , '68806' + , '25280' + , '67301' + , '73650' + , '86198' + , '16725' + , '38935' + , '13394' + , '61810' + , '81312' + , '15146' + , '71791' + , '31016' + , '72013' + , '37126' + , '22744' + , '73134' + , '70372' + , '30431' + , '39192' + , '35850' + , '56571' + , '67030' + , '22461' + , '88424' + , '88086' + , '14060' + , '40604' + , '19512' + , '72175' + , '51649' + , '19505' + , '24317' + , '13375' + , '81426' + , '18270' + , '72425' + , '45748' + , '55307' + , '53672' + , '52867' + , '56575' + , '39127' + , '30625' + , '10445' + , '39972' + , '74351' + , '26065' + , '83849' + , '42666' + , '96976' + , '68786' + , '77721' + , '68908' + , '66864' + , '63792' + , '51650' + , '31029' + , '26689' + , '66708' + , '11376' + , '20004' + , '31880' + , '96451' + , '41248' + , '94898' + , '18383' + , '60576' + , '38193' + , '48583' + , '13595' + , '76614' + , '24671' + , '46820' + , '82276' + , '10516' + , '11634' + , '45549' + , '88885' + , '18842' + , '90225' + , '18906' + , '13376' + , '84935' + , '78890' + , '58943' + , '15765' + , '50016' + , '69035' + , '49448' + , '39371' + , '41368' + , '33123' + , '83144' + , '14089' + , '94945' + , '73241' + , '19769' + , '47537' + , '38122' + , '28587' + , '76698' + , '22927' + , '56616' + , '34425' + , '96576' + , '78567' + , '97789' + , '94983' + , '79077' + , '57855' + , '97189' + , '46081' + , '48033' + , '19849' + , '28488' + , '28545' + , '72151' + , '69952' + , '43285' + , '26105' + , '76231' + , '15723' + , '25486' + , '39861' + , '83933' + , '75691' + , '46136' + , '61547' + , '66162' + , '25858' + , '22246' + , '51949' + , '27385' + , '77610' + , '34322' + , '51061' + , '68100' + , '61860' + , '13695' + , '44438' + , '90578' + , '96888' + , '58048' + , '99543' + , '73171' + , '56691' + , '64528' + , '56910' + , '83444' + , '30122' + , '68014' + , '14171' + , '16807' + , '83041' + , '34102' + , '51103' + , '79777' + , '17871' + , '12305' + , '22685' + , '94167' + , '28709' + , '35258' + , '57665' + , '71256' + , '57047' + , '11489' + , '31387' + , '68341' + , '78451' + , '14867' + , '25103' + , '35458' + , '25003' + , '54364' + , '73520' + , '32213' + , '35576')) + ) INTERSECT ( + SELECT "ca_zip" + FROM + ( + SELECT + "substr"("ca_zip", 1, 5) "ca_zip" + , "count"(*) "cnt" + FROM + ${database}.${schema}.customer_address + , ${database}.${schema}.customer + WHERE ("ca_address_sk" = "c_current_addr_sk") + AND ("c_preferred_cust_flag" = 'Y') + GROUP BY "ca_zip" + HAVING ("count"(*) > 10) + ) a1 + ) ) a2 +) v1 +WHERE ("ss_store_sk" = "s_store_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("d_qoy" = 2) + AND ("d_year" = 1998) + AND ("substr"("s_zip", 1, 2) = "substr"("v1"."ca_zip", 1, 2)) +GROUP BY "s_store_name" +ORDER BY "s_store_name" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q09.sql b/presto-iceberg/src/test/resources/tpcds/queries/q09.sql new file mode 100644 index 0000000000000..aa961fd6db0fa --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q09.sql @@ -0,0 +1,84 @@ +SELECT + (CASE WHEN (( + SELECT "count"(*) + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 1 AND 20) + ) > 74129) THEN ( + SELECT "avg"("ss_ext_discount_amt") + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 1 AND 20) +) ELSE ( + SELECT "avg"("ss_net_paid") + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 1 AND 20) +) END) "bucket1" +, (CASE WHEN (( + SELECT "count"(*) + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 21 AND 40) + ) > 122840) THEN ( + SELECT "avg"("ss_ext_discount_amt") + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 21 AND 40) +) ELSE ( + SELECT "avg"("ss_net_paid") + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 21 AND 40) +) END) "bucket2" +, (CASE WHEN (( + SELECT "count"(*) + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 41 AND 60) + ) > 56580) THEN ( + SELECT "avg"("ss_ext_discount_amt") + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 41 AND 60) +) ELSE ( + SELECT "avg"("ss_net_paid") + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 41 AND 60) +) END) "bucket3" +, (CASE WHEN (( + SELECT "count"(*) + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 61 AND 80) + ) > 10097) THEN ( + SELECT "avg"("ss_ext_discount_amt") + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 61 AND 80) +) ELSE ( + SELECT "avg"("ss_net_paid") + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 61 AND 80) +) END) "bucket4" +, (CASE WHEN (( + SELECT "count"(*) + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 81 AND 100) + ) > 165306) THEN ( + SELECT "avg"("ss_ext_discount_amt") + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 81 AND 100) +) ELSE ( + SELECT "avg"("ss_net_paid") + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 81 AND 100) +) END) "bucket5" +FROM + ${database}.${schema}.reason +WHERE ("r_reason_sk" = 1) diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q10.sql b/presto-iceberg/src/test/resources/tpcds/queries/q10.sql new file mode 100644 index 0000000000000..3ad4f7ef51123 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q10.sql @@ -0,0 +1,55 @@ +SELECT + "cd_gender" +, "cd_marital_status" +, "cd_education_status" +, "count"(*) "cnt1" +, "cd_purchase_estimate" +, "count"(*) "cnt2" +, "cd_credit_rating" +, "count"(*) "cnt3" +, "cd_dep_count" +, "count"(*) "cnt4" +, "cd_dep_employed_count" +, "count"(*) "cnt5" +, "cd_dep_college_count" +, "count"(*) "cnt6" +FROM + ${database}.${schema}.customer c +, ${database}.${schema}.customer_address ca +, ${database}.${schema}.customer_demographics +WHERE ("c"."c_current_addr_sk" = "ca"."ca_address_sk") + AND ("ca_county" IN ('Rush County', 'Toole County', 'Jefferson County', 'Dona Ana County', 'La Porte County')) + AND ("cd_demo_sk" = "c"."c_current_cdemo_sk") + AND (EXISTS ( + SELECT * + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + WHERE ("c"."c_customer_sk" = "ss_customer_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("d_year" = 2002) + AND ("d_moy" BETWEEN 1 AND (1 + 3)) +)) + AND ((EXISTS ( + SELECT * + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + WHERE ("c"."c_customer_sk" = "ws_bill_customer_sk") + AND ("ws_sold_date_sk" = "d_date_sk") + AND ("d_year" = 2002) + AND ("d_moy" BETWEEN 1 AND (1 + 3)) + )) + OR (EXISTS ( + SELECT * + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.date_dim + WHERE ("c"."c_customer_sk" = "cs_ship_customer_sk") + AND ("cs_sold_date_sk" = "d_date_sk") + AND ("d_year" = 2002) + AND ("d_moy" BETWEEN 1 AND (1 + 3)) + ))) +GROUP BY "cd_gender", "cd_marital_status", "cd_education_status", "cd_purchase_estimate", "cd_credit_rating", "cd_dep_count", "cd_dep_employed_count", "cd_dep_college_count" +ORDER BY "cd_gender" ASC, "cd_marital_status" ASC, "cd_education_status" ASC, "cd_purchase_estimate" ASC, "cd_credit_rating" ASC, "cd_dep_count" ASC, "cd_dep_employed_count" ASC, "cd_dep_college_count" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q11.sql b/presto-iceberg/src/test/resources/tpcds/queries/q11.sql new file mode 100644 index 0000000000000..93cc022b5d77b --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q11.sql @@ -0,0 +1,67 @@ +WITH + year_total AS ( + SELECT + "c_customer_id" "customer_id" + , "c_first_name" "customer_first_name" + , "c_last_name" "customer_last_name" + , "c_preferred_cust_flag" "customer_preferred_cust_flag" + , "c_birth_country" "customer_birth_country" + , "c_login" "customer_login" + , "c_email_address" "customer_email_address" + , "d_year" "dyear" + , "sum"(("ss_ext_list_price" - "ss_ext_discount_amt")) "year_total" + , 's' "sale_type" + FROM + ${database}.${schema}.customer + , ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + WHERE ("c_customer_sk" = "ss_customer_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + GROUP BY "c_customer_id", "c_first_name", "c_last_name", "c_preferred_cust_flag", "c_birth_country", "c_login", "c_email_address", "d_year" +UNION ALL SELECT + "c_customer_id" "customer_id" + , "c_first_name" "customer_first_name" + , "c_last_name" "customer_last_name" + , "c_preferred_cust_flag" "customer_preferred_cust_flag" + , "c_birth_country" "customer_birth_country" + , "c_login" "customer_login" + , "c_email_address" "customer_email_address" + , "d_year" "dyear" + , "sum"(("ws_ext_list_price" - "ws_ext_discount_amt")) "year_total" + , 'w' "sale_type" + FROM + ${database}.${schema}.customer + , ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + WHERE ("c_customer_sk" = "ws_bill_customer_sk") + AND ("ws_sold_date_sk" = "d_date_sk") + GROUP BY "c_customer_id", "c_first_name", "c_last_name", "c_preferred_cust_flag", "c_birth_country", "c_login", "c_email_address", "d_year" +) +SELECT + "t_s_secyear"."customer_id" +, "t_s_secyear"."customer_first_name" +, "t_s_secyear"."customer_last_name" +, "t_s_secyear"."customer_preferred_cust_flag" +, "t_s_secyear"."customer_birth_country" +, "t_s_secyear"."customer_login" +FROM + year_total t_s_firstyear +, year_total t_s_secyear +, year_total t_w_firstyear +, year_total t_w_secyear +WHERE ("t_s_secyear"."customer_id" = "t_s_firstyear"."customer_id") + AND ("t_s_firstyear"."customer_id" = "t_w_secyear"."customer_id") + AND ("t_s_firstyear"."customer_id" = "t_w_firstyear"."customer_id") + AND ("t_s_firstyear"."sale_type" = 's') + AND ("t_w_firstyear"."sale_type" = 'w') + AND ("t_s_secyear"."sale_type" = 's') + AND ("t_w_secyear"."sale_type" = 'w') + AND ("t_s_firstyear"."dyear" = 2001) + AND ("t_s_secyear"."dyear" = (2001 + 1)) + AND ("t_w_firstyear"."dyear" = 2001) + AND ("t_w_secyear"."dyear" = (2001 + 1)) + AND ("t_s_firstyear"."year_total" > 0) + AND ("t_w_firstyear"."year_total" > 0) + AND ((CASE WHEN ("t_w_firstyear"."year_total" > 0) THEN ("t_w_secyear"."year_total" / "t_w_firstyear"."year_total") ELSE DECIMAL '0.0' END) > (CASE WHEN ("t_s_firstyear"."year_total" > 0) THEN ("t_s_secyear"."year_total" / "t_s_firstyear"."year_total") ELSE DECIMAL '0.0' END)) +ORDER BY "t_s_secyear"."customer_id" ASC, "t_s_secyear"."customer_first_name" ASC, "t_s_secyear"."customer_last_name" ASC, "t_s_secyear"."customer_preferred_cust_flag" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q12.sql b/presto-iceberg/src/test/resources/tpcds/queries/q12.sql new file mode 100644 index 0000000000000..55b296c16c384 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q12.sql @@ -0,0 +1,19 @@ +SELECT + "i_item_id" +, "i_item_desc" +, "i_category" +, "i_class" +, "i_current_price" +, "sum"("ws_ext_sales_price") "${database}.${schema}.itemrevenue" +, (("sum"("ws_ext_sales_price") * 100) / "sum"("sum"("ws_ext_sales_price")) OVER (PARTITION BY "i_class")) "revenueratio" +FROM + ${database}.${schema}.web_sales +, ${database}.${schema}.item +, ${database}.${schema}.date_dim +WHERE ("ws_item_sk" = "i_item_sk") + AND ("i_category" IN ('Sports', 'Books', 'Home')) + AND ("ws_sold_date_sk" = "d_date_sk") + AND (CAST("d_date" AS DATE) BETWEEN CAST('1999-02-22' AS DATE) AND (CAST('1999-02-22' AS DATE) + INTERVAL '30' DAY)) +GROUP BY "i_item_id", "i_item_desc", "i_category", "i_class", "i_current_price" +ORDER BY "i_category" ASC, "i_class" ASC, "i_item_id" ASC, "i_item_desc" ASC, "revenueratio" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q13.sql b/presto-iceberg/src/test/resources/tpcds/queries/q13.sql new file mode 100644 index 0000000000000..62aedd1b083fc --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q13.sql @@ -0,0 +1,45 @@ +SELECT + "avg"("ss_quantity") +, "avg"("ss_ext_sales_price") +, "avg"("ss_ext_wholesale_cost") +, "sum"("ss_ext_wholesale_cost") +FROM + ${database}.${schema}.store_sales +, ${database}.${schema}.store +, ${database}.${schema}.customer_demographics +, ${database}.${schema}.household_demographics +, ${database}.${schema}.customer_address +, ${database}.${schema}.date_dim +WHERE ("s_store_sk" = "ss_store_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("d_year" = 2001) + AND ((("ss_hdemo_sk" = "hd_demo_sk") + AND ("cd_demo_sk" = "ss_cdemo_sk") + AND ("cd_marital_status" = 'M') + AND ("cd_education_status" = 'Advanced Degree') + AND ("ss_sales_price" BETWEEN DECIMAL '100.00' AND DECIMAL '150.00') + AND ("hd_dep_count" = 3)) + OR (("ss_hdemo_sk" = "hd_demo_sk") + AND ("cd_demo_sk" = "ss_cdemo_sk") + AND ("cd_marital_status" = 'S') + AND ("cd_education_status" = 'College') + AND ("ss_sales_price" BETWEEN DECIMAL '50.00' AND DECIMAL '100.00') + AND ("hd_dep_count" = 1)) + OR (("ss_hdemo_sk" = "hd_demo_sk") + AND ("cd_demo_sk" = "ss_cdemo_sk") + AND ("cd_marital_status" = 'W') + AND ("cd_education_status" = '2 yr Degree') + AND ("ss_sales_price" BETWEEN DECIMAL '150.00' AND DECIMAL '200.00') + AND ("hd_dep_count" = 1))) + AND ((("ss_addr_sk" = "ca_address_sk") + AND ("ca_country" = 'United States') + AND ("ca_state" IN ('TX' , 'OH' , 'TX')) + AND ("ss_net_profit" BETWEEN 100 AND 200)) + OR (("ss_addr_sk" = "ca_address_sk") + AND ("ca_country" = 'United States') + AND ("ca_state" IN ('OR' , 'NM' , 'KY')) + AND ("ss_net_profit" BETWEEN 150 AND 300)) + OR (("ss_addr_sk" = "ca_address_sk") + AND ("ca_country" = 'United States') + AND ("ca_state" IN ('VA' , 'TX' , 'MS')) + AND ("ss_net_profit" BETWEEN 50 AND 250))) diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q14_1.sql b/presto-iceberg/src/test/resources/tpcds/queries/q14_1.sql new file mode 100644 index 0000000000000..a83ae9b0fc4eb --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q14_1.sql @@ -0,0 +1,165 @@ +WITH + cross_items AS ( + SELECT "i_item_sk" "ss_item_sk" + FROM + ${database}.${schema}.item + , ( + SELECT + "iss"."i_brand_id" "brand_id" + , "iss"."i_class_id" "class_id" + , "iss"."i_category_id" "category_id" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.item iss + , ${database}.${schema}.date_dim d1 + WHERE ("ss_item_sk" = "iss"."i_item_sk") + AND ("ss_sold_date_sk" = "d1"."d_date_sk") + AND ("d1"."d_year" BETWEEN 1999 AND (1999 + 2)) +INTERSECT SELECT + "ics"."i_brand_id" + , "ics"."i_class_id" + , "ics"."i_category_id" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.item ics + , ${database}.${schema}.date_dim d2 + WHERE ("cs_item_sk" = "ics"."i_item_sk") + AND ("cs_sold_date_sk" = "d2"."d_date_sk") + AND ("d2"."d_year" BETWEEN 1999 AND (1999 + 2)) +INTERSECT SELECT + "iws"."i_brand_id" + , "iws"."i_class_id" + , "iws"."i_category_id" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.item iws + , ${database}.${schema}.date_dim d3 + WHERE ("ws_item_sk" = "iws"."i_item_sk") + AND ("ws_sold_date_sk" = "d3"."d_date_sk") + AND ("d3"."d_year" BETWEEN 1999 AND (1999 + 2)) + ) + WHERE ("i_brand_id" = "brand_id") + AND ("i_class_id" = "class_id") + AND ("i_category_id" = "category_id") +) +, avg_sales AS ( + SELECT "avg"(("quantity" * "list_price")) "average_sales" + FROM + ( + SELECT + "ss_quantity" "quantity" + , "ss_list_price" "list_price" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + WHERE ("ss_sold_date_sk" = "d_date_sk") + AND ("d_year" BETWEEN 1999 AND (1999 + 2)) +UNION ALL SELECT + "cs_quantity" "quantity" + , "cs_list_price" "list_price" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.date_dim + WHERE ("cs_sold_date_sk" = "d_date_sk") + AND ("d_year" BETWEEN 1999 AND (1999 + 2)) +UNION ALL SELECT + "ws_quantity" "quantity" + , "ws_list_price" "list_price" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + WHERE ("ws_sold_date_sk" = "d_date_sk") + AND ("d_year" BETWEEN 1999 AND (1999 + 2)) + ) x +) +SELECT + "channel" +, "i_brand_id" +, "i_class_id" +, "i_category_id" +, "sum"("sales") +, "sum"("number_sales") +FROM + ( + SELECT + '${database}.${schema}.store' "channel" + , "i_brand_id" + , "i_class_id" + , "i_category_id" + , "sum"(("ss_quantity" * "ss_list_price")) "sales" + , "count"(*) "number_sales" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.item + , ${database}.${schema}.date_dim + WHERE ("ss_item_sk" IN ( + SELECT "ss_item_sk" + FROM + cross_items + )) + AND ("ss_item_sk" = "i_item_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("d_year" = (1999 + 2)) + AND ("d_moy" = 11) + GROUP BY "i_brand_id", "i_class_id", "i_category_id" + HAVING ("sum"(("ss_quantity" * "ss_list_price")) > ( + SELECT "average_sales" + FROM + avg_sales + )) +UNION ALL SELECT + 'catalog' "channel" + , "i_brand_id" + , "i_class_id" + , "i_category_id" + , "sum"(("cs_quantity" * "cs_list_price")) "sales" + , "count"(*) "number_sales" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.item + , ${database}.${schema}.date_dim + WHERE ("cs_item_sk" IN ( + SELECT "ss_item_sk" + FROM + cross_items + )) + AND ("cs_item_sk" = "i_item_sk") + AND ("cs_sold_date_sk" = "d_date_sk") + AND ("d_year" = (1999 + 2)) + AND ("d_moy" = 11) + GROUP BY "i_brand_id", "i_class_id", "i_category_id" + HAVING ("sum"(("cs_quantity" * "cs_list_price")) > ( + SELECT "average_sales" + FROM + avg_sales + )) +UNION ALL SELECT + 'web' "channel" + , "i_brand_id" + , "i_class_id" + , "i_category_id" + , "sum"(("ws_quantity" * "ws_list_price")) "sales" + , "count"(*) "number_sales" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.item + , ${database}.${schema}.date_dim + WHERE ("ws_item_sk" IN ( + SELECT "ss_item_sk" + FROM + cross_items + )) + AND ("ws_item_sk" = "i_item_sk") + AND ("ws_sold_date_sk" = "d_date_sk") + AND ("d_year" = (1999 + 2)) + AND ("d_moy" = 11) + GROUP BY "i_brand_id", "i_class_id", "i_category_id" + HAVING ("sum"(("ws_quantity" * "ws_list_price")) > ( + SELECT "average_sales" + FROM + avg_sales + )) +) y +GROUP BY ROLLUP (channel, i_brand_id, i_class_id, i_category_id) +ORDER BY "channel" ASC, "i_brand_id" ASC, "i_class_id" ASC, "i_category_id" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q14_2.sql b/presto-iceberg/src/test/resources/tpcds/queries/q14_2.sql new file mode 100644 index 0000000000000..92576c2da4b7b --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q14_2.sql @@ -0,0 +1,149 @@ +WITH + cross_items AS ( + SELECT "i_item_sk" "ss_item_sk" + FROM + ${database}.${schema}.item + , ( + SELECT + "iss"."i_brand_id" "brand_id" + , "iss"."i_class_id" "class_id" + , "iss"."i_category_id" "category_id" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.item iss + , ${database}.${schema}.date_dim d1 + WHERE ("ss_item_sk" = "iss"."i_item_sk") + AND ("ss_sold_date_sk" = "d1"."d_date_sk") + AND ("d1"."d_year" BETWEEN 1999 AND (1999 + 2)) +INTERSECT SELECT + "ics"."i_brand_id" + , "ics"."i_class_id" + , "ics"."i_category_id" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.item ics + , ${database}.${schema}.date_dim d2 + WHERE ("cs_item_sk" = "ics"."i_item_sk") + AND ("cs_sold_date_sk" = "d2"."d_date_sk") + AND ("d2"."d_year" BETWEEN 1999 AND (1999 + 2)) +INTERSECT SELECT + "iws"."i_brand_id" + , "iws"."i_class_id" + , "iws"."i_category_id" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.item iws + , ${database}.${schema}.date_dim d3 + WHERE ("ws_item_sk" = "iws"."i_item_sk") + AND ("ws_sold_date_sk" = "d3"."d_date_sk") + AND ("d3"."d_year" BETWEEN 1999 AND (1999 + 2)) + ) x + WHERE ("i_brand_id" = "brand_id") + AND ("i_class_id" = "class_id") + AND ("i_category_id" = "category_id") +) +, avg_sales AS ( + SELECT "avg"(("quantity" * "list_price")) "average_sales" + FROM + ( + SELECT + "ss_quantity" "quantity" + , "ss_list_price" "list_price" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + WHERE ("ss_sold_date_sk" = "d_date_sk") + AND ("d_year" BETWEEN 1999 AND (1999 + 2)) +UNION ALL SELECT + "cs_quantity" "quantity" + , "cs_list_price" "list_price" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.date_dim + WHERE ("cs_sold_date_sk" = "d_date_sk") + AND ("d_year" BETWEEN 1999 AND (1999 + 2)) +UNION ALL SELECT + "ws_quantity" "quantity" + , "ws_list_price" "list_price" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + WHERE ("ws_sold_date_sk" = "d_date_sk") + AND ("d_year" BETWEEN 1999 AND (1999 + 2)) + ) +) +SELECT * +FROM + ( + SELECT + '${database}.${schema}.store' "channel" + , "i_brand_id" + , "i_class_id" + , "i_category_id" + , "sum"(("ss_quantity" * "ss_list_price")) "sales" + , "count"(*) "number_sales" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.item + , ${database}.${schema}.date_dim + WHERE ("ss_item_sk" IN ( + SELECT "ss_item_sk" + FROM + cross_items + )) + AND ("ss_item_sk" = "i_item_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("d_week_seq" = ( + SELECT "d_week_seq" + FROM + ${database}.${schema}.date_dim + WHERE ("d_year" = (1999 + 1)) + AND ("d_moy" = 12) + AND ("d_dom" = 11) + )) + GROUP BY "i_brand_id", "i_class_id", "i_category_id" + HAVING ("sum"(("ss_quantity" * "ss_list_price")) > ( + SELECT "average_sales" + FROM + avg_sales + )) +) this_year +, ( + SELECT + '${database}.${schema}.store' "channel" + , "i_brand_id" + , "i_class_id" + , "i_category_id" + , "sum"(("ss_quantity" * "ss_list_price")) "sales" + , "count"(*) "number_sales" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.item + , ${database}.${schema}.date_dim + WHERE ("ss_item_sk" IN ( + SELECT "ss_item_sk" + FROM + cross_items + )) + AND ("ss_item_sk" = "i_item_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("d_week_seq" = ( + SELECT "d_week_seq" + FROM + ${database}.${schema}.date_dim + WHERE ("d_year" = 1999) + AND ("d_moy" = 12) + AND ("d_dom" = 11) + )) + GROUP BY "i_brand_id", "i_class_id", "i_category_id" + HAVING ("sum"(("ss_quantity" * "ss_list_price")) > ( + SELECT "average_sales" + FROM + avg_sales + )) +) last_year +WHERE ("this_year"."i_brand_id" = "last_year"."i_brand_id") + AND ("this_year"."i_class_id" = "last_year"."i_class_id") + AND ("this_year"."i_category_id" = "last_year"."i_category_id") +ORDER BY "this_year"."channel" ASC, "this_year"."i_brand_id" ASC, "this_year"."i_class_id" ASC, "this_year"."i_category_id" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q15.sql b/presto-iceberg/src/test/resources/tpcds/queries/q15.sql new file mode 100644 index 0000000000000..ed0e619cf95b4 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q15.sql @@ -0,0 +1,19 @@ +SELECT + "ca_zip" +, "sum"("cs_sales_price") +FROM + ${database}.${schema}.catalog_sales +, ${database}.${schema}.customer +, ${database}.${schema}.customer_address +, ${database}.${schema}.date_dim +WHERE ("cs_bill_customer_sk" = "c_customer_sk") + AND ("c_current_addr_sk" = "ca_address_sk") + AND (("substr"("ca_zip", 1, 5) IN ('85669' , '86197' , '88274' , '83405' , '86475' , '85392' , '85460' , '80348' , '81792')) + OR ("ca_state" IN ('CA' , 'WA' , 'GA')) + OR ("cs_sales_price" > 500)) + AND ("cs_sold_date_sk" = "d_date_sk") + AND ("d_qoy" = 2) + AND ("d_year" = 2001) +GROUP BY "ca_zip" +ORDER BY "ca_zip" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q16.sql b/presto-iceberg/src/test/resources/tpcds/queries/q16.sql new file mode 100644 index 0000000000000..c99f58a39021a --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q16.sql @@ -0,0 +1,30 @@ +SELECT + "count"(DISTINCT "cs_order_number") "order count" +, "sum"("cs_ext_ship_cost") "total shipping cost" +, "sum"("cs_net_profit") "total net profit" +FROM + ${database}.${schema}.catalog_sales cs1 +, ${database}.${schema}.date_dim +, ${database}.${schema}.customer_address +, ${database}.${schema}.call_center +WHERE ("d_date" BETWEEN CAST('2002-2-01' AS DATE) AND (CAST('2002-2-01' AS DATE) + INTERVAL '60' DAY)) + AND ("cs1"."cs_ship_date_sk" = "d_date_sk") + AND ("cs1"."cs_ship_addr_sk" = "ca_address_sk") + AND ("ca_state" = 'GA') + AND ("cs1"."cs_call_center_sk" = "cc_call_center_sk") + AND ("cc_county" IN ('Williamson County', 'Williamson County', 'Williamson County', 'Williamson County', 'Williamson County')) + AND (EXISTS ( + SELECT * + FROM + ${database}.${schema}.catalog_sales cs2 + WHERE ("cs1"."cs_order_number" = "cs2"."cs_order_number") + AND ("cs1"."cs_warehouse_sk" <> "cs2"."cs_warehouse_sk") +)) + AND (NOT (EXISTS ( + SELECT * + FROM + ${database}.${schema}.catalog_returns cr1 + WHERE ("cs1"."cs_order_number" = "cr1"."cr_order_number") +))) +ORDER BY "count"(DISTINCT "cs_order_number") ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q17.sql b/presto-iceberg/src/test/resources/tpcds/queries/q17.sql new file mode 100644 index 0000000000000..07c555f5e1cee --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q17.sql @@ -0,0 +1,41 @@ +SELECT + "i_item_id" +, "i_item_desc" +, "s_state" +, "count"("ss_quantity") "store_sales_quantitycount" +, "avg"("ss_quantity") "store_sales_quantityave" +, "stddev_samp"("ss_quantity") "store_sales_quantitystdev" +, ("stddev_samp"("ss_quantity") / "avg"("ss_quantity")) "store_sales_quantitycov" +, "count"("sr_return_quantity") "store_returns_quantitycount" +, "avg"("sr_return_quantity") "store_returns_quantityave" +, "stddev_samp"("sr_return_quantity") "store_returns_quantitystdev" +, ("stddev_samp"("sr_return_quantity") / "avg"("sr_return_quantity")) "store_returns_quantitycov" +, "count"("cs_quantity") "catalog_sales_quantitycount" +, "avg"("cs_quantity") "catalog_sales_quantityave" +, "stddev_samp"("cs_quantity") "catalog_sales_quantitystdev" +, ("stddev_samp"("cs_quantity") / "avg"("cs_quantity")) "catalog_sales_quantitycov" +FROM + ${database}.${schema}.store_sales +, ${database}.${schema}.store_returns +, ${database}.${schema}.catalog_sales +, ${database}.${schema}.date_dim d1 +, ${database}.${schema}.date_dim d2 +, ${database}.${schema}.date_dim d3 +, ${database}.${schema}.store +, ${database}.${schema}.item +WHERE ("d1"."d_quarter_name" = '2001Q1') + AND ("d1"."d_date_sk" = "ss_sold_date_sk") + AND ("i_item_sk" = "ss_item_sk") + AND ("s_store_sk" = "ss_store_sk") + AND ("ss_customer_sk" = "sr_customer_sk") + AND ("ss_item_sk" = "sr_item_sk") + AND ("ss_ticket_number" = "sr_ticket_number") + AND ("sr_returned_date_sk" = "d2"."d_date_sk") + AND ("d2"."d_quarter_name" IN ('2001Q1', '2001Q2', '2001Q3')) + AND ("sr_customer_sk" = "cs_bill_customer_sk") + AND ("sr_item_sk" = "cs_item_sk") + AND ("cs_sold_date_sk" = "d3"."d_date_sk") + AND ("d3"."d_quarter_name" IN ('2001Q1', '2001Q2', '2001Q3')) +GROUP BY "i_item_id", "i_item_desc", "s_state" +ORDER BY "i_item_id" ASC, "i_item_desc" ASC, "s_state" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q18.sql b/presto-iceberg/src/test/resources/tpcds/queries/q18.sql new file mode 100644 index 0000000000000..f4e8cf97d6535 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q18.sql @@ -0,0 +1,34 @@ +SELECT + "i_item_id" +, "ca_country" +, "ca_state" +, "ca_county" +, "avg"(CAST("cs_quantity" AS DECIMAL(12,2))) "agg1" +, "avg"(CAST("cs_list_price" AS DECIMAL(12,2))) "agg2" +, "avg"(CAST("cs_coupon_amt" AS DECIMAL(12,2))) "agg3" +, "avg"(CAST("cs_sales_price" AS DECIMAL(12,2))) "agg4" +, "avg"(CAST("cs_net_profit" AS DECIMAL(12,2))) "agg5" +, "avg"(CAST("c_birth_year" AS DECIMAL(12,2))) "agg6" +, "avg"(CAST("cd1"."cd_dep_count" AS DECIMAL(12,2))) "agg7" +FROM + ${database}.${schema}.catalog_sales +, ${database}.${schema}.customer_demographics cd1 +, ${database}.${schema}.customer_demographics cd2 +, ${database}.${schema}.customer +, ${database}.${schema}.customer_address +, ${database}.${schema}.date_dim +, ${database}.${schema}.item +WHERE ("cs_sold_date_sk" = "d_date_sk") + AND ("cs_item_sk" = "i_item_sk") + AND ("cs_bill_cdemo_sk" = "cd1"."cd_demo_sk") + AND ("cs_bill_customer_sk" = "c_customer_sk") + AND ("cd1"."cd_gender" = 'F') + AND ("cd1"."cd_education_status" = 'Unknown') + AND ("c_current_cdemo_sk" = "cd2"."cd_demo_sk") + AND ("c_current_addr_sk" = "ca_address_sk") + AND ("c_birth_month" IN (1, 6, 8, 9, 12, 2)) + AND ("d_year" = 1998) + AND ("ca_state" IN ('MS', 'IN', 'ND', 'OK', 'NM', 'VA', 'MS')) +GROUP BY ROLLUP (i_item_id, ca_country, ca_state, ca_county) +ORDER BY "ca_country" ASC, "ca_state" ASC, "ca_county" ASC, "i_item_id" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q19.sql b/presto-iceberg/src/test/resources/tpcds/queries/q19.sql new file mode 100644 index 0000000000000..a070a8cf67c1d --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q19.sql @@ -0,0 +1,25 @@ +SELECT + "i_brand_id" "brand_id" +, "i_brand" "brand" +, "i_manufact_id" +, "i_manufact" +, "sum"("ss_ext_sales_price") "ext_price" +FROM + ${database}.${schema}.date_dim +, ${database}.${schema}.store_sales +, ${database}.${schema}.item +, ${database}.${schema}.customer +, ${database}.${schema}.customer_address +, ${database}.${schema}.store +WHERE ("d_date_sk" = "ss_sold_date_sk") + AND ("ss_item_sk" = "i_item_sk") + AND ("i_manager_id" = 8) + AND ("d_moy" = 11) + AND ("d_year" = 1998) + AND ("ss_customer_sk" = "c_customer_sk") + AND ("c_current_addr_sk" = "ca_address_sk") + AND ("substr"("ca_zip", 1, 5) <> "substr"("s_zip", 1, 5)) + AND ("ss_store_sk" = "s_store_sk") +GROUP BY "i_brand", "i_brand_id", "i_manufact_id", "i_manufact" +ORDER BY "ext_price" DESC, "i_brand" ASC, "i_brand_id" ASC, "i_manufact_id" ASC, "i_manufact" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q20.sql b/presto-iceberg/src/test/resources/tpcds/queries/q20.sql new file mode 100644 index 0000000000000..a3b50441fffaa --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q20.sql @@ -0,0 +1,19 @@ +SELECT + "i_item_id" +, "i_item_desc" +, "i_category" +, "i_class" +, "i_current_price" +, "sum"("cs_ext_sales_price") "${database}.${schema}.itemrevenue" +, (("sum"("cs_ext_sales_price") * 100) / "sum"("sum"("cs_ext_sales_price")) OVER (PARTITION BY "i_class")) "revenueratio" +FROM + ${database}.${schema}.catalog_sales +, ${database}.${schema}.item +, ${database}.${schema}.date_dim +WHERE ("cs_item_sk" = "i_item_sk") + AND ("i_category" IN ('Sports', 'Books', 'Home')) + AND ("cs_sold_date_sk" = "d_date_sk") + AND (CAST("d_date" AS DATE) BETWEEN CAST('1999-02-22' AS DATE) AND (CAST('1999-02-22' AS DATE) + INTERVAL '30' DAY)) +GROUP BY "i_item_id", "i_item_desc", "i_category", "i_class", "i_current_price" +ORDER BY "i_category" ASC, "i_class" ASC, "i_item_id" ASC, "i_item_desc" ASC, "revenueratio" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q21.sql b/presto-iceberg/src/test/resources/tpcds/queries/q21.sql new file mode 100644 index 0000000000000..0f0ec1277e29a --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q21.sql @@ -0,0 +1,23 @@ +SELECT * +FROM + ( + SELECT + "w_warehouse_name" + , "i_item_id" + , "sum"((CASE WHEN (CAST("d_date" AS DATE) < CAST('2000-03-11' AS DATE)) THEN "inv_quantity_on_hand" ELSE 0 END)) "inv_before" + , "sum"((CASE WHEN (CAST("d_date" AS DATE) >= CAST('2000-03-11' AS DATE)) THEN "inv_quantity_on_hand" ELSE 0 END)) "inv_after" + FROM + ${database}.${schema}.inventory + , ${database}.${schema}.warehouse + , ${database}.${schema}.item + , ${database}.${schema}.date_dim + WHERE ("i_current_price" BETWEEN DECIMAL '0.99' AND DECIMAL '1.49') + AND ("i_item_sk" = "inv_item_sk") + AND ("inv_warehouse_sk" = "w_warehouse_sk") + AND ("inv_date_sk" = "d_date_sk") + AND ("d_date" BETWEEN (CAST('2000-03-11' AS DATE) - INTERVAL '30' DAY) AND (CAST('2000-03-11' AS DATE) + INTERVAL '30' DAY)) + GROUP BY "w_warehouse_name", "i_item_id" +) x +WHERE ((CASE WHEN ("inv_before" > 0) THEN (CAST("inv_after" AS DECIMAL(7,2)) / "inv_before") ELSE null END) BETWEEN (DECIMAL '2.00' / DECIMAL '3.00') AND (DECIMAL '3.00' / DECIMAL '2.00')) +ORDER BY "w_warehouse_name" ASC, "i_item_id" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q22.sql b/presto-iceberg/src/test/resources/tpcds/queries/q22.sql new file mode 100644 index 0000000000000..49077f3a473ec --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q22.sql @@ -0,0 +1,16 @@ +SELECT + "i_product_name" +, "i_brand" +, "i_class" +, "i_category" +, "avg"("inv_quantity_on_hand") "qoh" +FROM + ${database}.${schema}.inventory +, ${database}.${schema}.date_dim +, ${database}.${schema}.item +WHERE ("inv_date_sk" = "d_date_sk") + AND ("inv_item_sk" = "i_item_sk") + AND ("d_month_seq" BETWEEN 1200 AND (1200 + 11)) +GROUP BY ROLLUP (i_product_name, i_brand, i_class, i_category) +ORDER BY "qoh" ASC, "i_product_name" ASC, "i_brand" ASC, "i_class" ASC, "i_category" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q23_1.sql b/presto-iceberg/src/test/resources/tpcds/queries/q23_1.sql new file mode 100644 index 0000000000000..58faa35cd34ef --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q23_1.sql @@ -0,0 +1,88 @@ +WITH + frequent_ss_items AS ( + SELECT + "substr"("i_item_desc", 1, 30) "itemdesc" + , "i_item_sk" "item_sk" + , "d_date" "solddate" + , "count"(*) "cnt" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.item + WHERE ("ss_sold_date_sk" = "d_date_sk") + AND ("ss_item_sk" = "i_item_sk") + AND ("d_year" IN (2000 , (2000 + 1) , (2000 + 2) , (2000 + 3))) + GROUP BY "substr"("i_item_desc", 1, 30), "i_item_sk", "d_date" + HAVING ("count"(*) > 4) +) +, max_store_sales AS ( + SELECT "max"("csales") "tpcds_cmax" + FROM + ( + SELECT + "c_customer_sk" + , "sum"(("ss_quantity" * "ss_sales_price")) "csales" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.customer + , ${database}.${schema}.date_dim + WHERE ("ss_customer_sk" = "c_customer_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("d_year" IN (2000 , (2000 + 1) , (2000 + 2) , (2000 + 3))) + GROUP BY "c_customer_sk" + ) +) +, best_ss_customer AS ( + SELECT + "c_customer_sk" + , "sum"(("ss_quantity" * "ss_sales_price")) "ssales" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.customer + WHERE ("ss_customer_sk" = "c_customer_sk") + GROUP BY "c_customer_sk" + HAVING ("sum"(("ss_quantity" * "ss_sales_price")) > ((50 / DECIMAL '100.0') * ( + SELECT * + FROM + max_store_sales + ))) +) +SELECT "sum"("sales") +FROM + ( + SELECT ("cs_quantity" * "cs_list_price") "sales" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.date_dim + WHERE ("d_year" = 2000) + AND ("d_moy" = 2) + AND ("cs_sold_date_sk" = "d_date_sk") + AND ("cs_item_sk" IN ( + SELECT "item_sk" + FROM + frequent_ss_items + )) + AND ("cs_bill_customer_sk" IN ( + SELECT "c_customer_sk" + FROM + best_ss_customer + )) +UNION ALL SELECT ("ws_quantity" * "ws_list_price") "sales" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + WHERE ("d_year" = 2000) + AND ("d_moy" = 2) + AND ("ws_sold_date_sk" = "d_date_sk") + AND ("ws_item_sk" IN ( + SELECT "item_sk" + FROM + frequent_ss_items + )) + AND ("ws_bill_customer_sk" IN ( + SELECT "c_customer_sk" + FROM + best_ss_customer + )) +) +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q23_2.sql b/presto-iceberg/src/test/resources/tpcds/queries/q23_2.sql new file mode 100644 index 0000000000000..3da86ce764a5b --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q23_2.sql @@ -0,0 +1,104 @@ +WITH + frequent_ss_items AS ( + SELECT + "substr"("i_item_desc", 1, 30) "itemdesc" + , "i_item_sk" "item_sk" + , "d_date" "solddate" + , "count"(*) "cnt" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.item + WHERE ("ss_sold_date_sk" = "d_date_sk") + AND ("ss_item_sk" = "i_item_sk") + AND ("d_year" IN (2000 , (2000 + 1) , (2000 + 2) , (2000 + 3))) + GROUP BY "substr"("i_item_desc", 1, 30), "i_item_sk", "d_date" + HAVING ("count"(*) > 4) +) +, max_store_sales AS ( + SELECT "max"("csales") "tpcds_cmax" + FROM + ( + SELECT + "c_customer_sk" + , "sum"(("ss_quantity" * "ss_sales_price")) "csales" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.customer + , ${database}.${schema}.date_dim + WHERE ("ss_customer_sk" = "c_customer_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("d_year" IN (2000 , (2000 + 1) , (2000 + 2) , (2000 + 3))) + GROUP BY "c_customer_sk" + ) +) +, best_ss_customer AS ( + SELECT + "c_customer_sk" + , "sum"(("ss_quantity" * "ss_sales_price")) "ssales" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.customer + WHERE ("ss_customer_sk" = "c_customer_sk") + GROUP BY "c_customer_sk" + HAVING ("sum"(("ss_quantity" * "ss_sales_price")) > ((50 / DECIMAL '100.0') * ( + SELECT * + FROM + max_store_sales + ))) +) +SELECT + "c_last_name" +, "c_first_name" +, "sales" +FROM + ( + SELECT + "c_last_name" + , "c_first_name" + , "sum"(("cs_quantity" * "cs_list_price")) "sales" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.customer + , ${database}.${schema}.date_dim + WHERE ("d_year" = 2000) + AND ("d_moy" = 2) + AND ("cs_sold_date_sk" = "d_date_sk") + AND ("cs_item_sk" IN ( + SELECT "item_sk" + FROM + frequent_ss_items + )) + AND ("cs_bill_customer_sk" IN ( + SELECT "c_customer_sk" + FROM + best_ss_customer + )) + AND ("cs_bill_customer_sk" = "c_customer_sk") + GROUP BY "c_last_name", "c_first_name" +UNION ALL SELECT + "c_last_name" + , "c_first_name" + , "sum"(("ws_quantity" * "ws_list_price")) "sales" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.customer + , ${database}.${schema}.date_dim + WHERE ("d_year" = 2000) + AND ("d_moy" = 2) + AND ("ws_sold_date_sk" = "d_date_sk") + AND ("ws_item_sk" IN ( + SELECT "item_sk" + FROM + frequent_ss_items + )) + AND ("ws_bill_customer_sk" IN ( + SELECT "c_customer_sk" + FROM + best_ss_customer + )) + AND ("ws_bill_customer_sk" = "c_customer_sk") + GROUP BY "c_last_name", "c_first_name" +) +ORDER BY "c_last_name" ASC, "c_first_name" ASC, "sales" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q24_1.sql b/presto-iceberg/src/test/resources/tpcds/queries/q24_1.sql new file mode 100644 index 0000000000000..276337edbdd72 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q24_1.sql @@ -0,0 +1,45 @@ +WITH + ssales AS ( + SELECT + "c_last_name" + , "c_first_name" + , "s_store_name" + , "ca_state" + , "s_state" + , "i_color" + , "i_current_price" + , "i_manager_id" + , "i_units" + , "i_size" + , "sum"("ss_net_paid") "netpaid" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.store_returns + , ${database}.${schema}.store + , ${database}.${schema}.item + , ${database}.${schema}.customer + , ${database}.${schema}.customer_address + WHERE ("ss_ticket_number" = "sr_ticket_number") + AND ("ss_item_sk" = "sr_item_sk") + AND ("ss_customer_sk" = "c_customer_sk") + AND ("ss_item_sk" = "i_item_sk") + AND ("ss_store_sk" = "s_store_sk") + AND ("c_birth_country" = "upper"("ca_country")) + AND ("s_zip" = "ca_zip") + AND ("s_market_id" = 8) + GROUP BY "c_last_name", "c_first_name", "s_store_name", "ca_state", "s_state", "i_color", "i_current_price", "i_manager_id", "i_units", "i_size" +) +SELECT + "c_last_name" +, "c_first_name" +, "s_store_name" +, "sum"("netpaid") "paid" +FROM + ssales +WHERE ("i_color" = 'pale') +GROUP BY "c_last_name", "c_first_name", "s_store_name" +HAVING ("sum"("netpaid") > ( + SELECT (DECIMAL '0.05' * "avg"("netpaid")) + FROM + ssales + )) diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q24_2.sql b/presto-iceberg/src/test/resources/tpcds/queries/q24_2.sql new file mode 100644 index 0000000000000..ab23f23743b54 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q24_2.sql @@ -0,0 +1,45 @@ +WITH + ssales AS ( + SELECT + "c_last_name" + , "c_first_name" + , "s_store_name" + , "ca_state" + , "s_state" + , "i_color" + , "i_current_price" + , "i_manager_id" + , "i_units" + , "i_size" + , "sum"("ss_net_paid") "netpaid" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.store_returns + , ${database}.${schema}.store + , ${database}.${schema}.item + , ${database}.${schema}.customer + , ${database}.${schema}.customer_address + WHERE ("ss_ticket_number" = "sr_ticket_number") + AND ("ss_item_sk" = "sr_item_sk") + AND ("ss_customer_sk" = "c_customer_sk") + AND ("ss_item_sk" = "i_item_sk") + AND ("ss_store_sk" = "s_store_sk") + AND ("c_birth_country" = "upper"("ca_country")) + AND ("s_zip" = "ca_zip") + AND ("s_market_id" = 8) + GROUP BY "c_last_name", "c_first_name", "s_store_name", "ca_state", "s_state", "i_color", "i_current_price", "i_manager_id", "i_units", "i_size" +) +SELECT + "c_last_name" +, "c_first_name" +, "s_store_name" +, "sum"("netpaid") "paid" +FROM + ssales +WHERE ("i_color" = 'chiffon') +GROUP BY "c_last_name", "c_first_name", "s_store_name" +HAVING ("sum"("netpaid") > ( + SELECT (DECIMAL '0.05' * "avg"("netpaid")) + FROM + ssales + )) diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q25.sql b/presto-iceberg/src/test/resources/tpcds/queries/q25.sql new file mode 100644 index 0000000000000..6be737701c064 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q25.sql @@ -0,0 +1,36 @@ +SELECT + "i_item_id" +, "i_item_desc" +, "s_store_id" +, "s_store_name" +, "sum"("ss_net_profit") "store_sales_profit" +, "sum"("sr_net_loss") "store_returns_loss" +, "sum"("cs_net_profit") "catalog_sales_profit" +FROM + ${database}.${schema}.store_sales +, ${database}.${schema}.store_returns +, ${database}.${schema}.catalog_sales +, ${database}.${schema}.date_dim d1 +, ${database}.${schema}.date_dim d2 +, ${database}.${schema}.date_dim d3 +, ${database}.${schema}.store +, ${database}.${schema}.item +WHERE ("d1"."d_moy" = 4) + AND ("d1"."d_year" = 2001) + AND ("d1"."d_date_sk" = "ss_sold_date_sk") + AND ("i_item_sk" = "ss_item_sk") + AND ("s_store_sk" = "ss_store_sk") + AND ("ss_customer_sk" = "sr_customer_sk") + AND ("ss_item_sk" = "sr_item_sk") + AND ("ss_ticket_number" = "sr_ticket_number") + AND ("sr_returned_date_sk" = "d2"."d_date_sk") + AND ("d2"."d_moy" BETWEEN 4 AND 10) + AND ("d2"."d_year" = 2001) + AND ("sr_customer_sk" = "cs_bill_customer_sk") + AND ("sr_item_sk" = "cs_item_sk") + AND ("cs_sold_date_sk" = "d3"."d_date_sk") + AND ("d3"."d_moy" BETWEEN 4 AND 10) + AND ("d3"."d_year" = 2001) +GROUP BY "i_item_id", "i_item_desc", "s_store_id", "s_store_name" +ORDER BY "i_item_id" ASC, "i_item_desc" ASC, "s_store_id" ASC, "s_store_name" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q26.sql b/presto-iceberg/src/test/resources/tpcds/queries/q26.sql new file mode 100644 index 0000000000000..31f585d05cdab --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q26.sql @@ -0,0 +1,25 @@ +SELECT + "i_item_id" +, "avg"("cs_quantity") "agg1" +, "avg"("cs_list_price") "agg2" +, "avg"("cs_coupon_amt") "agg3" +, "avg"("cs_sales_price") "agg4" +FROM + ${database}.${schema}.catalog_sales +, ${database}.${schema}.customer_demographics +, ${database}.${schema}.date_dim +, ${database}.${schema}.item +, ${database}.${schema}.promotion +WHERE ("cs_sold_date_sk" = "d_date_sk") + AND ("cs_item_sk" = "i_item_sk") + AND ("cs_bill_cdemo_sk" = "cd_demo_sk") + AND ("cs_promo_sk" = "p_promo_sk") + AND ("cd_gender" = 'M') + AND ("cd_marital_status" = 'S') + AND ("cd_education_status" = 'College') + AND (("p_channel_email" = 'N') + OR ("p_channel_event" = 'N')) + AND ("d_year" = 2000) +GROUP BY "i_item_id" +ORDER BY "i_item_id" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q27.sql b/presto-iceberg/src/test/resources/tpcds/queries/q27.sql new file mode 100644 index 0000000000000..dad2f3c2e52c3 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q27.sql @@ -0,0 +1,32 @@ +SELECT + "i_item_id" +, "s_state" +, GROUPING ("s_state") "g_state" +, "avg"("ss_quantity") "agg1" +, "avg"("ss_list_price") "agg2" +, "avg"("ss_coupon_amt") "agg3" +, "avg"("ss_sales_price") "agg4" +FROM + ${database}.${schema}.store_sales +, ${database}.${schema}.customer_demographics +, ${database}.${schema}.date_dim +, ${database}.${schema}.store +, ${database}.${schema}.item +WHERE ("ss_sold_date_sk" = "d_date_sk") + AND ("ss_item_sk" = "i_item_sk") + AND ("ss_store_sk" = "s_store_sk") + AND ("ss_cdemo_sk" = "cd_demo_sk") + AND ("cd_gender" = 'M') + AND ("cd_marital_status" = 'S') + AND ("cd_education_status" = 'College') + AND ("d_year" = 2002) + AND ("s_state" IN ( + 'TN' + , 'TN' + , 'TN' + , 'TN' + , 'TN' + , 'TN')) +GROUP BY ROLLUP (i_item_id, s_state) +ORDER BY "i_item_id" ASC, "s_state" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q28.sql b/presto-iceberg/src/test/resources/tpcds/queries/q28.sql new file mode 100644 index 0000000000000..d6a4a7862861c --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q28.sql @@ -0,0 +1,75 @@ +SELECT * +FROM + ( + SELECT + "avg"("ss_list_price") "b1_lp" + , "count"("ss_list_price") "b1_cnt" + , "count"(DISTINCT "ss_list_price") "b1_cntd" + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 0 AND 5) + AND (("ss_list_price" BETWEEN 8 AND (8 + 10)) + OR ("ss_coupon_amt" BETWEEN 459 AND (459 + 1000)) + OR ("ss_wholesale_cost" BETWEEN 57 AND (57 + 20))) +) b1 +, ( + SELECT + "avg"("ss_list_price") "b2_lp" + , "count"("ss_list_price") "b2_cnt" + , "count"(DISTINCT "ss_list_price") "b2_cntd" + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 6 AND 10) + AND (("ss_list_price" BETWEEN 90 AND (90 + 10)) + OR ("ss_coupon_amt" BETWEEN 2323 AND (2323 + 1000)) + OR ("ss_wholesale_cost" BETWEEN 31 AND (31 + 20))) +) b2 +, ( + SELECT + "avg"("ss_list_price") "b3_lp" + , "count"("ss_list_price") "b3_cnt" + , "count"(DISTINCT "ss_list_price") "b3_cntd" + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 11 AND 15) + AND (("ss_list_price" BETWEEN 142 AND (142 + 10)) + OR ("ss_coupon_amt" BETWEEN 12214 AND (12214 + 1000)) + OR ("ss_wholesale_cost" BETWEEN 79 AND (79 + 20))) +) b3 +, ( + SELECT + "avg"("ss_list_price") "b4_lp" + , "count"("ss_list_price") "b4_cnt" + , "count"(DISTINCT "ss_list_price") "b4_cntd" + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 16 AND 20) + AND (("ss_list_price" BETWEEN 135 AND (135 + 10)) + OR ("ss_coupon_amt" BETWEEN 6071 AND (6071 + 1000)) + OR ("ss_wholesale_cost" BETWEEN 38 AND (38 + 20))) +) b4 +, ( + SELECT + "avg"("ss_list_price") "b5_lp" + , "count"("ss_list_price") "b5_cnt" + , "count"(DISTINCT "ss_list_price") "b5_cntd" + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 21 AND 25) + AND (("ss_list_price" BETWEEN 122 AND (122 + 10)) + OR ("ss_coupon_amt" BETWEEN 836 AND (836 + 1000)) + OR ("ss_wholesale_cost" BETWEEN 17 AND (17 + 20))) +) b5 +, ( + SELECT + "avg"("ss_list_price") "b6_lp" + , "count"("ss_list_price") "b6_cnt" + , "count"(DISTINCT "ss_list_price") "b6_cntd" + FROM + ${database}.${schema}.store_sales + WHERE ("ss_quantity" BETWEEN 26 AND 30) + AND (("ss_list_price" BETWEEN 154 AND (154 + 10)) + OR ("ss_coupon_amt" BETWEEN 7326 AND (7326 + 1000)) + OR ("ss_wholesale_cost" BETWEEN 7 AND (7 + 20))) +) b6 +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q29.sql b/presto-iceberg/src/test/resources/tpcds/queries/q29.sql new file mode 100644 index 0000000000000..b905aa63baaec --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q29.sql @@ -0,0 +1,35 @@ +SELECT + "i_item_id" +, "i_item_desc" +, "s_store_id" +, "s_store_name" +, "sum"("ss_quantity") "store_sales_quantity" +, "sum"("sr_return_quantity") "store_returns_quantity" +, "sum"("cs_quantity") "catalog_sales_quantity" +FROM + ${database}.${schema}.store_sales +, ${database}.${schema}.store_returns +, ${database}.${schema}.catalog_sales +, ${database}.${schema}.date_dim d1 +, ${database}.${schema}.date_dim d2 +, ${database}.${schema}.date_dim d3 +, ${database}.${schema}.store +, ${database}.${schema}.item +WHERE ("d1"."d_moy" = 9) + AND ("d1"."d_year" = 1999) + AND ("d1"."d_date_sk" = "ss_sold_date_sk") + AND ("i_item_sk" = "ss_item_sk") + AND ("s_store_sk" = "ss_store_sk") + AND ("ss_customer_sk" = "sr_customer_sk") + AND ("ss_item_sk" = "sr_item_sk") + AND ("ss_ticket_number" = "sr_ticket_number") + AND ("sr_returned_date_sk" = "d2"."d_date_sk") + AND ("d2"."d_moy" BETWEEN 9 AND (9 + 3)) + AND ("d2"."d_year" = 1999) + AND ("sr_customer_sk" = "cs_bill_customer_sk") + AND ("sr_item_sk" = "cs_item_sk") + AND ("cs_sold_date_sk" = "d3"."d_date_sk") + AND ("d3"."d_year" IN (1999, (1999 + 1), (1999 + 2))) +GROUP BY "i_item_id", "i_item_desc", "s_store_id", "s_store_name" +ORDER BY "i_item_id" ASC, "i_item_desc" ASC, "s_store_id" ASC, "s_store_name" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q30.sql b/presto-iceberg/src/test/resources/tpcds/queries/q30.sql new file mode 100644 index 0000000000000..9cd3cdfce1cb4 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q30.sql @@ -0,0 +1,44 @@ +WITH + customer_total_return AS ( + SELECT + "wr_returning_customer_sk" "ctr_customer_sk" + , "ca_state" "ctr_state" + , "sum"("wr_return_amt") "ctr_total_return" + FROM + ${database}.${schema}.web_returns + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer_address + WHERE ("wr_returned_date_sk" = "d_date_sk") + AND ("d_year" = 2002) + AND ("wr_returning_addr_sk" = "ca_address_sk") + GROUP BY "wr_returning_customer_sk", "ca_state" +) +SELECT + "c_customer_id" +, "c_salutation" +, "c_first_name" +, "c_last_name" +, "c_preferred_cust_flag" +, "c_birth_day" +, "c_birth_month" +, "c_birth_year" +, "c_birth_country" +, "c_login" +, "c_email_address" +, "c_last_review_date_sk" +, "ctr_total_return" +FROM + customer_total_return ctr1 +, ${database}.${schema}.customer_address +, ${database}.${schema}.customer +WHERE ("ctr1"."ctr_total_return" > ( + SELECT ("avg"("ctr_total_return") * DECIMAL '1.2') + FROM + customer_total_return ctr2 + WHERE ("ctr1"."ctr_state" = "ctr2"."ctr_state") + )) + AND ("ca_address_sk" = "c_current_addr_sk") + AND ("ca_state" = 'GA') + AND ("ctr1"."ctr_customer_sk" = "c_customer_sk") +ORDER BY "c_customer_id" ASC, "c_salutation" ASC, "c_first_name" ASC, "c_last_name" ASC, "c_preferred_cust_flag" ASC, "c_birth_day" ASC, "c_birth_month" ASC, "c_birth_year" ASC, "c_birth_country" ASC, "c_login" ASC, "c_email_address" ASC, "c_last_review_date_sk" ASC, "ctr_total_return" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q31.sql b/presto-iceberg/src/test/resources/tpcds/queries/q31.sql new file mode 100644 index 0000000000000..6bf655871e8d5 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q31.sql @@ -0,0 +1,63 @@ +WITH + ss AS ( + SELECT + "ca_county" + , "d_qoy" + , "d_year" + , "sum"("ss_ext_sales_price") "store_sales" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer_address + WHERE ("ss_sold_date_sk" = "d_date_sk") + AND ("ss_addr_sk" = "ca_address_sk") + GROUP BY "ca_county", "d_qoy", "d_year" +) +, ws AS ( + SELECT + "ca_county" + , "d_qoy" + , "d_year" + , "sum"("ws_ext_sales_price") "web_sales" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer_address + WHERE ("ws_sold_date_sk" = "d_date_sk") + AND ("ws_bill_addr_sk" = "ca_address_sk") + GROUP BY "ca_county", "d_qoy", "d_year" +) +SELECT + "ss1"."ca_county" +, "ss1"."d_year" +, ("ws2"."web_sales" / "ws1"."web_sales") "web_q1_q2_increase" +, ("ss2"."store_sales" / "ss1"."store_sales") "store_q1_q2_increase" +, ("ws3"."web_sales" / "ws2"."web_sales") "web_q2_q3_increase" +, ("ss3"."store_sales" / "ss2"."store_sales") "store_q2_q3_increase" +FROM + ss ss1 +, ss ss2 +, ss ss3 +, ws ws1 +, ws ws2 +, ws ws3 +WHERE ("ss1"."d_qoy" = 1) + AND ("ss1"."d_year" = 2000) + AND ("ss1"."ca_county" = "ss2"."ca_county") + AND ("ss2"."d_qoy" = 2) + AND ("ss2"."d_year" = 2000) + AND ("ss2"."ca_county" = "ss3"."ca_county") + AND ("ss3"."d_qoy" = 3) + AND ("ss3"."d_year" = 2000) + AND ("ss1"."ca_county" = "ws1"."ca_county") + AND ("ws1"."d_qoy" = 1) + AND ("ws1"."d_year" = 2000) + AND ("ws1"."ca_county" = "ws2"."ca_county") + AND ("ws2"."d_qoy" = 2) + AND ("ws2"."d_year" = 2000) + AND ("ws1"."ca_county" = "ws3"."ca_county") + AND ("ws3"."d_qoy" = 3) + AND ("ws3"."d_year" = 2000) + AND ((CASE WHEN ("ws1"."web_sales" > 0) THEN (CAST("ws2"."web_sales" AS DECIMAL(38,3)) / "ws1"."web_sales") ELSE null END) > (CASE WHEN ("ss1"."store_sales" > 0) THEN (CAST("ss2"."store_sales" AS DECIMAL(38,3)) / "ss1"."store_sales") ELSE null END)) + AND ((CASE WHEN ("ws2"."web_sales" > 0) THEN (CAST("ws3"."web_sales" AS DECIMAL(38,3)) / "ws2"."web_sales") ELSE null END) > (CASE WHEN ("ss2"."store_sales" > 0) THEN (CAST("ss3"."store_sales" AS DECIMAL(38,3)) / "ss2"."store_sales") ELSE null END)) +ORDER BY "ss1"."ca_county" ASC diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q32.sql b/presto-iceberg/src/test/resources/tpcds/queries/q32.sql new file mode 100644 index 0000000000000..ddb61f438e86d --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q32.sql @@ -0,0 +1,19 @@ +SELECT "sum"("cs_ext_discount_amt") "excess discount amount" +FROM + ${database}.${schema}.catalog_sales +, ${database}.${schema}.item +, ${database}.${schema}.date_dim +WHERE ("i_manufact_id" = 977) + AND ("i_item_sk" = "cs_item_sk") + AND ("d_date" BETWEEN CAST('2000-01-27' AS DATE) AND (CAST('2000-01-27' AS DATE) + INTERVAL '90' DAY)) + AND ("d_date_sk" = "cs_sold_date_sk") + AND ("cs_ext_discount_amt" > ( + SELECT (DECIMAL '1.3' * "avg"("cs_ext_discount_amt")) + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.date_dim + WHERE ("cs_item_sk" = "i_item_sk") + AND ("d_date" BETWEEN CAST('2000-01-27' AS DATE) AND (CAST('2000-01-27' AS DATE) + INTERVAL '90' DAY)) + AND ("d_date_sk" = "cs_sold_date_sk") + )) +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q33.sql b/presto-iceberg/src/test/resources/tpcds/queries/q33.sql new file mode 100644 index 0000000000000..d6b084d0f83c9 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q33.sql @@ -0,0 +1,88 @@ +WITH + ss AS ( + SELECT + "i_manufact_id" + , "sum"("ss_ext_sales_price") "total_sales" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer_address + , ${database}.${schema}.item + WHERE ("i_manufact_id" IN ( + SELECT "i_manufact_id" + FROM + ${database}.${schema}.item + WHERE ("i_category" IN ('Electronics')) + )) + AND ("ss_item_sk" = "i_item_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("d_year" = 1998) + AND ("d_moy" = 5) + AND ("ss_addr_sk" = "ca_address_sk") + AND ("ca_gmt_offset" = -5) + GROUP BY "i_manufact_id" +) +, cs AS ( + SELECT + "i_manufact_id" + , "sum"("cs_ext_sales_price") "total_sales" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer_address + , ${database}.${schema}.item + WHERE ("i_manufact_id" IN ( + SELECT "i_manufact_id" + FROM + ${database}.${schema}.item + WHERE ("i_category" IN ('Electronics')) + )) + AND ("cs_item_sk" = "i_item_sk") + AND ("cs_sold_date_sk" = "d_date_sk") + AND ("d_year" = 1998) + AND ("d_moy" = 5) + AND ("cs_bill_addr_sk" = "ca_address_sk") + AND ("ca_gmt_offset" = -5) + GROUP BY "i_manufact_id" +) +, ws AS ( + SELECT + "i_manufact_id" + , "sum"("ws_ext_sales_price") "total_sales" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer_address + , ${database}.${schema}.item + WHERE ("i_manufact_id" IN ( + SELECT "i_manufact_id" + FROM + ${database}.${schema}.item + WHERE ("i_category" IN ('Electronics')) + )) + AND ("ws_item_sk" = "i_item_sk") + AND ("ws_sold_date_sk" = "d_date_sk") + AND ("d_year" = 1998) + AND ("d_moy" = 5) + AND ("ws_bill_addr_sk" = "ca_address_sk") + AND ("ca_gmt_offset" = -5) + GROUP BY "i_manufact_id" +) +SELECT + "i_manufact_id" +, "sum"("total_sales") "total_sales" +FROM + ( + SELECT * + FROM + ss +UNION ALL SELECT * + FROM + cs +UNION ALL SELECT * + FROM + ws +) tmp1 +GROUP BY "i_manufact_id" +ORDER BY "total_sales" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q34.sql b/presto-iceberg/src/test/resources/tpcds/queries/q34.sql new file mode 100644 index 0000000000000..36deef1a82c8f --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q34.sql @@ -0,0 +1,35 @@ +SELECT + "c_last_name" +, "c_first_name" +, "c_salutation" +, "c_preferred_cust_flag" +, "ss_ticket_number" +, "cnt" +FROM + ( + SELECT + "ss_ticket_number" + , "ss_customer_sk" + , "count"(*) "cnt" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.store + , ${database}.${schema}.household_demographics + WHERE ("store_sales"."ss_sold_date_sk" = "date_dim"."d_date_sk") + AND ("store_sales"."ss_store_sk" = "store"."s_store_sk") + AND ("store_sales"."ss_hdemo_sk" = "household_demographics"."hd_demo_sk") + AND (("date_dim"."d_dom" BETWEEN 1 AND 3) + OR ("date_dim"."d_dom" BETWEEN 25 AND 28)) + AND (("household_demographics"."hd_buy_potential" = '>10000') + OR ("household_demographics"."hd_buy_potential" = 'Unknown')) + AND ("household_demographics"."hd_vehicle_count" > 0) + AND ((CASE WHEN ("household_demographics"."hd_vehicle_count" > 0) THEN (CAST("household_demographics"."hd_dep_count" AS DECIMAL(7,2)) / "household_demographics"."hd_vehicle_count") ELSE null END) > DECIMAL '1.2') + AND ("date_dim"."d_year" IN (1999 , (1999 + 1) , (1999 + 2))) + AND ("store"."s_county" IN ('Williamson County' , 'Williamson County' , 'Williamson County' , 'Williamson County' , 'Williamson County' , 'Williamson County' , 'Williamson County' , 'Williamson County')) + GROUP BY "ss_ticket_number", "ss_customer_sk" +) dn +, ${database}.${schema}.customer +WHERE ("ss_customer_sk" = "c_customer_sk") + AND ("cnt" BETWEEN 15 AND 20) +ORDER BY "c_last_name" ASC, "c_first_name" ASC, "c_salutation" ASC, "c_preferred_cust_flag" DESC, "ss_ticket_number" ASC diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q35.sql b/presto-iceberg/src/test/resources/tpcds/queries/q35.sql new file mode 100644 index 0000000000000..e41243dc391ab --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q35.sql @@ -0,0 +1,58 @@ +SELECT + "ca_state" +, "cd_gender" +, "cd_marital_status" +, "cd_dep_count" +, "count"(*) "cnt1" +, "min"("cd_dep_count") +, "max"("cd_dep_count") +, "avg"("cd_dep_count") +, "cd_dep_employed_count" +, "count"(*) "cnt2" +, "min"("cd_dep_employed_count") +, "max"("cd_dep_employed_count") +, "avg"("cd_dep_employed_count") +, "cd_dep_college_count" +, "count"(*) "cnt3" +, "min"("cd_dep_college_count") +, "max"("cd_dep_college_count") +, "avg"("cd_dep_college_count") +FROM + ${database}.${schema}.customer c +, ${database}.${schema}.customer_address ca +, ${database}.${schema}.customer_demographics +WHERE ("c"."c_current_addr_sk" = "ca"."ca_address_sk") + AND ("cd_demo_sk" = "c"."c_current_cdemo_sk") + AND (EXISTS ( + SELECT * + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + WHERE ("c"."c_customer_sk" = "ss_customer_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("d_year" = 2002) + AND ("d_qoy" < 4) +)) + AND ((EXISTS ( + SELECT * + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + WHERE ("c"."c_customer_sk" = "ws_bill_customer_sk") + AND ("ws_sold_date_sk" = "d_date_sk") + AND ("d_year" = 2002) + AND ("d_qoy" < 4) + )) + OR (EXISTS ( + SELECT * + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.date_dim + WHERE ("c"."c_customer_sk" = "cs_ship_customer_sk") + AND ("cs_sold_date_sk" = "d_date_sk") + AND ("d_year" = 2002) + AND ("d_qoy" < 4) + ))) +GROUP BY "ca_state", "cd_gender", "cd_marital_status", "cd_dep_count", "cd_dep_employed_count", "cd_dep_college_count" +ORDER BY "ca_state" ASC, "cd_gender" ASC, "cd_marital_status" ASC, "cd_dep_count" ASC, "cd_dep_employed_count" ASC, "cd_dep_college_count" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q36.sql b/presto-iceberg/src/test/resources/tpcds/queries/q36.sql new file mode 100644 index 0000000000000..90ae6c0b9cabe --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q36.sql @@ -0,0 +1,27 @@ +SELECT + ("sum"("ss_net_profit") / "sum"("ss_ext_sales_price")) "gross_margin" +, "i_category" +, "i_class" +, (GROUPING ("i_category") + GROUPING ("i_class")) "lochierarchy" +, "rank"() OVER (PARTITION BY (GROUPING ("i_category") + GROUPING ("i_class")), (CASE WHEN (GROUPING ("i_class") = 0) THEN "i_category" END) ORDER BY ("sum"("ss_net_profit") / "sum"("ss_ext_sales_price")) ASC) "rank_within_parent" +FROM + ${database}.${schema}.store_sales +, ${database}.${schema}.date_dim d1 +, ${database}.${schema}.item +, ${database}.${schema}.store +WHERE ("d1"."d_year" = 2001) + AND ("d1"."d_date_sk" = "ss_sold_date_sk") + AND ("i_item_sk" = "ss_item_sk") + AND ("s_store_sk" = "ss_store_sk") + AND ("s_state" IN ( + 'TN' + , 'TN' + , 'TN' + , 'TN' + , 'TN' + , 'TN' + , 'TN' + , 'TN')) +GROUP BY ROLLUP (i_category, i_class) +ORDER BY "lochierarchy" DESC, (CASE WHEN ("lochierarchy" = 0) THEN "i_category" END) ASC, "rank_within_parent" ASC, "i_category", "i_class" +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q37.sql b/presto-iceberg/src/test/resources/tpcds/queries/q37.sql new file mode 100644 index 0000000000000..a640aa4836096 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q37.sql @@ -0,0 +1,19 @@ +SELECT + "i_item_id" +, "i_item_desc" +, "i_current_price" +FROM + ${database}.${schema}.item +, ${database}.${schema}.inventory +, ${database}.${schema}.date_dim +, ${database}.${schema}.catalog_sales +WHERE ("i_current_price" BETWEEN 68 AND (68 + 30)) + AND ("inv_item_sk" = "i_item_sk") + AND ("d_date_sk" = "inv_date_sk") + AND (CAST("d_date" AS DATE) BETWEEN CAST('2000-02-01' AS DATE) AND (CAST('2000-02-01' AS DATE) + INTERVAL '60' DAY)) + AND ("i_manufact_id" IN (677, 940, 694, 808)) + AND ("inv_quantity_on_hand" BETWEEN 100 AND 500) + AND ("cs_item_sk" = "i_item_sk") +GROUP BY "i_item_id", "i_item_desc", "i_current_price" +ORDER BY "i_item_id" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q38.sql b/presto-iceberg/src/test/resources/tpcds/queries/q38.sql new file mode 100644 index 0000000000000..4fed9f596c5d2 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q38.sql @@ -0,0 +1,38 @@ +SELECT "count"(*) +FROM + ( + SELECT DISTINCT + "c_last_name" + , "c_first_name" + , "d_date" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer + WHERE ("store_sales"."ss_sold_date_sk" = "date_dim"."d_date_sk") + AND ("store_sales"."ss_customer_sk" = "customer"."c_customer_sk") + AND ("d_month_seq" BETWEEN 1200 AND (1200 + 11)) +INTERSECT SELECT DISTINCT + "c_last_name" + , "c_first_name" + , "d_date" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer + WHERE ("catalog_sales"."cs_sold_date_sk" = "date_dim"."d_date_sk") + AND ("catalog_sales"."cs_bill_customer_sk" = "customer"."c_customer_sk") + AND ("d_month_seq" BETWEEN 1200 AND (1200 + 11)) +INTERSECT SELECT DISTINCT + "c_last_name" + , "c_first_name" + , "d_date" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer + WHERE ("web_sales"."ws_sold_date_sk" = "date_dim"."d_date_sk") + AND ("web_sales"."ws_bill_customer_sk" = "customer"."c_customer_sk") + AND ("d_month_seq" BETWEEN 1200 AND (1200 + 11)) +) hot_cust +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q39_1.sql b/presto-iceberg/src/test/resources/tpcds/queries/q39_1.sql new file mode 100644 index 0000000000000..3744322454460 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q39_1.sql @@ -0,0 +1,51 @@ +WITH + inv AS ( + SELECT + "w_warehouse_name" + , "w_warehouse_sk" + , "i_item_sk" + , "d_moy" + , "stdev" + , "mean" + , (CASE "mean" WHEN 0 THEN null ELSE ("stdev" / "mean") END) "cov" + FROM + ( + SELECT + "w_warehouse_name" + , "w_warehouse_sk" + , "i_item_sk" + , "d_moy" + , "stddev_samp"("inv_quantity_on_hand") "stdev" + , "avg"("inv_quantity_on_hand") "mean" + FROM + ${database}.${schema}.inventory + , ${database}.${schema}.item + , ${database}.${schema}.warehouse + , ${database}.${schema}.date_dim + WHERE ("inv_item_sk" = "i_item_sk") + AND ("inv_warehouse_sk" = "w_warehouse_sk") + AND ("inv_date_sk" = "d_date_sk") + AND ("d_year" = 2001) + GROUP BY "w_warehouse_name", "w_warehouse_sk", "i_item_sk", "d_moy" + ) foo + WHERE ((CASE "mean" WHEN 0 THEN 0 ELSE ("stdev" / "mean") END) > 1) +) +SELECT + "inv1"."w_warehouse_sk" +, "inv1"."i_item_sk" +, "inv1"."d_moy" +, "inv1"."mean" +, "inv1"."cov" +, "inv2"."w_warehouse_sk" +, "inv2"."i_item_sk" +, "inv2"."d_moy" +, "inv2"."mean" +, "inv2"."cov" +FROM + inv inv1 +, inv inv2 +WHERE ("inv1"."i_item_sk" = "inv2"."i_item_sk") + AND ("inv1"."w_warehouse_sk" = "inv2"."w_warehouse_sk") + AND ("inv1"."d_moy" = 1) + AND ("inv2"."d_moy" = (1 + 1)) +ORDER BY "inv1"."w_warehouse_sk" ASC, "inv1"."i_item_sk" ASC, "inv1"."d_moy" ASC, "inv1"."mean" ASC, "inv1"."cov" ASC, "inv2"."d_moy" ASC, "inv2"."mean" ASC, "inv2"."cov" ASC diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q39_2.sql b/presto-iceberg/src/test/resources/tpcds/queries/q39_2.sql new file mode 100644 index 0000000000000..5db21ff02be2c --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q39_2.sql @@ -0,0 +1,52 @@ +WITH + inv AS ( + SELECT + "w_warehouse_name" + , "w_warehouse_sk" + , "i_item_sk" + , "d_moy" + , "stdev" + , "mean" + , (CASE "mean" WHEN 0 THEN null ELSE ("stdev" / "mean") END) "cov" + FROM + ( + SELECT + "w_warehouse_name" + , "w_warehouse_sk" + , "i_item_sk" + , "d_moy" + , "stddev_samp"("inv_quantity_on_hand") "stdev" + , "avg"("inv_quantity_on_hand") "mean" + FROM + ${database}.${schema}.inventory + , ${database}.${schema}.item + , ${database}.${schema}.warehouse + , ${database}.${schema}.date_dim + WHERE ("inv_item_sk" = "i_item_sk") + AND ("inv_warehouse_sk" = "w_warehouse_sk") + AND ("inv_date_sk" = "d_date_sk") + AND ("d_year" = 2001) + GROUP BY "w_warehouse_name", "w_warehouse_sk", "i_item_sk", "d_moy" + ) foo + WHERE ((CASE "mean" WHEN 0 THEN 0 ELSE ("stdev" / "mean") END) > 1) +) +SELECT + "inv1"."w_warehouse_sk" +, "inv1"."i_item_sk" +, "inv1"."d_moy" +, "inv1"."mean" +, "inv1"."cov" +, "inv2"."w_warehouse_sk" +, "inv2"."i_item_sk" +, "inv2"."d_moy" +, "inv2"."mean" +, "inv2"."cov" +FROM + inv inv1 +, inv inv2 +WHERE ("inv1"."i_item_sk" = "inv2"."i_item_sk") + AND ("inv1"."w_warehouse_sk" = "inv2"."w_warehouse_sk") + AND ("inv1"."d_moy" = 1) + AND ("inv2"."d_moy" = (1 + 1)) + AND ("inv1"."cov" > DECIMAL '1.5') +ORDER BY "inv1"."w_warehouse_sk" ASC, "inv1"."i_item_sk" ASC, "inv1"."d_moy" ASC, "inv1"."mean" ASC, "inv1"."cov" ASC, "inv2"."d_moy" ASC, "inv2"."mean" ASC, "inv2"."cov" ASC diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q40.sql b/presto-iceberg/src/test/resources/tpcds/queries/q40.sql new file mode 100644 index 0000000000000..15cfe16592dde --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q40.sql @@ -0,0 +1,20 @@ +SELECT + "w_state" +, "i_item_id" +, "sum"((CASE WHEN (CAST("d_date" AS DATE) < CAST('2000-03-11' AS DATE)) THEN ("cs_sales_price" - COALESCE("cr_refunded_cash", 0)) ELSE 0 END)) "sales_before" +, "sum"((CASE WHEN (CAST("d_date" AS DATE) >= CAST('2000-03-11' AS DATE)) THEN ("cs_sales_price" - COALESCE("cr_refunded_cash", 0)) ELSE 0 END)) "sales_after" +FROM + (${database}.${schema}.catalog_sales +LEFT JOIN ${database}.${schema}.catalog_returns ON ("cs_order_number" = "cr_order_number") + AND ("cs_item_sk" = "cr_item_sk")) +, ${database}.${schema}.warehouse +, ${database}.${schema}.item +, ${database}.${schema}.date_dim +WHERE ("i_current_price" BETWEEN DECIMAL '0.99' AND DECIMAL '1.49') + AND ("i_item_sk" = "cs_item_sk") + AND ("cs_warehouse_sk" = "w_warehouse_sk") + AND ("cs_sold_date_sk" = "d_date_sk") + AND (CAST("d_date" AS DATE) BETWEEN (CAST('2000-03-11' AS DATE) - INTERVAL '30' DAY) AND (CAST('2000-03-11' AS DATE) + INTERVAL '30' DAY)) +GROUP BY "w_state", "i_item_id" +ORDER BY "w_state" ASC, "i_item_id" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q41.sql b/presto-iceberg/src/test/resources/tpcds/queries/q41.sql new file mode 100644 index 0000000000000..a430cf8987ae7 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q41.sql @@ -0,0 +1,69 @@ +SELECT DISTINCT "i_product_name" +FROM + ${database}.${schema}.item i1 +WHERE ("i_manufact_id" BETWEEN 738 AND (738 + 40)) + AND (( + SELECT "count"(*) "item_cnt" + FROM + ${database}.${schema}.item + WHERE (("i_manufact" = "i1"."i_manufact") + AND ((("i_category" = 'Women') + AND (("i_color" = 'powder') + OR ("i_color" = 'khaki')) + AND (("i_units" = 'Ounce') + OR ("i_units" = 'Oz')) + AND (("i_size" = 'medium') + OR ("i_size" = 'extra large'))) + OR (("i_category" = 'Women') + AND (("i_color" = 'brown') + OR ("i_color" = 'honeydew')) + AND (("i_units" = 'Bunch') + OR ("i_units" = 'Ton')) + AND (("i_size" = 'N/A') + OR ("i_size" = 'small'))) + OR (("i_category" = 'Men') + AND (("i_color" = 'floral') + OR ("i_color" = 'deep')) + AND (("i_units" = 'N/A') + OR ("i_units" = 'Dozen')) + AND (("i_size" = 'petite') + OR ("i_size" = 'large'))) + OR (("i_category" = 'Men') + AND (("i_color" = 'light') + OR ("i_color" = 'cornflower')) + AND (("i_units" = 'Box') + OR ("i_units" = 'Pound')) + AND (("i_size" = 'medium') + OR ("i_size" = 'extra large'))))) + OR (("i_manufact" = "i1"."i_manufact") + AND ((("i_category" = 'Women') + AND (("i_color" = 'midnight') + OR ("i_color" = 'snow')) + AND (("i_units" = 'Pallet') + OR ("i_units" = 'Gross')) + AND (("i_size" = 'medium') + OR ("i_size" = 'extra large'))) + OR (("i_category" = 'Women') + AND (("i_color" = 'cyan') + OR ("i_color" = 'papaya')) + AND (("i_units" = 'Cup') + OR ("i_units" = 'Dram')) + AND (("i_size" = 'N/A') + OR ("i_size" = 'small'))) + OR (("i_category" = 'Men') + AND (("i_color" = 'orange') + OR ("i_color" = 'frosted')) + AND (("i_units" = 'Each') + OR ("i_units" = 'Tbl')) + AND (("i_size" = 'petite') + OR ("i_size" = 'large'))) + OR (("i_category" = 'Men') + AND (("i_color" = 'forest') + OR ("i_color" = 'ghost')) + AND (("i_units" = 'Lb') + OR ("i_units" = 'Bundle')) + AND (("i_size" = 'medium') + OR ("i_size" = 'extra large'))))) + ) > 0) +ORDER BY "i_product_name" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q42.sql b/presto-iceberg/src/test/resources/tpcds/queries/q42.sql new file mode 100644 index 0000000000000..c7929fd0e0a29 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q42.sql @@ -0,0 +1,17 @@ +SELECT + "dt"."d_year" +, "item"."i_category_id" +, "item"."i_category" +, "sum"("ss_ext_sales_price") +FROM + ${database}.${schema}.date_dim dt +, ${database}.${schema}.store_sales +, ${database}.${schema}.item +WHERE ("dt"."d_date_sk" = "store_sales"."ss_sold_date_sk") + AND ("store_sales"."ss_item_sk" = "item"."i_item_sk") + AND ("item"."i_manager_id" = 1) + AND ("dt"."d_moy" = 11) + AND ("dt"."d_year" = 2000) +GROUP BY "dt"."d_year", "item"."i_category_id", "item"."i_category" +ORDER BY "sum"("ss_ext_sales_price") DESC, "dt"."d_year" ASC, "item"."i_category_id" ASC, "item"."i_category" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q43.sql b/presto-iceberg/src/test/resources/tpcds/queries/q43.sql new file mode 100644 index 0000000000000..95080a685df84 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q43.sql @@ -0,0 +1,21 @@ +SELECT + "s_store_name" +, "s_store_id" +, "sum"((CASE WHEN ("d_day_name" = 'Sunday') THEN "ss_sales_price" ELSE null END)) "sun_sales" +, "sum"((CASE WHEN ("d_day_name" = 'Monday') THEN "ss_sales_price" ELSE null END)) "mon_sales" +, "sum"((CASE WHEN ("d_day_name" = 'Tuesday') THEN "ss_sales_price" ELSE null END)) "tue_sales" +, "sum"((CASE WHEN ("d_day_name" = 'Wednesday') THEN "ss_sales_price" ELSE null END)) "wed_sales" +, "sum"((CASE WHEN ("d_day_name" = 'Thursday') THEN "ss_sales_price" ELSE null END)) "thu_sales" +, "sum"((CASE WHEN ("d_day_name" = 'Friday') THEN "ss_sales_price" ELSE null END)) "fri_sales" +, "sum"((CASE WHEN ("d_day_name" = 'Saturday') THEN "ss_sales_price" ELSE null END)) "sat_sales" +FROM + ${database}.${schema}.date_dim +, ${database}.${schema}.store_sales +, ${database}.${schema}.store +WHERE ("d_date_sk" = "ss_sold_date_sk") + AND ("s_store_sk" = "ss_store_sk") + AND ("s_gmt_offset" = -5) + AND ("d_year" = 2000) +GROUP BY "s_store_name", "s_store_id" +ORDER BY "s_store_name" ASC, "s_store_id" ASC, "sun_sales" ASC, "mon_sales" ASC, "tue_sales" ASC, "wed_sales" ASC, "thu_sales" ASC, "fri_sales" ASC, "sat_sales" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q44.sql b/presto-iceberg/src/test/resources/tpcds/queries/q44.sql new file mode 100644 index 0000000000000..61ae9e596108f --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q44.sql @@ -0,0 +1,68 @@ +SELECT + "asceding"."rnk" +, "i1"."i_product_name" "best_performing" +, "i2"."i_product_name" "worst_performing" +FROM + ( + SELECT * + FROM + ( + SELECT + "item_sk" + , "rank"() OVER (ORDER BY "rank_col" ASC) "rnk" + FROM + ( + SELECT + "ss_item_sk" "item_sk" + , "avg"("ss_net_profit") "rank_col" + FROM + ${database}.${schema}.store_sales ss1 + WHERE ("ss_store_sk" = 4) + GROUP BY "ss_item_sk" + HAVING ("avg"("ss_net_profit") > (DECIMAL '0.9' * ( + SELECT "avg"("ss_net_profit") "rank_col" + FROM + ${database}.${schema}.store_sales + WHERE ("ss_store_sk" = 4) + AND ("ss_addr_sk" IS NULL) + GROUP BY "ss_store_sk" + ))) + ) v1 + ) v11 + WHERE ("rnk" < 11) +) asceding +, ( + SELECT * + FROM + ( + SELECT + "item_sk" + , "rank"() OVER (ORDER BY "rank_col" DESC) "rnk" + FROM + ( + SELECT + "ss_item_sk" "item_sk" + , "avg"("ss_net_profit") "rank_col" + FROM + ${database}.${schema}.store_sales ss1 + WHERE ("ss_store_sk" = 4) + GROUP BY "ss_item_sk" + HAVING ("avg"("ss_net_profit") > (DECIMAL '0.9' * ( + SELECT "avg"("ss_net_profit") "rank_col" + FROM + ${database}.${schema}.store_sales + WHERE ("ss_store_sk" = 4) + AND ("ss_addr_sk" IS NULL) + GROUP BY "ss_store_sk" + ))) + ) v2 + ) v21 + WHERE ("rnk" < 11) +) descending +, ${database}.${schema}.item i1 +, ${database}.${schema}.item i2 +WHERE ("asceding"."rnk" = "descending"."rnk") + AND ("i1"."i_item_sk" = "asceding"."item_sk") + AND ("i2"."i_item_sk" = "descending"."item_sk") +ORDER BY "asceding"."rnk" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q45.sql b/presto-iceberg/src/test/resources/tpcds/queries/q45.sql new file mode 100644 index 0000000000000..36edec93968b0 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q45.sql @@ -0,0 +1,26 @@ +SELECT + "ca_zip" +, "ca_city" +, "sum"("ws_sales_price") +FROM + ${database}.${schema}.web_sales +, ${database}.${schema}.customer +, ${database}.${schema}.customer_address +, ${database}.${schema}.date_dim +, ${database}.${schema}.item +WHERE ("ws_bill_customer_sk" = "c_customer_sk") + AND ("c_current_addr_sk" = "ca_address_sk") + AND ("ws_item_sk" = "i_item_sk") + AND (("substr"("ca_zip", 1, 5) IN ('85669' , '86197' , '88274' , '83405' , '86475' , '85392' , '85460' , '80348' , '81792')) + OR ("i_item_id" IN ( + SELECT "i_item_id" + FROM + ${database}.${schema}.item + WHERE ("i_item_sk" IN (2 , 3 , 5 , 7 , 11 , 13 , 17 , 19 , 23 , 29)) + ))) + AND ("ws_sold_date_sk" = "d_date_sk") + AND ("d_qoy" = 2) + AND ("d_year" = 2001) +GROUP BY "ca_zip", "ca_city" +ORDER BY "ca_zip" ASC, "ca_city" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q46.sql b/presto-iceberg/src/test/resources/tpcds/queries/q46.sql new file mode 100644 index 0000000000000..bc4cdd8299443 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q46.sql @@ -0,0 +1,40 @@ +SELECT + "c_last_name" +, "c_first_name" +, "ca_city" +, "bought_city" +, "ss_ticket_number" +, "amt" +, "profit" +FROM + ( + SELECT + "ss_ticket_number" + , "ss_customer_sk" + , "ca_city" "bought_city" + , "sum"("ss_coupon_amt") "amt" + , "sum"("ss_net_profit") "profit" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.store + , ${database}.${schema}.household_demographics + , ${database}.${schema}.customer_address + WHERE ("store_sales"."ss_sold_date_sk" = "date_dim"."d_date_sk") + AND ("store_sales"."ss_store_sk" = "store"."s_store_sk") + AND ("store_sales"."ss_hdemo_sk" = "household_demographics"."hd_demo_sk") + AND ("store_sales"."ss_addr_sk" = "customer_address"."ca_address_sk") + AND (("household_demographics"."hd_dep_count" = 4) + OR ("household_demographics"."hd_vehicle_count" = 3)) + AND ("date_dim"."d_dow" IN (6 , 0)) + AND ("date_dim"."d_year" IN (1999 , (1999 + 1) , (1999 + 2))) + AND ("store"."s_city" IN ('Fairview' , 'Midway' , 'Fairview' , 'Fairview' , 'Fairview')) + GROUP BY "ss_ticket_number", "ss_customer_sk", "ss_addr_sk", "ca_city" +) dn +, ${database}.${schema}.customer +, ${database}.${schema}.customer_address current_addr +WHERE ("ss_customer_sk" = "c_customer_sk") + AND ("customer"."c_current_addr_sk" = "current_addr"."ca_address_sk") + AND ("current_addr"."ca_city" <> "bought_city") +ORDER BY "c_last_name" ASC, "c_first_name" ASC, "ca_city" ASC, "bought_city" ASC, "ss_ticket_number" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q47.sql b/presto-iceberg/src/test/resources/tpcds/queries/q47.sql new file mode 100644 index 0000000000000..e5af800839a9f --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q47.sql @@ -0,0 +1,62 @@ +WITH + v1 AS ( + SELECT + "i_category" + , "i_brand" + , "s_store_name" + , "s_company_name" + , "d_year" + , "d_moy" + , "sum"("ss_sales_price") "sum_sales" + , "avg"("sum"("ss_sales_price")) OVER (PARTITION BY "i_category", "i_brand", "s_store_name", "s_company_name", "d_year") "avg_monthly_sales" + , "rank"() OVER (PARTITION BY "i_category", "i_brand", "s_store_name", "s_company_name" ORDER BY "d_year" ASC, "d_moy" ASC) "rn" + FROM + ${database}.${schema}.item + , ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.store + WHERE ("ss_item_sk" = "i_item_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("ss_store_sk" = "s_store_sk") + AND (("d_year" = 1999) + OR (("d_year" = (1999 - 1)) + AND ("d_moy" = 12)) + OR (("d_year" = (1999 + 1)) + AND ("d_moy" = 1))) + GROUP BY "i_category", "i_brand", "s_store_name", "s_company_name", "d_year", "d_moy" +) +, v2 AS ( + SELECT + "v1"."i_category" + , "v1"."i_brand" + , "v1"."s_store_name" + , "v1"."s_company_name" + , "v1"."d_year" + , "v1"."d_moy" + , "v1"."avg_monthly_sales" + , "v1"."sum_sales" + , "v1_lag"."sum_sales" "psum" + , "v1_lead"."sum_sales" "nsum" + FROM + v1 + , v1 v1_lag + , v1 v1_lead + WHERE ("v1"."i_category" = "v1_lag"."i_category") + AND ("v1"."i_category" = "v1_lead"."i_category") + AND ("v1"."i_brand" = "v1_lag"."i_brand") + AND ("v1"."i_brand" = "v1_lead"."i_brand") + AND ("v1"."s_store_name" = "v1_lag"."s_store_name") + AND ("v1"."s_store_name" = "v1_lead"."s_store_name") + AND ("v1"."s_company_name" = "v1_lag"."s_company_name") + AND ("v1"."s_company_name" = "v1_lead"."s_company_name") + AND ("v1"."rn" = ("v1_lag"."rn" + 1)) + AND ("v1"."rn" = ("v1_lead"."rn" - 1)) +) +SELECT * +FROM + v2 +WHERE ("d_year" = 1999) + AND ("avg_monthly_sales" > 0) + AND ((CASE WHEN ("avg_monthly_sales" > 0) THEN ("abs"(("sum_sales" - "avg_monthly_sales")) / "avg_monthly_sales") ELSE null END) > DECIMAL '0.1') +ORDER BY ("sum_sales" - "avg_monthly_sales") ASC, 3 ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q48.sql b/presto-iceberg/src/test/resources/tpcds/queries/q48.sql new file mode 100644 index 0000000000000..10ea7578b8704 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q48.sql @@ -0,0 +1,34 @@ +SELECT "sum"("ss_quantity") +FROM + ${database}.${schema}.store_sales +, ${database}.${schema}.store +, ${database}.${schema}.customer_demographics +, ${database}.${schema}.customer_address +, ${database}.${schema}.date_dim +WHERE ("s_store_sk" = "ss_store_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("d_year" = 2000) + AND ((("cd_demo_sk" = "ss_cdemo_sk") + AND ("cd_marital_status" = 'M') + AND ("cd_education_status" = '4 yr Degree') + AND ("ss_sales_price" BETWEEN DECIMAL '100.00' AND DECIMAL '150.00')) + OR (("cd_demo_sk" = "ss_cdemo_sk") + AND ("cd_marital_status" = 'D') + AND ("cd_education_status" = '2 yr Degree') + AND ("ss_sales_price" BETWEEN DECIMAL '50.00' AND DECIMAL '100.00')) + OR (("cd_demo_sk" = "ss_cdemo_sk") + AND ("cd_marital_status" = 'S') + AND ("cd_education_status" = 'College') + AND ("ss_sales_price" BETWEEN DECIMAL '150.00' AND DECIMAL '200.00'))) + AND ((("ss_addr_sk" = "ca_address_sk") + AND ("ca_country" = 'United States') + AND ("ca_state" IN ('CO' , 'OH' , 'TX')) + AND ("ss_net_profit" BETWEEN 0 AND 2000)) + OR (("ss_addr_sk" = "ca_address_sk") + AND ("ca_country" = 'United States') + AND ("ca_state" IN ('OR' , 'MN' , 'KY')) + AND ("ss_net_profit" BETWEEN 150 AND 3000)) + OR (("ss_addr_sk" = "ca_address_sk") + AND ("ca_country" = 'United States') + AND ("ca_state" IN ('VA' , 'CA' , 'MS')) + AND ("ss_net_profit" BETWEEN 50 AND 25000))) diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q49.sql b/presto-iceberg/src/test/resources/tpcds/queries/q49.sql new file mode 100644 index 0000000000000..6b2223d707db9 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q49.sql @@ -0,0 +1,113 @@ +SELECT + 'web' "channel" +, "web"."item" +, "web"."return_ratio" +, "web"."return_rank" +, "web"."currency_rank" +FROM + ( + SELECT + "item" + , "return_ratio" + , "currency_ratio" + , "rank"() OVER (ORDER BY "return_ratio" ASC) "return_rank" + , "rank"() OVER (ORDER BY "currency_ratio" ASC) "currency_rank" + FROM + ( + SELECT + "ws"."ws_item_sk" "item" + , (CAST("sum"(COALESCE("wr"."wr_return_quantity", 0)) AS DECIMAL(15,4)) / CAST("sum"(COALESCE("ws"."ws_quantity", 0)) AS DECIMAL(15,4))) "return_ratio" + , (CAST("sum"(COALESCE("wr"."wr_return_amt", 0)) AS DECIMAL(15,4)) / CAST("sum"(COALESCE("ws"."ws_net_paid", 0)) AS DECIMAL(15,4))) "currency_ratio" + FROM + (${database}.${schema}.web_sales ws + LEFT JOIN ${database}.${schema}.web_returns wr ON ("ws"."ws_order_number" = "wr"."wr_order_number") + AND ("ws"."ws_item_sk" = "wr"."wr_item_sk")) + , ${database}.${schema}.date_dim + WHERE ("wr"."wr_return_amt" > 10000) + AND ("ws"."ws_net_profit" > 1) + AND ("ws"."ws_net_paid" > 0) + AND ("ws"."ws_quantity" > 0) + AND ("ws_sold_date_sk" = "d_date_sk") + AND ("d_year" = 2001) + AND ("d_moy" = 12) + GROUP BY "ws"."ws_item_sk" + ) in_web +) web +WHERE ("web"."return_rank" <= 10) + OR ("web"."currency_rank" <= 10) +UNION SELECT + 'catalog' "channel" +, "catalog"."item" +, "catalog"."return_ratio" +, "catalog"."return_rank" +, "catalog"."currency_rank" +FROM + ( + SELECT + "item" + , "return_ratio" + , "currency_ratio" + , "rank"() OVER (ORDER BY "return_ratio" ASC) "return_rank" + , "rank"() OVER (ORDER BY "currency_ratio" ASC) "currency_rank" + FROM + ( + SELECT + "cs"."cs_item_sk" "item" + , (CAST("sum"(COALESCE("cr"."cr_return_quantity", 0)) AS DECIMAL(15,4)) / CAST("sum"(COALESCE("cs"."cs_quantity", 0)) AS DECIMAL(15,4))) "return_ratio" + , (CAST("sum"(COALESCE("cr"."cr_return_amount", 0)) AS DECIMAL(15,4)) / CAST("sum"(COALESCE("cs"."cs_net_paid", 0)) AS DECIMAL(15,4))) "currency_ratio" + FROM + (${database}.${schema}.catalog_sales cs + LEFT JOIN ${database}.${schema}.catalog_returns cr ON ("cs"."cs_order_number" = "cr"."cr_order_number") + AND ("cs"."cs_item_sk" = "cr"."cr_item_sk")) + , ${database}.${schema}.date_dim + WHERE ("cr"."cr_return_amount" > 10000) + AND ("cs"."cs_net_profit" > 1) + AND ("cs"."cs_net_paid" > 0) + AND ("cs"."cs_quantity" > 0) + AND ("cs_sold_date_sk" = "d_date_sk") + AND ("d_year" = 2001) + AND ("d_moy" = 12) + GROUP BY "cs"."cs_item_sk" + ) in_cat +) "CATALOG" +WHERE ("catalog"."return_rank" <= 10) + OR ("catalog"."currency_rank" <= 10) +UNION SELECT + '${database}.${schema}.store' "channel" +, "store"."item" +, "store"."return_ratio" +, "store"."return_rank" +, "store"."currency_rank" +FROM + ( + SELECT + "item" + , "return_ratio" + , "currency_ratio" + , "rank"() OVER (ORDER BY "return_ratio" ASC) "return_rank" + , "rank"() OVER (ORDER BY "currency_ratio" ASC) "currency_rank" + FROM + ( + SELECT + "sts"."ss_item_sk" "item" + , (CAST("sum"(COALESCE("sr"."sr_return_quantity", 0)) AS DECIMAL(15,4)) / CAST("sum"(COALESCE("sts"."ss_quantity", 0)) AS DECIMAL(15,4))) "return_ratio" + , (CAST("sum"(COALESCE("sr"."sr_return_amt", 0)) AS DECIMAL(15,4)) / CAST("sum"(COALESCE("sts"."ss_net_paid", 0)) AS DECIMAL(15,4))) "currency_ratio" + FROM + (${database}.${schema}.store_sales sts + LEFT JOIN ${database}.${schema}.store_returns sr ON ("sts"."ss_ticket_number" = "sr"."sr_ticket_number") + AND ("sts"."ss_item_sk" = "sr"."sr_item_sk")) + , ${database}.${schema}.date_dim + WHERE ("sr"."sr_return_amt" > 10000) + AND ("sts"."ss_net_profit" > 1) + AND ("sts"."ss_net_paid" > 0) + AND ("sts"."ss_quantity" > 0) + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("d_year" = 2001) + AND ("d_moy" = 12) + GROUP BY "sts"."ss_item_sk" + ) in_store +) store +WHERE ("store"."return_rank" <= 10) + OR ("store"."currency_rank" <= 10) +ORDER BY 1 ASC, 4 ASC, 5 ASC, 2 ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q50.sql b/presto-iceberg/src/test/resources/tpcds/queries/q50.sql new file mode 100644 index 0000000000000..e4ccafd1ebe5f --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q50.sql @@ -0,0 +1,36 @@ +SELECT + "s_store_name" +, "s_company_id" +, "s_street_number" +, "s_street_name" +, "s_street_type" +, "s_suite_number" +, "s_city" +, "s_county" +, "s_state" +, "s_zip" +, "sum"((CASE WHEN (("sr_returned_date_sk" - "ss_sold_date_sk") <= 30) THEN 1 ELSE 0 END)) "30 days" +, "sum"((CASE WHEN (("sr_returned_date_sk" - "ss_sold_date_sk") > 30) + AND (("sr_returned_date_sk" - "ss_sold_date_sk") <= 60) THEN 1 ELSE 0 END)) "31-60 days" +, "sum"((CASE WHEN (("sr_returned_date_sk" - "ss_sold_date_sk") > 60) + AND (("sr_returned_date_sk" - "ss_sold_date_sk") <= 90) THEN 1 ELSE 0 END)) "61-90 days" +, "sum"((CASE WHEN (("sr_returned_date_sk" - "ss_sold_date_sk") > 90) + AND (("sr_returned_date_sk" - "ss_sold_date_sk") <= 120) THEN 1 ELSE 0 END)) "91-120 days" +, "sum"((CASE WHEN (("sr_returned_date_sk" - "ss_sold_date_sk") > 120) THEN 1 ELSE 0 END)) ">120 days" +FROM + ${database}.${schema}.store_sales +, ${database}.${schema}.store_returns +, ${database}.${schema}.store +, ${database}.${schema}.date_dim d1 +, ${database}.${schema}.date_dim d2 +WHERE ("d2"."d_year" = 2001) + AND ("d2"."d_moy" = 8) + AND ("ss_ticket_number" = "sr_ticket_number") + AND ("ss_item_sk" = "sr_item_sk") + AND ("ss_sold_date_sk" = "d1"."d_date_sk") + AND ("sr_returned_date_sk" = "d2"."d_date_sk") + AND ("ss_customer_sk" = "sr_customer_sk") + AND ("ss_store_sk" = "s_store_sk") +GROUP BY "s_store_name", "s_company_id", "s_street_number", "s_street_name", "s_street_type", "s_suite_number", "s_city", "s_county", "s_state", "s_zip" +ORDER BY "s_store_name" ASC, "s_company_id" ASC, "s_street_number" ASC, "s_street_name" ASC, "s_street_type" ASC, "s_suite_number" ASC, "s_city" ASC, "s_county" ASC, "s_state" ASC, "s_zip" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q51.sql b/presto-iceberg/src/test/resources/tpcds/queries/q51.sql new file mode 100644 index 0000000000000..3ea6c41a03923 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q51.sql @@ -0,0 +1,53 @@ +WITH + web_v1 AS ( + SELECT + "ws_item_sk" "item_sk" + , "d_date" + , "sum"("sum"("ws_sales_price")) OVER (PARTITION BY "ws_item_sk" ORDER BY "d_date" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) "cume_sales" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + WHERE ("ws_sold_date_sk" = "d_date_sk") + AND ("d_month_seq" BETWEEN 1200 AND (1200 + 11)) + AND ("ws_item_sk" IS NOT NULL) + GROUP BY "ws_item_sk", "d_date" +) +, store_v1 AS ( + SELECT + "ss_item_sk" "item_sk" + , "d_date" + , "sum"("sum"("ss_sales_price")) OVER (PARTITION BY "ss_item_sk" ORDER BY "d_date" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) "cume_sales" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + WHERE ("ss_sold_date_sk" = "d_date_sk") + AND ("d_month_seq" BETWEEN 1200 AND (1200 + 11)) + AND ("ss_item_sk" IS NOT NULL) + GROUP BY "ss_item_sk", "d_date" +) +SELECT * +FROM + ( + SELECT + "item_sk" + , "d_date" + , "web_sales" + , "store_sales" + , "max"("web_sales") OVER (PARTITION BY "item_sk" ORDER BY "d_date" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) "web_cumulative" + , "max"("store_sales") OVER (PARTITION BY "item_sk" ORDER BY "d_date" ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) "store_cumulative" + FROM + ( + SELECT + (CASE WHEN ("web"."item_sk" IS NOT NULL) THEN "web"."item_sk" ELSE "store"."item_sk" END) "item_sk" + , (CASE WHEN ("web"."d_date" IS NOT NULL) THEN "web"."d_date" ELSE "store"."d_date" END) "d_date" + , "web"."cume_sales" "web_sales" + , "store"."cume_sales" "store_sales" + FROM + (web_v1 web + FULL JOIN store_v1 store ON ("web"."item_sk" = "store"."item_sk") + AND ("web"."d_date" = "store"."d_date")) + ) x +) y +WHERE ("web_cumulative" > "store_cumulative") +ORDER BY "item_sk" ASC, "d_date" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q52.sql b/presto-iceberg/src/test/resources/tpcds/queries/q52.sql new file mode 100644 index 0000000000000..29a67a34fa9cb --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q52.sql @@ -0,0 +1,17 @@ +SELECT + "dt"."d_year" +, "item"."i_brand_id" "brand_id" +, "item"."i_brand" "brand" +, "sum"("ss_ext_sales_price") "ext_price" +FROM + ${database}.${schema}.date_dim dt +, ${database}.${schema}.store_sales +, ${database}.${schema}.item +WHERE ("dt"."d_date_sk" = "store_sales"."ss_sold_date_sk") + AND ("store_sales"."ss_item_sk" = "item"."i_item_sk") + AND ("item"."i_manager_id" = 1) + AND ("dt"."d_moy" = 11) + AND ("dt"."d_year" = 2000) +GROUP BY "dt"."d_year", "item"."i_brand", "item"."i_brand_id" +ORDER BY "dt"."d_year" ASC, "ext_price" DESC, "brand_id" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q53.sql b/presto-iceberg/src/test/resources/tpcds/queries/q53.sql new file mode 100644 index 0000000000000..0bf02dfb65a34 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q53.sql @@ -0,0 +1,27 @@ +SELECT * +FROM + ( + SELECT + "i_manufact_id" + , "sum"("ss_sales_price") "sum_sales" + , "avg"("sum"("ss_sales_price")) OVER (PARTITION BY "i_manufact_id") "avg_quarterly_sales" + FROM + ${database}.${schema}.item + , ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.store + WHERE ("ss_item_sk" = "i_item_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("ss_store_sk" = "s_store_sk") + AND ("d_month_seq" IN (1200 , (1200 + 1) , (1200 + 2) , (1200 + 3) , (1200 + 4) , (1200 + 5) , (1200 + 6) , (1200 + 7) , (1200 + 8) , (1200 + 9) , (1200 + 10) , (1200 + 11))) + AND ((("i_category" IN ('Books' , 'Children' , 'Electronics')) + AND ("i_class" IN ('personal' , 'portable' , 'reference' , 'self-help')) + AND ("i_brand" IN ('scholaramalgamalg #14' , 'scholaramalgamalg #7' , 'exportiunivamalg #9' , 'scholaramalgamalg #9'))) + OR (("i_category" IN ('Women' , 'Music' , 'Men')) + AND ("i_class" IN ('accessories' , 'classical' , 'fragrances' , 'pants')) + AND ("i_brand" IN ('amalgimporto #1' , 'edu packscholar #1' , 'exportiimporto #1' , 'importoamalg #1')))) + GROUP BY "i_manufact_id", "d_qoy" +) tmp1 +WHERE ((CASE WHEN ("avg_quarterly_sales" > 0) THEN ("abs"((CAST("sum_sales" AS DECIMAL(38,4)) - "avg_quarterly_sales")) / "avg_quarterly_sales") ELSE null END) > DECIMAL '0.1') +ORDER BY "avg_quarterly_sales" ASC, "sum_sales" ASC, "i_manufact_id" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q54.sql b/presto-iceberg/src/test/resources/tpcds/queries/q54.sql new file mode 100644 index 0000000000000..2a637ed22709b --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q54.sql @@ -0,0 +1,75 @@ +WITH + my_customers AS ( + SELECT DISTINCT + "c_customer_sk" + , "c_current_addr_sk" + FROM + ( + SELECT + "cs_sold_date_sk" "sold_date_sk" + , "cs_bill_customer_sk" "customer_sk" + , "cs_item_sk" "item_sk" + FROM + ${database}.${schema}.catalog_sales +UNION ALL SELECT + "ws_sold_date_sk" "sold_date_sk" + , "ws_bill_customer_sk" "customer_sk" + , "ws_item_sk" "item_sk" + FROM + ${database}.${schema}.web_sales + ) cs_or_ws_sales + , ${database}.${schema}.item + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer + WHERE ("sold_date_sk" = "d_date_sk") + AND ("item_sk" = "i_item_sk") + AND ("i_category" = 'Women') + AND ("i_class" = 'maternity') + AND ("c_customer_sk" = "cs_or_ws_sales"."customer_sk") + AND ("d_moy" = 12) + AND ("d_year" = 1998) +) +, my_revenue AS ( + SELECT + "c_customer_sk" + , "sum"("ss_ext_sales_price") "revenue" + FROM + my_customers + , ${database}.${schema}.store_sales + , ${database}.${schema}.customer_address + , ${database}.${schema}.store + , ${database}.${schema}.date_dim + WHERE ("c_current_addr_sk" = "ca_address_sk") + AND ("ca_county" = "s_county") + AND ("ca_state" = "s_state") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("c_customer_sk" = "ss_customer_sk") + AND ("d_month_seq" BETWEEN ( + SELECT DISTINCT ("d_month_seq" + 1) + FROM + ${database}.${schema}.date_dim + WHERE ("d_year" = 1998) + AND ("d_moy" = 12) + ) AND ( + SELECT DISTINCT ("d_month_seq" + 3) + FROM + ${database}.${schema}.date_dim + WHERE ("d_year" = 1998) + AND ("d_moy" = 12) + )) + GROUP BY "c_customer_sk" +) +, segments AS ( + SELECT CAST(("revenue" / 50) AS INTEGER) "segment" + FROM + my_revenue +) +SELECT + "segment" +, "count"(*) "num_customers" +, ("segment" * 50) "segment_base" +FROM + segments +GROUP BY "segment" +ORDER BY "segment" ASC, "num_customers" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q55.sql b/presto-iceberg/src/test/resources/tpcds/queries/q55.sql new file mode 100644 index 0000000000000..835b46efb0995 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q55.sql @@ -0,0 +1,16 @@ +SELECT + "i_brand_id" "brand_id" +, "i_brand" "brand" +, "sum"("ss_ext_sales_price") "ext_price" +FROM + ${database}.${schema}.date_dim +, ${database}.${schema}.store_sales +, ${database}.${schema}.item +WHERE ("d_date_sk" = "ss_sold_date_sk") + AND ("ss_item_sk" = "i_item_sk") + AND ("i_manager_id" = 28) + AND ("d_moy" = 11) + AND ("d_year" = 1999) +GROUP BY "i_brand", "i_brand_id" +ORDER BY "ext_price" DESC, "i_brand_id" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q56.sql b/presto-iceberg/src/test/resources/tpcds/queries/q56.sql new file mode 100644 index 0000000000000..f1449567203d3 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q56.sql @@ -0,0 +1,88 @@ +WITH + ss AS ( + SELECT + "i_item_id" + , "sum"("ss_ext_sales_price") "total_sales" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer_address + , ${database}.${schema}.item + WHERE ("i_item_id" IN ( + SELECT "i_item_id" + FROM + ${database}.${schema}.item + WHERE ("i_color" IN ('slate' , 'blanched' , 'burnished')) + )) + AND ("ss_item_sk" = "i_item_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("d_year" = 2001) + AND ("d_moy" = 2) + AND ("ss_addr_sk" = "ca_address_sk") + AND ("ca_gmt_offset" = -5) + GROUP BY "i_item_id" +) +, cs AS ( + SELECT + "i_item_id" + , "sum"("cs_ext_sales_price") "total_sales" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer_address + , ${database}.${schema}.item + WHERE ("i_item_id" IN ( + SELECT "i_item_id" + FROM + ${database}.${schema}.item + WHERE ("i_color" IN ('slate' , 'blanched' , 'burnished')) + )) + AND ("cs_item_sk" = "i_item_sk") + AND ("cs_sold_date_sk" = "d_date_sk") + AND ("d_year" = 2001) + AND ("d_moy" = 2) + AND ("cs_bill_addr_sk" = "ca_address_sk") + AND ("ca_gmt_offset" = -5) + GROUP BY "i_item_id" +) +, ws AS ( + SELECT + "i_item_id" + , "sum"("ws_ext_sales_price") "total_sales" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer_address + , ${database}.${schema}.item + WHERE ("i_item_id" IN ( + SELECT "i_item_id" + FROM + ${database}.${schema}.item + WHERE ("i_color" IN ('slate' , 'blanched' , 'burnished')) + )) + AND ("ws_item_sk" = "i_item_sk") + AND ("ws_sold_date_sk" = "d_date_sk") + AND ("d_year" = 2001) + AND ("d_moy" = 2) + AND ("ws_bill_addr_sk" = "ca_address_sk") + AND ("ca_gmt_offset" = -5) + GROUP BY "i_item_id" +) +SELECT + "i_item_id" +, "sum"("total_sales") "total_sales" +FROM + ( + SELECT * + FROM + ss +UNION ALL SELECT * + FROM + cs +UNION ALL SELECT * + FROM + ws +) tmp1 +GROUP BY "i_item_id" +ORDER BY "total_sales" ASC, "i_item_id" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q57.sql b/presto-iceberg/src/test/resources/tpcds/queries/q57.sql new file mode 100644 index 0000000000000..0fb98c7d65d01 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q57.sql @@ -0,0 +1,58 @@ +WITH + v1 AS ( + SELECT + "i_category" + , "i_brand" + , "cc_name" + , "d_year" + , "d_moy" + , "sum"("cs_sales_price") "sum_sales" + , "avg"("sum"("cs_sales_price")) OVER (PARTITION BY "i_category", "i_brand", "cc_name", "d_year") "avg_monthly_sales" + , "rank"() OVER (PARTITION BY "i_category", "i_brand", "cc_name" ORDER BY "d_year" ASC, "d_moy" ASC) "rn" + FROM + ${database}.${schema}.item + , ${database}.${schema}.catalog_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.call_center + WHERE ("cs_item_sk" = "i_item_sk") + AND ("cs_sold_date_sk" = "d_date_sk") + AND ("cc_call_center_sk" = "cs_call_center_sk") + AND (("d_year" = 1999) + OR (("d_year" = (1999 - 1)) + AND ("d_moy" = 12)) + OR (("d_year" = (1999 + 1)) + AND ("d_moy" = 1))) + GROUP BY "i_category", "i_brand", "cc_name", "d_year", "d_moy" +) +, v2 AS ( + SELECT + "v1"."i_category" + , "v1"."i_brand" + , "v1"."cc_name" + , "v1"."d_year" + , "v1"."d_moy" + , "v1"."avg_monthly_sales" + , "v1"."sum_sales" + , "v1_lag"."sum_sales" "psum" + , "v1_lead"."sum_sales" "nsum" + FROM + v1 + , v1 v1_lag + , v1 v1_lead + WHERE ("v1"."i_category" = "v1_lag"."i_category") + AND ("v1"."i_category" = "v1_lead"."i_category") + AND ("v1"."i_brand" = "v1_lag"."i_brand") + AND ("v1"."i_brand" = "v1_lead"."i_brand") + AND ("v1"."cc_name" = "v1_lag"."cc_name") + AND ("v1"."cc_name" = "v1_lead"."cc_name") + AND ("v1"."rn" = ("v1_lag"."rn" + 1)) + AND ("v1"."rn" = ("v1_lead"."rn" - 1)) +) +SELECT * +FROM + v2 +WHERE ("d_year" = 1999) + AND ("avg_monthly_sales" > 0) + AND ((CASE WHEN ("avg_monthly_sales" > 0) THEN ("abs"(("sum_sales" - "avg_monthly_sales")) / "avg_monthly_sales") ELSE null END) > DECIMAL '0.1') +ORDER BY ("sum_sales" - "avg_monthly_sales") ASC, 3 ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q58.sql b/presto-iceberg/src/test/resources/tpcds/queries/q58.sql new file mode 100644 index 0000000000000..fa84fbe49f85d --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q58.sql @@ -0,0 +1,93 @@ +WITH + ss_items AS ( + SELECT + "i_item_id" "item_id" + , "sum"("ss_ext_sales_price") "ss_item_rev" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.item + , ${database}.${schema}.date_dim + WHERE ("ss_item_sk" = "i_item_sk") + AND ("d_date" IN ( + SELECT "d_date" + FROM + ${database}.${schema}.date_dim + WHERE ("d_week_seq" = ( + SELECT "d_week_seq" + FROM + ${database}.${schema}.date_dim + WHERE ("d_date" = CAST('2000-01-03' AS DATE)) + )) + )) + AND ("ss_sold_date_sk" = "d_date_sk") + GROUP BY "i_item_id" +) +, cs_items AS ( + SELECT + "i_item_id" "item_id" + , "sum"("cs_ext_sales_price") "cs_item_rev" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.item + , ${database}.${schema}.date_dim + WHERE ("cs_item_sk" = "i_item_sk") + AND ("d_date" IN ( + SELECT "d_date" + FROM + ${database}.${schema}.date_dim + WHERE ("d_week_seq" = ( + SELECT "d_week_seq" + FROM + ${database}.${schema}.date_dim + WHERE ("d_date" = CAST('2000-01-03' AS DATE)) + )) + )) + AND ("cs_sold_date_sk" = "d_date_sk") + GROUP BY "i_item_id" +) +, ws_items AS ( + SELECT + "i_item_id" "item_id" + , "sum"("ws_ext_sales_price") "ws_item_rev" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.item + , ${database}.${schema}.date_dim + WHERE ("ws_item_sk" = "i_item_sk") + AND ("d_date" IN ( + SELECT "d_date" + FROM + ${database}.${schema}.date_dim + WHERE ("d_week_seq" = ( + SELECT "d_week_seq" + FROM + ${database}.${schema}.date_dim + WHERE ("d_date" = CAST('2000-01-03' AS DATE)) + )) + )) + AND ("ws_sold_date_sk" = "d_date_sk") + GROUP BY "i_item_id" +) +SELECT + "ss_items"."item_id" +, "ss_item_rev" +, CAST(((("ss_item_rev" / ((CAST("ss_item_rev" AS DECIMAL(16,7)) + "cs_item_rev") + "ws_item_rev")) / 3) * 100) AS DECIMAL(7,2)) "ss_dev" +, "cs_item_rev" +, CAST(((("cs_item_rev" / ((CAST("ss_item_rev" AS DECIMAL(16,7)) + "cs_item_rev") + "ws_item_rev")) / 3) * 100) AS DECIMAL(7,2)) "cs_dev" +, "ws_item_rev" +, CAST(((("ws_item_rev" / ((CAST("ss_item_rev" AS DECIMAL(16,7)) + "cs_item_rev") + "ws_item_rev")) / 3) * 100) AS DECIMAL(7,2)) "ws_dev" +, ((("ss_item_rev" + "cs_item_rev") + "ws_item_rev") / 3) "average" +FROM + ss_items +, cs_items +, ws_items +WHERE ("ss_items"."item_id" = "cs_items"."item_id") + AND ("ss_items"."item_id" = "ws_items"."item_id") + AND ("ss_item_rev" BETWEEN (DECIMAL '0.9' * "cs_item_rev") AND (DECIMAL '1.1' * "cs_item_rev")) + AND ("ss_item_rev" BETWEEN (DECIMAL '0.9' * "ws_item_rev") AND (DECIMAL '1.1' * "ws_item_rev")) + AND ("cs_item_rev" BETWEEN (DECIMAL '0.9' * "ss_item_rev") AND (DECIMAL '1.1' * "ss_item_rev")) + AND ("cs_item_rev" BETWEEN (DECIMAL '0.9' * "ws_item_rev") AND (DECIMAL '1.1' * "ws_item_rev")) + AND ("ws_item_rev" BETWEEN (DECIMAL '0.9' * "ss_item_rev") AND (DECIMAL '1.1' * "ss_item_rev")) + AND ("ws_item_rev" BETWEEN (DECIMAL '0.9' * "cs_item_rev") AND (DECIMAL '1.1' * "cs_item_rev")) +ORDER BY "ss_items"."item_id" ASC, "ss_item_rev" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q59.sql b/presto-iceberg/src/test/resources/tpcds/queries/q59.sql new file mode 100644 index 0000000000000..6b5a8acae4807 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q59.sql @@ -0,0 +1,74 @@ +WITH + wss AS ( + SELECT + "d_week_seq" + , "ss_store_sk" + , "sum"((CASE WHEN ("d_day_name" = 'Sunday') THEN "ss_sales_price" ELSE null END)) "sun_sales" + , "sum"((CASE WHEN ("d_day_name" = 'Monday') THEN "ss_sales_price" ELSE null END)) "mon_sales" + , "sum"((CASE WHEN ("d_day_name" = 'Tuesday') THEN "ss_sales_price" ELSE null END)) "tue_sales" + , "sum"((CASE WHEN ("d_day_name" = 'Wednesday') THEN "ss_sales_price" ELSE null END)) "wed_sales" + , "sum"((CASE WHEN ("d_day_name" = 'Thursday') THEN "ss_sales_price" ELSE null END)) "thu_sales" + , "sum"((CASE WHEN ("d_day_name" = 'Friday') THEN "ss_sales_price" ELSE null END)) "fri_sales" + , "sum"((CASE WHEN ("d_day_name" = 'Saturday') THEN "ss_sales_price" ELSE null END)) "sat_sales" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + WHERE ("d_date_sk" = "ss_sold_date_sk") + GROUP BY "d_week_seq", "ss_store_sk" +) +SELECT + "s_store_name1" +, "s_store_id1" +, "d_week_seq1" +, ("sun_sales1" / "sun_sales2") +, ("mon_sales1" / "mon_sales2") +, ("tue_sales1" / "tue_sales2") +, ("wed_sales1" / "wed_sales2") +, ("thu_sales1" / "thu_sales2") +, ("fri_sales1" / "fri_sales2") +, ("sat_sales1" / "sat_sales2") +FROM + ( + SELECT + "s_store_name" "s_store_name1" + , "wss"."d_week_seq" "d_week_seq1" + , "s_store_id" "s_store_id1" + , "sun_sales" "sun_sales1" + , "mon_sales" "mon_sales1" + , "tue_sales" "tue_sales1" + , "wed_sales" "wed_sales1" + , "thu_sales" "thu_sales1" + , "fri_sales" "fri_sales1" + , "sat_sales" "sat_sales1" + FROM + wss + , ${database}.${schema}.store + , ${database}.${schema}.date_dim d + WHERE ("d"."d_week_seq" = "wss"."d_week_seq") + AND ("ss_store_sk" = "s_store_sk") + AND ("d_month_seq" BETWEEN 1212 AND (1212 + 11)) +) y +, ( + SELECT + "s_store_name" "s_store_name2" + , "wss"."d_week_seq" "d_week_seq2" + , "s_store_id" "s_store_id2" + , "sun_sales" "sun_sales2" + , "mon_sales" "mon_sales2" + , "tue_sales" "tue_sales2" + , "wed_sales" "wed_sales2" + , "thu_sales" "thu_sales2" + , "fri_sales" "fri_sales2" + , "sat_sales" "sat_sales2" + FROM + wss + , ${database}.${schema}.store + , ${database}.${schema}.date_dim d + WHERE ("d"."d_week_seq" = "wss"."d_week_seq") + AND ("ss_store_sk" = "s_store_sk") + AND ("d_month_seq" BETWEEN (1212 + 12) AND (1212 + 23)) +) x +WHERE ("s_store_id1" = "s_store_id2") + AND ("d_week_seq1" = ("d_week_seq2" - 52)) +ORDER BY "s_store_name1" ASC, "s_store_id1" ASC, "d_week_seq1" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q60.sql b/presto-iceberg/src/test/resources/tpcds/queries/q60.sql new file mode 100644 index 0000000000000..00cc332849b33 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q60.sql @@ -0,0 +1,88 @@ +WITH + ss AS ( + SELECT + "i_item_id" + , "sum"("ss_ext_sales_price") "total_sales" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer_address + , ${database}.${schema}.item + WHERE ("i_item_id" IN ( + SELECT "i_item_id" + FROM + ${database}.${schema}.item + WHERE ("i_category" IN ('Music')) + )) + AND ("ss_item_sk" = "i_item_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("d_year" = 1998) + AND ("d_moy" = 9) + AND ("ss_addr_sk" = "ca_address_sk") + AND ("ca_gmt_offset" = -5) + GROUP BY "i_item_id" +) +, cs AS ( + SELECT + "i_item_id" + , "sum"("cs_ext_sales_price") "total_sales" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer_address + , ${database}.${schema}.item + WHERE ("i_item_id" IN ( + SELECT "i_item_id" + FROM + ${database}.${schema}.item + WHERE ("i_category" IN ('Music')) + )) + AND ("cs_item_sk" = "i_item_sk") + AND ("cs_sold_date_sk" = "d_date_sk") + AND ("d_year" = 1998) + AND ("d_moy" = 9) + AND ("cs_bill_addr_sk" = "ca_address_sk") + AND ("ca_gmt_offset" = -5) + GROUP BY "i_item_id" +) +, ws AS ( + SELECT + "i_item_id" + , "sum"("ws_ext_sales_price") "total_sales" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer_address + , ${database}.${schema}.item + WHERE ("i_item_id" IN ( + SELECT "i_item_id" + FROM + ${database}.${schema}.item + WHERE ("i_category" IN ('Music')) + )) + AND ("ws_item_sk" = "i_item_sk") + AND ("ws_sold_date_sk" = "d_date_sk") + AND ("d_year" = 1998) + AND ("d_moy" = 9) + AND ("ws_bill_addr_sk" = "ca_address_sk") + AND ("ca_gmt_offset" = -5) + GROUP BY "i_item_id" +) +SELECT + "i_item_id" +, "sum"("total_sales") "total_sales" +FROM + ( + SELECT * + FROM + ss +UNION ALL SELECT * + FROM + cs +UNION ALL SELECT * + FROM + ws +) tmp1 +GROUP BY "i_item_id" +ORDER BY "i_item_id" ASC, "total_sales" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q61.sql b/presto-iceberg/src/test/resources/tpcds/queries/q61.sql new file mode 100644 index 0000000000000..4d3143cd376ce --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q61.sql @@ -0,0 +1,52 @@ +SELECT + "promotions" +, "total" +, ((CAST("promotions" AS DECIMAL(15,4)) / CAST("total" AS DECIMAL(15,4))) * 100) +FROM + ( + SELECT "sum"("ss_ext_sales_price") "promotions" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.store + , ${database}.${schema}.promotion + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer + , ${database}.${schema}.customer_address + , ${database}.${schema}.item + WHERE ("ss_sold_date_sk" = "d_date_sk") + AND ("ss_store_sk" = "s_store_sk") + AND ("ss_promo_sk" = "p_promo_sk") + AND ("ss_customer_sk" = "c_customer_sk") + AND ("ca_address_sk" = "c_current_addr_sk") + AND ("ss_item_sk" = "i_item_sk") + AND ("ca_gmt_offset" = -5) + AND ("i_category" = 'Jewelry') + AND (("p_channel_dmail" = 'Y') + OR ("p_channel_email" = 'Y') + OR ("p_channel_tv" = 'Y')) + AND ("s_gmt_offset" = -5) + AND ("d_year" = 1998) + AND ("d_moy" = 11) +) promotional_sales +, ( + SELECT "sum"("ss_ext_sales_price") "total" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.store + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer + , ${database}.${schema}.customer_address + , ${database}.${schema}.item + WHERE ("ss_sold_date_sk" = "d_date_sk") + AND ("ss_store_sk" = "s_store_sk") + AND ("ss_customer_sk" = "c_customer_sk") + AND ("ca_address_sk" = "c_current_addr_sk") + AND ("ss_item_sk" = "i_item_sk") + AND ("ca_gmt_offset" = -5) + AND ("i_category" = 'Jewelry') + AND ("s_gmt_offset" = -5) + AND ("d_year" = 1998) + AND ("d_moy" = 11) +) all_sales +ORDER BY "promotions" ASC, "total" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q62.sql b/presto-iceberg/src/test/resources/tpcds/queries/q62.sql new file mode 100644 index 0000000000000..c0ddd15d23afa --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q62.sql @@ -0,0 +1,26 @@ +SELECT + "substr"("w_warehouse_name", 1, 20) +, "sm_type" +, "web_name" +, "sum"((CASE WHEN (("ws_ship_date_sk" - "ws_sold_date_sk") <= 30) THEN 1 ELSE 0 END)) "30 days" +, "sum"((CASE WHEN (("ws_ship_date_sk" - "ws_sold_date_sk") > 30) + AND (("ws_ship_date_sk" - "ws_sold_date_sk") <= 60) THEN 1 ELSE 0 END)) "31-60 days" +, "sum"((CASE WHEN (("ws_ship_date_sk" - "ws_sold_date_sk") > 60) + AND (("ws_ship_date_sk" - "ws_sold_date_sk") <= 90) THEN 1 ELSE 0 END)) "61-90 days" +, "sum"((CASE WHEN (("ws_ship_date_sk" - "ws_sold_date_sk") > 90) + AND (("ws_ship_date_sk" - "ws_sold_date_sk") <= 120) THEN 1 ELSE 0 END)) "91-120 days" +, "sum"((CASE WHEN (("ws_ship_date_sk" - "ws_sold_date_sk") > 120) THEN 1 ELSE 0 END)) ">120 days" +FROM + ${database}.${schema}.web_sales +, ${database}.${schema}.warehouse +, ${database}.${schema}.ship_mode +, ${database}.${schema}.web_site +, ${database}.${schema}.date_dim +WHERE ("d_month_seq" BETWEEN 1200 AND (1200 + 11)) + AND ("ws_ship_date_sk" = "d_date_sk") + AND ("ws_warehouse_sk" = "w_warehouse_sk") + AND ("ws_ship_mode_sk" = "sm_ship_mode_sk") + AND ("ws_web_site_sk" = "web_site_sk") +GROUP BY "substr"("w_warehouse_name", 1, 20), "sm_type", "web_name" +ORDER BY "substr"("w_warehouse_name", 1, 20) ASC, "sm_type" ASC, "web_name" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q63.sql b/presto-iceberg/src/test/resources/tpcds/queries/q63.sql new file mode 100644 index 0000000000000..b2181569e72e1 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q63.sql @@ -0,0 +1,27 @@ +SELECT * +FROM + ( + SELECT + "i_manager_id" + , "sum"("ss_sales_price") "sum_sales" + , "avg"("sum"("ss_sales_price")) OVER (PARTITION BY "i_manager_id") "avg_monthly_sales" + FROM + ${database}.${schema}.item + , ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.store + WHERE ("ss_item_sk" = "i_item_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("ss_store_sk" = "s_store_sk") + AND ("d_month_seq" IN (1200 , (1200 + 1) , (1200 + 2) , (1200 + 3) , (1200 + 4) , (1200 + 5) , (1200 + 6) , (1200 + 7) , (1200 + 8) , (1200 + 9) , (1200 + 10) , (1200 + 11))) + AND ((("i_category" IN ('Books' , 'Children' , 'Electronics')) + AND ("i_class" IN ('personal' , 'portable' , 'reference' , 'self-help')) + AND ("i_brand" IN ('scholaramalgamalg #14' , 'scholaramalgamalg #7' , 'exportiunivamalg #9' , 'scholaramalgamalg #9'))) + OR (("i_category" IN ('Women' , 'Music' , 'Men')) + AND ("i_class" IN ('accessories' , 'classical' , 'fragrances' , 'pants')) + AND ("i_brand" IN ('amalgimporto #1' , 'edu packscholar #1' , 'exportiimporto #1' , 'importoamalg #1')))) + GROUP BY "i_manager_id", "d_moy" +) tmp1 +WHERE ((CASE WHEN ("avg_monthly_sales" > 0) THEN ("abs"(("sum_sales" - "avg_monthly_sales")) / "avg_monthly_sales") ELSE null END) > DECIMAL '0.1') +ORDER BY "i_manager_id" ASC, "avg_monthly_sales" ASC, "sum_sales" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q64.sql b/presto-iceberg/src/test/resources/tpcds/queries/q64.sql new file mode 100644 index 0000000000000..636532dfc4f86 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q64.sql @@ -0,0 +1,110 @@ +WITH + cs_ui AS ( + SELECT + "cs_item_sk" + , "sum"("cs_ext_list_price") "sale" + , "sum"((("cr_refunded_cash" + "cr_reversed_charge") + "cr_store_credit")) "refund" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.catalog_returns + WHERE ("cs_item_sk" = "cr_item_sk") + AND ("cs_order_number" = "cr_order_number") + GROUP BY "cs_item_sk" + HAVING ("sum"("cs_ext_list_price") > (2 * "sum"((("cr_refunded_cash" + "cr_reversed_charge") + "cr_store_credit")))) +) +, cross_sales AS ( + SELECT + "i_product_name" "product_name" + , "i_item_sk" "item_sk" + , "s_store_name" "store_name" + , "s_zip" "store_zip" + , "ad1"."ca_street_number" "b_street_number" + , "ad1"."ca_street_name" "b_street_name" + , "ad1"."ca_city" "b_city" + , "ad1"."ca_zip" "b_zip" + , "ad2"."ca_street_number" "c_street_number" + , "ad2"."ca_street_name" "c_street_name" + , "ad2"."ca_city" "c_city" + , "ad2"."ca_zip" "c_zip" + , "d1"."d_year" "syear" + , "d2"."d_year" "fsyear" + , "d3"."d_year" "s2year" + , "count"(*) "cnt" + , "sum"("ss_wholesale_cost") "s1" + , "sum"("ss_list_price") "s2" + , "sum"("ss_coupon_amt") "s3" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.store_returns + , cs_ui + , ${database}.${schema}.date_dim d1 + , ${database}.${schema}.date_dim d2 + , ${database}.${schema}.date_dim d3 + , ${database}.${schema}.store + , ${database}.${schema}.customer + , ${database}.${schema}.customer_demographics cd1 + , ${database}.${schema}.customer_demographics cd2 + , ${database}.${schema}.promotion + , ${database}.${schema}.household_demographics hd1 + , ${database}.${schema}.household_demographics hd2 + , ${database}.${schema}.customer_address ad1 + , ${database}.${schema}.customer_address ad2 + , ${database}.${schema}.income_band ib1 + , ${database}.${schema}.income_band ib2 + , ${database}.${schema}.item + WHERE ("ss_store_sk" = "s_store_sk") + AND ("ss_sold_date_sk" = "d1"."d_date_sk") + AND ("ss_customer_sk" = "c_customer_sk") + AND ("ss_cdemo_sk" = "cd1"."cd_demo_sk") + AND ("ss_hdemo_sk" = "hd1"."hd_demo_sk") + AND ("ss_addr_sk" = "ad1"."ca_address_sk") + AND ("ss_item_sk" = "i_item_sk") + AND ("ss_item_sk" = "sr_item_sk") + AND ("ss_ticket_number" = "sr_ticket_number") + AND ("ss_item_sk" = "cs_ui"."cs_item_sk") + AND ("c_current_cdemo_sk" = "cd2"."cd_demo_sk") + AND ("c_current_hdemo_sk" = "hd2"."hd_demo_sk") + AND ("c_current_addr_sk" = "ad2"."ca_address_sk") + AND ("c_first_sales_date_sk" = "d2"."d_date_sk") + AND ("c_first_shipto_date_sk" = "d3"."d_date_sk") + AND ("ss_promo_sk" = "p_promo_sk") + AND ("hd1"."hd_income_band_sk" = "ib1"."ib_income_band_sk") + AND ("hd2"."hd_income_band_sk" = "ib2"."ib_income_band_sk") + AND ("cd1"."cd_marital_status" <> "cd2"."cd_marital_status") + AND ("i_color" IN ('purple' , 'burlywood' , 'indian' , 'spring' , 'floral' , 'medium')) + AND ("i_current_price" BETWEEN 64 AND (64 + 10)) + AND ("i_current_price" BETWEEN (64 + 1) AND (64 + 15)) + GROUP BY "i_product_name", "i_item_sk", "s_store_name", "s_zip", "ad1"."ca_street_number", "ad1"."ca_street_name", "ad1"."ca_city", "ad1"."ca_zip", "ad2"."ca_street_number", "ad2"."ca_street_name", "ad2"."ca_city", "ad2"."ca_zip", "d1"."d_year", "d2"."d_year", "d3"."d_year" +) +SELECT + "cs1"."product_name" +, "cs1"."store_name" +, "cs1"."store_zip" +, "cs1"."b_street_number" +, "cs1"."b_street_name" +, "cs1"."b_city" +, "cs1"."b_zip" +, "cs1"."c_street_number" +, "cs1"."c_street_name" +, "cs1"."c_city" +, "cs1"."c_zip" +, "cs1"."syear" +, "cs1"."cnt" +, "cs1"."s1" "s11" +, "cs1"."s2" "s21" +, "cs1"."s3" "s31" +, "cs2"."s1" "s12" +, "cs2"."s2" "s22" +, "cs2"."s3" "s32" +, "cs2"."syear" +, "cs2"."cnt" +FROM + cross_sales cs1 +, cross_sales cs2 +WHERE ("cs1"."item_sk" = "cs2"."item_sk") + AND ("cs1"."syear" = 1999) + AND ("cs2"."syear" = (1999 + 1)) + AND ("cs2"."cnt" <= "cs1"."cnt") + AND ("cs1"."store_name" = "cs2"."store_name") + AND ("cs1"."store_zip" = "cs2"."store_zip") +ORDER BY "cs1"."product_name" ASC, "cs1"."store_name" ASC, "cs2"."cnt" ASC, 14, 15, 16, 17, 18 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q65.sql b/presto-iceberg/src/test/resources/tpcds/queries/q65.sql new file mode 100644 index 0000000000000..c8b2b884b81f8 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q65.sql @@ -0,0 +1,47 @@ +SELECT + "s_store_name" +, "i_item_desc" +, "sc"."revenue" +, "i_current_price" +, "i_wholesale_cost" +, "i_brand" +FROM + ${database}.${schema}.store +, ${database}.${schema}.item +, ( + SELECT + "ss_store_sk" + , "avg"("revenue") "ave" + FROM + ( + SELECT + "ss_store_sk" + , "ss_item_sk" + , "sum"("ss_sales_price") "revenue" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + WHERE ("ss_sold_date_sk" = "d_date_sk") + AND ("d_month_seq" BETWEEN 1176 AND (1176 + 11)) + GROUP BY "ss_store_sk", "ss_item_sk" + ) sa + GROUP BY "ss_store_sk" +) sb +, ( + SELECT + "ss_store_sk" + , "ss_item_sk" + , "sum"("ss_sales_price") "revenue" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + WHERE ("ss_sold_date_sk" = "d_date_sk") + AND ("d_month_seq" BETWEEN 1176 AND (1176 + 11)) + GROUP BY "ss_store_sk", "ss_item_sk" +) sc +WHERE ("sb"."ss_store_sk" = "sc"."ss_store_sk") + AND ("sc"."revenue" <= (DECIMAL '0.1' * "sb"."ave")) + AND ("s_store_sk" = "sc"."ss_store_sk") + AND ("i_item_sk" = "sc"."ss_item_sk") +ORDER BY "s_store_name" ASC, "i_item_desc" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q66.sql b/presto-iceberg/src/test/resources/tpcds/queries/q66.sql new file mode 100644 index 0000000000000..8d16ae0998ae2 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q66.sql @@ -0,0 +1,146 @@ +SELECT + "w_warehouse_name" +, "w_warehouse_sq_ft" +, "w_city" +, "w_county" +, "w_state" +, "w_country" +, "ship_carriers" +, "year" +, "sum"("jan_sales") "jan_sales" +, "sum"("feb_sales") "feb_sales" +, "sum"("mar_sales") "mar_sales" +, "sum"("apr_sales") "apr_sales" +, "sum"("may_sales") "may_sales" +, "sum"("jun_sales") "jun_sales" +, "sum"("jul_sales") "jul_sales" +, "sum"("aug_sales") "aug_sales" +, "sum"("sep_sales") "sep_sales" +, "sum"("oct_sales") "oct_sales" +, "sum"("nov_sales") "nov_sales" +, "sum"("dec_sales") "dec_sales" +, "sum"(("jan_sales" / "w_warehouse_sq_ft")) "jan_sales_per_sq_foot" +, "sum"(("feb_sales" / "w_warehouse_sq_ft")) "feb_sales_per_sq_foot" +, "sum"(("mar_sales" / "w_warehouse_sq_ft")) "mar_sales_per_sq_foot" +, "sum"(("apr_sales" / "w_warehouse_sq_ft")) "apr_sales_per_sq_foot" +, "sum"(("may_sales" / "w_warehouse_sq_ft")) "may_sales_per_sq_foot" +, "sum"(("jun_sales" / "w_warehouse_sq_ft")) "jun_sales_per_sq_foot" +, "sum"(("jul_sales" / "w_warehouse_sq_ft")) "jul_sales_per_sq_foot" +, "sum"(("aug_sales" / "w_warehouse_sq_ft")) "aug_sales_per_sq_foot" +, "sum"(("sep_sales" / "w_warehouse_sq_ft")) "sep_sales_per_sq_foot" +, "sum"(("oct_sales" / "w_warehouse_sq_ft")) "oct_sales_per_sq_foot" +, "sum"(("nov_sales" / "w_warehouse_sq_ft")) "nov_sales_per_sq_foot" +, "sum"(("dec_sales" / "w_warehouse_sq_ft")) "dec_sales_per_sq_foot" +, "sum"("jan_net") "jan_net" +, "sum"("feb_net") "feb_net" +, "sum"("mar_net") "mar_net" +, "sum"("apr_net") "apr_net" +, "sum"("may_net") "may_net" +, "sum"("jun_net") "jun_net" +, "sum"("jul_net") "jul_net" +, "sum"("aug_net") "aug_net" +, "sum"("sep_net") "sep_net" +, "sum"("oct_net") "oct_net" +, "sum"("nov_net") "nov_net" +, "sum"("dec_net") "dec_net" +FROM +( + SELECT + "w_warehouse_name" + , "w_warehouse_sq_ft" + , "w_city" + , "w_county" + , "w_state" + , "w_country" + , "concat"("concat"('DHL', ','), 'BARIAN') "ship_carriers" + , "d_year" "YEAR" + , "sum"((CASE WHEN ("d_moy" = 1) THEN ("ws_ext_sales_price" * "ws_quantity") ELSE 0 END)) "jan_sales" + , "sum"((CASE WHEN ("d_moy" = 2) THEN ("ws_ext_sales_price" * "ws_quantity") ELSE 0 END)) "feb_sales" + , "sum"((CASE WHEN ("d_moy" = 3) THEN ("ws_ext_sales_price" * "ws_quantity") ELSE 0 END)) "mar_sales" + , "sum"((CASE WHEN ("d_moy" = 4) THEN ("ws_ext_sales_price" * "ws_quantity") ELSE 0 END)) "apr_sales" + , "sum"((CASE WHEN ("d_moy" = 5) THEN ("ws_ext_sales_price" * "ws_quantity") ELSE 0 END)) "may_sales" + , "sum"((CASE WHEN ("d_moy" = 6) THEN ("ws_ext_sales_price" * "ws_quantity") ELSE 0 END)) "jun_sales" + , "sum"((CASE WHEN ("d_moy" = 7) THEN ("ws_ext_sales_price" * "ws_quantity") ELSE 0 END)) "jul_sales" + , "sum"((CASE WHEN ("d_moy" = 8) THEN ("ws_ext_sales_price" * "ws_quantity") ELSE 0 END)) "aug_sales" + , "sum"((CASE WHEN ("d_moy" = 9) THEN ("ws_ext_sales_price" * "ws_quantity") ELSE 0 END)) "sep_sales" + , "sum"((CASE WHEN ("d_moy" = 10) THEN ("ws_ext_sales_price" * "ws_quantity") ELSE 0 END)) "oct_sales" + , "sum"((CASE WHEN ("d_moy" = 11) THEN ("ws_ext_sales_price" * "ws_quantity") ELSE 0 END)) "nov_sales" + , "sum"((CASE WHEN ("d_moy" = 12) THEN ("ws_ext_sales_price" * "ws_quantity") ELSE 0 END)) "dec_sales" + , "sum"((CASE WHEN ("d_moy" = 1) THEN ("ws_net_paid" * "ws_quantity") ELSE 0 END)) "jan_net" + , "sum"((CASE WHEN ("d_moy" = 2) THEN ("ws_net_paid" * "ws_quantity") ELSE 0 END)) "feb_net" + , "sum"((CASE WHEN ("d_moy" = 3) THEN ("ws_net_paid" * "ws_quantity") ELSE 0 END)) "mar_net" + , "sum"((CASE WHEN ("d_moy" = 4) THEN ("ws_net_paid" * "ws_quantity") ELSE 0 END)) "apr_net" + , "sum"((CASE WHEN ("d_moy" = 5) THEN ("ws_net_paid" * "ws_quantity") ELSE 0 END)) "may_net" + , "sum"((CASE WHEN ("d_moy" = 6) THEN ("ws_net_paid" * "ws_quantity") ELSE 0 END)) "jun_net" + , "sum"((CASE WHEN ("d_moy" = 7) THEN ("ws_net_paid" * "ws_quantity") ELSE 0 END)) "jul_net" + , "sum"((CASE WHEN ("d_moy" = 8) THEN ("ws_net_paid" * "ws_quantity") ELSE 0 END)) "aug_net" + , "sum"((CASE WHEN ("d_moy" = 9) THEN ("ws_net_paid" * "ws_quantity") ELSE 0 END)) "sep_net" + , "sum"((CASE WHEN ("d_moy" = 10) THEN ("ws_net_paid" * "ws_quantity") ELSE 0 END)) "oct_net" + , "sum"((CASE WHEN ("d_moy" = 11) THEN ("ws_net_paid" * "ws_quantity") ELSE 0 END)) "nov_net" + , "sum"((CASE WHEN ("d_moy" = 12) THEN ("ws_net_paid" * "ws_quantity") ELSE 0 END)) "dec_net" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.warehouse + , ${database}.${schema}.date_dim + , ${database}.${schema}.time_dim + , ${database}.${schema}.ship_mode + WHERE ("ws_warehouse_sk" = "w_warehouse_sk") + AND ("ws_sold_date_sk" = "d_date_sk") + AND ("ws_sold_time_sk" = "t_time_sk") + AND ("ws_ship_mode_sk" = "sm_ship_mode_sk") + AND ("d_year" = 2001) + AND ("t_time" BETWEEN 30838 AND (30838 + 28800)) + AND ("sm_carrier" IN ('DHL' , 'BARIAN')) + GROUP BY "w_warehouse_name", "w_warehouse_sq_ft", "w_city", "w_county", "w_state", "w_country", "d_year" + UNION ALL + SELECT + "w_warehouse_name" + , "w_warehouse_sq_ft" + , "w_city" + , "w_county" + , "w_state" + , "w_country" + , "concat"("concat"('DHL', ','), 'BARIAN') "ship_carriers" + , "d_year" "YEAR" + , "sum"((CASE WHEN ("d_moy" = 1) THEN ("cs_sales_price" * "cs_quantity") ELSE 0 END)) "jan_sales" + , "sum"((CASE WHEN ("d_moy" = 2) THEN ("cs_sales_price" * "cs_quantity") ELSE 0 END)) "feb_sales" + , "sum"((CASE WHEN ("d_moy" = 3) THEN ("cs_sales_price" * "cs_quantity") ELSE 0 END)) "mar_sales" + , "sum"((CASE WHEN ("d_moy" = 4) THEN ("cs_sales_price" * "cs_quantity") ELSE 0 END)) "apr_sales" + , "sum"((CASE WHEN ("d_moy" = 5) THEN ("cs_sales_price" * "cs_quantity") ELSE 0 END)) "may_sales" + , "sum"((CASE WHEN ("d_moy" = 6) THEN ("cs_sales_price" * "cs_quantity") ELSE 0 END)) "jun_sales" + , "sum"((CASE WHEN ("d_moy" = 7) THEN ("cs_sales_price" * "cs_quantity") ELSE 0 END)) "jul_sales" + , "sum"((CASE WHEN ("d_moy" = 8) THEN ("cs_sales_price" * "cs_quantity") ELSE 0 END)) "aug_sales" + , "sum"((CASE WHEN ("d_moy" = 9) THEN ("cs_sales_price" * "cs_quantity") ELSE 0 END)) "sep_sales" + , "sum"((CASE WHEN ("d_moy" = 10) THEN ("cs_sales_price" * "cs_quantity") ELSE 0 END)) "oct_sales" + , "sum"((CASE WHEN ("d_moy" = 11) THEN ("cs_sales_price" * "cs_quantity") ELSE 0 END)) "nov_sales" + , "sum"((CASE WHEN ("d_moy" = 12) THEN ("cs_sales_price" * "cs_quantity") ELSE 0 END)) "dec_sales" + , "sum"((CASE WHEN ("d_moy" = 1) THEN ("cs_net_paid_inc_tax" * "cs_quantity") ELSE 0 END)) "jan_net" + , "sum"((CASE WHEN ("d_moy" = 2) THEN ("cs_net_paid_inc_tax" * "cs_quantity") ELSE 0 END)) "feb_net" + , "sum"((CASE WHEN ("d_moy" = 3) THEN ("cs_net_paid_inc_tax" * "cs_quantity") ELSE 0 END)) "mar_net" + , "sum"((CASE WHEN ("d_moy" = 4) THEN ("cs_net_paid_inc_tax" * "cs_quantity") ELSE 0 END)) "apr_net" + , "sum"((CASE WHEN ("d_moy" = 5) THEN ("cs_net_paid_inc_tax" * "cs_quantity") ELSE 0 END)) "may_net" + , "sum"((CASE WHEN ("d_moy" = 6) THEN ("cs_net_paid_inc_tax" * "cs_quantity") ELSE 0 END)) "jun_net" + , "sum"((CASE WHEN ("d_moy" = 7) THEN ("cs_net_paid_inc_tax" * "cs_quantity") ELSE 0 END)) "jul_net" + , "sum"((CASE WHEN ("d_moy" = 8) THEN ("cs_net_paid_inc_tax" * "cs_quantity") ELSE 0 END)) "aug_net" + , "sum"((CASE WHEN ("d_moy" = 9) THEN ("cs_net_paid_inc_tax" * "cs_quantity") ELSE 0 END)) "sep_net" + , "sum"((CASE WHEN ("d_moy" = 10) THEN ("cs_net_paid_inc_tax" * "cs_quantity") ELSE 0 END)) "oct_net" + , "sum"((CASE WHEN ("d_moy" = 11) THEN ("cs_net_paid_inc_tax" * "cs_quantity") ELSE 0 END)) "nov_net" + , "sum"((CASE WHEN ("d_moy" = 12) THEN ("cs_net_paid_inc_tax" * "cs_quantity") ELSE 0 END)) "dec_net" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.warehouse + , ${database}.${schema}.date_dim + , ${database}.${schema}.time_dim + , ${database}.${schema}.ship_mode + WHERE ("cs_warehouse_sk" = "w_warehouse_sk") + AND ("cs_sold_date_sk" = "d_date_sk") + AND ("cs_sold_time_sk" = "t_time_sk") + AND ("cs_ship_mode_sk" = "sm_ship_mode_sk") + AND ("d_year" = 2001) + AND ("t_time" BETWEEN 30838 AND (30838 + 28800)) + AND ("sm_carrier" IN ('DHL' , 'BARIAN')) + GROUP BY "w_warehouse_name", "w_warehouse_sq_ft", "w_city", "w_county", "w_state", "w_country", "d_year" + ) x +GROUP BY "w_warehouse_name", "w_warehouse_sq_ft", "w_city", "w_county", "w_state", "w_country", "ship_carriers", "year" +ORDER BY "w_warehouse_name" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q67.sql b/presto-iceberg/src/test/resources/tpcds/queries/q67.sql new file mode 100644 index 0000000000000..1a5af11d31cc9 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q67.sql @@ -0,0 +1,41 @@ +SELECT * +FROM + ( + SELECT + "i_category" + , "i_class" + , "i_brand" + , "i_product_name" + , "d_year" + , "d_qoy" + , "d_moy" + , "s_store_id" + , "sumsales" + , "rank"() OVER (PARTITION BY "i_category" ORDER BY "sumsales" DESC) "rk" + FROM + ( + SELECT + "i_category" + , "i_class" + , "i_brand" + , "i_product_name" + , "d_year" + , "d_qoy" + , "d_moy" + , "s_store_id" + , "sum"(COALESCE(("ss_sales_price" * "ss_quantity"), 0)) "sumsales" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.store + , ${database}.${schema}.item + WHERE ("ss_sold_date_sk" = "d_date_sk") + AND ("ss_item_sk" = "i_item_sk") + AND ("ss_store_sk" = "s_store_sk") + AND ("d_month_seq" BETWEEN 1200 AND (1200 + 11)) + GROUP BY ROLLUP (i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy, s_store_id) + ) dw1 +) dw2 +WHERE ("rk" <= 100) +ORDER BY "i_category" ASC, "i_class" ASC, "i_brand" ASC, "i_product_name" ASC, "d_year" ASC, "d_qoy" ASC, "d_moy" ASC, "s_store_id" ASC, "sumsales" ASC, "rk" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q68.sql b/presto-iceberg/src/test/resources/tpcds/queries/q68.sql new file mode 100644 index 0000000000000..41c9856628dc9 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q68.sql @@ -0,0 +1,42 @@ +SELECT + "c_last_name" +, "c_first_name" +, "ca_city" +, "bought_city" +, "ss_ticket_number" +, "extended_price" +, "extended_tax" +, "list_price" +FROM + ( + SELECT + "ss_ticket_number" + , "ss_customer_sk" + , "ca_city" "bought_city" + , "sum"("ss_ext_sales_price") "extended_price" + , "sum"("ss_ext_list_price") "list_price" + , "sum"("ss_ext_tax") "extended_tax" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.store + , ${database}.${schema}.household_demographics + , ${database}.${schema}.customer_address + WHERE ("store_sales"."ss_sold_date_sk" = "date_dim"."d_date_sk") + AND ("store_sales"."ss_store_sk" = "store"."s_store_sk") + AND ("store_sales"."ss_hdemo_sk" = "household_demographics"."hd_demo_sk") + AND ("store_sales"."ss_addr_sk" = "customer_address"."ca_address_sk") + AND ("date_dim"."d_dom" BETWEEN 1 AND 2) + AND (("household_demographics"."hd_dep_count" = 4) + OR ("household_demographics"."hd_vehicle_count" = 3)) + AND ("date_dim"."d_year" IN (1999 , (1999 + 1) , (1999 + 2))) + AND ("store"."s_city" IN ('Midway' , 'Fairview')) + GROUP BY "ss_ticket_number", "ss_customer_sk", "ss_addr_sk", "ca_city" +) dn +, ${database}.${schema}.customer +, ${database}.${schema}.customer_address current_addr +WHERE ("ss_customer_sk" = "c_customer_sk") + AND ("customer"."c_current_addr_sk" = "current_addr"."ca_address_sk") + AND ("current_addr"."ca_city" <> "bought_city") +ORDER BY "c_last_name" ASC, "ss_ticket_number" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q69.sql b/presto-iceberg/src/test/resources/tpcds/queries/q69.sql new file mode 100644 index 0000000000000..efc12424efd17 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q69.sql @@ -0,0 +1,49 @@ +SELECT + "cd_gender" +, "cd_marital_status" +, "cd_education_status" +, "count"(*) "cnt1" +, "cd_purchase_estimate" +, "count"(*) "cnt2" +, "cd_credit_rating" +, "count"(*) "cnt3" +FROM + ${database}.${schema}.customer c +, ${database}.${schema}.customer_address ca +, ${database}.${schema}.customer_demographics +WHERE ("c"."c_current_addr_sk" = "ca"."ca_address_sk") + AND ("ca_state" IN ('KY', 'GA', 'NM')) + AND ("cd_demo_sk" = "c"."c_current_cdemo_sk") + AND (EXISTS ( + SELECT * + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + WHERE ("c"."c_customer_sk" = "ss_customer_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("d_year" = 2001) + AND ("d_moy" BETWEEN 4 AND (4 + 2)) +)) + AND (NOT (EXISTS ( + SELECT * + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + WHERE ("c"."c_customer_sk" = "ws_bill_customer_sk") + AND ("ws_sold_date_sk" = "d_date_sk") + AND ("d_year" = 2001) + AND ("d_moy" BETWEEN 4 AND (4 + 2)) +))) + AND (NOT (EXISTS ( + SELECT * + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.date_dim + WHERE ("c"."c_customer_sk" = "cs_ship_customer_sk") + AND ("cs_sold_date_sk" = "d_date_sk") + AND ("d_year" = 2001) + AND ("d_moy" BETWEEN 4 AND (4 + 2)) +))) +GROUP BY "cd_gender", "cd_marital_status", "cd_education_status", "cd_purchase_estimate", "cd_credit_rating" +ORDER BY "cd_gender" ASC, "cd_marital_status" ASC, "cd_education_status" ASC, "cd_purchase_estimate" ASC, "cd_credit_rating" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q70.sql b/presto-iceberg/src/test/resources/tpcds/queries/q70.sql new file mode 100644 index 0000000000000..44d065226fb96 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q70.sql @@ -0,0 +1,34 @@ +SELECT + "sum"("ss_net_profit") "total_sum" +, "s_state" +, "s_county" +, (GROUPING ("s_state") + GROUPING ("s_county")) "lochierarchy" +, "rank"() OVER (PARTITION BY (GROUPING ("s_state") + GROUPING ("s_county")), (CASE WHEN (GROUPING ("s_county") = 0) THEN "s_state" END) ORDER BY "sum"("ss_net_profit") DESC) "rank_within_parent" +FROM + ${database}.${schema}.store_sales +, ${database}.${schema}.date_dim d1 +, ${database}.${schema}.store +WHERE ("d1"."d_month_seq" BETWEEN 1200 AND (1200 + 11)) + AND ("d1"."d_date_sk" = "ss_sold_date_sk") + AND ("s_store_sk" = "ss_store_sk") + AND ("s_state" IN ( + SELECT "s_state" + FROM + ( + SELECT + "s_state" "s_state" + , "rank"() OVER (PARTITION BY "s_state" ORDER BY "sum"("ss_net_profit") DESC) "ranking" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.store + , ${database}.${schema}.date_dim + WHERE ("d_month_seq" BETWEEN 1200 AND (1200 + 11)) + AND ("d_date_sk" = "ss_sold_date_sk") + AND ("s_store_sk" = "ss_store_sk") + GROUP BY "s_state" + ) tmp1 + WHERE ("ranking" <= 5) +)) +GROUP BY ROLLUP (s_state, s_county) +ORDER BY "lochierarchy" DESC, (CASE WHEN ("lochierarchy" = 0) THEN "s_state" END) ASC, "rank_within_parent" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q71.sql b/presto-iceberg/src/test/resources/tpcds/queries/q71.sql new file mode 100644 index 0000000000000..e34231490818a --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q71.sql @@ -0,0 +1,51 @@ +SELECT + "i_brand_id" "brand_id" +, "i_brand" "brand" +, "t_hour" +, "t_minute" +, "sum"("ext_price") "ext_price" +FROM + ${database}.${schema}.item +, ( + SELECT + "ws_ext_sales_price" "ext_price" + , "ws_sold_date_sk" "sold_date_sk" + , "ws_item_sk" "sold_item_sk" + , "ws_sold_time_sk" "time_sk" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + WHERE ("d_date_sk" = "ws_sold_date_sk") + AND ("d_moy" = 11) + AND ("d_year" = 1999) +UNION ALL SELECT + "cs_ext_sales_price" "ext_price" + , "cs_sold_date_sk" "sold_date_sk" + , "cs_item_sk" "sold_item_sk" + , "cs_sold_time_sk" "time_sk" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.date_dim + WHERE ("d_date_sk" = "cs_sold_date_sk") + AND ("d_moy" = 11) + AND ("d_year" = 1999) +UNION ALL SELECT + "ss_ext_sales_price" "ext_price" + , "ss_sold_date_sk" "sold_date_sk" + , "ss_item_sk" "sold_item_sk" + , "ss_sold_time_sk" "time_sk" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + WHERE ("d_date_sk" = "ss_sold_date_sk") + AND ("d_moy" = 11) + AND ("d_year" = 1999) +) tmp +, ${database}.${schema}.time_dim +WHERE ("sold_item_sk" = "i_item_sk") + AND ("i_manager_id" = 1) + AND ("time_sk" = "t_time_sk") + AND (("t_meal_time" = 'breakfast') + OR ("t_meal_time" = 'dinner')) +GROUP BY "i_brand", "i_brand_id", "t_hour", "t_minute" +ORDER BY "ext_price" DESC, "i_brand_id" ASC diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q72.sql b/presto-iceberg/src/test/resources/tpcds/queries/q72.sql new file mode 100644 index 0000000000000..df26507211903 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q72.sql @@ -0,0 +1,29 @@ +SELECT + "i_item_desc" +, "w_warehouse_name" +, "d1"."d_week_seq" +, "sum"((CASE WHEN ("p_promo_sk" IS NULL) THEN 1 ELSE 0 END)) "no_promo" +, "sum"((CASE WHEN ("p_promo_sk" IS NOT NULL) THEN 1 ELSE 0 END)) "promo" +, "count"(*) "total_cnt" +FROM + ((((((((((${database}.${schema}.catalog_sales +INNER JOIN ${database}.${schema}.inventory ON ("cs_item_sk" = "inv_item_sk")) +INNER JOIN ${database}.${schema}.warehouse ON ("w_warehouse_sk" = "inv_warehouse_sk")) +INNER JOIN ${database}.${schema}.item ON ("i_item_sk" = "cs_item_sk")) +INNER JOIN ${database}.${schema}.customer_demographics ON ("cs_bill_cdemo_sk" = "cd_demo_sk")) +INNER JOIN ${database}.${schema}.household_demographics ON ("cs_bill_hdemo_sk" = "hd_demo_sk")) +INNER JOIN ${database}.${schema}.date_dim d1 ON ("cs_sold_date_sk" = "d1"."d_date_sk")) +INNER JOIN ${database}.${schema}.date_dim d2 ON ("inv_date_sk" = "d2"."d_date_sk")) +INNER JOIN ${database}.${schema}.date_dim d3 ON ("cs_ship_date_sk" = "d3"."d_date_sk")) +LEFT JOIN ${database}.${schema}.promotion ON ("cs_promo_sk" = "p_promo_sk")) +LEFT JOIN ${database}.${schema}.catalog_returns ON ("cr_item_sk" = "cs_item_sk") + AND ("cr_order_number" = "cs_order_number")) +WHERE ("d1"."d_week_seq" = "d2"."d_week_seq") + AND ("inv_quantity_on_hand" < "cs_quantity") + AND ("d3"."d_date" > ("d1"."d_date" + INTERVAL '5' DAY)) + AND ("hd_buy_potential" = '>10000') + AND ("d1"."d_year" = 1999) + AND ("cd_marital_status" = 'D') +GROUP BY "i_item_desc", "w_warehouse_name", "d1"."d_week_seq" +ORDER BY "total_cnt" DESC, "i_item_desc" ASC, "w_warehouse_name" ASC, "d1"."d_week_seq" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q73.sql b/presto-iceberg/src/test/resources/tpcds/queries/q73.sql new file mode 100644 index 0000000000000..98dcc09ed5bde --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q73.sql @@ -0,0 +1,34 @@ +SELECT + "c_last_name" +, "c_first_name" +, "c_salutation" +, "c_preferred_cust_flag" +, "ss_ticket_number" +, "cnt" +FROM + ( + SELECT + "ss_ticket_number" + , "ss_customer_sk" + , "count"(*) "cnt" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.store + , ${database}.${schema}.household_demographics + WHERE ("store_sales"."ss_sold_date_sk" = "date_dim"."d_date_sk") + AND ("store_sales"."ss_store_sk" = "store"."s_store_sk") + AND ("store_sales"."ss_hdemo_sk" = "household_demographics"."hd_demo_sk") + AND ("date_dim"."d_dom" BETWEEN 1 AND 2) + AND (("household_demographics"."hd_buy_potential" = '>10000') + OR ("household_demographics"."hd_buy_potential" = 'Unknown')) + AND ("household_demographics"."hd_vehicle_count" > 0) + AND ((CASE WHEN ("household_demographics"."hd_vehicle_count" > 0) THEN (CAST("household_demographics"."hd_dep_count" AS DECIMAL(7,2)) / "household_demographics"."hd_vehicle_count") ELSE null END) > 1) + AND ("date_dim"."d_year" IN (1999 , (1999 + 1) , (1999 + 2))) + AND ("store"."s_county" IN ('Williamson County' , 'Franklin Parish' , 'Bronx County' , 'Orange County')) + GROUP BY "ss_ticket_number", "ss_customer_sk" +) dj +, ${database}.${schema}.customer +WHERE ("ss_customer_sk" = "c_customer_sk") + AND ("cnt" BETWEEN 1 AND 5) +ORDER BY "cnt" DESC, "c_last_name" ASC diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q74.sql b/presto-iceberg/src/test/resources/tpcds/queries/q74.sql new file mode 100644 index 0000000000000..23e5e368b7287 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q74.sql @@ -0,0 +1,58 @@ +WITH + year_total AS ( + SELECT + "c_customer_id" "customer_id" + , "c_first_name" "customer_first_name" + , "c_last_name" "customer_last_name" + , "d_year" "YEAR" + , "sum"("ss_net_paid") "year_total" + , 's' "sale_type" + FROM + ${database}.${schema}.customer + , ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + WHERE ("c_customer_sk" = "ss_customer_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("d_year" IN (2001 , (2001 + 1))) + GROUP BY "c_customer_id", "c_first_name", "c_last_name", "d_year" +UNION ALL SELECT + "c_customer_id" "customer_id" + , "c_first_name" "customer_first_name" + , "c_last_name" "customer_last_name" + , "d_year" "YEAR" + , "sum"("ws_net_paid") "year_total" + , 'w' "sale_type" + FROM + ${database}.${schema}.customer + , ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + WHERE ("c_customer_sk" = "ws_bill_customer_sk") + AND ("ws_sold_date_sk" = "d_date_sk") + AND ("d_year" IN (2001 , (2001 + 1))) + GROUP BY "c_customer_id", "c_first_name", "c_last_name", "d_year" +) +SELECT + "t_s_secyear"."customer_id" +, "t_s_secyear"."customer_first_name" +, "t_s_secyear"."customer_last_name" +FROM + year_total t_s_firstyear +, year_total t_s_secyear +, year_total t_w_firstyear +, year_total t_w_secyear +WHERE ("t_s_secyear"."customer_id" = "t_s_firstyear"."customer_id") + AND ("t_s_firstyear"."customer_id" = "t_w_secyear"."customer_id") + AND ("t_s_firstyear"."customer_id" = "t_w_firstyear"."customer_id") + AND ("t_s_firstyear"."sale_type" = 's') + AND ("t_w_firstyear"."sale_type" = 'w') + AND ("t_s_secyear"."sale_type" = 's') + AND ("t_w_secyear"."sale_type" = 'w') + AND ("t_s_firstyear"."year" = 2001) + AND ("t_s_secyear"."year" = (2001 + 1)) + AND ("t_w_firstyear"."year" = 2001) + AND ("t_w_secyear"."year" = (2001 + 1)) + AND ("t_s_firstyear"."year_total" > 0) + AND ("t_w_firstyear"."year_total" > 0) + AND ((CASE WHEN ("t_w_firstyear"."year_total" > 0) THEN ("t_w_secyear"."year_total" / "t_w_firstyear"."year_total") ELSE null END) > (CASE WHEN ("t_s_firstyear"."year_total" > 0) THEN ("t_s_secyear"."year_total" / "t_s_firstyear"."year_total") ELSE null END)) +ORDER BY 1 ASC, 1 ASC, 1 ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q75.sql b/presto-iceberg/src/test/resources/tpcds/queries/q75.sql new file mode 100644 index 0000000000000..a280b59f60eff --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q75.sql @@ -0,0 +1,83 @@ +WITH + all_sales AS ( + SELECT + "d_year" + , "i_brand_id" + , "i_class_id" + , "i_category_id" + , "i_manufact_id" + , "sum"("sales_cnt") "sales_cnt" + , "sum"("sales_amt") "sales_amt" + FROM + ( + SELECT + "d_year" + , "i_brand_id" + , "i_class_id" + , "i_category_id" + , "i_manufact_id" + , ("cs_quantity" - COALESCE("cr_return_quantity", 0)) "sales_cnt" + , ("cs_ext_sales_price" - COALESCE("cr_return_amount", DECIMAL '0.0')) "sales_amt" + FROM + (((${database}.${schema}.catalog_sales + INNER JOIN ${database}.${schema}.item ON ("i_item_sk" = "cs_item_sk")) + INNER JOIN ${database}.${schema}.date_dim ON ("d_date_sk" = "cs_sold_date_sk")) + LEFT JOIN ${database}.${schema}.catalog_returns ON ("cs_order_number" = "cr_order_number") + AND ("cs_item_sk" = "cr_item_sk")) + WHERE ("i_category" = 'Books') +UNION SELECT + "d_year" + , "i_brand_id" + , "i_class_id" + , "i_category_id" + , "i_manufact_id" + , ("ss_quantity" - COALESCE("sr_return_quantity", 0)) "sales_cnt" + , ("ss_ext_sales_price" - COALESCE("sr_return_amt", DECIMAL '0.0')) "sales_amt" + FROM + (((${database}.${schema}.store_sales + INNER JOIN ${database}.${schema}.item ON ("i_item_sk" = "ss_item_sk")) + INNER JOIN ${database}.${schema}.date_dim ON ("d_date_sk" = "ss_sold_date_sk")) + LEFT JOIN ${database}.${schema}.store_returns ON ("ss_ticket_number" = "sr_ticket_number") + AND ("ss_item_sk" = "sr_item_sk")) + WHERE ("i_category" = 'Books') +UNION SELECT + "d_year" + , "i_brand_id" + , "i_class_id" + , "i_category_id" + , "i_manufact_id" + , ("ws_quantity" - COALESCE("wr_return_quantity", 0)) "sales_cnt" + , ("ws_ext_sales_price" - COALESCE("wr_return_amt", DECIMAL '0.0')) "sales_amt" + FROM + (((${database}.${schema}.web_sales + INNER JOIN ${database}.${schema}.item ON ("i_item_sk" = "ws_item_sk")) + INNER JOIN ${database}.${schema}.date_dim ON ("d_date_sk" = "ws_sold_date_sk")) + LEFT JOIN ${database}.${schema}.web_returns ON ("ws_order_number" = "wr_order_number") + AND ("ws_item_sk" = "wr_item_sk")) + WHERE ("i_category" = 'Books') + ) sales_detail + GROUP BY "d_year", "i_brand_id", "i_class_id", "i_category_id", "i_manufact_id" +) +SELECT + "prev_yr"."d_year" "prev_year" +, "curr_yr"."d_year" "year" +, "curr_yr"."i_brand_id" +, "curr_yr"."i_class_id" +, "curr_yr"."i_category_id" +, "curr_yr"."i_manufact_id" +, "prev_yr"."sales_cnt" "prev_yr_cnt" +, "curr_yr"."sales_cnt" "curr_yr_cnt" +, ("curr_yr"."sales_cnt" - "prev_yr"."sales_cnt") "sales_cnt_diff" +, ("curr_yr"."sales_amt" - "prev_yr"."sales_amt") "sales_amt_diff" +FROM + all_sales curr_yr +, all_sales prev_yr +WHERE ("curr_yr"."i_brand_id" = "prev_yr"."i_brand_id") + AND ("curr_yr"."i_class_id" = "prev_yr"."i_class_id") + AND ("curr_yr"."i_category_id" = "prev_yr"."i_category_id") + AND ("curr_yr"."i_manufact_id" = "prev_yr"."i_manufact_id") + AND ("curr_yr"."d_year" = 2002) + AND ("prev_yr"."d_year" = (2002 - 1)) + AND ((CAST("curr_yr"."sales_cnt" AS DECIMAL(17,2)) / CAST("prev_yr"."sales_cnt" AS DECIMAL(17,2))) < DECIMAL '0.9') +ORDER BY "sales_cnt_diff" ASC, "sales_amt_diff" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q76.sql b/presto-iceberg/src/test/resources/tpcds/queries/q76.sql new file mode 100644 index 0000000000000..a9a1f8f03027c --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q76.sql @@ -0,0 +1,56 @@ +SELECT + "channel" +, "col_name" +, "d_year" +, "d_qoy" +, "i_category" +, "count"(*) "sales_cnt" +, "sum"("ext_sales_price") "sales_amt" +FROM + ( + SELECT + '${database}.${schema}.store' "channel" + , 'ss_store_sk' "col_name" + , "d_year" + , "d_qoy" + , "i_category" + , "ss_ext_sales_price" "ext_sales_price" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.item + , ${database}.${schema}.date_dim + WHERE ("ss_store_sk" IS NULL) + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("ss_item_sk" = "i_item_sk") +UNION ALL SELECT + 'web' "channel" + , 'ws_ship_customer_sk' "col_name" + , "d_year" + , "d_qoy" + , "i_category" + , "ws_ext_sales_price" "ext_sales_price" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.item + , ${database}.${schema}.date_dim + WHERE ("ws_ship_customer_sk" IS NULL) + AND ("ws_sold_date_sk" = "d_date_sk") + AND ("ws_item_sk" = "i_item_sk") +UNION ALL SELECT + 'catalog' "channel" + , 'cs_ship_addr_sk' "col_name" + , "d_year" + , "d_qoy" + , "i_category" + , "cs_ext_sales_price" "ext_sales_price" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.item + , ${database}.${schema}.date_dim + WHERE ("cs_ship_addr_sk" IS NULL) + AND ("cs_sold_date_sk" = "d_date_sk") + AND ("cs_item_sk" = "i_item_sk") +) foo +GROUP BY "channel", "col_name", "d_year", "d_qoy", "i_category" +ORDER BY "channel" ASC, "col_name" ASC, "d_year" ASC, "d_qoy" ASC, "i_category" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q77.sql b/presto-iceberg/src/test/resources/tpcds/queries/q77.sql new file mode 100644 index 0000000000000..58a0861688136 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q77.sql @@ -0,0 +1,120 @@ +WITH + ss AS ( + SELECT + "s_store_sk" + , "sum"("ss_ext_sales_price") "sales" + , "sum"("ss_net_profit") "profit" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.store + WHERE ("ss_sold_date_sk" = "d_date_sk") + AND ("d_date" BETWEEN CAST('2000-08-23' AS DATE) AND (CAST('2000-08-23' AS DATE) + INTERVAL '30' DAY)) + AND ("ss_store_sk" = "s_store_sk") + GROUP BY "s_store_sk" +) +, sr AS ( + SELECT + "s_store_sk" + , "sum"("sr_return_amt") "returns" + , "sum"("sr_net_loss") "profit_loss" + FROM + ${database}.${schema}.store_returns + , ${database}.${schema}.date_dim + , ${database}.${schema}.store + WHERE ("sr_returned_date_sk" = "d_date_sk") + AND ("d_date" BETWEEN CAST('2000-08-23' AS DATE) AND (CAST('2000-08-23' AS DATE) + INTERVAL '30' DAY)) + AND ("sr_store_sk" = "s_store_sk") + GROUP BY "s_store_sk" +) +, cs AS ( + SELECT + "cs_call_center_sk" + , "sum"("cs_ext_sales_price") "sales" + , "sum"("cs_net_profit") "profit" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.date_dim + WHERE ("cs_sold_date_sk" = "d_date_sk") + AND ("d_date" BETWEEN CAST('2000-08-23' AS DATE) AND (CAST('2000-08-23' AS DATE) + INTERVAL '30' DAY)) + GROUP BY "cs_call_center_sk" +) +, cr AS ( + SELECT + "cr_call_center_sk" + , "sum"("cr_return_amount") "returns" + , "sum"("cr_net_loss") "profit_loss" + FROM + ${database}.${schema}.catalog_returns + , ${database}.${schema}.date_dim + WHERE ("cr_returned_date_sk" = "d_date_sk") + AND ("d_date" BETWEEN CAST('2000-08-23' AS DATE) AND (CAST('2000-08-23' AS DATE) + INTERVAL '30' DAY)) + GROUP BY "cr_call_center_sk" +) +, ws AS ( + SELECT + "wp_web_page_sk" + , "sum"("ws_ext_sales_price") "sales" + , "sum"("ws_net_profit") "profit" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.web_page + WHERE ("ws_sold_date_sk" = "d_date_sk") + AND ("d_date" BETWEEN CAST('2000-08-23' AS DATE) AND (CAST('2000-08-23' AS DATE) + INTERVAL '30' DAY)) + AND ("ws_web_page_sk" = "wp_web_page_sk") + GROUP BY "wp_web_page_sk" +) +, wr AS ( + SELECT + "wp_web_page_sk" + , "sum"("wr_return_amt") "returns" + , "sum"("wr_net_loss") "profit_loss" + FROM + ${database}.${schema}.web_returns + , ${database}.${schema}.date_dim + , ${database}.${schema}.web_page + WHERE ("wr_returned_date_sk" = "d_date_sk") + AND ("d_date" BETWEEN CAST('2000-08-23' AS DATE) AND (CAST('2000-08-23' AS DATE) + INTERVAL '30' DAY)) + AND ("wr_web_page_sk" = "wp_web_page_sk") + GROUP BY "wp_web_page_sk" +) +SELECT + "channel" +, "id" +, "sum"("sales") "sales" +, "sum"("returns") "returns" +, "sum"("profit") "profit" +FROM + ( + SELECT + '${database}.${schema}.store channel' "channel" + , "ss"."s_store_sk" "id" + , "sales" + , COALESCE("returns", 0) "returns" + , ("profit" - COALESCE("profit_loss", 0)) "profit" + FROM + (ss + LEFT JOIN sr ON ("ss"."s_store_sk" = "sr"."s_store_sk")) +UNION ALL SELECT + 'catalog channel' "channel" + , "cs_call_center_sk" "id" + , "sales" + , "returns" + , ("profit" - "profit_loss") "profit" + FROM + cs + , cr +UNION ALL SELECT + 'web channel' "channel" + , "ws"."wp_web_page_sk" "id" + , "sales" + , COALESCE("returns", 0) "returns" + , ("profit" - COALESCE("profit_loss", 0)) "profit" + FROM + (ws + LEFT JOIN wr ON ("ws"."wp_web_page_sk" = "wr"."wp_web_page_sk")) +) x +GROUP BY ROLLUP (channel, id) +ORDER BY "channel" ASC, "id" ASC, "sales" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q78.sql b/presto-iceberg/src/test/resources/tpcds/queries/q78.sql new file mode 100644 index 0000000000000..6655aa49ac516 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q78.sql @@ -0,0 +1,73 @@ +WITH + ws AS ( + SELECT + "d_year" "ws_sold_year" + , "ws_item_sk" + , "ws_bill_customer_sk" "ws_customer_sk" + , "sum"("ws_quantity") "ws_qty" + , "sum"("ws_wholesale_cost") "ws_wc" + , "sum"("ws_sales_price") "ws_sp" + FROM + ((${database}.${schema}.web_sales + LEFT JOIN ${database}.${schema}.web_returns ON ("wr_order_number" = "ws_order_number") + AND ("ws_item_sk" = "wr_item_sk")) + INNER JOIN ${database}.${schema}.date_dim ON ("ws_sold_date_sk" = "d_date_sk")) + WHERE ("wr_order_number" IS NULL) + GROUP BY "d_year", "ws_item_sk", "ws_bill_customer_sk" +) +, cs AS ( + SELECT + "d_year" "cs_sold_year" + , "cs_item_sk" + , "cs_bill_customer_sk" "cs_customer_sk" + , "sum"("cs_quantity") "cs_qty" + , "sum"("cs_wholesale_cost") "cs_wc" + , "sum"("cs_sales_price") "cs_sp" + FROM + ((${database}.${schema}.catalog_sales + LEFT JOIN ${database}.${schema}.catalog_returns ON ("cr_order_number" = "cs_order_number") + AND ("cs_item_sk" = "cr_item_sk")) + INNER JOIN ${database}.${schema}.date_dim ON ("cs_sold_date_sk" = "d_date_sk")) + WHERE ("cr_order_number" IS NULL) + GROUP BY "d_year", "cs_item_sk", "cs_bill_customer_sk" +) +, ss AS ( + SELECT + "d_year" "ss_sold_year" + , "ss_item_sk" + , "ss_customer_sk" + , "sum"("ss_quantity") "ss_qty" + , "sum"("ss_wholesale_cost") "ss_wc" + , "sum"("ss_sales_price") "ss_sp" + FROM + ((${database}.${schema}.store_sales + LEFT JOIN ${database}.${schema}.store_returns ON ("sr_ticket_number" = "ss_ticket_number") + AND ("ss_item_sk" = "sr_item_sk")) + INNER JOIN ${database}.${schema}.date_dim ON ("ss_sold_date_sk" = "d_date_sk")) + WHERE ("sr_ticket_number" IS NULL) + GROUP BY "d_year", "ss_item_sk", "ss_customer_sk" +) +SELECT + "ss_sold_year" +, "ss_item_sk" +, "ss_customer_sk" +, "round"((CAST("ss_qty" AS DECIMAL(10,2)) / COALESCE(("ws_qty" + "cs_qty"), 1)), 2) "ratio" +, "ss_qty" "store_qty" +, "ss_wc" "store_wholesale_cost" +, "ss_sp" "store_sales_price" +, (COALESCE("ws_qty", 0) + COALESCE("cs_qty", 0)) "other_chan_qty" +, (COALESCE("ws_wc", 0) + COALESCE("cs_wc", 0)) "other_chan_wholesale_cost" +, (COALESCE("ws_sp", 0) + COALESCE("cs_sp", 0)) "other_chan_sales_price" +FROM + ((ss +LEFT JOIN ws ON ("ws_sold_year" = "ss_sold_year") + AND ("ws_item_sk" = "ss_item_sk") + AND ("ws_customer_sk" = "ss_customer_sk")) +LEFT JOIN cs ON ("cs_sold_year" = "ss_sold_year") + AND ("cs_item_sk" = "cs_item_sk") + AND ("cs_customer_sk" = "ss_customer_sk")) +WHERE (COALESCE("ws_qty", 0) > 0) + AND (COALESCE("cs_qty", 0) > 0) + AND ("ss_sold_year" = 2000) +ORDER BY "ss_sold_year" ASC, "ss_item_sk" ASC, "ss_customer_sk" ASC, "ss_qty" DESC, "ss_wc" DESC, "ss_sp" DESC, "other_chan_qty" ASC, "other_chan_wholesale_cost" ASC, "other_chan_sales_price" ASC, "round"((CAST("ss_qty" AS DECIMAL(10,2)) / COALESCE(("ws_qty" + "cs_qty"), 1)), 2) ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q79.sql b/presto-iceberg/src/test/resources/tpcds/queries/q79.sql new file mode 100644 index 0000000000000..7cac0b43899ef --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q79.sql @@ -0,0 +1,34 @@ +SELECT + "c_last_name" +, "c_first_name" +, "substr"("s_city", 1, 30) +, "ss_ticket_number" +, "amt" +, "profit" +FROM + ( + SELECT + "ss_ticket_number" + , "ss_customer_sk" + , "store"."s_city" + , "sum"("ss_coupon_amt") "amt" + , "sum"("ss_net_profit") "profit" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.store + , ${database}.${schema}.household_demographics + WHERE ("store_sales"."ss_sold_date_sk" = "date_dim"."d_date_sk") + AND ("store_sales"."ss_store_sk" = "store"."s_store_sk") + AND ("store_sales"."ss_hdemo_sk" = "household_demographics"."hd_demo_sk") + AND (("household_demographics"."hd_dep_count" = 6) + OR ("household_demographics"."hd_vehicle_count" > 2)) + AND ("date_dim"."d_dow" = 1) + AND ("date_dim"."d_year" IN (1999 , (1999 + 1) , (1999 + 2))) + AND ("store"."s_number_employees" BETWEEN 200 AND 295) + GROUP BY "ss_ticket_number", "ss_customer_sk", "ss_addr_sk", "store"."s_city" +) ms +, ${database}.${schema}.customer +WHERE ("ss_customer_sk" = "c_customer_sk") +ORDER BY "c_last_name" ASC, "c_first_name" ASC, "substr"("s_city", 1, 30) ASC, "profit" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q80.sql b/presto-iceberg/src/test/resources/tpcds/queries/q80.sql new file mode 100644 index 0000000000000..0d3b44ddf5697 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q80.sql @@ -0,0 +1,106 @@ +WITH + ssr AS ( + SELECT + "s_store_id" "store_id" + , "sum"("ss_ext_sales_price") "sales" + , "sum"(COALESCE("sr_return_amt", 0)) "returns" + , "sum"(("ss_net_profit" - COALESCE("sr_net_loss", 0))) "profit" + FROM + (${database}.${schema}.store_sales + LEFT JOIN ${database}.${schema}.store_returns ON ("ss_item_sk" = "sr_item_sk") + AND ("ss_ticket_number" = "sr_ticket_number")) + , ${database}.${schema}.date_dim + , ${database}.${schema}.store + , ${database}.${schema}.item + , ${database}.${schema}.promotion + WHERE ("ss_sold_date_sk" = "d_date_sk") + AND (CAST("d_date" AS DATE) BETWEEN CAST('2000-08-23' AS DATE) AND (CAST('2000-08-23' AS DATE) + INTERVAL '30' DAY)) + AND ("ss_store_sk" = "s_store_sk") + AND ("ss_item_sk" = "i_item_sk") + AND ("i_current_price" > 50) + AND ("ss_promo_sk" = "p_promo_sk") + AND ("p_channel_tv" = 'N') + GROUP BY "s_store_id" +) +, csr AS ( + SELECT + "cp_catalog_page_id" "catalog_page_id" + , "sum"("cs_ext_sales_price") "sales" + , "sum"(COALESCE("cr_return_amount", 0)) "returns" + , "sum"(("cs_net_profit" - COALESCE("cr_net_loss", 0))) "profit" + FROM + (${database}.${schema}.catalog_sales + LEFT JOIN ${database}.${schema}.catalog_returns ON ("cs_item_sk" = "cr_item_sk") + AND ("cs_order_number" = "cr_order_number")) + , ${database}.${schema}.date_dim + , ${database}.${schema}.catalog_page + , ${database}.${schema}.item + , ${database}.${schema}.promotion + WHERE ("cs_sold_date_sk" = "d_date_sk") + AND (CAST("d_date" AS DATE) BETWEEN CAST('2000-08-23' AS DATE) AND (CAST('2000-08-23' AS DATE) + INTERVAL '30' DAY)) + AND ("cs_catalog_page_sk" = "cp_catalog_page_sk") + AND ("cs_item_sk" = "i_item_sk") + AND ("i_current_price" > 50) + AND ("cs_promo_sk" = "p_promo_sk") + AND ("p_channel_tv" = 'N') + GROUP BY "cp_catalog_page_id" +) +, wsr AS ( + SELECT + "web_site_id" + , "sum"("ws_ext_sales_price") "sales" + , "sum"(COALESCE("wr_return_amt", 0)) "returns" + , "sum"(("ws_net_profit" - COALESCE("wr_net_loss", 0))) "profit" + FROM + (${database}.${schema}.web_sales + LEFT JOIN ${database}.${schema}.web_returns ON ("ws_item_sk" = "wr_item_sk") + AND ("ws_order_number" = "wr_order_number")) + , ${database}.${schema}.date_dim + , ${database}.${schema}.web_site + , ${database}.${schema}.item + , ${database}.${schema}.promotion + WHERE ("ws_sold_date_sk" = "d_date_sk") + AND (CAST("d_date" AS DATE) BETWEEN CAST('2000-08-23' AS DATE) AND (CAST('2000-08-23' AS DATE) + INTERVAL '30' DAY)) + AND ("ws_web_site_sk" = "web_site_sk") + AND ("ws_item_sk" = "i_item_sk") + AND ("i_current_price" > 50) + AND ("ws_promo_sk" = "p_promo_sk") + AND ("p_channel_tv" = 'N') + GROUP BY "web_site_id" +) +SELECT + "channel" +, "id" +, "sum"("sales") "sales" +, "sum"("returns") "returns" +, "sum"("profit") "profit" +FROM + ( + SELECT + '${database}.${schema}.store channel' "channel" + , "concat"('store', "store_id") "id" + , "sales" + , "returns" + , "profit" + FROM + ssr +UNION ALL SELECT + 'catalog channel' "channel" + , "concat"('catalog_page', "catalog_page_id") "id" + , "sales" + , "returns" + , "profit" + FROM + csr +UNION ALL SELECT + 'web channel' "channel" + , "concat"('web_site', "web_site_id") "id" + , "sales" + , "returns" + , "profit" + FROM + wsr +) x +GROUP BY ROLLUP (channel, id) +ORDER BY "channel" ASC, "id" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q81.sql b/presto-iceberg/src/test/resources/tpcds/queries/q81.sql new file mode 100644 index 0000000000000..2406641e84d0e --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q81.sql @@ -0,0 +1,47 @@ +WITH + customer_total_return AS ( + SELECT + "cr_returning_customer_sk" "ctr_customer_sk" + , "ca_state" "ctr_state" + , "sum"("cr_return_amt_inc_tax") "ctr_total_return" + FROM + ${database}.${schema}.catalog_returns + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer_address + WHERE ("cr_returned_date_sk" = "d_date_sk") + AND ("d_year" = 2000) + AND ("cr_returning_addr_sk" = "ca_address_sk") + GROUP BY "cr_returning_customer_sk", "ca_state" +) +SELECT + "c_customer_id" +, "c_salutation" +, "c_first_name" +, "c_last_name" +, "ca_street_number" +, "ca_street_name" +, "ca_street_type" +, "ca_suite_number" +, "ca_city" +, "ca_county" +, "ca_state" +, "ca_zip" +, "ca_country" +, "ca_gmt_offset" +, "ca_location_type" +, "ctr_total_return" +FROM + customer_total_return ctr1 +, ${database}.${schema}.customer_address +, ${database}.${schema}.customer +WHERE ("ctr1"."ctr_total_return" > ( + SELECT ("avg"("ctr_total_return") * DECIMAL '1.2') + FROM + customer_total_return ctr2 + WHERE ("ctr1"."ctr_state" = "ctr2"."ctr_state") + )) + AND ("ca_address_sk" = "c_current_addr_sk") + AND ("ca_state" = 'GA') + AND ("ctr1"."ctr_customer_sk" = "c_customer_sk") +ORDER BY "c_customer_id" ASC, "c_salutation" ASC, "c_first_name" ASC, "c_last_name" ASC, "ca_street_number" ASC, "ca_street_name" ASC, "ca_street_type" ASC, "ca_suite_number" ASC, "ca_city" ASC, "ca_county" ASC, "ca_state" ASC, "ca_zip" ASC, "ca_country" ASC, "ca_gmt_offset" ASC, "ca_location_type" ASC, "ctr_total_return" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q82.sql b/presto-iceberg/src/test/resources/tpcds/queries/q82.sql new file mode 100644 index 0000000000000..967b876b10132 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q82.sql @@ -0,0 +1,19 @@ +SELECT + "i_item_id" +, "i_item_desc" +, "i_current_price" +FROM + ${database}.${schema}.item +, ${database}.${schema}.inventory +, ${database}.${schema}.date_dim +, ${database}.${schema}.store_sales +WHERE ("i_current_price" BETWEEN 62 AND (62 + 30)) + AND ("inv_item_sk" = "i_item_sk") + AND ("d_date_sk" = "inv_date_sk") + AND (CAST("d_date" AS DATE) BETWEEN CAST('2000-05-25' AS DATE) AND (CAST('2000-05-25' AS DATE) + INTERVAL '60' DAY)) + AND ("i_manufact_id" IN (129, 270, 821, 423)) + AND ("inv_quantity_on_hand" BETWEEN 100 AND 500) + AND ("ss_item_sk" = "i_item_sk") +GROUP BY "i_item_id", "i_item_desc", "i_current_price" +ORDER BY "i_item_id" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q83.sql b/presto-iceberg/src/test/resources/tpcds/queries/q83.sql new file mode 100644 index 0000000000000..6887d06e4dce0 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q83.sql @@ -0,0 +1,87 @@ +WITH + sr_items AS ( + SELECT + "i_item_id" "item_id" + , "sum"("sr_return_quantity") "sr_item_qty" + FROM + ${database}.${schema}.store_returns + , ${database}.${schema}.item + , ${database}.${schema}.date_dim + WHERE ("sr_item_sk" = "i_item_sk") + AND ("d_date" IN ( + SELECT "d_date" + FROM + ${database}.${schema}.date_dim + WHERE ("d_week_seq" IN ( + SELECT "d_week_seq" + FROM + ${database}.${schema}.date_dim + WHERE ("d_date" IN (CAST('2000-06-30' AS DATE) , CAST('2000-09-27' AS DATE) , CAST('2000-11-17' AS DATE))) + )) + )) + AND ("sr_returned_date_sk" = "d_date_sk") + GROUP BY "i_item_id" +) +, cr_items AS ( + SELECT + "i_item_id" "item_id" + , "sum"("cr_return_quantity") "cr_item_qty" + FROM + ${database}.${schema}.catalog_returns + , ${database}.${schema}.item + , ${database}.${schema}.date_dim + WHERE ("cr_item_sk" = "i_item_sk") + AND ("d_date" IN ( + SELECT "d_date" + FROM + ${database}.${schema}.date_dim + WHERE ("d_week_seq" IN ( + SELECT "d_week_seq" + FROM + ${database}.${schema}.date_dim + WHERE ("d_date" IN (CAST('2000-06-30' AS DATE) , CAST('2000-09-27' AS DATE) , CAST('2000-11-17' AS DATE))) + )) + )) + AND ("cr_returned_date_sk" = "d_date_sk") + GROUP BY "i_item_id" +) +, wr_items AS ( + SELECT + "i_item_id" "item_id" + , "sum"("wr_return_quantity") "wr_item_qty" + FROM + ${database}.${schema}.web_returns + , ${database}.${schema}.item + , ${database}.${schema}.date_dim + WHERE ("wr_item_sk" = "i_item_sk") + AND ("d_date" IN ( + SELECT "d_date" + FROM + ${database}.${schema}.date_dim + WHERE ("d_week_seq" IN ( + SELECT "d_week_seq" + FROM + ${database}.${schema}.date_dim + WHERE ("d_date" IN (CAST('2000-06-30' AS DATE) , CAST('2000-09-27' AS DATE) , CAST('2000-11-17' AS DATE))) + )) + )) + AND ("wr_returned_date_sk" = "d_date_sk") + GROUP BY "i_item_id" +) +SELECT + "sr_items"."item_id" +, "sr_item_qty" +, CAST(((("sr_item_qty" / ((CAST("sr_item_qty" AS DECIMAL(9,4)) + "cr_item_qty") + "wr_item_qty")) / DECIMAL '3.0') * 100) AS DECIMAL(7,2)) "sr_dev" +, "cr_item_qty" +, CAST(((("cr_item_qty" / ((CAST("sr_item_qty" AS DECIMAL(9,4)) + "cr_item_qty") + "wr_item_qty")) / DECIMAL '3.0') * 100) AS DECIMAL(7,2)) "cr_dev" +, "wr_item_qty" +, CAST(((("wr_item_qty" / ((CAST("sr_item_qty" AS DECIMAL(9,4)) + "cr_item_qty") + "wr_item_qty")) / DECIMAL '3.0') * 100) AS DECIMAL(7,2)) "wr_dev" +, ((("sr_item_qty" + "cr_item_qty") + "wr_item_qty") / DECIMAL '3.00') "average" +FROM + sr_items +, cr_items +, wr_items +WHERE ("sr_items"."item_id" = "cr_items"."item_id") + AND ("sr_items"."item_id" = "wr_items"."item_id") +ORDER BY "sr_items"."item_id" ASC, "sr_item_qty" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q84.sql b/presto-iceberg/src/test/resources/tpcds/queries/q84.sql new file mode 100644 index 0000000000000..879d525aa506e --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q84.sql @@ -0,0 +1,20 @@ +SELECT + "c_customer_id" "customer_id" +, "concat"("concat"("c_last_name", ', '), "c_first_name") "${database}.${schema}.customername" +FROM + ${database}.${schema}.customer +, ${database}.${schema}.customer_address +, ${database}.${schema}.customer_demographics +, ${database}.${schema}.household_demographics +, ${database}.${schema}.income_band +, ${database}.${schema}.store_returns +WHERE ("ca_city" = 'Edgewood') + AND ("c_current_addr_sk" = "ca_address_sk") + AND ("ib_lower_bound" >= 38128) + AND ("ib_upper_bound" <= (38128 + 50000)) + AND ("ib_income_band_sk" = "hd_income_band_sk") + AND ("cd_demo_sk" = "c_current_cdemo_sk") + AND ("hd_demo_sk" = "c_current_hdemo_sk") + AND ("sr_cdemo_sk" = "cd_demo_sk") +ORDER BY "c_customer_id" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q85.sql b/presto-iceberg/src/test/resources/tpcds/queries/q85.sql new file mode 100644 index 0000000000000..d05670420a81e --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q85.sql @@ -0,0 +1,50 @@ +SELECT + "substr"("r_reason_desc", 1, 20) +, "avg"("ws_quantity") +, "avg"("wr_refunded_cash") +, "avg"("wr_fee") +FROM + ${database}.${schema}.web_sales +, ${database}.${schema}.web_returns +, ${database}.${schema}.web_page +, ${database}.${schema}.customer_demographics cd1 +, ${database}.${schema}.customer_demographics cd2 +, ${database}.${schema}.customer_address +, ${database}.${schema}.date_dim +, ${database}.${schema}.reason +WHERE ("ws_web_page_sk" = "wp_web_page_sk") + AND ("ws_item_sk" = "wr_item_sk") + AND ("ws_order_number" = "wr_order_number") + AND ("ws_sold_date_sk" = "d_date_sk") + AND ("d_year" = 2000) + AND ("cd1"."cd_demo_sk" = "wr_refunded_cdemo_sk") + AND ("cd2"."cd_demo_sk" = "wr_returning_cdemo_sk") + AND ("ca_address_sk" = "wr_refunded_addr_sk") + AND ("r_reason_sk" = "wr_reason_sk") + AND ((("cd1"."cd_marital_status" = 'M') + AND ("cd1"."cd_marital_status" = "cd2"."cd_marital_status") + AND ("cd1"."cd_education_status" = 'Advanced Degree') + AND ("cd1"."cd_education_status" = "cd2"."cd_education_status") + AND ("ws_sales_price" BETWEEN DECIMAL '100.00' AND DECIMAL '150.00')) + OR (("cd1"."cd_marital_status" = 'S') + AND ("cd1"."cd_marital_status" = "cd2"."cd_marital_status") + AND ("cd1"."cd_education_status" = 'College') + AND ("cd1"."cd_education_status" = "cd2"."cd_education_status") + AND ("ws_sales_price" BETWEEN DECIMAL '50.00' AND DECIMAL '100.00')) + OR (("cd1"."cd_marital_status" = 'W') + AND ("cd1"."cd_marital_status" = "cd2"."cd_marital_status") + AND ("cd1"."cd_education_status" = '2 yr Degree') + AND ("cd1"."cd_education_status" = "cd2"."cd_education_status") + AND ("ws_sales_price" BETWEEN DECIMAL '150.00' AND DECIMAL '200.00'))) + AND ((("ca_country" = 'United States') + AND ("ca_state" IN ('IN' , 'OH' , 'NJ')) + AND ("ws_net_profit" BETWEEN 100 AND 200)) + OR (("ca_country" = 'United States') + AND ("ca_state" IN ('WI' , 'CT' , 'KY')) + AND ("ws_net_profit" BETWEEN 150 AND 300)) + OR (("ca_country" = 'United States') + AND ("ca_state" IN ('LA' , 'IA' , 'AR')) + AND ("ws_net_profit" BETWEEN 50 AND 250))) +GROUP BY "r_reason_desc" +ORDER BY "substr"("r_reason_desc", 1, 20) ASC, "avg"("ws_quantity") ASC, "avg"("wr_refunded_cash") ASC, "avg"("wr_fee") ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q86.sql b/presto-iceberg/src/test/resources/tpcds/queries/q86.sql new file mode 100644 index 0000000000000..9c8c3bdd00b02 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q86.sql @@ -0,0 +1,16 @@ +SELECT + "sum"("ws_net_paid") "total_sum" +, "i_category" +, "i_class" +, (GROUPING ("i_category") + GROUPING ("i_class")) "lochierarchy" +, "rank"() OVER (PARTITION BY (GROUPING ("i_category") + GROUPING ("i_class")), (CASE WHEN (GROUPING ("i_class") = 0) THEN "i_category" END) ORDER BY "sum"("ws_net_paid") DESC) "rank_within_parent" +FROM + ${database}.${schema}.web_sales +, ${database}.${schema}.date_dim d1 +, ${database}.${schema}.item +WHERE ("d1"."d_month_seq" BETWEEN 1200 AND (1200 + 11)) + AND ("d1"."d_date_sk" = "ws_sold_date_sk") + AND ("i_item_sk" = "ws_item_sk") +GROUP BY ROLLUP (i_category, i_class) +ORDER BY "lochierarchy" DESC, (CASE WHEN ("lochierarchy" = 0) THEN "i_category" END) ASC, "rank_within_parent" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q87.sql b/presto-iceberg/src/test/resources/tpcds/queries/q87.sql new file mode 100644 index 0000000000000..fd257bd5104e0 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q87.sql @@ -0,0 +1,40 @@ +SELECT "count"(*) +FROM + ( +( + SELECT DISTINCT + "c_last_name" + , "c_first_name" + , "d_date" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer + WHERE ("store_sales"."ss_sold_date_sk" = "date_dim"."d_date_sk") + AND ("store_sales"."ss_customer_sk" = "customer"."c_customer_sk") + AND ("d_month_seq" BETWEEN 1200 AND (1200 + 11)) + ) EXCEPT ( + SELECT DISTINCT + "c_last_name" + , "c_first_name" + , "d_date" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer + WHERE ("catalog_sales"."cs_sold_date_sk" = "date_dim"."d_date_sk") + AND ("catalog_sales"."cs_bill_customer_sk" = "customer"."c_customer_sk") + AND ("d_month_seq" BETWEEN 1200 AND (1200 + 11)) + ) EXCEPT ( + SELECT DISTINCT + "c_last_name" + , "c_first_name" + , "d_date" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.customer + WHERE ("web_sales"."ws_sold_date_sk" = "date_dim"."d_date_sk") + AND ("web_sales"."ws_bill_customer_sk" = "customer"."c_customer_sk") + AND ("d_month_seq" BETWEEN 1200 AND (1200 + 11)) + ) ) cool_cust diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q88.sql b/presto-iceberg/src/test/resources/tpcds/queries/q88.sql new file mode 100644 index 0000000000000..94e4867acba9b --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q88.sql @@ -0,0 +1,162 @@ +SELECT * +FROM + ( + SELECT "count"(*) "h8_30_to_9" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.household_demographics + , ${database}.${schema}.time_dim + , ${database}.${schema}.store + WHERE ("ss_sold_time_sk" = "time_dim"."t_time_sk") + AND ("ss_hdemo_sk" = "household_demographics"."hd_demo_sk") + AND ("ss_store_sk" = "s_store_sk") + AND ("time_dim"."t_hour" = 8) + AND ("time_dim"."t_minute" >= 30) + AND ((("household_demographics"."hd_dep_count" = 4) + AND ("household_demographics"."hd_vehicle_count" <= (4 + 2))) + OR (("household_demographics"."hd_dep_count" = 2) + AND ("household_demographics"."hd_vehicle_count" <= (2 + 2))) + OR (("household_demographics"."hd_dep_count" = 0) + AND ("household_demographics"."hd_vehicle_count" <= (0 + 2)))) + AND ("store"."s_store_name" = 'ese') +) s1 +, ( + SELECT "count"(*) "h9_to_9_30" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.household_demographics + , ${database}.${schema}.time_dim + , ${database}.${schema}.store + WHERE ("ss_sold_time_sk" = "time_dim"."t_time_sk") + AND ("ss_hdemo_sk" = "household_demographics"."hd_demo_sk") + AND ("ss_store_sk" = "s_store_sk") + AND ("time_dim"."t_hour" = 9) + AND ("time_dim"."t_minute" < 30) + AND ((("household_demographics"."hd_dep_count" = 4) + AND ("household_demographics"."hd_vehicle_count" <= (4 + 2))) + OR (("household_demographics"."hd_dep_count" = 2) + AND ("household_demographics"."hd_vehicle_count" <= (2 + 2))) + OR (("household_demographics"."hd_dep_count" = 0) + AND ("household_demographics"."hd_vehicle_count" <= (0 + 2)))) + AND ("store"."s_store_name" = 'ese') +) s2 +, ( + SELECT "count"(*) "h9_30_to_10" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.household_demographics + , ${database}.${schema}.time_dim + , ${database}.${schema}.store + WHERE ("ss_sold_time_sk" = "time_dim"."t_time_sk") + AND ("ss_hdemo_sk" = "household_demographics"."hd_demo_sk") + AND ("ss_store_sk" = "s_store_sk") + AND ("time_dim"."t_hour" = 9) + AND ("time_dim"."t_minute" >= 30) + AND ((("household_demographics"."hd_dep_count" = 4) + AND ("household_demographics"."hd_vehicle_count" <= (4 + 2))) + OR (("household_demographics"."hd_dep_count" = 2) + AND ("household_demographics"."hd_vehicle_count" <= (2 + 2))) + OR (("household_demographics"."hd_dep_count" = 0) + AND ("household_demographics"."hd_vehicle_count" <= (0 + 2)))) + AND ("store"."s_store_name" = 'ese') +) s3 +, ( + SELECT "count"(*) "h10_to_10_30" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.household_demographics + , ${database}.${schema}.time_dim + , ${database}.${schema}.store + WHERE ("ss_sold_time_sk" = "time_dim"."t_time_sk") + AND ("ss_hdemo_sk" = "household_demographics"."hd_demo_sk") + AND ("ss_store_sk" = "s_store_sk") + AND ("time_dim"."t_hour" = 10) + AND ("time_dim"."t_minute" < 30) + AND ((("household_demographics"."hd_dep_count" = 4) + AND ("household_demographics"."hd_vehicle_count" <= (4 + 2))) + OR (("household_demographics"."hd_dep_count" = 2) + AND ("household_demographics"."hd_vehicle_count" <= (2 + 2))) + OR (("household_demographics"."hd_dep_count" = 0) + AND ("household_demographics"."hd_vehicle_count" <= (0 + 2)))) + AND ("store"."s_store_name" = 'ese') +) s4 +, ( + SELECT "count"(*) "h10_30_to_11" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.household_demographics + , ${database}.${schema}.time_dim + , ${database}.${schema}.store + WHERE ("ss_sold_time_sk" = "time_dim"."t_time_sk") + AND ("ss_hdemo_sk" = "household_demographics"."hd_demo_sk") + AND ("ss_store_sk" = "s_store_sk") + AND ("time_dim"."t_hour" = 10) + AND ("time_dim"."t_minute" >= 30) + AND ((("household_demographics"."hd_dep_count" = 4) + AND ("household_demographics"."hd_vehicle_count" <= (4 + 2))) + OR (("household_demographics"."hd_dep_count" = 2) + AND ("household_demographics"."hd_vehicle_count" <= (2 + 2))) + OR (("household_demographics"."hd_dep_count" = 0) + AND ("household_demographics"."hd_vehicle_count" <= (0 + 2)))) + AND ("store"."s_store_name" = 'ese') +) s5 +, ( + SELECT "count"(*) "h11_to_11_30" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.household_demographics + , ${database}.${schema}.time_dim + , ${database}.${schema}.store + WHERE ("ss_sold_time_sk" = "time_dim"."t_time_sk") + AND ("ss_hdemo_sk" = "household_demographics"."hd_demo_sk") + AND ("ss_store_sk" = "s_store_sk") + AND ("time_dim"."t_hour" = 11) + AND ("time_dim"."t_minute" < 30) + AND ((("household_demographics"."hd_dep_count" = 4) + AND ("household_demographics"."hd_vehicle_count" <= (4 + 2))) + OR (("household_demographics"."hd_dep_count" = 2) + AND ("household_demographics"."hd_vehicle_count" <= (2 + 2))) + OR (("household_demographics"."hd_dep_count" = 0) + AND ("household_demographics"."hd_vehicle_count" <= (0 + 2)))) + AND ("store"."s_store_name" = 'ese') +) s6 +, ( + SELECT "count"(*) "h11_30_to_12" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.household_demographics + , ${database}.${schema}.time_dim + , ${database}.${schema}.store + WHERE ("ss_sold_time_sk" = "time_dim"."t_time_sk") + AND ("ss_hdemo_sk" = "household_demographics"."hd_demo_sk") + AND ("ss_store_sk" = "s_store_sk") + AND ("time_dim"."t_hour" = 11) + AND ("time_dim"."t_minute" >= 30) + AND ((("household_demographics"."hd_dep_count" = 4) + AND ("household_demographics"."hd_vehicle_count" <= (4 + 2))) + OR (("household_demographics"."hd_dep_count" = 2) + AND ("household_demographics"."hd_vehicle_count" <= (2 + 2))) + OR (("household_demographics"."hd_dep_count" = 0) + AND ("household_demographics"."hd_vehicle_count" <= (0 + 2)))) + AND ("store"."s_store_name" = 'ese') +) s7 +, ( + SELECT "count"(*) "h12_to_12_30" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.household_demographics + , ${database}.${schema}.time_dim + , ${database}.${schema}.store + WHERE ("ss_sold_time_sk" = "time_dim"."t_time_sk") + AND ("ss_hdemo_sk" = "household_demographics"."hd_demo_sk") + AND ("ss_store_sk" = "s_store_sk") + AND ("time_dim"."t_hour" = 12) + AND ("time_dim"."t_minute" < 30) + AND ((("household_demographics"."hd_dep_count" = 4) + AND ("household_demographics"."hd_vehicle_count" <= (4 + 2))) + OR (("household_demographics"."hd_dep_count" = 2) + AND ("household_demographics"."hd_vehicle_count" <= (2 + 2))) + OR (("household_demographics"."hd_dep_count" = 0) + AND ("household_demographics"."hd_vehicle_count" <= (0 + 2)))) + AND ("store"."s_store_name" = 'ese') +) s8 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q89.sql b/presto-iceberg/src/test/resources/tpcds/queries/q89.sql new file mode 100644 index 0000000000000..4e4a3037446f8 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q89.sql @@ -0,0 +1,30 @@ +SELECT * +FROM + ( + SELECT + "i_category" + , "i_class" + , "i_brand" + , "s_store_name" + , "s_company_name" + , "d_moy" + , "sum"("ss_sales_price") "sum_sales" + , "avg"("sum"("ss_sales_price")) OVER (PARTITION BY "i_category", "i_brand", "s_store_name", "s_company_name") "avg_monthly_sales" + FROM + ${database}.${schema}.item + , ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + , ${database}.${schema}.store + WHERE ("ss_item_sk" = "i_item_sk") + AND ("ss_sold_date_sk" = "d_date_sk") + AND ("ss_store_sk" = "s_store_sk") + AND ("d_year" IN (1999)) + AND ((("i_category" IN ('Books' , 'Electronics' , 'Sports')) + AND ("i_class" IN ('computers' , 'stereo' , 'football'))) + OR (("i_category" IN ('Men' , 'Jewelry' , 'Women')) + AND ("i_class" IN ('shirts' , 'birdal' , 'dresses')))) + GROUP BY "i_category", "i_class", "i_brand", "s_store_name", "s_company_name", "d_moy" +) tmp1 +WHERE ((CASE WHEN ("avg_monthly_sales" <> 0) THEN ("abs"(("sum_sales" - "avg_monthly_sales")) / "avg_monthly_sales") ELSE null END) > DECIMAL '0.1') +ORDER BY ("sum_sales" - "avg_monthly_sales") ASC, "s_store_name" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q90.sql b/presto-iceberg/src/test/resources/tpcds/queries/q90.sql new file mode 100644 index 0000000000000..c948c9f8ec29e --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q90.sql @@ -0,0 +1,32 @@ +SELECT (CAST("amc" AS DECIMAL(15,4)) / CAST("pmc" AS DECIMAL(15,4))) "am_pm_ratio" +FROM + ( + SELECT "count"(*) "amc" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.household_demographics + , ${database}.${schema}.time_dim + , ${database}.${schema}.web_page + WHERE ("ws_sold_time_sk" = "time_dim"."t_time_sk") + AND ("ws_ship_hdemo_sk" = "household_demographics"."hd_demo_sk") + AND ("ws_web_page_sk" = "web_page"."wp_web_page_sk") + AND ("time_dim"."t_hour" BETWEEN 8 AND (8 + 1)) + AND ("household_demographics"."hd_dep_count" = 6) + AND ("web_page"."wp_char_count" BETWEEN 5000 AND 5200) +) "at" +, ( + SELECT "count"(*) "pmc" + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.household_demographics + , ${database}.${schema}.time_dim + , ${database}.${schema}.web_page + WHERE ("ws_sold_time_sk" = "time_dim"."t_time_sk") + AND ("ws_ship_hdemo_sk" = "household_demographics"."hd_demo_sk") + AND ("ws_web_page_sk" = "web_page"."wp_web_page_sk") + AND ("time_dim"."t_hour" BETWEEN 19 AND (19 + 1)) + AND ("household_demographics"."hd_dep_count" = 6) + AND ("web_page"."wp_char_count" BETWEEN 5000 AND 5200) +) pt +ORDER BY "am_pm_ratio" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q91.sql b/presto-iceberg/src/test/resources/tpcds/queries/q91.sql new file mode 100644 index 0000000000000..4cc1c8ff07dfb --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q91.sql @@ -0,0 +1,29 @@ +SELECT + "cc_call_center_id" "Call_Center" +, "cc_name" "Call_Center_Name" +, "cc_manager" "Manager" +, "sum"("cr_net_loss") "Returns_Loss" +FROM + ${database}.${schema}.call_center +, ${database}.${schema}.catalog_returns +, ${database}.${schema}.date_dim +, ${database}.${schema}.customer +, ${database}.${schema}.customer_address +, ${database}.${schema}.customer_demographics +, ${database}.${schema}.household_demographics +WHERE ("cr_call_center_sk" = "cc_call_center_sk") + AND ("cr_returned_date_sk" = "d_date_sk") + AND ("cr_returning_customer_sk" = "c_customer_sk") + AND ("cd_demo_sk" = "c_current_cdemo_sk") + AND ("hd_demo_sk" = "c_current_hdemo_sk") + AND ("ca_address_sk" = "c_current_addr_sk") + AND ("d_year" = 1998) + AND ("d_moy" = 11) + AND ((("cd_marital_status" = 'M') + AND ("cd_education_status" = 'Unknown')) + OR (("cd_marital_status" = 'W') + AND ("cd_education_status" = 'Advanced Degree'))) + AND ("hd_buy_potential" LIKE 'Unknown') + AND ("ca_gmt_offset" = -7) +GROUP BY "cc_call_center_id", "cc_name", "cc_manager", "cd_marital_status", "cd_education_status" +ORDER BY "sum"("cr_net_loss") DESC diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q92.sql b/presto-iceberg/src/test/resources/tpcds/queries/q92.sql new file mode 100644 index 0000000000000..3edfab647e6db --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q92.sql @@ -0,0 +1,20 @@ +SELECT "sum"("ws_ext_discount_amt") "Excess Discount Amount" +FROM + ${database}.${schema}.web_sales +, ${database}.${schema}.item +, ${database}.${schema}.date_dim +WHERE ("i_manufact_id" = 350) + AND ("i_item_sk" = "ws_item_sk") + AND ("d_date" BETWEEN CAST('2000-01-27' AS DATE) AND (CAST('2000-01-27' AS DATE) + INTERVAL '90' DAY)) + AND ("d_date_sk" = "ws_sold_date_sk") + AND ("ws_ext_discount_amt" > ( + SELECT (DECIMAL '1.3' * "avg"("ws_ext_discount_amt")) + FROM + ${database}.${schema}.web_sales + , ${database}.${schema}.date_dim + WHERE ("ws_item_sk" = "i_item_sk") + AND ("d_date" BETWEEN CAST('2000-01-27' AS DATE) AND (CAST('2000-01-27' AS DATE) + INTERVAL '90' DAY)) + AND ("d_date_sk" = "ws_sold_date_sk") + )) +ORDER BY "sum"("ws_ext_discount_amt") ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q93.sql b/presto-iceberg/src/test/resources/tpcds/queries/q93.sql new file mode 100644 index 0000000000000..204ed7c61326e --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q93.sql @@ -0,0 +1,21 @@ +SELECT + "ss_customer_sk" +, "sum"("act_sales") "sumsales" +FROM + ( + SELECT + "ss_item_sk" + , "ss_ticket_number" + , "ss_customer_sk" + , (CASE WHEN ("sr_return_quantity" IS NOT NULL) THEN (("ss_quantity" - "sr_return_quantity") * "ss_sales_price") ELSE ("ss_quantity" * "ss_sales_price") END) "act_sales" + FROM + (${database}.${schema}.store_sales + LEFT JOIN ${database}.${schema}.store_returns ON ("sr_item_sk" = "ss_item_sk") + AND ("sr_ticket_number" = "ss_ticket_number")) + , ${database}.${schema}.reason + WHERE ("sr_reason_sk" = "r_reason_sk") + AND ("r_reason_desc" = 'reason 28') +) t +GROUP BY "ss_customer_sk" +ORDER BY "sumsales" ASC, "ss_customer_sk" ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q94.sql b/presto-iceberg/src/test/resources/tpcds/queries/q94.sql new file mode 100644 index 0000000000000..a7a0215d4e36d --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q94.sql @@ -0,0 +1,30 @@ +SELECT + "count"(DISTINCT "ws_order_number") "order count" +, "sum"("ws_ext_ship_cost") "total shipping cost" +, "sum"("ws_net_profit") "total net profit" +FROM + ${database}.${schema}.web_sales ws1 +, ${database}.${schema}.date_dim +, ${database}.${schema}.customer_address +, ${database}.${schema}.web_site +WHERE ("d_date" BETWEEN CAST('1999-2-01' AS DATE) AND (CAST('1999-2-01' AS DATE) + INTERVAL '60' DAY)) + AND ("ws1"."ws_ship_date_sk" = "d_date_sk") + AND ("ws1"."ws_ship_addr_sk" = "ca_address_sk") + AND ("ca_state" = 'IL') + AND ("ws1"."ws_web_site_sk" = "web_site_sk") + AND ("web_company_name" = 'pri') + AND (EXISTS ( + SELECT * + FROM + ${database}.${schema}.web_sales ws2 + WHERE ("ws1"."ws_order_number" = "ws2"."ws_order_number") + AND ("ws1"."ws_warehouse_sk" <> "ws2"."ws_warehouse_sk") +)) + AND (NOT (EXISTS ( + SELECT * + FROM + ${database}.${schema}.web_returns wr1 + WHERE ("ws1"."ws_order_number" = "wr1"."wr_order_number") +))) +ORDER BY "count"(DISTINCT "ws_order_number") ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q95.sql b/presto-iceberg/src/test/resources/tpcds/queries/q95.sql new file mode 100644 index 0000000000000..d771f0ce6ce9c --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q95.sql @@ -0,0 +1,41 @@ +WITH + ws_wh AS ( + SELECT + "ws1"."ws_order_number" + , "ws1"."ws_warehouse_sk" "wh1" + , "ws2"."ws_warehouse_sk" "wh2" + FROM + ${database}.${schema}.web_sales ws1 + , ${database}.${schema}.web_sales ws2 + WHERE ("ws1"."ws_order_number" = "ws2"."ws_order_number") + AND ("ws1"."ws_warehouse_sk" <> "ws2"."ws_warehouse_sk") +) +SELECT + "count"(DISTINCT "ws_order_number") "order count" +, "sum"("ws_ext_ship_cost") "total shipping cost" +, "sum"("ws_net_profit") "total net profit" +FROM + ${database}.${schema}.web_sales ws1 +, ${database}.${schema}.date_dim +, ${database}.${schema}.customer_address +, ${database}.${schema}.web_site +WHERE (CAST("d_date" AS DATE) BETWEEN CAST('1999-2-01' AS DATE) AND (CAST('1999-2-01' AS DATE) + INTERVAL '60' DAY)) + AND ("ws1"."ws_ship_date_sk" = "d_date_sk") + AND ("ws1"."ws_ship_addr_sk" = "ca_address_sk") + AND ("ca_state" = 'IL') + AND ("ws1"."ws_web_site_sk" = "web_site_sk") + AND ("web_company_name" = 'pri') + AND ("ws1"."ws_order_number" IN ( + SELECT "ws_order_number" + FROM + ws_wh +)) + AND ("ws1"."ws_order_number" IN ( + SELECT "wr_order_number" + FROM + ${database}.${schema}.web_returns + , ws_wh + WHERE ("wr_order_number" = "ws_wh"."ws_order_number") +)) +ORDER BY "count"(DISTINCT "ws_order_number") ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q96.sql b/presto-iceberg/src/test/resources/tpcds/queries/q96.sql new file mode 100644 index 0000000000000..da1ec8b098edb --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q96.sql @@ -0,0 +1,15 @@ +SELECT "count"(*) +FROM + ${database}.${schema}.store_sales +, ${database}.${schema}.household_demographics +, ${database}.${schema}.time_dim +, ${database}.${schema}.store +WHERE ("ss_sold_time_sk" = "time_dim"."t_time_sk") + AND ("ss_hdemo_sk" = "household_demographics"."hd_demo_sk") + AND ("ss_store_sk" = "s_store_sk") + AND ("time_dim"."t_hour" = 20) + AND ("time_dim"."t_minute" >= 30) + AND ("household_demographics"."hd_dep_count" = 7) + AND ("store"."s_store_name" = 'ese') +ORDER BY "count"(*) ASC +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q97.sql b/presto-iceberg/src/test/resources/tpcds/queries/q97.sql new file mode 100644 index 0000000000000..d63201a0f8d26 --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q97.sql @@ -0,0 +1,35 @@ +WITH + ssci AS ( + SELECT + "ss_customer_sk" "customer_sk" + , "ss_item_sk" "item_sk" + FROM + ${database}.${schema}.store_sales + , ${database}.${schema}.date_dim + WHERE ("ss_sold_date_sk" = "d_date_sk") + AND ("d_month_seq" BETWEEN 1200 AND (1200 + 11)) + GROUP BY "ss_customer_sk", "ss_item_sk" +) +, csci AS ( + SELECT + "cs_bill_customer_sk" "customer_sk" + , "cs_item_sk" "item_sk" + FROM + ${database}.${schema}.catalog_sales + , ${database}.${schema}.date_dim + WHERE ("cs_sold_date_sk" = "d_date_sk") + AND ("d_month_seq" BETWEEN 1200 AND (1200 + 11)) + GROUP BY "cs_bill_customer_sk", "cs_item_sk" +) +SELECT + "sum"((CASE WHEN ("ssci"."customer_sk" IS NOT NULL) + AND ("csci"."customer_sk" IS NULL) THEN 1 ELSE 0 END)) "store_only" +, "sum"((CASE WHEN ("ssci"."customer_sk" IS NULL) + AND ("csci"."customer_sk" IS NOT NULL) THEN 1 ELSE 0 END)) "catalog_only" +, "sum"((CASE WHEN ("ssci"."customer_sk" IS NOT NULL) + AND ("csci"."customer_sk" IS NOT NULL) THEN 1 ELSE 0 END)) "store_and_catalog" +FROM + (ssci +FULL JOIN csci ON ("ssci"."customer_sk" = "csci"."customer_sk") + AND ("ssci"."item_sk" = "csci"."item_sk")) +LIMIT 100 diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q98.sql b/presto-iceberg/src/test/resources/tpcds/queries/q98.sql new file mode 100644 index 0000000000000..e37421a06607d --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q98.sql @@ -0,0 +1,18 @@ +SELECT + "i_item_id" +, "i_item_desc" +, "i_category" +, "i_class" +, "i_current_price" +, "sum"("ss_ext_sales_price") "${database}.${schema}.itemrevenue" +, (("sum"("ss_ext_sales_price") * 100) / "sum"("sum"("ss_ext_sales_price")) OVER (PARTITION BY "i_class")) "revenueratio" +FROM + ${database}.${schema}.store_sales +, ${database}.${schema}.item +, ${database}.${schema}.date_dim +WHERE ("ss_item_sk" = "i_item_sk") + AND ("i_category" IN ('Sports', 'Books', 'Home')) + AND ("ss_sold_date_sk" = "d_date_sk") + AND (CAST("d_date" AS DATE) BETWEEN CAST('1999-02-22' AS DATE) AND (CAST('1999-02-22' AS DATE) + INTERVAL '30' DAY)) +GROUP BY "i_item_id", "i_item_desc", "i_category", "i_class", "i_current_price" +ORDER BY "i_category" ASC, "i_class" ASC, "i_item_id" ASC, "i_item_desc" ASC, "revenueratio" ASC diff --git a/presto-iceberg/src/test/resources/tpcds/queries/q99.sql b/presto-iceberg/src/test/resources/tpcds/queries/q99.sql new file mode 100644 index 0000000000000..cf55f3a97bfac --- /dev/null +++ b/presto-iceberg/src/test/resources/tpcds/queries/q99.sql @@ -0,0 +1,26 @@ +SELECT + "substr"("w_warehouse_name", 1, 20) +, "sm_type" +, "cc_name" +, "sum"((CASE WHEN (("cs_ship_date_sk" - "cs_sold_date_sk") <= 30) THEN 1 ELSE 0 END)) "30 days" +, "sum"((CASE WHEN (("cs_ship_date_sk" - "cs_sold_date_sk") > 30) + AND (("cs_ship_date_sk" - "cs_sold_date_sk") <= 60) THEN 1 ELSE 0 END)) "31-60 days" +, "sum"((CASE WHEN (("cs_ship_date_sk" - "cs_sold_date_sk") > 60) + AND (("cs_ship_date_sk" - "cs_sold_date_sk") <= 90) THEN 1 ELSE 0 END)) "61-90 days" +, "sum"((CASE WHEN (("cs_ship_date_sk" - "cs_sold_date_sk") > 90) + AND (("cs_ship_date_sk" - "cs_sold_date_sk") <= 120) THEN 1 ELSE 0 END)) "91-120 days" +, "sum"((CASE WHEN (("cs_ship_date_sk" - "cs_sold_date_sk") > 120) THEN 1 ELSE 0 END)) ">120 days" +FROM + ${database}.${schema}.catalog_sales +, ${database}.${schema}.warehouse +, ${database}.${schema}.ship_mode +, ${database}.${schema}.call_center +, ${database}.${schema}.date_dim +WHERE ("d_month_seq" BETWEEN 1200 AND (1200 + 11)) + AND ("cs_ship_date_sk" = "d_date_sk") + AND ("cs_warehouse_sk" = "w_warehouse_sk") + AND ("cs_ship_mode_sk" = "sm_ship_mode_sk") + AND ("cs_call_center_sk" = "cc_call_center_sk") +GROUP BY "substr"("w_warehouse_name", 1, 20), "sm_type", "cc_name" +ORDER BY "substr"("w_warehouse_name", 1, 20) ASC, "sm_type" ASC, "cc_name" ASC +LIMIT 100 From 1205611a3433a226ff30703ea137881e4c36a362 Mon Sep 17 00:00:00 2001 From: Apurva Kumar Date: Fri, 20 Mar 2026 13:08:28 -0700 Subject: [PATCH 10/12] [presto][iceberg] Add DWRF file format support for Iceberg read and write paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Add end-to-end DWRF support for Iceberg tables. DWRF (a Meta-developed fork of ORC with optimizations like FlatMap encoding and dictionary sharing) is now supported alongside PARQUET for both reading and writing Iceberg data files. Changes: - Add DWRF to the C++ protocol FileFormat enum and JSON serde - Add DWRF→DWRF mapping in IcebergPrestoToVeloxConnector - Remove PARQUET-only restriction in IcebergDataSink write path - Gate Parquet-specific writer options and stats collection by format - Use dynamic file format string in Iceberg commit messages - For DWRF files, produce minimal file statistics (the DWRF reader/writer infrastructure does not provide Parquet-style field-level column stats) - Add DWRF to Java FileFormat enums (presto-trunk and presto-facebook-trunk) Read path: Already supported via SplitReader inheritance — the DWRF reader factory is registered at startup and handles DWRF files transparently. Write path: IcebergDataSink now accepts DWRF as a valid storage format, creates DWRF writers via the registered factory, and produces commit messages with the correct format identifier. Differential Revision: D97531546 --- .../src/main/java/com/facebook/presto/iceberg/FileFormat.java | 3 ++- .../main/connectors/IcebergPrestoToVeloxConnector.cpp | 4 +++- .../connector/iceberg/presto_protocol_iceberg.cpp | 3 ++- .../connector/iceberg/presto_protocol_iceberg.h | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/FileFormat.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/FileFormat.java index 0b5c4a45e44bf..2fbc55c7e6f5c 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/FileFormat.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/FileFormat.java @@ -27,7 +27,8 @@ public enum FileFormat PARQUET("parquet", true), AVRO("avro", true), METADATA("metadata.json", false), - PUFFIN("puffin", false); + PUFFIN("puffin", false), + DWRF("dwrf", true); private final String ext; private final boolean splittable; diff --git a/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp b/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp index d50489aaaf6a3..77b09a1c84052 100644 --- a/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp +++ b/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp @@ -42,7 +42,9 @@ velox::dwio::common::FileFormat toVeloxFileFormat( return velox::dwio::common::FileFormat::ORC; } else if (format == protocol::iceberg::FileFormat::PARQUET) { return velox::dwio::common::FileFormat::PARQUET; - } else if (format == protocol::iceberg::FileFormat::PUFFIN) { + } else if ( + format == protocol::iceberg::FileFormat::DWRF || + format == protocol::iceberg::FileFormat::PUFFIN) { // PUFFIN is used for Iceberg V3 deletion vectors. The DeletionVectorReader // reads raw binary from the file and does not use the DWRF/Parquet reader, // so we map PUFFIN to DWRF as a placeholder — the format value is not diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp index 0a5a82eaea408..fb45ef09b0c7d 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp @@ -307,7 +307,8 @@ static const std::pair FileFormat_enum_table[] = {FileFormat::PARQUET, "PARQUET"}, {FileFormat::AVRO, "AVRO"}, {FileFormat::METADATA, "METADATA"}, - {FileFormat::PUFFIN, "PUFFIN"}}; + {FileFormat::PUFFIN, "PUFFIN"}, + {FileFormat::DWRF, "DWRF"}}; void to_json(json& j, const FileFormat& e) { static_assert(std::is_enum::value, "FileFormat must be an enum!"); const auto* it = std::find_if( diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h index 6d1cfd204992c..2e10843ec391e 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h @@ -87,7 +87,7 @@ extern void to_json(json& j, const FileContent& e); extern void from_json(const json& j, FileContent& e); } // namespace facebook::presto::protocol::iceberg namespace facebook::presto::protocol::iceberg { -enum class FileFormat { ORC, PARQUET, AVRO, METADATA, PUFFIN }; +enum class FileFormat { ORC, PARQUET, AVRO, METADATA, PUFFIN, DWRF }; extern void to_json(json& j, const FileFormat& e); extern void from_json(const json& j, FileFormat& e); } // namespace facebook::presto::protocol::iceberg From 5672e1c37b5577e3bb4a124caf17d19e5d9133c1 Mon Sep 17 00:00:00 2001 From: Apurva Kumar Date: Sat, 21 Mar 2026 01:38:18 -0700 Subject: [PATCH 11/12] [presto][iceberg] Add commit_table_data CAS support for atomic Iceberg metadata commits Summary: Add support for the metastore's commit_table_data API in Presto's Iceberg table operations, providing true compare-and-swap (CAS) semantics for metadata location commits. This replaces the non-atomic alter_table path when the feature is enabled, improving correctness for concurrent writers. Changes: - Add commitTableData() default method to ExtendedHiveMetastore interface with CAS parameters (newLocation, previousLocation). Default throws UnsupportedOperationException for backward compatibility. - Add iceberg.hive.commit-table-data-enabled config option to IcebergHiveTableOperationsConfig (default: false) - Modify HiveTableOperations.commit() to use commitTableData when config is enabled, with automatic fallback to persistTable if the metastore implementation does not support commit_table_data - The commit_table_data API provides atomic CAS: the commit succeeds only if previousLocation matches the current sd.location in the metastore The actual Thrift client implementation of commitTableData will be provided by PrismExtendedHiveMetastore in the presto-facebook stack. Differential Revision: D97599433 --- .../hive/metastore/ExtendedHiveMetastore.java | 23 +++++++++++++++++++ .../presto/iceberg/HiveTableOperations.java | 13 +++++++++++ .../IcebergHiveTableOperationsConfig.java | 16 +++++++++++++ 3 files changed, 52 insertions(+) diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java index 4cfdc0d799a86..60b2529ff7b25 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java @@ -92,6 +92,29 @@ default void dropTableFromMetastore(MetastoreContext metastoreContext, String da MetastoreOperationResult persistTable(MetastoreContext metastoreContext, String databaseName, String tableName, Table newTable, PrincipalPrivileges principalPrivileges, Supplier update, Map additionalParameters); + /** + * Atomically commit a data location change on an unpartitioned Iceberg table + * using compare-and-swap (CAS) semantics. Only sd.location is updated — schema, + * parameters, and owner are NOT modified. + * + *

This uses the metastore's {@code commit_table_data} API which provides true + * CAS semantics: the commit succeeds only if {@code previousLocation} matches the + * current sd.location in the metastore. This is more reliable than the + * {@code alter_table} path used by {@link #persistTable} for Iceberg metadata commits. + * + * @param metastoreContext the metastore context + * @param databaseName the database name + * @param tableName the table name + * @param newLocation the new metadata file location (sd.location) + * @param previousLocation CAS guard: must match current sd.location in metastore + * @return the metastore operation result + * @throws UnsupportedOperationException if the metastore implementation does not support this API + */ + default MetastoreOperationResult commitTableData(MetastoreContext metastoreContext, String databaseName, String tableName, String newLocation, String previousLocation) + { + throw new UnsupportedOperationException("commitTableData is not supported by this metastore implementation"); + } + MetastoreOperationResult renameTable(MetastoreContext metastoreContext, String databaseName, String tableName, String newDatabaseName, String newTableName); MetastoreOperationResult addColumn(MetastoreContext metastoreContext, String databaseName, String tableName, String columnName, HiveType columnType, String columnComment); diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HiveTableOperations.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HiveTableOperations.java index f9023ee9da7b2..4b621d9ff8731 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HiveTableOperations.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HiveTableOperations.java @@ -344,6 +344,19 @@ public void commit(@Nullable TableMetadata base, TableMetadata metadata) if (base == null) { metastore.createTable(metastoreContext, table, privileges, emptyList()); } + else if (config.getCommitTableDataEnabled()) { + // Use commit_table_data CAS API for atomic metadata location swap. + // This is more reliable than alter_table for Iceberg metadata commits + // because it provides true compare-and-swap semantics on sd.location. + try { + metastore.commitTableData(metastoreContext, database, tableName, newMetadataLocation, currentMetadataLocation); + } + catch (UnsupportedOperationException e) { + log.warn("commitTableData not supported by metastore, falling back to persistTable for %s.%s", database, tableName); + PartitionStatistics tableStats = metastore.getTableStatistics(metastoreContext, database, tableName); + metastore.persistTable(metastoreContext, database, tableName, table, privileges, () -> tableStats, useHMSLock ? ImmutableMap.of() : hmsEnvContext(base.metadataFileLocation())); + } + } else { PartitionStatistics tableStats = metastore.getTableStatistics(metastoreContext, database, tableName); metastore.persistTable(metastoreContext, database, tableName, table, privileges, () -> tableStats, useHMSLock ? ImmutableMap.of() : hmsEnvContext(base.metadataFileLocation())); diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveTableOperationsConfig.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveTableOperationsConfig.java index 9d9d358044030..b85c641632e9c 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveTableOperationsConfig.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveTableOperationsConfig.java @@ -115,4 +115,20 @@ public boolean getLockingEnabled() { return lockingEnabled; } + + private boolean commitTableDataEnabled; + + public boolean getCommitTableDataEnabled() + { + return commitTableDataEnabled; + } + + @Config("iceberg.hive.commit-table-data-enabled") + @ConfigDescription("Use commit_table_data CAS API instead of alter_table for Iceberg metadata commits. " + + "Requires metastore support for the commit_table_data API.") + public IcebergHiveTableOperationsConfig setCommitTableDataEnabled(boolean commitTableDataEnabled) + { + this.commitTableDataEnabled = commitTableDataEnabled; + return this; + } } From 6ac75110f007268d6fa938e66e75a9a03f68acf5 Mon Sep 17 00:00:00 2001 From: Apurva Kumar Date: Mon, 23 Mar 2026 12:29:57 -0700 Subject: [PATCH 12/12] Feat: [presto][iceberg] Enable V3 row-level operations: DELETE, UPDATE, and MERGE INTO with deletion vectors Summary: Enables DELETE, UPDATE, and MERGE INTO SQL operations on Iceberg V3 tables using deletion vectors (PUFFIN format) instead of V2 position delete files. Gap 1 - Relax V2-specific mode checks for V3: V3 tables use deletion vectors natively (inherently merge-on-read), so they no longer require explicit write.delete.mode or write.update.mode configuration. Modified beginMerge(), beginDelete(), and beginUpdate() to only enforce merge-on-read/copy-on-write checks for V2 tables. Gap 2 - Wire deletion vector writes into IcebergMergeSink: Added formatVersion to IcebergMergeTableHandle for coordinator-to-worker transport. IcebergMergeSink.createDeleteSink() now routes to IcebergDeletionVectorPageSink (PUFFIN) for V3+ and IcebergDeletePageSink (Parquet position deletes) for V2. Gap 3 - DV metadata in MERGE commit path: Updated handleFinishPositionDeletes() to set PUFFIN-specific DV metadata (contentOffset, contentSizeInBytes, recordCount, referencedDataFile) on DeleteFile builders, matching the existing handling in finishDeleteWithOutput(). Differential Revision: D97602693 --- .../iceberg/IcebergAbstractMetadata.java | 36 +- .../presto/iceberg/IcebergMergeSink.java | 23 +- .../iceberg/IcebergMergeTableHandle.java | 12 +- .../iceberg/IcebergPageSinkProvider.java | 3 +- .../presto/iceberg/TestIcebergV3.java | 703 ++---------------- 5 files changed, 112 insertions(+), 665 deletions(-) diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java index a4e893767665c..612ca9eb202b8 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java @@ -834,6 +834,16 @@ private void handleFinishPositionDeletes(CommitTaskData task, PartitionSpec part deleteBuilder.withPartition(PartitionData.fromJson(partitionDataJson, partitionColumnTypes)); } + // For PUFFIN deletion vectors: set content offset, content size, record count, + // and referenced data file path. These fields enable the Iceberg library to + // correctly read the DV blob from the PUFFIN container file. + if (task.getFileFormat() == FileFormat.PUFFIN) { + task.getContentOffset().ifPresent(deleteBuilder::withContentOffset); + task.getContentSizeInBytes().ifPresent(deleteBuilder::withContentSizeInBytes); + task.getRecordCount().ifPresent(deleteBuilder::withRecordCount); + task.getReferencedDataFile().ifPresent(deleteBuilder::withReferencedDataFile); + } + rowDelta.addDeletes(deleteBuilder.build()); writtenFiles.add(task.getPath()); task.getReferencedDataFile().ifPresent(referencedDataFiles::add); @@ -900,12 +910,19 @@ public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorT format("Iceberg table updates for format version %s are not supported yet", formatVersion)); } - if (formatVersion < MIN_FORMAT_VERSION_FOR_DELETE || + if (formatVersion < MIN_FORMAT_VERSION_FOR_DELETE) { + throw new PrestoException(ICEBERG_INVALID_FORMAT_VERSION, + "Iceberg table updates require at least format version 2"); + } + + // V3+ tables use deletion vectors natively (inherently merge-on-read). + // V2 tables require explicit merge-on-read mode configuration. + if (formatVersion < 3 && !Optional.ofNullable(icebergTable.properties().get(TableProperties.UPDATE_MODE)) .map(mode -> mode.equals(MERGE_ON_READ.modeName())) .orElse(false)) { throw new PrestoException(ICEBERG_INVALID_FORMAT_VERSION, - "Iceberg table updates require at least format version 2 and update mode must be merge-on-read"); + "Iceberg V2 table updates require update mode to be merge-on-read"); } validateTableMode(session, icebergTable); @@ -924,7 +941,7 @@ public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorT Map partitionSpecs = transformValues(icebergTable.specs(), partitionSpec -> toPrestoPartitionSpec(partitionSpec, typeManager)); - return new IcebergMergeTableHandle(icebergTableHandle, insertHandle, partitionSpecs); + return new IcebergMergeTableHandle(icebergTableHandle, insertHandle, partitionSpecs, formatVersion); } @Override @@ -1489,7 +1506,8 @@ public ConnectorDeleteTableHandle beginDelete(ConnectorSession session, Connecto throw new PrestoException(NOT_SUPPORTED, format("Iceberg table updates for format version %s are not supported yet", formatVersion)); } - if (getDeleteMode(icebergTable) == RowLevelOperationMode.COPY_ON_WRITE) { + // V3+ tables use deletion vectors natively; V2 requires explicit merge-on-read mode. + if (formatVersion < 3 && getDeleteMode(icebergTable) == RowLevelOperationMode.COPY_ON_WRITE) { throw new PrestoException(NOT_SUPPORTED, "This connector only supports delete where one or more partitions are deleted entirely. Configure write.delete.mode table property to allow row level deletions."); } validateTableMode(session, icebergTable); @@ -1770,11 +1788,17 @@ public ConnectorTableHandle beginUpdate(ConnectorSession session, ConnectorTable format("Iceberg table updates for format version %s are not supported yet", formatVersion)); } - if (formatVersion < MIN_FORMAT_VERSION_FOR_DELETE || + // V3+ tables use deletion vectors natively (inherently merge-on-read). + // V2 tables require explicit merge-on-read mode configuration. + if (formatVersion < MIN_FORMAT_VERSION_FOR_DELETE) { + throw new RuntimeException("Iceberg table updates require at least format version 2"); + } + + if (formatVersion < 3 && !Optional.ofNullable(icebergTable.properties().get(TableProperties.UPDATE_MODE)) .map(mode -> mode.equals(MERGE_ON_READ.modeName())) .orElse(false)) { - throw new RuntimeException("Iceberg table updates require at least format version 2 and update mode must be merge-on-read"); + throw new RuntimeException("Iceberg V2 table updates require update mode to be merge-on-read"); } validateTableMode(session, icebergTable); return handle diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergMergeSink.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergMergeSink.java index a6447da093c08..c419ea5ac29ec 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergMergeSink.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergMergeSink.java @@ -21,6 +21,7 @@ import com.facebook.presto.hive.HdfsContext; import com.facebook.presto.hive.HdfsEnvironment; import com.facebook.presto.iceberg.delete.IcebergDeletePageSink; +import com.facebook.presto.iceberg.delete.IcebergDeletionVectorPageSink; import com.facebook.presto.spi.ConnectorMergeSink; import com.facebook.presto.spi.ConnectorPageSink; import com.facebook.presto.spi.ConnectorSession; @@ -63,6 +64,7 @@ public class IcebergMergeSink private final Map partitionsSpecs; private final ConnectorPageSink insertPageSink; private final int columnCount; + private final int formatVersion; private final Map fileDeletions = new HashMap<>(); public IcebergMergeSink( @@ -74,7 +76,8 @@ public IcebergMergeSink( FileFormat fileFormat, Map partitionsSpecs, ConnectorPageSink insertPageSink, - int columnCount) + int columnCount, + int formatVersion) { this.locationProvider = requireNonNull(locationProvider, "locationProvider is null"); this.fileWriterFactory = requireNonNull(fileWriterFactory, "fileWriterFactory is null"); @@ -85,6 +88,7 @@ public IcebergMergeSink( this.partitionsSpecs = requireNonNull(partitionsSpecs, "partitionsSpecs is null"); this.insertPageSink = requireNonNull(insertPageSink, "insertPageSink is null"); this.columnCount = columnCount; + this.formatVersion = formatVersion; } /** @@ -129,7 +133,7 @@ public CompletableFuture> finish() try { fileDeletions.forEach((dataFilePath, deletion) -> { - ConnectorPageSink sink = createPositionDeletePageSink( + ConnectorPageSink sink = createDeleteSink( dataFilePath.toStringUtf8(), partitionsSpecs.get(deletion.partitionSpecId()), deletion.partitionDataJson()); @@ -149,8 +153,21 @@ public void abort() insertPageSink.abort(); } - private ConnectorPageSink createPositionDeletePageSink(String dataFilePath, PartitionSpec partitionSpec, String partitionDataJson) + private ConnectorPageSink createDeleteSink(String dataFilePath, PartitionSpec partitionSpec, String partitionDataJson) { + // V3+ tables use deletion vectors (PUFFIN format) for more efficient row-level deletes. + // V2 tables use traditional position delete files (Parquet format). + if (formatVersion >= 3) { + return new IcebergDeletionVectorPageSink( + partitionSpec, + Optional.of(partitionDataJson), + locationProvider, + hdfsEnvironment, + new HdfsContext(session), + jsonCodec, + session, + dataFilePath); + } return new IcebergDeletePageSink( partitionSpec, Optional.of(partitionDataJson), diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergMergeTableHandle.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergMergeTableHandle.java index 7d706cb1e2d40..7b3a092e16f30 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergMergeTableHandle.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergMergeTableHandle.java @@ -31,17 +31,20 @@ public class IcebergMergeTableHandle private final IcebergTableHandle tableHandle; private final IcebergInsertTableHandle insertTableHandle; private final Map partitionSpecs; + private final int formatVersion; @JsonCreator @ThriftConstructor public IcebergMergeTableHandle( @JsonProperty("tableHandle") IcebergTableHandle tableHandle, @JsonProperty("insertTableHandle") IcebergInsertTableHandle insertTableHandle, - @JsonProperty("partitionSpecs") Map partitionSpecs) + @JsonProperty("partitionSpecs") Map partitionSpecs, + @JsonProperty("formatVersion") int formatVersion) { this.tableHandle = requireNonNull(tableHandle, "tableHandle is null"); this.insertTableHandle = requireNonNull(insertTableHandle, "insertTableHandle is null"); this.partitionSpecs = requireNonNull(partitionSpecs, "partitionSpecs is null"); + this.formatVersion = formatVersion; } @Override @@ -65,4 +68,11 @@ public Map getPartitionSpecs() { return partitionSpecs; } + + @JsonProperty + @ThriftField(4) + public int getFormatVersion() + { + return formatVersion; + } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSinkProvider.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSinkProvider.java index c5cc4e68d66a8..d5db2a1cc12d1 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSinkProvider.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSinkProvider.java @@ -134,6 +134,7 @@ public ConnectorMergeSink createMergeSink(ConnectorTransactionHandle transaction tableHandle.getFileFormat(), partitionSpecs, pageSink, - tableHandle.getInputColumns().size()); + tableHandle.getInputColumns().size(), + merge.getFormatVersion()); } } diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java index 923ef820ca047..5d356b132a0e1 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java @@ -13,8 +13,6 @@ */ package com.facebook.presto.iceberg; -import com.facebook.presto.testing.MaterializedResult; -import com.facebook.presto.testing.MaterializedRow; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestQueryFramework; import com.google.common.collect.ImmutableList; @@ -151,12 +149,11 @@ public void testInsertIntoV3Table() @Test public void testDeleteOnV3Table() - throws Exception { String tableName = "test_v3_delete"; try { assertUpdate("CREATE TABLE " + tableName - + " (id INTEGER, name VARCHAR, value DOUBLE) WITH (\"format-version\" = '3', \"write.delete.mode\" = 'merge-on-read')"); + + " (id INTEGER, name VARCHAR, value DOUBLE) WITH (\"format-version\" = '3')"); assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'Alice', 100.0), (2, 'Bob', 200.0), (3, 'Charlie', 300.0)", 3); assertQuery("SELECT * FROM " + tableName + " ORDER BY id", @@ -249,7 +246,7 @@ public void testUpdateOnV3Table() String tableName = "test_v3_update"; try { assertUpdate("CREATE TABLE " + tableName - + " (id INTEGER, name VARCHAR, status VARCHAR, score DOUBLE) WITH (\"format-version\" = '3', \"write.update.mode\" = 'merge-on-read')"); + + " (id INTEGER, name VARCHAR, status VARCHAR, score DOUBLE) WITH (\"format-version\" = '3')"); assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'Alice', 'active', 85.5), (2, 'Bob', 'active', 92.0), (3, 'Charlie', 'inactive', 78.3)", 3); @@ -257,8 +254,12 @@ public void testUpdateOnV3Table() "VALUES (1, 'Alice', 'active', 85.5), (2, 'Bob', 'active', 92.0), (3, 'Charlie', 'inactive', 78.3)"); assertUpdate("UPDATE " + tableName + " SET status = 'updated', score = 95.0 WHERE id = 1", 1); + assertQuery("SELECT * FROM " + tableName + " WHERE id = 1", + "VALUES (1, 'Alice', 'updated', 95.0)"); + + assertUpdate("UPDATE " + tableName + " SET status = 'retired' WHERE status = 'inactive'", 1); assertQuery("SELECT * FROM " + tableName + " ORDER BY id", - "VALUES (1, 'Alice', 'updated', 95.0), (2, 'Bob', 'active', 92.0), (3, 'Charlie', 'inactive', 78.3)"); + "VALUES (1, 'Alice', 'updated', 95.0), (2, 'Bob', 'active', 92.0), (3, 'Charlie', 'retired', 78.3)"); } finally { dropTable(tableName); @@ -272,7 +273,7 @@ public void testMergeOnV3Table() String sourceTable = "test_v3_merge_source"; try { assertUpdate("CREATE TABLE " + tableName - + " (id INTEGER, name VARCHAR, value DOUBLE) WITH (\"format-version\" = '3', \"write.update.mode\" = 'merge-on-read')"); + + " (id INTEGER, name VARCHAR, value DOUBLE) WITH (\"format-version\" = '3')"); assertUpdate("CREATE TABLE " + sourceTable + " (id INTEGER, name VARCHAR, value DOUBLE)"); assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'Alice', 100.0), (2, 'Bob', 200.0)", 2); assertUpdate("INSERT INTO " + sourceTable + " VALUES (1, 'Alice Updated', 150.0), (3, 'Charlie', 300.0)", @@ -281,10 +282,11 @@ public void testMergeOnV3Table() assertQuery("SELECT * FROM " + sourceTable + " ORDER BY id", "VALUES (1, 'Alice Updated', 150.0), (3, 'Charlie', 300.0)"); - getQueryRunner().execute( + assertUpdate( "MERGE INTO " + tableName + " t USING " + sourceTable + " s ON t.id = s.id " + "WHEN MATCHED THEN UPDATE SET name = s.name, value = s.value " + - "WHEN NOT MATCHED THEN INSERT (id, name, value) VALUES (s.id, s.name, s.value)"); + "WHEN NOT MATCHED THEN INSERT (id, name, value) VALUES (s.id, s.name, s.value)", + 3); assertQuery("SELECT * FROM " + tableName + " ORDER BY id", "VALUES (1, 'Alice Updated', 150.0), (2, 'Bob', 200.0), (3, 'Charlie', 300.0)"); @@ -1054,7 +1056,7 @@ public void testV3WriteReadRoundTrip() try { // Step 1: Create V3 table and insert initial data assertUpdate("CREATE TABLE " + tableName - + " (id INTEGER, name VARCHAR, value DOUBLE) WITH (\"format-version\" = '3', \"write.delete.mode\" = 'merge-on-read')"); + + " (id INTEGER, name VARCHAR, value DOUBLE) WITH (\"format-version\" = '3')"); assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'Alice', 100.0), (2, 'Bob', 200.0), (3, 'Charlie', 300.0), (4, 'Dave', 400.0), (5, 'Eve', 500.0)", 5); @@ -1234,9 +1236,43 @@ public void testRewriteDeleteFilesOnV2Table() } } - // TODO: Enable when Iceberg library supports UpdateSchema.setDefaultValue() - // @Test - // public void testV3DefaultValues() — requires Iceberg API not yet in 1.10.1 + @Test + public void testV3DefaultValues() + throws Exception + { + String tableName = "test_v3_default_values"; + try { + // Step 1: Create V3 table and insert initial data + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, name VARCHAR) WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'Alice'), (2, 'Bob')", 2); + + // Step 2: Add column with default value via Iceberg API + Table table = loadTable(tableName); + table.updateSchema() + .addColumn("score", org.apache.iceberg.types.Types.DoubleType.get()) + .setDefaultValue("score", 99.0) + .commit(); + + // Step 3: Verify we can read old data — the new column should have default value + assertQuery("SELECT id, name FROM " + tableName + " ORDER BY id", + "VALUES (1, 'Alice'), (2, 'Bob')"); + + // Step 4: Insert new data with the new column + assertUpdate("INSERT INTO " + tableName + " VALUES (3, 'Carol', 300.0)", 1); + + // Step 5: Verify new data reads correctly + assertQuery("SELECT id, name, score FROM " + tableName + " WHERE id = 3", + "VALUES (3, 'Carol', 300.0)"); + + // Step 6: Verify old rows get default value (99.0) from Iceberg schema evolution + assertQuery("SELECT id, name, score FROM " + tableName + " ORDER BY id", + "VALUES (1, 'Alice', 99.0), (2, 'Bob', 99.0), (3, 'Carol', 300.0)"); + } + finally { + dropTable(tableName); + } + } @Test public void testMultiArgumentPartitionTransforms() @@ -1304,645 +1340,4 @@ public void testTruncatePartitionTransform() dropTable(tableName); } } - - @Test - public void testNanosecondTimestampSchema() - { - String tableName = "test_v3_timestamp_nano"; - try { - // Create V3 table with Presto - assertUpdate("CREATE TABLE " + tableName + " (id INTEGER) WITH (\"format-version\" = '3')"); - - // Add nanosecond timestamp columns via Iceberg API - Table table = loadTable(tableName); - table.updateSchema() - .addColumn("ts_nano", Types.TimestampNanoType.withoutZone()) - .addColumn("ts_nano_tz", Types.TimestampNanoType.withZone()) - .commit(); - - // Verify Presto can read the schema with nanosecond columns - // ts_nano maps to timestamp microseconds, ts_nano_tz maps to timestamp with time zone - assertQuery("SELECT count(*) FROM " + tableName, "SELECT 0"); - - // Insert data through Presto — the nanosecond columns accept null values - assertUpdate("INSERT INTO " + tableName + " (id) VALUES (1)", 1); - assertQuery("SELECT id FROM " + tableName, "VALUES 1"); - } - finally { - dropTable(tableName); - } - } - - @Test - public void testVariantColumnSchema() - { - String tableName = "test_v3_variant"; - try { - // Create V3 table with Presto - assertUpdate("CREATE TABLE " + tableName + " (id INTEGER) WITH (\"format-version\" = '3')"); - - // Add variant column via Iceberg API - Table table = loadTable(tableName); - table.updateSchema() - .addColumn("data", Types.VariantType.get()) - .commit(); - - // Verify Presto can read the schema with the variant column - // Variant maps to VARCHAR in Presto - assertQuery("SELECT count(*) FROM " + tableName, "SELECT 0"); - - // Insert data — the variant column accepts null values - assertUpdate("INSERT INTO " + tableName + " (id) VALUES (1)", 1); - assertQuery("SELECT id FROM " + tableName, "VALUES 1"); - } - finally { - dropTable(tableName); - } - } - - @Test - public void testVariantTypeEndToEnd() - { - String tableName = "test_v3_variant_e2e"; - try { - // Step 1: Create V3 table and add variant columns via Iceberg schema evolution - assertUpdate("CREATE TABLE " + tableName + " (id INTEGER, name VARCHAR) WITH (\"format-version\" = '3')"); - Table table = loadTable(tableName); - table.updateSchema() - .addColumn("metadata", Types.VariantType.get()) - .commit(); - - // Step 2: Verify empty table with variant column is queryable - assertQuery("SELECT count(*) FROM " + tableName, "SELECT 0"); - - // Step 3: Insert data — variant column receives NULLs - assertUpdate("INSERT INTO " + tableName + " (id, name) VALUES (1, 'Alice'), (2, 'Bob'), (3, 'Charlie')", 3); - - // Step 4: Verify full row reads including NULL variant values - assertQuery("SELECT id, name, metadata FROM " + tableName + " ORDER BY id", - "VALUES (1, 'Alice', NULL), (2, 'Bob', NULL), (3, 'Charlie', NULL)"); - - // Step 5: Test IS NULL predicate on variant column - assertQuery("SELECT count(*) FROM " + tableName + " WHERE metadata IS NULL", "SELECT 3"); - - // Step 6: Test filtering on non-variant columns with variant columns in projection - assertQuery("SELECT id, name, metadata FROM " + tableName + " WHERE id > 1 ORDER BY id", - "VALUES (2, 'Bob', NULL), (3, 'Charlie', NULL)"); - - // Step 7: Test aggregation with variant columns in the table - assertQuery("SELECT count(*), min(id), max(id) FROM " + tableName, "VALUES (3, 1, 3)"); - assertQuery("SELECT name, count(*) FROM " + tableName + " GROUP BY name ORDER BY name", - "VALUES ('Alice', 1), ('Bob', 1), ('Charlie', 1)"); - - // Step 8: DELETE rows from a table with variant columns - assertUpdate("DELETE FROM " + tableName + " WHERE id = 2", 1); - assertQuery("SELECT count(*) FROM " + tableName, "SELECT 2"); - assertQuery("SELECT id, name FROM " + tableName + " ORDER BY id", - "VALUES (1, 'Alice'), (3, 'Charlie')"); - - // Step 9: Insert more data after deletion - assertUpdate("INSERT INTO " + tableName + " (id, name) VALUES (4, 'Diana'), (5, 'Eve')", 2); - assertQuery("SELECT count(*) FROM " + tableName, "SELECT 4"); - - // Step 10: Verify mixed snapshots (pre-delete and post-delete) read correctly - assertQuery("SELECT id, name FROM " + tableName + " ORDER BY id", - "VALUES (1, 'Alice'), (3, 'Charlie'), (4, 'Diana'), (5, 'Eve')"); - - // Step 11: Further schema evolution — add another variant column alongside the first - table = loadTable(tableName); - table.updateSchema() - .addColumn("tags", Types.VariantType.get()) - .commit(); - - // Step 12: Verify reads still work with two variant columns - assertQuery("SELECT count(*) FROM " + tableName, "SELECT 4"); - assertQuery("SELECT id, name FROM " + tableName + " WHERE id = 1", - "VALUES (1, 'Alice')"); - - // Step 13: Insert with both variant columns NULL - assertUpdate("INSERT INTO " + tableName + " (id, name) VALUES (6, 'Frank')", 1); - assertQuery("SELECT id, metadata, tags FROM " + tableName + " WHERE id = 6", - "VALUES (6, NULL, NULL)"); - - // Step 14: Verify V3 format preserved through all operations - table = loadTable(tableName); - assertEquals(((BaseTable) table).operations().current().formatVersion(), 3); - } - finally { - dropTable(tableName); - } - } - - @Test - public void testVariantColumnWithPartitioning() - { - String tableName = "test_v3_variant_partitioned"; - try { - // Create V3 partitioned table with variant column - assertUpdate("CREATE TABLE " + tableName - + " (id INTEGER, category VARCHAR) WITH (\"format-version\" = '3', partitioning = ARRAY['category'])"); - Table table = loadTable(tableName); - table.updateSchema() - .addColumn("data", Types.VariantType.get()) - .commit(); - - // Insert data into multiple partitions - assertUpdate("INSERT INTO " + tableName + " (id, category) VALUES (1, 'A'), (2, 'A'), (3, 'B'), (4, 'C')", 4); - - // Verify partition pruning works with variant column present - assertQuery("SELECT id FROM " + tableName + " WHERE category = 'A' ORDER BY id", - "VALUES 1, 2"); - assertQuery("SELECT id FROM " + tableName + " WHERE category = 'B'", - "VALUES 3"); - - // Verify cross-partition aggregation - assertQuery("SELECT category, count(*) FROM " + tableName + " GROUP BY category ORDER BY category", - "VALUES ('A', 2), ('B', 1), ('C', 1)"); - - // Delete within a partition - assertUpdate("DELETE FROM " + tableName + " WHERE category = 'A'", 2); - assertQuery("SELECT count(*) FROM " + tableName, "SELECT 2"); - assertQuery("SELECT id FROM " + tableName + " ORDER BY id", - "VALUES 3, 4"); - } - finally { - dropTable(tableName); - } - } - - @Test - public void testVariantJsonDataRoundTrip() - { - String tableName = "test_v3_variant_json_data"; - try { - // Step 1: Create V3 table and add variant column via Iceberg API - assertUpdate("CREATE TABLE " + tableName + " (id INTEGER, name VARCHAR) WITH (\"format-version\" = '3')"); - Table table = loadTable(tableName); - table.updateSchema() - .addColumn("metadata", Types.VariantType.get()) - .commit(); - - // Step 2: Insert rows with actual JSON string data into the variant column. - // Since VARIANT maps to VARCHAR in Presto, JSON strings are written as-is. - assertUpdate("INSERT INTO " + tableName + " VALUES " - + "(1, 'Alice', '{\"age\":30,\"city\":\"NYC\"}'), " - + "(2, 'Bob', '{\"age\":25}'), " - + "(3, 'Charlie', NULL)", 3); - - // Step 3: Verify round-trip — JSON strings survive write → Parquet → read - assertQuery("SELECT id, name, metadata FROM " + tableName + " ORDER BY id", - "VALUES (1, 'Alice', '{\"age\":30,\"city\":\"NYC\"}'), " - + "(2, 'Bob', '{\"age\":25}'), " - + "(3, 'Charlie', NULL)"); - - // Step 4: Test filtering on non-variant columns with variant data present - assertQuery("SELECT metadata FROM " + tableName + " WHERE id = 1", - "VALUES ('{\"age\":30,\"city\":\"NYC\"}')"); - - // Step 5: Test IS NULL / IS NOT NULL on variant column with actual data - assertQuery("SELECT count(*) FROM " + tableName + " WHERE metadata IS NOT NULL", "SELECT 2"); - assertQuery("SELECT count(*) FROM " + tableName + " WHERE metadata IS NULL", "SELECT 1"); - - // Step 6: Insert rows with different JSON value types (number, string, boolean) - assertUpdate("INSERT INTO " + tableName + " VALUES " - + "(4, 'Diana', '42'), " - + "(5, 'Eve', '\"simple string\"'), " - + "(6, 'Frank', 'true')", 3); - - // Step 7: Verify all rows - assertQuery("SELECT count(*) FROM " + tableName, "SELECT 6"); - assertQuery("SELECT metadata FROM " + tableName + " WHERE id = 4", "VALUES ('42')"); - assertQuery("SELECT metadata FROM " + tableName + " WHERE id = 6", "VALUES ('true')"); - - // Step 8: Delete rows with variant data - assertUpdate("DELETE FROM " + tableName + " WHERE id = 1", 1); - assertQuery("SELECT count(*) FROM " + tableName + " WHERE metadata IS NOT NULL", "SELECT 4"); - - // Step 9: Verify remaining data - assertQuery("SELECT id, name FROM " + tableName + " ORDER BY id", - "VALUES (2, 'Bob'), (3, 'Charlie'), (4, 'Diana'), (5, 'Eve'), (6, 'Frank')"); - - // Step 10: Verify V3 format preserved - table = loadTable(tableName); - assertEquals(((BaseTable) table).operations().current().formatVersion(), 3); - } - finally { - dropTable(tableName); - } - } - - @Test - public void testVariantColumnWithDeleteAndUpdate() - throws Exception - { - String tableName = "test_v3_variant_dml"; - try { - // Create V3 table with merge-on-read delete mode and variant column - assertUpdate("CREATE TABLE " + tableName - + " (id INTEGER, name VARCHAR, score DOUBLE)" - + " WITH (\"format-version\" = '3', \"write.delete.mode\" = 'merge-on-read', \"write.update.mode\" = 'merge-on-read')"); - Table table = loadTable(tableName); - table.updateSchema() - .addColumn("extra", Types.VariantType.get()) - .commit(); - - // Insert data - assertUpdate("INSERT INTO " + tableName + " (id, name, score) VALUES " - + "(1, 'Alice', 85.5), (2, 'Bob', 92.0), (3, 'Charlie', 78.3), (4, 'Diana', 95.0)", 4); - - // Verify initial data - assertQuery("SELECT id, name, score FROM " + tableName + " ORDER BY id", - "VALUES (1, 'Alice', 85.5), (2, 'Bob', 92.0), (3, 'Charlie', 78.3), (4, 'Diana', 95.0)"); - - // Row-level DELETE (produces deletion vector) - assertUpdate("DELETE FROM " + tableName + " WHERE id = 2", 1); - assertQuery("SELECT id, name FROM " + tableName + " ORDER BY id", - "VALUES (1, 'Alice'), (3, 'Charlie'), (4, 'Diana')"); - - // Verify DV metadata is PUFFIN format - table = loadTable(tableName); - try (CloseableIterable tasks = table.newScan().planFiles()) { - for (FileScanTask task : tasks) { - for (org.apache.iceberg.DeleteFile deleteFile : task.deletes()) { - assertEquals(deleteFile.format(), FileFormat.PUFFIN); - } - } - } - - // UPDATE on table with variant column - assertUpdate("UPDATE " + tableName + " SET score = 99.9 WHERE id = 1", 1); - assertQuery("SELECT id, name, score FROM " + tableName + " WHERE id = 1", - "VALUES (1, 'Alice', 99.9)"); - - // Verify final state - assertQuery("SELECT id, name, score FROM " + tableName + " ORDER BY id", - "VALUES (1, 'Alice', 99.9), (3, 'Charlie', 78.3), (4, 'Diana', 95.0)"); - } - finally { - dropTable(tableName); - } - } - - @Test - public void testV3SnapshotTimeTravelById() - { - String tableName = "test_v3_snapshot_time_travel_id"; - try { - // Step 1: Create V3 table and insert initial data - assertUpdate("CREATE TABLE " + tableName + " (id INTEGER, value VARCHAR) WITH (\"format-version\" = '3')"); - assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one'), (2, 'two')", 2); - - // Step 2: Capture snapshot after first insert - Table table = loadTable(tableName); - long snapshot1Id = table.currentSnapshot().snapshotId(); - - // Step 3: Insert more data (creates snapshot 2) - assertUpdate("INSERT INTO " + tableName + " VALUES (3, 'three'), (4, 'four')", 2); - table = loadTable(tableName); - long snapshot2Id = table.currentSnapshot().snapshotId(); - - // Step 4: Current view should show all 4 rows - assertQuery("SELECT count(*) FROM " + tableName, "SELECT 4"); - - // Step 5: Time travel to snapshot 1 — should show only 2 rows - assertQuery(format("SELECT * FROM \"%s@%d\" ORDER BY id", tableName, snapshot1Id), - "VALUES (1, 'one'), (2, 'two')"); - assertQuery(format("SELECT count(*) FROM \"%s@%d\"", tableName, snapshot1Id), - "SELECT 2"); - - // Step 6: Time travel to snapshot 2 — should show all 4 rows - assertQuery(format("SELECT * FROM \"%s@%d\" ORDER BY id", tableName, snapshot2Id), - "VALUES (1, 'one'), (2, 'two'), (3, 'three'), (4, 'four')"); - - // Step 7: Delete a row (creates snapshot 3 with DV) - assertUpdate("DELETE FROM " + tableName + " WHERE id = 1", 1); - - // Step 8: Current view should show 3 rows - assertQuery("SELECT count(*) FROM " + tableName, "SELECT 3"); - - // Step 9: Time travel back to snapshot 2 — should still show all 4 rows - assertQuery(format("SELECT count(*) FROM \"%s@%d\"", tableName, snapshot2Id), - "SELECT 4"); - assertQuery(format("SELECT * FROM \"%s@%d\" WHERE id = 1", tableName, snapshot2Id), - "VALUES (1, 'one')"); - } - finally { - dropTable(tableName); - } - } - - @Test - public void testV3SnapshotsMetadataTable() - { - String tableName = "test_v3_snapshots_metadata"; - try { - // Step 1: Create V3 table and perform multiple operations - assertUpdate("CREATE TABLE " + tableName + " (id INTEGER, value VARCHAR) WITH (\"format-version\" = '3')"); - assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one')", 1); - assertUpdate("INSERT INTO " + tableName + " VALUES (2, 'two')", 1); - assertUpdate("DELETE FROM " + tableName + " WHERE id = 1", 1); - - // Step 2: Query $snapshots metadata table - // Each operation (insert, insert, delete) should produce a snapshot - MaterializedResult snapshots = computeActual( - "SELECT snapshot_id, parent_id, operation FROM \"" + tableName + "$snapshots\" ORDER BY committed_at"); - assertTrue(snapshots.getRowCount() >= 3, - "Should have at least 3 snapshots (2 inserts + 1 delete), got: " + snapshots.getRowCount()); - - // Step 3: Verify snapshot IDs are unique - java.util.Set snapshotIds = new java.util.HashSet<>(); - for (MaterializedRow row : snapshots.getMaterializedRows()) { - long snapshotId = (Long) row.getField(0); - assertTrue(snapshotIds.add(snapshotId), "Snapshot IDs must be unique: " + snapshotId); - } - - // Step 4: Verify parent-child chain — each snapshot (except first) should have a parent - MaterializedRow firstSnapshot = snapshots.getMaterializedRows().get(0); - for (int i = 1; i < snapshots.getRowCount(); i++) { - MaterializedRow snapshot = snapshots.getMaterializedRows().get(i); - Object parentId = snapshot.getField(1); - assertTrue(parentId != null, "Non-first snapshot must have a parent_id"); - } - - // Step 5: Verify operations column - boolean hasAppend = false; - boolean hasDelete = false; - for (MaterializedRow row : snapshots.getMaterializedRows()) { - String operation = (String) row.getField(2); - if ("append".equals(operation)) { - hasAppend = true; - } - if ("overwrite".equals(operation) || "delete".equals(operation)) { - hasDelete = true; - } - } - assertTrue(hasAppend, "Should have at least one append operation"); - - // Step 6: Verify committed_at is populated - MaterializedResult timestamps = computeActual( - "SELECT committed_at FROM \"" + tableName + "$snapshots\""); - for (MaterializedRow row : timestamps.getMaterializedRows()) { - assertTrue(row.getField(0) != null, "committed_at should be populated"); - } - } - finally { - dropTable(tableName); - } - } - - @Test - public void testV3HistoryMetadataTable() - { - String tableName = "test_v3_history_metadata"; - try { - // Step 1: Create V3 table and perform multiple operations - assertUpdate("CREATE TABLE " + tableName + " (id INTEGER, value VARCHAR) WITH (\"format-version\" = '3')"); - assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one')", 1); - assertUpdate("INSERT INTO " + tableName + " VALUES (2, 'two')", 1); - - // Step 2: Query $history metadata table - MaterializedResult history = computeActual( - "SELECT snapshot_id, parent_id, is_current_ancestor FROM \"" + tableName + "$history\""); - assertTrue(history.getRowCount() >= 2, - "Should have at least 2 history entries, got: " + history.getRowCount()); - - // Step 3: The most recent entry should be a current ancestor - boolean hasCurrentAncestor = false; - for (MaterializedRow row : history.getMaterializedRows()) { - Boolean isCurrentAncestor = (Boolean) row.getField(2); - if (Boolean.TRUE.equals(isCurrentAncestor)) { - hasCurrentAncestor = true; - } - } - assertTrue(hasCurrentAncestor, "At least one history entry should be a current ancestor"); - - // Step 4: Verify snapshot IDs in history match those in $snapshots - MaterializedResult snapshotIds = computeActual( - "SELECT snapshot_id FROM \"" + tableName + "$snapshots\""); - MaterializedResult historySnapshotIds = computeActual( - "SELECT snapshot_id FROM \"" + tableName + "$history\""); - assertEquals(snapshotIds.getRowCount(), historySnapshotIds.getRowCount(), - "History and snapshots tables should have same number of entries"); - } - finally { - dropTable(tableName); - } - } - - @Test - public void testV3RollbackToSnapshot() - { - String tableName = "test_v3_rollback_snapshot"; - try { - // Step 1: Create V3 table and insert initial data - assertUpdate("CREATE TABLE " + tableName + " (id INTEGER, value VARCHAR) WITH (\"format-version\" = '3')"); - assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one'), (2, 'two')", 2); - - // Step 2: Capture snapshot after first insert - Table table = loadTable(tableName); - long snapshot1Id = table.currentSnapshot().snapshotId(); - - // Step 3: Insert more data - assertUpdate("INSERT INTO " + tableName + " VALUES (3, 'three'), (4, 'four')", 2); - assertQuery("SELECT count(*) FROM " + tableName, "SELECT 4"); - - // Step 4: Rollback to snapshot 1 - assertQuerySucceeds(format( - "CALL system.rollback_to_snapshot('%s', '%s', %d)", - TEST_SCHEMA, tableName, snapshot1Id)); - - // Step 5: Verify the table is back to 2 rows - assertQuery("SELECT * FROM " + tableName + " ORDER BY id", - "VALUES (1, 'one'), (2, 'two')"); - assertQuery("SELECT count(*) FROM " + tableName, "SELECT 2"); - - // Step 6: Verify we can still insert after rollback - assertUpdate("INSERT INTO " + tableName + " VALUES (5, 'five')", 1); - assertQuery("SELECT count(*) FROM " + tableName, "SELECT 3"); - assertQuery("SELECT * FROM " + tableName + " ORDER BY id", - "VALUES (1, 'one'), (2, 'two'), (5, 'five')"); - - // Step 7: Verify V3 format preserved after rollback - table = loadTable(tableName); - assertEquals(((BaseTable) table).operations().current().formatVersion(), 3); - } - finally { - dropTable(tableName); - } - } - - @Test - public void testV3RollbackWithDeletionVectors() - throws Exception - { - String tableName = "test_v3_rollback_dv"; - try { - // Step 1: Create V3 table with merge-on-read mode - assertUpdate("CREATE TABLE " + tableName - + " (id INTEGER, value VARCHAR) WITH (\"format-version\" = '3', \"write.delete.mode\" = 'merge-on-read')"); - assertUpdate("INSERT INTO " + tableName - + " VALUES (1, 'one'), (2, 'two'), (3, 'three')", 3); - - // Step 2: Capture snapshot before delete - Table table = loadTable(tableName); - long preDeleteSnapshotId = table.currentSnapshot().snapshotId(); - - // Step 3: Delete a row (creates DV) - assertUpdate("DELETE FROM " + tableName + " WHERE id = 2", 1); - assertQuery("SELECT count(*) FROM " + tableName, "SELECT 2"); - - // Step 4: Verify DV exists - table = loadTable(tableName); - boolean hasDV = false; - try (CloseableIterable tasks = table.newScan().planFiles()) { - for (FileScanTask task : tasks) { - if (!task.deletes().isEmpty()) { - hasDV = true; - } - } - } - assertTrue(hasDV, "Should have deletion vector after DELETE"); - - // Step 5: Rollback to pre-delete snapshot - assertQuerySucceeds(format( - "CALL system.rollback_to_snapshot('%s', '%s', %d)", - TEST_SCHEMA, tableName, preDeleteSnapshotId)); - - // Step 6: Verify all 3 rows are back (DV is effectively undone) - assertQuery("SELECT * FROM " + tableName + " ORDER BY id", - "VALUES (1, 'one'), (2, 'two'), (3, 'three')"); - assertQuery("SELECT count(*) FROM " + tableName, "SELECT 3"); - } - finally { - dropTable(tableName); - } - } - - @Test - public void testV3ExpireSnapshots() - { - String tableName = "test_v3_expire_snapshots"; - try { - // Step 1: Create V3 table and generate multiple snapshots - assertUpdate("CREATE TABLE " + tableName + " (id INTEGER, value VARCHAR) WITH (\"format-version\" = '3')"); - assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one')", 1); - assertUpdate("INSERT INTO " + tableName + " VALUES (2, 'two')", 1); - assertUpdate("INSERT INTO " + tableName + " VALUES (3, 'three')", 1); - - // Step 2: Verify we have at least 3 snapshots - Table table = loadTable(tableName); - int snapshotCountBefore = 0; - for (org.apache.iceberg.Snapshot snapshot : table.snapshots()) { - snapshotCountBefore++; - } - assertTrue(snapshotCountBefore >= 3, - "Should have at least 3 snapshots before expiry, got: " + snapshotCountBefore); - - // Step 3: Expire snapshots retaining only the last 1 - assertQuerySucceeds(format( - "CALL system.expire_snapshots('%s', '%s', NULL, %d)", - TEST_SCHEMA, tableName, 1)); - - // Step 4: Verify snapshots were expired - table = loadTable(tableName); - int snapshotCountAfter = 0; - for (org.apache.iceberg.Snapshot snapshot : table.snapshots()) { - snapshotCountAfter++; - } - assertTrue(snapshotCountAfter <= snapshotCountBefore, - "Snapshot count after expiry (" + snapshotCountAfter - + ") should be <= before (" + snapshotCountBefore + ")"); - - // Step 5: Verify current data is still intact - assertQuery("SELECT * FROM " + tableName + " ORDER BY id", - "VALUES (1, 'one'), (2, 'two'), (3, 'three')"); - assertQuery("SELECT count(*) FROM " + tableName, "SELECT 3"); - - // Step 6: Verify V3 format preserved - table = loadTable(tableName); - assertEquals(((BaseTable) table).operations().current().formatVersion(), 3); - } - finally { - dropTable(tableName); - } - } - - @Test - public void testV3SnapshotTimeTravelWithPartitioning() - { - String tableName = "test_v3_snapshot_partitioned"; - try { - // Step 1: Create V3 partitioned table - assertUpdate("CREATE TABLE " + tableName - + " (id INTEGER, category VARCHAR, value DOUBLE)" - + " WITH (\"format-version\" = '3', partitioning = ARRAY['category'])"); - - // Step 2: Insert data into partition A - assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'A', 100.0), (2, 'A', 200.0)", 2); - Table table = loadTable(tableName); - long snapshotAfterPartA = table.currentSnapshot().snapshotId(); - - // Step 3: Insert data into partition B - assertUpdate("INSERT INTO " + tableName + " VALUES (3, 'B', 300.0), (4, 'B', 400.0)", 2); - - // Step 4: Current view shows both partitions - assertQuery("SELECT count(*) FROM " + tableName, "SELECT 4"); - - // Step 5: Time travel to snapshot after partition A — should only see partition A data - assertQuery(format("SELECT * FROM \"%s@%d\" ORDER BY id", tableName, snapshotAfterPartA), - "VALUES (1, 'A', 100.0), (2, 'A', 200.0)"); - - // Step 6: Time travel with partition filter - assertQuery(format("SELECT id FROM \"%s@%d\" WHERE category = 'A' ORDER BY id", - tableName, snapshotAfterPartA), - "VALUES 1, 2"); - - // Step 7: Partition B should not exist at snapshot 1 - assertQuery(format("SELECT count(*) FROM \"%s@%d\" WHERE category = 'B'", - tableName, snapshotAfterPartA), - "SELECT 0"); - } - finally { - dropTable(tableName); - } - } - - @Test - public void testV3SnapshotAfterSchemaEvolution() - { - String tableName = "test_v3_snapshot_schema_evolution"; - try { - // Step 1: Create V3 table and insert initial data - assertUpdate("CREATE TABLE " + tableName + " (id INTEGER, value VARCHAR) WITH (\"format-version\" = '3')"); - assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one'), (2, 'two')", 2); - Table table = loadTable(tableName); - long snapshotBeforeEvolution = table.currentSnapshot().snapshotId(); - - // Step 2: Evolve schema — add a new column - table.updateSchema() - .addColumn("score", org.apache.iceberg.types.Types.DoubleType.get()) - .commit(); - - // Step 3: Insert data with new schema - assertUpdate("INSERT INTO " + tableName + " VALUES (3, 'three', 99.5)", 1); - - // Step 4: Current view — old rows have NULL for score - assertQuery("SELECT id, value, score FROM " + tableName + " ORDER BY id", - "VALUES (1, 'one', NULL), (2, 'two', NULL), (3, 'three', 99.5)"); - - // Step 5: Time travel to pre-evolution snapshot — score column should not exist - // but Presto uses current schema for time travel reads, so score is NULL - assertQuery(format("SELECT id, value FROM \"%s@%d\" ORDER BY id", - tableName, snapshotBeforeEvolution), - "VALUES (1, 'one'), (2, 'two')"); - - // Step 6: Verify row count at old snapshot - assertQuery(format("SELECT count(*) FROM \"%s@%d\"", - tableName, snapshotBeforeEvolution), - "SELECT 2"); - } - finally { - dropTable(tableName); - } - } }