Fix reading variant null values in Delta Lake

In cases where a value chunk includes both null and non-null
entries, the chunk must be treated as potentially nullable.

Author: chenjian2664

lib/trino-parquet/src/main/java/io/trino/parquet/ParquetTypeUtils.java

@@ -299,14 +299,14 @@ private static Optional<Field> constructField(Type type, ColumnIO columnIO, bool
             if (!(columnIO instanceof GroupColumnIO groupColumnIo)) {
                 throw new IllegalStateException("Expected columnIO to be GroupColumnIO but got %s".formatted(columnIO.getClass().getSimpleName()));
             }
-            Field valueField = constructField(VARBINARY, groupColumnIo.getChild(0), false).orElseThrow();
+            PrimitiveField valueField = (PrimitiveField) constructField(VARBINARY, groupColumnIo.getChild(0), false).orElseThrow();
             PrimitiveField metadataField = (PrimitiveField) constructField(VARBINARY, groupColumnIo.getChild(1), false).orElseThrow();
             return Optional.of(new VariantField(
                     type,
                     repetitionLevel,
                     definitionLevel,
                     required,
-                    valueField,
+                    new PrimitiveField(valueField.getType(), false, valueField.getDescriptor(), valueField.getId()),
                     // Mark the metadata field as optional, this is because the metadata field is not present when the actual Variant value is null
                     new PrimitiveField(metadataField.getType(), false, metadataField.getDescriptor(), metadataField.getId())));
         }

plugin/trino-delta-lake/src/test/resources/databricks154/test_variant_null/README.md

@@ -11,5 +11,6 @@ INSERT INTO test_variant_null values
 (1, parse_json('{"a":1}')), 
 (2, parse_json('{"a":2}')), 
 (3, parse_json('null')), 
-(4, NULL);
+(4, NULL),
+(5, parse_json('{"a":5}'));
 ```

plugin/trino-delta-lake/src/test/resources/databricks154/test_variant_null/_delta_log/00000000000000000000.json

@@ -1,3 +1,3 @@
-{"commitInfo":{"timestamp":1750987703896,"userId":"4222103729284476","userName":"[email protected]","operation":"CREATE TABLE","operationParameters":{"partitionBy":"[]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.enableDeletionVectors\":\"true\"}","statsOnLoad":false},"notebook":{"notebookId":"2325234880148729"},"clusterId":"1002-064054-nbosugsx","isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/15.4.x-scala2.12","txnId":"c677ecaa-0b37-4998-a698-7fe58fdddc2e"}}
-{"metaData":{"id":"7113656c-e75c-486e-bcca-35fb29c6e4ce","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"x\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableDeletionVectors":"true"},"createdTime":1750987703781}}
+{"commitInfo":{"timestamp":1752418845893,"userId":"4222103729284476","userName":"[email protected]","operation":"CREATE TABLE","operationParameters":{"partitionBy":"[]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.enableDeletionVectors\":\"true\"}","statsOnLoad":false},"notebook":{"notebookId":"2325234880148729"},"clusterId":"1002-064054-nbosugsx","isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/15.4.x-scala2.12","txnId":"7a8ee623-3291-4bb7-9b54-e1d7fa6f7bbb"}}
+{"metaData":{"id":"f363d3d3-6ad6-4228-a48a-53d4e60f58a3","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"x\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableDeletionVectors":"true"},"createdTime":1752418845102}}
 {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors","variantType-preview"],"writerFeatures":["deletionVectors","variantType-preview"]}}

plugin/trino-delta-lake/src/test/resources/databricks154/test_variant_null/_delta_log/00000000000000000001.json

@@ -1,2 +1,2 @@
-{"commitInfo":{"timestamp":1750987827929,"userId":"4222103729284476","userName":"[email protected]","operation":"WRITE","operationParameters":{"mode":"Append","statsOnLoad":false,"partitionBy":"[]"},"notebook":{"notebookId":"2325234880148729"},"clusterId":"1002-064054-nbosugsx","readVersion":0,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"4","numOutputBytes":"1082"},"tags":{"noRowsCopied":"true","restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/15.4.x-scala2.12","txnId":"f08b2495-6911-4949-b784-9c1ac49f96d9"}}
-{"add":{"path":"part-00000-370c7cfb-47f7-4c81-81bd-3ecd7b5b4c41-c000.snappy.parquet","partitionValues":{},"size":1082,"modificationTime":1750987828000,"dataChange":true,"stats":"{\"numRecords\":4,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0,\"x\":1},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1750987828000000","MIN_INSERTION_TIME":"1750987828000000","MAX_INSERTION_TIME":"1750987828000000","OPTIMIZE_TARGET_SIZE":"268435456"}}}
+{"commitInfo":{"timestamp":1752418883651,"userId":"4222103729284476","userName":"[email protected]","operation":"WRITE","operationParameters":{"mode":"Append","statsOnLoad":false,"partitionBy":"[]"},"notebook":{"notebookId":"2325234880148729"},"clusterId":"1002-064054-nbosugsx","readVersion":0,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"1094"},"tags":{"noRowsCopied":"true","restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/15.4.x-scala2.12","txnId":"57b16f5c-8c44-4349-b47e-7a70373f38e5"}}
+{"add":{"path":"part-00000-3dae12c4-61bc-4177-bd36-2c936db81e90-c000.snappy.parquet","partitionValues":{},"size":1094,"modificationTime":1752418883000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":5},\"nullCount\":{\"id\":0,\"x\":1},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1752418883000000","MIN_INSERTION_TIME":"1752418883000000","MAX_INSERTION_TIME":"1752418883000000","OPTIMIZE_TARGET_SIZE":"268435456"}}}