From 4996e9b43167cde812f1a851b8f8426bb7a0d29a Mon Sep 17 00:00:00 2001 From: Fokko Date: Tue, 18 Feb 2025 19:58:11 +0100 Subject: [PATCH 1/2] Update tests with `UnknownType` from `Required` to `Optional` --- .../apache/iceberg/types/TestTypeUtil.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java index 137d7c663068..078c0180b5e7 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java @@ -665,9 +665,9 @@ private static Stream testTypes() { @MethodSource("testTypes") public void testAssignIdsWithType(Type testType) { Types.StructType sourceType = - Types.StructType.of(required(0, "id", IntegerType.get()), required(1, "data", testType)); + Types.StructType.of(required(0, "id", IntegerType.get()), optional(1, "data", testType)); Type expectedType = - Types.StructType.of(required(10, "id", IntegerType.get()), required(11, "data", testType)); + Types.StructType.of(required(10, "id", IntegerType.get()), optional(11, "data", testType)); Type assignedType = TypeUtil.assignIds(sourceType, oldId -> oldId + 10); assertThat(assignedType).isEqualTo(expectedType); @@ -676,20 +676,20 @@ public void testAssignIdsWithType(Type testType) { @ParameterizedTest @MethodSource("testTypes") public void testAssignFreshIdsWithType(Type testType) { - Schema schema = new Schema(required(0, "id", IntegerType.get()), required(1, "data", testType)); + Schema schema = new Schema(required(0, "id", IntegerType.get()), optional(1, "data", testType)); Schema assignedSchema = TypeUtil.assignFreshIds(schema, new AtomicInteger(10)::incrementAndGet); Schema expectedSchema = - new Schema(required(11, "id", IntegerType.get()), required(12, "data", testType)); + new Schema(required(11, "id", IntegerType.get()), optional(12, "data", testType)); assertThat(assignedSchema.asStruct()).isEqualTo(expectedSchema.asStruct()); } @ParameterizedTest @MethodSource("testTypes") public void testReassignIdsWithType(Type testType) { - Schema schema = new Schema(required(0, "id", IntegerType.get()), required(1, "data", testType)); + Schema schema = new Schema(required(0, "id", IntegerType.get()), optional(1, "data", testType)); Schema sourceSchema = - new Schema(required(1, "id", IntegerType.get()), required(2, "data", testType)); + new Schema(required(1, "id", IntegerType.get()), optional(2, "data", testType)); Schema reassignedSchema = TypeUtil.reassignIds(schema, sourceSchema); assertThat(reassignedSchema.asStruct()).isEqualTo(sourceSchema.asStruct()); @@ -698,7 +698,7 @@ public void testReassignIdsWithType(Type testType) { @ParameterizedTest @MethodSource("testTypes") public void testIndexByIdWithType(Type testType) { - Schema schema = new Schema(required(0, "id", IntegerType.get()), required(1, "data", testType)); + Schema schema = new Schema(required(0, "id", IntegerType.get()), optional(1, "data", testType)); Map indexByIds = TypeUtil.indexById(schema.asStruct()); assertThat(indexByIds.get(1).type()).isEqualTo(testType); @@ -707,7 +707,7 @@ public void testIndexByIdWithType(Type testType) { @ParameterizedTest @MethodSource("testTypes") public void testIndexNameByIdWithType(Type testType) { - Schema schema = new Schema(required(0, "id", IntegerType.get()), required(1, "data", testType)); + Schema schema = new Schema(required(0, "id", IntegerType.get()), optional(1, "data", testType)); Map indexNameByIds = TypeUtil.indexNameById(schema.asStruct()); assertThat(indexNameByIds.get(1)).isEqualTo("data"); @@ -716,9 +716,9 @@ public void testIndexNameByIdWithType(Type testType) { @ParameterizedTest @MethodSource("testTypes") public void testProjectWithType(Type testType) { - Schema schema = new Schema(required(0, "id", IntegerType.get()), required(1, "data", testType)); + Schema schema = new Schema(required(0, "id", IntegerType.get()), optional(1, "data", testType)); - Schema expectedSchema = new Schema(required(1, "data", testType)); + Schema expectedSchema = new Schema(optional(1, "data", testType)); Schema projectedSchema = TypeUtil.project(schema, Sets.newHashSet(1)); assertThat(projectedSchema.asStruct()).isEqualTo(expectedSchema.asStruct()); } @@ -726,7 +726,7 @@ public void testProjectWithType(Type testType) { @ParameterizedTest @MethodSource("testTypes") public void testGetProjectedIdsWithType(Type testType) { - Schema schema = new Schema(required(0, "id", IntegerType.get()), required(1, "data", testType)); + Schema schema = new Schema(required(0, "id", IntegerType.get()), optional(1, "data", testType)); Set projectedIds = TypeUtil.getProjectedIds(schema); assertThat(Set.of(0, 1)).isEqualTo(projectedIds); @@ -735,10 +735,10 @@ public void testGetProjectedIdsWithType(Type testType) { @ParameterizedTest @MethodSource("testTypes") public void testReassignDocWithType(Type testType) { - Schema schema = new Schema(required(0, "id", IntegerType.get()), required(1, "data", testType)); + Schema schema = new Schema(required(0, "id", IntegerType.get()), optional(1, "data", testType)); Schema docSourceSchema = new Schema( - required(0, "id", IntegerType.get(), "id"), required(1, "data", testType, "data")); + required(0, "id", IntegerType.get(), "id"), optional(1, "data", testType, "data")); Schema reassignedSchema = TypeUtil.reassignDoc(schema, docSourceSchema); assertThat(reassignedSchema.asStruct()).isEqualTo(docSourceSchema.asStruct()); From 7481abd45dab02bfc74573e4900e23742a0bcd11 Mon Sep 17 00:00:00 2001 From: Fokko Date: Tue, 18 Feb 2025 20:24:20 +0100 Subject: [PATCH 2/2] Skip UnknownType as MapType key --- .../java/org/apache/iceberg/TestSchemaUnionByFieldName.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/src/test/java/org/apache/iceberg/TestSchemaUnionByFieldName.java b/core/src/test/java/org/apache/iceberg/TestSchemaUnionByFieldName.java index c6f9cfdb241a..aa478f85260e 100644 --- a/core/src/test/java/org/apache/iceberg/TestSchemaUnionByFieldName.java +++ b/core/src/test/java/org/apache/iceberg/TestSchemaUnionByFieldName.java @@ -122,6 +122,12 @@ public void testAddTopLevelListOfPrimitives() { @Test public void testAddTopLevelMapOfPrimitives() { for (Type primitiveType : primitiveTypes()) { + if (primitiveType.equals(UnknownType.get())) { + // The UnknownType has to be optional, and this conflicts with the map key that must be + // required + continue; + } + Schema newSchema = new Schema( optional(1, "aMap", Types.MapType.ofOptional(2, 3, primitiveType, primitiveType)));