Skip to content

API: Reject unknown type for required fields and validate defaults #12302

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rdblue merged 4 commits into from
Feb 18, 2025
Merged

API: Reject unknown type for required fields and validate defaults #12302

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cfe64b8
API: Reject unknown type for required fields, validate defaults.
rdblue Feb 17, 2025
1cb085e
Fix TestSchema.
rdblue Feb 17, 2025
34846cf
Fix TestPartitionSpecValidation.
rdblue Feb 17, 2025
88d998a
Update api/src/test/java/org/apache/iceberg/TestSchema.java
rdblue Feb 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion api/src/main/java/org/apache/iceberg/types/Types.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -680,6 +680,10 @@ private NestedField(
Literal<?> writeDefault) {
Preconditions.checkNotNull(name, "Name cannot be null");
Preconditions.checkNotNull(type, "Type cannot be null");
Preconditions.checkArgument(
isOptional || !type.equals(UnknownType.get()),
"Cannot create required field with unknown type: %s",
name);
this.isOptional = isOptional;
this.id = id;
this.name = name;
Expand All @@ -694,7 +698,10 @@ private static Literal<?> castDefault(Literal<?> defaultValue, Type type) {
throw new IllegalArgumentException(
String.format("Invalid default value for %s: %s (must be null)", type, defaultValue));
} else if (defaultValue != null) {
return defaultValue.to(type);
Literal<?> typedDefault = defaultValue.to(type);

@Fokko Fokko Feb 18, 2025

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thinking of it, would a NoneLiteral make sense 🤔

@rdblue rdblue Feb 18, 2025

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right now, we make a lot of assumptions that Literal does not contain null or NaN. That way, we don't need to implement 3-value boolean logic or worry about how to handle NaN in comparisons.

If you want to have a null default, then you don't set a default value for an optional column. And this also passes through null without a problem if the Literal was null.

Preconditions.checkArgument(
typedDefault != null, "Cannot cast default value to %s: %s", type, defaultValue);
return typedDefault;
}

return null;
Expand Down
2 changes: 1 addition & 1 deletion api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public class TestPartitionSpecValidation {
NestedField.required(5, "another_d", Types.TimestampType.withZone()),
NestedField.required(6, "s", Types.StringType.get()),
NestedField.required(7, "v", Types.VariantType.get()),
NestedField.required(8, "u", Types.UnknownType.get()));
NestedField.optional(8, "u", Types.UnknownType.get()));

@Test
public void testMultipleTimestampPartitions() {
Expand Down
54 changes: 52 additions & 2 deletions api/src/test/java/org/apache/iceberg/TestSchema.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.FieldSource;
Expand All @@ -41,8 +42,7 @@ public class TestSchema {
ImmutableList.of(
Types.TimestampNanoType.withoutZone(),
Types.TimestampNanoType.withZone(),
Types.VariantType.get(),
Types.UnknownType.get());
Types.VariantType.get());

private static final Schema INITIAL_DEFAULT_SCHEMA =
new Schema(
Expand Down Expand Up @@ -122,6 +122,56 @@ private static Stream<Arguments> supportedTypes() {
.mapToObj(supportedVersion -> Arguments.of(type, supportedVersion)));
}

@Test
public void testUnknownSupport() {
// this needs a different schema becuase it cannot be used in required fields
Comment thread
rdblue marked this conversation as resolved.
Outdated
Schema schemaWithUnknown =
new Schema(
Types.NestedField.required(1, "id", Types.LongType.get()),
Types.NestedField.optional(2, "top", Types.UnknownType.get()),
Types.NestedField.optional(
3, "arr", Types.ListType.ofOptional(4, Types.UnknownType.get())),
Types.NestedField.required(
5,
"struct",
Types.StructType.of(
Types.NestedField.optional(6, "inner_op", Types.UnknownType.get()),
Types.NestedField.optional(
7,
"inner_map",
Types.MapType.ofOptional(
8, 9, Types.StringType.get(), Types.UnknownType.get())),
Types.NestedField.optional(
10,
"struct_arr",
Types.StructType.of(
Types.NestedField.optional(11, "deep", Types.UnknownType.get()))))));

assertThatThrownBy(() -> Schema.checkCompatibility(schemaWithUnknown, 2))
.isInstanceOf(IllegalStateException.class)
.hasMessage(
"Invalid schema for v%s:\n"
+ "- Invalid type for top: %s is not supported until v%s\n"
+ "- Invalid type for arr.element: %s is not supported until v%s\n"
+ "- Invalid type for struct.inner_op: %s is not supported until v%s\n"
+ "- Invalid type for struct.inner_map.value: %s is not supported until v%s\n"
+ "- Invalid type for struct.struct_arr.deep: %s is not supported until v%s",
2,
Types.UnknownType.get(),
MIN_FORMAT_VERSIONS.get(Type.TypeID.UNKNOWN),
Types.UnknownType.get(),
MIN_FORMAT_VERSIONS.get(Type.TypeID.UNKNOWN),
Types.UnknownType.get(),
MIN_FORMAT_VERSIONS.get(Type.TypeID.UNKNOWN),
Types.UnknownType.get(),
MIN_FORMAT_VERSIONS.get(Type.TypeID.UNKNOWN),
Types.UnknownType.get(),
MIN_FORMAT_VERSIONS.get(Type.TypeID.UNKNOWN));

assertThatCode(() -> Schema.checkCompatibility(schemaWithUnknown, 3))
.doesNotThrowAnyException();
}

@ParameterizedTest
@MethodSource("supportedTypes")
public void testTypeSupported(Type type, int supportedVersion) {
Expand Down
43 changes: 43 additions & 0 deletions core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2439,4 +2439,47 @@ public void testMoveDeletedNestedStructFieldToFirst() {

assertThat(actual.asStruct()).isEqualTo(expected.asStruct());
}

@Test
public void testAddUnknown() {
Schema schema = new Schema(required(1, "id", Types.LongType.get()));
Schema expected =
new Schema(
required(1, "id", Types.LongType.get()), optional(2, "unk", Types.UnknownType.get()));

Schema actual =
new SchemaUpdate(schema, schema.highestFieldId())
.addColumn("unk", Types.UnknownType.get())
.apply();

assertThat(actual.asStruct()).isEqualTo(expected.asStruct());
}

@Test
public void testAddUnknownNonNullDefault() {
Schema schema = new Schema(required(1, "id", Types.LongType.get()));

assertThatThrownBy(
() ->
new SchemaUpdate(schema, schema.highestFieldId())
.allowIncompatibleChanges()
.addColumn("unk", Types.UnknownType.get(), Literal.of("string!"))
.apply())
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot cast default value to unknown: \"string!\"");
}

@Test
public void testAddRequiredUnknown() {
Schema schema = new Schema(required(1, "id", Types.LongType.get()));

assertThatThrownBy(
() ->
new SchemaUpdate(schema, schema.highestFieldId())
.allowIncompatibleChanges()
.addRequiredColumn("unk", Types.UnknownType.get())
.apply())
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot create required field with unknown type: unk");
}
}