Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,15 @@ default boolean isIdentity() {
return false;
}

/**
* Return whether this transform is the void transform.
*
* @return true if this is an void transform, false otherwise
*/
default boolean isVoid() {
return false;
}

/**
* Returns a human-readable String representation of a transformed value.
* <p>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,11 @@ public UnboundPredicate<Void> project(String name, BoundPredicate<S> predicate)
return null;
}

@Override
public boolean isVoid() {
return true;
}

@Override
public String toHumanString(Void value) {
return "null";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,9 @@ public PartitionSpec apply() {
// field IDs were not required for v1 and were assigned sequentially in each partition spec starting at 1,000.
// to maintain consistent field ids across partition specs in v1 tables, any partition field that is removed
// must be replaced with a null transform. null values are always allowed in partition data.
builder.add(field.sourceId(), field.fieldId(), field.name(), Transforms.alwaysNull());
// To avoid name conflict when add and remove same partition transform multiple times, field name will be
// replaced by field name append with field id.
builder.add(field.sourceId(), field.fieldId(), field.name() + "_" + field.fieldId(), Transforms.alwaysNull());
Comment thread
vinson0526 marked this conversation as resolved.
Outdated
Comment thread
vinson0526 marked this conversation as resolved.
Outdated
}
}

Expand Down Expand Up @@ -290,7 +292,9 @@ private static Map<Pair<Integer, String>, PartitionField> indexSpecByTransform(P
ImmutableMap.Builder<Pair<Integer, String>, PartitionField> builder = ImmutableMap.builder();
List<PartitionField> fields = spec.fields();
for (PartitionField field : fields) {
builder.put(Pair.of(field.sourceId(), field.transform().toString()), field);
if (!field.transform().isVoid()) {
Comment thread
vinson0526 marked this conversation as resolved.
Outdated
builder.put(Pair.of(field.sourceId(), field.transform().toString()), field);
}
}

return builder.build();
Expand Down
33 changes: 33 additions & 0 deletions core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java
Original file line number Diff line number Diff line change
Expand Up @@ -569,6 +569,39 @@ public void testRenameAndDelete() {
.renameField("shard", "id_bucket"));
}

@Test
public void testRemoveAndAddMultiTimes() {
PartitionSpec addFirstTime = new BaseUpdatePartitionSpec(formatVersion, UNPARTITIONED)
.addField(day("ts"))
.apply();
PartitionSpec removeFirstTime = new BaseUpdatePartitionSpec(formatVersion, addFirstTime)
.removeField(day("ts"))
.apply();
PartitionSpec addSecondTime = new BaseUpdatePartitionSpec(formatVersion, removeFirstTime)
.addField(day("ts"))
.apply();
PartitionSpec removeSecondTime = new BaseUpdatePartitionSpec(formatVersion, addSecondTime)
.removeField(day("ts"))
.apply();
PartitionSpec updated = new BaseUpdatePartitionSpec(formatVersion, removeSecondTime)
.addField(day("ts"))
.apply();

PartitionSpec v1Expected = PartitionSpec.builderFor(SCHEMA)
.alwaysNull("ts", "ts_day_1000")
.alwaysNull("ts", "ts_day_1001")
.day("ts")
.build();

V1Assert.assertEquals("Should match expected spec", v1Expected, updated);

PartitionSpec v2Expected = PartitionSpec.builderFor(SCHEMA)
.day("ts")
.build();

V2Assert.assertEquals("Should match expected spec", v2Expected, updated);
}

private static int id(String name) {
return SCHEMA.findField(name).fieldId();
}
Expand Down