Skip to content

[#8940] improvement(core): Support serialize/deserialize for distributionImpl and sortOrderImpl #8975

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jerryshao merged 7 commits into from
Nov 4, 2025
Merged

[#8940] improvement(core): Support serialize/deserialize for distributionImpl and sortOrderImpl #8975

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5d7c1e0
Support serialize/deserialize for distributionImpl and sortOrderImpl
yuqi1129 Oct 30, 2025
be2010e
fix
yuqi1129 Oct 30, 2025
ed94bb3
fix code format
yuqi1129 Oct 31, 2025
a8b3d11
Fix test error.
yuqi1129 Oct 31, 2025
29db690
Polish PRs
yuqi1129 Oct 31, 2025
93cf67a
Revert changes in code
yuqi1129 Nov 3, 2025
f47953f
Resolve comments
yuqi1129 Nov 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 21 additions & 84 deletions api/src/main/java/org/apache/gravitino/rel/indexes/Indexes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,8 @@
*/
package org.apache.gravitino.rel.indexes;

import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import java.io.IOException;
import java.util.List;
import java.util.Locale;
import com.google.common.base.Objects;
import java.util.Arrays;

/** Helper methods to create index to pass into Apache Gravitino. */
public class Indexes {
Expand Down Expand Up @@ -89,76 +76,7 @@ public static Index of(Index.IndexType indexType, String name, String[][] fieldN
.build();
}

/** Custom JSON serializer for Index objects. */
public static class IndexSerializer extends JsonSerializer<Index> {
@Override
public void serialize(Index value, JsonGenerator gen, SerializerProvider serializers)
throws IOException {
gen.writeStartObject();
gen.writeStringField("indexType", value.type().name().toUpperCase(Locale.ROOT));
if (null != value.name()) {
gen.writeStringField("name", value.name());
}
gen.writeFieldName("fieldNames");
gen.writeObject(value.fieldNames());
gen.writeEndObject();
}
}

/** Custom JSON deserializer for Index objects. */
public static class IndexDeserializer extends JsonDeserializer<Index> {

@Override
public Index deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
JsonNode node = p.getCodec().readTree(p);
Preconditions.checkArgument(
node != null && !node.isNull() && node.isObject(),
"Index must be a valid JSON object, but found: %s",
node);

IndexImpl.Builder builder = IndexImpl.builder();
Preconditions.checkArgument(
node.has("indexType"), "Cannot parse index from missing type: %s", node);
String indexType = getString("indexType", node);
builder.withIndexType(Index.IndexType.valueOf(indexType.toUpperCase(Locale.ROOT)));
if (node.has("name")) {
builder.withName(getString("name", node));
}
Preconditions.checkArgument(
node.has("fieldNames"), "Cannot parse index from missing field names: %s", node);
List<String[]> fieldNames = Lists.newArrayList();
node.get("fieldNames").forEach(field -> fieldNames.add(getStringArray((ArrayNode) field)));
builder.withFieldNames(fieldNames.toArray(new String[0][0]));
return builder.build();
}

private static String[] getStringArray(ArrayNode node) {
String[] array = new String[node.size()];
for (int i = 0; i < node.size(); i++) {
array[i] = node.get(i).asText();
}
return array;
}

private static String getString(String property, JsonNode node) {
Preconditions.checkArgument(node.has(property), "Cannot parse missing string: %s", property);
JsonNode pNode = node.get(property);
return convertToString(property, pNode);
}

private static String convertToString(String property, JsonNode pNode) {
Preconditions.checkArgument(
pNode != null && !pNode.isNull() && pNode.isTextual(),
"Cannot parse to a string value %s: %s",
property,
pNode);
return pNode.asText();
}
}

/** The user side implementation of the index. */
@JsonSerialize(using = IndexSerializer.class)
@JsonDeserialize(using = IndexDeserializer.class)
public static final class IndexImpl implements Index {
private final IndexType indexType;

Expand Down Expand Up @@ -203,6 +121,25 @@ public String[][] fieldNames() {
return fieldNames;
}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
IndexImpl index = (IndexImpl) o;
return indexType == index.indexType
&& Objects.equal(name, index.name)
&& Arrays.deepEquals(fieldNames, index.fieldNames);
}

@Override
public int hashCode() {
return Objects.hashCode(indexType, name, Arrays.hashCode(fieldNames));
}

/**
* @return the builder for creating a new instance of IndexImpl.
*/
Expand Down
57 changes: 0 additions & 57 deletions api/src/test/java/org/apache/gravitino/rel/TestIndex.java
View file
Open in desktop

This file was deleted.

52 changes: 49 additions & 3 deletions ...g/apache/gravitino/catalog/lakehouse/integration/test/CatalogGenericLakehouseLanceIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,22 @@
import org.apache.gravitino.integration.test.util.GravitinoITUtils;
import org.apache.gravitino.rel.Column;
import org.apache.gravitino.rel.Table;
import org.apache.gravitino.rel.expressions.NamedReference;
import org.apache.gravitino.rel.expressions.distributions.Distribution;
import org.apache.gravitino.rel.expressions.distributions.Distributions;
import org.apache.gravitino.rel.expressions.distributions.Strategy;
import org.apache.gravitino.rel.expressions.literals.Literals;
import org.apache.gravitino.rel.expressions.sorts.NullOrdering;
import org.apache.gravitino.rel.expressions.sorts.SortDirection;
import org.apache.gravitino.rel.expressions.sorts.SortOrder;
import org.apache.gravitino.rel.expressions.sorts.SortOrders;
import org.apache.gravitino.rel.expressions.transforms.Transform;
import org.apache.gravitino.rel.expressions.transforms.Transforms;
import org.apache.gravitino.rel.indexes.Index;
import org.apache.gravitino.rel.indexes.Index.IndexType;
import org.apache.gravitino.rel.indexes.Indexes;
import org.apache.gravitino.rel.partitions.Partitions;
import org.apache.gravitino.rel.partitions.RangePartition;
import org.apache.gravitino.rel.types.Types;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
Expand Down Expand Up @@ -176,6 +191,30 @@ public void testCreateLanceTable() throws InterruptedException {
properties = createProperties();
properties.put("format", "lance");

Distribution distribution =
Distributions.of(Strategy.EVEN, 10, NamedReference.field(LANCE_COL_NAME1));
SortOrder[] sortOrders =
new SortOrder[] {
SortOrders.of(
NamedReference.field(LANCE_COL_NAME2),
SortDirection.ASCENDING,
NullOrdering.NULLS_FIRST)
};
Index[] indexes =
new Index[] {
Indexes.of(IndexType.UNIQUE_KEY, "unique_index", new String[][] {{LANCE_COL_NAME3}})
};

RangePartition p1 =
Partitions.range(
"p1", Literals.stringLiteral("20220101"), Literals.NULL, Collections.emptyMap());
RangePartition p2 =
Partitions.range(
"p2", Literals.stringLiteral("20220301"), Literals.NULL, Collections.emptyMap());
Transform[] partitioning = {
Transforms.range(new String[] {LANCE_COL_NAME3}, new RangePartition[] {p1, p2})
};

createdTable =
catalog
.asTableCatalog()
Expand All @@ -184,9 +223,10 @@ public void testCreateLanceTable() throws InterruptedException {
columns,
TABLE_COMMENT,
properties,
Transforms.EMPTY_TRANSFORM,
null,
null);
partitioning,
distribution,
sortOrders,
indexes);
Assertions.assertEquals(createdTable.name(), tableName);
createdTableProperties = createdTable.properties();
Assertions.assertEquals("lance", createdTableProperties.get("format"));
Expand All @@ -198,6 +238,12 @@ public void testCreateLanceTable() throws InterruptedException {
Assertions.assertEquals(expectedTableLocation, createdTableProperties.get("location"));
Assertions.assertTrue(new File(expectedTableLocation).exists());

Table loadTable = catalog.asTableCatalog().loadTable(nameIdentifier);
Assertions.assertEquals(distribution, loadTable.distribution());
Assertions.assertArrayEquals(sortOrders, loadTable.sortOrder());
Assertions.assertArrayEquals(indexes, loadTable.index());
Assertions.assertArrayEquals(partitioning, loadTable.partitioning());

// Now try to load table
Table loadedTable = catalog.asTableCatalog().loadTable(nameIdentifier);
Assertions.assertEquals(createdTable.name(), loadedTable.name());
Expand Down
6 changes: 6 additions & 0 deletions common/src/main/java/org/apache/gravitino/json/JsonUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@
*/
package org.apache.gravitino.json;

import static org.apache.gravitino.dto.rel.expressions.FunctionArg.ArgType.FIELD;
import static org.apache.gravitino.dto.rel.expressions.FunctionArg.ArgType.FUNCTION;
import static org.apache.gravitino.dto.rel.expressions.FunctionArg.ArgType.LITERAL;

import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.core.JacksonException;
Expand Down Expand Up @@ -304,6 +308,8 @@ private static class AnyFieldMapperHolder {
new SimpleModule()
.addDeserializer(Type.class, new TypeDeserializer())
.addSerializer(Type.class, new TypeSerializer())
.addDeserializer(Partitioning.class, new PartitioningDeserializer())
.addSerializer(Partitioning.class, new PartitioningSerializer())
.addDeserializer(Expression.class, new ColumnDefaultValueDeserializer())
.addSerializer(Expression.class, new ColumnDefaultValueSerializer())
.addDeserializer(StatisticValue.class, new StatisticValueDeserializer())
Expand Down
Loading