Skip to content

Fix SetDigest, BingTile, Kdb Tree, Color types (de)serialization in client protocol #24612

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Jan 7, 2025
Merged

Fix SetDigest, BingTile, Kdb Tree, Color types (de)serialization in client protocol #24612

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
1ecefda
Synchronize types in (Client)StandardTypes
wendigo Jan 2, 2025
5af3f4f
Enumerate all types while decoding
wendigo Jan 2, 2025
c2ac79a
Fix deserialization of KDB_Tree and BingTile
wendigo Jan 2, 2025
c47bc20
Improve SetDigest serialization
wendigo Jan 2, 2025
83b4b57
Inline SetDigestType.NAME
wendigo Jan 2, 2025
db4ace5
Use StandardTypes.BING_TILE const
wendigo Jan 2, 2025
d5f7c0c
Use StandardTypes.GEOMETRY const
wendigo Jan 2, 2025
7594099
Use StandardTypes.KDB_TREE const
wendigo Jan 2, 2025
7fda4b0
Use StandardTypes.SPHERICAL_GEOMETRY const
wendigo Jan 2, 2025
14b9947
Fix deserialization of Color type
wendigo Jan 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions client/trino-client/src/main/java/io/trino/client/ClientStandardTypes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
*/
package io.trino.client;

// Should have the same types as in the StandardTypes
public final class ClientStandardTypes
{
public static final String BIGINT = "bigint";
Expand All @@ -26,6 +27,8 @@ public final class ClientStandardTypes
public static final String DOUBLE = "double";
public static final String HYPER_LOG_LOG = "HyperLogLog";
public static final String QDIGEST = "qdigest";
public static final String TDIGEST = "tdigest";
public static final String SET_DIGEST = "SetDigest";
public static final String P4_HYPER_LOG_LOG = "P4HyperLogLog";
public static final String INTERVAL_DAY_TO_SECOND = "interval day to second";
public static final String INTERVAL_YEAR_TO_MONTH = "interval year to month";
Expand All @@ -40,11 +43,14 @@ public final class ClientStandardTypes
public static final String ARRAY = "array";
public static final String MAP = "map";
public static final String JSON = "json";
public static final String JSON_2016 = "json2016";
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is an internal type, not meant to be exposed to clients

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not exposed. It's here so the files are the same (server/client ones) for easier comparison

public static final String IPADDRESS = "ipaddress";
public static final String UUID = "uuid";
public static final String GEOMETRY = "Geometry";
public static final String SPHERICAL_GEOGRAPHY = "SphericalGeography";
public static final String BING_TILE = "BingTile";
public static final String KDB_TREE = "KdbTree";
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same with this one. This type is just there to support some types of geospatial joins

public static final String COLOR = "color";

private ClientStandardTypes() {}
}
33 changes: 33 additions & 0 deletions client/trino-client/src/main/java/io/trino/client/JsonDecodingUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;

import java.io.IOException;
Expand All @@ -31,20 +32,27 @@
import static com.google.common.base.Verify.verify;
import static io.trino.client.ClientStandardTypes.ARRAY;
import static io.trino.client.ClientStandardTypes.BIGINT;
import static io.trino.client.ClientStandardTypes.BING_TILE;
import static io.trino.client.ClientStandardTypes.BOOLEAN;
import static io.trino.client.ClientStandardTypes.CHAR;
import static io.trino.client.ClientStandardTypes.COLOR;
import static io.trino.client.ClientStandardTypes.DATE;
import static io.trino.client.ClientStandardTypes.DECIMAL;
import static io.trino.client.ClientStandardTypes.DOUBLE;
import static io.trino.client.ClientStandardTypes.GEOMETRY;
import static io.trino.client.ClientStandardTypes.HYPER_LOG_LOG;
import static io.trino.client.ClientStandardTypes.INTEGER;
import static io.trino.client.ClientStandardTypes.INTERVAL_DAY_TO_SECOND;
import static io.trino.client.ClientStandardTypes.INTERVAL_YEAR_TO_MONTH;
import static io.trino.client.ClientStandardTypes.IPADDRESS;
import static io.trino.client.ClientStandardTypes.JSON;
import static io.trino.client.ClientStandardTypes.KDB_TREE;
import static io.trino.client.ClientStandardTypes.MAP;
import static io.trino.client.ClientStandardTypes.P4_HYPER_LOG_LOG;
import static io.trino.client.ClientStandardTypes.QDIGEST;
import static io.trino.client.ClientStandardTypes.REAL;
import static io.trino.client.ClientStandardTypes.ROW;
import static io.trino.client.ClientStandardTypes.SET_DIGEST;
import static io.trino.client.ClientStandardTypes.SMALLINT;
import static io.trino.client.ClientStandardTypes.SPHERICAL_GEOGRAPHY;
import static io.trino.client.ClientStandardTypes.TIME;
Expand All @@ -53,6 +61,7 @@
import static io.trino.client.ClientStandardTypes.TIME_WITH_TIME_ZONE;
import static io.trino.client.ClientStandardTypes.TINYINT;
import static io.trino.client.ClientStandardTypes.UUID;
import static io.trino.client.ClientStandardTypes.VARBINARY;
import static io.trino.client.ClientStandardTypes.VARCHAR;
import static java.lang.String.format;
import static java.util.Collections.unmodifiableList;
Expand All @@ -72,6 +81,7 @@ private JsonDecodingUtils() {}
private static final BooleanDecoder BOOLEAN_DECODER = new BooleanDecoder();
private static final StringDecoder STRING_DECODER = new StringDecoder();
private static final Base64Decoder BASE_64_DECODER = new Base64Decoder();
private static final ObjectDecoder OBJECT_DECODER = new ObjectDecoder();

public static TypeDecoder[] createTypeDecoders(List<Column> columns)
{
Expand Down Expand Up @@ -127,7 +137,16 @@ private static TypeDecoder createTypeDecoder(ClientTypeSignature signature)
case CHAR:
case GEOMETRY:
case SPHERICAL_GEOGRAPHY:
case COLOR:
return STRING_DECODER;
case KDB_TREE:
case BING_TILE:
return OBJECT_DECODER;
case QDIGEST:
case P4_HYPER_LOG_LOG:
case HYPER_LOG_LOG:
case SET_DIGEST:
case VARBINARY:
default:
return BASE_64_DECODER;
}
Expand Down Expand Up @@ -387,6 +406,7 @@ private Object decodeKey(String value)
case CHAR:
case GEOMETRY:
case SPHERICAL_GEOGRAPHY:
case BING_TILE:
return value;
default:
return Base64.getDecoder().decode(value);
Expand Down Expand Up @@ -447,6 +467,19 @@ public Object decode(JsonParser parser)
}
}

private static class ObjectDecoder
implements TypeDecoder
{
private final ObjectMapper objectMapper = new ObjectMapper();

@Override
public Object decode(JsonParser parser)
throws IOException
{
return objectMapper.readValue(parser, Object.class);
}
}

private static IllegalTokenException illegalToken(JsonParser parser)
{
throw new IllegalTokenException(parser);
Expand Down
3 changes: 2 additions & 1 deletion core/trino-main/src/main/java/io/trino/type/ColorType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import io.trino.spi.block.Block;
import io.trino.spi.connector.ConnectorSession;
import io.trino.spi.type.AbstractIntType;
import io.trino.spi.type.StandardTypes;
import io.trino.spi.type.TypeSignature;

import java.util.HexFormat;
Expand All @@ -26,7 +27,7 @@ public class ColorType
{
private static final HexFormat HEX_FORMAT = HexFormat.of();
public static final ColorType COLOR = new ColorType();
public static final String NAME = "color";
public static final String NAME = StandardTypes.COLOR;

private ColorType()
{
Expand Down
3 changes: 1 addition & 2 deletions core/trino-main/src/main/java/io/trino/type/TypeCoercion.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
import io.trino.spi.type.TypeSignature;
import io.trino.spi.type.TypeSignatureParameter;
import io.trino.spi.type.VarcharType;
import io.trino.type.setdigest.SetDigestType;

import java.util.List;
import java.util.Optional;
Expand Down Expand Up @@ -342,7 +341,7 @@ public Optional<Type> coerceTypeBase(Type sourceType, String resultTypeBase)
StandardTypes.TIMESTAMP,
StandardTypes.TIMESTAMP_WITH_TIME_ZONE,
StandardTypes.HYPER_LOG_LOG,
SetDigestType.NAME,
StandardTypes.SET_DIGEST,
StandardTypes.P4_HYPER_LOG_LOG,
StandardTypes.JSON,
StandardTypes.INTERVAL_YEAR_TO_MONTH,
Expand Down
3 changes: 2 additions & 1 deletion core/trino-main/src/main/java/io/trino/type/setdigest/BuildSetDigestAggregation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import io.trino.spi.function.OutputFunction;
import io.trino.spi.function.SqlType;
import io.trino.spi.function.TypeParameter;
import io.trino.spi.type.StandardTypes;

@AggregationFunction("make_set_digest")
public final class BuildSetDigestAggregation
Expand Down Expand Up @@ -64,7 +65,7 @@ public static void combine(SetDigestState state, SetDigestState otherState)
}
}

@OutputFunction(SetDigestType.NAME)
@OutputFunction(StandardTypes.SET_DIGEST)
public static void output(SetDigestState state, BlockBuilder out)
{
SERIALIZER.serialize(state, out);
Expand Down
5 changes: 3 additions & 2 deletions core/trino-main/src/main/java/io/trino/type/setdigest/MergeSetDigestAggregation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import io.trino.spi.function.InputFunction;
import io.trino.spi.function.OutputFunction;
import io.trino.spi.function.SqlType;
import io.trino.spi.type.StandardTypes;

import static io.trino.type.setdigest.SetDigestType.SET_DIGEST;

Expand All @@ -30,7 +31,7 @@ public final class MergeSetDigestAggregation
private MergeSetDigestAggregation() {}

@InputFunction
public static void input(SetDigestState state, @SqlType(SetDigestType.NAME) Slice value)
public static void input(SetDigestState state, @SqlType(StandardTypes.SET_DIGEST) Slice value)
{
SetDigest instance = SetDigest.newInstance(value);
merge(state, instance);
Expand All @@ -52,7 +53,7 @@ private static void merge(SetDigestState state, SetDigest instance)
}
}

@OutputFunction(SetDigestType.NAME)
@OutputFunction(StandardTypes.SET_DIGEST)
public static void output(SetDigestState state, BlockBuilder out)
{
if (state.getDigest() == null) {
Expand Down
8 changes: 4 additions & 4 deletions core/trino-main/src/main/java/io/trino/type/setdigest/SetDigestFunctions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,14 @@ private SetDigestFunctions() {}

@ScalarFunction
@SqlType(StandardTypes.BIGINT)
public static long cardinality(@SqlType(SetDigestType.NAME) Slice digest)
public static long cardinality(@SqlType(StandardTypes.SET_DIGEST) Slice digest)
{
return SetDigest.newInstance(digest).cardinality();
}

@ScalarFunction
@SqlType(StandardTypes.BIGINT)
public static long intersectionCardinality(@SqlType(SetDigestType.NAME) Slice slice1, @SqlType(SetDigestType.NAME) Slice slice2)
public static long intersectionCardinality(@SqlType(StandardTypes.SET_DIGEST) Slice slice1, @SqlType(StandardTypes.SET_DIGEST) Slice slice2)
{
SetDigest digest1 = SetDigest.newInstance(slice1);
SetDigest digest2 = SetDigest.newInstance(slice2);
Expand All @@ -65,7 +65,7 @@ public static long intersectionCardinality(@SqlType(SetDigestType.NAME) Slice sl

@ScalarFunction
@SqlType(StandardTypes.DOUBLE)
public static double jaccardIndex(@SqlType(SetDigestType.NAME) Slice slice1, @SqlType(SetDigestType.NAME) Slice slice2)
public static double jaccardIndex(@SqlType(StandardTypes.SET_DIGEST) Slice slice1, @SqlType(StandardTypes.SET_DIGEST) Slice slice2)
{
SetDigest digest1 = SetDigest.newInstance(slice1);
SetDigest digest2 = SetDigest.newInstance(slice2);
Expand All @@ -75,7 +75,7 @@ public static double jaccardIndex(@SqlType(SetDigestType.NAME) Slice slice1, @Sq

@ScalarFunction
@SqlType("map(bigint,smallint)")
public static SqlMap hashCounts(@TypeParameter("map(bigint,smallint)") Type mapType, @SqlType(SetDigestType.NAME) Slice slice)
public static SqlMap hashCounts(@TypeParameter("map(bigint,smallint)") Type mapType, @SqlType(StandardTypes.SET_DIGEST) Slice slice)
{
SetDigest digest = SetDigest.newInstance(slice);
return buildMapValue(
Expand Down
6 changes: 3 additions & 3 deletions core/trino-main/src/main/java/io/trino/type/setdigest/SetDigestOperators.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,21 +31,21 @@ private SetDigestOperators() {}

@ScalarOperator(CAST)
@SqlType(StandardTypes.VARBINARY)
public static Slice castToBinary(@SqlType(SetDigestType.NAME) Slice slice)
public static Slice castToBinary(@SqlType(StandardTypes.SET_DIGEST) Slice slice)
{
return slice;
}

@ScalarOperator(CAST)
@SqlType(SetDigestType.NAME)
@SqlType(StandardTypes.SET_DIGEST)
public static Slice castFromBinary(@SqlType(StandardTypes.VARBINARY) Slice slice)
{
return slice;
}

@ScalarOperator(CAST)
@SqlType(StandardTypes.HYPER_LOG_LOG)
public static Slice castToHyperLogLog(@SqlType(SetDigestType.NAME) Slice slice)
public static Slice castToHyperLogLog(@SqlType(StandardTypes.SET_DIGEST) Slice slice)
{
checkArgument(slice.getByte(0) == 1, "Legacy version of SetDigest cannot cast to HyperLogLog");
int hllLength = slice.getInt(SIZE_OF_BYTE);
Expand Down
4 changes: 2 additions & 2 deletions core/trino-main/src/main/java/io/trino/type/setdigest/SetDigestType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import io.trino.spi.connector.ConnectorSession;
import io.trino.spi.type.AbstractVariableWidthType;
import io.trino.spi.type.SqlVarbinary;
import io.trino.spi.type.StandardTypes;
import io.trino.spi.type.TypeSignature;

// Layout is <size>:<digest>, where
Expand All @@ -31,11 +32,10 @@ public class SetDigestType
extends AbstractVariableWidthType
{
public static final SetDigestType SET_DIGEST = new SetDigestType();
public static final String NAME = "SetDigest";

private SetDigestType()
{
super(new TypeSignature(NAME), Slice.class);
super(new TypeSignature(StandardTypes.SET_DIGEST), Slice.class);
}

@Override
Expand Down
8 changes: 7 additions & 1 deletion core/trino-spi/src/main/java/io/trino/spi/type/StandardTypes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
*/
package io.trino.spi.type;

// Should have the same types as in the ClientStandardTypes
public final class StandardTypes
{
public static final String BIGINT = "bigint";
Expand All @@ -27,6 +28,7 @@ public final class StandardTypes
public static final String HYPER_LOG_LOG = "HyperLogLog";
public static final String QDIGEST = "qdigest";
public static final String TDIGEST = "tdigest";
public static final String SET_DIGEST = "SetDigest";
public static final String P4_HYPER_LOG_LOG = "P4HyperLogLog";
public static final String INTERVAL_DAY_TO_SECOND = "interval day to second";
public static final String INTERVAL_YEAR_TO_MONTH = "interval year to month";
Expand All @@ -43,8 +45,12 @@ public final class StandardTypes
public static final String JSON = "json";
public static final String JSON_2016 = "json2016";
public static final String IPADDRESS = "ipaddress";
public static final String GEOMETRY = "Geometry";
public static final String UUID = "uuid";
public static final String GEOMETRY = "Geometry";
public static final String SPHERICAL_GEOGRAPHY = "SphericalGeography"; // SphericalGeographyType.SPHERICAL_GEOGRAPHY_TYPE_NAME
public static final String BING_TILE = "BingTile"; // BingTileType.NAME
public static final String KDB_TREE = "KdbTree"; // KdbTreeType.NAME
public static final String COLOR = "color"; // Color.NAME

private StandardTypes() {}
}
Loading
Loading