Skip to content

Improvements to numeric literals (underscores, hex, octal, binary) #17776

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
martint merged 2 commits into from
Jun 8, 2023
Merged

Improvements to numeric literals (underscores, hex, octal, binary) #17776

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6f02ea9
Add support for underscores in numeric literals
martint Jun 6, 2023
7cba489
Add support for hexadecimal, octal and binary integer literals
martint Jun 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions core/trino-main/src/main/java/io/trino/sql/analyzer/ExpressionAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -997,7 +997,7 @@ protected Type visitSubscriptExpression(SubscriptExpression node, StackableAstVi
if (!indexType.equals(INTEGER)) {
throw semanticException(TYPE_MISMATCH, node.getIndex(), "Subscript expression on ROW requires integer index, found %s", indexType);
}
int indexValue = toIntExact(((LongLiteral) node.getIndex()).getValue());
int indexValue = toIntExact(((LongLiteral) node.getIndex()).getParsedValue());
if (indexValue <= 0) {
throw semanticException(INVALID_FUNCTION_ARGUMENT, node.getIndex(), "Invalid subscript index: %s. ROW indices start at 1", indexValue);
}
Expand Down Expand Up @@ -1043,7 +1043,7 @@ protected Type visitBinaryLiteral(BinaryLiteral node, StackableAstVisitorContext
@Override
protected Type visitLongLiteral(LongLiteral node, StackableAstVisitorContext<Context> context)
{
if (node.getValue() >= Integer.MIN_VALUE && node.getValue() <= Integer.MAX_VALUE) {
if (node.getParsedValue() >= Integer.MIN_VALUE && node.getParsedValue() <= Integer.MAX_VALUE) {
return setExpressionType(node, INTEGER);
}

Expand Down Expand Up @@ -1686,7 +1686,7 @@ private Type analyzePatternRecognitionFunction(FunctionCall node, StackableAstVi
if (!(node.getArguments().get(1) instanceof LongLiteral)) {
throw semanticException(INVALID_FUNCTION_ARGUMENT, node, "%s pattern recognition navigation function requires a number as the second argument", node.getName());
}
long offset = ((LongLiteral) node.getArguments().get(1)).getValue();
long offset = ((LongLiteral) node.getArguments().get(1)).getParsedValue();
if (offset < 0) {
throw semanticException(NUMERIC_VALUE_OUT_OF_RANGE, node, "%s pattern recognition navigation function requires a non-negative number as the second argument (actual: %s)", node.getName(), offset);
}
Expand Down
4 changes: 2 additions & 2 deletions core/trino-main/src/main/java/io/trino/sql/analyzer/PatternRecognitionAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ public static PatternRecognitionAnalysis analyze(
.filter(RangeQuantifier.class::isInstance)
.map(RangeQuantifier.class::cast)
.forEach(quantifier -> {
Optional<Long> atLeast = quantifier.getAtLeast().map(LongLiteral::getValue);
Optional<Long> atLeast = quantifier.getAtLeast().map(LongLiteral::getParsedValue);
atLeast.ifPresent(value -> {
if (value < 0) {
throw semanticException(NUMERIC_VALUE_OUT_OF_RANGE, quantifier, "Pattern quantifier lower bound must be greater than or equal to 0");
Expand All @@ -133,7 +133,7 @@ public static PatternRecognitionAnalysis analyze(
throw semanticException(NUMERIC_VALUE_OUT_OF_RANGE, quantifier, "Pattern quantifier lower bound must not exceed " + Integer.MAX_VALUE);
}
});
Optional<Long> atMost = quantifier.getAtMost().map(LongLiteral::getValue);
Optional<Long> atMost = quantifier.getAtMost().map(LongLiteral::getParsedValue);
atMost.ifPresent(value -> {
if (value < 1) {
throw semanticException(NUMERIC_VALUE_OUT_OF_RANGE, quantifier, "Pattern quantifier upper bound must be greater than or equal to 1");
Expand Down
10 changes: 5 additions & 5 deletions core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4039,7 +4039,7 @@ private GroupingSetAnalysis analyzeGroupBy(QuerySpecification node, Scope scope,
for (Expression column : groupingElement.getExpressions()) {
// simple GROUP BY expressions allow ordinals or arbitrary expressions
if (column instanceof LongLiteral) {
long ordinal = ((LongLiteral) column).getValue();
long ordinal = ((LongLiteral) column).getParsedValue();
if (ordinal < 1 || ordinal > outputExpressions.size()) {
throw semanticException(INVALID_COLUMN_REFERENCE, column, "GROUP BY position %s is not in select list", ordinal);
}
Expand Down Expand Up @@ -5206,7 +5206,7 @@ private List<Expression> analyzeOrderBy(Node node, List<SortItem> sortItems, Sco
if (expression instanceof LongLiteral) {
// this is an ordinal in the output tuple

long ordinal = ((LongLiteral) expression).getValue();
long ordinal = ((LongLiteral) expression).getParsedValue();
if (ordinal < 1 || ordinal > orderByScope.getRelationType().getVisibleFieldCount()) {
throw semanticException(INVALID_COLUMN_REFERENCE, expression, "ORDER BY position %s is not in select list", ordinal);
}
Expand Down Expand Up @@ -5241,7 +5241,7 @@ private void analyzeOffset(Offset node, Scope scope)
{
long rowCount;
if (node.getRowCount() instanceof LongLiteral) {
rowCount = ((LongLiteral) node.getRowCount()).getValue();
rowCount = ((LongLiteral) node.getRowCount()).getParsedValue();
}
else {
checkState(node.getRowCount() instanceof Parameter, "unexpected OFFSET rowCount: " + node.getRowCount().getClass().getSimpleName());
Expand Down Expand Up @@ -5275,7 +5275,7 @@ private boolean analyzeLimit(FetchFirst node, Scope scope)
if (node.getRowCount().isPresent()) {
Expression count = node.getRowCount().get();
if (count instanceof LongLiteral) {
rowCount = ((LongLiteral) count).getValue();
rowCount = ((LongLiteral) count).getParsedValue();
}
else {
checkState(count instanceof Parameter, "unexpected FETCH FIRST rowCount: " + count.getClass().getSimpleName());
Expand All @@ -5300,7 +5300,7 @@ private boolean analyzeLimit(Limit node, Scope scope)
rowCount = OptionalLong.empty();
}
else if (node.getRowCount() instanceof LongLiteral) {
rowCount = OptionalLong.of(((LongLiteral) node.getRowCount()).getValue());
rowCount = OptionalLong.of(((LongLiteral) node.getRowCount()).getParsedValue());
}
else {
checkState(node.getRowCount() instanceof Parameter, "unexpected LIMIT rowCount: " + node.getRowCount().getClass().getSimpleName());
Expand Down
2 changes: 1 addition & 1 deletion core/trino-main/src/main/java/io/trino/sql/planner/ConnectorExpressionTranslator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -836,7 +836,7 @@ protected Optional<ConnectorExpression> visitSubscriptExpression(SubscriptExpres
return Optional.empty();
}

return Optional.of(new FieldDereference(typeOf(node), translatedBase.get(), toIntExact(((LongLiteral) node.getIndex()).getValue() - 1)));
return Optional.of(new FieldDereference(typeOf(node), translatedBase.get(), toIntExact(((LongLiteral) node.getIndex()).getParsedValue() - 1)));
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion core/trino-main/src/main/java/io/trino/sql/planner/LiteralEncoder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ public Expression toExpression(Session session, @Nullable Object object, Type ty

if (type.equals(BIGINT)) {
LongLiteral expression = new LongLiteral(object.toString());
if (expression.getValue() >= Integer.MIN_VALUE && expression.getValue() <= Integer.MAX_VALUE) {
if (expression.getParsedValue() >= Integer.MIN_VALUE && expression.getParsedValue() <= Integer.MAX_VALUE) {
return new GenericLiteral("BIGINT", object.toString());
}
return new LongLiteral(object.toString());
Expand Down
2 changes: 1 addition & 1 deletion core/trino-main/src/main/java/io/trino/sql/planner/LiteralInterpreter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ protected Object visitBooleanLiteral(BooleanLiteral node, Void context)
@Override
protected Long visitLongLiteral(LongLiteral node, Void context)
{
return node.getValue();
return node.getParsedValue();
}

@Override
Expand Down
4 changes: 2 additions & 2 deletions core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/UnwrapRowSubscript.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public Expression rewriteSubscriptExpression(SubscriptExpression node, Void cont
break;
}

int index = (int) ((LongLiteral) node.getIndex()).getValue();
int index = (int) ((LongLiteral) node.getIndex()).getParsedValue();
DataType type = rowType.getFields().get(index - 1).getType();
if (!(type instanceof GenericDataType) || !((GenericDataType) type).getName().getValue().equalsIgnoreCase(UnknownType.NAME)) {
coercions.push(new Coercion(type, cast.isTypeOnly(), cast.isSafe()));
Expand All @@ -69,7 +69,7 @@ public Expression rewriteSubscriptExpression(SubscriptExpression node, Void cont
}

if (base instanceof Row row) {
int index = (int) ((LongLiteral) node.getIndex()).getValue();
int index = (int) ((LongLiteral) node.getIndex()).getParsedValue();
Expression result = row.getItems().get(index - 1);

while (!coercions.isEmpty()) {
Expand Down
2 changes: 1 addition & 1 deletion core/trino-main/src/main/java/io/trino/sql/planner/planprinter/CounterBasedAnonymizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ private String anonymizeLiteral(Literal node)
return anonymizeLiteral("double", literal.getValue());
}
if (node instanceof LongLiteral literal) {
return anonymizeLiteral("long", literal.getValue());
return anonymizeLiteral("long", literal.getParsedValue());
}
if (node instanceof TimestampLiteral literal) {
return anonymizeLiteral("timestamp", literal.getValue());
Expand Down
2 changes: 1 addition & 1 deletion core/trino-main/src/main/java/io/trino/sql/planner/rowpattern/LogicalIndexExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ private Expression rewritePatternNavigationFunction(FunctionCall node, LogicalIn
Optional<ProcessingMode> processingMode = node.getProcessingMode();
OptionalInt offset = OptionalInt.empty();
if (node.getArguments().size() > 1) {
offset = OptionalInt.of(toIntExact(((LongLiteral) node.getArguments().get(1)).getValue()));
offset = OptionalInt.of(toIntExact(((LongLiteral) node.getArguments().get(1)).getParsedValue()));
}
return switch (functionName) {
case "PREV" -> treeRewriter.rewrite(argument, context.withPhysicalOffset(-offset.orElse(1)));
Expand Down
6 changes: 3 additions & 3 deletions core/trino-main/src/main/java/io/trino/sql/relational/SqlToRowExpressionTranslator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,10 +206,10 @@ protected RowExpression visitBooleanLiteral(BooleanLiteral node, Void context)
@Override
protected RowExpression visitLongLiteral(LongLiteral node, Void context)
{
if (node.getValue() >= Integer.MIN_VALUE && node.getValue() <= Integer.MAX_VALUE) {
return constant(node.getValue(), INTEGER);
if (node.getParsedValue() >= Integer.MIN_VALUE && node.getParsedValue() <= Integer.MAX_VALUE) {
return constant(node.getParsedValue(), INTEGER);
}
return constant(node.getValue(), BIGINT);
return constant(node.getParsedValue(), BIGINT);
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion core/trino-main/src/test/java/io/trino/sql/planner/assertions/ExpressionVerifier.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ protected Boolean visitNullLiteral(NullLiteral node, Node expectedExpression)
private static String getValueFromLiteral(Node expression)
{
if (expression instanceof LongLiteral) {
return String.valueOf(((LongLiteral) expression).getValue());
return String.valueOf(((LongLiteral) expression).getParsedValue());
}

if (expression instanceof BooleanLiteral) {
Expand Down
25 changes: 22 additions & 3 deletions core/trino-parser/src/main/antlr4/io/trino/sql/parser/SqlBase.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1223,12 +1223,15 @@ BINARY_LITERAL
;

INTEGER_VALUE
: DIGIT+
: DECIMAL_INTEGER
| HEXADECIMAL_INTEGER
| OCTAL_INTEGER
| BINARY_INTEGER
;

DECIMAL_VALUE
: DIGIT+ '.' DIGIT*
| '.' DIGIT+
: DECIMAL_INTEGER '.' DECIMAL_INTEGER?
| '.' DECIMAL_INTEGER
;

DOUBLE_VALUE
Expand All @@ -1252,6 +1255,22 @@ BACKQUOTED_IDENTIFIER
: '`' ( ~'`' | '``' )* '`'
;

fragment DECIMAL_INTEGER
: DIGIT ('_'? DIGIT)*
;

fragment HEXADECIMAL_INTEGER
: '0X' ('_'? (DIGIT | [A-F]))+
;

fragment OCTAL_INTEGER
: '0O' ('_'? [0-7])+
;

fragment BINARY_INTEGER
: '0B' ('_'? [01])+
;

fragment EXPONENT
: 'E' [+-]? DIGIT+
;
Expand Down
2 changes: 1 addition & 1 deletion core/trino-parser/src/main/java/io/trino/sql/ExpressionFormatter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ protected String visitLongLiteral(LongLiteral node, Void context)
{
return literalFormatter
.map(formatter -> formatter.apply(node))
.orElseGet(() -> Long.toString(node.getValue()));
.orElseGet(node::getValue);
}

@Override
Expand Down
37 changes: 31 additions & 6 deletions core/trino-parser/src/main/java/io/trino/sql/tree/LongLiteral.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@
public class LongLiteral
extends Literal
{
private final long value;
private final String value;
private final long parsedValue;

public LongLiteral(String value)
{
Expand All @@ -39,18 +40,24 @@ private LongLiteral(Optional<NodeLocation> location, String value)
super(location);
requireNonNull(value, "value is null");
try {
this.value = Long.parseLong(value);
this.value = value;
this.parsedValue = parse(value);
}
catch (NumberFormatException e) {
throw new ParsingException("Invalid numeric literal: " + value);
}
}

public long getValue()
public String getValue()
{
return value;
}

public long getParsedValue()
{
return parsedValue;
}

@Override
public <R, C> R accept(AstVisitor<R, C> visitor, C context)
{
Expand All @@ -69,7 +76,7 @@ public boolean equals(Object o)

LongLiteral that = (LongLiteral) o;

if (value != that.value) {
if (parsedValue != that.parsedValue) {
return false;
}

Expand All @@ -79,7 +86,7 @@ public boolean equals(Object o)
@Override
public int hashCode()
{
return (int) (value ^ (value >>> 32));
return (int) (parsedValue ^ (parsedValue >>> 32));
}

@Override
Expand All @@ -89,6 +96,24 @@ public boolean shallowEquals(Node other)
return false;
}

return value == ((LongLiteral) other).value;
return parsedValue == ((LongLiteral) other).parsedValue;
}

private static long parse(String value)
{
value = value.replace("_", "");

if (value.startsWith("0x") || value.startsWith("0X")) {
return Long.parseLong(value.substring(2), 16);
Copy link
Copy Markdown
Member

@dain dain Jun 8, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

BTW, in general this syntax is dangerous because parse long supports a leading sign. It isn't a problem here because the grammar doesn't allow that, but I ran into code in Hive that did this and ended up supporting 0x-123

}
else if (value.startsWith("0b") || value.startsWith("0B")) {
return Long.parseLong(value.substring(2), 2);
}
else if (value.startsWith("0o") || value.startsWith("0O")) {
return Long.parseLong(value.substring(2), 8);
}
else {
return Long.parseLong(value);
}
}
}
71 changes: 71 additions & 0 deletions core/trino-parser/src/test/java/io/trino/sql/parser/TestSqlParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -456,6 +456,77 @@ public void testNumbers()
.isEqualTo(new DecimalLiteral(location, "1.2"));
assertThat(expression("-1.2"))
.isEqualTo(new DecimalLiteral(location, "-1.2"));

assertThat(expression("123_456_789"))
.isEqualTo(new LongLiteral(new NodeLocation(1, 1), "123_456_789"))
.satisfies(value -> assertThat(((LongLiteral) value).getParsedValue()).isEqualTo(123456789L));

assertThatThrownBy(() -> SQL_PARSER.createExpression("123_456_789_", new ParsingOptions()))
.isInstanceOf(ParsingException.class);

assertThat(expression("123_456.789_0123"))
.isEqualTo(new DecimalLiteral(new NodeLocation(1, 1), "123_456.789_0123"));

assertThatThrownBy(() -> SQL_PARSER.createExpression("123_456.789_0123_", new ParsingOptions()))
.isInstanceOf(ParsingException.class);

assertThatThrownBy(() -> SQL_PARSER.createExpression("_123_456.789_0123", new ParsingOptions()))
.isInstanceOf(ParsingException.class);

assertThatThrownBy(() -> SQL_PARSER.createExpression("123_456_.789_0123", new ParsingOptions()))
.isInstanceOf(ParsingException.class);

assertThatThrownBy(() -> SQL_PARSER.createExpression("123_456._789_0123", new ParsingOptions()))
.isInstanceOf(ParsingException.class);

assertThat(expression("0x123_abc_def"))
.isEqualTo(new LongLiteral(new NodeLocation(1, 1), "0x123_abc_def"))
.satisfies(value -> assertThat(((LongLiteral) value).getParsedValue()).isEqualTo(4893429231L));

assertThat(expression("0X123_ABC_DEF"))
.isEqualTo(new LongLiteral(new NodeLocation(1, 1), "0X123_ABC_DEF"))
.satisfies(value -> assertThat(((LongLiteral) value).getParsedValue()).isEqualTo(4893429231L));

assertThatThrownBy(() -> SQL_PARSER.createExpression("0x123_ABC_DEF_", new ParsingOptions()))
.isInstanceOf(ParsingException.class);

assertThat(expression("0O012_345"))
.isEqualTo(new LongLiteral(new NodeLocation(1, 1), "0O012_345"))
.satisfies(value -> assertThat(((LongLiteral) value).getParsedValue()).isEqualTo(5349L));

assertThat(expression("0o012_345"))
.isEqualTo(new LongLiteral(new NodeLocation(1, 1), "0o012_345"))
.satisfies(value -> assertThat(((LongLiteral) value).getParsedValue()).isEqualTo(5349L));

assertThatThrownBy(() -> SQL_PARSER.createExpression("0o012_345_", new ParsingOptions()))
.isInstanceOf(ParsingException.class);

assertThat(expression("0B110_010"))
.isEqualTo(new LongLiteral(new NodeLocation(1, 1), "0B110_010"))
.satisfies(value -> assertThat(((LongLiteral) value).getParsedValue()).isEqualTo(50L));

assertThat(expression("0b110_010"))
.isEqualTo(new LongLiteral(new NodeLocation(1, 1), "0b110_010"))
.satisfies(value -> assertThat(((LongLiteral) value).getParsedValue()).isEqualTo(50L));

assertThatThrownBy(() -> SQL_PARSER.createExpression("0b110_010_", new ParsingOptions()))
.isInstanceOf(ParsingException.class);
}

@Test
public void testIdentifier()
{
assertThat(expression("_123_456"))
.isEqualTo(new Identifier(new NodeLocation(1, 1), "_123_456", false));

assertThat(expression("_0x123_ABC_DEF"))
.isEqualTo(new Identifier(new NodeLocation(1, 1), "_0x123_ABC_DEF", false));

assertThat(expression("_0o012_345"))
.isEqualTo(new Identifier(new NodeLocation(1, 1), "_0o012_345", false));

assertThat(expression("_0b110_010"))
.isEqualTo(new Identifier(new NodeLocation(1, 1), "_0b110_010", false));
}

@Test
Expand Down
Loading