Repository navigation
Define unified SQL language spec with composable extensions #5360
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.api.spec; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import org.apache.calcite.sql.SqlNode; | ||
| import org.apache.calcite.sql.SqlOperatorTable; | ||
| import org.apache.calcite.sql.fun.SqlStdOperatorTable; | ||
| import org.apache.calcite.sql.parser.SqlParser; | ||
| import org.apache.calcite.sql.util.SqlOperatorTables; | ||
| import org.apache.calcite.sql.util.SqlVisitor; | ||
| import org.apache.calcite.sql.validate.SqlValidator; | ||
|
|
||
| /** | ||
| * Language specification defining the dialect the engine accepts. Provides parser configuration, | ||
| * validator configuration, and composable {@link LanguageExtension}s that contribute operators and | ||
| * post-parse rewrite rules. | ||
| * | ||
| * <p>Implementations define a complete language surface — for example, {@link UnifiedSqlSpec} | ||
| * provides ANSI and extended SQL modes. A future PPL spec would implement this same interface once | ||
| * PPL converges on the Calcite pipeline. | ||
| */ | ||
| public interface LanguageSpec { | ||
|
|
||
| /** | ||
| * A composable language extension that contributes operators and post-parse rewrite rules. All | ||
| * methods have defaults so extensions only override what they need. | ||
| */ | ||
| interface LanguageExtension { | ||
|
|
||
| /** | ||
| * Operators (functions, aggregates) this extension adds. Chained with the standard operator | ||
| * table during validation. | ||
| */ | ||
| default SqlOperatorTable operators() { | ||
| return SqlOperatorTables.of(); | ||
| } | ||
|
|
||
| /** | ||
| * AST rewrite rules applied after parsing and before validation. Each visitor transforms the | ||
| * parse tree (e.g., rewriting named arguments into MAP literals). | ||
| */ | ||
| default List<SqlVisitor<SqlNode>> postParseRules() { | ||
| return List.of(); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Parser configuration controlling how SQL text is tokenized and parsed into a parse tree, | ||
| * including parser factory, lexical rules, and conformance. | ||
| */ | ||
| SqlParser.Config parserConfig(); | ||
|
|
||
| /** | ||
| * Validator configuration controlling what SQL semantics the validator accepts, such as GROUP BY | ||
| * behavior, LIMIT syntax, and type coercion. | ||
| */ | ||
| SqlValidator.Config validatorConfig(); | ||
|
|
||
| /** | ||
| * Language extensions registered with this spec. Each extension contributes operators and | ||
| * post-parse rewrite rules that are composed by {@link #operatorTable()} and {@link | ||
| * #postParseRules()}. | ||
| */ | ||
| List<LanguageExtension> extensions(); | ||
|
|
||
| /** | ||
| * Chained operator table combining the standard Calcite operators with all operators contributed | ||
| * by registered extensions. | ||
| */ | ||
| default SqlOperatorTable operatorTable() { | ||
| List<SqlOperatorTable> tables = new ArrayList<>(); | ||
| tables.add(SqlStdOperatorTable.instance()); | ||
| extensions().forEach(ext -> tables.add(ext.operators())); | ||
| return SqlOperatorTables.chain(tables); | ||
| } | ||
|
|
||
| /** | ||
| * All post-parse rewrite rules from registered extensions, flattened in registration order. | ||
| * Applied to the parse tree after parsing and before validation. | ||
| */ | ||
| default List<SqlVisitor<SqlNode>> postParseRules() { | ||
| return extensions().stream().flatMap(ext -> ext.postParseRules().stream()).toList(); | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.api.spec; | ||
|
|
||
| import java.util.List; | ||
| import lombok.AccessLevel; | ||
| import lombok.NoArgsConstructor; | ||
| import org.apache.calcite.sql.parser.SqlParser; | ||
| import org.apache.calcite.sql.validate.SqlValidator; | ||
|
|
||
| /** | ||
| * PPL language specification. | ||
| * | ||
| * <p>Note: PPL currently has its own parsing and analyzing pipeline, so only configuration and | ||
| * extensions applied after RelNode construction are in use. The parser and validator configs | ||
| * returned here are inert for the PPL path. | ||
| */ | ||
| @NoArgsConstructor(access = AccessLevel.PRIVATE) | ||
| public class UnifiedPplSpec implements LanguageSpec { | ||
|
|
||
| public static UnifiedPplSpec create() { | ||
| return new UnifiedPplSpec(); | ||
| } | ||
|
|
||
| @Override | ||
| public SqlParser.Config parserConfig() { | ||
| return SqlParser.config(); | ||
| } | ||
|
|
||
| @Override | ||
| public SqlValidator.Config validatorConfig() { | ||
| return SqlValidator.Config.DEFAULT; | ||
| } | ||
|
|
||
| @Override | ||
| public List<LanguageExtension> extensions() { | ||
| return List.of(); | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.api.spec; | ||
|
|
||
| import java.util.List; | ||
| import lombok.AccessLevel; | ||
| import lombok.Getter; | ||
| import lombok.RequiredArgsConstructor; | ||
| import lombok.experimental.Accessors; | ||
| import org.apache.calcite.config.Lex; | ||
| import org.apache.calcite.sql.parser.SqlParser; | ||
| import org.apache.calcite.sql.parser.SqlParserImplFactory; | ||
| import org.apache.calcite.sql.parser.babel.SqlBabelParserImpl; | ||
| import org.apache.calcite.sql.validate.SqlConformanceEnum; | ||
| import org.apache.calcite.sql.validate.SqlValidator; | ||
| import org.opensearch.sql.api.spec.search.SearchExtension; | ||
|
|
||
| /** | ||
| * SQL language specification. Configures Calcite's parser, validator, and composable extensions for | ||
| * OpenSearch SQL compatibility. | ||
| * | ||
| * <p>Use {@link #extended()} for the default configuration with lenient syntax, hyphenated | ||
| * identifiers, and search functions. | ||
| */ | ||
| @RequiredArgsConstructor(access = AccessLevel.PRIVATE) | ||
| @Accessors(fluent = true) | ||
| public class UnifiedSqlSpec implements LanguageSpec { | ||
|
|
||
| /** Lexical rules: identifier quoting, character escaping, and special identifier support. */ | ||
| private final Lex lex; | ||
|
|
||
| /** Parser implementation: controls keyword reservation and grammar extensions. */ | ||
| private final SqlParserImplFactory parserFactory; | ||
|
|
||
| /** Validation rules: what SQL semantics the validator accepts (GROUP BY, LIMIT, coercion). */ | ||
| private final SqlConformanceEnum conformance; | ||
|
|
||
| /** Composable extensions contributing operators and post-parse rewrite rules. */ | ||
| @Getter private final List<LanguageExtension> extensions; | ||
|
|
||
| /** | ||
| * Extended SQL spec: Babel parser, BIG_QUERY lex (hyphenated identifiers, backtick quoting), | ||
| * BABEL conformance (lenient GROUP BY, LIMIT, optional FROM), and search functions. | ||
| */ | ||
| public static UnifiedSqlSpec extended() { | ||
| return new UnifiedSqlSpec( | ||
| Lex.BIG_QUERY, | ||
| SqlBabelParserImpl.FACTORY, | ||
| SqlConformanceEnum.BABEL, | ||
| List.of(new SearchExtension())); | ||
| } | ||
|
|
||
| @Override | ||
| public SqlParser.Config parserConfig() { | ||
| return SqlParser.config() | ||
| .withParserFactory(parserFactory) | ||
| .withLex(lex) | ||
|
Collaborator
There was a problem hiding this comment. Potential behavior change:
Collaborator
Author
There was a problem hiding this comment. Sure, I've added a TODO item above. Will double confirm. Thanks! |
||
| .withConformance(conformance); | ||
| } | ||
|
|
||
| @Override | ||
| public SqlValidator.Config validatorConfig() { | ||
| return SqlValidator.Config.DEFAULT.withConformance(conformance); | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.api.spec.search; | ||
|
|
||
| import java.util.List; | ||
| import org.apache.calcite.sql.SqlNode; | ||
| import org.apache.calcite.sql.SqlOperatorTable; | ||
| import org.apache.calcite.sql.util.SqlVisitor; | ||
| import org.opensearch.sql.api.spec.LanguageSpec; | ||
| import org.opensearch.sql.api.spec.UnifiedFunctionSpec; | ||
|
|
||
| /** Search Extension: relevance functions and named argument rewriting. */ | ||
| public class SearchExtension implements LanguageSpec.LanguageExtension { | ||
|
|
||
| @Override | ||
| public SqlOperatorTable operators() { | ||
| return UnifiedFunctionSpec.RELEVANCE.operatorTable(); | ||
| } | ||
|
|
||
| @Override | ||
| public List<SqlVisitor<SqlNode>> postParseRules() { | ||
| return List.of(NamedArgRewriter.INSTANCE); | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Small thing to flag: the
SqlKind.QUERYgate runs before this loop, so extensions would not be able to register rewrite rules for non-QUERY statements (e.g. DDL). Should the gate move afterpostParseRules()?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Sure, after I finish this TODO item, the rewrite logic will be moved to
UnifiedQueryParser.