From 68663d1553120d32621b48f7ae1fd9be451b59c4 Mon Sep 17 00:00:00 2001 From: MitchellGale-BitQuill <104795536+MitchellGale-BitQuill@users.noreply.github.com> Date: Tue, 2 Aug 2022 14:23:12 -0700 Subject: [PATCH 01/20] Two single or double quote escapes single or double quote when string is surrounded by same type of quote (#696) Signed-off-by: mitchellg Co-authored-by: Andrew Carbonetto --- .../sql/common/utils/StringUtils.java | 42 +++++++++++++++- .../sql/common/utils/StringUtilsTest.java | 48 +++++++++++++++++++ .../correctness/expressions/literals.txt | 5 +- 3 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 core/src/test/java/org/opensearch/sql/common/utils/StringUtilsTest.java diff --git a/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java b/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java index 2e5ef7803e6..06992453383 100644 --- a/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java +++ b/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java @@ -27,16 +27,54 @@ public static String unquote(String text, String mark) { /** * Unquote Identifier which has " or ' or ` as mark. + * Strings quoted by ' or " with two of these quotes appearing next to each other in the quote + * acts as an escape + * Example: 'Test''s' will result in 'Test's', similar with those single quotes being replaced + * with double. * @param text string * @return An unquoted string whose outer pair of (single/double/back-tick) quotes have been * removed */ public static String unquoteText(String text) { - if (isQuoted(text, "\"") || isQuoted(text, "'") || isQuoted(text, "`")) { - return text.substring(1, text.length() - 1); + + if (text.length() < 2) { + return text; + } + + char enclosingQuote; + char firstChar = text.charAt(0); + char lastChar = text.charAt(text.length() - 1); + + if (firstChar == lastChar + && (firstChar == '\'' + || firstChar == '"' + || firstChar == '`')) { + enclosingQuote = firstChar; } else { return text; } + + if (enclosingQuote == '`') { + return text.substring(1, text.length() - 1); + } + + char currentChar; + char nextChar; + + StringBuilder textSB = new StringBuilder(); + + // Ignores first and last character as they are the quotes that should be removed + for (int chIndex = 1; chIndex < text.length() - 1; chIndex++) { + currentChar = text.charAt(chIndex); + nextChar = text.charAt(chIndex + 1); + if (currentChar == enclosingQuote + && nextChar == currentChar) { + chIndex++; + } + textSB.append(currentChar); + } + + return textSB.toString(); } /** diff --git a/core/src/test/java/org/opensearch/sql/common/utils/StringUtilsTest.java b/core/src/test/java/org/opensearch/sql/common/utils/StringUtilsTest.java new file mode 100644 index 00000000000..06ea404fc29 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/common/utils/StringUtilsTest.java @@ -0,0 +1,48 @@ +package org.opensearch.sql.common.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.common.utils.StringUtils.unquoteText; + +import org.junit.jupiter.api.Test; + +class StringUtilsTest { + @Test + void unquoteTest() { + assertEquals("test", unquoteText("test")); + assertEquals("test", unquoteText("'test'")); + assertEquals("test", unquoteText("`test`")); + + assertEquals("test'", unquoteText("'test'''")); + assertEquals("test\"", unquoteText("\"test\"\"\"")); + + assertEquals("te``st", unquoteText("'te``st'")); + assertEquals("te``st", unquoteText("\"te``st\"")); + assertEquals("te``st", unquoteText("`te``st`")); + + assertEquals("te'st", unquoteText("'te''st'")); + assertEquals("te''st", unquoteText("\"te''st\"")); + assertEquals("te''st", unquoteText("`te''st`")); + + assertEquals("te\"\"st", unquoteText("'te\"\"st'")); + assertEquals("te\"st", unquoteText("\"te\"\"st\"")); + assertEquals("te\"\"st", unquoteText("`te\"\"st`")); + + assertEquals("''", unquoteText("''''''")); + assertEquals("\"\"", unquoteText("\"\"\"\"\"\"")); + assertEquals("````", unquoteText("``````")); + + assertEquals("test'", unquoteText("'test''")); + + assertEquals("", unquoteText("")); + assertEquals("'", unquoteText("'")); + assertEquals("`", unquoteText("`")); + assertEquals("\"", unquoteText("\"")); + + assertEquals("hello'", unquoteText("'hello''")); + assertEquals("don't", unquoteText("'don't'")); + assertEquals("hello`", unquoteText("`hello``")); + assertEquals("don\"t", unquoteText("\"don\"t\"")); + + } + +} diff --git a/integ-test/src/test/resources/correctness/expressions/literals.txt b/integ-test/src/test/resources/correctness/expressions/literals.txt index 39eedc28e7b..c7dfcc81561 100644 --- a/integ-test/src/test/resources/correctness/expressions/literals.txt +++ b/integ-test/src/test/resources/correctness/expressions/literals.txt @@ -8,4 +8,7 @@ true 2147483647 -2147483648 2147483648 --2147483649 \ No newline at end of file +-2147483649 +'im' +'i''m' +'i""m' From d26cde3331c31924515ff38c4f24bd594c111ff8 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 4 Aug 2022 09:58:39 -0700 Subject: [PATCH 02/20] Remove BWC tests (#721) Signed-off-by: Yury Fridlyand --- .github/workflows/sql-test-and-build-workflow.yml | 3 --- integ-test/build.gradle | 4 ++++ scripts/bwctest.sh | 0 3 files changed, 4 insertions(+), 3 deletions(-) mode change 100755 => 100644 scripts/bwctest.sh diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index fcc63433a8f..70d1c3a3e59 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -22,9 +22,6 @@ jobs: - name: Build with Gradle run: ./gradlew build assemble - - name: Run backward compatibility tests - run: ./scripts/bwctest.sh - - name: Create Artifact Path run: | mkdir -p opensearch-sql-builds diff --git a/integ-test/build.gradle b/integ-test/build.gradle index e640c41025a..0baf1af6cc8 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -187,6 +187,10 @@ task compileJdbc(type: Exec) { } } +/* +BWC test suite was running on OpenDistro which was discontinued and no available anymore for testing. +Test suite is not removed, because it could be reused later between different OpenSearch versions. +*/ String bwcVersion = "1.13.2.0"; String baseName = "sqlBwcCluster" String bwcFilePath = "src/test/resources/bwc/" diff --git a/scripts/bwctest.sh b/scripts/bwctest.sh old mode 100755 new mode 100644 From 6a99b56d4b898aaa604240298043f1a8ec7eb0fc Mon Sep 17 00:00:00 2001 From: MitchellGale-BitQuill <104795536+MitchellGale-BitQuill@users.noreply.github.com> Date: Thu, 4 Aug 2022 14:56:06 -0700 Subject: [PATCH 03/20] Github Actions fix for reference to OpenSearch-Dashboard not existing outside of main OS sql project (#704) Signed-off-by: mitchellg --- .../sql-workbench-test-and-build-workflow.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index 536a99a7b49..d4c3f12996c 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -22,40 +22,40 @@ jobs: with: repository: opensearch-project/Opensearch-Dashboards ref: ${{ env.OPENSEARCH_VERSION }} - path: sql/OpenSearch-Dashboards + path: OpenSearch-Dashboards - name: Setup Node uses: actions/setup-node@v2 with: - node-version-file: "./OpenSearch-Dashboards/.nvmrc" + node-version-file: "../OpenSearch-Dashboards/.nvmrc" registry-url: 'https://registry.npmjs.org' - name: Move Workbench to Plugins Dir run: | - mv workbench OpenSearch-Dashboards/plugins + mv workbench ../OpenSearch-Dashboards/plugins - name: OpenSearch Dashboards Plugin Bootstrap uses: nick-invision/retry@v1 with: timeout_minutes: 60 max_attempts: 3 - command: cd OpenSearch-Dashboards/plugins/workbench; yarn osd bootstrap + command: cd ../OpenSearch-Dashboards/plugins/workbench; yarn osd bootstrap - name: Test run: | - cd OpenSearch-Dashboards/plugins/workbench + cd ../OpenSearch-Dashboards/plugins/workbench yarn test:jest --coverage - name: Upload coverage uses: codecov/codecov-action@v1 with: flags: query-workbench - directory: ./OpenSearch-Dashboards/plugins/workbench + directory: ../OpenSearch-Dashboards/plugins/workbench token: ${{ secrets.CODECOV_TOKEN }} - name: Build Artifact run: | - cd OpenSearch-Dashboards/plugins/workbench + cd ../OpenSearch-Dashboards/plugins/workbench yarn build mv ./build/*.zip ./build/${{ env.PLUGIN_NAME }}-${{ env.OPENSEARCH_PLUGIN_VERSION }}.zip @@ -63,4 +63,4 @@ jobs: uses: actions/upload-artifact@v1 with: name: workbench - path: OpenSearch-Dashboards/plugins/workbench/build + path: ../OpenSearch-Dashboards/plugins/workbench/build From 8f1b31d4d1f1cbbdbb6c36d56776b47d5f49c809 Mon Sep 17 00:00:00 2001 From: Forest Vey <36905077+forestmvey@users.noreply.github.com> Date: Fri, 5 Aug 2022 10:48:29 -0700 Subject: [PATCH 04/20] Add Highlight In SQL (#717) * Add support for highlight to parser and AstExpressionBuilder Signed-off-by: MaxKsyunz * Add unit test for highlight in AstExpressionBuilder Signed-off-by: MaxKsyunz * Add unit test for highlight in AstBuilderTest Signed-off-by: MaxKsyunz * Support highlight as an Unresolved expression. Signed-off-by: MaxKsyunz * Represent highlight as UnresolvedExpression. Signed-off-by: MaxKsyunz * Support highlight in Analyzer. Signed-off-by: MaxKsyunz * Treat highlight as a proper function in AST In particular, highlightField is an expression now. Signed-off-by: MaxKsyunz * Add support for highlight in Analyzer HighlightFunction is converted to LogicalHighlight logical plan. Signed-off-by: MaxKsyunz * Add a simple IT test for highlight. Signed-off-by: MaxKsyunz * Register highlight function in the BuiltInFunctionRepository Signed-off-by: MaxKsyunz * Partial support for highlight in physical plan. Signed-off-by: MaxKsyunz * Add HighlightOperator. Signed-off-by: MaxKsyunz * Highlight alpha complete. Signed-off-by: MaxKsyunz * Initial implementation to upporting highlight('*') Signed-off-by: forestmvey * Add support for multiple highlight calls in select statement. Signed-off-by: forestmvey * Removed OpenSearchLogicalIndexScan highlightFields and dependencies. Improved test coverage and fixing checkstyle errors. Signed-off-by: forestmvey * Added HighlightExpressionTest Signed-off-by: forestmvey * Added javadocs, minor PR revisions, and fixed jacoco errors by improving coverage for OpenSearchIndexScan Signed-off-by: forestmvey * Code cleanup, adding parsing failure tests, and adding tests for highlight acceptance as a string literal as well as qualified name. Signed-off-by: forestmvey * Removing HighlightOperator functionality and unnecessary visitHighlight call in PhysicalPlanNodeVisitor.. Signed-off-by: forestmvey * Adding highlight function to functions.rst and removing unecessary function call in OpenSearchIndexScan. Signed-off-by: forestmvey * Change highlight fields returned format to array list. Changed highlight all and wildcard to unsupported to open up output formatting changes for multiple returned highlight fields. Change tests for updated coverage and disable highlight all and wildcard tests. Signed-off-by: forestmvey * Fix bug where invalid schema name was being used for returned highlight fields Signed-off-by: forestmvey * Fix failing integration tests due to schema changes for highlight expression type. Signed-off-by: forestmvey Co-authored-by: MaxKsyunz --- .../org/opensearch/sql/analysis/Analyzer.java | 6 ++ .../sql/analysis/ExpressionAnalyzer.java | 11 ++- .../sql/analysis/HighlightAnalyzer.java | 41 +++++++++ .../sql/ast/AbstractNodeVisitor.java | 5 ++ .../org/opensearch/sql/ast/dsl/AstDSL.java | 5 ++ .../sql/ast/expression/HighlightFunction.java | 34 +++++++ .../sql/expression/ExpressionNodeVisitor.java | 4 + .../sql/expression/HighlightExpression.java | 57 ++++++++++++ .../function/BuiltinFunctionName.java | 1 + .../function/OpenSearchFunctions.java | 11 ++- .../sql/planner/logical/LogicalHighlight.java | 29 ++++++ .../sql/planner/logical/LogicalPlanDSL.java | 4 + .../logical/LogicalPlanNodeVisitor.java | 4 + .../sql/planner/physical/PhysicalPlanDSL.java | 1 - .../physical/PhysicalPlanNodeVisitor.java | 2 - .../opensearch/sql/analysis/AnalyzerTest.java | 19 ++++ .../sql/analysis/ExpressionAnalyzerTest.java | 9 +- .../analysis/NamedExpressionAnalyzerTest.java | 16 ++++ .../expression/ExpressionNodeVisitorTest.java | 1 + .../expression/HighlightExpressionTest.java | 67 ++++++++++++++ .../logical/LogicalPlanNodeVisitorTest.java | 8 +- docs/user/dql/functions.rst | 26 ++++++ .../sql/sql/HighlightFunctionIT.java | 88 +++++++++++++++++++ .../response/OpenSearchResponse.java | 20 ++++- .../opensearch/storage/OpenSearchIndex.java | 7 ++ .../storage/OpenSearchIndexScan.java | 17 ++++ .../OpenSearchExecutionProtectorTest.java | 1 - .../response/OpenSearchResponseTest.java | 37 ++++++++ .../OpenSearchDefaultImplementorTest.java | 15 ++++ .../storage/OpenSearchIndexScanTest.java | 27 ++++++ sql/src/main/antlr/OpenSearchSQLLexer.g4 | 1 + sql/src/main/antlr/OpenSearchSQLParser.g4 | 5 ++ .../sql/sql/parser/AstExpressionBuilder.java | 7 ++ .../sql/sql/antlr/HighlightTest.java | 45 ++++++++++ .../sql/sql/parser/AstBuilderTest.java | 20 +++++ .../sql/parser/AstExpressionBuilderTest.java | 17 ++++ 36 files changed, 657 insertions(+), 11 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/analysis/HighlightAnalyzer.java create mode 100644 core/src/main/java/org/opensearch/sql/ast/expression/HighlightFunction.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/logical/LogicalHighlight.java create mode 100644 core/src/test/java/org/opensearch/sql/expression/HighlightExpressionTest.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/sql/HighlightFunctionIT.java create mode 100644 sql/src/test/java/org/opensearch/sql/sql/antlr/HighlightTest.java diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 9054f80e93d..dc12bdab73f 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -292,6 +292,12 @@ public LogicalPlan visitProject(Project node, AnalysisContext context) { child = windowAnalyzer.analyze(expr, context); } + for (UnresolvedExpression expr : node.getProjectList()) { + HighlightAnalyzer highlightAnalyzer = new HighlightAnalyzer(expressionAnalyzer, child); + child = highlightAnalyzer.analyze(expr, context); + + } + List namedExpressions = selectExpressionAnalyzer.analyze(node.getProjectList(), context, new ExpressionReferenceOptimizer(expressionAnalyzer.getRepository(), child)); diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java index 91b964d0749..670da5c85ce 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java @@ -11,9 +11,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import lombok.Getter; @@ -29,6 +27,7 @@ import org.opensearch.sql.ast.expression.EqualTo; import org.opensearch.sql.ast.expression.Field; import org.opensearch.sql.ast.expression.Function; +import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.In; import org.opensearch.sql.ast.expression.Interval; import org.opensearch.sql.ast.expression.Literal; @@ -44,12 +43,12 @@ import org.opensearch.sql.ast.expression.WindowFunction; import org.opensearch.sql.ast.expression.Xor; import org.opensearch.sql.common.antlr.SyntaxCheckException; -import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.HighlightExpression; import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.NamedArgumentExpression; import org.opensearch.sql.expression.NamedExpression; @@ -191,6 +190,12 @@ public Expression visitWindowFunction(WindowFunction node, AnalysisContext conte return expr; } + @Override + public Expression visitHighlight(HighlightFunction node, AnalysisContext context) { + Expression expr = node.getHighlightField().accept(this, context); + return new HighlightExpression(expr); + } + @Override public Expression visitIn(In node, AnalysisContext context) { return visitIn(node.getField(), node.getValueList(), context); diff --git a/core/src/main/java/org/opensearch/sql/analysis/HighlightAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/HighlightAnalyzer.java new file mode 100644 index 00000000000..06a601327ce --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/analysis/HighlightAnalyzer.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.analysis; + +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.Alias; +import org.opensearch.sql.ast.expression.HighlightFunction; +import org.opensearch.sql.ast.expression.UnresolvedExpression; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.planner.logical.LogicalHighlight; +import org.opensearch.sql.planner.logical.LogicalPlan; + +/** + * Analyze the highlight in the {@link AnalysisContext} to construct the {@link + * LogicalPlan}. + */ +@RequiredArgsConstructor +public class HighlightAnalyzer extends AbstractNodeVisitor { + private final ExpressionAnalyzer expressionAnalyzer; + private final LogicalPlan child; + + public LogicalPlan analyze(UnresolvedExpression projectItem, AnalysisContext context) { + LogicalPlan highlight = projectItem.accept(this, context); + return (highlight == null) ? child : highlight; + } + + @Override + public LogicalPlan visitAlias(Alias node, AnalysisContext context) { + if (!(node.getDelegated() instanceof HighlightFunction)) { + return null; + } + + HighlightFunction unresolved = (HighlightFunction) node.getDelegated(); + Expression field = expressionAnalyzer.analyze(unresolved.getHighlightField(), context); + return new LogicalHighlight(child, field); + } +} diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index 37163821e32..17321bc4739 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -18,6 +18,7 @@ import org.opensearch.sql.ast.expression.EqualTo; import org.opensearch.sql.ast.expression.Field; import org.opensearch.sql.ast.expression.Function; +import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.In; import org.opensearch.sql.ast.expression.Interval; import org.opensearch.sql.ast.expression.Let; @@ -254,4 +255,8 @@ public T visitKmeans(Kmeans node, C context) { public T visitAD(AD node, C context) { return visitChildren(node, context); } + + public T visitHighlight(HighlightFunction node, C context) { + return visitChildren(node, context); + } } diff --git a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java index 4056347d44f..510482c6455 100644 --- a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java @@ -22,6 +22,7 @@ import org.opensearch.sql.ast.expression.EqualTo; import org.opensearch.sql.ast.expression.Field; import org.opensearch.sql.ast.expression.Function; +import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.In; import org.opensearch.sql.ast.expression.Interval; import org.opensearch.sql.ast.expression.Let; @@ -261,6 +262,10 @@ public When when(UnresolvedExpression condition, UnresolvedExpression result) { return new When(condition, result); } + public UnresolvedExpression highlight(UnresolvedExpression fieldName) { + return new HighlightFunction(fieldName); + } + public UnresolvedExpression window(UnresolvedExpression function, List partitionByList, List> sortList) { diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/HighlightFunction.java b/core/src/main/java/org/opensearch/sql/ast/expression/HighlightFunction.java new file mode 100644 index 00000000000..5f1bb652d95 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/expression/HighlightFunction.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.expression; + +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; + +/** + * Expression node of Highlight function. + */ +@AllArgsConstructor +@EqualsAndHashCode(callSuper = false) +@Getter +@ToString +public class HighlightFunction extends UnresolvedExpression { + private final UnresolvedExpression highlightField; + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitHighlight(this, context); + } + + @Override + public List getChild() { + return List.of(highlightField); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java b/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java index b05b0924a8e..d53371dd58f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java @@ -55,6 +55,10 @@ public T visitNamed(NamedExpression node, C context) { return node.getDelegated().accept(this, context); } + public T visitHighlight(HighlightExpression node, C context) { + return visitNode(node, context); + } + public T visitReference(ReferenceExpression node, C context) { return visitNode(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java b/core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java new file mode 100644 index 00000000000..97456961114 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression; + +import java.util.List; +import lombok.Getter; +import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.expression.env.Environment; +import org.opensearch.sql.expression.function.BuiltinFunctionName; + +/** + * Highlight Expression. + */ +@Getter +public class HighlightExpression extends FunctionExpression { + private final Expression highlightField; + + /** + * HighlightExpression Constructor. + * @param highlightField : Highlight field for expression. + */ + public HighlightExpression(Expression highlightField) { + super(BuiltinFunctionName.HIGHLIGHT.getName(), List.of(highlightField)); + this.highlightField = highlightField; + } + + /** + * Return collection value matching highlight field. + * @param valueEnv : Dataset to parse value from. + * @return : collection value of highlight fields. + */ + @Override + public ExprValue valueOf(Environment valueEnv) { + String refName = "_highlight" + "." + StringUtils.unquoteText(getHighlightField().toString()); + return valueEnv.resolve(DSL.ref(refName, ExprCoreType.STRING)); + } + + /** + * Get type for HighlightExpression. + * @return : String type. + */ + @Override + public ExprType type() { + return ExprCoreType.ARRAY; + } + + @Override + public T accept(ExpressionNodeVisitor visitor, C context) { + return visitor.visitHighlight(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 5eb9e715b83..cd343284530 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -192,6 +192,7 @@ public enum BuiltinFunctionName { MATCHPHRASE(FunctionName.of("matchphrase")), QUERY_STRING(FunctionName.of("query_string")), MATCH_BOOL_PREFIX(FunctionName.of("match_bool_prefix")), + HIGHLIGHT(FunctionName.of("highlight")), MATCH_PHRASE_PREFIX(FunctionName.of("match_phrase_prefix")), /** * Legacy Relevance Function. diff --git a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java index dff3ec1f200..c3e5cc55947 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java @@ -8,7 +8,6 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.ArrayList; import java.util.Collections; @@ -16,11 +15,13 @@ import java.util.Map; import java.util.stream.Collectors; import lombok.experimental.UtilityClass; +import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.HighlightExpression; import org.opensearch.sql.expression.NamedArgumentExpression; import org.opensearch.sql.expression.env.Environment; @@ -50,6 +51,14 @@ public void register(BuiltinFunctionRepository repository) { repository.register(match_phrase(BuiltinFunctionName.MATCH_PHRASE)); repository.register(match_phrase(BuiltinFunctionName.MATCHPHRASE)); repository.register(match_phrase_prefix()); + repository.register(highlight()); + } + + private static FunctionResolver highlight() { + FunctionName functionName = BuiltinFunctionName.HIGHLIGHT.getName(); + FunctionSignature functionSignature = new FunctionSignature(functionName, List.of(STRING)); + FunctionBuilder functionBuilder = arguments -> new HighlightExpression(arguments.get(0)); + return new FunctionResolver(functionName, ImmutableMap.of(functionSignature, functionBuilder)); } private static FunctionResolver match_bool_prefix() { diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalHighlight.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalHighlight.java new file mode 100644 index 00000000000..986a5454860 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalHighlight.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.logical; + +import java.util.Collections; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.opensearch.sql.expression.Expression; + +@EqualsAndHashCode(callSuper = true) +@Getter +@ToString +public class LogicalHighlight extends LogicalPlan { + private final Expression highlightField; + + public LogicalHighlight(LogicalPlan childPlan, Expression field) { + super(Collections.singletonList(childPlan)); + highlightField = field; + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitHighlight(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java index d5fef43c0db..4185d55c559 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java @@ -62,6 +62,10 @@ public LogicalPlan window(LogicalPlan input, return new LogicalWindow(input, windowFunction, windowDefinition); } + public LogicalPlan highlight(LogicalPlan input, Expression field) { + return new LogicalHighlight(input, field); + } + public static LogicalPlan remove(LogicalPlan input, ReferenceExpression... fields) { return new LogicalRemove(input, ImmutableSet.copyOf(fields)); } diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java index 5163e44edb2..df23b9cd20d 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java @@ -26,6 +26,10 @@ public R visitFilter(LogicalFilter plan, C context) { return visitNode(plan, context); } + public R visitHighlight(LogicalHighlight plan, C context) { + return visitNode(plan, context); + } + public R visitAggregation(LogicalAggregation plan, C context) { return visitNode(plan, context); } diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanDSL.java b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanDSL.java index 938a4c532c8..e6e59990c82 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanDSL.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanDSL.java @@ -105,5 +105,4 @@ public ValuesOperator values(List... values) { public static LimitOperator limit(PhysicalPlan input, Integer limit, Integer offset) { return new LimitOperator(input, limit, offset); } - } diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java index 87582df3bbf..646aae8220a 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java @@ -79,6 +79,4 @@ public R visitMLCommons(PhysicalPlan node, C context) { public R visitAD(PhysicalPlan node, C context) { return visitNode(node, context); } - - } diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index 797c5ab11ad..b9de96b30a3 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -22,6 +22,7 @@ import static org.opensearch.sql.ast.dsl.AstDSL.qualifiedName; import static org.opensearch.sql.ast.dsl.AstDSL.relation; import static org.opensearch.sql.ast.dsl.AstDSL.span; +import static org.opensearch.sql.ast.dsl.AstDSL.stringLiteral; import static org.opensearch.sql.ast.tree.Sort.NullOrder; import static org.opensearch.sql.ast.tree.Sort.SortOption; import static org.opensearch.sql.ast.tree.Sort.SortOption.DEFAULT_ASC; @@ -45,6 +46,7 @@ import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.ast.expression.DataType; +import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.tree.AD; @@ -52,6 +54,7 @@ import org.opensearch.sql.ast.tree.RareTopN.CommandType; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.HighlightExpression; import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.window.WindowDefinition; import org.opensearch.sql.planner.logical.LogicalAD; @@ -231,6 +234,22 @@ public void project_source() { AstDSL.alias("double_value", AstDSL.field("double_value")))); } + @Test + public void project_highlight() { + assertAnalyzeEqual( + LogicalPlanDSL.project( + LogicalPlanDSL.highlight(LogicalPlanDSL.relation("schema"), + DSL.literal("fieldA")), + DSL.named("highlight(fieldA)", new HighlightExpression(DSL.literal("fieldA"))) + ), + AstDSL.projectWithArg( + AstDSL.relation("schema"), + AstDSL.defaultFieldsArgs(), + AstDSL.alias("highlight(fieldA)", new HighlightFunction(AstDSL.stringLiteral("fieldA"))) + ) + ); + } + @Test public void remove_source() { assertAnalyzeEqual( diff --git a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java index dd0e199cf4c..72db4025522 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java @@ -10,7 +10,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.opensearch.sql.ast.dsl.AstDSL.field; -import static org.opensearch.sql.ast.dsl.AstDSL.floatLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.function; import static org.opensearch.sql.ast.dsl.AstDSL.intLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.qualifiedName; @@ -35,6 +34,7 @@ import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.DataType; +import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.RelevanceFieldList; import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.expression.UnresolvedExpression; @@ -45,6 +45,7 @@ import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.HighlightExpression; import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.window.aggregation.AggregateWindowFunction; import org.springframework.context.annotation.Configuration; @@ -536,6 +537,12 @@ public void match_phrase_prefix_all_params() { ); } + @Test + void highlight() { + assertAnalyzeEqual(new HighlightExpression(DSL.literal("fieldA")), + new HighlightFunction(stringLiteral("fieldA"))); + } + protected Expression analyze(UnresolvedExpression unresolvedExpression) { return expressionAnalyzer.analyze(unresolvedExpression, analysisContext); } diff --git a/core/src/test/java/org/opensearch/sql/analysis/NamedExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/NamedExpressionAnalyzerTest.java index 738e81bfd60..2293d125aa2 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/NamedExpressionAnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/NamedExpressionAnalyzerTest.java @@ -12,6 +12,11 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.Alias; +import org.opensearch.sql.ast.expression.HighlightFunction; +import org.opensearch.sql.ast.expression.QualifiedName; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.config.ExpressionConfig; import org.springframework.context.annotation.Configuration; @@ -32,4 +37,15 @@ void visit_named_select_item() { NamedExpression analyze = analyzer.analyze(alias, analysisContext); assertEquals("integer_value", analyze.getNameOrAlias()); } + + @Test + void visit_highlight() { + Alias alias = AstDSL.alias("highlight(fieldA)", + new HighlightFunction(AstDSL.stringLiteral("fieldA"))); + NamedExpressionAnalyzer analyzer = + new NamedExpressionAnalyzer(expressionAnalyzer); + + NamedExpression analyze = analyzer.analyze(alias, analysisContext); + assertEquals("highlight(fieldA)", analyze.getNameOrAlias()); + } } diff --git a/core/src/test/java/org/opensearch/sql/expression/ExpressionNodeVisitorTest.java b/core/src/test/java/org/opensearch/sql/expression/ExpressionNodeVisitorTest.java index caf11064ae8..b0b2bc5b2bf 100644 --- a/core/src/test/java/org/opensearch/sql/expression/ExpressionNodeVisitorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/ExpressionNodeVisitorTest.java @@ -34,6 +34,7 @@ class ExpressionNodeVisitorTest { @Test void should_return_null_by_default() { ExpressionNodeVisitor visitor = new ExpressionNodeVisitor(){}; + assertNull(new HighlightExpression(DSL.literal("Title")).accept(visitor, null)); assertNull(literal(10).accept(visitor, null)); assertNull(ref("name", STRING).accept(visitor, null)); assertNull(named("bool", literal(true)).accept(visitor, null)); diff --git a/core/src/test/java/org/opensearch/sql/expression/HighlightExpressionTest.java b/core/src/test/java/org/opensearch/sql/expression/HighlightExpressionTest.java new file mode 100644 index 00000000000..c6e2dccf692 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/HighlightExpressionTest.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; +import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; + +import com.google.common.collect.ImmutableMap; +import com.google.errorprone.annotations.DoNotCall; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.expression.env.Environment; + +public class HighlightExpressionTest extends ExpressionTestBase { + + @Test + public void single_highlight_test() { + Environment hlTuple = ExprValueUtils.tupleValue( + ImmutableMap.of("_highlight.Title", "result value")).bindingTuples(); + HighlightExpression expr = new HighlightExpression(DSL.literal("Title")); + ExprValue resultVal = expr.valueOf(hlTuple); + + assertEquals(expr.type(), ARRAY); + assertEquals("result value", resultVal.stringValue()); + } + + @Test + public void missing_highlight_test() { + Environment hlTuple = ExprValueUtils.tupleValue( + ImmutableMap.of("_highlight.Title", "result value")).bindingTuples(); + HighlightExpression expr = new HighlightExpression(DSL.literal("invalid")); + ExprValue resultVal = expr.valueOf(hlTuple); + + assertTrue(resultVal.isMissing()); + } + + /** + * Enable me when '*' is supported in highlight. + */ + @DoNotCall + public void highlight_all_test() { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + var hlBuilder = ImmutableMap.builder(); + hlBuilder.put("Title", ExprValueUtils.stringValue("correct result value")); + hlBuilder.put("Body", ExprValueUtils.stringValue("incorrect result value")); + builder.put("_highlight", ExprTupleValue.fromExprValueMap(hlBuilder.build())); + + HighlightExpression hlExpr = new HighlightExpression(DSL.literal("*")); + ExprValue resultVal = hlExpr.valueOf( + ExprTupleValue.fromExprValueMap(builder.build()).bindingTuples()); + assertEquals(ARRAY, resultVal.type()); + for (var field : resultVal.tupleValue().entrySet()) { + assertTrue(field.toString().contains(hlExpr.getHighlightField().toString())); + } + assertTrue(resultVal.tupleValue().containsValue( + ExprValueUtils.stringValue("\"correct result value\""))); + assertTrue(resultVal.tupleValue().containsValue( + ExprValueUtils.stringValue("\"correct result value\""))); + } +} diff --git a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java index a2455a9a3d5..e899351d4f9 100644 --- a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java @@ -19,13 +19,14 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.tree.RareTopN.CommandType; import org.opensearch.sql.ast.tree.Sort.SortOption; +import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.aggregation.Aggregator; import org.opensearch.sql.expression.window.WindowDefinition; @@ -113,6 +114,11 @@ public void testAbstractPlanNodeVisitorShouldReturnNull() { assertNull(rareTopN.accept(new LogicalPlanNodeVisitor() { }, null)); + LogicalPlan highlight = new LogicalHighlight(filter, + new LiteralExpression(ExprValueUtils.stringValue("fieldA"))); + assertNull(highlight.accept(new LogicalPlanNodeVisitor() { + }, null)); + LogicalPlan mlCommons = new LogicalMLCommons(LogicalPlanDSL.relation("schema"), "kmeans", ImmutableMap.builder() diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 240bf32dfb4..f1d4d987a37 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -2479,3 +2479,29 @@ Another example to show how to set custom values for the optional parameters:: |------+--------------------------+----------------------| | 1 | The House at Pooh Corner | Alan Alexander Milne | +------+--------------------------+----------------------+ + + +HIGHLIGHT +------------ + +Description +>>>>>>>>>>> + +``highlight(field_expression)`` + +The highlight function maps to the highlight function used in search engine to return highlight fields for the given search. +The syntax allows to specify the field in double quotes or single quotes or without any wrap. +Please refer to examples below: + +| ``highlight(title)`` + +Example searching for field Tags:: + + os> select highlight(title) from books where query_string(['title'], 'Pooh House'); + fetched rows / total rows = 2/2 + +----------------------------------------------+ + | highlight(title) | + |----------------------------------------------| + | [The House at Pooh Corner] | + | [Winnie-the-Pooh] | + +----------------------------------------------+ diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/HighlightFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/HighlightFunctionIT.java new file mode 100644 index 00000000000..422f71968fb --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/sql/HighlightFunctionIT.java @@ -0,0 +1,88 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.sql; + +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import com.google.errorprone.annotations.DoNotCall; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.legacy.TestsConstants; + +public class HighlightFunctionIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.BEER); + } + + @Test + public void single_highlight_test() { + String query = "SELECT Tags, highlight('Tags') FROM %s WHERE match(Tags, 'yeast') LIMIT 1"; + JSONObject response = executeJdbcRequest(String.format(query, TestsConstants.TEST_INDEX_BEER)); + verifySchema(response, schema("Tags", null, "text"), + schema("highlight('Tags')", null, "nested")); + assertEquals(1, response.getInt("total")); + } + + @Test + public void accepts_unquoted_test() { + String query = "SELECT Tags, highlight(Tags) FROM %s WHERE match(Tags, 'yeast') LIMIT 1"; + JSONObject response = executeJdbcRequest(String.format(query, TestsConstants.TEST_INDEX_BEER)); + verifySchema(response, schema("Tags", null, "text"), + schema("highlight(Tags)", null, "nested")); + assertEquals(1, response.getInt("total")); + } + + @Test + public void multiple_highlight_test() { + String query = "SELECT highlight(Title), highlight(Body) FROM %s WHERE MULTI_MATCH([Title, Body], 'hops') LIMIT 1"; + JSONObject response = executeJdbcRequest(String.format(query, TestsConstants.TEST_INDEX_BEER)); + verifySchema(response, schema("highlight(Title)", null, "nested"), + schema("highlight(Body)", null, "nested")); + assertEquals(1, response.getInt("total")); + } + + // Enable me when * is supported + @DoNotCall + public void wildcard_highlight_test() { + String query = "SELECT highlight('*itle') FROM %s WHERE MULTI_MATCH([Title, Body], 'hops') LIMIT 1"; + JSONObject response = executeJdbcRequest(String.format(query, TestsConstants.TEST_INDEX_BEER)); + verifySchema(response, schema("highlight('*itle')", null, "nested")); + assertEquals(1, response.getInt("total")); + } + + // Enable me when * is supported + @DoNotCall + public void wildcard_multi_field_highlight_test() { + String query = "SELECT highlight('T*') FROM %s WHERE MULTI_MATCH([Title, Tags], 'hops') LIMIT 1"; + JSONObject response = executeJdbcRequest(String.format(query, TestsConstants.TEST_INDEX_BEER)); + verifySchema(response, schema("highlight('T*')", null, "nested")); + var resultMap = response.getJSONArray("datarows").getJSONArray(0).getJSONObject(0); + assertEquals(1, response.getInt("total")); + assertTrue(resultMap.has("highlight(\"T*\").Title")); + assertTrue(resultMap.has("highlight(\"T*\").Tags")); + } + + // Enable me when * is supported + @DoNotCall + public void highlight_all_test() { + String query = "SELECT highlight('*') FROM %s WHERE MULTI_MATCH([Title, Body], 'hops') LIMIT 1"; + JSONObject response = executeJdbcRequest(String.format(query, TestsConstants.TEST_INDEX_BEER)); + verifySchema(response, schema("highlight('*')", null, "nested")); + assertEquals(1, response.getInt("total")); + } + + @Test + public void highlight_no_limit_test() { + String query = "SELECT highlight(Body) FROM %s WHERE MATCH(Body, 'hops')"; + JSONObject response = executeJdbcRequest(String.format(query, TestsConstants.TEST_INDEX_BEER)); + verifySchema(response, schema("highlight(Body)", null, "nested")); + assertEquals(2, response.getInt("total")); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java index 7dc77d7d295..aadd73efdde 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java @@ -10,6 +10,7 @@ import java.util.Arrays; import java.util.Iterator; import java.util.Map; +import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.ToString; import org.opensearch.action.search.SearchResponse; @@ -17,6 +18,7 @@ import org.opensearch.search.aggregations.Aggregations; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; /** @@ -91,7 +93,23 @@ public Iterator iterator() { }).iterator(); } else { return Arrays.stream(hits.getHits()) - .map(hit -> (ExprValue) exprValueFactory.construct(hit.getSourceAsString())).iterator(); + .map(hit -> { + ExprValue docData = exprValueFactory.construct(hit.getSourceAsString()); + if (hit.getHighlightFields().isEmpty()) { + return docData; + } else { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + builder.putAll(docData.tupleValue()); + var hlBuilder = ImmutableMap.builder(); + for (var es : hit.getHighlightFields().entrySet()) { + hlBuilder.put(es.getKey(), ExprValueUtils.collectionValue( + Arrays.stream(es.getValue().fragments()).map( + t -> (t.toString())).collect(Collectors.toList()))); + } + builder.put("_highlight", ExprTupleValue.fromExprValueMap(hlBuilder.build())); + return ExprTupleValue.fromExprValueMap(builder.build()); + } + }).iterator(); } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java index 49301cbf536..c028f283a27 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java @@ -33,6 +33,7 @@ import org.opensearch.sql.opensearch.storage.serialization.DefaultExpressionSerializer; import org.opensearch.sql.planner.DefaultImplementor; import org.opensearch.sql.planner.logical.LogicalAD; +import org.opensearch.sql.planner.logical.LogicalHighlight; import org.opensearch.sql.planner.logical.LogicalMLCommons; import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.logical.LogicalRelation; @@ -187,5 +188,11 @@ public PhysicalPlan visitAD(LogicalAD node, OpenSearchIndexScan context) { return new ADOperator(visitChild(node, context), node.getArguments(), client.getNodeClient()); } + + @Override + public PhysicalPlan visitHighlight(LogicalHighlight node, OpenSearchIndexScan context) { + context.pushDownHighlight(node.getHighlightField().toString()); + return visitChild(node, context); + } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScan.java index c35a5ba9dbf..6e88f3de891 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScan.java @@ -25,8 +25,10 @@ import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; import org.opensearch.search.sort.SortBuilder; import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.ReferenceExpression; @@ -158,6 +160,21 @@ public void pushDownLimit(Integer limit, Integer offset) { sourceBuilder.from(offset).size(limit); } + /** + * Add highlight to DSL requests. + * @param field name of the field to highlight + */ + public void pushDownHighlight(String field) { + SearchSourceBuilder sourceBuilder = request.getSourceBuilder(); + if (sourceBuilder.highlighter() != null) { + sourceBuilder.highlighter().field(StringUtils.unquoteText(field)); + } else { + HighlightBuilder highlightBuilder = + new HighlightBuilder().field(StringUtils.unquoteText(field)); + sourceBuilder.highlighter(highlightBuilder); + } + } + /** * Push down project list to DSL requets. */ diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java index 5bffa1cfa86..ee981a4abca 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java @@ -36,7 +36,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.client.node.NodeClient; -import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.tree.RareTopN.CommandType; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java index fa25b4f4086..0a60503415d 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java @@ -17,18 +17,24 @@ import com.google.common.collect.ImmutableMap; import java.util.Arrays; +import java.util.Map; +import java.util.stream.Collectors; import org.apache.lucene.search.TotalHits; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.bytes.BytesArray; +import org.opensearch.common.text.Text; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.Aggregations; +import org.opensearch.search.fetch.subphase.highlight.HighlightField; import org.opensearch.sql.data.model.ExprIntegerValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; @@ -148,4 +154,35 @@ void aggregation_iterator() { i++; } } + + @Test + void highlight_iterator() { + SearchHit searchHit = new SearchHit(1); + searchHit.sourceRef( + new BytesArray("{\"name\":\"John\"}")); + Map highlightMap = Map.of("highlights", + new HighlightField("Title", new Text[] {new Text("field")})); + searchHit.highlightFields(Map.of("highlights", new HighlightField("Title", + new Text[] {new Text("field")}))); + ExprValue resultTuple = ExprValueUtils.tupleValue(searchHit.getSourceAsMap()); + + when(searchResponse.getHits()) + .thenReturn( + new SearchHits( + new SearchHit[]{searchHit1}, + new TotalHits(1L, TotalHits.Relation.EQUAL_TO), + 1.0F)); + + when(searchHit1.getHighlightFields()).thenReturn(highlightMap); + when(factory.construct(any())).thenReturn(resultTuple); + + for (ExprValue resultHit : new OpenSearchResponse(searchResponse, factory)) { + var expected = ExprValueUtils.collectionValue( + Arrays.stream(searchHit.getHighlightFields().get("highlights").getFragments()) + .map(t -> (t.toString())).collect(Collectors.toList())); + var result = resultHit.tupleValue().get( + "_highlight").tupleValue().get("highlights"); + assertTrue(expected.equals(result)); + } + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchDefaultImplementorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchDefaultImplementorTest.java index 0770ea3938c..c83172955c1 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchDefaultImplementorTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchDefaultImplementorTest.java @@ -9,6 +9,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.relation; import org.junit.jupiter.api.Test; @@ -19,6 +21,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.planner.logical.LogicalAD; +import org.opensearch.sql.planner.logical.LogicalHighlight; import org.opensearch.sql.planner.logical.LogicalMLCommons; import org.opensearch.sql.planner.logical.LogicalPlan; @@ -67,4 +70,16 @@ public void visitAD() { new OpenSearchIndex.OpenSearchDefaultImplementor(indexScan, client); assertNotNull(implementor.visitAD(node, indexScan)); } + + @Test + public void visitHighlight() { + LogicalHighlight node = Mockito.mock(LogicalHighlight.class, + Answers.RETURNS_DEEP_STUBS); + Mockito.when(node.getChild().get(0)).thenReturn(Mockito.mock(LogicalPlan.class)); + OpenSearchIndex.OpenSearchDefaultImplementor implementor = + new OpenSearchIndex.OpenSearchDefaultImplementor(indexScan, client); + + implementor.visitHighlight(node, indexScan); + verify(indexScan).pushDownHighlight(any()); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScanTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScanTest.java index 429c639da98..41769914d95 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScanTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScanTest.java @@ -31,6 +31,7 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; +import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; @@ -110,6 +111,16 @@ void pushDownFilters() { .filter(QueryBuilders.rangeQuery("balance").gte(10000))); } + @Test + void pushDownHighlight() { + assertThat() + .pushDown(QueryBuilders.termQuery("name", "John")) + .pushDownHighlight("Title") + .pushDownHighlight("Body") + .shouldQueryHighlight(QueryBuilders.termQuery("name", "John"), + new HighlightBuilder().field("Title").field("Body")); + } + private PushDownAssertion assertThat() { return new PushDownAssertion(client, exprValueFactory, settings); } @@ -135,6 +146,22 @@ PushDownAssertion pushDown(QueryBuilder query) { return this; } + PushDownAssertion pushDownHighlight(String query) { + indexScan.pushDownHighlight(query); + return this; + } + + PushDownAssertion shouldQueryHighlight(QueryBuilder query, HighlightBuilder highlight) { + OpenSearchRequest request = new OpenSearchQueryRequest("test", 200, factory); + request.getSourceBuilder() + .query(query) + .highlighter(highlight) + .sort(DOC_FIELD_NAME, ASC); + when(client.search(request)).thenReturn(response); + indexScan.open(); + return this; + } + PushDownAssertion shouldQuery(QueryBuilder expected) { OpenSearchRequest request = new OpenSearchQueryRequest("test", 200, factory); request.getSourceBuilder() diff --git a/sql/src/main/antlr/OpenSearchSQLLexer.g4 b/sql/src/main/antlr/OpenSearchSQLLexer.g4 index 8b74e31aacc..9fca2942cff 100644 --- a/sql/src/main/antlr/OpenSearchSQLLexer.g4 +++ b/sql/src/main/antlr/OpenSearchSQLLexer.g4 @@ -346,6 +346,7 @@ TIE_BREAKER: 'TIE_BREAKER'; TIME_ZONE: 'TIME_ZONE'; TYPE: 'TYPE'; ZERO_TERMS_QUERY: 'ZERO_TERMS_QUERY'; +HIGHLIGHT: 'HIGHLIGHT'; // RELEVANCE FUNCTIONS MATCH_BOOL_PREFIX: 'MATCH_BOOL_PREFIX'; diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index d75893b6961..a9316b55a4d 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -300,6 +300,11 @@ functionCall | aggregateFunction #aggregateFunctionCall | aggregateFunction (orderByClause)? filterClause #filteredAggregationFunctionCall | relevanceFunction #relevanceFunctionCall + | highlightFunction #highlightFunctionCall + ; + +highlightFunction + : HIGHLIGHT LR_BRACKET relevanceField RR_BRACKET ; scalarFunctionName diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index 3c686b5e8e3..453162e3356 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -63,6 +63,7 @@ import org.opensearch.sql.ast.expression.Cast; import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Function; +import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.Interval; import org.opensearch.sql.ast.expression.IntervalUnit; import org.opensearch.sql.ast.expression.Literal; @@ -130,6 +131,12 @@ public UnresolvedExpression visitScalarFunctionCall(ScalarFunctionCallContext ct return visitFunction(ctx.scalarFunctionName().getText(), ctx.functionArgs()); } + @Override + public UnresolvedExpression visitHighlightFunctionCall( + OpenSearchSQLParser.HighlightFunctionCallContext ctx) { + return new HighlightFunction(visit(ctx.highlightFunction().relevanceField())); + } + @Override public UnresolvedExpression visitTableFilter(TableFilterContext ctx) { return new Function( diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/HighlightTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/HighlightTest.java new file mode 100644 index 00000000000..6826a37c0b5 --- /dev/null +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/HighlightTest.java @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.sql.antlr; + +import org.junit.jupiter.api.Test; + +public class HighlightTest extends SQLParserTest { + @Test + void single_field_test() { + acceptQuery("SELECT HIGHLIGHT(Tags) FROM Index WHERE MATCH(Tags, 'Time')"); + } + + @Test + void multiple_highlights_test() { + acceptQuery("SELECT HIGHLIGHT(Tags), HIGHLIGHT(Body) FROM Index " + + "WHERE MULTI_MATCH([Tags, Body], 'Time')"); + } + + @Test + void wildcard_test() { + acceptQuery("SELECT HIGHLIGHT('T*') FROM Index " + + "WHERE MULTI_MATCH([Tags, Body], 'Time')"); + } + + @Test + void highlight_all_test() { + + acceptQuery("SELECT HIGHLIGHT('*') FROM Index WHERE MULTI_MATCH([Tags, Body], 'Time')"); + } + + @Test + void multiple_parameters_failure_test() { + rejectQuery("SELECT HIGHLIGHT(Tags1, Tags2) FROM Index " + + "WHERE MULTI_MATCH([Tags, Body], 'Time')"); + } + + @Test + void no_parameters_failure_test() { + rejectQuery("SELECT HIGHLIGHT() FROM Index " + + "WHERE MULTI_MATCH([Tags, Body], 'Time')"); + } +} diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java index d576389595a..8bf38b14a67 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java @@ -18,6 +18,7 @@ import static org.opensearch.sql.ast.dsl.AstDSL.field; import static org.opensearch.sql.ast.dsl.AstDSL.filter; import static org.opensearch.sql.ast.dsl.AstDSL.function; +import static org.opensearch.sql.ast.dsl.AstDSL.highlight; import static org.opensearch.sql.ast.dsl.AstDSL.intLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.limit; import static org.opensearch.sql.ast.dsl.AstDSL.project; @@ -33,6 +34,7 @@ import com.google.common.collect.ImmutableList; import org.antlr.v4.runtime.tree.ParseTree; import org.junit.jupiter.api.Test; +import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.antlr.SyntaxCheckException; @@ -667,6 +669,24 @@ public void can_build_limit_clause_with_offset() { buildAST("SELECT name FROM test LIMIT 5, 10")); } + @Test + public void can_build_qualified_name_highlight() { + assertEquals( + project(relation("test"), + alias("highlight(fieldA)", highlight(AstDSL.qualifiedName("fieldA")))), + buildAST("SELECT highlight(fieldA) FROM test") + ); + } + + @Test + public void can_build_string_literal_highlight() { + assertEquals( + project(relation("test"), + alias("highlight(\"fieldA\")", highlight(AstDSL.stringLiteral("fieldA")))), + buildAST("SELECT highlight(\"fieldA\") FROM test") + ); + } + private UnresolvedPlan buildAST(String query) { ParseTree parseTree = parser.parse(query); return parseTree.accept(new AstBuilder(query)); diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java index 9ce3dd6714d..ef881275e5b 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -15,6 +15,7 @@ import static org.opensearch.sql.ast.dsl.AstDSL.doubleLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.floatLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.function; +import static org.opensearch.sql.ast.dsl.AstDSL.highlight; import static org.opensearch.sql.ast.dsl.AstDSL.intLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.intervalLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.longLiteral; @@ -308,6 +309,22 @@ public void canBuildWindowFunctionWithNullOrderSpecified() { buildExprAst("DENSE_RANK() OVER (ORDER BY age ASC NULLS LAST)")); } + @Test + public void canBuildStringLiteralHighlightFunction() { + assertEquals( + highlight(AstDSL.stringLiteral("fieldA")), + buildExprAst("highlight(\"fieldA\")") + ); + } + + @Test + public void canBuildQualifiedNameHighlightFunction() { + assertEquals( + highlight(AstDSL.qualifiedName("fieldA")), + buildExprAst("highlight(fieldA)") + ); + } + @Test public void canBuildWindowFunctionWithoutOrderBy() { assertEquals( From df3238488673591d4dd6bc76879e7df0ed352a76 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Fri, 5 Aug 2022 16:45:57 -0700 Subject: [PATCH 05/20] Bump version to 2.2.0 (#729) Signed-off-by: penghuo --- .github/workflows/draft-release-notes-workflow.yml | 2 +- .github/workflows/sql-odbc-release-workflow.yml | 2 +- .github/workflows/sql-workbench-release-workflow.yml | 2 +- .../workflows/sql-workbench-test-and-build-workflow.yml | 2 +- build.gradle | 2 +- core/build.gradle | 6 +++--- integ-test/build.gradle | 6 +++--- opensearch/build.gradle | 6 +++--- plugin/build.gradle | 8 ++++---- ppl/build.gradle | 4 ++-- sql-jdbc/build.gradle | 2 +- sql/build.gradle | 4 ++-- workbench/opensearch_dashboards.json | 2 +- workbench/package.json | 2 +- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml index 6497e94c95a..b0b92441b16 100644 --- a/.github/workflows/draft-release-notes-workflow.yml +++ b/.github/workflows/draft-release-notes-workflow.yml @@ -16,6 +16,6 @@ jobs: with: config-name: draft-release-notes-config.yml tag: (None) - version: 2.1.0.0 + version: 2.2.0.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/sql-odbc-release-workflow.yml b/.github/workflows/sql-odbc-release-workflow.yml index 83d4f89b0c2..00920fffd77 100644 --- a/.github/workflows/sql-odbc-release-workflow.yml +++ b/.github/workflows/sql-odbc-release-workflow.yml @@ -12,7 +12,7 @@ env: ODBC_BUILD_PATH: "./build/odbc/build" AWS_SDK_INSTALL_PATH: "./build/aws-sdk/install" PLUGIN_NAME: opensearch-sql-odbc - OD_VERSION: 2.1.0.0 + OD_VERSION: 2.2.0.0 jobs: build-mac: diff --git a/.github/workflows/sql-workbench-release-workflow.yml b/.github/workflows/sql-workbench-release-workflow.yml index 35a6396a850..ef23bff98a0 100644 --- a/.github/workflows/sql-workbench-release-workflow.yml +++ b/.github/workflows/sql-workbench-release-workflow.yml @@ -8,7 +8,7 @@ on: env: PLUGIN_NAME: query-workbench-dashboards OPENSEARCH_VERSION: 'main' - OPENSEARCH_PLUGIN_VERSION: 2.1.0.0 + OPENSEARCH_PLUGIN_VERSION: 2.2.0.0 jobs: diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index d4c3f12996c..c0ae593c1d4 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -5,7 +5,7 @@ on: [pull_request, push] env: PLUGIN_NAME: query-workbench-dashboards OPENSEARCH_VERSION: 'main' - OPENSEARCH_PLUGIN_VERSION: 2.1.0.0 + OPENSEARCH_PLUGIN_VERSION: 2.2.0.0 jobs: diff --git a/build.gradle b/build.gradle index 9e2c1c6b6d8..28ecdc9dbe2 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "2.1.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.2.0-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') diff --git a/core/build.gradle b/core/build.gradle index 3ecc7a0ef9d..342d5673cd7 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -40,8 +40,8 @@ repositories { dependencies { api group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' - api group: 'org.springframework', name: 'spring-context', version: '5.2.20.RELEASE' - api group: 'org.springframework', name: 'spring-beans', version: '5.2.20.RELEASE' + api group: 'org.springframework', name: 'spring-context', version: '5.3.22' + api group: 'org.springframework', name: 'spring-beans', version: '5.3.22' api group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' api group: 'com.facebook.presto', name: 'presto-matching', version: '0.240' api group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1' @@ -49,7 +49,7 @@ dependencies { testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' - testImplementation group: 'org.springframework', name: 'spring-test', version: '5.2.20.RELEASE' + testImplementation group: 'org.springframework', name: 'spring-test', version: '5.3.22' testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.12.4' testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.12.4' } diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 0baf1af6cc8..864b4df0971 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -53,9 +53,9 @@ configurations.all { // enforce 1.1.3, https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379 resolutionStrategy.force 'commons-codec:commons-codec:1.13' resolutionStrategy.force 'com.google.guava:guava:31.0.1-jre' - resolutionStrategy.force 'com.fasterxml.jackson.core:jackson-core:2.13.2' - resolutionStrategy.force 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.13.2' - resolutionStrategy.force 'com.fasterxml.jackson.core:jackson-databind:2.13.2.2' + resolutionStrategy.force 'com.fasterxml.jackson.core:jackson-core:2.13.3' + resolutionStrategy.force 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.13.3' + resolutionStrategy.force 'com.fasterxml.jackson.core:jackson-databind:2.13.3' } dependencies { diff --git a/opensearch/build.gradle b/opensearch/build.gradle index e4489330050..2b26943d5bf 100644 --- a/opensearch/build.gradle +++ b/opensearch/build.gradle @@ -32,9 +32,9 @@ dependencies { api project(':core') api group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" implementation "io.github.resilience4j:resilience4j-retry:1.5.0" - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.13.2' - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.13.2.2' - implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: '2.13.2' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.13.3' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.13.3' + implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: '2.13.3' implementation group: 'org.json', name: 'json', version:'20180813' compileOnly group: 'org.opensearch.client', name: 'opensearch-rest-high-level-client', version: "${opensearch_version}" implementation group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}" diff --git a/plugin/build.gradle b/plugin/build.gradle index 8e59efcfb29..5c4d635ef02 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -82,12 +82,12 @@ configurations.all { // conflict with spring-jcl exclude group: "commons-logging", module: "commons-logging" // enforce 2.12.6, https://github.com/opensearch-project/sql/issues/424 - resolutionStrategy.force 'com.fasterxml.jackson.core:jackson-core:2.13.2' + resolutionStrategy.force 'com.fasterxml.jackson.core:jackson-core:2.13.3' // enforce 1.1.3, https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379 resolutionStrategy.force 'commons-codec:commons-codec:1.13' resolutionStrategy.force 'com.google.guava:guava:31.0.1-jre' - resolutionStrategy.force 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.13.2' - resolutionStrategy.force 'com.fasterxml.jackson.core:jackson-databind:2.13.2.2' + resolutionStrategy.force 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.13.3' + resolutionStrategy.force 'com.fasterxml.jackson.core:jackson-databind:2.13.3' } compileJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) @@ -98,7 +98,7 @@ compileTestJava { } dependencies { - api group: 'org.springframework', name: 'spring-beans', version: '5.2.20.RELEASE' + api group: 'org.springframework', name: 'spring-beans', version: '5.3.22' api project(":ppl") api project(':legacy') api project(':opensearch') diff --git a/ppl/build.gradle b/ppl/build.gradle index c4becfa3d87..1fb4c776425 100644 --- a/ppl/build.gradle +++ b/ppl/build.gradle @@ -48,8 +48,8 @@ dependencies { implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' api group: 'org.opensearch', name: 'opensearch-x-content', version: "${opensearch_version}" api group: 'org.json', name: 'json', version: '20180813' - implementation group: 'org.springframework', name: 'spring-context', version: '5.2.20.RELEASE' - implementation group: 'org.springframework', name: 'spring-beans', version: '5.2.20.RELEASE' + implementation group: 'org.springframework', name: 'spring-context', version: '5.3.22' + implementation group: 'org.springframework', name: 'spring-beans', version: '5.3.22' implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.17.1' api project(':common') api project(':core') diff --git a/sql-jdbc/build.gradle b/sql-jdbc/build.gradle index 0937619dc18..6af5e28c286 100644 --- a/sql-jdbc/build.gradle +++ b/sql-jdbc/build.gradle @@ -24,7 +24,7 @@ plugins { group 'org.opensearch.client' // keep version in sync with version in Driver source -version '2.1.0.0' +version '2.2.0.0' boolean snapshot = "true".equals(System.getProperty("build.snapshot", "false")); if (snapshot) { diff --git a/sql/build.gradle b/sql/build.gradle index d04e28c03d7..e6d1a65a53a 100644 --- a/sql/build.gradle +++ b/sql/build.gradle @@ -47,8 +47,8 @@ dependencies { implementation "org.antlr:antlr4-runtime:4.7.1" implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' implementation group: 'org.json', name: 'json', version:'20180813' - implementation group: 'org.springframework', name: 'spring-context', version: '5.2.20.RELEASE' - implementation group: 'org.springframework', name: 'spring-beans', version: '5.2.20.RELEASE' + implementation group: 'org.springframework', name: 'spring-context', version: '5.3.22' + implementation group: 'org.springframework', name: 'spring-beans', version: '5.3.22' implementation project(':common') implementation project(':core') api project(':protocol') diff --git a/workbench/opensearch_dashboards.json b/workbench/opensearch_dashboards.json index df1aa01300e..1599f5e4a82 100644 --- a/workbench/opensearch_dashboards.json +++ b/workbench/opensearch_dashboards.json @@ -1,7 +1,7 @@ { "id": "queryWorkbenchDashboards", "version": "2.1.0.0", - "opensearchDashboardsVersion": "2.1.0", + "opensearchDashboardsVersion": "2.2.0", "server": true, "ui": true, "requiredPlugins": ["navigation"], diff --git a/workbench/package.json b/workbench/package.json index 0ba5a3ad0ca..74cf2c9f410 100644 --- a/workbench/package.json +++ b/workbench/package.json @@ -1,6 +1,6 @@ { "name": "opensearch-query-workbench", - "version": "2.1.0.0", + "version": "2.2.0.0", "description": "Query Workbench", "main": "index.js", "license": "Apache-2.0", From 1df89e3a0dbe4991cae5ecd726d1d69ca3a6c565 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Fri, 5 Aug 2022 18:07:59 -0700 Subject: [PATCH 06/20] Add release notes for 2.2.0.0 (#730) Signed-off-by: Joshua Li --- .../opensearch-sql.release-notes-2.2.0.0.md | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 release-notes/opensearch-sql.release-notes-2.2.0.0.md diff --git a/release-notes/opensearch-sql.release-notes-2.2.0.0.md b/release-notes/opensearch-sql.release-notes-2.2.0.0.md new file mode 100644 index 00000000000..e7f2e6ee260 --- /dev/null +++ b/release-notes/opensearch-sql.release-notes-2.2.0.0.md @@ -0,0 +1,32 @@ +### Version 2.2.0.0 Release Notes + +Compatible with OpenSearch and OpenSearch Dashboards Version 2.2.0 + +### Features +* Add match_phrase_prefix ([#661](https://github.com/opensearch-project/sql/pull/661)) +* `query_string` Relevance Function Implementation in SQL and PPL ([#675](https://github.com/opensearch-project/sql/pull/675)) +* Add Highlight In SQL ([#717](https://github.com/opensearch-project/sql/pull/717)) + +### Enhancements +* implement transport api for PPL inter-plugin communication ([#533](https://github.com/opensearch-project/sql/pull/533)) +* Two single or double quote escapes single or double quote when string is surrounded by same type of quote ([#696](https://github.com/opensearch-project/sql/pull/696)) + +### Bug Fixes +* Reverted UseSSL flag to false and removed invalid test case ([#671](https://github.com/opensearch-project/sql/pull/671)) +* Update BI connectors and drivers readme files ([#665](https://github.com/opensearch-project/sql/pull/665)) +* Bump moment from 2.29.2 to 2.29.4 in /workbench ([#702](https://github.com/opensearch-project/sql/pull/702)) + +### Infrastructure +* Staging for version increment automation ([#684](https://github.com/opensearch-project/sql/pull/684)) +* Update tests and test data for relevancy search functions ([#707](https://github.com/opensearch-project/sql/pull/707)) +* Remove ODFE BWC tests ([#721](https://github.com/opensearch-project/sql/pull/721)) +* Github Actions fix for reference to OpenSearch-Dashboard not existing outside of main OS sql project ([#704](https://github.com/opensearch-project/sql/pull/704)) + +### Documentation +* Update support link for Tableau connector ([#643](https://github.com/opensearch-project/sql/pull/643)) +* Fix broken forum link ([#694](https://github.com/opensearch-project/sql/pull/694)) +* Fix links in the doc file. ([#705](https://github.com/opensearch-project/sql/pull/705)) + +### Maintenance +* change version bump under maintenance ([#679](https://github.com/opensearch-project/sql/pull/679)) +* Bump version to 2.2.0 ([#729](https://github.com/opensearch-project/sql/pull/729)) From 792d3ff8f4d594c1f6f54c55b0b0231e9a922bf4 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Mon, 8 Aug 2022 10:57:41 -0700 Subject: [PATCH 07/20] Bump workbench version to 2.2.0.0 (#736) Signed-off-by: penghuo --- workbench/opensearch_dashboards.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workbench/opensearch_dashboards.json b/workbench/opensearch_dashboards.json index 1599f5e4a82..b992549d7da 100644 --- a/workbench/opensearch_dashboards.json +++ b/workbench/opensearch_dashboards.json @@ -1,6 +1,6 @@ { "id": "queryWorkbenchDashboards", - "version": "2.1.0.0", + "version": "2.2.0.0", "opensearchDashboardsVersion": "2.2.0", "server": true, "ui": true, From 921a28cae8352dd5a934cd6166a18c775b5b8d53 Mon Sep 17 00:00:00 2001 From: MitchellGale-BitQuill <104795536+MitchellGale-BitQuill@users.noreply.github.com> Date: Mon, 8 Aug 2022 16:43:11 -0700 Subject: [PATCH 08/20] Adding documentation about double quote implementation (#723) Signed-off-by: mitchellg Co-authored-by: Yury Fridlyand --- docs/user/general/datatypes.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/user/general/datatypes.rst b/docs/user/general/datatypes.rst index dd8cc68a442..63b57f87e89 100644 --- a/docs/user/general/datatypes.rst +++ b/docs/user/general/datatypes.rst @@ -353,16 +353,15 @@ A string can also represent and be converted to date and time types (except to i String Data Types ================= -A string is a sequence of characters enclosed in either single or double quotes. For example, both 'text' and "text" will be treated as string literal. To use quote characters in a string literal, you can include double quotes within single quoted string or single quotes within double quoted string:: +A string is a sequence of characters enclosed in either single or double quotes. For example, both 'text' and "text" will be treated as string literal. To use quote characters in a string literal, you can use two quotes of the same type as the enclosing quotes:: - os> SELECT 'hello', "world", '"hello"', "'world'" + os> SELECT 'hello', "world", '"hello"', "'world'", '''hello''', """world""" fetched rows / total rows = 1/1 - +-----------+-----------+-------------+-------------+ - | 'hello' | "world" | '"hello"' | "'world'" | - |-----------+-----------+-------------+-------------| - | hello | world | "hello" | 'world' | - +-----------+-----------+-------------+-------------+ - + +-----------+-----------+-------------+-------------+---------------+---------------+ + | 'hello' | "world" | '"hello"' | "'world'" | '''hello''' | """world""" | + |-----------+-----------+-------------+-------------+---------------+---------------| + | hello | world | "hello" | 'world' | 'hello' | "world" | + +-----------+-----------+-------------+-------------+---------------+---------------+ Boolean Data Types ================== From e6a5ac91489059cd02a9d53917d4dda275f25500 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 9 Aug 2022 10:50:20 -0700 Subject: [PATCH 09/20] Update snapshots for jest tests on workbench (#740) Signed-off-by: Joshua Li --- .../Main/__snapshots__/main.test.tsx.snap | 101 +- .../__snapshots__/QueryResults.test.tsx.snap | 938 ++++++++++++++++-- .../QueryResultsBody.test.tsx.snap | 662 +++++++++++- 3 files changed, 1578 insertions(+), 123 deletions(-) diff --git a/workbench/public/components/Main/__snapshots__/main.test.tsx.snap b/workbench/public/components/Main/__snapshots__/main.test.tsx.snap index f40a1813a76..defafbdc77b 100644 --- a/workbench/public/components/Main/__snapshots__/main.test.tsx.snap +++ b/workbench/public/components/Main/__snapshots__/main.test.tsx.snap @@ -99,7 +99,20 @@ exports[`
spec click clear button 1`] = ` - EuiIconMock + @@ -470,7 +483,20 @@ exports[`
spec click run button, and response causes an error 1`] = ` - EuiIconMock + @@ -841,7 +867,20 @@ exports[`
spec click run button, and response is not ok 1`] = ` - EuiIconMock + @@ -1212,7 +1251,20 @@ exports[`
spec click run button, and response is ok 1`] = ` - EuiIconMock + @@ -1583,7 +1635,20 @@ exports[`
spec click run button, response fills null and missing values - EuiIconMock + @@ -1957,7 +2022,20 @@ exports[`
spec click translation button, and response is ok 1`] = ` - EuiIconMock + @@ -2328,7 +2406,16 @@ exports[`
spec renders the component 1`] = ` - EuiIconMock + diff --git a/workbench/public/components/QueryResults/__snapshots__/QueryResults.test.tsx.snap b/workbench/public/components/QueryResults/__snapshots__/QueryResults.test.tsx.snap index ece3aaed624..3b5dbbc6036 100644 --- a/workbench/public/components/QueryResults/__snapshots__/QueryResults.test.tsx.snap +++ b/workbench/public/components/QueryResults/__snapshots__/QueryResults.test.tsx.snap @@ -36,7 +36,21 @@ exports[` spec renders the component to test tabs down - EuiIconMock + @@ -216,7 +230,21 @@ exports[` spec renders the component to test tabs down
- EuiIconMock +
@@ -277,7 +305,20 @@ exports[` spec renders the component to test tabs down - EuiIconMock + @@ -307,7 +348,21 @@ exports[` spec renders the component to test tabs down - EuiIconMock + @@ -778,7 +833,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -1146,7 +1215,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -1514,7 +1597,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -1882,7 +1979,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -2250,7 +2361,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -2618,7 +2743,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -2986,7 +3125,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -3354,7 +3507,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -3722,7 +3889,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -4090,7 +4271,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -4468,7 +4663,21 @@ exports[` spec renders the component to test tabs down - EuiIconMock + @@ -4494,7 +4703,21 @@ exports[` spec renders the component to test tabs down disabled="" type="button" > - EuiIconMock +
    spec renders the component to test tabs down data-test-subj="pagination-button-next" type="button" > - EuiIconMock + @@ -4591,7 +4828,21 @@ exports[` spec renders the component to test tabs down - EuiIconMock + @@ -4771,7 +5022,21 @@ exports[` spec renders the component to test tabs down
    - EuiIconMock +
    @@ -5240,7 +5505,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock +     @@ -5608,7 +5887,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock +     @@ -5976,7 +6269,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -6344,7 +6651,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -6712,7 +7033,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -7080,7 +7415,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -7448,7 +7797,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -7816,7 +8179,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -8184,7 +8561,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -8552,7 +8943,21 @@ exports[` spec renders the component to test tabs down style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -8930,7 +9335,21 @@ exports[` spec renders the component to test tabs down - EuiIconMock + @@ -8956,7 +9375,21 @@ exports[` spec renders the component to test tabs down disabled="" type="button" > - EuiIconMock +
      spec renders the component to test tabs down data-test-subj="pagination-button-next" type="button" > - EuiIconMock + @@ -9053,7 +9500,16 @@ exports[` spec renders the component with mock query re - EuiIconMock + @@ -9233,7 +9689,16 @@ exports[` spec renders the component with mock query re
      - EuiIconMock +
      @@ -9294,7 +9759,16 @@ exports[` spec renders the component with mock query re - EuiIconMock + @@ -9324,7 +9798,16 @@ exports[` spec renders the component with mock query re - EuiIconMock + @@ -9795,7 +10278,16 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -10163,7 +10655,16 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -10531,7 +11032,16 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock +       @@ -10899,7 +11409,16 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock +       @@ -11267,7 +11786,16 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -11635,7 +12163,16 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -12003,7 +12540,16 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -12371,7 +12917,16 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -12739,7 +13294,16 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -13107,7 +13671,16 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -13485,7 +14058,16 @@ exports[` spec renders the component with mock query re - EuiIconMock + @@ -13511,7 +14093,16 @@ exports[` spec renders the component with mock query re disabled="" type="button" > - EuiIconMock +
        spec renders the component with mock query re data-test-subj="pagination-button-next" type="button" > - EuiIconMock + @@ -13608,7 +14208,21 @@ exports[` spec renders the component with mock query re - EuiIconMock + @@ -13788,7 +14402,21 @@ exports[` spec renders the component with mock query re
        - EuiIconMock +
        @@ -14257,7 +14885,21 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock +         @@ -14625,7 +15267,21 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock +         @@ -14993,7 +15649,21 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -15361,7 +16031,21 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -15729,7 +16413,21 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -16097,7 +16795,21 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -16465,7 +17177,21 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -16833,7 +17559,21 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -17201,7 +17941,21 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -17569,7 +18323,21 @@ exports[` spec renders the component with mock query re style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -17947,7 +18715,21 @@ exports[` spec renders the component with mock query re - EuiIconMock + @@ -17973,7 +18755,21 @@ exports[` spec renders the component with mock query re disabled="" type="button" > - EuiIconMock +
          spec renders the component with mock query re data-test-subj="pagination-button-next" type="button" > - EuiIconMock + diff --git a/workbench/public/components/QueryResults/__snapshots__/QueryResultsBody.test.tsx.snap b/workbench/public/components/QueryResults/__snapshots__/QueryResultsBody.test.tsx.snap index 1463c90b055..5fff13c6dac 100644 --- a/workbench/public/components/QueryResults/__snapshots__/QueryResultsBody.test.tsx.snap +++ b/workbench/public/components/QueryResults/__snapshots__/QueryResultsBody.test.tsx.snap @@ -53,7 +53,16 @@ exports[` spec renders component with mock QueryResults data - EuiIconMock + @@ -83,7 +92,16 @@ exports[` spec renders component with mock QueryResults data - EuiIconMock + @@ -554,7 +572,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -922,7 +949,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -1290,7 +1326,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -1658,7 +1703,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -2026,7 +2080,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -2394,7 +2457,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -2762,7 +2834,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -3130,7 +3211,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -3498,7 +3588,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -3866,7 +3965,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -4234,7 +4342,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -4612,7 +4729,16 @@ exports[` spec renders component with mock QueryResults data - EuiIconMock + @@ -4638,7 +4764,16 @@ exports[` spec renders component with mock QueryResults data disabled="" type="button" > - EuiIconMock +
            spec renders component with mock QueryResults data data-test-subj="pagination-button-next" type="button" > - EuiIconMock + @@ -4712,7 +4856,16 @@ exports[` spec renders component with mock QueryResults data - EuiIconMock +
            spec renders component with mock QueryResults data class="euiFormControlLayoutClearButton" type="button" > - EuiIconMock +
            @@ -4753,7 +4915,16 @@ exports[` spec renders component with mock QueryResults data - EuiIconMock + @@ -5223,7 +5394,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -5620,7 +5800,21 @@ exports[` spec renders component with mock QueryResults data class="euiButtonIcon euiButtonIcon--primary euiButtonIcon--empty euiButtonIcon--xSmall euiSideNavItemButton__icon" type="button" > - EuiIconMock + spec renders component with mock QueryResults data class="euiButtonIcon euiButtonIcon--primary euiButtonIcon--empty euiButtonIcon--xSmall euiSideNavItemButton__icon" type="button" > - EuiIconMock + spec renders component with mock QueryResults data class="euiButtonIcon euiButtonIcon--primary euiButtonIcon--empty euiButtonIcon--xSmall euiSideNavItemButton__icon" type="button" > - EuiIconMock + spec renders component with mock QueryResults data class="euiButtonIcon euiButtonIcon--primary euiButtonIcon--empty euiButtonIcon--xSmall euiSideNavItemButton__icon" type="button" > - EuiIconMock + spec renders component with mock QueryResults data class="euiButtonIcon euiButtonIcon--primary euiButtonIcon--empty euiButtonIcon--xSmall euiSideNavItemButton__icon" type="button" > - EuiIconMock + spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -6204,7 +6458,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -6572,7 +6835,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -6940,7 +7212,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -7308,7 +7589,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -7676,7 +7966,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -8044,7 +8343,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -8412,7 +8720,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -8780,7 +9097,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -9148,7 +9474,16 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -9526,7 +9861,16 @@ exports[` spec renders component with mock QueryResults data - EuiIconMock + @@ -9552,7 +9896,16 @@ exports[` spec renders component with mock QueryResults data disabled="" type="button" > - EuiIconMock +
              spec renders component with mock QueryResults data data-test-subj="pagination-button-next" type="button" > - EuiIconMock + @@ -10034,7 +10396,21 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -10402,7 +10778,21 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -10770,7 +11160,21 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -11138,7 +11542,21 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -11506,7 +11924,21 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -11874,7 +12306,21 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -12242,7 +12688,21 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -12610,7 +13070,21 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -12978,7 +13452,21 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -13346,7 +13834,21 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -13714,7 +14216,21 @@ exports[` spec renders component with mock QueryResults data style="margin-left: -4px;" type="button" > - EuiIconMock + @@ -14092,7 +14608,21 @@ exports[` spec renders component with mock QueryResults data - EuiIconMock + @@ -14118,7 +14648,21 @@ exports[` spec renders component with mock QueryResults data disabled="" type="button" > - EuiIconMock +
                spec renders component with mock QueryResults data data-test-subj="pagination-button-next" type="button" > - EuiIconMock + From 29f1c52de7139e1d65422830f015ad52557d822b Mon Sep 17 00:00:00 2001 From: Chen Dai <46505291+dai-chen@users.noreply.github.com> Date: Tue, 9 Aug 2022 11:04:00 -0700 Subject: [PATCH 10/20] Bump JDBC driver's dependency version (#735) * Bump wiremock version Signed-off-by: Chen Dai * Bump guava and json-smart version Signed-off-by: Chen Dai * Bump httpclient version Signed-off-by: Chen Dai --- sql-jdbc/build.gradle | 8 ++++++-- .../auth/aws/AWSRequestSigningApacheInterceptorTests.java | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/sql-jdbc/build.gradle b/sql-jdbc/build.gradle index 6af5e28c286..3dea2b49a7f 100644 --- a/sql-jdbc/build.gradle +++ b/sql-jdbc/build.gradle @@ -45,18 +45,22 @@ repositories { } dependencies { - implementation group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.6' + implementation group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.13' implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.13.2.2' implementation group: 'com.amazonaws', name: 'aws-java-sdk-core', version: '1.11.452' testImplementation('org.junit.jupiter:junit-jupiter-api:5.3.1') testImplementation('org.junit.jupiter:junit-jupiter-params:5.3.1') - testImplementation('com.github.tomakehurst:wiremock:2.20.0') + testImplementation('com.github.tomakehurst:wiremock:2.27.2') testImplementation('org.mockito:mockito-core:2.23.0') testImplementation('org.junit.jupiter:junit-jupiter-engine:5.3.1') testImplementation('org.junit-pioneer:junit-pioneer:0.3.0') testImplementation('org.eclipse.jetty:jetty-server:9.2.24.v20180105') + // Enforce wiremock to use latest guava and json-smart + testImplementation('com.google.guava:guava:31.1-jre') + testImplementation('net.minidev:json-smart:2.4.8') + testRuntimeOnly('org.slf4j:slf4j-simple:1.7.25') // capture WireMock logging } diff --git a/sql-jdbc/src/test/java/org/opensearch/jdbc/transport/http/auth/aws/AWSRequestSigningApacheInterceptorTests.java b/sql-jdbc/src/test/java/org/opensearch/jdbc/transport/http/auth/aws/AWSRequestSigningApacheInterceptorTests.java index 2ce90ca06b1..ec39e6f5807 100644 --- a/sql-jdbc/src/test/java/org/opensearch/jdbc/transport/http/auth/aws/AWSRequestSigningApacheInterceptorTests.java +++ b/sql-jdbc/src/test/java/org/opensearch/jdbc/transport/http/auth/aws/AWSRequestSigningApacheInterceptorTests.java @@ -24,8 +24,8 @@ import java.io.IOException; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; public class AWSRequestSigningApacheInterceptorTests { From 6fb3fac98f5917d13eb5cb37e10acc009f1cbee4 Mon Sep 17 00:00:00 2001 From: Chen Dai <46505291+dai-chen@users.noreply.github.com> Date: Thu, 11 Aug 2022 12:33:20 -0700 Subject: [PATCH 11/20] Bump Jackson dependency version (#745) * Bump jackson version and add version config in root gradle script Signed-off-by: Chen Dai * Bump jackson version in JDBC driver Signed-off-by: Chen Dai Signed-off-by: Chen Dai --- build.gradle | 2 ++ core/build.gradle | 6 +++--- integ-test/build.gradle | 6 +++--- opensearch/build.gradle | 6 +++--- plugin/build.gradle | 8 ++++---- ppl/build.gradle | 4 ++-- protocol/build.gradle | 8 ++++---- sql-jdbc/build.gradle | 2 +- sql/build.gradle | 4 ++-- 9 files changed, 24 insertions(+), 22 deletions(-) diff --git a/build.gradle b/build.gradle index 28ecdc9dbe2..8a2f9046bfc 100644 --- a/build.gradle +++ b/build.gradle @@ -7,6 +7,8 @@ buildscript { ext { opensearch_version = System.getProperty("opensearch.version", "2.2.0-SNAPSHOT") + spring_version = "5.3.22" + jackson_version = "2.13.3" isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') diff --git a/core/build.gradle b/core/build.gradle index 342d5673cd7..1fa3e19e269 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -40,8 +40,8 @@ repositories { dependencies { api group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' - api group: 'org.springframework', name: 'spring-context', version: '5.3.22' - api group: 'org.springframework', name: 'spring-beans', version: '5.3.22' + api group: 'org.springframework', name: 'spring-context', version: "${spring_version}" + api group: 'org.springframework', name: 'spring-beans', version: "${spring_version}" api group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' api group: 'com.facebook.presto', name: 'presto-matching', version: '0.240' api group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1' @@ -49,7 +49,7 @@ dependencies { testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' - testImplementation group: 'org.springframework', name: 'spring-test', version: '5.3.22' + testImplementation group: 'org.springframework', name: 'spring-test', version: "${spring_version}" testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.12.4' testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.12.4' } diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 864b4df0971..429c360a1ba 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -53,9 +53,9 @@ configurations.all { // enforce 1.1.3, https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379 resolutionStrategy.force 'commons-codec:commons-codec:1.13' resolutionStrategy.force 'com.google.guava:guava:31.0.1-jre' - resolutionStrategy.force 'com.fasterxml.jackson.core:jackson-core:2.13.3' - resolutionStrategy.force 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.13.3' - resolutionStrategy.force 'com.fasterxml.jackson.core:jackson-databind:2.13.3' + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${jackson_version}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jackson_version}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_version}" } dependencies { diff --git a/opensearch/build.gradle b/opensearch/build.gradle index 2b26943d5bf..8b5f917dffd 100644 --- a/opensearch/build.gradle +++ b/opensearch/build.gradle @@ -32,9 +32,9 @@ dependencies { api project(':core') api group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" implementation "io.github.resilience4j:resilience4j-retry:1.5.0" - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.13.3' - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.13.3' - implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: '2.13.3' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${jackson_version}" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_version}" + implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${jackson_version}" implementation group: 'org.json', name: 'json', version:'20180813' compileOnly group: 'org.opensearch.client', name: 'opensearch-rest-high-level-client', version: "${opensearch_version}" implementation group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}" diff --git a/plugin/build.gradle b/plugin/build.gradle index 5c4d635ef02..5c3b3974ef8 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -82,12 +82,12 @@ configurations.all { // conflict with spring-jcl exclude group: "commons-logging", module: "commons-logging" // enforce 2.12.6, https://github.com/opensearch-project/sql/issues/424 - resolutionStrategy.force 'com.fasterxml.jackson.core:jackson-core:2.13.3' + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${jackson_version}" // enforce 1.1.3, https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379 resolutionStrategy.force 'commons-codec:commons-codec:1.13' resolutionStrategy.force 'com.google.guava:guava:31.0.1-jre' - resolutionStrategy.force 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.13.3' - resolutionStrategy.force 'com.fasterxml.jackson.core:jackson-databind:2.13.3' + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jackson_version}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_version}" } compileJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) @@ -98,7 +98,7 @@ compileTestJava { } dependencies { - api group: 'org.springframework', name: 'spring-beans', version: '5.3.22' + api group: 'org.springframework', name: 'spring-beans', version: "${spring_version}" api project(":ppl") api project(':legacy') api project(':opensearch') diff --git a/ppl/build.gradle b/ppl/build.gradle index 1fb4c776425..12b0787efcd 100644 --- a/ppl/build.gradle +++ b/ppl/build.gradle @@ -48,8 +48,8 @@ dependencies { implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' api group: 'org.opensearch', name: 'opensearch-x-content', version: "${opensearch_version}" api group: 'org.json', name: 'json', version: '20180813' - implementation group: 'org.springframework', name: 'spring-context', version: '5.3.22' - implementation group: 'org.springframework', name: 'spring-beans', version: '5.3.22' + implementation group: 'org.springframework', name: 'spring-context', version: "${spring_version}" + implementation group: 'org.springframework', name: 'spring-beans', version: "${spring_version}" implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.17.1' api project(':common') api project(':core') diff --git a/protocol/build.gradle b/protocol/build.gradle index 7cca4aa0a96..fc35b94d343 100644 --- a/protocol/build.gradle +++ b/protocol/build.gradle @@ -30,9 +30,9 @@ plugins { dependencies { implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.13.2' - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.13.2.2' - implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: '2.13.2' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${jackson_version}" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_version}" + implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${jackson_version}" implementation 'com.google.code.gson:gson:2.8.9' implementation project(':core') implementation project(':opensearch') @@ -44,7 +44,7 @@ dependencies { } configurations.all { - resolutionStrategy.force 'com.fasterxml.jackson.core:jackson-databind:2.13.2.2' + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_version}" } test { diff --git a/sql-jdbc/build.gradle b/sql-jdbc/build.gradle index 3dea2b49a7f..dd629e438f6 100644 --- a/sql-jdbc/build.gradle +++ b/sql-jdbc/build.gradle @@ -46,7 +46,7 @@ repositories { dependencies { implementation group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.13' - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.13.2.2' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "2.13.3" implementation group: 'com.amazonaws', name: 'aws-java-sdk-core', version: '1.11.452' testImplementation('org.junit.jupiter:junit-jupiter-api:5.3.1') diff --git a/sql/build.gradle b/sql/build.gradle index e6d1a65a53a..222ad92ac62 100644 --- a/sql/build.gradle +++ b/sql/build.gradle @@ -47,8 +47,8 @@ dependencies { implementation "org.antlr:antlr4-runtime:4.7.1" implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' implementation group: 'org.json', name: 'json', version:'20180813' - implementation group: 'org.springframework', name: 'spring-context', version: '5.3.22' - implementation group: 'org.springframework', name: 'spring-beans', version: '5.3.22' + implementation group: 'org.springframework', name: 'spring-context', version: "${spring_version}" + implementation group: 'org.springframework', name: 'spring-beans', version: "${spring_version}" implementation project(':common') implementation project(':core') api project(':protocol') From f3c9c29538abdf93780e156edba3558fd43479da Mon Sep 17 00:00:00 2001 From: Sean Kao Date: Fri, 12 Aug 2022 09:54:31 -0700 Subject: [PATCH 12/20] Extend query size limit using scroll (#716) * add maxResultWindow to LogicalRelation Signed-off-by: Sean Kao * add maxResultWindow to OpenSearchLogicalIndexScan Signed-off-by: Sean Kao * OpenSearchRequestBuilder init Signed-off-by: Sean Kao * request builder: push down and build Signed-off-by: Sean Kao * plan.build() for building request Signed-off-by: Sean Kao * maxResultWindow for test utils Signed-off-by: Sean Kao * fix style Signed-off-by: Sean Kao * remove plan.build() Signed-off-by: Sean Kao * fetch result in batches Signed-off-by: Sean Kao * get index.max_result_window settings Signed-off-by: Sean Kao * use index.max_result_window to decide scroll Signed-off-by: Sean Kao * maxResultWindow for aggregation Signed-off-by: Sean Kao * fix fetch size & for aggregation query Signed-off-by: Sean Kao * fix rest client get max result window Signed-off-by: Sean Kao * remove maxResultWindow from logical plan Signed-off-by: Sean Kao * get max result window when building physical plan Signed-off-by: Sean Kao * move source builder init to request builder Signed-off-by: Sean Kao * fix max result window for test & rest client Signed-off-by: Sean Kao * include request builder in equal comparison Signed-off-by: Sean Kao * rename getIndexMaxResultWindows Signed-off-by: Sean Kao * open search rest client test Signed-off-by: Sean Kao * test: request builder, scroll index scan Signed-off-by: Sean Kao * fix style Signed-off-by: Sean Kao * remove getMaxResultWindow from base Table Signed-off-by: Sean Kao * remove unused import from OpenSearchIndexScan Signed-off-by: Sean Kao * test index scan Signed-off-by: Sean Kao * integ test for head command Signed-off-by: Sean Kao * keep request query size for aggregation Signed-off-by: Sean Kao * fix rest client test coverage Signed-off-by: Sean Kao * test for node client Signed-off-by: Sean Kao * test node client default settings Signed-off-by: Sean Kao * change Elasticsearch to OpenSearch in comment Signed-off-by: Sean Kao * fix comments Signed-off-by: Sean Kao * more test for Head IT Signed-off-by: Sean Kao * ignore some head IT Signed-off-by: Sean Kao Signed-off-by: Sean Kao --- .../sql/planner/logical/LogicalPlanDSL.java | 1 - .../sql/legacy/SQLIntegTestCase.java | 24 +++ .../org/opensearch/sql/ppl/HeadCommandIT.java | 72 +++++++ .../opensearch/client/OpenSearchClient.java | 8 + .../client/OpenSearchNodeClient.java | 26 +++ .../client/OpenSearchRestClient.java | 34 +++ .../request/OpenSearchQueryRequest.java | 16 +- .../request/OpenSearchRequestBuilder.java | 202 ++++++++++++++++++ .../request/OpenSearchScrollRequest.java | 14 +- .../OpenSearchDescribeIndexRequest.java | 10 + .../opensearch/storage/OpenSearchIndex.java | 34 ++- .../storage/OpenSearchIndexScan.java | 171 ++++----------- .../client/OpenSearchNodeClientTest.java | 59 +++++ .../client/OpenSearchRestClientTest.java | 64 ++++++ .../OpenSearchExecutionEngineTest.java | 3 +- .../OpenSearchExecutionProtectorTest.java | 5 +- .../request/OpenSearchRequestBuilderTest.java | 76 +++++++ .../OpenSearchDefaultImplementorTest.java | 5 +- .../storage/OpenSearchIndexScanTest.java | 93 +++++++- .../storage/OpenSearchIndexTest.java | 32 ++- .../sql/opensearch/utils/Utils.java | 4 +- .../test/resources/mappings/accounts2.json | 93 ++++++++ 22 files changed, 880 insertions(+), 166 deletions(-) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java create mode 100644 opensearch/src/test/resources/mappings/accounts2.json diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java index 4185d55c559..cdd3d3a103b 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java @@ -18,7 +18,6 @@ import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.NamedExpression; -import org.opensearch.sql.expression.ParseExpression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.aggregation.NamedAggregator; import org.opensearch.sql.expression.window.WindowDefinition; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 15f0261f0d0..5c339cc7bb4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -31,6 +31,7 @@ import java.nio.file.Paths; import java.util.Locale; +import static com.google.common.base.Strings.isNullOrEmpty; import static org.opensearch.sql.legacy.TestUtils.createIndexByRestClient; import static org.opensearch.sql.legacy.TestUtils.getAccountIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getBankIndexMapping; @@ -71,6 +72,8 @@ public abstract class SQLIntegTestCase extends OpenSearchSQLRestTestCase { public static final String TRANSIENT = "transient"; public static final Integer DEFAULT_QUERY_SIZE_LIMIT = Integer.parseInt(System.getProperty("defaultQuerySizeLimit", "200")); + public static final Integer DEFAULT_MAX_RESULT_WINDOW = + Integer.parseInt(System.getProperty("defaultMaxResultWindow", "10000")); public boolean shouldResetQuerySizeLimit() { return true; @@ -161,6 +164,15 @@ protected static void wipeAllClusterSettings() throws IOException { updateClusterSettings(new ClusterSetting("transient", "*", null)); } + protected void setMaxResultWindow(String indexName, Integer window) throws IOException { + updateIndexSettings(indexName, "{ \"index\": { \"max_result_window\":" + window + " } }"); + } + + protected void resetMaxResultWindow(String indexName) throws IOException { + updateIndexSettings(indexName, + "{ \"index\": { \"max_result_window\": " + DEFAULT_MAX_RESULT_WINDOW + " } }"); + } + /** * Provide for each test to load test index, data and other setup work */ @@ -378,6 +390,18 @@ public String toString() { } } + protected static JSONObject updateIndexSettings(String indexName, String setting) + throws IOException { + Request request = new Request("PUT", "/" + indexName + "/_settings"); + if (!isNullOrEmpty(setting)) { + request.setJsonEntity(setting); + } + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + return new JSONObject(executeRequest(request)); + } + protected String makeRequest(String query) { return makeRequest(query, 0); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/HeadCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/HeadCommandIT.java index 1ae45ab4694..48c489ce109 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/HeadCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/HeadCommandIT.java @@ -14,6 +14,7 @@ import org.json.JSONObject; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.jupiter.api.Test; public class HeadCommandIT extends PPLIntegTestCase { @@ -26,6 +27,7 @@ public void beforeTest() throws IOException { @After public void afterTest() throws IOException { resetQuerySizeLimit(); + resetMaxResultWindow(TEST_INDEX_ACCOUNT); } @Override @@ -60,6 +62,76 @@ public void testHeadWithNumber() throws IOException { rows("Nanette", 28)); } + @Ignore("Fix https://github.com/opensearch-project/sql/issues/703#issuecomment-1211422130") + @Test + public void testHeadWithNumberLargerThanQuerySizeLimit() throws IOException { + setQuerySizeLimit(5); + JSONObject result = + executeQuery(String.format( + "source=%s | fields firstname, age | head 10", TEST_INDEX_ACCOUNT)); + verifyDataRows(result, + rows("Amber", 32), + rows("Hattie", 36), + rows("Nanette", 28), + rows("Dale", 33), + rows("Elinor", 36), + rows("Virginia", 39), + rows("Dillard", 34), + rows("Mcgee", 39), + rows("Aurelia", 37), + rows("Fulton", 23)); + } + + @Test + public void testHeadWithNumberLargerThanMaxResultWindow() throws IOException { + setMaxResultWindow(TEST_INDEX_ACCOUNT, 10); + JSONObject result = + executeQuery(String.format( + "source=%s | fields firstname, age | head 15", TEST_INDEX_ACCOUNT)); + verifyDataRows(result, + rows("Amber", 32), + rows("Hattie", 36), + rows("Nanette", 28), + rows("Dale", 33), + rows("Elinor", 36), + rows("Virginia", 39), + rows("Dillard", 34), + rows("Mcgee", 39), + rows("Aurelia", 37), + rows("Fulton", 23), + rows("Burton", 31), + rows("Josie", 32), + rows("Hughes", 30), + rows("Hall", 25), + rows("Deidre", 33)); + } + + @Ignore("Fix https://github.com/opensearch-project/sql/issues/703#issuecomment-1211422130") + @Test + public void testHeadWithLargeNumber() throws IOException { + setQuerySizeLimit(5); + setMaxResultWindow(TEST_INDEX_ACCOUNT, 10); + JSONObject result = + executeQuery(String.format( + "source=%s | fields firstname, age | head 15", TEST_INDEX_ACCOUNT)); + verifyDataRows(result, + rows("Amber", 32), + rows("Hattie", 36), + rows("Nanette", 28), + rows("Dale", 33), + rows("Elinor", 36), + rows("Virginia", 39), + rows("Dillard", 34), + rows("Mcgee", 39), + rows("Aurelia", 37), + rows("Fulton", 23), + rows("Burton", 31), + rows("Josie", 32), + rows("Hughes", 30), + rows("Hall", 25), + rows("Deidre", 33)); + } + @Test public void testHeadWithNumberAndFrom() throws IOException { JSONObject result = diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java index c1b7d782d27..09a83f65a57 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java @@ -30,6 +30,14 @@ public interface OpenSearchClient { */ Map getIndexMappings(String... indexExpression); + /** + * Fetch index.max_result_window settings according to index expression given. + * + * @param indexExpression index expression + * @return map from index name to its max result window + */ + Map getIndexMaxResultWindows(String... indexExpression); + /** * Perform search query in the search request. * diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index fe262808121..db35f3580c3 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -24,11 +24,14 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.AliasMetadata; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.collect.ImmutableOpenMap; +import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.index.IndexSettings; import org.opensearch.sql.opensearch.mapping.IndexMapping; import org.opensearch.sql.opensearch.request.OpenSearchRequest; import org.opensearch.sql.opensearch.response.OpenSearchResponse; @@ -86,6 +89,29 @@ public Map getIndexMappings(String... indexExpression) { } } + /** + * Fetch index.max_result_window settings according to index expression given. + * + * @param indexExpression index expression + * @return map from index name to its max result window + */ + @Override + public Map getIndexMaxResultWindows(String... indexExpression) { + ClusterState state = clusterService.state(); + ImmutableOpenMap indicesMetadata = state.metadata().getIndices(); + String[] concreteIndices = resolveIndexExpression(state, indexExpression); + + ImmutableMap.Builder result = ImmutableMap.builder(); + for (String index : concreteIndices) { + Settings settings = indicesMetadata.get(index).getSettings(); + Integer maxResultWindow = settings.getAsInt("index.max_result_window", + IndexSettings.MAX_RESULT_WINDOW_SETTING.getDefault(settings)); + result.put(index, maxResultWindow); + } + + return result.build(); + } + /** * TODO: Scroll doesn't work for aggregation. Support aggregation later. */ diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index 9da8c442e09..f354215e058 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -11,12 +11,15 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.RequiredArgsConstructor; import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.search.ClearScrollRequest; import org.opensearch.client.RequestOptions; import org.opensearch.client.RestHighLevelClient; @@ -26,6 +29,7 @@ import org.opensearch.client.indices.GetMappingsResponse; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.metadata.AliasMetadata; +import org.opensearch.common.collect.ImmutableOpenMap; import org.opensearch.common.settings.Settings; import org.opensearch.sql.opensearch.mapping.IndexMapping; import org.opensearch.sql.opensearch.request.OpenSearchRequest; @@ -54,6 +58,36 @@ public Map getIndexMappings(String... indexExpression) { } } + @Override + public Map getIndexMaxResultWindows(String... indexExpression) { + GetSettingsRequest request = new GetSettingsRequest() + .indices(indexExpression).includeDefaults(true); + try { + GetSettingsResponse response = client.indices().getSettings(request, RequestOptions.DEFAULT); + ImmutableOpenMap settings = response.getIndexToSettings(); + ImmutableOpenMap defaultSettings = response.getIndexToDefaultSettings(); + Map result = new HashMap<>(); + + defaultSettings.forEach(entry -> { + Integer maxResultWindow = entry.value.getAsInt("index.max_result_window", null); + if (maxResultWindow != null) { + result.put(entry.key, maxResultWindow); + } + }); + + settings.forEach(entry -> { + Integer maxResultWindow = entry.value.getAsInt("index.max_result_window", null); + if (maxResultWindow != null) { + result.put(entry.key, maxResultWindow); + } + }); + + return result; + } catch (IOException e) { + throw new IllegalStateException("Failed to get max result window for " + indexExpression, e); + } + } + @Override public OpenSearchResponse search(OpenSearchRequest request) { return request.search( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java index 5ca3670ca16..6f6fea841b6 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java @@ -49,7 +49,7 @@ public class OpenSearchQueryRequest implements OpenSearchRequest { /** - * ElasticsearchExprValueFactory. + * OpenSearchExprValueFactory. */ @EqualsAndHashCode.Exclude @ToString.Exclude @@ -61,7 +61,7 @@ public class OpenSearchQueryRequest implements OpenSearchRequest { private boolean searchDone = false; /** - * Constructor of ElasticsearchQueryRequest. + * Constructor of OpenSearchQueryRequest. */ public OpenSearchQueryRequest(String indexName, int size, OpenSearchExprValueFactory factory) { @@ -69,7 +69,7 @@ public OpenSearchQueryRequest(String indexName, int size, } /** - * Constructor of ElasticsearchQueryRequest. + * Constructor of OpenSearchQueryRequest. */ public OpenSearchQueryRequest(IndexName indexName, int size, OpenSearchExprValueFactory factory) { @@ -81,6 +81,16 @@ public OpenSearchQueryRequest(IndexName indexName, int size, this.exprValueFactory = factory; } + /** + * Constructor of OpenSearchQueryRequest. + */ + public OpenSearchQueryRequest(IndexName indexName, SearchSourceBuilder sourceBuilder, + OpenSearchExprValueFactory factory) { + this.indexName = indexName; + this.sourceBuilder = sourceBuilder; + this.exprValueFactory = factory; + } + @Override public OpenSearchResponse search(Function searchAction, Function scrollAction) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java new file mode 100644 index 00000000000..646395d790c --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java @@ -0,0 +1,202 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.opensearch.request; + +import static org.opensearch.search.sort.FieldSortBuilder.DOC_FIELD_NAME; +import static org.opensearch.search.sort.SortOrder.ASC; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.opensearch.search.sort.SortBuilder; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.expression.ReferenceExpression; +import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; +import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; + +/** + * OpenSearch search request builder. + */ +@EqualsAndHashCode +@Getter +@ToString +public class OpenSearchRequestBuilder { + + /** + * Default query timeout in minutes. + */ + public static final TimeValue DEFAULT_QUERY_TIMEOUT = TimeValue.timeValueMinutes(1L); + + /** + * {@link OpenSearchRequest.IndexName}. + */ + private final OpenSearchRequest.IndexName indexName; + + /** + * Index max result window. + */ + private final Integer maxResultWindow; + + /** + * Search request source builder. + */ + private final SearchSourceBuilder sourceBuilder; + + /** + * OpenSearchExprValueFactory. + */ + @EqualsAndHashCode.Exclude + @ToString.Exclude + private final OpenSearchExprValueFactory exprValueFactory; + + /** + * Query size of the request. + */ + private Integer querySize; + + public OpenSearchRequestBuilder(String indexName, + Integer maxResultWindow, + Settings settings, + OpenSearchExprValueFactory exprValueFactory) { + this(new OpenSearchRequest.IndexName(indexName), maxResultWindow, settings, exprValueFactory); + } + + /** + * Constructor. + */ + public OpenSearchRequestBuilder(OpenSearchRequest.IndexName indexName, + Integer maxResultWindow, + Settings settings, + OpenSearchExprValueFactory exprValueFactory) { + this.indexName = indexName; + this.maxResultWindow = maxResultWindow; + this.sourceBuilder = new SearchSourceBuilder(); + this.exprValueFactory = exprValueFactory; + this.querySize = settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT); + sourceBuilder.from(0); + sourceBuilder.size(querySize); + sourceBuilder.timeout(DEFAULT_QUERY_TIMEOUT); + } + + /** + * Build DSL request. + * + * @return query request or scroll request + */ + public OpenSearchRequest build() { + Integer from = sourceBuilder.from(); + Integer size = sourceBuilder.size(); + + if (from + size <= maxResultWindow) { + return new OpenSearchQueryRequest(indexName, sourceBuilder, exprValueFactory); + } else { + sourceBuilder.size(maxResultWindow - from); + return new OpenSearchScrollRequest(indexName, sourceBuilder, exprValueFactory); + } + } + + /** + * Push down query to DSL request. + * + * @param query query request + */ + public void pushDown(QueryBuilder query) { + QueryBuilder current = sourceBuilder.query(); + + if (current == null) { + sourceBuilder.query(query); + } else { + if (isBoolFilterQuery(current)) { + ((BoolQueryBuilder) current).filter(query); + } else { + sourceBuilder.query(QueryBuilders.boolQuery() + .filter(current) + .filter(query)); + } + } + + if (sourceBuilder.sorts() == null) { + sourceBuilder.sort(DOC_FIELD_NAME, ASC); // Make sure consistent order + } + } + + /** + * Push down aggregation to DSL request. + * + * @param aggregationBuilder pair of aggregation query and aggregation parser. + */ + public void pushDownAggregation( + Pair, OpenSearchAggregationResponseParser> aggregationBuilder) { + aggregationBuilder.getLeft().forEach(builder -> sourceBuilder.aggregation(builder)); + sourceBuilder.size(0); + exprValueFactory.setParser(aggregationBuilder.getRight()); + } + + /** + * Push down sort to DSL request. + * + * @param sortBuilders sortBuilders. + */ + public void pushDownSort(List> sortBuilders) { + for (SortBuilder sortBuilder : sortBuilders) { + sourceBuilder.sort(sortBuilder); + } + } + + /** + * Push down size (limit) and from (offset) to DSL request. + */ + public void pushDownLimit(Integer limit, Integer offset) { + querySize = limit; + sourceBuilder.from(offset).size(limit); + } + + /** + * Add highlight to DSL requests. + * @param field name of the field to highlight + */ + public void pushDownHighlight(String field) { + if (sourceBuilder.highlighter() != null) { + sourceBuilder.highlighter().field(StringUtils.unquoteText(field)); + } else { + HighlightBuilder highlightBuilder = + new HighlightBuilder().field(StringUtils.unquoteText(field)); + sourceBuilder.highlighter(highlightBuilder); + } + } + + /** + * Push down project list to DSL requets. + */ + public void pushDownProjects(Set projects) { + final Set projectsSet = + projects.stream().map(ReferenceExpression::getAttr).collect(Collectors.toSet()); + sourceBuilder.fetchSource(projectsSet.toArray(new String[0]), new String[0]); + } + + public void pushTypeMapping(Map typeMapping) { + exprValueFactory.setTypeMapping(typeMapping); + } + + private boolean isBoolFilterQuery(QueryBuilder current) { + return (current instanceof BoolQueryBuilder); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java index ebbebcd8ebe..4509e443c0a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java @@ -54,10 +54,12 @@ public class OpenSearchScrollRequest implements OpenSearchRequest { private String scrollId; /** Search request source builder. */ - private final SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + private final SearchSourceBuilder sourceBuilder; + /** Constructor. */ public OpenSearchScrollRequest(IndexName indexName, OpenSearchExprValueFactory exprValueFactory) { this.indexName = indexName; + this.sourceBuilder = new SearchSourceBuilder(); this.exprValueFactory = exprValueFactory; } @@ -65,6 +67,16 @@ public OpenSearchScrollRequest(String indexName, OpenSearchExprValueFactory expr this(new IndexName(indexName), exprValueFactory); } + /** Constructor. */ + public OpenSearchScrollRequest(IndexName indexName, + SearchSourceBuilder sourceBuilder, + OpenSearchExprValueFactory exprValueFactory) { + this.indexName = indexName; + this.sourceBuilder = sourceBuilder; + this.exprValueFactory = exprValueFactory; + } + + /** Constructor. */ @Override public OpenSearchResponse search(Function searchAction, Function scrollAction) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java index 5c6d3687c6a..f3214970997 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java @@ -121,6 +121,16 @@ public Map getFieldTypes() { return fieldTypes; } + /** + * Get the minimum of the max result windows of the indices. + * + * @return max result window + */ + public Integer getMaxResultWindow() { + return client.getIndexMaxResultWindows(indexName.getIndexNames()) + .values().stream().min(Integer::compare).get(); + } + private ExprType transformESTypeToExprType(String openSearchType) { return OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING.getOrDefault(openSearchType, ExprCoreType.UNKNOWN); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java index c028f283a27..ef6159020f9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java @@ -58,6 +58,11 @@ public class OpenSearchIndex implements Table { */ private Map cachedFieldTypes = null; + /** + * The cached max result window setting of index. + */ + private Integer cachedMaxResultWindow = null; + /** * Constructor. */ @@ -80,13 +85,24 @@ public Map getFieldTypes() { return cachedFieldTypes; } + /** + * Get the max result window setting of the table. + */ + public Integer getMaxResultWindow() { + if (cachedMaxResultWindow == null) { + cachedMaxResultWindow = + new OpenSearchDescribeIndexRequest(client, indexName).getMaxResultWindow(); + } + return cachedMaxResultWindow; + } + /** * TODO: Push down operations to index scan operator as much as possible in future. */ @Override public PhysicalPlan implement(LogicalPlan plan) { OpenSearchIndexScan indexScan = new OpenSearchIndexScan(client, settings, indexName, - new OpenSearchExprValueFactory(getFieldTypes())); + getMaxResultWindow(), new OpenSearchExprValueFactory(getFieldTypes())); /* * Visit logical plan with index scan as context so logical operators visited, such as @@ -128,7 +144,7 @@ public PhysicalPlan visitIndexScan(OpenSearchLogicalIndexScan node, OpenSearchIndexScan context) { if (null != node.getSortList()) { final SortQueryBuilder builder = new SortQueryBuilder(); - context.pushDownSort(node.getSortList().stream() + context.getRequestBuilder().pushDownSort(node.getSortList().stream() .map(sort -> builder.build(sort.getValue(), sort.getKey())) .collect(Collectors.toList())); } @@ -136,15 +152,15 @@ public PhysicalPlan visitIndexScan(OpenSearchLogicalIndexScan node, if (null != node.getFilter()) { FilterQueryBuilder queryBuilder = new FilterQueryBuilder(new DefaultExpressionSerializer()); QueryBuilder query = queryBuilder.build(node.getFilter()); - context.pushDown(query); + context.getRequestBuilder().pushDown(query); } if (node.getLimit() != null) { - context.pushDownLimit(node.getLimit(), node.getOffset()); + context.getRequestBuilder().pushDownLimit(node.getLimit(), node.getOffset()); } if (node.hasProjects()) { - context.pushDownProjects(node.getProjectList()); + context.getRequestBuilder().pushDownProjects(node.getProjectList()); } return indexScan; } @@ -158,15 +174,15 @@ public PhysicalPlan visitIndexAggregation(OpenSearchLogicalIndexAgg node, FilterQueryBuilder queryBuilder = new FilterQueryBuilder( new DefaultExpressionSerializer()); QueryBuilder query = queryBuilder.build(node.getFilter()); - context.pushDown(query); + context.getRequestBuilder().pushDown(query); } AggregationQueryBuilder builder = new AggregationQueryBuilder(new DefaultExpressionSerializer()); Pair, OpenSearchAggregationResponseParser> aggregationBuilder = builder.buildAggregationBuilder(node.getAggregatorList(), node.getGroupByList(), node.getSortList()); - context.pushDownAggregation(aggregationBuilder); - context.pushTypeMapping( + context.getRequestBuilder().pushDownAggregation(aggregationBuilder); + context.getRequestBuilder().pushTypeMapping( builder.buildTypeMapping(node.getAggregatorList(), node.getGroupByList())); return indexScan; @@ -191,7 +207,7 @@ public PhysicalPlan visitAD(LogicalAD node, OpenSearchIndexScan context) { @Override public PhysicalPlan visitHighlight(LogicalHighlight node, OpenSearchIndexScan context) { - context.pushDownHighlight(node.getHighlightField().toString()); + context.getRequestBuilder().pushDownHighlight(node.getHighlightField().toString()); return visitChild(node, context); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScan.java index 6e88f3de891..e9746e1fae1 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScan.java @@ -6,38 +6,18 @@ package org.opensearch.sql.opensearch.storage; -import static org.opensearch.search.sort.FieldSortBuilder.DOC_FIELD_NAME; -import static org.opensearch.search.sort.SortOrder.ASC; - -import com.google.common.collect.Iterables; -import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; -import org.apache.commons.lang3.tuple.Pair; -import org.opensearch.index.query.BoolQueryBuilder; -import org.opensearch.index.query.QueryBuilder; -import org.opensearch.index.query.QueryBuilders; -import org.opensearch.search.aggregations.AggregationBuilder; -import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; -import org.opensearch.search.sort.SortBuilder; import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; -import org.opensearch.sql.opensearch.request.OpenSearchQueryRequest; import org.opensearch.sql.opensearch.request.OpenSearchRequest; +import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; import org.opensearch.sql.opensearch.response.OpenSearchResponse; -import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; import org.opensearch.sql.storage.TableScanOperator; /** @@ -50,11 +30,24 @@ public class OpenSearchIndexScan extends TableScanOperator { /** OpenSearch client. */ private final OpenSearchClient client; - /** Search request. */ + /** Search request builder. */ @EqualsAndHashCode.Include @Getter @ToString.Include - private final OpenSearchRequest request; + private final OpenSearchRequestBuilder requestBuilder; + + /** Search request. */ + @EqualsAndHashCode.Include + @ToString.Include + private OpenSearchRequest request; + + /** Total query size. */ + @EqualsAndHashCode.Include + @ToString.Include + private Integer querySize; + + /** Number of rows returned. */ + private Integer queryCount; /** Search response for current batch. */ private Iterator iterator; @@ -62,133 +55,57 @@ public class OpenSearchIndexScan extends TableScanOperator { /** * Constructor. */ - public OpenSearchIndexScan(OpenSearchClient client, - Settings settings, String indexName, + public OpenSearchIndexScan(OpenSearchClient client, Settings settings, + String indexName, Integer maxResultWindow, OpenSearchExprValueFactory exprValueFactory) { - this(client, settings, new OpenSearchRequest.IndexName(indexName), exprValueFactory); + this(client, settings, + new OpenSearchRequest.IndexName(indexName),maxResultWindow, exprValueFactory); } /** * Constructor. */ - public OpenSearchIndexScan(OpenSearchClient client, - Settings settings, OpenSearchRequest.IndexName indexName, - OpenSearchExprValueFactory exprValueFactory) { + public OpenSearchIndexScan(OpenSearchClient client, Settings settings, + OpenSearchRequest.IndexName indexName, Integer maxResultWindow, + OpenSearchExprValueFactory exprValueFactory) { this.client = client; - this.request = new OpenSearchQueryRequest(indexName, - settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT), exprValueFactory); + this.requestBuilder = new OpenSearchRequestBuilder( + indexName, maxResultWindow, settings,exprValueFactory); } @Override public void open() { super.open(); - - // For now pull all results immediately once open - List responses = new ArrayList<>(); - OpenSearchResponse response = client.search(request); - while (!response.isEmpty()) { - responses.add(response); - response = client.search(request); - } - iterator = Iterables.concat(responses.toArray(new OpenSearchResponse[0])).iterator(); + querySize = requestBuilder.getQuerySize(); + request = requestBuilder.build(); + iterator = Collections.emptyIterator(); + queryCount = 0; + fetchNextBatch(); } @Override public boolean hasNext() { + if (queryCount >= querySize) { + iterator = Collections.emptyIterator(); + } else if (!iterator.hasNext()) { + fetchNextBatch(); + } return iterator.hasNext(); } @Override public ExprValue next() { + queryCount++; return iterator.next(); } - /** - * Push down query to DSL request. - * @param query query request - */ - public void pushDown(QueryBuilder query) { - SearchSourceBuilder source = request.getSourceBuilder(); - QueryBuilder current = source.query(); - - if (current == null) { - source.query(query); - } else { - if (isBoolFilterQuery(current)) { - ((BoolQueryBuilder) current).filter(query); - } else { - source.query(QueryBuilders.boolQuery() - .filter(current) - .filter(query)); - } - } - - if (source.sorts() == null) { - source.sort(DOC_FIELD_NAME, ASC); // Make sure consistent order - } - } - - /** - * Push down aggregation to DSL request. - * @param aggregationBuilder pair of aggregation query and aggregation parser. - */ - public void pushDownAggregation( - Pair, OpenSearchAggregationResponseParser> aggregationBuilder) { - SearchSourceBuilder source = request.getSourceBuilder(); - aggregationBuilder.getLeft().forEach(builder -> source.aggregation(builder)); - source.size(0); - request.getExprValueFactory().setParser(aggregationBuilder.getRight()); - } - - /** - * Push down sort to DSL request. - * - * @param sortBuilders sortBuilders. - */ - public void pushDownSort(List> sortBuilders) { - SearchSourceBuilder source = request.getSourceBuilder(); - for (SortBuilder sortBuilder : sortBuilders) { - source.sort(sortBuilder); - } - } - - /** - * Push down size (limit) and from (offset) to DSL request. - */ - public void pushDownLimit(Integer limit, Integer offset) { - SearchSourceBuilder sourceBuilder = request.getSourceBuilder(); - sourceBuilder.from(offset).size(limit); - } - - /** - * Add highlight to DSL requests. - * @param field name of the field to highlight - */ - public void pushDownHighlight(String field) { - SearchSourceBuilder sourceBuilder = request.getSourceBuilder(); - if (sourceBuilder.highlighter() != null) { - sourceBuilder.highlighter().field(StringUtils.unquoteText(field)); - } else { - HighlightBuilder highlightBuilder = - new HighlightBuilder().field(StringUtils.unquoteText(field)); - sourceBuilder.highlighter(highlightBuilder); + private void fetchNextBatch() { + OpenSearchResponse response = client.search(request); + if (!response.isEmpty()) { + iterator = response.iterator(); } } - /** - * Push down project list to DSL requets. - */ - public void pushDownProjects(Set projects) { - SearchSourceBuilder sourceBuilder = request.getSourceBuilder(); - final Set projectsSet = - projects.stream().map(ReferenceExpression::getAttr).collect(Collectors.toSet()); - sourceBuilder.fetchSource(projectsSet.toArray(new String[0]), new String[0]); - } - - public void pushTypeMapping(Map typeMapping) { - request.getExprValueFactory().setTypeMapping(typeMapping); - } - @Override public void close() { super.close(); @@ -196,12 +113,8 @@ public void close() { client.cleanup(request); } - private boolean isBoolFilterQuery(QueryBuilder current) { - return (current instanceof BoolQueryBuilder); - } - @Override public String explain() { - return getRequest().toString(); + return getRequestBuilder().build().toString(); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java index 50410e07ccd..8fdb93427b7 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.net.URL; import java.util.Arrays; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -72,6 +73,7 @@ class OpenSearchNodeClientTest { private static final String TEST_MAPPING_FILE = "mappings/accounts.json"; + private static final String TEST_MAPPING_SETTINGS_FILE = "mappings/accounts2.json"; @Mock(answer = RETURNS_DEEP_STUBS) private NodeClient nodeClient; @@ -151,6 +153,36 @@ public void getIndexMappingsWithNonExistIndex() { assertThrows(IndexNotFoundException.class, () -> client.getIndexMappings("non_exist_index")); } + @Test + public void getIndexMaxResultWindows() throws IOException { + URL url = Resources.getResource(TEST_MAPPING_SETTINGS_FILE); + String mappings = Resources.toString(url, Charsets.UTF_8); + String indexName = "accounts"; + ClusterService clusterService = mockClusterServiceForSettings(indexName, mappings); + OpenSearchNodeClient client = new OpenSearchNodeClient(clusterService, nodeClient); + + Map indexMaxResultWindows = client.getIndexMaxResultWindows(indexName); + assertEquals(1, indexMaxResultWindows.size()); + + Integer indexMaxResultWindow = indexMaxResultWindows.values().iterator().next(); + assertEquals(100, indexMaxResultWindow); + } + + @Test + public void getIndexMaxResultWindowsWithDefaultSettings() throws IOException { + URL url = Resources.getResource(TEST_MAPPING_FILE); + String mappings = Resources.toString(url, Charsets.UTF_8); + String indexName = "accounts"; + ClusterService clusterService = mockClusterServiceForSettings(indexName, mappings); + OpenSearchNodeClient client = new OpenSearchNodeClient(clusterService, nodeClient); + + Map indexMaxResultWindows = client.getIndexMaxResultWindows(indexName); + assertEquals(1, indexMaxResultWindows.size()); + + Integer indexMaxResultWindow = indexMaxResultWindows.values().iterator().next(); + assertEquals(10000, indexMaxResultWindow); + } + /** Jacoco enforce this constant lambda be tested. */ @Test public void testAllFieldsPredicate() { @@ -353,6 +385,33 @@ public ClusterService mockClusterService(String indexName, Throwable t) { return mockService; } + public ClusterService mockClusterServiceForSettings(String indexName, String mappings) { + ClusterService mockService = mock(ClusterService.class); + ClusterState mockState = mock(ClusterState.class); + Metadata mockMetaData = mock(Metadata.class); + + when(mockService.state()).thenReturn(mockState); + when(mockState.metadata()).thenReturn(mockMetaData); + try { + ImmutableOpenMap.Builder indexBuilder = + ImmutableOpenMap.builder(); + IndexMetadata indexMetadata = IndexMetadata.fromXContent(createParser(mappings)); + + indexBuilder.put(indexName, indexMetadata); + when(mockMetaData.getIndices()).thenReturn(indexBuilder.build()); + + // IndexNameExpressionResolver use this method to check if index exists. If not, + // IndexNotFoundException is thrown. + IndexAbstraction indexAbstraction = mock(IndexAbstraction.class); + when(indexAbstraction.getIndices()).thenReturn(Collections.singletonList(indexMetadata)); + when(mockMetaData.getIndicesLookup()) + .thenReturn(ImmutableSortedMap.of(indexName, indexAbstraction)); + } catch (IOException e) { + throw new IllegalStateException("Failed to mock cluster service", e); + } + return mockService; + } + private XContentParser createParser(String mappings) throws IOException { return XContentType.JSON .xContent() diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java index 0c2503ea57a..bc334aaf399 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java @@ -34,6 +34,8 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsResponse; +import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.RequestOptions; import org.opensearch.client.RestHighLevelClient; @@ -43,6 +45,7 @@ import org.opensearch.client.indices.GetMappingsResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; +import org.opensearch.common.collect.ImmutableOpenMap; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.DeprecationHandler; import org.opensearch.common.xcontent.NamedXContentRegistry; @@ -128,6 +131,61 @@ void getIndexMappingsWithIOException() throws IOException { assertThrows(IllegalStateException.class, () -> client.getIndexMappings("test")); } + @Test + void getIndexMaxResultWindowsSettings() throws IOException { + String indexName = "test"; + Integer maxResultWindow = 1000; + + GetSettingsResponse response = mock(GetSettingsResponse.class); + Settings maxResultWindowSettings = Settings.builder() + .put("index.max_result_window", maxResultWindow) + .build(); + Settings emptySettings = Settings.builder().build(); + ImmutableOpenMap indexToSettings = + mockSettings(indexName, maxResultWindowSettings); + ImmutableOpenMap indexToDefaultSettings = + mockSettings(indexName, emptySettings); + when(response.getIndexToSettings()).thenReturn(indexToSettings); + when(response.getIndexToDefaultSettings()).thenReturn(indexToDefaultSettings); + when(restClient.indices().getSettings(any(GetSettingsRequest.class), any())) + .thenReturn(response); + + Map indexMaxResultWindows = client.getIndexMaxResultWindows(indexName); + assertEquals(1, indexMaxResultWindows.size()); + assertEquals(maxResultWindow, indexMaxResultWindows.values().iterator().next()); + } + + @Test + void getIndexMaxResultWindowsDefaultSettings() throws IOException { + String indexName = "test"; + Integer maxResultWindow = 10000; + + GetSettingsResponse response = mock(GetSettingsResponse.class); + Settings maxResultWindowSettings = Settings.builder() + .put("index.max_result_window", maxResultWindow) + .build(); + Settings emptySettings = Settings.builder().build(); + ImmutableOpenMap indexToSettings = + mockSettings(indexName, emptySettings); + ImmutableOpenMap indexToDefaultSettings = + mockSettings(indexName, maxResultWindowSettings); + when(response.getIndexToSettings()).thenReturn(indexToSettings); + when(response.getIndexToDefaultSettings()).thenReturn(indexToDefaultSettings); + when(restClient.indices().getSettings(any(GetSettingsRequest.class), any())) + .thenReturn(response); + + Map indexMaxResultWindows = client.getIndexMaxResultWindows(indexName); + assertEquals(1, indexMaxResultWindows.size()); + assertEquals(maxResultWindow, indexMaxResultWindows.values().iterator().next()); + } + + @Test + void getIndexMaxResultWindowsWithIOException() throws IOException { + when(restClient.indices().getSettings(any(GetSettingsRequest.class), any())) + .thenThrow(new IOException()); + assertThrows(IllegalStateException.class, () -> client.getIndexMaxResultWindows("test")); + } + @Test void search() throws IOException { // Mock first scroll request @@ -277,6 +335,12 @@ private Map mockFieldMappings(String indexName, String return ImmutableMap.of(indexName, IndexMetadata.fromXContent(createParser(mappings)).mapping()); } + private ImmutableOpenMap mockSettings(String indexName, Settings settings) { + ImmutableOpenMap.Builder indexToSettingsBuilder = ImmutableOpenMap.builder(); + indexToSettingsBuilder.put(indexName, settings); + return indexToSettingsBuilder.build(); + } + private XContentParser createParser(String mappings) throws IOException { return XContentType.JSON .xContent() diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java index 24c305a75ef..f1a0a7d5d76 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java @@ -24,6 +24,7 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import lombok.RequiredArgsConstructor; import org.junit.jupiter.api.BeforeEach; @@ -126,7 +127,7 @@ void explainSuccessfully() { Settings settings = mock(Settings.class); when(settings.getSettingValue(QUERY_SIZE_LIMIT)).thenReturn(100); PhysicalPlan plan = new OpenSearchIndexScan(mock(OpenSearchClient.class), - settings, "test", mock(OpenSearchExprValueFactory.class)); + settings, "test", 10000, mock(OpenSearchExprValueFactory.class)); AtomicReference result = new AtomicReference<>(); executor.explain(plan, new ResponseListener() { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java index ee981a4abca..e5c5046b817 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java @@ -88,6 +88,7 @@ public void testProtectIndexScan() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); String indexName = "test"; + Integer maxResultWindow = 10000; NamedExpression include = named("age", ref("age", INTEGER)); ReferenceExpression exclude = ref("name", STRING); ReferenceExpression dedupeField = ref("name", STRING); @@ -124,7 +125,7 @@ public void testProtectIndexScan() { resourceMonitor( new OpenSearchIndexScan( client, settings, indexName, - exprValueFactory)), + maxResultWindow, exprValueFactory)), filterExpr), aggregators, groupByExprs), @@ -152,7 +153,7 @@ public void testProtectIndexScan() { filter( new OpenSearchIndexScan( client, settings, indexName, - exprValueFactory), + maxResultWindow, exprValueFactory), filterExpr), aggregators, groupByExprs), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java new file mode 100644 index 00000000000..43b93531906 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java @@ -0,0 +1,76 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.opensearch.request; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; + +@ExtendWith(MockitoExtension.class) +public class OpenSearchRequestBuilderTest { + + public static final TimeValue DEFAULT_QUERY_TIMEOUT = TimeValue.timeValueMinutes(1L); + @Mock + private Settings settings; + + @Mock + private OpenSearchExprValueFactory factory; + + @BeforeEach + void setup() { + when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); + } + + @Test + void buildQueryRequest() { + Integer maxResultWindow = 500; + Integer limit = 200; + Integer offset = 0; + OpenSearchRequestBuilder builder = + new OpenSearchRequestBuilder("test", maxResultWindow, settings, factory); + builder.pushDownLimit(limit, offset); + + assertEquals( + new OpenSearchQueryRequest( + new OpenSearchRequest.IndexName("test"), + new SearchSourceBuilder() + .from(offset) + .size(limit) + .timeout(DEFAULT_QUERY_TIMEOUT), + factory), + builder.build()); + } + + @Test + void buildScrollRequestWithCorrectSize() { + Integer maxResultWindow = 500; + Integer limit = 800; + Integer offset = 10; + OpenSearchRequestBuilder builder = + new OpenSearchRequestBuilder("test", maxResultWindow, settings, factory); + builder.pushDownLimit(limit, offset); + + assertEquals( + new OpenSearchScrollRequest( + new OpenSearchRequest.IndexName("test"), + new SearchSourceBuilder() + .from(offset) + .size(maxResultWindow - offset) + .timeout(DEFAULT_QUERY_TIMEOUT), + factory), + builder.build()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchDefaultImplementorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchDefaultImplementorTest.java index c83172955c1..b85d60c1fb4 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchDefaultImplementorTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchDefaultImplementorTest.java @@ -20,6 +20,7 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; import org.opensearch.sql.planner.logical.LogicalAD; import org.opensearch.sql.planner.logical.LogicalHighlight; import org.opensearch.sql.planner.logical.LogicalMLCommons; @@ -76,10 +77,12 @@ public void visitHighlight() { LogicalHighlight node = Mockito.mock(LogicalHighlight.class, Answers.RETURNS_DEEP_STUBS); Mockito.when(node.getChild().get(0)).thenReturn(Mockito.mock(LogicalPlan.class)); + OpenSearchRequestBuilder requestBuilder = Mockito.mock(OpenSearchRequestBuilder.class); + Mockito.when(indexScan.getRequestBuilder()).thenReturn(requestBuilder); OpenSearchIndex.OpenSearchDefaultImplementor implementor = new OpenSearchIndex.OpenSearchDefaultImplementor(indexScan, client); implementor.visitHighlight(node, indexScan); - verify(indexScan).pushDownHighlight(any()); + verify(requestBuilder).pushDownHighlight(any()); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScanTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScanTest.java index 41769914d95..a1f2869ca51 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScanTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScanTest.java @@ -62,7 +62,7 @@ void setup() { void queryEmptyResult() { mockResponse(); try (OpenSearchIndexScan indexScan = - new OpenSearchIndexScan(client, settings, "test", exprValueFactory)) { + new OpenSearchIndexScan(client, settings, "test", 3, exprValueFactory)) { indexScan.open(); assertFalse(indexScan.hasNext()); } @@ -70,13 +70,90 @@ void queryEmptyResult() { } @Test - void queryAllResults() { + void queryAllResultsWithQuery() { + mockResponse(new ExprValue[]{ + employee(1, "John", "IT"), + employee(2, "Smith", "HR"), + employee(3, "Allen", "IT")}); + + try (OpenSearchIndexScan indexScan = + new OpenSearchIndexScan(client, settings, "employees", 10, exprValueFactory)) { + indexScan.open(); + + assertTrue(indexScan.hasNext()); + assertEquals(employee(1, "John", "IT"), indexScan.next()); + + assertTrue(indexScan.hasNext()); + assertEquals(employee(2, "Smith", "HR"), indexScan.next()); + + assertTrue(indexScan.hasNext()); + assertEquals(employee(3, "Allen", "IT"), indexScan.next()); + + assertFalse(indexScan.hasNext()); + } + verify(client).cleanup(any()); + } + + @Test + void queryAllResultsWithScroll() { mockResponse( new ExprValue[]{employee(1, "John", "IT"), employee(2, "Smith", "HR")}, new ExprValue[]{employee(3, "Allen", "IT")}); try (OpenSearchIndexScan indexScan = - new OpenSearchIndexScan(client, settings, "employees", exprValueFactory)) { + new OpenSearchIndexScan(client, settings, "employees", 2, exprValueFactory)) { + indexScan.open(); + + assertTrue(indexScan.hasNext()); + assertEquals(employee(1, "John", "IT"), indexScan.next()); + + assertTrue(indexScan.hasNext()); + assertEquals(employee(2, "Smith", "HR"), indexScan.next()); + + assertTrue(indexScan.hasNext()); + assertEquals(employee(3, "Allen", "IT"), indexScan.next()); + + assertFalse(indexScan.hasNext()); + } + verify(client).cleanup(any()); + } + + @Test + void querySomeResultsWithQuery() { + mockResponse(new ExprValue[]{ + employee(1, "John", "IT"), + employee(2, "Smith", "HR"), + employee(3, "Allen", "IT"), + employee(4, "Bob", "HR")}); + + try (OpenSearchIndexScan indexScan = + new OpenSearchIndexScan(client, settings, "employees", 10, exprValueFactory)) { + indexScan.getRequestBuilder().pushDownLimit(3, 0); + indexScan.open(); + + assertTrue(indexScan.hasNext()); + assertEquals(employee(1, "John", "IT"), indexScan.next()); + + assertTrue(indexScan.hasNext()); + assertEquals(employee(2, "Smith", "HR"), indexScan.next()); + + assertTrue(indexScan.hasNext()); + assertEquals(employee(3, "Allen", "IT"), indexScan.next()); + + assertFalse(indexScan.hasNext()); + } + verify(client).cleanup(any()); + } + + @Test + void querySomeResultsWithScroll() { + mockResponse( + new ExprValue[]{employee(1, "John", "IT"), employee(2, "Smith", "HR")}, + new ExprValue[]{employee(3, "Allen", "IT"), employee(4, "Bob", "HR")}); + + try (OpenSearchIndexScan indexScan = + new OpenSearchIndexScan(client, settings, "employees", 2, exprValueFactory)) { + indexScan.getRequestBuilder().pushDownLimit(3, 0); indexScan.open(); assertTrue(indexScan.hasNext()); @@ -135,19 +212,19 @@ public PushDownAssertion(OpenSearchClient client, OpenSearchExprValueFactory valueFactory, Settings settings) { this.client = client; - this.indexScan = new OpenSearchIndexScan(client, settings, "test", valueFactory); + this.indexScan = new OpenSearchIndexScan(client, settings, "test", 10000, valueFactory); this.response = mock(OpenSearchResponse.class); this.factory = valueFactory; when(response.isEmpty()).thenReturn(true); } PushDownAssertion pushDown(QueryBuilder query) { - indexScan.pushDown(query); + indexScan.getRequestBuilder().pushDown(query); return this; } PushDownAssertion pushDownHighlight(String query) { - indexScan.pushDownHighlight(query); + indexScan.getRequestBuilder().pushDownHighlight(query); return this; } @@ -187,10 +264,8 @@ public OpenSearchResponse answer(InvocationOnMock invocation) { when(response.isEmpty()).thenReturn(false); ExprValue[] searchHit = searchHitBatches[batchNum]; when(response.iterator()).thenReturn(Arrays.asList(searchHit).iterator()); - } else if (batchNum == totalBatch) { - when(response.isEmpty()).thenReturn(true); } else { - fail("Search request after empty response returned already"); + when(response.isEmpty()).thenReturn(true); } batchNum++; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java index 847ac8dfc03..f1754a455dd 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java @@ -70,7 +70,6 @@ import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.planner.physical.PhysicalPlanDSL; import org.opensearch.sql.planner.physical.ProjectOperator; -import org.opensearch.sql.storage.Table; @ExtendWith(MockitoExtension.class) class OpenSearchIndexTest { @@ -109,7 +108,7 @@ void getFieldTypes() { .put("blob", "binary") .build()))); - Table index = new OpenSearchIndex(client, settings, "test"); + OpenSearchIndex index = new OpenSearchIndex(client, settings, "test"); Map fieldTypes = index.getFieldTypes(); assertThat( fieldTypes, @@ -134,30 +133,35 @@ void getFieldTypes() { @Test void implementRelationOperatorOnly() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); + when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); String indexName = "test"; LogicalPlan plan = relation(indexName); - Table index = new OpenSearchIndex(client, settings, indexName); + OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); + Integer maxResultWindow = index.getMaxResultWindow(); assertEquals( - new OpenSearchIndexScan(client, settings, indexName, exprValueFactory), + new OpenSearchIndexScan(client, settings, indexName, maxResultWindow, exprValueFactory), index.implement(plan)); } @Test void implementRelationOperatorWithOptimization() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); + when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); String indexName = "test"; LogicalPlan plan = relation(indexName); - Table index = new OpenSearchIndex(client, settings, indexName); + OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); + Integer maxResultWindow = index.getMaxResultWindow(); assertEquals( - new OpenSearchIndexScan(client, settings, indexName, exprValueFactory), + new OpenSearchIndexScan(client, settings, indexName, maxResultWindow, exprValueFactory), index.implement(index.optimize(plan))); } @Test void implementOtherLogicalOperators() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); + when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); String indexName = "test"; NamedExpression include = named("age", ref("age", INTEGER)); @@ -191,7 +195,8 @@ void implementOtherLogicalOperators() { dedupeField), include); - Table index = new OpenSearchIndex(client, settings, indexName); + OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); + Integer maxResultWindow = index.getMaxResultWindow(); assertEquals( PhysicalPlanDSL.project( PhysicalPlanDSL.dedupe( @@ -200,7 +205,7 @@ void implementOtherLogicalOperators() { PhysicalPlanDSL.remove( PhysicalPlanDSL.rename( new OpenSearchIndexScan(client, settings, indexName, - exprValueFactory), + maxResultWindow, exprValueFactory), mappings), exclude), newEvalField), @@ -213,6 +218,7 @@ void implementOtherLogicalOperators() { @Test void shouldImplLogicalIndexScan() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); + when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); ReferenceExpression field = ref("name", STRING); NamedExpression named = named("n", field); @@ -235,6 +241,7 @@ void shouldImplLogicalIndexScan() { @Test void shouldNotPushDownFilterFarFromRelation() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); + when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); ReferenceExpression field = ref("name", STRING); Expression filterExpr = dsl.equal(field, literal("John")); @@ -260,6 +267,7 @@ void shouldNotPushDownFilterFarFromRelation() { @Test void shouldImplLogicalIndexScanAgg() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); + when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); ReferenceExpression field = ref("name", STRING); Expression filterExpr = dsl.equal(field, literal("John")); @@ -296,6 +304,7 @@ void shouldImplLogicalIndexScanAgg() { @Test void shouldNotPushDownAggregationFarFromRelation() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); + when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); ReferenceExpression field = ref("name", STRING); Expression filterExpr = dsl.equal(field, literal("John")); @@ -320,6 +329,7 @@ void shouldNotPushDownAggregationFarFromRelation() { @Test void shouldImplIndexScanWithSort() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); + when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); ReferenceExpression field = ref("name", STRING); NamedExpression named = named("n", field); @@ -342,6 +352,7 @@ void shouldImplIndexScanWithSort() { @Test void shouldImplIndexScanWithLimit() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); + when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); ReferenceExpression field = ref("name", STRING); NamedExpression named = named("n", field); @@ -363,6 +374,7 @@ void shouldImplIndexScanWithLimit() { @Test void shouldImplIndexScanWithSortAndLimit() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); + when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); ReferenceExpression field = ref("name", STRING); NamedExpression named = named("n", field); @@ -387,6 +399,7 @@ void shouldImplIndexScanWithSortAndLimit() { @Test void shouldNotPushDownLimitFarFromRelationButUpdateScanSize() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); + when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); String indexName = "test"; OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); @@ -411,6 +424,7 @@ void shouldNotPushDownLimitFarFromRelationButUpdateScanSize() { @Test void shouldPushDownProjects() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); + when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); String indexName = "test"; OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); @@ -425,7 +439,7 @@ indexName, projects(ref("intV", INTEGER)) assertTrue(((ProjectOperator) plan).getInput() instanceof OpenSearchIndexScan); final FetchSourceContext fetchSource = - ((OpenSearchIndexScan) ((ProjectOperator) plan).getInput()).getRequest() + ((OpenSearchIndexScan) ((ProjectOperator) plan).getInput()).getRequestBuilder() .getSourceBuilder().fetchSource(); assertThat(fetchSource.includes(), arrayContaining("intV")); assertThat(fetchSource.excludes(), emptyArray()); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/utils/Utils.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/utils/Utils.java index 15ca9d491ff..2ed9a16434f 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/utils/Utils.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/utils/Utils.java @@ -31,7 +31,9 @@ public class Utils { * Build ElasticsearchLogicalIndexScan. */ public static LogicalPlan indexScan(String tableName, Expression filter) { - return OpenSearchLogicalIndexScan.builder().relationName(tableName).filter(filter).build(); + return OpenSearchLogicalIndexScan.builder().relationName(tableName) + .filter(filter) + .build(); } /** diff --git a/opensearch/src/test/resources/mappings/accounts2.json b/opensearch/src/test/resources/mappings/accounts2.json new file mode 100644 index 00000000000..d300b8c523a --- /dev/null +++ b/opensearch/src/test/resources/mappings/accounts2.json @@ -0,0 +1,93 @@ +{ + "accounts": { + "mappings": { + "_doc": { + "properties": { + "address": { + "type": "text" + }, + "age": { + "type": "integer" + }, + "balance": { + "type": "double" + }, + "city": { + "type": "keyword" + }, + "birthday": { + "type": "date" + }, + "location": { + "type": "geo_point" + }, + "new_field": { + "type": "some_new_es_type_outside_type_system" + }, + "field with spaces": { + "type": "text" + }, + "employer": { + "type": "text", + "fields": { + "raw": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "projects": { + "type": "nested", + "properties": { + "members": { + "type": "nested", + "properties": { + "name": { + "type": "text" + } + } + }, + "active": { + "type": "boolean" + }, + "release": { + "type": "date" + } + } + }, + "manager": { + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "address": { + "type": "keyword" + }, + "salary": { + "type": "long" + } + } + } + } + } + }, + "settings": { + "index": { + "number_of_shards": 5, + "number_of_replicas": 0, + "max_result_window": 100, + "version": { + "created": "6050399" + } + } + }, + "mapping_version": "1", + "settings_version": "1" + } +} \ No newline at end of file From deececbb03a9dcf9647bf8f2f651a4e3fe0a6c2e Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Fri, 12 Aug 2022 10:29:33 -0700 Subject: [PATCH 13/20] Replace 2x `LogUtils` by `QueryContext` (#747) Signed-off-by: Yury Fridlyand --- .../{LogUtils.java => QueryContext.java} | 20 ++++-- .../legacy/executor/AsyncRestExecutor.java | 16 ++--- .../cursor/CursorAsyncRestExecutor.java | 12 ++-- .../sql/legacy/plugin/RestSqlAction.java | 16 ++--- .../sql/legacy/plugin/RestSqlStatsAction.java | 4 +- .../opensearch/sql/legacy/utils/LogUtils.java | 67 ------------------- ...ogUtilsTest.java => QueryContextTest.java} | 18 ++--- .../sql/plugin/rest/RestPPLStatsAction.java | 4 +- .../plugin/rest/RestQuerySettingsAction.java | 4 +- .../transport/TransportPPLQueryAction.java | 4 +- .../org/opensearch/sql/ppl/PPLService.java | 5 +- 11 files changed, 55 insertions(+), 115 deletions(-) rename common/src/main/java/org/opensearch/sql/common/utils/{LogUtils.java => QueryContext.java} (78%) delete mode 100644 legacy/src/main/java/org/opensearch/sql/legacy/utils/LogUtils.java rename legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/{LogUtilsTest.java => QueryContextTest.java} (79%) diff --git a/common/src/main/java/org/opensearch/sql/common/utils/LogUtils.java b/common/src/main/java/org/opensearch/sql/common/utils/QueryContext.java similarity index 78% rename from common/src/main/java/org/opensearch/sql/common/utils/LogUtils.java rename to common/src/main/java/org/opensearch/sql/common/utils/QueryContext.java index 2f8c22c059b..372dbae3874 100644 --- a/common/src/main/java/org/opensearch/sql/common/utils/LogUtils.java +++ b/common/src/main/java/org/opensearch/sql/common/utils/QueryContext.java @@ -7,13 +7,14 @@ package org.opensearch.sql.common.utils; import java.util.Map; +import java.util.Optional; import java.util.UUID; import org.apache.logging.log4j.ThreadContext; /** - * Utility class for generating/accessing the request id from logging context. + * Utility class for recording and accessing context for the query being executed. */ -public class LogUtils { +public class QueryContext { /** * The key of the request id in the context map. @@ -29,8 +30,10 @@ public class LogUtils { * call this method twice on the same thread within the lifetime of the request. *

                */ - public static void addRequestId() { - ThreadContext.put(REQUEST_ID_KEY, UUID.randomUUID().toString()); + public static String addRequestId() { + var id = UUID.randomUUID().toString(); + ThreadContext.put(REQUEST_ID_KEY, id); + return id; } /** @@ -38,8 +41,11 @@ public static void addRequestId() { * @return the current request id from {@link ThreadContext}. */ public static String getRequestId() { - final String requestId = ThreadContext.get(REQUEST_ID_KEY); - return requestId; + var id = ThreadContext.get(REQUEST_ID_KEY); + if (null == id) { + id = addRequestId(); + } + return id; } /** @@ -57,7 +63,7 @@ public static Runnable withCurrentContext(final Runnable task) { }; } - private LogUtils() { + private QueryContext() { throw new AssertionError( getClass().getCanonicalName() + " is a utility class and must not be initialized"); } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/AsyncRestExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/AsyncRestExecutor.java index e6406e8b3ee..4ad6e557771 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/AsyncRestExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/AsyncRestExecutor.java @@ -19,13 +19,13 @@ import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestStatus; import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.exception.SqlParseException; import org.opensearch.sql.legacy.metrics.MetricName; import org.opensearch.sql.legacy.metrics.Metrics; import org.opensearch.sql.legacy.query.QueryAction; import org.opensearch.sql.legacy.query.join.BackOffRetryStrategy; -import org.opensearch.sql.legacy.utils.LogUtils; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transports; @@ -73,13 +73,13 @@ public void execute(Client client, Map params, QueryAction query if (isBlockingAction(queryAction) && isRunningInTransportThread()) { if (LOG.isDebugEnabled()) { LOG.debug("[{}] Async blocking query action [{}] for executor [{}] in current thread [{}]", - LogUtils.getRequestId(), name(executor), name(queryAction), Thread.currentThread().getName()); + QueryContext.getRequestId(), name(executor), name(queryAction), Thread.currentThread().getName()); } async(client, params, queryAction, channel); } else { if (LOG.isDebugEnabled()) { LOG.debug("[{}] Continue running query action [{}] for executor [{}] in current thread [{}]", - LogUtils.getRequestId(), name(executor), name(queryAction), Thread.currentThread().getName()); + QueryContext.getRequestId(), name(executor), name(queryAction), Thread.currentThread().getName()); } doExecuteWithTimeMeasured(client, params, queryAction, channel); } @@ -110,18 +110,18 @@ private void async(Client client, Map params, QueryAction queryA doExecuteWithTimeMeasured(client, params, queryAction, channel); } catch (IOException | SqlParseException | OpenSearchException e) { Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); - LOG.warn("[{}] [MCB] async task got an IO/SQL exception: {}", LogUtils.getRequestId(), + LOG.warn("[{}] [MCB] async task got an IO/SQL exception: {}", QueryContext.getRequestId(), e.getMessage()); channel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage())); } catch (IllegalStateException e) { Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); - LOG.warn("[{}] [MCB] async task got a runtime exception: {}", LogUtils.getRequestId(), + LOG.warn("[{}] [MCB] async task got a runtime exception: {}", QueryContext.getRequestId(), e.getMessage()); channel.sendResponse(new BytesRestResponse(RestStatus.INSUFFICIENT_STORAGE, "Memory circuit is broken.")); } catch (Throwable t) { Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); - LOG.warn("[{}] [MCB] async task got an unknown throwable: {}", LogUtils.getRequestId(), + LOG.warn("[{}] [MCB] async task got an unknown throwable: {}", QueryContext.getRequestId(), t.getMessage()); channel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, String.valueOf(t.getMessage()))); @@ -132,7 +132,7 @@ private void async(Client client, Map params, QueryAction queryA // Preserve context of calling thread to ensure headers of requests are forwarded when running blocking actions threadPool.schedule( - LogUtils.withCurrentContext(runnable), + QueryContext.withCurrentContext(runnable), new TimeValue(0L), SQL_WORKER_THREAD_POOL_NAME ); @@ -152,7 +152,7 @@ private void doExecuteWithTimeMeasured(Client client, Duration elapsed = Duration.ofNanos(System.nanoTime() - startTime); int slowLogThreshold = LocalClusterState.state().getSettingValue(Settings.Key.SQL_SLOWLOG); if (elapsed.getSeconds() >= slowLogThreshold) { - LOG.warn("[{}] Slow query: elapsed={} (ms)", LogUtils.getRequestId(), elapsed.toMillis()); + LOG.warn("[{}] Slow query: elapsed={} (ms)", QueryContext.getRequestId(), elapsed.toMillis()); } } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorAsyncRestExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorAsyncRestExecutor.java index 0dc1fe301f5..7bb64215022 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorAsyncRestExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorAsyncRestExecutor.java @@ -17,11 +17,11 @@ import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestStatus; import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.metrics.MetricName; import org.opensearch.sql.legacy.metrics.Metrics; import org.opensearch.sql.legacy.query.join.BackOffRetryStrategy; -import org.opensearch.sql.legacy.utils.LogUtils; import org.opensearch.threadpool.ThreadPool; public class CursorAsyncRestExecutor { @@ -57,20 +57,20 @@ private void async(Client client, Map params, RestChannel channe doExecuteWithTimeMeasured(client, params, channel); } catch (IOException e) { Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); - LOG.warn("[{}] [MCB] async task got an IO/SQL exception: {}", LogUtils.getRequestId(), + LOG.warn("[{}] [MCB] async task got an IO/SQL exception: {}", QueryContext.getRequestId(), e.getMessage()); e.printStackTrace(); channel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage())); } catch (IllegalStateException e) { Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); - LOG.warn("[{}] [MCB] async task got a runtime exception: {}", LogUtils.getRequestId(), + LOG.warn("[{}] [MCB] async task got a runtime exception: {}", QueryContext.getRequestId(), e.getMessage()); e.printStackTrace(); channel.sendResponse(new BytesRestResponse(RestStatus.INSUFFICIENT_STORAGE, "Memory circuit is broken.")); } catch (Throwable t) { Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); - LOG.warn("[{}] [MCB] async task got an unknown throwable: {}", LogUtils.getRequestId(), + LOG.warn("[{}] [MCB] async task got an unknown throwable: {}", QueryContext.getRequestId(), t.getMessage()); t.printStackTrace(); channel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, @@ -82,7 +82,7 @@ private void async(Client client, Map params, RestChannel channe // Preserve context of calling thread to ensure headers of requests are forwarded when running blocking actions threadPool.schedule( - LogUtils.withCurrentContext(runnable), + QueryContext.withCurrentContext(runnable), new TimeValue(0L), SQL_WORKER_THREAD_POOL_NAME ); @@ -101,7 +101,7 @@ private void doExecuteWithTimeMeasured(Client client, Duration elapsed = Duration.ofNanos(System.nanoTime() - startTime); int slowLogThreshold = LocalClusterState.state().getSettingValue(Settings.Key.SQL_SLOWLOG); if (elapsed.getSeconds() >= slowLogThreshold) { - LOG.warn("[{}] Slow query: elapsed={} (ms)", LogUtils.getRequestId(), elapsed.toMillis()); + LOG.warn("[{}] Slow query: elapsed={} (ms)", QueryContext.getRequestId(), elapsed.toMillis()); } } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java index 10d9dab0fa0..6f7579c9c7d 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java @@ -35,6 +35,7 @@ import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestStatus; import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.legacy.antlr.OpenSearchLegacySqlAnalyzer; @@ -60,7 +61,6 @@ import org.opensearch.sql.legacy.request.SqlRequestParam; import org.opensearch.sql.legacy.rewriter.matchtoterm.VerificationException; import org.opensearch.sql.legacy.utils.JsonPrettyFormatter; -import org.opensearch.sql.legacy.utils.LogUtils; import org.opensearch.sql.legacy.utils.QueryDataAnonymizer; import org.opensearch.sql.sql.domain.SQLQueryRequest; @@ -123,7 +123,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli Metrics.getInstance().getNumericalMetric(MetricName.REQ_TOTAL).increment(); Metrics.getInstance().getNumericalMetric(MetricName.REQ_COUNT_TOTAL).increment(); - LogUtils.addRequestId(); + QueryContext.addRequestId(); try { if (!isSQLFeatureEnabled()) { @@ -137,12 +137,12 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli if (isExplainRequest(request)) { throw new IllegalArgumentException("Invalid request. Cannot explain cursor"); } else { - LOG.info("[{}] Cursor request {}: {}", LogUtils.getRequestId(), request.uri(), sqlRequest.cursor()); + LOG.info("[{}] Cursor request {}: {}", QueryContext.getRequestId(), request.uri(), sqlRequest.cursor()); return channel -> handleCursorRequest(request, sqlRequest.cursor(), client, channel); } } - LOG.info("[{}] Incoming request {}: {}", LogUtils.getRequestId(), request.uri(), + LOG.info("[{}] Incoming request {}: {}", QueryContext.getRequestId(), request.uri(), QueryDataAnonymizer.anonymizeData(sqlRequest.getSql())); Format format = SqlRequestParam.getFormat(request.params()); @@ -152,11 +152,11 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli sqlRequest.getSql(), request.path(), request.params()); RestChannelConsumer result = newSqlQueryHandler.prepareRequest(newSqlRequest, client); if (result != RestSQLQueryAction.NOT_SUPPORTED_YET) { - LOG.info("[{}] Request is handled by new SQL query engine", LogUtils.getRequestId()); + LOG.info("[{}] Request is handled by new SQL query engine", QueryContext.getRequestId()); return result; } LOG.debug("[{}] Request {} is not supported and falling back to old SQL engine", - LogUtils.getRequestId(), newSqlRequest); + QueryContext.getRequestId(), newSqlRequest); final QueryAction queryAction = explainRequest(client, sqlRequest, format); return channel -> executeSqlRequest(request, queryAction, client, channel); @@ -182,10 +182,10 @@ private void handleCursorRequest(final RestRequest request, final String cursor, private static void logAndPublishMetrics(final Exception e) { if (isClientError(e)) { - LOG.error(LogUtils.getRequestId() + " Client side error during query execution", e); + LOG.error(QueryContext.getRequestId() + " Client side error during query execution", e); Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_CUS).increment(); } else { - LOG.error(LogUtils.getRequestId() + " Server side error during query execution", e); + LOG.error(QueryContext.getRequestId() + " Server side error during query execution", e); Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment(); } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlStatsAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlStatsAction.java index 70ec21c3fab..5b48ef67104 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlStatsAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlStatsAction.java @@ -22,9 +22,9 @@ import org.opensearch.rest.RestController; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestStatus; +import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.legacy.executor.format.ErrorMessageFactory; import org.opensearch.sql.legacy.metrics.Metrics; -import org.opensearch.sql.legacy.utils.LogUtils; /** * Currently this interface is for node level. @@ -67,7 +67,7 @@ public List replacedRoutes() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { - LogUtils.addRequestId(); + QueryContext.addRequestId(); try { return channel -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/utils/LogUtils.java b/legacy/src/main/java/org/opensearch/sql/legacy/utils/LogUtils.java deleted file mode 100644 index 4830dd44135..00000000000 --- a/legacy/src/main/java/org/opensearch/sql/legacy/utils/LogUtils.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.legacy.utils; - -import java.util.Map; -import java.util.Optional; -import java.util.UUID; -import org.apache.logging.log4j.ThreadContext; - -/** - * Utility class for generating/accessing the request id from logging context. - */ -public class LogUtils { - - /** - * The key of the request id in the context map - */ - private static final String REQUEST_ID_KEY = "request_id"; - - private static final String EMPTY_ID = "ID"; - - /** - * Generates a random UUID and adds to the {@link ThreadContext} as the request id. - *

                - * Note: If a request id already present, this method will overwrite it with a new - * one. This is to pre-vent re-using the same request id for different requests in - * case the same thread handles both of them. But this also means one should not - * call this method twice on the same thread within the lifetime of the request. - *

                - */ - public static void addRequestId() { - - ThreadContext.put(REQUEST_ID_KEY, UUID.randomUUID().toString()); - } - - /** - * @return the current request id from {@link ThreadContext} - */ - public static String getRequestId() { - return Optional.ofNullable(ThreadContext.get(REQUEST_ID_KEY)).orElseGet(() -> EMPTY_ID); - } - - /** - * Wraps a given instance of {@link Runnable} into a new one which gets all the - * entries from current ThreadContext map. - * - * @param task the instance of Runnable to wrap - * @return the new task - */ - public static Runnable withCurrentContext(final Runnable task) { - - final Map currentContext = ThreadContext.getImmutableContext(); - return () -> { - ThreadContext.putAll(currentContext); - task.run(); - }; - } - - private LogUtils() { - - throw new AssertionError(getClass().getCanonicalName() + " is a utility class and must not be initialized"); - } -} diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/LogUtilsTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/QueryContextTest.java similarity index 79% rename from legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/LogUtilsTest.java rename to legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/QueryContextTest.java index 564ce8c9ea7..55b78af0d7e 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/LogUtilsTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/QueryContextTest.java @@ -8,15 +8,15 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import org.apache.logging.log4j.ThreadContext; import org.junit.After; import org.junit.Assert; import org.junit.Test; -import org.opensearch.sql.legacy.utils.LogUtils; +import org.opensearch.sql.common.utils.QueryContext; -public class LogUtilsTest { +public class QueryContextTest { private static final String REQUEST_ID_KEY = "request_id"; @@ -30,7 +30,7 @@ public void cleanUpContext() { public void addRequestId() { Assert.assertNull(ThreadContext.get(REQUEST_ID_KEY)); - LogUtils.addRequestId(); + QueryContext.addRequestId(); final String requestId = ThreadContext.get(REQUEST_ID_KEY); Assert.assertNotNull(requestId); } @@ -38,16 +38,16 @@ public void addRequestId() { @Test public void addRequestId_alreadyExists() { - LogUtils.addRequestId(); + QueryContext.addRequestId(); final String requestId = ThreadContext.get(REQUEST_ID_KEY); - LogUtils.addRequestId(); + QueryContext.addRequestId(); final String requestId2 = ThreadContext.get(REQUEST_ID_KEY); Assert.assertThat(requestId2, not(equalTo(requestId))); } @Test public void getRequestId_doesNotExist() { - assertEquals("ID", LogUtils.getRequestId()); + assertNotNull(QueryContext.getRequestId()); } @Test @@ -55,7 +55,7 @@ public void getRequestId() { final String test_request_id = "test_id_111"; ThreadContext.put(REQUEST_ID_KEY, test_request_id); - final String requestId = LogUtils.getRequestId(); + final String requestId = QueryContext.getRequestId(); Assert.assertThat(requestId, equalTo(test_request_id)); } @@ -68,6 +68,6 @@ public void withCurrentContext() throws InterruptedException { }; ThreadContext.put("test11", "value11"); ThreadContext.put("test22", "value11"); - new Thread(LogUtils.withCurrentContext(task)).join(); + new Thread(QueryContext.withCurrentContext(task)).join(); } } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLStatsAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLStatsAction.java index 6b5cbd4135b..5b9c792c7dd 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLStatsAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLStatsAction.java @@ -22,9 +22,9 @@ import org.opensearch.rest.RestController; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestStatus; +import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.legacy.executor.format.ErrorMessageFactory; import org.opensearch.sql.legacy.metrics.Metrics; -import org.opensearch.sql.legacy.utils.LogUtils; /** * PPL Node level status. @@ -67,7 +67,7 @@ public List replacedRoutes() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { - LogUtils.addRequestId(); + QueryContext.addRequestId(); try { return channel -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestQuerySettingsAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestQuerySettingsAction.java index 0d8ca66cc69..14d06dfc71a 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestQuerySettingsAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestQuerySettingsAction.java @@ -28,8 +28,8 @@ import org.opensearch.rest.RestController; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.legacy.executor.format.ErrorMessageFactory; -import org.opensearch.sql.legacy.utils.LogUtils; public class RestQuerySettingsAction extends BaseRestHandler { private static final Logger LOG = LogManager.getLogger(RestQuerySettingsAction.class); @@ -74,7 +74,7 @@ public List replacedRoutes() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - LogUtils.addRequestId(); + QueryContext.addRequestId(); final ClusterUpdateSettingsRequest clusterUpdateSettingsRequest = Requests.clusterUpdateSettingsRequest(); clusterUpdateSettingsRequest.timeout(request.paramAsTime( diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java index e4699b6f9f7..31317c1962e 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java @@ -20,7 +20,7 @@ import org.opensearch.common.inject.Inject; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.common.utils.LogUtils; +import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.legacy.metrics.MetricName; import org.opensearch.sql.legacy.metrics.Metrics; @@ -77,7 +77,7 @@ protected void doExecute( Metrics.getInstance().getNumericalMetric(MetricName.PPL_REQ_TOTAL).increment(); Metrics.getInstance().getNumericalMetric(MetricName.PPL_REQ_COUNT_TOTAL).increment(); - LogUtils.addRequestId(); + QueryContext.addRequestId(); PPLService pplService = createPPLService(client); TransportPPLQueryRequest transportRequest = TransportPPLQueryRequest.fromActionRequest(request); diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java index a1a831c7cd9..866326f5627 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java @@ -16,7 +16,7 @@ import org.opensearch.sql.analysis.Analyzer; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.response.ResponseListener; -import org.opensearch.sql.common.utils.LogUtils; +import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; import org.opensearch.sql.expression.DSL; @@ -84,7 +84,8 @@ private PhysicalPlan plan(PPLQueryRequest request) { UnresolvedPlan ast = cst.accept( new AstBuilder(new AstExpressionBuilder(), request.getRequest())); - LOG.info("[{}] Incoming request {}", LogUtils.getRequestId(), anonymizer.anonymizeData(ast)); + LOG.info("[{}] Incoming request {}", QueryContext.getRequestId(), + anonymizer.anonymizeData(ast)); // 2.Analyze abstract syntax to generate logical plan LogicalPlan logicalPlan = analyzer.analyze(UnresolvedPlanHelper.addSelectAll(ast), From ac9b63aba45d2df9d62677f1171af91ae09be2aa Mon Sep 17 00:00:00 2001 From: MitchellGale-BitQuill <104795536+MitchellGale-BitQuill@users.noreply.github.com> Date: Mon, 15 Aug 2022 13:21:51 -0700 Subject: [PATCH 14/20] Add any case of arguments in relevancy based functions to be allowed (#744) * Added new functionality around argument names not case specific for relevancy based functions. Added unit tests to cover updates. Signed-off-by: MitchellGale-BitQuill * Fixed test cases that used uppercase values in arguments. Signed-off-by: MitchellGale-BitQuill Signed-off-by: MitchellGale-BitQuill --- .../relevance/MatchPhrasePrefixQuery.java | 2 +- .../lucene/relevance/MatchPhraseQuery.java | 2 +- .../filter/lucene/relevance/MatchQuery.java | 4 +-- .../lucene/relevance/MultiMatchQuery.java | 9 +++--- .../lucene/relevance/QueryStringQuery.java | 9 +++--- .../lucene/relevance/RelevanceQuery.java | 15 ++++++++-- .../relevance/SimpleQueryStringQuery.java | 9 +++--- .../script/filter/FilterQueryBuilderTest.java | 2 +- .../lucene/MatchPhrasePrefixQueryTest.java | 9 ++++++ .../filter/lucene/MatchPhraseQueryTest.java | 10 +++++++ .../script/filter/lucene/MatchQueryTest.java | 5 ++++ .../script/filter/lucene/MultiMatchTest.java | 5 ++++ .../script/filter/lucene/QueryStringTest.java | 8 ++++- .../script/filter/lucene/RangeQueryTest.java | 2 +- .../filter/lucene/SimpleQueryStringTest.java | 29 +++++++++++++++++++ .../relevance/RelevanceQueryBuildTest.java | 2 +- 16 files changed, 99 insertions(+), 23 deletions(-) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhrasePrefixQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhrasePrefixQuery.java index 83ccf256860..b8d0d4f18d1 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhrasePrefixQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhrasePrefixQuery.java @@ -23,7 +23,7 @@ public MatchPhrasePrefixQuery() { .put("slop", (b, v) -> b.slop(Integer.parseInt(v.stringValue()))) .put("max_expansions", (b, v) -> b.maxExpansions(Integer.parseInt(v.stringValue()))) .put("zero_terms_query", (b, v) -> b.zeroTermsQuery( - org.opensearch.index.search.MatchQuery.ZeroTermsQuery.valueOf(v.stringValue()))) + org.opensearch.index.search.MatchQuery.ZeroTermsQuery.valueOf(valueOfToUpper(v)))) .put("boost", (b, v) -> b.boost(Float.parseFloat(v.stringValue()))) .build()); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhraseQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhraseQuery.java index 9004fa78b45..333d8eff899 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhraseQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhraseQuery.java @@ -34,7 +34,7 @@ public MatchPhraseQuery() { .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) .put("slop", (b, v) -> b.slop(Integer.parseInt(v.stringValue()))) .put("zero_terms_query", (b, v) -> b.zeroTermsQuery( - org.opensearch.index.search.MatchQuery.ZeroTermsQuery.valueOf(v.stringValue()))) + org.opensearch.index.search.MatchQuery.ZeroTermsQuery.valueOf(valueOfToUpper(v)))) .build()); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchQuery.java index c69b43cbcba..4095ffba4ed 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchQuery.java @@ -24,7 +24,7 @@ public MatchQuery() { .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) .put("auto_generate_synonyms_phrase_query", (b, v) -> b.autoGenerateSynonymsPhraseQuery(Boolean.parseBoolean(v.stringValue()))) - .put("fuzziness", (b, v) -> b.fuzziness(v.stringValue())) + .put("fuzziness", (b, v) -> b.fuzziness(valueOfToUpper(v))) .put("max_expansions", (b, v) -> b.maxExpansions(Integer.parseInt(v.stringValue()))) .put("prefix_length", (b, v) -> b.prefixLength(Integer.parseInt(v.stringValue()))) .put("fuzzy_transpositions", @@ -34,7 +34,7 @@ public MatchQuery() { .put("operator", (b, v) -> b.operator(Operator.fromString(v.stringValue()))) .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) .put("zero_terms_query", (b, v) -> b.zeroTermsQuery( - org.opensearch.index.search.MatchQuery.ZeroTermsQuery.valueOf(v.stringValue()))) + org.opensearch.index.search.MatchQuery.ZeroTermsQuery.valueOf(valueOfToUpper(v)))) .put("boost", (b, v) -> b.boost(Float.parseFloat(v.stringValue()))) .build()); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiMatchQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiMatchQuery.java index b2865c12263..524d42f0b6e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiMatchQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiMatchQuery.java @@ -41,7 +41,7 @@ public MultiMatchQuery() { .put("type", (b, v) -> b.type(v.stringValue())) .put("slop", (b, v) -> b.slop(Integer.parseInt(v.stringValue()))) .put("zero_terms_query", (b, v) -> b.zeroTermsQuery( - org.opensearch.index.search.MatchQuery.ZeroTermsQuery.valueOf(v.stringValue()))) + org.opensearch.index.search.MatchQuery.ZeroTermsQuery.valueOf(valueOfToUpper(v)))) .build()); } @@ -67,14 +67,15 @@ public QueryBuilder build(FunctionExpression func) { .fields(fieldsAndWeights); while (iterator.hasNext()) { NamedArgumentExpression arg = (NamedArgumentExpression) iterator.next(); - if (!queryBuildActions.containsKey(arg.getArgName())) { + String argNormalized = arg.getArgName().toLowerCase(); + if (!queryBuildActions.containsKey(argNormalized)) { throw new SemanticCheckException( String.format("Parameter %s is invalid for %s function.", - arg.getArgName(), queryBuilder.getWriteableName())); + argNormalized, queryBuilder.getWriteableName())); } (Objects.requireNonNull( queryBuildActions - .get(arg.getArgName()))) + .get(argNormalized))) .apply(queryBuilder, arg.getValue().valueOf(null)); } return queryBuilder; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryStringQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryStringQuery.java index 53f4c3008b6..54ffea6158f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryStringQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryStringQuery.java @@ -59,7 +59,7 @@ public QueryStringQuery() { .put("phrase_slop", (b, v) -> b.phraseSlop(Integer.parseInt(v.stringValue()))) .put("quote_field_suffix", (b, v) -> b.quoteFieldSuffix(v.stringValue())) .put("rewrite", (b, v) -> b.rewrite(v.stringValue())) - .put("type", (b, v) -> b.type(MultiMatchQueryBuilder.Type.parse(v.stringValue(), + .put("type", (b, v) -> b.type(MultiMatchQueryBuilder.Type.parse(valueOfToLower(v), LoggingDeprecationHandler.INSTANCE))) .put("tie_breaker", (b, v) -> b.tieBreaker(Float.parseFloat(v.stringValue()))) .put("time_zone", (b, v) -> b.timeZone(v.stringValue())) @@ -93,14 +93,15 @@ public QueryBuilder build(FunctionExpression func) { .fields(fieldsAndWeights); while (iterator.hasNext()) { NamedArgumentExpression arg = (NamedArgumentExpression) iterator.next(); - if (!queryBuildActions.containsKey(arg.getArgName())) { + String argNormalized = arg.getArgName().toLowerCase(); + if (!queryBuildActions.containsKey(argNormalized)) { throw new SemanticCheckException( String.format("Parameter %s is invalid for %s function.", - arg.getArgName(), queryBuilder.getWriteableName())); + argNormalized, queryBuilder.getWriteableName())); } (Objects.requireNonNull( queryBuildActions - .get(arg.getArgName()))) + .get(argNormalized))) .apply(queryBuilder, arg.getValue().valueOf(null)); } return queryBuilder; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java index fb0852c18b8..fb997646f4f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java @@ -46,14 +46,15 @@ public QueryBuilder build(FunctionExpression func) { Iterator iterator = arguments.listIterator(2); while (iterator.hasNext()) { NamedArgumentExpression arg = (NamedArgumentExpression) iterator.next(); - if (!queryBuildActions.containsKey(arg.getArgName())) { + String argNormalized = arg.getArgName().toLowerCase(); + if (!queryBuildActions.containsKey(argNormalized)) { throw new SemanticCheckException( String.format("Parameter %s is invalid for %s function.", - arg.getArgName(), queryBuilder.getWriteableName())); + argNormalized, queryBuilder.getWriteableName())); } (Objects.requireNonNull( queryBuildActions - .get(arg.getArgName()))) + .get(argNormalized))) .apply(queryBuilder, arg.getValue().valueOf(null)); } return queryBuilder; @@ -70,4 +71,12 @@ public interface QueryBuilderStep extends BiFunction { } + + public static String valueOfToUpper(ExprValue v) { + return v.stringValue().toUpperCase(); + } + + public static String valueOfToLower(ExprValue v) { + return v.stringValue().toLowerCase(); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SimpleQueryStringQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SimpleQueryStringQuery.java index 539fc26382c..45637e98a6a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SimpleQueryStringQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SimpleQueryStringQuery.java @@ -32,7 +32,7 @@ public SimpleQueryStringQuery() { b.autoGenerateSynonymsPhraseQuery(Boolean.parseBoolean(v.stringValue()))) .put("boost", (b, v) -> b.boost(Float.parseFloat(v.stringValue()))) .put("default_operator", (b, v) -> b.defaultOperator(Operator.fromString(v.stringValue()))) - .put("flags", (b, v) -> b.flags(Arrays.stream(v.stringValue().split("\\|")) + .put("flags", (b, v) -> b.flags(Arrays.stream(valueOfToUpper(v).split("\\|")) .map(SimpleQueryStringFlag::valueOf) .toArray(SimpleQueryStringFlag[]::new))) .put("fuzzy_max_expansions", (b, v) -> @@ -69,14 +69,15 @@ public QueryBuilder build(FunctionExpression func) { .fields(fieldsAndWeights); while (iterator.hasNext()) { NamedArgumentExpression arg = (NamedArgumentExpression) iterator.next(); - if (!queryBuildActions.containsKey(arg.getArgName())) { + String argNormalized = arg.getArgName().toLowerCase(); + if (!queryBuildActions.containsKey(argNormalized)) { throw new SemanticCheckException( String.format("Parameter %s is invalid for %s function.", - arg.getArgName(), queryBuilder.getWriteableName())); + argNormalized, queryBuilder.getWriteableName())); } (Objects.requireNonNull( queryBuildActions - .get(arg.getArgName()))) + .get(argNormalized))) .apply(queryBuilder, arg.getValue().valueOf(null)); } return queryBuilder; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java index 11c9afe1743..b1efe86d018 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java @@ -851,7 +851,7 @@ void match_phrase_invalid_value_ztq() { dsl.namedArgument("query", literal("search query")), dsl.namedArgument("zero_terms_query", literal("meow"))); var msg = assertThrows(IllegalArgumentException.class, () -> buildQuery(expr)).getMessage(); - assertEquals("No enum constant org.opensearch.index.search.MatchQuery.ZeroTermsQuery.meow", + assertEquals("No enum constant org.opensearch.index.search.MatchQuery.ZeroTermsQuery.MEOW", msg); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhrasePrefixQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhrasePrefixQueryTest.java index 9a5eeb14b55..8e1a2fcdf0b 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhrasePrefixQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhrasePrefixQueryTest.java @@ -94,6 +94,15 @@ public void test_zero_terms_query_parameter() { Assertions.assertNotNull(matchPhrasePrefixQuery.build(new MatchPhraseExpression(arguments))); } + @Test + public void test_zero_terms_query_parameter_lower_case() { + List arguments = List.of( + dsl.namedArgument("field", "t1"), + dsl.namedArgument("query", "t2"), + dsl.namedArgument("zero_terms_query", "all") + ); + Assertions.assertNotNull(matchPhrasePrefixQuery.build(new MatchPhraseExpression(arguments))); + } @Test public void test_boost_parameter() { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhraseQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhraseQueryTest.java index fef3d64f95f..4e8895a12a8 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhraseQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhraseQueryTest.java @@ -99,6 +99,16 @@ public void test_zero_terms_query_parameter() { Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression(arguments))); } + @Test + public void test_zero_terms_query_parameter_lower_case() { + List arguments = List.of( + namedArgument("field", "t1"), + namedArgument("query", "t2"), + namedArgument("zero_terms_query", "all") + ); + Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression(arguments))); + } + private class MatchPhraseExpression extends FunctionExpression { public MatchPhraseExpression(List arguments) { super(MatchPhraseQueryTest.this.matchPhrase, arguments); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchQueryTest.java index 99cf132a3ee..60dd938f78d 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchQueryTest.java @@ -96,6 +96,11 @@ static Stream> generateValidData() { dsl.namedArgument("query", DSL.literal("query_value")), dsl.namedArgument("zero_terms_query", DSL.literal("NONE")) ), + List.of( + dsl.namedArgument("field", DSL.literal("field_value")), + dsl.namedArgument("query", DSL.literal("query_value")), + dsl.namedArgument("zero_terms_query", DSL.literal("none")) + ), List.of( dsl.namedArgument("field", DSL.literal("field_value")), dsl.namedArgument("query", DSL.literal("query_value")), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MultiMatchTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MultiMatchTest.java index 51499faf036..4a6e1d2ed9b 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MultiMatchTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MultiMatchTest.java @@ -120,6 +120,11 @@ static Stream> generateValidData() { dsl.namedArgument("fields", fields_value), dsl.namedArgument("query", query_value), dsl.namedArgument("zero_terms_query", DSL.literal("ALL")) + ), + List.of( + dsl.namedArgument("fields", fields_value), + dsl.namedArgument("query", query_value), + dsl.namedArgument("zero_terms_query", DSL.literal("all")) ) ); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryStringTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryStringTest.java index 31c9f981a53..fce835bf43d 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryStringTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryStringTest.java @@ -54,6 +54,7 @@ static Stream> generateValidData() { dsl.namedArgument("auto_generate_synonyms_phrase_query", DSL.literal("true")), dsl.namedArgument("boost", DSL.literal("1")), dsl.namedArgument("default_operator", DSL.literal("AND")), + dsl.namedArgument("default_operator", DSL.literal("and")), dsl.namedArgument("enable_position_increments", DSL.literal("true")), dsl.namedArgument("escape", DSL.literal("false")), dsl.namedArgument("fuzziness", DSL.literal("1")), @@ -70,7 +71,12 @@ static Stream> generateValidData() { dsl.namedArgument("rewrite", DSL.literal("constant_score")), dsl.namedArgument("type", DSL.literal("best_fields")), dsl.namedArgument("tie_breaker", DSL.literal("0.3")), - dsl.namedArgument("time_zone", DSL.literal("Canada/Pacific")) + dsl.namedArgument("time_zone", DSL.literal("Canada/Pacific")), + dsl.namedArgument("ANALYZER", DSL.literal("standard")), + dsl.namedArgument("ANALYZE_wildcard", DSL.literal("true")), + dsl.namedArgument("Allow_Leading_wildcard", DSL.literal("true")), + dsl.namedArgument("Auto_Generate_Synonyms_Phrase_Query", DSL.literal("true")), + dsl.namedArgument("Boost", DSL.literal("1")) ).stream().map(arg -> List.of(field, query, arg)); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQueryTest.java index 2d1f36a81de..208c7825933 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQueryTest.java @@ -20,7 +20,7 @@ class RangeQueryTest { @Test void should_throw_exception_for_unsupported_comparison() { - // Note that since we do switch check on enum comparison, this should'be impossible + // Note that since we do switch check on enum comparison, this should be impossible assertThrows(IllegalStateException.class, () -> new RangeQuery(Comparison.BETWEEN) .doBuild("name", STRING, ExprValueUtils.stringValue("John"))); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/SimpleQueryStringTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/SimpleQueryStringTest.java index 954d82474d3..048f6e1cb92 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/SimpleQueryStringTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/SimpleQueryStringTest.java @@ -81,6 +81,16 @@ static Stream> generateValidData() { dsl.namedArgument("query", query_value), dsl.namedArgument("flags", DSL.literal("NOT|AND")) ), + List.of( + dsl.namedArgument("fields", fields_value), + dsl.namedArgument("query", query_value), + dsl.namedArgument("flags", DSL.literal("PREFIX|not|AND")) + ), + List.of( + dsl.namedArgument("fields", fields_value), + dsl.namedArgument("query", query_value), + dsl.namedArgument("flags", DSL.literal("not|and")) + ), List.of( dsl.namedArgument("fields", fields_value), dsl.namedArgument("query", query_value), @@ -106,6 +116,11 @@ static Stream> generateValidData() { dsl.namedArgument("query", query_value), dsl.namedArgument("default_operator", DSL.literal("AND")) ), + List.of( + dsl.namedArgument("fields", fields_value), + dsl.namedArgument("query", query_value), + dsl.namedArgument("default_operator", DSL.literal("and")) + ), List.of( dsl.namedArgument("fields", fields_value), dsl.namedArgument("query", query_value), @@ -120,6 +135,20 @@ static Stream> generateValidData() { dsl.namedArgument("fields", fields_value), dsl.namedArgument("query", query_value), dsl.namedArgument("boost", DSL.literal("1")) + ), + List.of( + dsl.namedArgument("FIELDS", fields_value), + dsl.namedArgument("QUERY", query_value) + ), + List.of( + dsl.namedArgument("FIELDS", fields_value), + dsl.namedArgument("query", query_value), + dsl.namedArgument("ANALYZE_wildcard", DSL.literal("true")) + ), + List.of( + dsl.namedArgument("fields", fields_value), + dsl.namedArgument("query", query_value), + dsl.namedArgument("analyZER", DSL.literal("standard")) ) ); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQueryBuildTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQueryBuildTest.java index 1186031f5ff..a67f0f34a7f 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQueryBuildTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQueryBuildTest.java @@ -72,7 +72,7 @@ void throws_SemanticCheckException_when_wrong_argument_name() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, () -> query.build(expr)); - assertEquals("Parameter wrongArg is invalid for mock_query function.", exception.getMessage()); + assertEquals("Parameter wrongarg is invalid for mock_query function.", exception.getMessage()); } @Test From 45d84d005ebbca8bb88e3cff48327131f35cb6e6 Mon Sep 17 00:00:00 2001 From: Max Ksyunz Date: Mon, 15 Aug 2022 14:39:08 -0700 Subject: [PATCH 15/20] Callout distinction between original and current SQL engines. (#728) Signed-off-by: MaxKsyunz Signed-off-by: MaxKsyunz --- docs/user/beyond/fulltext.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/user/beyond/fulltext.rst b/docs/user/beyond/fulltext.rst index c2edbfa97d6..558ddfadbf4 100644 --- a/docs/user/beyond/fulltext.rst +++ b/docs/user/beyond/fulltext.rst @@ -14,6 +14,8 @@ Introduction Full-text search is for searching a single stored document which is distinguished from regular search based on original texts in database. It tries to match search criteria by examining all of the words in each document. In OpenSearch, full-text queries provided enables you to search text fields analyzed during indexing. +This document describes the original SQL engine. It only applies to queries that cannot be handled by `SQL engine V2 <../../dev/NewSQLEngine.md>`. Documentation for full-text search using SQL Engine V2 can be found `here <../dql/functions.rst#relevance>`_. + Match Query =========== From 95a24b25e1bcf80a99c5ab1201e10c315e2d277d Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 16 Aug 2022 09:20:26 -0700 Subject: [PATCH 16/20] Update PPL docs link for workbench (#758) Signed-off-by: Joshua Li Signed-off-by: Joshua Li --- workbench/public/components/Main/main.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workbench/public/components/Main/main.tsx b/workbench/public/components/Main/main.tsx index e928fefe9ae..9e0964988a4 100644 --- a/workbench/public/components/Main/main.tsx +++ b/workbench/public/components/Main/main.tsx @@ -640,7 +640,7 @@ export class Main extends React.Component { updatePPLQueries={this.updatePPLQueries} /> ); - link = 'https://opensearch.org/docs/latest/observability/ppl/index/'; + link = 'https://opensearch.org/docs/latest/observability-plugin/ppl/index/'; linkTitle = 'PPL documentation'; } From 9f602c382aea1b481219b6a853b83f4214d9f342 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Tue, 16 Aug 2022 09:28:14 -0700 Subject: [PATCH 17/20] Add `maketime` and `makedate` (#102) (#755) Signed-off-by: Yury Fridlyand Signed-off-by: Yury Fridlyand --- .../expression/datetime/DateTimeFunction.java | 60 ++++++ .../function/BuiltinFunctionName.java | 2 + .../datetime/DateTimeFunctionTest.java | 12 ++ .../sql/expression/datetime/MakeDateTest.java | 174 ++++++++++++++++++ .../sql/expression/datetime/MakeTimeTest.java | 172 +++++++++++++++++ docs/user/dql/functions.rst | 57 +++++- docs/user/ppl/functions/datetime.rst | 57 +++++- .../sql/ppl/DateTimeFunctionIT.java | 17 ++ .../sql/sql/DateTimeFunctionIT.java | 19 ++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 +- sql/src/main/antlr/OpenSearchSQLLexer.g4 | 1 + sql/src/main/antlr/OpenSearchSQLParser.g4 | 2 +- 13 files changed, 573 insertions(+), 4 deletions(-) create mode 100644 core/src/test/java/org/opensearch/sql/expression/datetime/MakeDateTest.java create mode 100644 core/src/test/java/org/opensearch/sql/expression/datetime/MakeTimeTest.java diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index c4de0e13ad9..0fccacd1362 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.INTERVAL; import static org.opensearch.sql.data.type.ExprCoreType.LONG; @@ -19,6 +20,8 @@ import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import java.time.LocalDate; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; import java.time.format.TextStyle; import java.util.Locale; import java.util.concurrent.TimeUnit; @@ -27,6 +30,7 @@ import org.opensearch.sql.data.model.ExprDatetimeValue; import org.opensearch.sql.data.model.ExprIntegerValue; import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; @@ -64,6 +68,8 @@ public void register(BuiltinFunctionRepository repository) { repository.register(dayOfYear()); repository.register(from_days()); repository.register(hour()); + repository.register(makedate()); + repository.register(maketime()); repository.register(microsecond()); repository.register(minute()); repository.register(month()); @@ -236,6 +242,16 @@ private FunctionResolver hour() { ); } + private FunctionResolver makedate() { + return define(BuiltinFunctionName.MAKEDATE.getName(), + impl(nullMissingHandling(DateTimeFunction::exprMakeDate), DATE, DOUBLE, DOUBLE)); + } + + private FunctionResolver maketime() { + return define(BuiltinFunctionName.MAKETIME.getName(), + impl(nullMissingHandling(DateTimeFunction::exprMakeTime), TIME, DOUBLE, DOUBLE, DOUBLE)); + } + /** * MICROSECOND(STRING/TIME/DATETIME/TIMESTAMP). return the microsecond value for time. */ @@ -512,6 +528,50 @@ private ExprValue exprHour(ExprValue time) { return new ExprIntegerValue(time.timeValue().getHour()); } + /** + * Following MySQL, function receives arguments of type double and rounds them before use. + * Furthermore: + * - zero year interpreted as 2000 + * - negative year is not accepted + * - @dayOfYear should be greater than 1 + * - if @dayOfYear is greater than 365/366, calculation goes to the next year(s) + * + * @param yearExpr year + * @param dayOfYearExp day of the @year, starting from 1 + * @return Date - ExprDateValue object with LocalDate + */ + private ExprValue exprMakeDate(ExprValue yearExpr, ExprValue dayOfYearExp) { + var year = Math.round(yearExpr.doubleValue()); + var dayOfYear = Math.round(dayOfYearExp.doubleValue()); + // We need to do this to comply with MySQL + if (0 >= dayOfYear || 0 > year) { + return ExprNullValue.of(); + } + if (0 == year) { + year = 2000; + } + return new ExprDateValue(LocalDate.ofYearDay((int)year, 1).plusDays(dayOfYear - 1)); + } + + /** + * Following MySQL, function receives arguments of type double. @hour and @minute are rounded, + * while @second used as is, including fraction part. + * @param hourExpr hour + * @param minuteExpr minute + * @param secondExpr second + * @return Time - ExprTimeValue object with LocalTime + */ + private ExprValue exprMakeTime(ExprValue hourExpr, ExprValue minuteExpr, ExprValue secondExpr) { + var hour = Math.round(hourExpr.doubleValue()); + var minute = Math.round(minuteExpr.doubleValue()); + var second = secondExpr.doubleValue(); + if (0 > hour || 0 > minute || 0 > second) { + return ExprNullValue.of(); + } + return new ExprTimeValue(LocalTime.parse(String.format("%02d:%02d:%012.9f", + hour, minute, second), DateTimeFormatter.ISO_TIME)); + } + /** * Microsecond implementation for ExprValue. * diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index cd343284530..b3821d6e415 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -68,6 +68,8 @@ public enum BuiltinFunctionName { DAYOFYEAR(FunctionName.of("dayofyear")), FROM_DAYS(FunctionName.of("from_days")), HOUR(FunctionName.of("hour")), + MAKEDATE(FunctionName.of("makedate")), + MAKETIME(FunctionName.of("maketime")), MICROSECOND(FunctionName.of("microsecond")), MINUTE(FunctionName.of("minute")), MONTH(FunctionName.of("month")), diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 79efa2a0158..89415e0560c 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -16,6 +16,7 @@ import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.INTERVAL; import static org.opensearch.sql.data.type.ExprCoreType.LONG; @@ -24,7 +25,15 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import com.google.common.collect.ImmutableList; +import java.time.Duration; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.Year; +import java.util.HashSet; import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.stream.IntStream; import lombok.AllArgsConstructor; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -43,7 +52,10 @@ import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ExpressionTestBase; import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.env.Environment; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.expression.function.FunctionSignature; @ExtendWith(MockitoExtension.class) class DateTimeFunctionTest extends ExpressionTestBase { diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/MakeDateTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/MakeDateTest.java new file mode 100644 index 00000000000..497e73ea519 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/MakeDateTest.java @@ -0,0 +1,174 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.expression.datetime; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.model.ExprValueUtils.missingValue; +import static org.opensearch.sql.data.model.ExprValueUtils.nullValue; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; + +import java.time.LocalDate; +import java.time.Year; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.ExpressionTestBase; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.config.ExpressionConfig; +import org.opensearch.sql.expression.env.Environment; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.expression.function.FunctionSignature; + +@ExtendWith(MockitoExtension.class) +public class MakeDateTest extends ExpressionTestBase { + + @Mock + Environment env; + + @Mock + Expression nullRef; + + @Mock + Expression missingRef; + + private FunctionExpression makedate(Expression year, Expression dayOfYear) { + var repo = new ExpressionConfig().functionRepository(); + var func = repo.resolve(new FunctionSignature(new FunctionName("makedate"), + List.of(DOUBLE, DOUBLE))); + return (FunctionExpression)func.apply(List.of(year, dayOfYear)); + } + + private LocalDate makedate(Double year, Double dayOfYear) { + return makedate(DSL.literal(year), DSL.literal(dayOfYear)).valueOf(null).dateValue(); + } + + @Test + public void checkEdgeCases() { + assertEquals(LocalDate.ofYearDay(2002, 1), makedate(2001., 366.), + "No switch to the next year on getting 366th day of a non-leap year"); + assertEquals(LocalDate.ofYearDay(2005, 1), makedate(2004., 367.), + "No switch to the next year on getting 367th day of a leap year"); + assertEquals(LocalDate.ofYearDay(2000, 42), makedate(0., 42.), + "0 year is not interpreted as 2000 as in MySQL"); + assertEquals(nullValue(), eval(makedate(DSL.literal(-1.), DSL.literal(42.))), + "Negative year doesn't produce NULL"); + assertEquals(nullValue(), eval(makedate(DSL.literal(42.), DSL.literal(-1.))), + "Negative dayOfYear doesn't produce NULL"); + assertEquals(nullValue(), eval(makedate(DSL.literal(42.), DSL.literal(0.))), + "Zero dayOfYear doesn't produce NULL"); + + assertEquals(LocalDate.of(1999, 3, 1), makedate(1999., 60.), + "Got Feb 29th of a non-lear year"); + assertEquals(LocalDate.of(1999, 12, 31), makedate(1999., 365.)); + assertEquals(LocalDate.of(2004, 12, 31), makedate(2004., 366.)); + } + + @Test + public void checkRounding() { + assertEquals(LocalDate.of(42, 1, 1), makedate(42.49, 1.49)); + assertEquals(LocalDate.of(43, 1, 2), makedate(42.50, 1.50)); + } + + @Test + public void checkNullValues() { + when(nullRef.valueOf(env)).thenReturn(nullValue()); + + assertEquals(nullValue(), eval(makedate(nullRef, DSL.literal(42.)))); + assertEquals(nullValue(), eval(makedate(DSL.literal(42.), nullRef))); + assertEquals(nullValue(), eval(makedate(nullRef, nullRef))); + } + + @Test + public void checkMissingValues() { + when(missingRef.valueOf(env)).thenReturn(missingValue()); + + assertEquals(missingValue(), eval(makedate(missingRef, DSL.literal(42.)))); + assertEquals(missingValue(), eval(makedate(DSL.literal(42.), missingRef))); + assertEquals(missingValue(), eval(makedate(missingRef, missingRef))); + } + + private static Stream getTestData() { + return Stream.of( + Arguments.of(3755.421154, 9.300720), + Arguments.of(3416.922084, 850.832172), + Arguments.of(498.717527, 590.831215), + Arguments.of(1255.402786, 846.041171), + Arguments.of(2491.200868, 832.929840), + Arguments.of(1140.775582, 345.592629), + Arguments.of(2087.208382, 110.392189), + Arguments.of(4582.515870, 763.629197), + Arguments.of(1654.431245, 476.360251), + Arguments.of(1342.494306, 70.108352), + Arguments.of(171.841206, 794.470738), + Arguments.of(5000.103926, 441.461842), + Arguments.of(2957.828371, 273.909052), + Arguments.of(2232.699033, 171.537097), + Arguments.of(4650.163672, 226.857148), + Arguments.of(495.943520, 735.062451), + Arguments.of(4568.187019, 552.394124), + Arguments.of(688.085482, 283.574200), + Arguments.of(4627.662672, 791.729059), + Arguments.of(2812.837393, 397.688304), + Arguments.of(3050.030341, 596.714966), + Arguments.of(3617.452566, 619.795467), + Arguments.of(2210.322073, 106.914268), + Arguments.of(675.757974, 147.702828), + Arguments.of(1101.801820, 40.055318) + ); + } + + /** + * Test function with given pseudo-random values. + * @param year year + * @param dayOfYear day of year + */ + @ParameterizedTest(name = "year = {0}, dayOfYear = {1}") + @MethodSource("getTestData") + public void checkRandomValues(double year, double dayOfYear) { + LocalDate actual = makedate(year, dayOfYear); + LocalDate expected = getReferenceValue(year, dayOfYear); + + assertEquals(expected, actual, + String.format("year = %f, dayOfYear = %f", year, dayOfYear)); + } + + /** + * Using another algorithm to get reference value. + * We should go to the next year until remaining @dayOfYear is bigger than 365/366. + * @param year Year. + * @param dayOfYear Day of the year. + * @return The calculated date. + */ + private LocalDate getReferenceValue(double year, double dayOfYear) { + var yearL = (int)Math.round(year); + var dayL = (int)Math.round(dayOfYear); + while (true) { + int daysInYear = Year.isLeap(yearL) ? 366 : 365; + if (dayL > daysInYear) { + dayL -= daysInYear; + yearL++; + } else { + break; + } + } + return LocalDate.ofYearDay(yearL, dayL); + } + + private ExprValue eval(Expression expression) { + return expression.valueOf(env); + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/MakeTimeTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/MakeTimeTest.java new file mode 100644 index 00000000000..8269f74ccce --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/MakeTimeTest.java @@ -0,0 +1,172 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.expression.datetime; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.model.ExprValueUtils.missingValue; +import static org.opensearch.sql.data.model.ExprValueUtils.nullValue; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; + +import java.time.Duration; +import java.time.LocalTime; +import java.time.format.DateTimeParseException; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.ExpressionTestBase; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.config.ExpressionConfig; +import org.opensearch.sql.expression.env.Environment; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.expression.function.FunctionSignature; + +@ExtendWith(MockitoExtension.class) +public class MakeTimeTest extends ExpressionTestBase { + + @Mock + Environment env; + + @Mock + Expression nullRef; + + @Mock + Expression missingRef; + + private FunctionExpression maketime(Expression hour, Expression minute, Expression second) { + var repo = new ExpressionConfig().functionRepository(); + var func = repo.resolve(new FunctionSignature(new FunctionName("maketime"), + List.of(DOUBLE, DOUBLE, DOUBLE))); + return (FunctionExpression)func.apply(List.of(hour, minute, second)); + } + + private LocalTime maketime(Double hour, Double minute, Double second) { + return maketime(DSL.literal(hour), DSL.literal(minute), DSL.literal(second)) + .valueOf(null).timeValue(); + } + + @Test + public void checkEdgeCases() { + assertEquals(nullValue(), eval(maketime(DSL.literal(-1.), DSL.literal(42.), DSL.literal(42.))), + "Negative hour doesn't produce NULL"); + assertEquals(nullValue(), eval(maketime(DSL.literal(42.), DSL.literal(-1.), DSL.literal(42.))), + "Negative minute doesn't produce NULL"); + assertEquals(nullValue(), eval(maketime(DSL.literal(12.), DSL.literal(42.), DSL.literal(-1.))), + "Negative second doesn't produce NULL"); + + assertThrows(DateTimeParseException.class, + () -> eval(maketime(DSL.literal(24.), DSL.literal(42.), DSL.literal(42.)))); + assertThrows(DateTimeParseException.class, + () -> eval(maketime(DSL.literal(12.), DSL.literal(60.), DSL.literal(42.)))); + assertThrows(DateTimeParseException.class, + () -> eval(maketime(DSL.literal(12.), DSL.literal(42.), DSL.literal(60.)))); + + assertEquals(LocalTime.of(23, 59, 59), maketime(23., 59., 59.)); + assertEquals(LocalTime.of(0, 0, 0), maketime(0., 0., 0.)); + } + + @Test + public void checkRounding() { + assertEquals(LocalTime.of(0, 0, 0), maketime(0.49, 0.49, 0.)); + assertEquals(LocalTime.of(1, 1, 0), maketime(0.50, 0.50, 0.)); + } + + @Test + public void checkSecondFraction() { + assertEquals(LocalTime.of(0, 0, 0).withNano(999999999), maketime(0., 0., 0.999999999)); + assertEquals(LocalTime.of(0, 0, 0).withNano(100502000), maketime(0., 0., 0.100502)); + } + + @Test + public void checkNullValues() { + when(nullRef.valueOf(env)).thenReturn(nullValue()); + + assertEquals(nullValue(), eval(maketime(nullRef, DSL.literal(42.), DSL.literal(42.)))); + assertEquals(nullValue(), eval(maketime(DSL.literal(42.), nullRef, DSL.literal(42.)))); + assertEquals(nullValue(), eval(maketime(DSL.literal(42.), DSL.literal(42.), nullRef))); + assertEquals(nullValue(), eval(maketime(nullRef, nullRef, DSL.literal(42.)))); + assertEquals(nullValue(), eval(maketime(nullRef, DSL.literal(42.), nullRef))); + assertEquals(nullValue(), eval(maketime(nullRef, nullRef, nullRef))); + assertEquals(nullValue(), eval(maketime(DSL.literal(42.), nullRef, nullRef))); + } + + @Test + public void checkMissingValues() { + when(missingRef.valueOf(env)).thenReturn(missingValue()); + + assertEquals(missingValue(), eval(maketime(missingRef, DSL.literal(42.), DSL.literal(42.)))); + assertEquals(missingValue(), eval(maketime(DSL.literal(42.), missingRef, DSL.literal(42.)))); + assertEquals(missingValue(), eval(maketime(DSL.literal(42.), DSL.literal(42.), missingRef))); + assertEquals(missingValue(), eval(maketime(missingRef, missingRef, DSL.literal(42.)))); + assertEquals(missingValue(), eval(maketime(missingRef, DSL.literal(42.), missingRef))); + assertEquals(missingValue(), eval(maketime(missingRef, missingRef, missingRef))); + assertEquals(missingValue(), eval(maketime(DSL.literal(42.), missingRef, missingRef))); + } + + private static Stream getTestData() { + return Stream.of( + Arguments.of(20., 30., 40.), + Arguments.of(18.392650, 32.625996, 52.877904), + Arguments.of(20.115442, 7.393619, 27.006809), + Arguments.of(1.231453, 36.462770, 28.736317), + Arguments.of(3.586288, 13.180347, 22.665265), + Arguments.of(4.284613, 40.426888, 19.631883), + Arguments.of(14.843040, 44.682624, 53.484064), + Arguments.of(19.797981, 41.826666, 2.635713), + Arguments.of(4.194618, 10.934165, 32.019225), + Arguments.of(13.240491, 53.625706, 34.506773), + Arguments.of(7.606246, 27.344016, 30.117284), + Arguments.of(13.922934, 26.936002, 42.599373), + Arguments.of(23.114911, 37.764516, 7.677971), + Arguments.of(7.388466, 31.973471, 35.131596), + Arguments.of(19.777173, 44.926077, 24.613693), + Arguments.of(5.773249, 52.693275, 10.190731), + Arguments.of(17.812324, 36.549285, 4.620326), + Arguments.of(9.774054, 41.955251, 23.995705), + Arguments.of(19.619894, 54.933941, 48.788633), + Arguments.of(18.731704, 48.510363, 50.444896), + Arguments.of(10.345095, 27.593594, 23.083821), + Arguments.of(22.925545, 25.113236, 10.645589), + Arguments.of(7.494112, 9.761983, 17.444988), + Arguments.of(17.867756, 10.313120, 36.391815), + Arguments.of(19.712155, 3.197562, 6.607233), + Arguments.of(2.385090, 41.761568, 33.342590) + ); + } + + /** + * Test function with given pseudo-random values. + * @param hour hour + * @param minute minute + * @param second second + */ + @ParameterizedTest(name = "hour = {0}, minute = {1}, second = {2}") + @MethodSource("getTestData") + public void checkRandomValues(double hour, double minute, double second) { + // results could have 1 nanosec diff because of rounding FP + var expected = LocalTime.of((int)Math.round(hour), (int)Math.round(minute), + // pick fraction second part as nanos + (int)Math.floor(second)).withNano((int)((second % 1) * 1E9)); + var delta = Duration.between(expected, maketime(hour, minute, second)).getNano(); + assertEquals(0, delta, 1, + String.format("hour = %f, minute = %f, second = %f", hour, minute, second)); + } + + private ExprValue eval(Expression expression) { + return expression.valueOf(env); + } +} diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index f1d4d987a37..736b1f148ea 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -1219,15 +1219,70 @@ Example:: +---------------------------+ +MAKEDATE +-------- + +Description +>>>>>>>>>>> + +Returns a date, given `year` and `day-of-year` values. `dayofyear` must be greater than 0 or the result is `NULL`. The result is also `NULL` if either argument is `NULL`. +Arguments are rounded to an integer. + +Limitations: +- Zero `year` interpreted as 2000; +- Negative `year` is not accepted; +- `day-of-year` should be greater than zero; +- `day-of-year` could be greater than 365/366, calculation switches to the next year(s) (see example). + +Specifications: + +1. MAKEDATE(DOUBLE, DOUBLE) -> DATE + +Argument type: DOUBLE + +Return type: DATE + +Example:: + + os> select MAKEDATE(1945, 5.9), MAKEDATE(1984, 1984) + fetched rows / total rows = 1/1 + +-----------------------+------------------------+ + | MAKEDATE(1945, 5.9) | MAKEDATE(1984, 1984) | + |-----------------------+------------------------| + | 1945-01-06 | 1989-06-06 | + +-----------------------+------------------------+ + + MAKETIME -------- Description >>>>>>>>>>> +Returns a time value calculated from the hour, minute, and second arguments. Returns `NULL` if any of its arguments are `NULL`. +The second argument can have a fractional part, rest arguments are rounded to an integer. + +Limitations: +- 24-hour clock is used, available time range is [00:00:00.0 - 23:59:59.(9)]; +- Up to 9 digits of second fraction part is taken (nanosecond precision). + Specifications: -1. MAKETIME(INTEGER, INTEGER, INTEGER) -> DATE +1. MAKETIME(DOUBLE, DOUBLE, DOUBLE) -> TIME + +Argument type: DOUBLE + +Return type: TIME + +Example:: + + os> select MAKETIME(20, 30, 40), MAKETIME(20.2, 49.5, 42.100502) + fetched rows / total rows = 1/1 + +------------------------+-----------------------------------+ + | MAKETIME(20, 30, 40) | MAKETIME(20.2, 49.5, 42.100502) | + |------------------------+-----------------------------------| + | 20:30:40 | 20:50:42.100502 | + +------------------------+-----------------------------------+ MICROSECOND diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index 5be5686c34d..3680dc22720 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -388,15 +388,70 @@ Example:: +--------------------------+ +MAKEDATE +-------- + +Description +>>>>>>>>>>> + +Returns a date, given `year` and `day-of-year` values. `dayofyear` must be greater than 0 or the result is `NULL`. The result is also `NULL` if either argument is `NULL`. +Arguments are rounded to an integer. + +Limitations: +- Zero `year` interpreted as 2000; +- Negative `year` is not accepted; +- `day-of-year` should be greater than zero; +- `day-of-year` could be greater than 365/366, calculation switches to the next year(s) (see example). + +Specifications: + +1. MAKEDATE(DOUBLE, DOUBLE) -> DATE + +Argument type: DOUBLE + +Return type: DATE + +Example:: + + os> source=people | eval `MAKEDATE(1945, 5.9)` = MAKEDATE(1945, 5.9), `MAKEDATE(1984, 1984)` = MAKEDATE(1984, 1984) | fields `MAKEDATE(1945, 5.9)`, `MAKEDATE(1984, 1984)` + fetched rows / total rows = 1/1 + +-----------------------+------------------------+ + | MAKEDATE(1945, 5.9) | MAKEDATE(1984, 1984) | + |-----------------------+------------------------| + | 1945-01-06 | 1989-06-06 | + +-----------------------+------------------------+ + + MAKETIME -------- Description >>>>>>>>>>> +Returns a time value calculated from the hour, minute, and second arguments. Returns `NULL` if any of its arguments are `NULL`. +The second argument can have a fractional part, rest arguments are rounded to an integer. + +Limitations: +- 24-hour clock is used, available time range is [00:00:00.0 - 23:59:59.(9)]; +- Up to 9 digits of second fraction part is taken (nanosecond precision). + Specifications: -1. MAKETIME(INTEGER, INTEGER, INTEGER) -> DATE +1. MAKETIME(DOUBLE, DOUBLE, DOUBLE) -> TIME + +Argument type: DOUBLE + +Return type: TIME + +Example:: + + os> source=people | eval `MAKETIME(20, 30, 40)` = MAKETIME(20, 30, 40), `MAKETIME(20.2, 49.5, 42.100502)` = MAKETIME(20.2, 49.5, 42.100502) | fields `MAKETIME(20, 30, 40)`, `MAKETIME(20.2, 49.5, 42.100502)` + fetched rows / total rows = 1/1 + +------------------------+-----------------------------------+ + | MAKETIME(20, 30, 40) | MAKETIME(20.2, 49.5, 42.100502) | + |------------------------+-----------------------------------| + | 20:30:40 | 20:50:42.100502 | + +------------------------+-----------------------------------+ MICROSECOND diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java index fcbfc27710c..7e0169d174d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java @@ -13,6 +13,8 @@ import static org.opensearch.sql.util.MatcherUtils.verifySome; import java.io.IOException; +import java.time.LocalTime; + import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.common.utils.StringUtils; @@ -463,4 +465,19 @@ public void testDateFormatISO8601() throws IOException { verifyDateFormat(date, "date", dateFormat, dateFormatted); } + @Test + public void testMakeTime() throws IOException { + var result = executeQuery(String.format( + "source=%s | eval f1 = MAKETIME(20, 30, 40), f2 = MAKETIME(20.2, 49.5, 42.100502) | fields f1, f2", TEST_INDEX_DATE)); + verifySchema(result, schema("f1", null, "time"), schema("f2", null, "time")); + verifySome(result.getJSONArray("datarows"), rows("20:30:40", "20:50:42.100502")); + } + + @Test + public void testMakeDate() throws IOException { + var result = executeQuery(String.format( + "source=%s | eval f1 = MAKEDATE(1945, 5.9), f2 = MAKEDATE(1984, 1984) | fields f1, f2", TEST_INDEX_DATE)); + verifySchema(result, schema("f1", null, "date"), schema("f2", null, "date")); + verifySome(result.getJSONArray("datarows"), rows("1945-01-06", "1989-06-06")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index d19c3719b6d..7c6bd7efe2b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -7,11 +7,13 @@ package org.opensearch.sql.sql; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; +import static org.opensearch.sql.util.MatcherUtils.verifySome; import static org.opensearch.sql.util.TestUtils.getResponseBody; import java.io.IOException; @@ -452,6 +454,23 @@ public void testDateFormat() throws IOException { verifyDateFormat(date, "date", dateFormat, dateFormatted); } + + @Test + public void testMakeTime() throws IOException { + var result = executeQuery(String.format( + "select MAKETIME(20, 30, 40) as f1, MAKETIME(20.2, 49.5, 42.100502) as f2", TEST_INDEX_DATE)); + verifySchema(result, schema("MAKETIME(20, 30, 40)", "f1", "time"), schema("MAKETIME(20.2, 49.5, 42.100502)", "f2", "time")); + verifySome(result.getJSONArray("datarows"), rows("20:30:40", "20:50:42.100502")); + } + + @Test + public void testMakeDate() throws IOException { + var result = executeQuery(String.format( + "select MAKEDATE(1945, 5.9) as f1, MAKEDATE(1984, 1984) as f2", TEST_INDEX_DATE)); + verifySchema(result, schema("MAKEDATE(1945, 5.9)", "f1", "date"), schema("MAKEDATE(1984, 1984)", "f2", "date")); + verifySome(result.getJSONArray("datarows"), rows("1945-01-06", "1989-06-06")); + } + protected JSONObject executeQuery(String query) throws IOException { Request request = new Request("POST", QUERY_API_ENDPOINT); request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index e01d4337789..93df64d0b38 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -231,6 +231,8 @@ DAYOFWEEK: 'DAYOFWEEK'; DAYOFYEAR: 'DAYOFYEAR'; DAYNAME: 'DAYNAME'; FROM_DAYS: 'FROM_DAYS'; +MAKEDATE: 'MAKEDATE'; +MAKETIME: 'MAKETIME'; MONTHNAME: 'MONTHNAME'; SUBDATE: 'SUBDATE'; TIME: 'TIME'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 79d0b5f92e4..c83297459d6 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -375,7 +375,7 @@ trigonometricFunctionName dateAndTimeFunctionBase : ADDDATE | DATE | DATE_ADD | DATE_SUB | DAY | DAYNAME | DAYOFMONTH | DAYOFWEEK | DAYOFYEAR | FROM_DAYS | HOUR | MICROSECOND | MINUTE | MONTH | MONTHNAME | QUARTER | SECOND | SUBDATE | TIME | TIME_TO_SEC - | TIMESTAMP | TO_DAYS | YEAR | WEEK | DATE_FORMAT + | TIMESTAMP | TO_DAYS | YEAR | WEEK | DATE_FORMAT | MAKETIME | MAKEDATE ; /** condition function return boolean value */ diff --git a/sql/src/main/antlr/OpenSearchSQLLexer.g4 b/sql/src/main/antlr/OpenSearchSQLLexer.g4 index 9fca2942cff..6d2d7d8a64c 100644 --- a/sql/src/main/antlr/OpenSearchSQLLexer.g4 +++ b/sql/src/main/antlr/OpenSearchSQLLexer.g4 @@ -216,6 +216,7 @@ LOG10: 'LOG10'; LOG2: 'LOG2'; LOWER: 'LOWER'; LTRIM: 'LTRIM'; +MAKEDATE: 'MAKEDATE'; MAKETIME: 'MAKETIME'; MODULUS: 'MODULUS'; MONTHNAME: 'MONTHNAME'; diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index a9316b55a4d..40207df82a7 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -385,7 +385,7 @@ trigonometricFunctionName dateTimeFunctionName : ADDDATE | DATE | DATE_ADD | DATE_SUB | DAY | DAYNAME | DAYOFMONTH | DAYOFWEEK | DAYOFYEAR | FROM_DAYS | HOUR | MICROSECOND | MINUTE | MONTH | MONTHNAME | QUARTER | SECOND | SUBDATE | TIME | TIME_TO_SEC - | TIMESTAMP | TO_DAYS | YEAR | WEEK | DATE_FORMAT + | TIMESTAMP | TO_DAYS | YEAR | WEEK | DATE_FORMAT | MAKETIME | MAKEDATE ; textFunctionName From 23a4a88e1f8506dc1fd1c58e4f32ad2ac1299df6 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 16 Aug 2022 15:58:47 -0700 Subject: [PATCH 18/20] Upgrade yarn dependencies for workbench (#765) Signed-off-by: Joshua Li --- workbench/yarn.lock | 1021 +++++++++++++++++++++++-------------------- 1 file changed, 537 insertions(+), 484 deletions(-) diff --git a/workbench/yarn.lock b/workbench/yarn.lock index b9759fc669a..026ec307436 100644 --- a/workbench/yarn.lock +++ b/workbench/yarn.lock @@ -3,37 +3,42 @@ "@babel/code-frame@^7.0.0": - version "7.12.11" - resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.11.tgz#f4ad435aa263db935b8f10f2c552d23fb716a63f" - integrity sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw== + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.18.6.tgz#3b25d38c89600baa2dcc219edfa88a74eb2c427a" + integrity sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q== dependencies: - "@babel/highlight" "^7.10.4" + "@babel/highlight" "^7.18.6" -"@babel/helper-validator-identifier@^7.10.4": - version "7.12.11" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz#c9a1f021917dcb5ccf0d4e453e399022981fc9ed" - integrity sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw== +"@babel/helper-validator-identifier@^7.18.6": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.18.6.tgz#9c97e30d31b2b8c72a1d08984f2ca9b574d7a076" + integrity sha512-MmetCkz9ej86nJQV+sFCxoGGrUbU3q02kgLciwkrt9QqEB7cP39oKEY0PakknEO0Gu20SskMRi+AYZ3b1TpN9g== -"@babel/highlight@^7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.10.4.tgz#7d1bdfd65753538fabe6c38596cdb76d9ac60143" - integrity sha512-i6rgnR/YgPEQzZZnbTHHuZdlE8qyoBNalD6F+q4vAFlcMEcqmkoG+mPqJYJCo63qPf74+Y1UZsl3l6f7/RIkmA== +"@babel/highlight@^7.18.6": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.18.6.tgz#81158601e93e2563795adcbfbdf5d64be3f2ecdf" + integrity sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g== dependencies: - "@babel/helper-validator-identifier" "^7.10.4" + "@babel/helper-validator-identifier" "^7.18.6" chalk "^2.0.0" js-tokens "^4.0.0" -"@babel/runtime@^7.12.5": - version "7.14.6" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.14.6.tgz#535203bc0892efc7dec60bdc27b2ecf6e409062d" - integrity sha512-/PCB2uJ7oM44tz8YhC4Z/6PeOKXp4K588f+5M3clr1M4zbqztlo0XEfJ2LEzj/FgwfgGcIdl8n7YYjTCI0BYwg== +"@babel/runtime@^7.12.5", "@babel/runtime@^7.7.6": + version "7.18.9" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.18.9.tgz#b4fcfce55db3d2e5e080d2490f608a3b9f407f4a" + integrity sha512-lkqXDcvlFT5rvEjiu6+QYO+1GXrEHRo2LOtS7E4GtX5ESIZOgepqsZBVIj6Pv+a6zqsya9VCgiK1KAK4BvJDAw== dependencies: regenerator-runtime "^0.13.4" +"@colors/colors@1.5.0": + version "1.5.0" + resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" + integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ== + "@cypress/listr-verbose-renderer@^0.4.1": version "0.4.1" resolved "https://registry.yarnpkg.com/@cypress/listr-verbose-renderer/-/listr-verbose-renderer-0.4.1.tgz#a77492f4b11dcc7c446a34b3e28721afd33c642a" - integrity sha1-p3SS9LEdzHxEajSz4ochr9M8ZCo= + integrity sha512-EDiBsVPWC27DDLEJCo+dpl9ODHhdrwU57ccr9tspwCdG2ni0QVkf6LF0FGbhfujcjPxnXLIwsaks4sOrwrA4Qw== dependencies: chalk "^1.1.3" cli-cursor "^1.0.2" @@ -41,9 +46,9 @@ figures "^1.7.0" "@cypress/request@^2.88.5": - version "2.88.5" - resolved "https://registry.yarnpkg.com/@cypress/request/-/request-2.88.5.tgz#8d7ecd17b53a849cfd5ab06d5abe7d84976375d7" - integrity sha512-TzEC1XMi1hJkywWpRfD2clreTa/Z+lOrXDCxxBTBPEcY5azdPi56A6Xw+O4tWJnaJH3iIE7G5aDXZC6JgRZLcA== + version "2.88.10" + resolved "https://registry.yarnpkg.com/@cypress/request/-/request-2.88.10.tgz#b66d76b07f860d3a4b8d7a0604d020c662752cce" + integrity sha512-Zp7F+R93N0yZyG34GutyTNr+okam7s/Fzc1+i3kcqOP8vk6OuajuE9qZJ6Rs+10/1JFtXFYMdyarnU1rZuJesg== dependencies: aws-sign2 "~0.7.0" aws4 "^1.8.0" @@ -52,19 +57,17 @@ extend "~3.0.2" forever-agent "~0.6.1" form-data "~2.3.2" - har-validator "~5.1.3" - http-signature "~1.2.0" + http-signature "~1.3.6" is-typedarray "~1.0.0" isstream "~0.1.2" json-stringify-safe "~5.0.1" mime-types "~2.1.19" - oauth-sign "~0.9.0" performance-now "^2.1.0" qs "~6.5.2" safe-buffer "^5.1.2" tough-cookie "~2.5.0" tunnel-agent "^0.6.0" - uuid "^3.3.2" + uuid "^8.3.2" "@cypress/xvfb@^1.2.4": version "1.2.4" @@ -74,6 +77,18 @@ debug "^3.1.0" lodash.once "^4.1.1" +"@hapi/hoek@^9.0.0": + version "9.3.0" + resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.3.0.tgz#8368869dcb735be2e7f5cb7647de78e167a251fb" + integrity sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ== + +"@hapi/topo@^5.0.0": + version "5.1.0" + resolved "https://registry.yarnpkg.com/@hapi/topo/-/topo-5.1.0.tgz#dc448e332c6c6e37a4dc02fd84ba8d44b9afb012" + integrity sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg== + dependencies: + "@hapi/hoek" "^9.0.0" + "@jest/types@^24.9.0": version "24.9.0" resolved "https://registry.yarnpkg.com/@jest/types/-/types-24.9.0.tgz#63cb26cb7500d069e5a389441a7c6ab5e909fc59" @@ -101,10 +116,27 @@ dependencies: any-observable "^0.3.0" +"@sideway/address@^4.1.3": + version "4.1.4" + resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.4.tgz#03dccebc6ea47fdc226f7d3d1ad512955d4783f0" + integrity sha512-7vwq+rOHVWjyXxVlR76Agnvhy8I9rpzjosTESvmhNeXOXdZZB15Fl+TI9x1SiHZH5Jv2wTGduSxFDIaq0m3DUw== + dependencies: + "@hapi/hoek" "^9.0.0" + +"@sideway/formula@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@sideway/formula/-/formula-3.0.0.tgz#fe158aee32e6bd5de85044be615bc08478a0a13c" + integrity sha512-vHe7wZ4NOXVfkoRb8T5otiENVlT7a3IAiw7H5M2+GO+9CDgcVUUsX1zalAztCmwyOr2RUTGJdgB+ZvSVqmdHmg== + +"@sideway/pinpoint@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@sideway/pinpoint/-/pinpoint-2.0.0.tgz#cff8ffadc372ad29fd3f78277aeb29e632cc70df" + integrity sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ== + "@testing-library/user-event@^13.1.9": - version "13.1.9" - resolved "https://registry.yarnpkg.com/@testing-library/user-event/-/user-event-13.1.9.tgz#29e49a42659ac3c1023565ff56819e0153a82e99" - integrity sha512-NZr0zL2TMOs2qk+dNlqrAdbaRW5dAmYwd1yuQ4r7HpkVEOj0MWuUjDWwKhcLd/atdBy8ZSMHSKp+kXSQe47ezg== + version "13.5.0" + resolved "https://registry.yarnpkg.com/@testing-library/user-event/-/user-event-13.5.0.tgz#69d77007f1e124d55314a2b73fd204b333b13295" + integrity sha512-5Kwtbo3Y/NowpkbRuSepbyMFkZmHgD+vPzYB/RJ4oxt5Gj/avFFBYjhw27cqSVPVw/3a67NK1PbiIr9k4Gwmdg== dependencies: "@babel/runtime" "^7.12.5" @@ -114,14 +146,14 @@ integrity sha512-D3KB0PdaxdwtA44yOpK+NtptTscKWgUzXmf8fiLaaVxnX+b7QQ+dUMsyeVDweCQ6VX4PMwkd6x2hJ0X+ISIsoQ== "@types/boom@*": - version "7.3.0" - resolved "https://registry.yarnpkg.com/@types/boom/-/boom-7.3.0.tgz#33280c5552d4cfabc21b8b7e0f6d29292decd985" - integrity sha512-PH7bfkt1nu4pnlxz+Ws+wwJJF1HE12W3ia+Iace2JT7q56DLH3hbyjOJyNHJYRxk3PkKaC36fHfHKyeG1rMgCA== + version "7.3.2" + resolved "https://registry.yarnpkg.com/@types/boom/-/boom-7.3.2.tgz#6269480d11098c26cfbfca5059403902a530847a" + integrity sha512-W8+6CANZz/BmgzATboKXvEDtO3SHSj86omcU/2pMCkUQhqhgucB8sgcmdRadXDEvXFRiRhME0rcyj1GIj3LC9w== "@types/catbox@*": - version "10.0.6" - resolved "https://registry.yarnpkg.com/@types/catbox/-/catbox-10.0.6.tgz#8a4c91261cf0afca03351bb82a95b2d6cf43a5d0" - integrity sha512-qS0VHlL6eBUUoUeBnI/ASCffoniS62zdV6IUtLSIjGKmRhZNawotaOMsTYivZOTZVktfe9koAJkD9XFac7tEEg== + version "10.0.7" + resolved "https://registry.yarnpkg.com/@types/catbox/-/catbox-10.0.7.tgz#50a0f927924ace358cc0193c6d9f1790d4219fa6" + integrity sha512-8vjnM1eyHKwWfcbq7qb2NN4ltUgm4nVsT9QIM6IW4KqxbQbfQLbRApNTN+B3hvHvuFTHyvhJjZrBY3wNPp4tjA== "@types/hapi-latest@npm:@types/hapi@18.0.3": version "18.0.3" @@ -138,9 +170,11 @@ "@types/shot" "*" "@types/history@*": - version "4.7.8" - resolved "https://registry.yarnpkg.com/@types/history/-/history-4.7.8.tgz#49348387983075705fe8f4e02fb67f7daaec4934" - integrity sha512-S78QIYirQcUoo6UJZx9CSP0O2ix9IaeAXwQi26Rhr/+mg7qqPy8TzaxHSUut7eGjL8WmLccT7/MXf304WjqHcA== + version "5.0.0" + resolved "https://registry.yarnpkg.com/@types/history/-/history-5.0.0.tgz#29f919f0c8e302763798118f45b19cab4a886f14" + integrity sha512-hy8b7Y1J8OGe6LbAjj3xniQrj3v6lsivCcrmf4TzSgPzLkhIeKgc5IZnT7ReIqmEuodjfO8EYAuoFvIrHi/+jQ== + dependencies: + history "*" "@types/history@^4.7.11": version "4.7.11" @@ -148,9 +182,9 @@ integrity sha512-qjDJRrmvBMiTx+jyLxvLfJU7UznFuokDv4f3WRuriHKERccVpFU+8XMQUAbDzoiJCsmexxRExQeMwwCdamSKDA== "@types/iron@*": - version "5.0.1" - resolved "https://registry.yarnpkg.com/@types/iron/-/iron-5.0.1.tgz#5420bbda8623c48ee51b9a78ebad05d7305b4b24" - integrity sha512-Ng5BkVGPt7Tw9k1OJ6qYwuD9+dmnWgActmsnnrdvs4075N8V2go1f6Pz8omG3q5rbHjXN6yzzZDYo3JOgAE/Ug== + version "5.0.2" + resolved "https://registry.yarnpkg.com/@types/iron/-/iron-5.0.2.tgz#d64771b5c1d87d3c1a8a5bcaa3e02259069b8672" + integrity sha512-u1l5xO5BtLlPfeiy60vrC9Y30cR8gM7chD5npIAETZaOlGoT8KHqtQQpmlU5dZejhJSfIBSX0zBHnKW+0fw5qA== dependencies: "@types/node" "*" @@ -189,19 +223,21 @@ jest-diff "^24.3.0" "@types/joi@*": - version "14.3.4" - resolved "https://registry.yarnpkg.com/@types/joi/-/joi-14.3.4.tgz#eed1e14cbb07716079c814138831a520a725a1e0" - integrity sha512-1TQNDJvIKlgYXGNIABfgFp9y0FziDpuGrd799Q5RcnsDu+krD+eeW/0Fs5PHARvWWFelOhIG2OPCo6KbadBM4A== + version "17.2.3" + resolved "https://registry.yarnpkg.com/@types/joi/-/joi-17.2.3.tgz#b7768ed9d84f1ebd393328b9f97c1cf3d2b94798" + integrity sha512-dGjs/lhrWOa+eO0HwgxCSnDm5eMGCsXuvLglMghJq32F6q5LyyNuXb41DHzrg501CKNOSSAHmfB7FDGeUnDmzw== + dependencies: + joi "*" "@types/mime-db@*": - version "1.43.0" - resolved "https://registry.yarnpkg.com/@types/mime-db/-/mime-db-1.43.0.tgz#c91a4fcae0d6cfe19f2452b3b3e9fc2b95aa9a52" - integrity sha512-nCqnKWfvDzTVRh8NS9Zn+rnZYs5+iw5hx3G0nFgKM/R2mJQwPChin4DQ4zXlXnxTtc/3Ys5FnWweKW6NW+d2aw== + version "1.43.1" + resolved "https://registry.yarnpkg.com/@types/mime-db/-/mime-db-1.43.1.tgz#c2a0522453bb9b6e84ee48b7eef765d19bcd519e" + integrity sha512-kGZJY+R+WnR5Rk+RPHUMERtb2qBRViIHCBdtUrY+NmwuGb8pQdfTqQiCKPrxpdoycl8KWm2DLdkpoSdt479XoQ== "@types/mimos@*": - version "3.0.2" - resolved "https://registry.yarnpkg.com/@types/mimos/-/mimos-3.0.2.tgz#9595c66ead4e0d1fd022ff5f984851cdd5d5dc7b" - integrity sha512-UG3sdP/9HOk0oA1l8VylaZ0fjy6O/QGEeivOK9JhMjgJkcBJRdfsI6FtXFtg6UH17txmxuDOiIsTlrpBkayK0A== + version "3.0.3" + resolved "https://registry.yarnpkg.com/@types/mimos/-/mimos-3.0.3.tgz#c1a35a5c31ae99eb86e6607962280c05989f3e57" + integrity sha512-jtStUPuHpJ+LVK+e1K9SbD2V00IOAqAKgVVkBRmCdHG20VQaE3KOt7FincknY1uyyZNq7MKF44Gs0fon2v9oig== dependencies: "@types/mime-db" "*" @@ -216,14 +252,14 @@ integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== "@types/podium@*": - version "1.0.0" - resolved "https://registry.yarnpkg.com/@types/podium/-/podium-1.0.0.tgz#bfaa2151be2b1d6109cc69f7faa9dac2cba3bb20" - integrity sha1-v6ohUb4rHWEJzGn3+qnawsujuyA= + version "1.0.1" + resolved "https://registry.yarnpkg.com/@types/podium/-/podium-1.0.1.tgz#f51a419959584f2ffb8f849e5b33f2b99f7ee1f9" + integrity sha512-dcg8V5wKgrvNIBk2hzgK3gHobZ0u7pXW36NigfJq+lBA+/fqI6GAI0bdjLAEkZC67EsaVErtePkqipJPsszN7Q== "@types/react-dom@^16.0.5": - version "16.9.10" - resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-16.9.10.tgz#4485b0bec3d41f856181b717f45fd7831101156f" - integrity sha512-ItatOrnXDMAYpv6G8UCk2VhbYVTjZT9aorLtA/OzDN9XJ2GKcfam68jutoAcILdRjsRUO8qb7AmyObF77Q8QFw== + version "16.9.16" + resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-16.9.16.tgz#c591f2ed1c6f32e9759dfa6eb4abfd8041f29e39" + integrity sha512-Oqc0RY4fggGA3ltEgyPLc3IV9T73IGoWjkONbsyJ3ZBn+UPPCYpU2ec0i3cEbJuEdZtkqcCF2l1zf2pBdgUGSg== dependencies: "@types/react" "^16" @@ -246,11 +282,11 @@ "@types/react-router" "*" "@types/react-router@*": - version "5.1.8" - resolved "https://registry.yarnpkg.com/@types/react-router/-/react-router-5.1.8.tgz#4614e5ba7559657438e17766bb95ef6ed6acc3fa" - integrity sha512-HzOyJb+wFmyEhyfp4D4NYrumi+LQgQL/68HvJO+q6XtuHSDvw6Aqov7sCAhjbNq3bUPgPqbdvjXC5HeB2oEAPg== + version "5.1.18" + resolved "https://registry.yarnpkg.com/@types/react-router/-/react-router-5.1.18.tgz#c8851884b60bc23733500d86c1266e1cfbbd9ef3" + integrity sha512-YYknwy0D0iOwKQgz9v8nOzt2J6l4gouBmDnWqUUznltOTaon+r8US8ky8HvN0tXvc38U9m6z/t2RsVsnd1zM0g== dependencies: - "@types/history" "*" + "@types/history" "^4.7.11" "@types/react" "*" "@types/react@*", "@types/react@16.3.14", "@types/react@^16": @@ -261,26 +297,26 @@ csstype "^2.2.0" "@types/shot@*": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@types/shot/-/shot-4.0.0.tgz#7545500c489b65c69b5bc5446ba4fef3bd26af92" - integrity sha512-Xv+n8yfccuicMlwBY58K5PVVNtXRm7uDzcwwmCarBxMP+XxGfnh1BI06YiVAsPbTAzcnYsrzpoS5QHeyV7LS8A== + version "4.0.1" + resolved "https://registry.yarnpkg.com/@types/shot/-/shot-4.0.1.tgz#a50f56bde37db57184cdb43174ba4fea9c0c7897" + integrity sha512-LUJWxDKYWmmUpVXDWZ78hFIrqb/ho9RoKqBAFRJw4OHOS3qCKCoXCyzrfmf24lbO4COYfqL0iEWNArOfKEYW9g== dependencies: "@types/node" "*" "@types/sinonjs__fake-timers@^6.0.1": - version "6.0.2" - resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-6.0.2.tgz#3a84cf5ec3249439015e14049bd3161419bf9eae" - integrity sha512-dIPoZ3g5gcx9zZEszaxLSVTvMReD3xxyyDnQUjA6IYDG9Ba2AV0otMPs+77sG9ojB4Qr2N2Vk5RnKeuA0X/0bg== + version "6.0.4" + resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-6.0.4.tgz#0ecc1b9259b76598ef01942f547904ce61a6a77d" + integrity sha512-IFQTJARgMUBF+xVd2b+hIgXWrZEjND3vJtRCvIelcFB5SIXfjV4bOHbHJ0eXKh+0COrBRc8MqteKAz/j88rE0A== "@types/sizzle@^2.3.2": - version "2.3.2" - resolved "https://registry.yarnpkg.com/@types/sizzle/-/sizzle-2.3.2.tgz#a811b8c18e2babab7d542b3365887ae2e4d9de47" - integrity sha512-7EJYyKTL7tFR8+gDbB6Wwz/arpGa0Mywk1TJbNzKzHtzbwVmY4HR9WqS5VV7dsBUKQmPNr192jHr/VpBluj/hg== + version "2.3.3" + resolved "https://registry.yarnpkg.com/@types/sizzle/-/sizzle-2.3.3.tgz#ff5e2f1902969d305225a047c8a0fd5c915cebef" + integrity sha512-JYM8x9EGF163bEyhdJBpR2QX1R5naCJHC8ucJylJ3w9/CVBaskdQ8WqBf8MmQrd1kRvp/a4TS8HJ+bxzR7ZJYQ== "@types/yargs-parser@*": - version "20.2.1" - resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-20.2.1.tgz#3b9ce2489919d9e4fea439b76916abc34b2df129" - integrity sha512-7tFImggNeNBVMsn0vLrpn1H1uPrUBdnARPTpZoitY37ZrdJREzf7I16tMrlK3hen349gr1NYh8CmZQa7CTG6Aw== + version "21.0.0" + resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.0.tgz#0c60e537fa790f5f9472ed2776c2b71ec117351b" + integrity sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA== "@types/yargs@^13.0.0": version "13.0.12" @@ -297,9 +333,9 @@ "@types/yargs-parser" "*" acorn-jsx@^5.2.0: - version "5.3.1" - resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.1.tgz#fc8661e11b7ac1539c47dbfea2e72b3af34d267b" - integrity sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng== + version "5.3.2" + resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" + integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== acorn@^7.1.1: version "7.4.1" @@ -314,7 +350,7 @@ aggregate-error@^3.0.0: clean-stack "^2.0.0" indent-string "^4.0.0" -ajv@^6.10.0, ajv@^6.10.2, ajv@^6.12.3: +ajv@^6.10.0, ajv@^6.10.2: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== @@ -325,9 +361,9 @@ ajv@^6.10.0, ajv@^6.10.2, ajv@^6.12.3: uri-js "^4.2.2" ansi-colors@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.1.tgz#cbb9ae256bf750af1eab344f229aa27fe94ba348" - integrity sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA== + version "4.1.3" + resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.3.tgz#37611340eb2243e70cc604cad35d63270d48781b" + integrity sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw== ansi-escapes@^3.0.0: version "3.2.0" @@ -335,28 +371,28 @@ ansi-escapes@^3.0.0: integrity sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ== ansi-escapes@^4.2.1, ansi-escapes@^4.3.0: - version "4.3.1" - resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-4.3.1.tgz#a5c47cc43181f1f38ffd7076837700d395522a61" - integrity sha512-JWF7ocqNrp8u9oqpgV+wH5ftbt+cfvv+PTjOvKLT3AdYly/LmORARfEVT1iyjwN+4MqE5UmVKoAdIBqeoCHgLA== + version "4.3.2" + resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-4.3.2.tgz#6b2291d1db7d98b6521d5f1efa42d0f3a9feb65e" + integrity sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ== dependencies: - type-fest "^0.11.0" + type-fest "^0.21.3" ansi-regex@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" - integrity sha1-w7M6te42DYbg5ijwRorn7yfWVN8= + integrity sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA== ansi-regex@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-3.0.0.tgz#ed0317c322064f79466c02966bddb605ab37d998" - integrity sha1-7QMXwyIGT3lGbAKWa922Bas32Zg= + version "3.0.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-3.0.1.tgz#123d6479e92ad45ad897d4054e3c7ca7db4944e1" + integrity sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw== ansi-regex@^4.0.0, ansi-regex@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-4.1.0.tgz#8b9f8f08cf1acb843756a839ca8c7e3168c51997" - integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg== + version "4.1.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-4.1.1.tgz#164daac87ab2d6f6db3a29875e2d1766582dabed" + integrity sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g== -ansi-regex@^5.0.0: +ansi-regex@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== @@ -364,7 +400,7 @@ ansi-regex@^5.0.0: ansi-styles@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" - integrity sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4= + integrity sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA== ansi-styles@^3.2.0, ansi-styles@^3.2.1: version "3.2.1" @@ -403,16 +439,16 @@ argparse@^1.0.7: sprintf-js "~1.0.2" asn1@~0.2.3: - version "0.2.4" - resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136" - integrity sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg== + version "0.2.6" + resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.6.tgz#0d3a7bb6e64e02a90c0303b31f292868ea09a08d" + integrity sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ== dependencies: safer-buffer "~2.1.0" assert-plus@1.0.0, assert-plus@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" - integrity sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU= + integrity sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw== astral-regex@^1.0.0: version "1.0.0" @@ -425,14 +461,14 @@ astral-regex@^2.0.0: integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ== async@^3.2.0: - version "3.2.3" - resolved "https://registry.yarnpkg.com/async/-/async-3.2.3.tgz#ac53dafd3f4720ee9e8a160628f18ea91df196c9" - integrity sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g== + version "3.2.4" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" + integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" - integrity sha1-x57Zf380y48robyXkLzDZkdLS3k= + integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== at-least-node@^1.0.0: version "1.0.0" @@ -442,7 +478,7 @@ at-least-node@^1.0.0: aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" - integrity sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg= + integrity sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA== aws4@^1.8.0: version "1.11.0" @@ -450,14 +486,14 @@ aws4@^1.8.0: integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== balanced-match@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" - integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= + version "1.0.2" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" + integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== bcrypt-pbkdf@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" - integrity sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4= + integrity sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w== dependencies: tweetnacl "^0.14.3" @@ -482,9 +518,9 @@ brace-expansion@^1.1.7: brace@0.11.1: version "0.11.1" resolved "https://registry.yarnpkg.com/brace/-/brace-0.11.1.tgz#4896fcc9d544eef45f4bb7660db320d3b379fe58" - integrity sha1-SJb8ydVE7vRfS7dmDbMg07N5/lg= + integrity sha512-Fc8Ne62jJlKHiG/ajlonC4Sd66Pq68fFwK4ihJGNZpGqboc324SQk+lRvMzpPRuJOmfrJefdG8/7JdWX4bzJ2Q== -braces@^3.0.1: +braces@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== @@ -501,22 +537,17 @@ bs-logger@0.x: buffer-crc32@~0.2.3: version "0.2.13" resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" - integrity sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI= + integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ== -buffer-from@1.x: +buffer-from@1.x, buffer-from@^1.0.0: version "1.1.2" resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== -buffer-from@^1.0.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" - integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== - builtin-modules@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f" - integrity sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8= + integrity sha512-wxXCdllwGhI2kCC0MnvTGYTMvnVZTvqgypkiTI8Pa5tcz2i6VqsqwYGgqwXji+4RgCzms6EajE4IxiUH6HH8nQ== cachedir@^2.3.0: version "2.3.0" @@ -531,12 +562,12 @@ callsites@^3.0.0: caseless@~0.12.0: version "0.12.0" resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" - integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw= + integrity sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw== chalk@^1.0.0, chalk@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" - integrity sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg= + integrity sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A== dependencies: ansi-styles "^2.2.1" escape-string-regexp "^1.0.2" @@ -553,7 +584,7 @@ chalk@^2.0.0, chalk@^2.0.1, chalk@^2.1.0, chalk@^2.3.0, chalk@^2.4.1: escape-string-regexp "^1.0.5" supports-color "^5.3.0" -chalk@^4.0.0: +chalk@^4.0.0, chalk@^4.1.0: version "4.1.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== @@ -561,14 +592,6 @@ chalk@^4.0.0: ansi-styles "^4.1.0" supports-color "^7.1.0" -chalk@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.0.tgz#4e14870a618d9e2edd97dd8345fd9d9dc315646a" - integrity sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A== - dependencies: - ansi-styles "^4.1.0" - supports-color "^7.1.0" - chardet@^0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e" @@ -577,7 +600,7 @@ chardet@^0.7.0: check-more-types@^2.24.0: version "2.24.0" resolved "https://registry.yarnpkg.com/check-more-types/-/check-more-types-2.24.0.tgz#1420ffb10fd444dcfc79b43891bbfffd32a84600" - integrity sha1-FCD/sQ/URNz8ebQ4kbv//TKoRgA= + integrity sha512-Pj779qHxV2tuapviy1bSZNEL1maXr13bPYpsvSDB68HlYcYuhlDrmGd63i0JHMCLKzc7rUSNIrpdJlhVlNwrxA== ci-info@^2.0.0: version "2.0.0" @@ -592,14 +615,14 @@ clean-stack@^2.0.0: cli-cursor@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-1.0.2.tgz#64da3f7d56a54412e59794bd62dc35295e8f2987" - integrity sha1-ZNo/fValRBLll5S9Ytw1KV6PKYc= + integrity sha512-25tABq090YNKkF6JH7lcwO0zFJTRke4Jcq9iX2nr/Sz0Cjjv4gckmwlW6Ty/aoyFd6z3ysR2hMGC2GFugmBo6A== dependencies: restore-cursor "^1.0.1" cli-cursor@^2.0.0, cli-cursor@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-2.1.0.tgz#b35dac376479facc3e94747d41d0d0f5238ffcb5" - integrity sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU= + integrity sha512-8lgKz8LmCRYZZQDpRyT2m5rKJ08TnU4tR9FFFW2rxpxR1FzWi4PQ/NfyODchAatHaUgnSPVcx/R5w6NuTBzFiw== dependencies: restore-cursor "^2.0.0" @@ -611,19 +634,18 @@ cli-cursor@^3.1.0: restore-cursor "^3.1.0" cli-table3@~0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/cli-table3/-/cli-table3-0.6.0.tgz#b7b1bc65ca8e7b5cef9124e13dc2b21e2ce4faee" - integrity sha512-gnB85c3MGC7Nm9I/FkiasNBOKjOiO1RNuXXarQms37q4QMpWdlbBgD/VnOStA2faG1dpXMv31RFApjX1/QdgWQ== + version "0.6.2" + resolved "https://registry.yarnpkg.com/cli-table3/-/cli-table3-0.6.2.tgz#aaf5df9d8b5bf12634dc8b3040806a0c07120d2a" + integrity sha512-QyavHCaIC80cMivimWu4aWHilIpiDpfm3hGmqAmXVL1UsnbLuBSMd21hTX6VY4ZSDSM73ESLeF8TOYId3rBTbw== dependencies: - object-assign "^4.1.0" string-width "^4.2.0" optionalDependencies: - colors "^1.1.2" + "@colors/colors" "1.5.0" cli-truncate@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-0.2.1.tgz#9f15cfbb0705005369216c626ac7d05ab90dd574" - integrity sha1-nxXPuwcFAFNpIWxiasfQWrkN1XQ= + integrity sha512-f4r4yJnbT++qUPI9NR4XLDLq41gQ+uqnPItWG0F5ZkehuNiTTa3EY0S4AqTSUOeJ7/zU41oWPQSNkW5BqPL9bg== dependencies: slice-ansi "0.0.4" string-width "^1.0.1" @@ -644,7 +666,7 @@ cli-width@^3.0.0: code-point-at@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" - integrity sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c= + integrity sha512-RpAVKQA5T63xEj6/giIbUEtZwJ4UFIc3ZtvEkiaUERylqe8xb5IvqcgOurZLahv93CLKfxcw5YI+DZcUBRyLXA== color-convert@^1.9.0: version "1.9.3" @@ -663,17 +685,17 @@ color-convert@^2.0.1: color-name@1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" - integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= + integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== color-name@~1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== -colors@^1.1.2: - version "1.4.0" - resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" - integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA== +colorette@^2.0.16: + version "2.0.19" + resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.19.tgz#cdf044f47ad41a0f4b56b3a0d5b4e6e1a2d5a798" + integrity sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ== combined-stream@^1.0.6, combined-stream@~1.0.6: version "1.0.8" @@ -698,9 +720,9 @@ commander@^6.2.0: integrity sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA== common-tags@^1.8.0: - version "1.8.0" - resolved "https://registry.yarnpkg.com/common-tags/-/common-tags-1.8.0.tgz#8e3153e542d4a39e9b10554434afaaf98956a937" - integrity sha512-6P6g0uetGpW/sdyUy/iQQCbFF0kWVMSIVSyYz7Zgjcgh8mgw8PQzDNZeyZ5DQ2gM7LBoZPHmnjz8rUthkBG5tw== + version "1.8.2" + resolved "https://registry.yarnpkg.com/common-tags/-/common-tags-1.8.2.tgz#94ebb3c076d26032745fd54face7f688ef5ac9c6" + integrity sha512-gk/Z852D2Wtb//0I+kRFNKKE9dIIVirjoqPoA1wJU+XePVXZfGeBpk45+A1rKO4Q43prqWBNY/MiIeRLbPWUaA== compare-versions@^3.6.0: version "3.6.0" @@ -710,7 +732,7 @@ compare-versions@^3.6.0: concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" - integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= + integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== concat-stream@^1.6.2: version "1.6.2" @@ -722,15 +744,20 @@ concat-stream@^1.6.2: readable-stream "^2.2.2" typedarray "^0.0.6" -core-util-is@1.0.2, core-util-is@~1.0.0: +core-util-is@1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" - integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= + integrity sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ== + +core-util-is@~1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" + integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== cosmiconfig@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-7.0.0.tgz#ef9b44d773959cae63ddecd122de23853b60f8d3" - integrity sha512-pondGvTuVYDk++upghXJabWzL6Kxu6f26ljFw64Swq9v6sQPUL3EUlVDV56diOjpCayKihL6hVe8exIACU4XcA== + version "7.0.1" + resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-7.0.1.tgz#714d756522cace867867ccb4474c5d01bbae5d6d" + integrity sha512-a1YWNUV2HwGimB7dU2s1wUMurNKjpx60HxBB6xUM8Re+2s1g1IIfJvFR0/iCF+XHdE0GMTKTuLR32UQff4TEyQ== dependencies: "@types/parse-json" "^4.0.0" import-fresh "^3.2.1" @@ -759,9 +786,9 @@ cross-spawn@^7.0.0: which "^2.0.1" csstype@^2.2.0: - version "2.6.14" - resolved "https://registry.yarnpkg.com/csstype/-/csstype-2.6.14.tgz#004822a4050345b55ad4dcc00be1d9cf2f4296de" - integrity sha512-2mSc+VEpGPblzAxyeR+vZhJKgYg0Og0nnRi7pmRXFYYxSfnOnW8A5wwQb4n4cE2nIOzqKOAzLCaEX6aBmNEv8A== + version "2.6.20" + resolved "https://registry.yarnpkg.com/csstype/-/csstype-2.6.20.tgz#9229c65ea0b260cf4d3d997cb06288e36a8d6dda" + integrity sha512-/WwNkdXfckNgw6S5R125rrW8ez139lBHWouiBvX8dfMFtcn6V81REDqnH7+CRpRipfYlyU1CmOnOxrmGcFOjeA== cypress@^5.0.0: version "5.6.0" @@ -810,7 +837,7 @@ cypress@^5.0.0: dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" - integrity sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA= + integrity sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g== dependencies: assert-plus "^1.0.0" @@ -834,26 +861,26 @@ debug@^3.1.0: ms "^2.1.1" debug@^4.0.1, debug@^4.1.1, debug@^4.2.0: - version "4.3.1" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" - integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ== + version "4.3.4" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" + integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== dependencies: ms "2.1.2" dedent@^0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/dedent/-/dedent-0.7.0.tgz#2495ddbaf6eb874abb0e1be9df22d2e5a544326c" - integrity sha1-JJXduvbrh0q7Dhvp3yLS5aVEMmw= + integrity sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA== deep-is@~0.1.3: - version "0.1.3" - resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" - integrity sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ= + version "0.1.4" + resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" + integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ== delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" - integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= + integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== diff-sequences@^24.9.0: version "24.9.0" @@ -875,7 +902,7 @@ doctrine@^3.0.0: ecc-jsbn@~0.1.1: version "0.1.2" resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" - integrity sha1-OoOpBOVDUyh4dMVkt1SThoSamMk= + integrity sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw== dependencies: jsbn "~0.1.0" safer-buffer "^2.1.0" @@ -883,7 +910,7 @@ ecc-jsbn@~0.1.1: elegant-spinner@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/elegant-spinner/-/elegant-spinner-1.0.1.tgz#db043521c95d7e303fd8f345bedc3349cfb0729e" - integrity sha1-2wQ1IcldfjA/2PNFvtwzSc+wcp4= + integrity sha512-B+ZM+RXvRqQaAmkMlO/oSe5nMUOaUnyfGYCEHoR8wrXsZR2mA0XVibsxV1bvTwxdRWah1PkQqso2EzhILGHtEQ== emoji-regex@^7.0.1: version "7.0.3" @@ -919,17 +946,17 @@ error-ex@^1.3.1: escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" - integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ= + integrity sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg== eslint-plugin-no-unsanitized@^3.0.2: - version "3.1.4" - resolved "https://registry.yarnpkg.com/eslint-plugin-no-unsanitized/-/eslint-plugin-no-unsanitized-3.1.4.tgz#21d9f3cb5cbed91ac45cb30eec9195c387b918c7" - integrity sha512-WF1+eZo2Sh+bQNjZuVNwT0dA61zuJORsLh+1Sww7+O6GOPw+WPWIIRfTWNqrmaXaDMhM4SXAqYPcNlhRMiH13g== + version "3.2.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-no-unsanitized/-/eslint-plugin-no-unsanitized-3.2.0.tgz#a74239ae51363a5edbe6920eb34fe09aab925a9c" + integrity sha512-92opuXbjWmXcod94EyCKhp36V1QHLM/ArAST2ssgKOojALne0eZvSPfrg4oyr0EwTXvy0RJNe/Tkm33VkDUrKQ== eslint-plugin-prefer-object-spread@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/eslint-plugin-prefer-object-spread/-/eslint-plugin-prefer-object-spread-1.2.1.tgz#27fb91853690cceb3ae6101d9c8aecc6a67a402c" - integrity sha1-J/uRhTaQzOs65hAdnIrsxqZ6QCw= + integrity sha512-upA2W299LZNr49tGHmF0wUtFZw/fkQ+PkudsjZ0bP7dGFtRCFxItFtP39nDiet8u6rN/pOqhEYhBltW6T1OF5g== eslint-plugin-prettier@^2.2.0: version "2.7.0" @@ -1017,9 +1044,9 @@ esprima@^4.0.0: integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== esquery@^1.0.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.3.1.tgz#b78b5828aa8e214e29fb74c4d5b752e1c033da57" - integrity sha512-olpvt9QG0vniUBZspVRN6lwB7hOZoTRtT+jzR+tS4ffYx2mzbw+z0XCOk44aaLYKApNX5nMm+E+P6o25ip/DHQ== + version "1.4.0" + resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.4.0.tgz#2148ffc38b82e8c7057dfed48425b3e61f0f24a5" + integrity sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w== dependencies: estraverse "^5.1.0" @@ -1036,9 +1063,9 @@ estraverse@^4.1.1: integrity sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw== estraverse@^5.1.0, estraverse@^5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.2.0.tgz#307df42547e6cc7324d3cf03c155d5cdb8c53880" - integrity sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ== + version "5.3.0" + resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.3.0.tgz#2eea5290702f26ab8fe5370370ff86c965d21123" + integrity sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA== esutils@^2.0.2: version "2.0.3" @@ -1046,9 +1073,9 @@ esutils@^2.0.2: integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== eventemitter2@^6.4.2: - version "6.4.3" - resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-6.4.3.tgz#35c563619b13f3681e7eb05cbdaf50f56ba58820" - integrity sha512-t0A2msp6BzOf+QAcI6z9XMktLj52OjGQg+8SJH6v5+3uxNpWYRR3wQmfA+6xtMU9kOC59qk9licus5dYcrYkMQ== + version "6.4.7" + resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-6.4.7.tgz#a7f6c4d7abf28a14c1ef3442f21cb306a054271d" + integrity sha512-tYUSVOGeQPKt/eC1ABfhHy5Xd96N3oIijJvN3O9+TsC28T5V9yX9oEfEK5faP0EFSNVOG97qtAS68GBrQB2hDg== execa@^4.0.2, execa@^4.1.0: version "4.1.0" @@ -1075,7 +1102,7 @@ executable@^4.1.1: exit-hook@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/exit-hook/-/exit-hook-1.1.1.tgz#f05ca233b48c05d54fff07765df8507e95c02ff8" - integrity sha1-8FyiM7SMBdVP/wd2XfhQfpXAL/g= + integrity sha512-MsG3prOVw1WtLXAZbM3KiYtooKR1LvxHh3VHsVtIy0uiUu8usxgB/94DP2HxtD/661lLdB6yzQ09lGJSQr6nkg== extend@~3.0.2: version "3.0.2" @@ -1104,12 +1131,12 @@ extract-zip@^1.7.0: extsprintf@1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" - integrity sha1-lpGEQOMEGnpBT4xS48V06zw+HgU= + integrity sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g== extsprintf@^1.2.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f" - integrity sha1-4mifjzVvrWLMplo6kcXfX5VRaS8= + version "1.4.1" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.1.tgz#8d172c064867f235c0c84a596806d279bf4bcc07" + integrity sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA== fast-deep-equal@^3.1.1: version "3.1.3" @@ -1129,19 +1156,19 @@ fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0: fast-levenshtein@~2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" - integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc= + integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== fd-slicer@~1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e" - integrity sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4= + integrity sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g== dependencies: pend "~1.2.0" figures@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/figures/-/figures-1.7.0.tgz#cbe1e3affcf1cd44b80cadfed28dc793a9701d2e" - integrity sha1-y+Hjr/zxzUS4DK3+0o3Hk6lwHS4= + integrity sha512-UxKlfCRuCBxSXU4C6t9scbDyWZ4VlaFFdojKtzJuSkuOBQ5CNFum+zZXFwHjo+CxBC1t6zlYPgHIgFjL8ggoEQ== dependencies: escape-string-regexp "^1.0.5" object-assign "^4.1.0" @@ -1149,11 +1176,11 @@ figures@^1.7.0: figures@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/figures/-/figures-2.0.0.tgz#3ab1a2d2a62c8bfb431a0c94cb797a2fce27c962" - integrity sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI= + integrity sha512-Oa2M9atig69ZkfwiApY8F2Yy+tzMbazyvqv21R0NsSC8floSOC09BbT1ITWAdoMGQvJ/aZnR1KMwdx9tvHnTNA== dependencies: escape-string-regexp "^1.0.5" -figures@^3.0.0, figures@^3.2.0: +figures@^3.0.0: version "3.2.0" resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" integrity sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg== @@ -1174,20 +1201,20 @@ fill-range@^7.0.1: dependencies: to-regex-range "^5.0.1" -find-up@^4.0.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/find-up/-/find-up-4.1.0.tgz#97afe7d6cdc0bc5928584b7c8d7b16e8a9aa5d19" - integrity sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw== +find-up@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc" + integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== dependencies: - locate-path "^5.0.0" + locate-path "^6.0.0" path-exists "^4.0.0" -find-versions@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/find-versions/-/find-versions-3.2.0.tgz#10297f98030a786829681690545ef659ed1d254e" - integrity sha512-P8WRou2S+oe222TOCHitLy8zj+SIsVJh52VP4lvXkaFVnOFFdoWv1H1Jjvel1aI6NCFOAaeAVm8qrI0odiLcww== +find-versions@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/find-versions/-/find-versions-4.0.0.tgz#3c57e573bf97769b8cb8df16934b627915da4965" + integrity sha512-wgpWy002tA+wgmO27buH/9KzyEOQnKsG/R0yrcjPT9BOFm0zRBVQbZ95nRGXWMywS8YR5knRbpohio0bcJABxQ== dependencies: - semver-regex "^2.0.0" + semver-regex "^3.1.2" flat-cache@^2.0.1: version "2.0.1" @@ -1206,7 +1233,7 @@ flatted@^2.0.0: forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" - integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE= + integrity sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw== form-data@~2.3.2: version "2.3.3" @@ -1218,19 +1245,19 @@ form-data@~2.3.2: mime-types "^2.1.12" fs-extra@^9.0.1: - version "9.0.1" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.0.1.tgz#910da0062437ba4c39fedd863f1675ccfefcb9fc" - integrity sha512-h2iAoN838FqAFJY2/qVpzFXy+EBxfVE220PalAqQLDVsFOHLJrZvut5puAbCdNv6WJk+B8ihI+k0c7JK5erwqQ== + version "9.1.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d" + integrity sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ== dependencies: at-least-node "^1.0.0" graceful-fs "^4.2.0" jsonfile "^6.0.1" - universalify "^1.0.0" + universalify "^2.0.0" fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" - integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= + integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== function-bind@^1.1.1: version "1.1.1" @@ -1240,7 +1267,7 @@ function-bind@^1.1.1: functional-red-black-tree@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327" - integrity sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc= + integrity sha512-dsKNQNdj6xA3T+QlADDA7mOSlX0qiMINjn0cgr+eGHGsbSHzTabcIogz2+p/iqP1Xs6EP/sS2SbqH+brGTbq0g== get-own-enumerable-property-symbols@^3.0.0: version "3.0.2" @@ -1264,7 +1291,7 @@ getos@^3.2.1: getpass@^0.1.1: version "0.1.7" resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" - integrity sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo= + integrity sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng== dependencies: assert-plus "^1.0.0" @@ -1276,14 +1303,14 @@ glob-parent@^5.0.0: is-glob "^4.0.1" glob@^7.1.1, glob@^7.1.3: - version "7.1.6" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6" - integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA== + version "7.2.3" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" + integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== dependencies: fs.realpath "^1.0.0" inflight "^1.0.4" inherits "2" - minimatch "^3.0.4" + minimatch "^3.1.1" once "^1.3.0" path-is-absolute "^1.0.0" @@ -1301,40 +1328,22 @@ globals@^12.1.0: dependencies: type-fest "^0.8.1" -graceful-fs@^4.1.6, graceful-fs@^4.2.0: - version "4.2.4" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.4.tgz#2256bde14d3632958c465ebc96dc467ca07a29fb" - integrity sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw== - -graceful-fs@^4.2.4: - version "4.2.9" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.9.tgz#041b05df45755e587a24942279b9d113146e1c96" - integrity sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ== - -har-schema@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" - integrity sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI= - -har-validator@~5.1.3: - version "5.1.5" - resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.1.5.tgz#1f0803b9f8cb20c0fa13822df1ecddb36bde1efd" - integrity sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w== - dependencies: - ajv "^6.12.3" - har-schema "^2.0.0" +graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.4: + version "4.2.10" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" + integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== has-ansi@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" - integrity sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE= + integrity sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg== dependencies: ansi-regex "^2.0.0" has-flag@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" - integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0= + integrity sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw== has-flag@^4.0.0: version "4.0.0" @@ -1348,14 +1357,21 @@ has@^1.0.3: dependencies: function-bind "^1.1.1" -http-signature@~1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" - integrity sha1-muzZJRFHcvPZW2WmCruPfBj7rOE= +history@*: + version "5.3.0" + resolved "https://registry.yarnpkg.com/history/-/history-5.3.0.tgz#1548abaa245ba47992f063a0783db91ef201c73b" + integrity sha512-ZqaKwjjrAYUYfLG+htGaIIZ4nioX2L70ZUMIFysS3xvBsSG4x/n1V6TXV3N8ZYNuFGlDirFg32T7B6WOUPDYcQ== + dependencies: + "@babel/runtime" "^7.7.6" + +http-signature@~1.3.6: + version "1.3.6" + resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.3.6.tgz#cb6fbfdf86d1c974f343be94e87f7fc128662cf9" + integrity sha512-3adrsD6zqo4GsTqtO7FyrejHNv+NgiIfAfv68+jVlFmSr9OGy7zrxONceFRLKvnnZA5jbxQBX1u9PpB6Wi32Gw== dependencies: assert-plus "^1.0.0" - jsprim "^1.2.2" - sshpk "^1.7.0" + jsprim "^2.0.2" + sshpk "^1.14.1" human-signals@^1.1.1: version "1.1.1" @@ -1363,17 +1379,17 @@ human-signals@^1.1.1: integrity sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw== husky@^4.2.5: - version "4.3.6" - resolved "https://registry.yarnpkg.com/husky/-/husky-4.3.6.tgz#ebd9dd8b9324aa851f1587318db4cccb7665a13c" - integrity sha512-o6UjVI8xtlWRL5395iWq9LKDyp/9TE7XMOTvIpEVzW638UcGxTmV5cfel6fsk/jbZSTlvfGVJf2svFtybcIZag== + version "4.3.8" + resolved "https://registry.yarnpkg.com/husky/-/husky-4.3.8.tgz#31144060be963fd6850e5cc8f019a1dfe194296d" + integrity sha512-LCqqsB0PzJQ/AlCgfrfzRe3e3+NvmefAdKQhRYpxS4u6clblBoDdzzvHi8fmxKRzvMxPY/1WZWzomPZww0Anow== dependencies: chalk "^4.0.0" ci-info "^2.0.0" compare-versions "^3.6.0" cosmiconfig "^7.0.0" - find-versions "^3.2.0" + find-versions "^4.0.0" opencollective-postinstall "^2.0.2" - pkg-dir "^4.2.0" + pkg-dir "^5.0.0" please-upgrade-node "^3.2.0" slash "^3.0.0" which-pm-runs "^1.0.0" @@ -1391,9 +1407,9 @@ ignore@^4.0.6: integrity sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg== import-fresh@^3.0.0, import-fresh@^3.2.1: - version "3.2.2" - resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.2.2.tgz#fc129c160c5d68235507f4331a6baad186bdbc3e" - integrity sha512-cTPNrlvJT6twpYy+YmKUKrTSjWFs3bjYjAhCwm+z4EOCubZxAuO+hHpRN64TqjEaYSHs7tJAE0w1CKMGmsG/lw== + version "3.3.0" + resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b" + integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw== dependencies: parent-module "^1.0.0" resolve-from "^4.0.0" @@ -1401,12 +1417,12 @@ import-fresh@^3.0.0, import-fresh@^3.2.1: imurmurhash@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" - integrity sha1-khi5srkoojixPcT7a21XbyMUU+o= + integrity sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA== indent-string@^3.0.0: version "3.2.0" resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-3.2.0.tgz#4a5fd6d27cc332f37e5419a504dbb837105c9289" - integrity sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok= + integrity sha512-BYqTHXTGUIvg7t1r4sJNKcbDZkL92nkXA8YtRpbjFHRHGDL/NtUeiBJMeE60kIFN/Mg8ESaWQvftaYMGJzQZCQ== indent-string@^4.0.0: version "4.0.0" @@ -1416,7 +1432,7 @@ indent-string@^4.0.0: inflight@^1.0.4: version "1.0.6" resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" - integrity sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk= + integrity sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA== dependencies: once "^1.3.0" wrappy "1" @@ -1453,7 +1469,7 @@ inquirer@^7.0.0: is-arrayish@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" - integrity sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0= + integrity sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg== is-ci@^2.0.0: version "2.0.0" @@ -1462,29 +1478,29 @@ is-ci@^2.0.0: dependencies: ci-info "^2.0.0" -is-core-module@^2.1.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.2.0.tgz#97037ef3d52224d85163f5597b2b63d9afed981a" - integrity sha512-XRAfAdyyY5F5cOXn7hYQDqh2Xmii+DEfIcQGxK/uNwMHhIkPWO0g8msXcbzLe+MpGoR951MlqM/2iIlU4vKDdQ== +is-core-module@^2.9.0: + version "2.10.0" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.10.0.tgz#9012ede0a91c69587e647514e1d5277019e728ed" + integrity sha512-Erxj2n/LDAZ7H8WNJXd9tw38GYM3dv8rk8Zcs+jJuxYTW7sozH+SS8NtrSjVL1/vpLvWi1hxy96IzjJ3EHTJJg== dependencies: has "^1.0.3" is-extglob@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" - integrity sha1-qIwCU1eR8C7TfHahueqXc8gz+MI= + integrity sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ== is-fullwidth-code-point@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb" - integrity sha1-754xOG8DGn8NZDr4L95QxFfvAMs= + integrity sha512-1pqUqRjkhPJ9miNq9SwMfdvi6lBJcd6eFxvfaivQhaH3SgisfiuudvFntdKOmxuee/77l+FPjKrQjWvmPjWrRw== dependencies: number-is-nan "^1.0.0" is-fullwidth-code-point@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz#a3b30a5c4f199183167aaab93beefae3ddfb654f" - integrity sha1-o7MKXE8ZkYMWeqq5O+764937ZU8= + integrity sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w== is-fullwidth-code-point@^3.0.0: version "3.0.0" @@ -1492,9 +1508,9 @@ is-fullwidth-code-point@^3.0.0: integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== is-glob@^4.0.0, is-glob@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.1.tgz#7567dbe9f2f5e2467bc77ab83c4a29482407a5dc" - integrity sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg== + version "4.0.3" + resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084" + integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg== dependencies: is-extglob "^2.1.1" @@ -1514,7 +1530,7 @@ is-number@^7.0.0: is-obj@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/is-obj/-/is-obj-1.0.1.tgz#3e4729ac1f5fde025cd7d83a896dab9f4f67db0f" - integrity sha1-PkcprB9f3gJc19g6iW2rn09n2w8= + integrity sha512-l4RyHgRqGN4Y3+9JHVrNqO+tN0rV5My76uW5/nuO4K1b6vw5G8d/cmFjP9tRfEsdhZNt0IFdZuK/c2Vr4Nb+Qg== is-observable@^1.1.0: version "1.1.0" @@ -1524,9 +1540,9 @@ is-observable@^1.1.0: symbol-observable "^1.1.0" is-path-inside@^3.0.1: - version "3.0.2" - resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.2.tgz#f5220fc82a3e233757291dddc9c5877f2a1f3017" - integrity sha512-/2UGPSgmtqwo1ktx8NDHjuPwZWmHhO+gj0f93EkhLB5RgW9RZevWYYlIkS6zePc6U2WpOdQYIwHe9YC4DWEBVg== + version "3.0.3" + resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" + integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== is-promise@^2.1.0: version "2.2.2" @@ -1536,37 +1552,42 @@ is-promise@^2.1.0: is-regexp@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-regexp/-/is-regexp-1.0.0.tgz#fd2d883545c46bac5a633e7b9a09e87fa2cb5069" - integrity sha1-/S2INUXEa6xaYz57mgnof6LLUGk= + integrity sha512-7zjFAPO4/gwyQAAgRRmqeEeyIICSdmCqa3tsVHMdBzaXXRiqopZL4Cyghg/XulGWrtABTpbnYYzzIRffLkP4oA== is-stream@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" - integrity sha1-EtSj3U5o4Lec6428hBc66A2RykQ= + integrity sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ== is-stream@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.0.tgz#bde9c32680d6fae04129d6ac9d921ce7815f78e3" - integrity sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw== + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" + integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== is-typedarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" - integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= + integrity sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA== + +is-unicode-supported@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz#3f26c76a809593b52bfa2ecb5710ed2779b522a7" + integrity sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw== isarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" - integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= + integrity sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ== isexe@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" - integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA= + integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw== isstream@~0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" - integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo= + integrity sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g== jest-diff@^24.3.0: version "24.9.0" @@ -1591,7 +1612,7 @@ jest-get-type@^24.9.0: jest-raw-loader@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/jest-raw-loader/-/jest-raw-loader-1.0.1.tgz#ce9f56d54650f157c4a7d16d224ba5d613bcd626" - integrity sha1-zp9W1UZQ8VfEp9FtIkul1hO81iY= + integrity sha512-g9oaAjeC4/rIJk1Wd3RxVbOfMizowM7LSjEJqa4R9qDX0OjQNABXOhH+GaznUp+DjTGVPi2vPPbQXyX87DOnYg== jest-util@^26.1.0: version "26.6.2" @@ -1605,6 +1626,17 @@ jest-util@^26.1.0: is-ci "^2.0.0" micromatch "^4.0.2" +joi@*: + version "17.6.0" + resolved "https://registry.yarnpkg.com/joi/-/joi-17.6.0.tgz#0bb54f2f006c09a96e75ce687957bd04290054b2" + integrity sha512-OX5dG6DTbcr/kbMFj0KGYxuew69HPcAE3K/sZpEV2nP6e/j/C0HV+HNiBPCASxdx5T7DMoa0s8UeHWMnb6n2zw== + dependencies: + "@hapi/hoek" "^9.0.0" + "@hapi/topo" "^5.0.0" + "@sideway/address" "^4.1.3" + "@sideway/formula" "^3.0.0" + "@sideway/pinpoint" "^2.0.0" + js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" @@ -1621,7 +1653,7 @@ js-yaml@^3.13.1: jsbn@~0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" - integrity sha1-peZUwuWi3rXyAdls77yoDA7y9RM= + integrity sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg== json-parse-even-better-errors@^2.3.0: version "2.3.1" @@ -1633,7 +1665,7 @@ json-schema-traverse@^0.4.1: resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== -json-schema@0.2.3, json-schema@^0.4.0: +json-schema@0.4.0, json-schema@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.4.0.tgz#f7de4cf6efab838ebaeb3236474cbba5a1930ab5" integrity sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA== @@ -1641,19 +1673,17 @@ json-schema@0.2.3, json-schema@^0.4.0: json-stable-stringify-without-jsonify@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" - integrity sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE= + integrity sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw== json-stringify-safe@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" - integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus= + integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA== json5@2.x: - version "2.2.0" - resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.0.tgz#2dfefe720c6ba525d9ebd909950f0515316c89a3" - integrity sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA== - dependencies: - minimist "^1.2.5" + version "2.2.1" + resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.1.tgz#655d50ed1e6f95ad1a3caababd2b0efda10b395c" + integrity sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA== jsonfile@^6.0.1: version "6.1.0" @@ -1664,38 +1694,38 @@ jsonfile@^6.0.1: optionalDependencies: graceful-fs "^4.1.6" -jsprim@^1.2.2: - version "1.4.1" - resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" - integrity sha1-MT5mvB5cwG5Di8G3SZwuXFastqI= +jsprim@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-2.0.2.tgz#77ca23dbcd4135cd364800d22ff82c2185803d4d" + integrity sha512-gqXddjPqQ6G40VdnI6T6yObEC+pDNvyP95wdQhkWkg7crHH3km5qP1FsOXEkzEQwnz6gz5qGTn1c2Y52wP3OyQ== dependencies: assert-plus "1.0.0" extsprintf "1.3.0" - json-schema "0.2.3" + json-schema "0.4.0" verror "1.10.0" lazy-ass@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/lazy-ass/-/lazy-ass-1.6.0.tgz#7999655e8646c17f089fdd187d150d3324d54513" - integrity sha1-eZllXoZGwX8In90YfRUNMyTVRRM= + integrity sha512-cc8oEVoctTvsFZ/Oje/kGnHbpWHYBe8IAJe4C0QNc3t8uM/0Y8+erSz/7Y1ALuXTEZTMvxXwO6YbX1ey3ujiZw== levn@^0.3.0, levn@~0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/levn/-/levn-0.3.0.tgz#3b09924edf9f083c0490fdd4c0bc4421e04764ee" - integrity sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4= + integrity sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA== dependencies: prelude-ls "~1.1.2" type-check "~0.3.2" lines-and-columns@^1.1.6: - version "1.1.6" - resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00" - integrity sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA= + version "1.2.4" + resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" + integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg== lint-staged@^10.2.0: - version "10.5.3" - resolved "https://registry.yarnpkg.com/lint-staged/-/lint-staged-10.5.3.tgz#c682838b3eadd4c864d1022da05daa0912fb1da5" - integrity sha512-TanwFfuqUBLufxCc3RUtFEkFraSPNR3WzWcGF39R3f2J7S9+iF9W0KTVLfSy09lYGmZS5NDCxjNvhGMSJyFCWg== + version "10.5.4" + resolved "https://registry.yarnpkg.com/lint-staged/-/lint-staged-10.5.4.tgz#cd153b5f0987d2371fc1d2847a409a2fe705b665" + integrity sha512-EechC3DdFic/TdOPgj/RB3FicqE6932LTHCUm0Y2fsD9KGlLB+RwJl2q1IYBIvEsKzDOgn0D4gll+YxG5RsrKg== dependencies: chalk "^4.1.0" cli-truncate "^2.1.0" @@ -1716,7 +1746,7 @@ lint-staged@^10.2.0: listr-silent-renderer@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/listr-silent-renderer/-/listr-silent-renderer-1.1.1.tgz#924b5a3757153770bf1a8e3fbf74b8bbf3f9242e" - integrity sha1-kktaN1cVN3C/Go4/v3S4u/P5JC4= + integrity sha512-L26cIFm7/oZeSNVhWB6faeorXhMg4HNlb/dS/7jHhr708jxlXrtrBWo4YUxZQkc6dGoxEAe6J/D3juTRBUzjtA== listr-update-renderer@^0.5.0: version "0.5.0" @@ -1743,18 +1773,18 @@ listr-verbose-renderer@^0.5.0: figures "^2.0.0" listr2@^3.2.2: - version "3.2.3" - resolved "https://registry.yarnpkg.com/listr2/-/listr2-3.2.3.tgz#ef9e0d790862f038dde8a9837be552b1adfd1c07" - integrity sha512-vUb80S2dSUi8YxXahO8/I/s29GqnOL8ozgHVLjfWQXa03BNEeS1TpBLjh2ruaqq5ufx46BRGvfymdBSuoXET5w== + version "3.14.0" + resolved "https://registry.yarnpkg.com/listr2/-/listr2-3.14.0.tgz#23101cc62e1375fd5836b248276d1d2b51fdbe9e" + integrity sha512-TyWI8G99GX9GjE54cJ+RrNMcIFBfwMPxc3XTFiAYGN4s10hWROGtOg7+O6u6LE3mNkyld7RSLE6nrKBvTfcs3g== dependencies: - chalk "^4.1.0" cli-truncate "^2.1.0" - figures "^3.2.0" - indent-string "^4.0.0" + colorette "^2.0.16" log-update "^4.0.0" p-map "^4.0.0" - rxjs "^6.6.3" + rfdc "^1.3.0" + rxjs "^7.5.1" through "^2.3.8" + wrap-ansi "^7.0.0" listr@^0.14.3: version "0.14.3" @@ -1771,17 +1801,17 @@ listr@^0.14.3: p-map "^2.0.0" rxjs "^6.3.3" -locate-path@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-5.0.0.tgz#1afba396afd676a6d42504d0a67a3a7eb9f62aa0" - integrity sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g== +locate-path@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286" + integrity sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw== dependencies: - p-locate "^4.1.0" + p-locate "^5.0.0" lodash.once@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac" - integrity sha1-DdOXEhPHxW34gJd9UEyI+0cal6w= + integrity sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg== lodash@4.x, lodash@^4.17.14, lodash@^4.17.19: version "4.17.21" @@ -1791,21 +1821,22 @@ lodash@4.x, lodash@^4.17.14, lodash@^4.17.19: log-symbols@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-1.0.2.tgz#376ff7b58ea3086a0f09facc74617eca501e1a18" - integrity sha1-N2/3tY6jCGoPCfrMdGF+ylAeGhg= + integrity sha512-mmPrW0Fh2fxOzdBbFv4g1m6pR72haFLPJ2G5SJEELf1y+iaQrDG6cWCPjy54RHYbZAt7X+ls690Kw62AdWXBzQ== dependencies: chalk "^1.0.0" log-symbols@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-4.0.0.tgz#69b3cc46d20f448eccdb75ea1fa733d9e821c920" - integrity sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA== + version "4.1.0" + resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-4.1.0.tgz#3fbdbb95b4683ac9fc785111e792e558d4abd503" + integrity sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg== dependencies: - chalk "^4.0.0" + chalk "^4.1.0" + is-unicode-supported "^0.1.0" log-update@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/log-update/-/log-update-2.3.0.tgz#88328fd7d1ce7938b29283746f0b1bc126b24708" - integrity sha1-iDKP19HOeTiykoN0bwsbwSayRwg= + integrity sha512-vlP11XfFGyeNQlmEn9tJ66rEW1coA/79m5z6BCkudjbAGE83uhAcGYrBFwfs3AdLiLzGRusRPAbSPK9xZteCmg== dependencies: ansi-escapes "^3.0.0" cli-cursor "^2.0.0" @@ -1839,24 +1870,24 @@ merge-stream@^2.0.0: integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w== micromatch@^4.0.2: - version "4.0.4" - resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.4.tgz#896d519dfe9db25fce94ceb7a500919bf881ebf9" - integrity sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg== + version "4.0.5" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6" + integrity sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA== dependencies: - braces "^3.0.1" - picomatch "^2.2.3" + braces "^3.0.2" + picomatch "^2.3.1" -mime-db@1.44.0: - version "1.44.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92" - integrity sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg== +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== mime-types@^2.1.12, mime-types@~2.1.19: - version "2.1.27" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f" - integrity sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w== + version "2.1.35" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== dependencies: - mime-db "1.44.0" + mime-db "1.52.0" mimic-fn@^1.0.0: version "1.2.0" @@ -1868,14 +1899,14 @@ mimic-fn@^2.1.0: resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== -minimatch@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" - integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== +minimatch@^3.0.4, minimatch@^3.1.1: + version "3.1.2" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" + integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== dependencies: brace-expansion "^1.1.7" -minimist@^1.2.5: +minimist@^1.2.5, minimist@^1.2.6: version "1.2.6" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q== @@ -1886,11 +1917,11 @@ mkdirp@1.x: integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@^0.5.4: - version "0.5.5" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" - integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ== + version "0.5.6" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.6.tgz#7def03d2432dcae4ba1d611445c48396062255f6" + integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw== dependencies: - minimist "^1.2.5" + minimist "^1.2.6" moment@^2.27.0: version "2.29.4" @@ -1900,7 +1931,7 @@ moment@^2.27.0: ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" - integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= + integrity sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A== ms@2.1.2: version "2.1.2" @@ -1925,7 +1956,7 @@ mute-stream@0.0.8: natural-compare@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" - integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= + integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw== nice-try@^1.0.4: version "1.0.5" @@ -1947,34 +1978,29 @@ npm-run-path@^4.0.0: number-is-nan@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" - integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0= - -oauth-sign@~0.9.0: - version "0.9.0" - resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" - integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== + integrity sha512-4jbtZXNAsfZbAHiiqjLPBiCl16dES1zI4Hpzzxw61Tk+loF+sBDBKx1ICKKKwIqQ7M0mFn1TmkN7euSncWgHiQ== object-assign@^4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" - integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM= + integrity sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg== once@^1.3.0, once@^1.3.1, once@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" - integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= + integrity sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w== dependencies: wrappy "1" onetime@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/onetime/-/onetime-1.1.0.tgz#a1f7838f8314c516f05ecefcbc4ccfe04b4ed789" - integrity sha1-ofeDj4MUxRbwXs78vEzP4EtO14k= + integrity sha512-GZ+g4jayMqzCRMgB2sol7GiCLjKfS1PINkjmx8spcKce1LiVqcbQreXwqs2YAFXC6R03VIG28ZS31t8M866v6A== onetime@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/onetime/-/onetime-2.0.1.tgz#067428230fd67443b2794b22bba528b6867962d4" - integrity sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ= + integrity sha512-oyyPpiMaKARvvcgip+JV+7zci5L8D1W9RZIz2l1o08AM3pfspitVWnPt3mzHcBPp12oYMTy0pqrFs/C+m3EwsQ== dependencies: mimic-fn "^1.0.0" @@ -2005,26 +2031,26 @@ optionator@^0.8.3: os-tmpdir@~1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" - integrity sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ= + integrity sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g== ospath@^1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/ospath/-/ospath-1.2.2.tgz#1276639774a3f8ef2572f7fe4280e0ea4550c07b" - integrity sha1-EnZjl3Sj+O8lcvf+QoDg6kVQwHs= + integrity sha512-o6E5qJV5zkAbIDNhGSIlyOhScKXgQrSRMilfph0clDfM0nEnBOlKlH4sWDmG95BW/CvwNz0vmm7dJVtU2KlMiA== -p-limit@^2.2.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1" - integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w== +p-limit@^3.0.2: + version "3.1.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" + integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== dependencies: - p-try "^2.0.0" + yocto-queue "^0.1.0" -p-locate@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-4.1.0.tgz#a3428bb7088b3a60292f66919278b7c297ad4f07" - integrity sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A== +p-locate@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834" + integrity sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw== dependencies: - p-limit "^2.2.0" + p-limit "^3.0.2" p-map@^2.0.0: version "2.1.0" @@ -2038,11 +2064,6 @@ p-map@^4.0.0: dependencies: aggregate-error "^3.0.0" -p-try@^2.0.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" - integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== - parent-module@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" @@ -2051,9 +2072,9 @@ parent-module@^1.0.0: callsites "^3.0.0" parse-json@^5.0.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-5.1.0.tgz#f96088cdf24a8faa9aea9a009f2d9d942c999646" - integrity sha512-+mi/lmVVNKFNVyLXV31ERiy2CY5E1/F6QtJFEzoChPRwwngMNXRDQ9GJ5WdE2Z2P4AujsOi0/+2qHID68KwfIQ== + version "5.2.0" + resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-5.2.0.tgz#c76fc66dee54231c962b22bcc8a72cf2f99753cd" + integrity sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg== dependencies: "@babel/code-frame" "^7.0.0" error-ex "^1.3.1" @@ -2068,19 +2089,19 @@ path-exists@^4.0.0: path-is-absolute@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" - integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18= + integrity sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg== path-key@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40" - integrity sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A= + integrity sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw== path-key@^3.0.0, path-key@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== -path-parse@^1.0.6: +path-parse@^1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== @@ -2093,14 +2114,14 @@ path-type@^4.0.0: pend@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" - integrity sha1-elfrVQpng/kRUzH89GY9XI4AelA= + integrity sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg== performance-now@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" - integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= + integrity sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow== -picomatch@^2.2.3: +picomatch@^2.3.1: version "2.3.1" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== @@ -2108,14 +2129,14 @@ picomatch@^2.2.3: pify@^2.2.0: version "2.3.0" resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" - integrity sha1-7RQaasBDqEnqWISY59yosVMw6Qw= + integrity sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog== -pkg-dir@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" - integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ== +pkg-dir@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-5.0.0.tgz#a02d6aebe6ba133a928f74aec20bafdfe6b8e760" + integrity sha512-NPE8TDbzl/3YQYY7CSS228s3g2ollTFnc+Qi3tqmqJp9Vg2ovUpixcJEo2HJScN2Ez+kEaal6y70c0ehqJBJeA== dependencies: - find-up "^4.0.0" + find-up "^5.0.0" please-upgrade-node@^3.2.0: version "3.2.0" @@ -2127,12 +2148,12 @@ please-upgrade-node@^3.2.0: prelude-ls@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" - integrity sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ= + integrity sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w== pretty-bytes@^5.4.1: - version "5.4.1" - resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-5.4.1.tgz#cd89f79bbcef21e3d21eb0da68ffe93f803e884b" - integrity sha512-s1Iam6Gwz3JI5Hweaz4GoCD1WUNUIyzePFy5+Js2hjwGVt2Z79wNN+ZKOZ2vB6C+Xs6njyB84Z1IthQg8d9LxA== + version "5.6.0" + resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-5.6.0.tgz#356256f643804773c82f64723fe78c92c62beaeb" + integrity sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg== pretty-format@^24.9.0: version "24.9.0" @@ -2155,9 +2176,9 @@ progress@^2.0.0: integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== psl@^1.1.28: - version "1.8.0" - resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" - integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ== + version "1.9.0" + resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" + integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== pump@^3.0.0: version "3.0.0" @@ -2170,7 +2191,7 @@ pump@^3.0.0: punycode@1.3.2: version "1.3.2" resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" - integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0= + integrity sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw== punycode@^2.1.0, punycode@^2.1.1: version "2.1.1" @@ -2178,14 +2199,14 @@ punycode@^2.1.0, punycode@^2.1.1: integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== qs@~6.5.2: - version "6.5.2" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" - integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA== + version "6.5.3" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.3.tgz#3aeeffc91967ef6e35c0e488ef46fb296ab76aad" + integrity sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA== querystring@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" - integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= + integrity sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g== ramda@~0.26.1: version "0.26.1" @@ -2195,7 +2216,7 @@ ramda@~0.26.1: react-double-scrollbar@^0.0.15: version "0.0.15" resolved "https://registry.yarnpkg.com/react-double-scrollbar/-/react-double-scrollbar-0.0.15.tgz#e915ab8cb3b959877075f49436debfdb04288fe4" - integrity sha1-6RWrjLO5WYdwdfSUNt6/2wQoj+Q= + integrity sha512-dLz3/WBIpgFnzFY0Kb4aIYBMT2BWomHuW2DH6/9jXfS6/zxRRBUFQ04My4HIB7Ma7QoRBpcy8NtkPeFgcGBpgg== react-is@^16.8.4: version "16.13.1" @@ -2216,9 +2237,9 @@ readable-stream@^2.2.2: util-deprecate "~1.0.1" regenerator-runtime@^0.13.4: - version "0.13.7" - resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz#cac2dacc8a1ea675feaabaeb8ae833898ae46f55" - integrity sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew== + version "0.13.9" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz#8925742a98ffd90814988d7566ad30ca3b263b52" + integrity sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA== regexpp@^2.0.1: version "2.0.1" @@ -2228,7 +2249,7 @@ regexpp@^2.0.1: request-progress@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/request-progress/-/request-progress-3.0.0.tgz#4ca754081c7fec63f505e4faa825aa06cd669dbe" - integrity sha1-TKdUCBx/7GP1BeT6qCWqBs1mnb4= + integrity sha512-MnWzEHHaxHO2iWiQuHrUPBi/1WeBf5PkxQqNyNvLl9VAYSdXkP8tQ3pBSeCPD+yw0v0Aq1zosWLz0BdeXpWwZg== dependencies: throttleit "^1.0.0" @@ -2238,17 +2259,18 @@ resolve-from@^4.0.0: integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== resolve@^1.3.2: - version "1.19.0" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.19.0.tgz#1af5bf630409734a067cae29318aac7fa29a267c" - integrity sha512-rArEXAgsBG4UgRGcynxWIWKFvh/XZCcS8UJdHhwy91zwAvCZIbcs+vAbflgBnNjYMs/i/i+/Ux6IZhML1yPvxg== + version "1.22.1" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.1.tgz#27cb2ebb53f91abb49470a928bba7558066ac177" + integrity sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw== dependencies: - is-core-module "^2.1.0" - path-parse "^1.0.6" + is-core-module "^2.9.0" + path-parse "^1.0.7" + supports-preserve-symlinks-flag "^1.0.0" restore-cursor@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-1.0.1.tgz#34661f46886327fed2991479152252df92daa541" - integrity sha1-NGYfRohjJ/7SmRR5FSJS35LapUE= + integrity sha512-reSjH4HuiFlxlaBaFCiS6O76ZGG2ygKoSlCsipKdaZuKSPx/+bt9mULkn4l0asVzbEfQQmXRg6Wp6gv6m0wElw== dependencies: exit-hook "^1.0.0" onetime "^1.0.0" @@ -2256,7 +2278,7 @@ restore-cursor@^1.0.1: restore-cursor@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-2.0.0.tgz#9f7ee287f82fd326d4fd162923d62129eee0dfaf" - integrity sha1-n37ih/gv0ybU/RYpI9YhKe7g368= + integrity sha512-6IzJLuGi4+R14vwagDHX+JrXmPVtPpn4mffDJ1UdR7/Edm87fl6yi8mMBIVvFtJaNTUvjughmW4hwLhRG7gC1Q== dependencies: onetime "^2.0.0" signal-exit "^3.0.2" @@ -2269,6 +2291,11 @@ restore-cursor@^3.1.0: onetime "^5.1.0" signal-exit "^3.0.2" +rfdc@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/rfdc/-/rfdc-1.3.0.tgz#d0b7c441ab2720d05dc4cf26e01c89631d9da08b" + integrity sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA== + rimraf@2.6.3: version "2.6.3" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab" @@ -2288,13 +2315,20 @@ run-async@^2.4.0: resolved "https://registry.yarnpkg.com/run-async/-/run-async-2.4.1.tgz#8440eccf99ea3e70bd409d49aab88e10c189a455" integrity sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ== -rxjs@^6.3.3, rxjs@^6.6.0, rxjs@^6.6.3: - version "6.6.3" - resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.6.3.tgz#8ca84635c4daa900c0d3967a6ee7ac60271ee552" - integrity sha512-trsQc+xYYXZ3urjOiJOuCOa5N3jAZ3eiSpQB5hIT8zGlL2QfnHLJ2r7GMkBGuIausdJN1OneaI6gQlsqNHHmZQ== +rxjs@^6.3.3, rxjs@^6.6.0: + version "6.6.7" + resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.6.7.tgz#90ac018acabf491bf65044235d5863c4dab804c9" + integrity sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ== dependencies: tslib "^1.9.0" +rxjs@^7.5.1: + version "7.5.6" + resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-7.5.6.tgz#0446577557862afd6903517ce7cae79ecb9662bc" + integrity sha512-dnyv2/YsXhnm461G+R/Pe5bWP41Nm6LBXEYWI6eiFP4fiwx6WRI/CD0zbdVAudd9xwLEF2IDcKXLHit0FYjUzw== + dependencies: + tslib "^2.1.0" + safe-buffer@^5.0.1, safe-buffer@^5.1.2: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" @@ -2313,17 +2347,17 @@ safe-buffer@~5.1.0, safe-buffer@~5.1.1: semver-compare@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/semver-compare/-/semver-compare-1.0.0.tgz#0dee216a1c941ab37e9efb1788f6afc5ff5537fc" - integrity sha1-De4hahyUGrN+nvsXiPavxf9VN/w= + integrity sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow== -semver-regex@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/semver-regex/-/semver-regex-2.0.0.tgz#a93c2c5844539a770233379107b38c7b4ac9d338" - integrity sha512-mUdIBBvdn0PLOeP3TEkMH7HHeUP3GjsXCwKarjv/kGmUFOYg1VqEemKhoQpWMu6X2I8kHeuVdGibLGkVK+/5Qw== +semver-regex@^3.1.2: + version "3.1.4" + resolved "https://registry.yarnpkg.com/semver-regex/-/semver-regex-3.1.4.tgz#13053c0d4aa11d070a2f2872b6b1e3ae1e1971b4" + integrity sha512-6IiqeZNgq01qGf0TId0t3NvKzSvUsjcpdEO3AQNeIjR6A2+ckTnQlDpl4qu1bjRv0RzN3FP9hzFmws3lKqRWkA== semver@7.x: - version "7.3.5" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7" - integrity sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ== + version "7.3.7" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f" + integrity sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g== dependencies: lru-cache "^6.0.0" @@ -2340,7 +2374,7 @@ semver@^6.1.2: shebang-command@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea" - integrity sha1-RKrGW2lbAzmJaMOfNj/uXer98eo= + integrity sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg== dependencies: shebang-regex "^1.0.0" @@ -2354,7 +2388,7 @@ shebang-command@^2.0.0: shebang-regex@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3" - integrity sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM= + integrity sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ== shebang-regex@^3.0.0: version "3.0.0" @@ -2362,9 +2396,9 @@ shebang-regex@^3.0.0: integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== signal-exit@^3.0.2: - version "3.0.3" - resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" - integrity sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA== + version "3.0.7" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" + integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== slash@^3.0.0: version "3.0.0" @@ -2374,7 +2408,7 @@ slash@^3.0.0: slice-ansi@0.0.4: version "0.0.4" resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-0.0.4.tgz#edbf8903f66f7ce2f8eafd6ceed65e264c831b35" - integrity sha1-7b+JA/ZvfOL46v1s7tZeJkyDGzU= + integrity sha512-up04hB2hR92PgjpyU3y/eg91yIBILyjVY26NvvciY3EVVPjybkMszMpXQ9QAkcS3I5rtJBDLoTxxg+qvW8c7rw== slice-ansi@^2.1.0: version "2.1.0" @@ -2404,9 +2438,9 @@ slice-ansi@^4.0.0: is-fullwidth-code-point "^3.0.0" source-map-support@^0.5.17: - version "0.5.19" - resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.19.tgz#a98b62f86dcaf4f67399648c085291ab9e8fed61" - integrity sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw== + version "0.5.21" + resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f" + integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w== dependencies: buffer-from "^1.0.0" source-map "^0.6.0" @@ -2419,12 +2453,12 @@ source-map@^0.6.0: sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" - integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= + integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g== -sshpk@^1.7.0: - version "1.16.1" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877" - integrity sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg== +sshpk@^1.14.1: + version "1.17.0" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.17.0.tgz#578082d92d4fe612b13007496e543fa0fbcbe4c5" + integrity sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ== dependencies: asn1 "~0.2.3" assert-plus "^1.0.0" @@ -2444,7 +2478,7 @@ string-argv@0.3.1: string-width@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3" - integrity sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M= + integrity sha512-0XsVpQLnVCXHJfyEs8tC0zpTVIr5PKKsQtkT29IwupnPTjtPmQ3xT/4yCREF9hYkV/3M3kzcUTSAZT6a6h81tw== dependencies: code-point-at "^1.0.0" is-fullwidth-code-point "^1.0.0" @@ -2468,13 +2502,13 @@ string-width@^3.0.0: strip-ansi "^5.1.0" string-width@^4.1.0, string-width@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.0.tgz#952182c46cc7b2c313d1596e623992bd163b72b5" - integrity sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg== + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== dependencies: emoji-regex "^8.0.0" is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.0" + strip-ansi "^6.0.1" string_decoder@~1.1.1: version "1.1.1" @@ -2495,14 +2529,14 @@ stringify-object@^3.3.0: strip-ansi@^3.0.0, strip-ansi@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" - integrity sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8= + integrity sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg== dependencies: ansi-regex "^2.0.0" strip-ansi@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-4.0.0.tgz#a8479022eb1ac368a871389b635262c505ee368f" - integrity sha1-qEeQIusaw2iocTibY1JixQXuNo8= + integrity sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow== dependencies: ansi-regex "^3.0.0" @@ -2513,12 +2547,12 @@ strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532" - integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w== +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== dependencies: - ansi-regex "^5.0.0" + ansi-regex "^5.0.1" strip-final-newline@^2.0.0: version "2.0.0" @@ -2533,7 +2567,7 @@ strip-json-comments@^3.0.1: supports-color@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" - integrity sha1-U10EXOa2Nj+kARcIRimZXp3zJMc= + integrity sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g== supports-color@^5.3.0: version "5.5.0" @@ -2549,6 +2583,11 @@ supports-color@^7.1.0, supports-color@^7.2.0: dependencies: has-flag "^4.0.0" +supports-preserve-symlinks-flag@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" + integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== + symbol-observable@^1.1.0: version "1.2.0" resolved "https://registry.yarnpkg.com/symbol-observable/-/symbol-observable-1.2.0.tgz#c22688aed4eab3cdc2dfeacbb561660560a00804" @@ -2567,17 +2606,17 @@ table@^5.2.3: text-table@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" - integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ= + integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw== throttleit@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/throttleit/-/throttleit-1.0.0.tgz#9e785836daf46743145a5984b6268d828528ac6c" - integrity sha1-nnhYNtr0Z0MUWlmEtiaNgoUorGw= + integrity sha512-rkTVqu6IjfQ/6+uNuuc3sZek4CEYxTJom3IktzgdSxcZqdARuebbA/f4QmAxMQIxqq9ZLEUkSYqvuk1I6VKq4g== through@^2.3.6, through@^2.3.8: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" - integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= + integrity sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg== tmp@^0.0.33: version "0.0.33" @@ -2640,6 +2679,11 @@ tslib@^1.13.0, tslib@^1.7.1, tslib@^1.8.1, tslib@^1.9.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== +tslib@^2.1.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.0.tgz#7cecaa7f073ce680a05847aa77be941098f36dc3" + integrity sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ== + tslint-config-prettier@^1.18.0: version "1.18.0" resolved "https://registry.yarnpkg.com/tslint-config-prettier/-/tslint-config-prettier-1.18.0.tgz#75f140bde947d35d8f0d238e0ebf809d64592c37" @@ -2683,26 +2727,26 @@ tsutils@^2.29.0: tunnel-agent@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" - integrity sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0= + integrity sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w== dependencies: safe-buffer "^5.0.1" tweetnacl@^0.14.3, tweetnacl@~0.14.0: version "0.14.5" resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" - integrity sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q= + integrity sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA== type-check@~0.3.2: version "0.3.2" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.3.2.tgz#5884cab512cf1d355e3fb784f30804b2b520db72" - integrity sha1-WITKtRLPHTVeP7eE8wgEsrUg23I= + integrity sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg== dependencies: prelude-ls "~1.1.2" -type-fest@^0.11.0: - version "0.11.0" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.11.0.tgz#97abf0872310fed88a5c466b25681576145e33f1" - integrity sha512-OdjXJxnCN1AvyLSzeKIgXTXxV+99ZuXl3Hpo9XpJAv9MBcHrrJOQ5kV7ypXOuQie+AmWG25hLbiKdwYTifzcfQ== +type-fest@^0.21.3: + version "0.21.3" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.21.3.tgz#d260a24b0198436e133fa26a524a6d65fa3b2e37" + integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w== type-fest@^0.8.1: version "0.8.1" @@ -2712,12 +2756,7 @@ type-fest@^0.8.1: typedarray@^0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" - integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= - -universalify@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/universalify/-/universalify-1.0.0.tgz#b61a1da173e8435b2fe3c67d29b9adf8594bd16d" - integrity sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug== + integrity sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA== universalify@^2.0.0: version "2.0.0" @@ -2730,16 +2769,16 @@ untildify@^4.0.0: integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw== uri-js@^4.2.2: - version "4.4.0" - resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.0.tgz#aa714261de793e8a82347a7bcc9ce74e86f28602" - integrity sha512-B0yRTzYdUCCn9n+F4+Gh4yIDtMQcaJsmYBDsTSG8g/OejKBodLQ2IHfN3bM7jUsRXndopT7OIXWdYqc1fjmV6g== + version "4.4.1" + resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" + integrity sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg== dependencies: punycode "^2.1.0" url@^0.11.0: version "0.11.0" resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1" - integrity sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE= + integrity sha512-kbailJa29QrtXnxgq+DdCEGlbTeYM2eJUxsz6vjZavrCYPMIFHMKQmSKYAIuUK2i7hgPm28a8piX5NTUtM/LKQ== dependencies: punycode "1.3.2" querystring "0.2.0" @@ -2747,31 +2786,31 @@ url@^0.11.0: util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" - integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= + integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== -uuid@^3.3.2: - version "3.4.0" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" - integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== +uuid@^8.3.2: + version "8.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" + integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== v8-compile-cache@^2.0.3: - version "2.2.0" - resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.2.0.tgz#9471efa3ef9128d2f7c6a7ca39c4dd6b5055b132" - integrity sha512-gTpR5XQNKFwOd4clxfnhaqvfqMpqEwr4tOtCyz4MtYZX2JYhfr1JvBFKdS+7K/9rfpZR3VLX+YWBbKoxCgS43Q== + version "2.3.0" + resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" + integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA== verror@1.10.0: version "1.10.0" resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" - integrity sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA= + integrity sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw== dependencies: assert-plus "^1.0.0" core-util-is "1.0.2" extsprintf "^1.2.0" which-pm-runs@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/which-pm-runs/-/which-pm-runs-1.0.0.tgz#670b3afbc552e0b55df6b7780ca74615f23ad1cb" - integrity sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs= + version "1.1.0" + resolved "https://registry.yarnpkg.com/which-pm-runs/-/which-pm-runs-1.1.0.tgz#35ccf7b1a0fce87bd8b92a478c9d045785d3bf35" + integrity sha512-n1brCuqClxfFfq/Rb0ICg9giSZqCS+pLtccdag6C2HyufBrh3fBOiy9nb6ggRMvWOVH5GrdJskj5iGTZNxd7SA== which@^1.2.9: version "1.3.1" @@ -2795,7 +2834,7 @@ word-wrap@~1.2.3: wrap-ansi@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-3.0.1.tgz#288a04d87eda5c286e060dfe8f135ce8d007f8ba" - integrity sha1-KIoE2H7aXChuBg3+jxNc6NAH+Lo= + integrity sha512-iXR3tDXpbnTpzjKSylUJRkLuOrEC7hwEB221cgn6wtF8wpmz28puFXAEfPT5zrjM3wahygB//VuWEr1vTkDcNQ== dependencies: string-width "^2.1.1" strip-ansi "^4.0.0" @@ -2809,10 +2848,19 @@ wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" - integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= + integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== write@1.0.3: version "1.0.3" @@ -2827,9 +2875,9 @@ yallist@^4.0.0: integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== yaml@^1.10.0: - version "1.10.0" - resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.0.tgz#3b593add944876077d4d683fee01081bd9fff31e" - integrity sha512-yr2icI4glYaNG+KWONODapy2/jDdMSDnrONSjblABjD9B4Z5LgiircSt8m8sRZFNi08kG9Sm0uSHtEmP3zaEGg== + version "1.10.2" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" + integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== yargs-parser@20.x: version "20.2.9" @@ -2839,7 +2887,7 @@ yargs-parser@20.x: yauzl@^2.10.0: version "2.10.0" resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9" - integrity sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk= + integrity sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g== dependencies: buffer-crc32 "~0.2.3" fd-slicer "~1.1.0" @@ -2848,3 +2896,8 @@ yn@3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50" integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q== + +yocto-queue@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" + integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== From ce15448fbd005931b4a7272a643e2d076334ee63 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Wed, 17 Aug 2022 08:56:01 -0700 Subject: [PATCH 19/20] Schedule request in worker thread (#748) Signed-off-by: penghuo --- .../sql/legacy/plugin/RestSqlAction.java | 35 ++++++---- .../sql/opensearch/executor/Scheduler.java | 33 ++++++++++ .../opensearch/executor/SchedulerTest.java | 41 ++++++++++++ .../sql/plugin/rest/RestPPLQueryAction.java | 66 ++++++++++--------- 4 files changed, 130 insertions(+), 45 deletions(-) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/executor/Scheduler.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/executor/SchedulerTest.java diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java index 6f7579c9c7d..06d1ba1c73c 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java @@ -9,6 +9,7 @@ import static org.opensearch.rest.RestStatus.BAD_REQUEST; import static org.opensearch.rest.RestStatus.OK; import static org.opensearch.rest.RestStatus.SERVICE_UNAVAILABLE; +import static org.opensearch.sql.opensearch.executor.Scheduler.schedule; import com.alibaba.druid.sql.parser.ParserException; import com.google.common.collect.ImmutableList; @@ -147,19 +148,27 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli Format format = SqlRequestParam.getFormat(request.params()); - // Route request to new query engine if it's supported already - SQLQueryRequest newSqlRequest = new SQLQueryRequest(sqlRequest.getJsonContent(), - sqlRequest.getSql(), request.path(), request.params()); - RestChannelConsumer result = newSqlQueryHandler.prepareRequest(newSqlRequest, client); - if (result != RestSQLQueryAction.NOT_SUPPORTED_YET) { - LOG.info("[{}] Request is handled by new SQL query engine", QueryContext.getRequestId()); - return result; - } - LOG.debug("[{}] Request {} is not supported and falling back to old SQL engine", - QueryContext.getRequestId(), newSqlRequest); - - final QueryAction queryAction = explainRequest(client, sqlRequest, format); - return channel -> executeSqlRequest(request, queryAction, client, channel); + return channel -> schedule(client, () -> { + try { + // Route request to new query engine if it's supported already + SQLQueryRequest newSqlRequest = new SQLQueryRequest(sqlRequest.getJsonContent(), + sqlRequest.getSql(), request.path(), request.params()); + RestChannelConsumer result = newSqlQueryHandler.prepareRequest(newSqlRequest, client); + if (result != RestSQLQueryAction.NOT_SUPPORTED_YET) { + LOG.info("[{}] Request is handled by new SQL query engine", + QueryContext.getRequestId()); + result.accept(channel); + } else { + LOG.debug("[{}] Request {} is not supported and falling back to old SQL engine", + QueryContext.getRequestId(), newSqlRequest); + QueryAction queryAction = explainRequest(client, sqlRequest, format); + executeSqlRequest(request, queryAction, client, channel); + } + } catch (Exception e) { + logAndPublishMetrics(e); + reportError(channel, e, isClientError(e) ? BAD_REQUEST : SERVICE_UNAVAILABLE); + } + }); } catch (Exception e) { logAndPublishMetrics(e); return channel -> reportError(channel, e, isClientError(e) ? BAD_REQUEST : SERVICE_UNAVAILABLE); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/Scheduler.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/Scheduler.java new file mode 100644 index 00000000000..5567d1f9b20 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/Scheduler.java @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.executor; + +import java.util.Map; +import lombok.experimental.UtilityClass; +import org.apache.logging.log4j.ThreadContext; +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.threadpool.ThreadPool; + +/** The scheduler which schedule the task run in sql-worker thread pool. */ +@UtilityClass +public class Scheduler { + + public static final String SQL_WORKER_THREAD_POOL_NAME = "sql-worker"; + + public static void schedule(NodeClient client, Runnable task) { + ThreadPool threadPool = client.threadPool(); + threadPool.schedule(withCurrentContext(task), new TimeValue(0), SQL_WORKER_THREAD_POOL_NAME); + } + + private static Runnable withCurrentContext(final Runnable task) { + final Map currentContext = ThreadContext.getImmutableContext(); + return () -> { + ThreadContext.putAll(currentContext); + task.run(); + }; + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/SchedulerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/SchedulerTest.java new file mode 100644 index 00000000000..f14bda7a95c --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/SchedulerTest.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.executor; + +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.client.node.NodeClient; +import org.opensearch.threadpool.ThreadPool; + +@ExtendWith(MockitoExtension.class) +class SchedulerTest { + @Test + public void schedule() { + NodeClient nodeClient = mock(NodeClient.class); + ThreadPool threadPool = mock(ThreadPool.class); + when(nodeClient.threadPool()).thenReturn(threadPool); + + doAnswer( + invocation -> { + Runnable task = invocation.getArgument(0); + task.run(); + return null; + }) + .when(threadPool) + .schedule(any(), any(), any()); + AtomicBoolean isRun = new AtomicBoolean(false); + Scheduler.schedule(nodeClient, () -> isRun.set(true)); + assertTrue(isRun.get()); + } +} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java index 0a65fb7b0d4..e9202d96e8f 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java @@ -9,6 +9,7 @@ import static org.opensearch.rest.RestStatus.INTERNAL_SERVER_ERROR; import static org.opensearch.rest.RestStatus.OK; import static org.opensearch.rest.RestStatus.SERVICE_UNAVAILABLE; +import static org.opensearch.sql.opensearch.executor.Scheduler.schedule; import com.google.common.collect.ImmutableList; import java.util.Arrays; @@ -112,42 +113,43 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nod PPLQueryRequestFactory.getPPLRequest(request) ); - return channel -> - nodeClient.execute( - PPLQueryAction.INSTANCE, - transportPPLQueryRequest, - new ActionListener<>() { - @Override - public void onResponse(TransportPPLQueryResponse response) { - sendResponse(channel, OK, response.getResult()); - } - - @Override - public void onFailure(Exception e) { - if (transportPPLQueryRequest.isExplainRequest()) { - LOG.error("Error happened during explain", e); - sendResponse( - channel, - INTERNAL_SERVER_ERROR, - "Failed to explain the query due to error: " + e.getMessage()); - } else if (e instanceof IllegalAccessException) { + return channel -> schedule(nodeClient, () -> + nodeClient.execute( + PPLQueryAction.INSTANCE, + transportPPLQueryRequest, + new ActionListener<>() { + @Override + public void onResponse(TransportPPLQueryResponse response) { + sendResponse(channel, OK, response.getResult()); + } + + @Override + public void onFailure(Exception e) { + if (transportPPLQueryRequest.isExplainRequest()) { + LOG.error("Error happened during explain", e); + sendResponse( + channel, + INTERNAL_SERVER_ERROR, + "Failed to explain the query due to error: " + e.getMessage()); + } else if (e instanceof IllegalAccessException) { + reportError(channel, e, BAD_REQUEST); + } else { + LOG.error("Error happened during query handling", e); + if (isClientError(e)) { + Metrics.getInstance() + .getNumericalMetric(MetricName.PPL_FAILED_REQ_COUNT_CUS) + .increment(); reportError(channel, e, BAD_REQUEST); } else { - LOG.error("Error happened during query handling", e); - if (isClientError(e)) { - Metrics.getInstance() - .getNumericalMetric(MetricName.PPL_FAILED_REQ_COUNT_CUS) - .increment(); - reportError(channel, e, BAD_REQUEST); - } else { - Metrics.getInstance() - .getNumericalMetric(MetricName.PPL_FAILED_REQ_COUNT_SYS) - .increment(); - reportError(channel, e, SERVICE_UNAVAILABLE); - } + Metrics.getInstance() + .getNumericalMetric(MetricName.PPL_FAILED_REQ_COUNT_SYS) + .increment(); + reportError(channel, e, SERVICE_UNAVAILABLE); } } - }); + } + } + )); } private void sendResponse(RestChannel channel, RestStatus status, String content) { From 8185127ec8dab63dd735bd74eb5edd88d42d3742 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Wed, 17 Aug 2022 18:01:41 +0000 Subject: [PATCH 20/20] Resolve conflicts Signed-off-by: Joshua Li --- build.gradle | 12 ------------ integ-test/build.gradle | 4 ---- .../sql/legacy/plugin/RestSqlAction.java | 16 ---------------- opensearch/build.gradle | 4 ---- plugin/build.gradle | 4 ---- ppl/build.gradle | 1 + protocol/build.gradle | 8 -------- sql-jdbc/build.gradle | 5 ----- 8 files changed, 1 insertion(+), 53 deletions(-) diff --git a/build.gradle b/build.gradle index 5c851739d90..8a2f9046bfc 100644 --- a/build.gradle +++ b/build.gradle @@ -6,16 +6,9 @@ buildscript { ext { -<<<<<<< HEAD - opensearch_version = System.getProperty("opensearch.version", "2.1.0-SNAPSHOT") - spring_version = "5.3.22" - jackson_version = "2.13.3" - jackson_databind_version = "2.13.3" -======= opensearch_version = System.getProperty("opensearch.version", "2.2.0-SNAPSHOT") spring_version = "5.3.22" jackson_version = "2.13.3" ->>>>>>> main isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') @@ -143,11 +136,6 @@ subprojects { "org.checkstyle.google.suppressionfilter.config": rootProject.file("config/checkstyle/suppressions.xml")] ignoreFailures = false } - - configurations.all { - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${jackson_version}" - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jackson_version}" - } } checkstyle { configFile file("config/checkstyle/checkstyle.xml") diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 80dd54e4900..429c360a1ba 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -55,11 +55,7 @@ configurations.all { resolutionStrategy.force 'com.google.guava:guava:31.0.1-jre' resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${jackson_version}" resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jackson_version}" -<<<<<<< HEAD - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" -======= resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_version}" ->>>>>>> main } dependencies { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java index 5bf6ba181ec..06d1ba1c73c 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java @@ -148,21 +148,6 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli Format format = SqlRequestParam.getFormat(request.params()); -<<<<<<< HEAD - // Route request to new query engine if it's supported already - SQLQueryRequest newSqlRequest = new SQLQueryRequest(sqlRequest.getJsonContent(), - sqlRequest.getSql(), request.path(), request.params()); - RestChannelConsumer result = newSqlQueryHandler.prepareRequest(newSqlRequest, client); - if (result != RestSQLQueryAction.NOT_SUPPORTED_YET) { - LOG.info("[{}] Request is handled by new SQL query engine", QueryContext.getRequestId()); - return result; - } - LOG.debug("[{}] Request {} is not supported and falling back to old SQL engine", - QueryContext.getRequestId(), newSqlRequest); - - final QueryAction queryAction = explainRequest(client, sqlRequest, format); - return channel -> executeSqlRequest(request, queryAction, client, channel); -======= return channel -> schedule(client, () -> { try { // Route request to new query engine if it's supported already @@ -184,7 +169,6 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli reportError(channel, e, isClientError(e) ? BAD_REQUEST : SERVICE_UNAVAILABLE); } }); ->>>>>>> main } catch (Exception e) { logAndPublishMetrics(e); return channel -> reportError(channel, e, isClientError(e) ? BAD_REQUEST : SERVICE_UNAVAILABLE); diff --git a/opensearch/build.gradle b/opensearch/build.gradle index 0e0fa54c0f9..8b5f917dffd 100644 --- a/opensearch/build.gradle +++ b/opensearch/build.gradle @@ -33,11 +33,7 @@ dependencies { api group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" implementation "io.github.resilience4j:resilience4j-retry:1.5.0" implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${jackson_version}" -<<<<<<< HEAD - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_databind_version}" -======= implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_version}" ->>>>>>> main implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${jackson_version}" implementation group: 'org.json', name: 'json', version:'20180813' compileOnly group: 'org.opensearch.client', name: 'opensearch-rest-high-level-client', version: "${opensearch_version}" diff --git a/plugin/build.gradle b/plugin/build.gradle index 1aa3f7739da..5c3b3974ef8 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -87,11 +87,7 @@ configurations.all { resolutionStrategy.force 'commons-codec:commons-codec:1.13' resolutionStrategy.force 'com.google.guava:guava:31.0.1-jre' resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jackson_version}" -<<<<<<< HEAD - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" -======= resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_version}" ->>>>>>> main } compileJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) diff --git a/ppl/build.gradle b/ppl/build.gradle index 2c3c648478b..12b0787efcd 100644 --- a/ppl/build.gradle +++ b/ppl/build.gradle @@ -46,6 +46,7 @@ dependencies { implementation "org.antlr:antlr4-runtime:4.7.1" implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' + api group: 'org.opensearch', name: 'opensearch-x-content', version: "${opensearch_version}" api group: 'org.json', name: 'json', version: '20180813' implementation group: 'org.springframework', name: 'spring-context', version: "${spring_version}" implementation group: 'org.springframework', name: 'spring-beans', version: "${spring_version}" diff --git a/protocol/build.gradle b/protocol/build.gradle index bc6410fa931..fc35b94d343 100644 --- a/protocol/build.gradle +++ b/protocol/build.gradle @@ -31,11 +31,7 @@ plugins { dependencies { implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${jackson_version}" -<<<<<<< HEAD - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_databind_version}" -======= implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_version}" ->>>>>>> main implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${jackson_version}" implementation 'com.google.code.gson:gson:2.8.9' implementation project(':core') @@ -48,11 +44,7 @@ dependencies { } configurations.all { -<<<<<<< HEAD - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" -======= resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_version}" ->>>>>>> main } test { diff --git a/sql-jdbc/build.gradle b/sql-jdbc/build.gradle index 15df9e4b9ec..dd629e438f6 100644 --- a/sql-jdbc/build.gradle +++ b/sql-jdbc/build.gradle @@ -45,13 +45,8 @@ repositories { } dependencies { -<<<<<<< HEAD - implementation group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.6' - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.13.3' -======= implementation group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.13' implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "2.13.3" ->>>>>>> main implementation group: 'com.amazonaws', name: 'aws-java-sdk-core', version: '1.11.452' testImplementation('org.junit.jupiter:junit-jupiter-api:5.3.1')