From 30695b86be3371ad0fe0dc6b559ee584085416ac Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Mon, 5 Dec 2022 19:07:42 -0800 Subject: [PATCH 1/6] Enable CI for windows Signed-off-by: Peng Huo --- .../workflows/sql-test-and-build-workflow.yml | 28 +- .../sql-workbench-test-and-build-workflow.yml | 13 +- .../unittest/utils/PrettyFormatterTest.java | 6 +- .../AggregationQueryBuilderTest.java | 693 +++++++++--------- .../dsl/MetricAggregationBuilderTest.java | 247 +++---- .../format/CsvResponseFormatterTest.java | 49 +- .../format/RawResponseFormatterTest.java | 61 +- workbench/package.json | 2 +- 8 files changed, 559 insertions(+), 540 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index aa7fc3c8be9..73586d5da49 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -23,25 +23,31 @@ env: jobs: build: strategy: + # Run all jobs + fail-fast: false matrix: - java: - - 8 - - 11 - - 14 - runs-on: ubuntu-latest + entry: + - { os: ubuntu-latest, java: 8 } + - { os: windows-latest, java: 8, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc} + - { os: ubuntu-latest, java: 11 } + - { os: windows-latest, java: 11, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc} + - { os: ubuntu-latest, java: 14 } + - { os: windows-latest, java: 14, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc } + runs-on: ${{ matrix.entry.os }} steps: - uses: actions/checkout@v3 - - name: Set up JDK ${{ matrix.java }} + - name: Set up JDK ${{ matrix.entry.java }} uses: actions/setup-java@v1 with: - java-version: ${{ matrix.java }} + java-version: ${{ matrix.entry.java }} - name: Build with Gradle - run: ./gradlew --continue build assemble -Dopensearch.version=${{ env.OPENSEARCH_VERSION }} + run: ./gradlew --continue build ${{ matrix.entry.os_build_args }} -Dopensearch.version=${{ env.OPENSEARCH_VERSION }} - name: Run backward compatibility tests + if: ${{ matrix.entry.os == 'ubuntu-latest' }} run: ./bwctest.sh - name: Create Artifact Path @@ -51,7 +57,7 @@ jobs: # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action - name: Upload SQL Coverage Report - if: always() + if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} uses: codecov/codecov-action@v3 with: flags: sql-engine @@ -60,11 +66,11 @@ jobs: - name: Upload Artifacts uses: actions/upload-artifact@v2 with: - name: opensearch-sql + name: opensearch-sql-${{ matrix.entry.os }} path: opensearch-sql-builds - name: Upload test reports - if: always() + if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} uses: actions/upload-artifact@v2 with: name: test-reports diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index 9d7662dd305..c6cbacfeb0d 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -16,8 +16,15 @@ env: jobs: build: - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} steps: + - name: Enable longer filenames + if: ${{ matrix.os == 'windows-latest' }} + run: git config --system core.longpaths true + - name: Checkout Plugin uses: actions/checkout@v3 @@ -51,7 +58,7 @@ jobs: yarn test:jest --coverage - name: Upload coverage - if: always() + if: ${{ matrix.os == 'ubuntu-latest' }} uses: codecov/codecov-action@v3 with: flags: query-workbench @@ -68,5 +75,5 @@ jobs: if: always() uses: actions/upload-artifact@v1 # can't update to v3 because upload fails with: - name: workbench + name: workbench-${{ matrix.os }} path: ../OpenSearch-Dashboards/plugins/workbench/build diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/PrettyFormatterTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/PrettyFormatterTest.java index fc20d818e6e..7dbe8956cbd 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/PrettyFormatterTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/PrettyFormatterTest.java @@ -31,11 +31,13 @@ public void assertFormatterWithoutContentInside() throws IOException { public void assertFormatterOutputsPrettyJson() throws IOException { String explainFormattedPrettyFilePath = TestUtils.getResourceFilePath( "/src/test/resources/expectedOutput/explain_format_pretty.json"); - String explainFormattedPretty = Files.toString(new File(explainFormattedPrettyFilePath), StandardCharsets.UTF_8); + String explainFormattedPretty = Files.toString(new File(explainFormattedPrettyFilePath), StandardCharsets.UTF_8) + .replaceAll("\r", ""); String explainFormattedOnelineFilePath = TestUtils.getResourceFilePath( "/src/test/resources/explain_format_oneline.json"); - String explainFormattedOneline = Files.toString(new File(explainFormattedOnelineFilePath), StandardCharsets.UTF_8); + String explainFormattedOneline = Files.toString(new File(explainFormattedOnelineFilePath), StandardCharsets.UTF_8) + .replaceAll("\r", ""); String result = JsonPrettyFormatter.format(explainFormattedOneline); assertThat(result, equalTo(explainFormattedPretty)); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java index 04aedc0f01e..3614d82e596 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java @@ -12,6 +12,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; +import static org.opensearch.sql.common.utils.StringUtils.format; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; @@ -72,31 +73,31 @@ void set_up() { @Test void should_build_composite_aggregation_for_field_reference() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"name\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"name\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(age)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"name\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"name\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(age)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("avg(age)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER))), @@ -105,31 +106,31 @@ void should_build_composite_aggregation_for_field_reference() { @Test void should_build_composite_aggregation_for_field_reference_with_order() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"name\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"name\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"last\",\n" - + " \"order\" : \"desc\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(age)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"name\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"name\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"last\",%n" + + " \"order\" : \"desc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(age)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("avg(age)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER))), @@ -152,31 +153,31 @@ void should_build_type_mapping_for_field_reference() { @Test void should_build_composite_aggregation_for_field_reference_of_keyword() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"name\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"name.keyword\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(age)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"name\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"name.keyword\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(age)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("avg(age)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER))), @@ -201,37 +202,37 @@ void should_build_composite_aggregation_for_expression() { Expression expr = invocation.getArgument(0); return expr.toString(); }).when(serializer).serialize(any()); - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"age\" : {\n" - + " \"terms\" : {\n" - + " \"script\" : {\n" - + " \"source\" : \"asin(age)\",\n" - + " \"lang\" : \"opensearch_query_expression\"\n" - + " },\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(balance)\" : {\n" - + " \"avg\" : {\n" - + " \"script\" : {\n" - + " \"source\" : \"abs(balance)\",\n" - + " \"lang\" : \"opensearch_query_expression\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"age\" : {%n" + + " \"terms\" : {%n" + + " \"script\" : {%n" + + " \"source\" : \"asin(age)\",%n" + + " \"lang\" : \"opensearch_query_expression\"%n" + + " },%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(balance)\" : {%n" + + " \"avg\" : {%n" + + " \"script\" : {%n" + + " \"source\" : \"abs(balance)\",%n" + + " \"lang\" : \"opensearch_query_expression\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("avg(balance)", new AvgAggregator( @@ -241,40 +242,40 @@ void should_build_composite_aggregation_for_expression() { @Test void should_build_composite_aggregation_follow_with_order_by_position() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"name\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"name\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"last\",\n" - + " \"order\" : \"desc\"\n" - + " }\n" - + " }\n" - + " }, {\n" - + " \"age\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"age\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(balance)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"balance\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"name\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"name\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"last\",%n" + + " \"order\" : \"desc\"%n" + + " }%n" + + " }%n" + + " }, {%n" + + " \"age\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"age\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(balance)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"balance\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( agg(named("avg(balance)", avg(ref("balance", INTEGER), INTEGER))), group(named("age", ref("age", INTEGER)), named("name", ref("name", STRING))), @@ -298,14 +299,14 @@ void should_build_type_mapping_for_expression() { @Test void should_build_aggregation_without_bucket() { - assertEquals( - "{\n" - + " \"avg(balance)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"balance\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"avg(balance)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"balance\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("avg(balance)", new AvgAggregator( @@ -315,29 +316,29 @@ void should_build_aggregation_without_bucket() { @Test void should_build_filter_aggregation() { - assertEquals( - "{\n" - + " \"avg(age) filter(where age > 34)\" : {\n" - + " \"filter\" : {\n" - + " \"range\" : {\n" - + " \"age\" : {\n" - + " \"from\" : 20,\n" - + " \"to\" : null,\n" - + " \"include_lower\" : false,\n" - + " \"include_upper\" : true,\n" - + " \"boost\" : 1.0\n" - + " }\n" - + " }\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(age) filter(where age > 34)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"avg(age) filter(where age > 34)\" : {%n" + + " \"filter\" : {%n" + + " \"range\" : {%n" + + " \"age\" : {%n" + + " \"from\" : 20,%n" + + " \"to\" : null,%n" + + " \"include_lower\" : false,%n" + + " \"include_upper\" : true,%n" + + " \"boost\" : 1.0%n" + + " }%n" + + " }%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(age) filter(where age > 34)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList(named("avg(age) filter(where age > 34)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER) @@ -347,46 +348,46 @@ void should_build_filter_aggregation() { @Test void should_build_filter_aggregation_group_by() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"gender\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"gender\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(age) filter(where age > 34)\" : {\n" - + " \"filter\" : {\n" - + " \"range\" : {\n" - + " \"age\" : {\n" - + " \"from\" : 20,\n" - + " \"to\" : null,\n" - + " \"include_lower\" : false,\n" - + " \"include_upper\" : true,\n" - + " \"boost\" : 1.0\n" - + " }\n" - + " }\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(age) filter(where age > 34)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"gender\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"gender\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(age) filter(where age > 34)\" : {%n" + + " \"filter\" : {%n" + + " \"range\" : {%n" + + " \"age\" : {%n" + + " \"from\" : 20,%n" + + " \"to\" : null,%n" + + " \"include_lower\" : false,%n" + + " \"include_upper\" : true,%n" + + " \"boost\" : 1.0%n" + + " }%n" + + " }%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(age) filter(where age > 34)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList(named("avg(age) filter(where age > 34)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER) @@ -408,32 +409,32 @@ void should_build_type_mapping_without_bucket() { @Test void should_build_histogram() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"SpanExpression(field=age, value=10, unit=NONE)\" : {\n" - + " \"histogram\" : {\n" - + " \"field\" : \"age\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\",\n" - + " \"interval\" : 10.0\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"count(a)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"a\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=age, value=10, unit=NONE)\" : {%n" + + " \"histogram\" : {%n" + + " \"field\" : \"age\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\",%n" + + " \"interval\" : 10.0%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"count(a)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"a\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(a)", new CountAggregator(Arrays.asList(ref("a", INTEGER)), INTEGER))), @@ -442,37 +443,37 @@ void should_build_histogram() { @Test void should_build_histogram_two_metrics() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"SpanExpression(field=age, value=10, unit=NONE)\" : {\n" - + " \"histogram\" : {\n" - + " \"field\" : \"age\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\",\n" - + " \"interval\" : 10.0\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"count(a)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"a\"\n" - + " }\n" - + " },\n" - + " \"avg(b)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"b\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=age, value=10, unit=NONE)\" : {%n" + + " \"histogram\" : {%n" + + " \"field\" : \"age\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\",%n" + + " \"interval\" : 10.0%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"count(a)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"a\"%n" + + " }%n" + + " },%n" + + " \"avg(b)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"b\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(a)", new CountAggregator(Arrays.asList(ref("a", INTEGER)), INTEGER)), @@ -482,32 +483,32 @@ void should_build_histogram_two_metrics() { @Test void fixed_interval_time_span() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"SpanExpression(field=timestamp, value=1, unit=H)\" : {\n" - + " \"date_histogram\" : {\n" - + " \"field\" : \"timestamp\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\",\n" - + " \"fixed_interval\" : \"1h\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"count(a)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"a\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=timestamp, value=1, unit=H)\" : {%n" + + " \"date_histogram\" : {%n" + + " \"field\" : \"timestamp\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\",%n" + + " \"fixed_interval\" : \"1h\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"count(a)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"a\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(a)", new CountAggregator(Arrays.asList(ref("a", INTEGER)), INTEGER))), @@ -516,32 +517,32 @@ void fixed_interval_time_span() { @Test void calendar_interval_time_span() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"SpanExpression(field=date, value=1, unit=W)\" : {\n" - + " \"date_histogram\" : {\n" - + " \"field\" : \"date\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\",\n" - + " \"calendar_interval\" : \"1w\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"count(a)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"a\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=date, value=1, unit=W)\" : {%n" + + " \"date_histogram\" : {%n" + + " \"field\" : \"date\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\",%n" + + " \"calendar_interval\" : \"1w\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"count(a)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"a\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(a)", new CountAggregator(Arrays.asList(ref("a", INTEGER)), INTEGER))), @@ -550,32 +551,32 @@ void calendar_interval_time_span() { @Test void general_span() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"SpanExpression(field=age, value=1, unit=NONE)\" : {\n" - + " \"histogram\" : {\n" - + " \"field\" : \"age\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\",\n" - + " \"interval\" : 1.0\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"count(a)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"a\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=age, value=1, unit=NONE)\" : {%n" + + " \"histogram\" : {%n" + + " \"field\" : \"age\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\",%n" + + " \"interval\" : 1.0%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"count(a)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"a\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(a)", new CountAggregator(Arrays.asList(ref("a", INTEGER)), INTEGER))), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java index 845e32ba835..5161b35021b 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java @@ -9,6 +9,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.when; +import static org.opensearch.sql.common.utils.StringUtils.format; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.DSL.literal; @@ -62,14 +63,14 @@ void set_up() { @Test void should_build_avg_aggregation() { - assertEquals( - "{\n" - + " \"avg(age)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"avg(age)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("avg(age)", @@ -78,14 +79,14 @@ void should_build_avg_aggregation() { @Test void should_build_sum_aggregation() { - assertEquals( - "{\n" - + " \"sum(age)\" : {\n" - + " \"sum\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"sum(age)\" : {%n" + + " \"sum\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("sum(age)", @@ -94,14 +95,14 @@ void should_build_sum_aggregation() { @Test void should_build_count_aggregation() { - assertEquals( - "{\n" - + " \"count(age)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"count(age)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(age)", @@ -110,14 +111,14 @@ void should_build_count_aggregation() { @Test void should_build_count_star_aggregation() { - assertEquals( - "{\n" - + " \"count(*)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"_index\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"count(*)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"_index\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(*)", @@ -126,14 +127,14 @@ void should_build_count_star_aggregation() { @Test void should_build_count_other_literal_aggregation() { - assertEquals( - "{\n" - + " \"count(1)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"_index\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"count(1)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"_index\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(1)", @@ -142,14 +143,14 @@ void should_build_count_other_literal_aggregation() { @Test void should_build_min_aggregation() { - assertEquals( - "{\n" - + " \"min(age)\" : {\n" - + " \"min\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"min(age)\" : {%n" + + " \"min\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("min(age)", @@ -158,14 +159,14 @@ void should_build_min_aggregation() { @Test void should_build_max_aggregation() { - assertEquals( - "{\n" - + " \"max(age)\" : {\n" - + " \"max\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"max(age)\" : {%n" + + " \"max\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("max(age)", @@ -174,15 +175,15 @@ void should_build_max_aggregation() { @Test void should_build_varPop_aggregation() { - assertEquals( - "{\n" - + " \"var_pop(age)\" : {\n" - + " \"extended_stats\" : {\n" - + " \"field\" : \"age\",\n" - + " \"sigma\" : 2.0\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"var_pop(age)\" : {%n" + + " \"extended_stats\" : {%n" + + " \"field\" : \"age\",%n" + + " \"sigma\" : 2.0%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("var_pop(age)", @@ -191,15 +192,15 @@ void should_build_varPop_aggregation() { @Test void should_build_varSamp_aggregation() { - assertEquals( - "{\n" - + " \"var_samp(age)\" : {\n" - + " \"extended_stats\" : {\n" - + " \"field\" : \"age\",\n" - + " \"sigma\" : 2.0\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"var_samp(age)\" : {%n" + + " \"extended_stats\" : {%n" + + " \"field\" : \"age\",%n" + + " \"sigma\" : 2.0%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("var_samp(age)", @@ -208,15 +209,15 @@ void should_build_varSamp_aggregation() { @Test void should_build_stddevPop_aggregation() { - assertEquals( - "{\n" - + " \"stddev_pop(age)\" : {\n" - + " \"extended_stats\" : {\n" - + " \"field\" : \"age\",\n" - + " \"sigma\" : 2.0\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"stddev_pop(age)\" : {%n" + + " \"extended_stats\" : {%n" + + " \"field\" : \"age\",%n" + + " \"sigma\" : 2.0%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("stddev_pop(age)", @@ -225,15 +226,15 @@ void should_build_stddevPop_aggregation() { @Test void should_build_stddevSamp_aggregation() { - assertEquals( - "{\n" - + " \"stddev_samp(age)\" : {\n" - + " \"extended_stats\" : {\n" - + " \"field\" : \"age\",\n" - + " \"sigma\" : 2.0\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"stddev_samp(age)\" : {%n" + + " \"extended_stats\" : {%n" + + " \"field\" : \"age\",%n" + + " \"sigma\" : 2.0%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("stddev_samp(age)", @@ -242,14 +243,14 @@ void should_build_stddevSamp_aggregation() { @Test void should_build_cardinality_aggregation() { - assertEquals( - "{\n" - + " \"count(distinct name)\" : {\n" - + " \"cardinality\" : {\n" - + " \"field\" : \"name\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"count(distinct name)\" : {%n" + + " \"cardinality\" : {%n" + + " \"field\" : \"name\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Collections.singletonList(named("count(distinct name)", new CountAggregator( Collections.singletonList(ref("name", STRING)), INTEGER).distinct(true))))); @@ -257,29 +258,29 @@ void should_build_cardinality_aggregation() { @Test void should_build_filtered_cardinality_aggregation() { - assertEquals( - "{\n" - + " \"count(distinct name) filter(where age > 30)\" : {\n" - + " \"filter\" : {\n" - + " \"range\" : {\n" - + " \"age\" : {\n" - + " \"from\" : 30,\n" - + " \"to\" : null,\n" - + " \"include_lower\" : false,\n" - + " \"include_upper\" : true,\n" - + " \"boost\" : 1.0\n" - + " }\n" - + " }\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"count(distinct name) filter(where age > 30)\" : {\n" - + " \"cardinality\" : {\n" - + " \"field\" : \"name\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"count(distinct name) filter(where age > 30)\" : {%n" + + " \"filter\" : {%n" + + " \"range\" : {%n" + + " \"age\" : {%n" + + " \"from\" : 30,%n" + + " \"to\" : null,%n" + + " \"include_lower\" : false,%n" + + " \"include_upper\" : true,%n" + + " \"boost\" : 1.0%n" + + " }%n" + + " }%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"count(distinct name) filter(where age > 30)\" : {%n" + + " \"cardinality\" : {%n" + + " \"field\" : \"name\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery(Collections.singletonList(named( "count(distinct name) filter(where age > 30)", new CountAggregator(Collections.singletonList(ref("name", STRING)), INTEGER) diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java index 8998086afca..949781ce9f3 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java @@ -7,6 +7,7 @@ package org.opensearch.sql.protocol.response.format; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.common.utils.StringUtils.format; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; @@ -37,8 +38,8 @@ void formatResponse() { tupleValue(ImmutableMap.of("name", "John", "age", 20)), tupleValue(ImmutableMap.of("name", "Smith", "age", 30)))); CsvResponseFormatter formatter = new CsvResponseFormatter(); - String expected = "name,age\nJohn,20\nSmith,30"; - assertEquals(expected, formatter.format(response)); + String expected = "name|age%nJohn|20%nSmith|30"; + assertEquals(format(expected), formatter.format(response)); } @Test @@ -51,9 +52,9 @@ void sanitizeHeaders() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of( "=firstname", "John", "+lastname", "Smith", "-city", "Seattle", "@age", 20)))); - String expected = "'=firstname,'+lastname,'-city,'@age\n" - + "John,Smith,Seattle,20"; - assertEquals(expected, formatter.format(response)); + String expected = "=firstname|+lastname|-city|@age%n" + + "John|Smith|Seattle|20"; + assertEquals(format(expected), formatter.format(response)); } @Test @@ -67,14 +68,14 @@ void sanitizeData() { tupleValue(ImmutableMap.of("city", "-Seattle")), tupleValue(ImmutableMap.of("city", "@Seattle")), tupleValue(ImmutableMap.of("city", "Seattle=")))); - String expected = "city\n" - + "Seattle\n" - + "'=Seattle\n" - + "'+Seattle\n" - + "'-Seattle\n" - + "'@Seattle\n" + String expected = "city%n" + + "Seattle%n" + + "=Seattle%n" + + "+Seattle%n" + + "-Seattle%n" + + "@Seattle%n" + "Seattle="; - assertEquals(expected, formatter.format(response)); + assertEquals(format(expected), formatter.format(response)); } @Test @@ -84,9 +85,9 @@ void quoteIfRequired() { new ExecutionEngine.Schema.Column(",,age", ",,age", INTEGER))); QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("na,me", "John,Smith", ",,age", "30,,,")))); - String expected = "\"na,me\",\",,age\"\n" - + "\"John,Smith\",\"30,,,\""; - assertEquals(expected, formatter.format(response)); + String expected = "\"na|me\"|\"||age\"%n" + + "\"John|Smith\"|\"30|||\""; + assertEquals(format(expected), formatter.format(response)); } @Test @@ -105,10 +106,10 @@ void escapeSanitize() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("city", "=Seattle")), tupleValue(ImmutableMap.of("city", ",,Seattle")))); - String expected = "city\n" - + "=Seattle\n" - + "\",,Seattle\""; - assertEquals(expected, escapeFormatter.format(response)); + String expected = "city%n" + + "=Seattle%n" + + "\"||Seattle\""; + assertEquals(format(expected), escapeFormatter.format(response)); } @Test @@ -122,11 +123,11 @@ void replaceNullValues() { ImmutableMap.of("firstname", LITERAL_NULL, "city", stringValue("Seattle"))), ExprTupleValue.fromExprValueMap( ImmutableMap.of("firstname", stringValue("John"), "city", LITERAL_MISSING)))); - String expected = "name,city\n" - + "John,Seattle\n" - + ",Seattle\n" - + "John,"; - assertEquals(expected, formatter.format(response)); + String expected = "name|city%n" + + "John|Seattle%n" + + "|Seattle%n" + + "John|"; + assertEquals(format(expected), formatter.format(response)); } } diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java index 87d2d6f57fc..63e325a83d6 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java @@ -7,6 +7,7 @@ package org.opensearch.sql.protocol.response.format; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.common.utils.StringUtils.format; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; @@ -36,8 +37,8 @@ void formatResponse() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("name", "John", "age", 20)), tupleValue(ImmutableMap.of("name", "Smith", "age", 30)))); - String expected = "name|age\nJohn|20\nSmith|30"; - assertEquals(expected, rawFormater.format(response)); + String expected = "name|age%nJohn|20%nSmith|30"; + assertEquals(format(expected), rawFormater.format(response)); } @Test @@ -50,9 +51,9 @@ void sanitizeHeaders() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of( "=firstname", "John", "+lastname", "Smith", "-city", "Seattle", "@age", 20)))); - String expected = "=firstname|+lastname|-city|@age\n" + String expected = "=firstname|+lastname|-city|@age%n" + "John|Smith|Seattle|20"; - assertEquals(expected, rawFormater.format(response)); + assertEquals(format(expected), rawFormater.format(response)); } @Test @@ -66,14 +67,14 @@ void sanitizeData() { tupleValue(ImmutableMap.of("city", "-Seattle")), tupleValue(ImmutableMap.of("city", "@Seattle")), tupleValue(ImmutableMap.of("city", "Seattle=")))); - String expected = "city\n" - + "Seattle\n" - + "=Seattle\n" - + "+Seattle\n" - + "-Seattle\n" - + "@Seattle\n" + String expected = "city%n" + + "Seattle%n" + + "=Seattle%n" + + "+Seattle%n" + + "-Seattle%n" + + "@Seattle%n" + "Seattle="; - assertEquals(expected, rawFormater.format(response)); + assertEquals(format(expected), rawFormater.format(response)); } @Test @@ -83,9 +84,9 @@ void quoteIfRequired() { new ExecutionEngine.Schema.Column("||age", "||age", INTEGER))); QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("na|me", "John|Smith", "||age", "30|||")))); - String expected = "\"na|me\"|\"||age\"\n" - + "\"John|Smith\"|\"30|||\""; - assertEquals(expected, rawFormater.format(response)); + String expected = "\"na|me\"|\"||age\"%n" + + "\"John|Smith\"|\"30|||\""; + assertEquals(format(expected), rawFormater.format(response)); } @Test @@ -104,10 +105,10 @@ void escapeSanitize() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("city", "=Seattle")), tupleValue(ImmutableMap.of("city", "||Seattle")))); - String expected = "city\n" - + "=Seattle\n" - + "\"||Seattle\""; - assertEquals(expected, escapeFormatter.format(response)); + String expected = "city%n" + + "=Seattle%n" + + "\"||Seattle\""; + assertEquals(format(expected), escapeFormatter.format(response)); } @Test @@ -117,10 +118,10 @@ void senstiveCharater() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("city", "@Seattle")), tupleValue(ImmutableMap.of("city", "++Seattle")))); - String expected = "city\n" - + "@Seattle\n" - + "++Seattle"; - assertEquals(expected, rawFormater.format(response)); + String expected = "city%n" + + "@Seattle%n" + + "++Seattle"; + assertEquals(format(expected), rawFormater.format(response)); } @Test @@ -131,10 +132,10 @@ void senstiveCharaterWithSanitize() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("city", "@Seattle")), tupleValue(ImmutableMap.of("city", "++Seattle|||")))); - String expected = "city\n" - + "@Seattle\n" - + "\"++Seattle|||\""; - assertEquals(expected, testFormater.format(response)); + String expected = "city%n" + + "@Seattle%n" + + "\"++Seattle|||\""; + assertEquals(format(expected), testFormater.format(response)); } @Test @@ -148,11 +149,11 @@ void replaceNullValues() { ImmutableMap.of("firstname", LITERAL_NULL, "city", stringValue("Seattle"))), ExprTupleValue.fromExprValueMap( ImmutableMap.of("firstname", stringValue("John"), "city", LITERAL_MISSING)))); - String expected = "name|city\n" - + "John|Seattle\n" - + "|Seattle\n" + String expected = "name|city%n" + + "John|Seattle%n" + + "|Seattle%n" + "John|"; - assertEquals(expected, rawFormater.format(response)); + assertEquals(format(expected), rawFormater.format(response)); } } diff --git a/workbench/package.json b/workbench/package.json index 8bf4b6ff98c..1e5c20f6467 100644 --- a/workbench/package.json +++ b/workbench/package.json @@ -16,7 +16,7 @@ "start": "plugin-helpers start", "test:server": "plugin-helpers test:server", "test:browser": "plugin-helpers test:browser", - "test:jest": "NODE_PATH=../../node_modules ../../node_modules/.bin/jest --config ./test/jest.config.js", + "test:jest": "../../node_modules/.bin/jest --config ./test/jest.config.js", "build": "yarn plugin_helpers build", "plugin_helpers": "node ../../scripts/plugin_helpers" }, From 8aeeab2f2afa46201298d9dbc97dcca70821aa36 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Mon, 5 Dec 2022 19:19:11 -0800 Subject: [PATCH 2/6] fix build issue Signed-off-by: Peng Huo --- .github/workflows/sql-test-and-build-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 73586d5da49..31d168d33b8 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -44,7 +44,7 @@ jobs: java-version: ${{ matrix.entry.java }} - name: Build with Gradle - run: ./gradlew --continue build ${{ matrix.entry.os_build_args }} -Dopensearch.version=${{ env.OPENSEARCH_VERSION }} + run: ./gradlew --continue build ${{ matrix.entry.os_build_args }} - name: Run backward compatibility tests if: ${{ matrix.entry.os == 'ubuntu-latest' }} From c75d8e9b39ed0780c4cc9137ff81c1e5f32f7d54 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Mon, 5 Dec 2022 19:44:03 -0800 Subject: [PATCH 3/6] update failed test Signed-off-by: Peng Huo --- .../workflows/sql-test-and-build-workflow.yml | 3 +- .../format/CsvResponseFormatterTest.java | 28 +++++++++---------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 31d168d33b8..ad5e860b196 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -39,8 +39,9 @@ jobs: - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.entry.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: + distribution: 'temurin' java-version: ${{ matrix.entry.java }} - name: Build with Gradle diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java index 949781ce9f3..7008b51fa60 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java @@ -38,7 +38,7 @@ void formatResponse() { tupleValue(ImmutableMap.of("name", "John", "age", 20)), tupleValue(ImmutableMap.of("name", "Smith", "age", 30)))); CsvResponseFormatter formatter = new CsvResponseFormatter(); - String expected = "name|age%nJohn|20%nSmith|30"; + String expected = "name,age%nJohn,20%nSmith,30"; assertEquals(format(expected), formatter.format(response)); } @@ -52,8 +52,8 @@ void sanitizeHeaders() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of( "=firstname", "John", "+lastname", "Smith", "-city", "Seattle", "@age", 20)))); - String expected = "=firstname|+lastname|-city|@age%n" - + "John|Smith|Seattle|20"; + String expected = "'=firstname,'+lastname,'-city,'@age%n" + + "John,Smith,Seattle,20"; assertEquals(format(expected), formatter.format(response)); } @@ -70,10 +70,10 @@ void sanitizeData() { tupleValue(ImmutableMap.of("city", "Seattle=")))); String expected = "city%n" + "Seattle%n" - + "=Seattle%n" - + "+Seattle%n" - + "-Seattle%n" - + "@Seattle%n" + + "'=Seattle%n" + + "'+Seattle%n" + + "'-Seattle%n" + + "'@Seattle%n" + "Seattle="; assertEquals(format(expected), formatter.format(response)); } @@ -85,8 +85,8 @@ void quoteIfRequired() { new ExecutionEngine.Schema.Column(",,age", ",,age", INTEGER))); QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("na,me", "John,Smith", ",,age", "30,,,")))); - String expected = "\"na|me\"|\"||age\"%n" - + "\"John|Smith\"|\"30|||\""; + String expected = "\"na,me\",\",,age\"%n" + + "\"John,Smith\",\"30,,,\""; assertEquals(format(expected), formatter.format(response)); } @@ -108,7 +108,7 @@ void escapeSanitize() { tupleValue(ImmutableMap.of("city", ",,Seattle")))); String expected = "city%n" + "=Seattle%n" - + "\"||Seattle\""; + + "\",,Seattle\""; assertEquals(format(expected), escapeFormatter.format(response)); } @@ -123,10 +123,10 @@ void replaceNullValues() { ImmutableMap.of("firstname", LITERAL_NULL, "city", stringValue("Seattle"))), ExprTupleValue.fromExprValueMap( ImmutableMap.of("firstname", stringValue("John"), "city", LITERAL_MISSING)))); - String expected = "name|city%n" - + "John|Seattle%n" - + "|Seattle%n" - + "John|"; + String expected = "name,city%n" + + "John,Seattle%n" + + ",Seattle%n" + + "John,"; assertEquals(format(expected), formatter.format(response)); } From 7e0f6d4db60683b192f67bf593133eb05cccb3cc Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Mon, 5 Dec 2022 19:59:10 -0800 Subject: [PATCH 4/6] fix compile issue Signed-off-by: Peng Huo --- .github/workflows/sql-test-and-build-workflow.yml | 1 - .../java/org/opensearch/sql/legacy/domain/Condition.java | 2 +- .../main/java/org/opensearch/sql/legacy/domain/Field.java | 2 +- .../java/org/opensearch/sql/legacy/domain/MethodField.java | 2 +- .../main/java/org/opensearch/sql/legacy/domain/Order.java | 2 +- .../java/org/opensearch/sql/legacy/domain/SearchResult.java | 6 +++--- .../main/java/org/opensearch/sql/legacy/domain/Select.java | 2 +- .../java/org/opensearch/sql/legacy/parser/FieldMaker.java | 2 +- .../org/opensearch/sql/legacy/query/maker/AggMaker.java | 2 +- .../java/org/opensearch/sql/legacy/query/maker/Maker.java | 2 +- .../org/opensearch/sql/legacy/query/maker/QueryMaker.java | 4 ++-- 11 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index ad5e860b196..324b3214d68 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -41,7 +41,6 @@ jobs: - name: Set up JDK ${{ matrix.entry.java }} uses: actions/setup-java@v3 with: - distribution: 'temurin' java-version: ${{ matrix.entry.java }} - name: Build with Gradle diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Condition.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Condition.java index 793fb271a62..a43fcef2248 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Condition.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Condition.java @@ -18,7 +18,7 @@ import org.opensearch.sql.legacy.utils.StringUtils; /** - * 过滤条件 + * Filter Condition. * * @author ansj */ diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Field.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Field.java index 856f5905109..39538513033 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Field.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Field.java @@ -13,7 +13,7 @@ import org.opensearch.sql.legacy.parser.NestedType; /** - * 搜索域 + * Search Scope. * * @author ansj */ diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/MethodField.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/MethodField.java index 62eff626c44..09df7bce14c 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/MethodField.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/MethodField.java @@ -14,7 +14,7 @@ import org.opensearch.sql.legacy.utils.Util; /** - * 搜索域 + * Search Scope. * * @author ansj */ diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Order.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Order.java index 2aee7cdabf3..8fffc735688 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Order.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Order.java @@ -7,7 +7,7 @@ package org.opensearch.sql.legacy.domain; /** - * 排序规则 + * Ordering. * * @author ansj */ diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/SearchResult.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/SearchResult.java index 655da9848c1..30d0c4157b0 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/SearchResult.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/SearchResult.java @@ -30,7 +30,7 @@ public class SearchResult { /** - * 查询结果 + * Search Result. */ private List> results; @@ -82,7 +82,7 @@ public SearchResult(SearchResponse resp, Select select) throws SqlParseException } /** - * 讲es的field域转换为你Object + * Rewrite field to Object. * * @param fields * @return @@ -101,7 +101,7 @@ private Map toFieldsMap(Map fields) { } /** - * 讲es的field域转换为你Object + * Rewrite field to Object. * * @param fields * @return diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Select.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Select.java index 485efc1a2b5..7372afda88e 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/Select.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/Select.java @@ -18,7 +18,7 @@ /** - * 将sql语句转换为select 对象 + * Rewrite SQL statement to select object. * * @author ansj */ diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/parser/FieldMaker.java b/legacy/src/main/java/org/opensearch/sql/legacy/parser/FieldMaker.java index 0f5a2af631d..eb59fac083f 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/parser/FieldMaker.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/parser/FieldMaker.java @@ -41,7 +41,7 @@ import org.opensearch.sql.legacy.utils.Util; /** - * 一些具有参数的一般在 select 函数.或者group by 函数 + * FieldMaker. * * @author ansj */ diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/AggMaker.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/AggMaker.java index b56692e4537..ae6a5abbcdf 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/AggMaker.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/AggMaker.java @@ -71,7 +71,7 @@ public class AggMaker { private Where where; /** - * 分组查的聚合函数 + * group aggregation. * * @param field * @return diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/Maker.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/Maker.java index a3040c9f44b..93c638d5826 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/Maker.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/Maker.java @@ -93,7 +93,7 @@ protected Maker(Boolean isQuery) { } /** - * 构建过滤条件 + * Construct filter. * * @param cond * @return diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/QueryMaker.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/QueryMaker.java index d6d2f1e58e5..cd925bf691d 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/QueryMaker.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/QueryMaker.java @@ -19,7 +19,7 @@ public class QueryMaker extends Maker { /** - * 将where条件构建成query + * Rewrite where condition to query. * * @param where * @return @@ -58,7 +58,7 @@ private void explanWhere(BoolQueryBuilder boolQuery, Where where) throws SqlPars } /** - * 增加嵌套插 + * Add sub query. * * @param boolQuery * @param where From fd1101ec3d4a367e37e94c5375ad40bdc7a44500 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Mon, 5 Dec 2022 20:01:58 -0800 Subject: [PATCH 5/6] use v1 Signed-off-by: Peng Huo --- .github/workflows/sql-test-and-build-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 324b3214d68..31d168d33b8 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -39,7 +39,7 @@ jobs: - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.entry.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v1 with: java-version: ${{ matrix.entry.java }} From 1da3904e9a5119a29db876f28cc5ce0a77a1109d Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Mon, 5 Dec 2022 20:33:26 -0800 Subject: [PATCH 6/6] delete doctest Signed-off-by: Peng Huo --- integ-test/build.gradle | 22 -- .../sql/doctest/admin/MonitoringIT.java | 76 ----- .../sql/doctest/beyond/FullTextIT.java | 137 -------- .../sql/doctest/beyond/PartiQLIT.java | 145 -------- .../opensearch/sql/doctest/core/DocTest.java | 136 -------- .../opensearch/sql/doctest/core/Template.java | 49 --- .../opensearch/sql/doctest/core/TestData.java | 72 ---- .../core/annotation/DocTestConfig.java | 36 -- .../sql/doctest/core/annotation/Section.java | 28 -- .../sql/doctest/core/builder/Body.java | 32 -- .../sql/doctest/core/builder/DocBuilder.java | 227 ------------- .../sql/doctest/core/builder/Example.java | 104 ------ .../sql/doctest/core/builder/Formats.java | 41 --- .../sql/doctest/core/builder/ListItems.java | 31 -- .../sql/doctest/core/builder/Requests.java | 52 --- .../sql/doctest/core/markup/Document.java | 34 -- .../sql/doctest/core/markup/RstDocument.java | 101 ------ .../sql/doctest/core/request/SqlRequest.java | 98 ------ .../core/request/SqlRequestFormat.java | 125 ------- .../sql/doctest/core/response/DataTable.java | 83 ----- .../doctest/core/response/SqlResponse.java | 70 ---- .../core/response/SqlResponseFormat.java | 149 --------- .../sql/doctest/core/test/DataTableTest.java | 54 --- .../sql/doctest/core/test/DocBuilderTest.java | 229 ------------- .../sql/doctest/core/test/DocTestTests.java | 26 -- .../doctest/core/test/RstDocumentTest.java | 126 ------- .../core/test/SqlRequestFormatTest.java | 78 ----- .../sql/doctest/core/test/SqlRequestTest.java | 65 ---- .../core/test/SqlResponseFormatTest.java | 177 ---------- .../doctest/core/test/SqlResponseTest.java | 39 --- .../opensearch/sql/doctest/dml/DeleteIT.java | 44 --- .../sql/doctest/dql/BasicQueryIT.java | 310 ------------------ .../sql/doctest/dql/ComplexQueryIT.java | 192 ----------- .../sql/doctest/dql/MetaDataQueryIT.java | 66 ---- .../sql/doctest/dql/SQLFunctionsIT.java | 48 --- .../sql/doctest/interfaces/EndpointIT.java | 75 ----- .../sql/doctest/interfaces/ProtocolIT.java | 135 -------- 37 files changed, 3512 deletions(-) delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/admin/MonitoringIT.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/beyond/FullTextIT.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/beyond/PartiQLIT.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/DocTest.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/Template.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/TestData.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/annotation/DocTestConfig.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/annotation/Section.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Body.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/DocBuilder.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Example.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Formats.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/ListItems.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Requests.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/markup/Document.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/markup/RstDocument.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/request/SqlRequest.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/request/SqlRequestFormat.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/response/DataTable.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/response/SqlResponse.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/response/SqlResponseFormat.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DataTableTest.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DocBuilderTest.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DocTestTests.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/test/RstDocumentTest.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlRequestFormatTest.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlRequestTest.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlResponseFormatTest.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlResponseTest.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/dml/DeleteIT.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/dql/BasicQueryIT.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/dql/ComplexQueryIT.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/dql/MetaDataQueryIT.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/dql/SQLFunctionsIT.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/interfaces/EndpointIT.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/doctest/interfaces/ProtocolIT.java diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 3a828d8677e..d273442c9fa 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -140,28 +140,6 @@ integTest { exclude 'org/opensearch/sql/legacy/OrderIT.class' } - -task docTest(type: RestIntegTestTask) { - dependsOn ':plugin:bundlePlugin' - - systemProperty 'tests.security.manager', 'false' - systemProperty('project.root', project.projectDir.absolutePath) - - // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for - // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time. - doFirst { systemProperty 'cluster.debug', getDebug() } - - if (System.getProperty("test.debug") != null) { - jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' - } - - include 'org/opensearch/sql/doctest/**/*IT.class' - exclude 'org/opensearch/sql/correctness/**/*IT.class' - exclude 'org/opensearch/sql/ppl/**/*IT.class' - exclude 'org/opensearch/sql/sql/**/*IT.class' - exclude 'org/opensearch/sql/legacy/**/*IT.class' -} - task comparisonTest(type: RestIntegTestTask) { dependsOn ':plugin:bundlePlugin' diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/admin/MonitoringIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/admin/MonitoringIT.java deleted file mode 100644 index e391379244e..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/admin/MonitoringIT.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.admin; - -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.CURL_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; -import static org.opensearch.sql.legacy.metrics.MetricName.DEFAULT_CURSOR_REQUEST_COUNT_TOTAL; -import static org.opensearch.sql.legacy.metrics.MetricName.DEFAULT_CURSOR_REQUEST_TOTAL; -import static org.opensearch.sql.legacy.metrics.MetricName.FAILED_REQ_COUNT_CB; -import static org.opensearch.sql.legacy.metrics.MetricName.FAILED_REQ_COUNT_CUS; -import static org.opensearch.sql.legacy.metrics.MetricName.FAILED_REQ_COUNT_SYS; -import static org.opensearch.sql.legacy.metrics.MetricName.REQ_COUNT_TOTAL; -import static org.opensearch.sql.legacy.metrics.MetricName.REQ_TOTAL; -import static org.opensearch.sql.legacy.plugin.RestSqlStatsAction.STATS_API_ENDPOINT; - -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; -import org.opensearch.sql.doctest.core.builder.Requests; -import org.opensearch.sql.doctest.core.request.SqlRequest; -import org.opensearch.sql.doctest.core.response.DataTable; -import org.opensearch.sql.legacy.metrics.MetricName; - -/** - * Doc test for plugin monitoring functionality - */ -@DocTestConfig(template = "admin/monitoring.rst") -public class MonitoringIT extends DocTest { - - @Section - public void nodeStats() { - section( - title("Node Stats"), - description( - "The meaning of fields in the response is as follows:\n\n" + fieldDescriptions()), - example( - description(), - getStats(), - queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - - private String fieldDescriptions() { - DataTable table = new DataTable(new String[] {"Field name", "Description"}); - table.addRow(row(REQ_TOTAL, "Total count of request")); - table.addRow(row(REQ_COUNT_TOTAL, "Total count of request within the interval")); - table.addRow(row(DEFAULT_CURSOR_REQUEST_TOTAL, "Total count of simple cursor request")); - table.addRow(row(DEFAULT_CURSOR_REQUEST_COUNT_TOTAL, - "Total count of simple cursor request within the interval")); - table.addRow(row(FAILED_REQ_COUNT_SYS, - "Count of failed request due to system error within the interval")); - table.addRow(row(FAILED_REQ_COUNT_CUS, - "Count of failed request due to bad request within the interval")); - table.addRow( - row(FAILED_REQ_COUNT_CB, "Indicate if plugin is being circuit broken within the interval")); - - return table.toString(); - } - - private String[] row(MetricName name, String description) { - return new String[] {name.getName(), description}; - } - - private Requests getStats() { - return new Requests(restClient(), new SqlRequest("GET", STATS_API_ENDPOINT, "")); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/beyond/FullTextIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/beyond/FullTextIT.java deleted file mode 100644 index ded89f9ab29..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/beyond/FullTextIT.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.beyond; - -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; - -@DocTestConfig(template = "beyond/fulltext.rst", testData = {"accounts.json"}) -public class FullTextIT extends DocTest { - - @Section(1) - public void matchQuery() { - section( - title("Match Query"), - description( - "Match query is the standard query for full-text search in OpenSearch. Both ``MATCHQUERY`` and", - "``MATCH_QUERY`` are functions for performing match query." - ), - example( - description( - "Both functions can accept field name as first argument and a text as second argument."), - post(multiLine( - "SELECT account_number, address", - "FROM accounts", - "WHERE MATCH_QUERY(address, 'Holmes')" - )) - ), - example( - description( - "Both functions can also accept single argument and be used in the following manner."), - post(multiLine( - "SELECT account_number, address", - "FROM accounts", - "WHERE address = MATCH_QUERY('Holmes')" - )) - ) - ); - } - - @Section(2) - public void multiMatchQuery() { - section( - title("Multi-match Query"), - description( - "Besides match query against a single field, you can search for a text with multiple fields.", - "Function ``MULTI_MATCH``, ``MULTIMATCH`` and ``MULTIMATCHQUERY`` are provided for this." - ), - example( - description( - "Each preceding function accepts ``query`` for a text and ``fields`` for field names or pattern", - "that the text given is searched against. For example, the following query is searching for", - "documents in index accounts with 'Dale' as either firstname or lastname." - ), - post(multiLine( - "SELECT firstname, lastname", - "FROM accounts", - "WHERE MULTI_MATCH('query'='Dale', 'fields'='*name')" - )) - ) - ); - } - - @Section(3) - public void queryStringQuery() { - section( - title("Query String Query"), - description( - "Query string query parses and splits a query string provided based on Lucene query string syntax.", - "The mini language supports logical connectives, wildcard, regex and proximity search. Please refer", - "to official documentation for more details. Note that an error is thrown in the case of any invalid", - "syntax in query string." - ), - example( - description( - "``QUERY`` function accepts query string and returns true or false respectively for document", - "that matches the query string or not." - ), - post(multiLine( - "SELECT account_number, address", - "FROM accounts", - "WHERE QUERY('address:Lane OR address:Street')" - )) - ) - ); - } - - @Section(4) - public void matchPhraseQuery() { - section( - title("Match Phrase Query"), - description( - "Match phrase query is similar to match query but it is used for matching exact phrases.", - "``MATCHPHRASE``, ``MATCH_PHRASE`` and ``MATCHPHRASEQUERY`` are provided for this purpose." - ), - example( - description(), - post(multiLine( - "SELECT account_number, address", - "FROM accounts", - "WHERE MATCH_PHRASE(address, '880 Holmes Lane')" - )) - ) - ); - } - - @Section(5) - public void scoreQuery() { - section( - title("Score Query"), - description( - "OpenSearch supports to wrap a filter query so as to return a relevance score along with", - "every matching document. ``SCORE``, ``SCOREQUERY`` and ``SCORE_QUERY`` can be used for this." - ), - example( - description( - "The first argument is a match query expression and the second argument is for an optional", - "floating point number to boost the score. The default value is 1.0. Apart from this, an", - "implicit variable ``_score`` is available so you can return score for each document or", - "use it for sorting." - ), - post(multiLine( - "SELECT account_number, address, _score", - "FROM accounts", - "WHERE SCORE(MATCH_QUERY(address, 'Lane'), 0.5) OR", - " SCORE(MATCH_QUERY(address, 'Street'), 100)", - "ORDER BY _score" - )) - ) - ); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/beyond/PartiQLIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/beyond/PartiQLIT.java deleted file mode 100644 index dfce25e0ec7..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/beyond/PartiQLIT.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.beyond; - -import static org.opensearch.sql.doctest.core.TestData.TEST_DATA_FOLDER_ROOT; -import static org.opensearch.sql.util.TestUtils.getResourceFilePath; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; -import org.opensearch.sql.doctest.core.builder.Example; -import org.opensearch.sql.legacy.utils.JsonPrettyFormatter; - -@DocTestConfig(template = "beyond/partiql.rst", testData = {"employees_nested.json"}) -public class PartiQLIT extends DocTest { - - @Section(1) - public void showTestData() { - section( - title("Test Data"), - description( - "The test index ``employees_nested`` used by all examples in this document is very similar to", - "the one used in official PartiQL documentation." - ), - createDummyExampleForTestData("employees_nested.json") - ); - } - - @Section(2) - public void queryNestedCollection() { - section( - title("Querying Nested Collection"), - description( - "In SQL-92, a database table can only have tuples that consists of scalar values.", - "PartiQL extends SQL-92 to allow you query and unnest nested collection conveniently.", - "In OpenSearch world, this is very useful for index with object or nested field." - ), - example( - title("Unnesting a Nested Collection"), - description( - "In the following example, it finds nested document (project) with field value (name)", - "that satisfies the predicate (contains 'security'). Note that because each parent document", - "can have more than one nested documents, the matched nested document is flattened. In other", - "word, the final result is the Cartesian Product between parent and nested documents." - ), - post(multiLine( - "SELECT e.name AS employeeName,", - " p.name AS projectName", - "FROM employees_nested AS e,", - " e.projects AS p", - "WHERE p.name LIKE '%security%'" - )) - ), - /* - Issue: https://github.com/opendistro-for-elasticsearch/sql/issues/397 - example( - title("Preserving Parent Information with LEFT JOIN"), - description( - "The query in the preceding example is very similar to traditional join queries, except ``ON`` clause missing.", - "This is because it is implicitly in the nesting of nested documents (projects) into parent (employee). Therefore,", - "you can use ``LEFT JOIN`` to preserve the information in parent document associated." - ), - post( - "SELECT e.id AS id, " + - " e.name AS employeeName, " + - " e.title AS title, " + - " p.name AS projectName " + - "FROM employees_nested AS e " + - "LEFT JOIN e.projects AS p" - ) - )*/ - example( - title("Unnesting in Existential Subquery"), - description( - "Alternatively, a nested collection can be unnested in subquery to check if it", - "satisfies a condition." - ), - post(multiLine( - "SELECT e.name AS employeeName", - "FROM employees_nested AS e", - "WHERE EXISTS (", - " SELECT *", - " FROM e.projects AS p", - " WHERE p.name LIKE '%security%'", - ")" - )) - )/*, - Issue: https://github.com/opendistro-for-elasticsearch/sql/issues/398 - example( - title("Aggregating over a Nested Collection"), - description( - "After unnested, a nested collection can be aggregated just like a regular field." - ), - post(multiLine( - "SELECT", - " e.name AS employeeName,", - " COUNT(p) AS cnt", - "FROM employees_nested AS e,", - " e.projects AS p", - "WHERE p.name LIKE '%security%'", - "GROUP BY e.id, e.name", - "HAVING COUNT(p) >= 1" - ) - )) - */ - ); - } - - private Example createDummyExampleForTestData(String fileName) { - Example example = new Example(); - example.setTitle("Employees"); - example.setDescription(""); - example.setResult(parseJsonFromTestData(fileName)); - return example; - } - - /** - * Concat and pretty format document at odd number line in bulk request file - */ - private String parseJsonFromTestData(String fileName) { - Path path = Paths.get(getResourceFilePath(TEST_DATA_FOLDER_ROOT + fileName)); - try { - List lines = Files.readAllLines(path); - String json = IntStream.range(0, lines.size()). - filter(i -> i % 2 == 1). - mapToObj(lines::get). - collect(Collectors.joining(",", "{\"employees\":[", "]}")); - return JsonPrettyFormatter.format(json); - } catch (IOException e) { - throw new IllegalStateException("Failed to load test data: " + path, e); - } - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/DocTest.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/DocTest.java deleted file mode 100644 index afce22c0f38..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/DocTest.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core; - -import static com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope; -import static java.nio.file.StandardOpenOption.APPEND; -import static org.opensearch.test.OpenSearchIntegTestCase.Scope.SUITE; - -import com.carrotsearch.randomizedtesting.AnnotatedMethodProvider; -import com.carrotsearch.randomizedtesting.TestMethodAndParams; -import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; -import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import java.io.IOException; -import java.io.PrintWriter; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Comparator; -import org.opensearch.client.RestClient; -import org.opensearch.common.Strings; -import org.opensearch.common.transport.TransportAddress; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; -import org.opensearch.sql.doctest.core.builder.DocBuilder; -import org.opensearch.sql.doctest.core.markup.Document; -import org.opensearch.sql.doctest.core.markup.RstDocument; -import org.opensearch.sql.legacy.CustomExternalTestCluster; -import org.opensearch.sql.legacy.TestUtils; -import org.opensearch.test.OpenSearchIntegTestCase; -import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; -import org.opensearch.test.TestCluster; - -/** - * Documentation test base class - */ -@TestMethodProviders({DocTest.SectionMethod.class}) -@TestCaseOrdering(DocTest.SectionOrder.class) -@OpenSearchIntegTestCase.SuiteScopeTestCase -@ClusterScope(scope = SUITE, numDataNodes = 1, supportsDedicatedMasters = false, transportClientRatio = 1) -@ThreadLeakScope(Scope.NONE) -public abstract class DocTest extends OpenSearchIntegTestCase implements DocBuilder { - - @Override - protected void setupSuiteScopeCluster() { - DocTestConfig config = getClass().getAnnotation(DocTestConfig.class); - loadTestData(config); - copyTemplateToDocument(config); - } - - @Override - public RestClient restClient() { - return getRestClient(); - } - - @Override - public Document openDocument() { - DocTestConfig config = getClass().getAnnotation(DocTestConfig.class); - Path docPath = absolutePath(config.template()); - try { - PrintWriter docWriter = new PrintWriter(Files.newBufferedWriter(docPath, APPEND)); - return new RstDocument(docWriter); - } catch (IOException e) { - throw new IllegalStateException("Failed to open document file " + docPath, e); - } - } - - private void loadTestData(DocTestConfig config) { - String[] testFilePaths = config.testData(); - TestData testData = new TestData(testFilePaths); - testData.loadToES(this); - } - - private void copyTemplateToDocument(DocTestConfig config) { - Path docPath = absolutePath(config.template()); - Template template = new Template(config.template()); - template.copyToDocument(docPath); - } - - /** - * Method annotated by {@link Section} will be treated as test method. - */ - public static class SectionMethod extends AnnotatedMethodProvider { - public SectionMethod() { - super(Section.class); - } - } - - /** - * Test methods will execute in order defined by value in {@link Section} annotation. - */ - public static class SectionOrder implements Comparator { - @Override - public int compare(TestMethodAndParams method1, TestMethodAndParams method2) { - return Integer.compare(order(method1), order(method2)); - } - - private int order(TestMethodAndParams method) { - Section section = method.getTestMethod().getAnnotation(Section.class); - return section.value(); - } - } - - private Path absolutePath(String templateRelativePath) { - return Paths.get(TestUtils.getResourceFilePath(DOCUMENT_FOLDER_ROOT + templateRelativePath)); - } - - @Override - protected TestCluster buildTestCluster(Scope scope, long seed) throws IOException { - - String clusterAddresses = System.getProperty(TESTS_CLUSTER); - - if (Strings.hasLength(clusterAddresses)) { - String[] stringAddresses = clusterAddresses.split(","); - TransportAddress[] transportAddresses = new TransportAddress[stringAddresses.length]; - int i = 0; - for (String stringAddress : stringAddresses) { - URL url = new URL("http://" + stringAddress); - InetAddress inetAddress = InetAddress.getByName(url.getHost()); - transportAddresses[i++] = - new TransportAddress(new InetSocketAddress(inetAddress, url.getPort())); - } - return new CustomExternalTestCluster(createTempDir(), externalClusterClientSettings(), - transportClientPlugins(), transportAddresses); - } - return super.buildTestCluster(scope, seed); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/Template.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/Template.java deleted file mode 100644 index c84a2c1a11e..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/Template.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core; - -import static java.nio.file.StandardCopyOption.COPY_ATTRIBUTES; -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import org.opensearch.sql.legacy.utils.StringUtils; -import org.opensearch.sql.util.TestUtils; - -/** - * Abstraction for document template file - */ -public class Template { - - private static final String TEMPLATE_FOLDER_ROOT = "src/test/resources/doctest/templates/"; - - private final Path templateFullPath; - - public Template(String templateRelativePath) { - this.templateFullPath = - Paths.get(TestUtils.getResourceFilePath(TEMPLATE_FOLDER_ROOT + templateRelativePath)); - } - - /** - * Copy template file to target document. Replace it if existing. - * - * @param docFullPath full path of target document - */ - public void copyToDocument(Path docFullPath) { - try { - Files.createDirectories(docFullPath.getParent()); - Files.copy(templateFullPath, docFullPath, REPLACE_EXISTING, COPY_ATTRIBUTES); - } catch (IOException e) { - throw new IllegalStateException(StringUtils.format( - "Failed to copy from template [%s] to document file [%s]", templateFullPath, docFullPath), - e); - } - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/TestData.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/TestData.java deleted file mode 100644 index 962ce1ecfff..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/TestData.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core; - -import static org.opensearch.sql.util.TestUtils.createIndexByRestClient; -import static org.opensearch.sql.util.TestUtils.getResourceFilePath; -import static org.opensearch.sql.util.TestUtils.loadDataByRestClient; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import org.opensearch.sql.legacy.utils.StringUtils; - -/** - * Test data for document generation - */ -public class TestData { - - public static final String MAPPINGS_FOLDER_ROOT = "src/test/resources/doctest/mappings/"; - public static final String TEST_DATA_FOLDER_ROOT = "src/test/resources/doctest/testdata/"; - - private final String[] testFilePaths; - - public TestData(String[] testFilePaths) { - this.testFilePaths = testFilePaths; - } - - /** - * Load test data in file to Elaticsearch cluster via client. - * - * @param test current test instance - */ - public void loadToES(DocTest test) { - for (String filePath : testFilePaths) { - String indexName = indexName(filePath); - try { - createIndexByRestClient(test.restClient(), indexName, getIndexMapping(filePath)); - loadDataByRestClient(test.restClient(), indexName, TEST_DATA_FOLDER_ROOT + filePath); - } catch (Exception e) { - throw new IllegalStateException(StringUtils.format( - "Failed to load mapping and test filePath from %s", filePath), e); - } - test.ensureGreen(indexName); - } - } - - /** - * Use file name (without file extension) as index name implicitly. - * For example, for 'testdata/accounts.json', 'accounts' will be used. - */ - private String indexName(String filePath) { - return filePath.substring( - filePath.lastIndexOf(File.separatorChar) + 1, - filePath.lastIndexOf('.') - ); - } - - private String getIndexMapping(String filePath) throws IOException { - Path path = Paths.get(getResourceFilePath(MAPPINGS_FOLDER_ROOT + filePath)); - if (Files.notExists(path)) { - return ""; - } - return new String(Files.readAllBytes(path)); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/annotation/DocTestConfig.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/annotation/DocTestConfig.java deleted file mode 100644 index 0a2ddf8a2d1..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/annotation/DocTestConfig.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.annotation; - -import static java.lang.annotation.ElementType.TYPE; -import static java.lang.annotation.RetentionPolicy.RUNTIME; - -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - -/** - * Configuration to initializing the set up for a doc test. - */ -@Retention(RUNTIME) -@Target(value = TYPE) -public @interface DocTestConfig { - - /** - * Path of the template - * - * @return path - */ - String template(); - - /** - * Path of the test data used. - * - * @return path - */ - String[] testData() default {}; - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/annotation/Section.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/annotation/Section.java deleted file mode 100644 index 215542c36e7..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/annotation/Section.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.annotation; - -import static java.lang.annotation.ElementType.METHOD; -import static java.lang.annotation.RetentionPolicy.RUNTIME; - -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - -/** - * This annotation is used to indicate current method is a valid method for doc generation - * and it is supposed to run in the specified order. - */ -@Retention(RUNTIME) -@Target(value = METHOD) -public @interface Section { - - /** - * @return section order - */ - int value() default 0; - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Body.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Body.java deleted file mode 100644 index 45da0618df3..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Body.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.builder; - -import java.util.Arrays; -import java.util.stream.Collectors; - -/** - * Request body. - */ -class Body { - - private final String[] fieldValues; - - /** - * Request body built from field value pairs. - * - * @param fieldValues field and values in "'field": 'value'" format that can assemble to JSON directly - */ - Body(String... fieldValues) { - this.fieldValues = fieldValues; - } - - @Override - public String toString() { - return Arrays.stream(fieldValues).collect(Collectors.joining(",", "{", "}")); - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/DocBuilder.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/DocBuilder.java deleted file mode 100644 index 1c6361fa65b..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/DocBuilder.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.builder; - -import static org.opensearch.sql.doctest.core.request.SqlRequest.UrlParam; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.OPENSEARCH_DASHBOARD_REQUEST; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.TABLE_RESPONSE; -import static org.opensearch.sql.legacy.plugin.RestSqlAction.EXPLAIN_API_ENDPOINT; -import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; -import static org.opensearch.sql.plugin.rest.RestQuerySettingsAction.LEGACY_SQL_SETTINGS_API_ENDPOINT; - - -import com.google.common.base.Strings; -import java.util.Arrays; -import org.opensearch.client.RestClient; -import org.opensearch.sql.doctest.core.markup.Document; -import org.opensearch.sql.doctest.core.request.SqlRequest; -import org.opensearch.sql.doctest.core.request.SqlRequestFormat; -import org.opensearch.sql.doctest.core.response.SqlResponseFormat; -import org.opensearch.sql.legacy.utils.StringUtils; - -/** - * Build document by custom DSL. To make it more readable, each doc test needs to implement this interface - * and provide things required, such as client connection and document handle. As benefit, they can use the - * DSL to build document in readable and fluent way. - */ -public interface DocBuilder { - - String DOCUMENT_FOLDER_ROOT = "/docs/user/"; - String IMAGE_FOLDER_PATH = DOCUMENT_FOLDER_ROOT + "img/"; - - /** - * Get client connection to cluster for sending request - * - * @return REST client - */ - RestClient restClient(); - - /** - * Open document file to write - * - * @return document class - */ - Document openDocument(); - - default void section(String title, String description, Example... examples) { - section(title, description, new String[0], examples); - } - - /** - * Entry point to start building document by DSL. - * Each section consists of: - * 1. Title - * 2. Description - * 3. Example(s) - * 3.1 Description - * 3.2 [Sample request] - * 3.3 [Explain request] - * 3.4 [Explain output] - * 3.5 [Result set] - * - * @param title title of the section - * @param description description paragraph - * @param examples examples for the section - */ - default void section(String title, String description, String[] images, Example... examples) { - try (Document document = openDocument()) { - document.section(title); - - if (!description.isEmpty()) { - document.subSection("Description").paragraph(description); - } - - if (images.length > 0) { - document.subSection("Syntax"); - for (String image : images) { - // Convert image name ex. "rdd/queryStatement.png" to "queryStatement" as description. - String imageDesc = image.substring(image.lastIndexOf('/') + 1, image.lastIndexOf('.')); - document.image("Rule ``" + imageDesc + "``", IMAGE_FOLDER_PATH + image); - } - } - - for (int i = 0; i < examples.length; i++) { - String exampleTitle; - if (examples.length > 1) { - exampleTitle = "Example " + (i + 1); - } else { - exampleTitle = "Example"; - } - if (!Strings.isNullOrEmpty(examples[i].getTitle())) { - exampleTitle += (": " + examples[i].getTitle()); - } - document.subSection(exampleTitle); - - Example example = examples[i]; - if (!example.getDescription().isEmpty()) { - document.paragraph(example.getDescription()); - } - - document.codeBlock("SQL query", example.getQuery()). - codeBlock("Explain query", example.getExplainQuery()). - codeBlock("Explain", example.getExplainResult()); - - if (example.isTable()) { - document.table("Result set", example.getResult()); - } else { - document.codeBlock("Result set", example.getResult()); - } - } - } - } - - default Example example(String description, Requests requests) { - return example("", description, requests); - } - - /** - * Construct an example by default query and explain format - */ - default Example example(String title, String description, Requests requests) { - return example(title, description, requests, - queryFormat(OPENSEARCH_DASHBOARD_REQUEST, TABLE_RESPONSE), - explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) - ); - } - - default Example example(String description, - Requests requests, - Formats queryFormat, - Formats explainFormat) { - return example("", description, requests, queryFormat, explainFormat); - } - - default Example example(String title, - String description, - Requests requests, - Formats queryFormat, - Formats explainFormat) { - Example example = new Example(); - example.setTitle(title); - example.setDescription(description); - example.setQuery(queryFormat.format(requests.query())); - example.setTable(queryFormat.isTableFormat()); - example.setResult(queryFormat.format(requests.queryResponse())); - example.setExplainQuery(explainFormat.format(requests.explain())); - example.setExplainResult(explainFormat.format(requests.explainResponse())); - return example; - } - - /** - * Simple method just for readability - */ - default String title(String title) { - return title; - } - - default String description(String... sentences) { - return String.join(" ", sentences); - } - - default String[] images(String... images) { - return images; - } - - default Formats queryFormat(SqlRequestFormat requestFormat, SqlResponseFormat responseFormat) { - return new Formats(requestFormat, responseFormat); - } - - default Formats explainFormat(SqlRequestFormat requestFormat, SqlResponseFormat responseFormat) { - return new Formats(requestFormat, responseFormat); - } - - default Requests get(String sql) { - Body body = new Body("\"query\":\"" + sql + "\""); - return new Requests( - restClient(), - new SqlRequest("GET", QUERY_API_ENDPOINT, "", new UrlParam("sql", sql)), - new SqlRequest("POST", EXPLAIN_API_ENDPOINT, body.toString()) - ); - } - - default Requests put(String name, Object value) { - String setting = value == null ? - StringUtils.format("\"%s\": {\"%s\": null}", "transient", name) : - StringUtils.format("\"%s\": {\"%s\": \"%s\"}", "transient", name, value); - return new Requests( - restClient(), - new SqlRequest("PUT", LEGACY_SQL_SETTINGS_API_ENDPOINT, new Body(setting).toString()), - null - ); - } - - default String multiLine(String... lines) { - return String.join("\\n", lines); - } - - /** - * Query by a simple SQL is too common and deserve a dedicated overload method - */ - default Requests post(String sql, UrlParam... params) { - return post(new Body("\"query\":\"" + sql + "\""), params); - } - - default Requests post(Body body, UrlParam... params) { - String bodyStr = body.toString(); - return new Requests( - restClient(), - new SqlRequest("POST", QUERY_API_ENDPOINT, bodyStr, params), - new SqlRequest("POST", EXPLAIN_API_ENDPOINT, bodyStr) - ); - } - - default Body body(String... fieldValues) { - return new Body(fieldValues); - } - - default UrlParam[] params(String... keyValues) { - return Arrays.stream(keyValues).map(UrlParam::new).toArray(UrlParam[]::new); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Example.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Example.java deleted file mode 100644 index d35165845c1..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Example.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.builder; - -/** - * Example value object. - */ -public class Example { - - /** - * Title for the example section - */ - private String title; - - /** - * Description for the example - */ - private String description; - - /** - * Sample SQL query - */ - private String query; - - /** - * Query result set - */ - private String result; - - /** - * Is result set formatted in table (markup handle table in different way - */ - private boolean isTable; - - /** - * Explain query correspondent to the sample query - */ - private String explainQuery; - - /** - * Result of explain - */ - private String explainResult; - - public String getTitle() { - return title; - } - - public String getDescription() { - return description; - } - - public String getQuery() { - return query; - } - - public String getResult() { - return result; - } - - public boolean isTable() { - return isTable; - } - - public String getExplainQuery() { - return explainQuery; - } - - public String getExplainResult() { - return explainResult; - } - - public void setTitle(String title) { - this.title = title; - } - - public void setDescription(String description) { - this.description = description; - } - - public void setQuery(String query) { - this.query = query; - } - - public void setResult(String result) { - this.result = result; - } - - public void setTable(boolean table) { - isTable = table; - } - - public void setExplainQuery(String explainQuery) { - this.explainQuery = explainQuery; - } - - public void setExplainResult(String explainResult) { - this.explainResult = explainResult; - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Formats.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Formats.java deleted file mode 100644 index c281dd2c505..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Formats.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.builder; - -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.TABLE_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.TABLE_UNSORTED_RESPONSE; - -import org.opensearch.sql.doctest.core.request.SqlRequest; -import org.opensearch.sql.doctest.core.request.SqlRequestFormat; -import org.opensearch.sql.doctest.core.response.SqlResponse; -import org.opensearch.sql.doctest.core.response.SqlResponseFormat; - -/** - * Request and response format tuple. - */ -class Formats { - - private final SqlRequestFormat requestFormat; - private final SqlResponseFormat responseFormat; - - Formats(SqlRequestFormat requestFormat, SqlResponseFormat responseFormat) { - this.requestFormat = requestFormat; - this.responseFormat = responseFormat; - } - - String format(SqlRequest request) { - return requestFormat.format(request); - } - - String format(SqlResponse response) { - return responseFormat.format(response); - } - - boolean isTableFormat() { - return responseFormat == TABLE_RESPONSE || responseFormat == TABLE_UNSORTED_RESPONSE; - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/ListItems.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/ListItems.java deleted file mode 100644 index a3e0418956c..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/ListItems.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.builder; - -import org.opensearch.sql.legacy.utils.StringUtils; - -/** - * Item list - */ -public class ListItems { - private final StringBuilder list = new StringBuilder(); - private int index = 0; - - public void addItem(String text) { - list.append(index()).append(text).append('\n'); - } - - private String index() { - index++; - return StringUtils.format("%d. ", index); - } - - @Override - public String toString() { - return list.toString(); - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Requests.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Requests.java deleted file mode 100644 index cba0ee84288..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Requests.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.builder; - -import java.util.Objects; -import org.opensearch.client.RestClient; -import org.opensearch.sql.doctest.core.request.SqlRequest; -import org.opensearch.sql.doctest.core.response.SqlResponse; - -/** - * Query and explain request tuple. - */ -public class Requests { - - private final RestClient client; - private final SqlRequest query; - private final SqlRequest explain; - - public Requests(RestClient client, SqlRequest query) { - this(client, query, SqlRequest.NONE); - } - - public Requests(RestClient client, SqlRequest query, SqlRequest explain) { - this.client = client; - this.query = query; - this.explain = explain; - } - - public SqlRequest query() { - return query; - } - - public SqlResponse queryResponse() { - Objects.requireNonNull(query, "Query request is required"); - return query.send(client); - } - - public SqlRequest explain() { - return explain; - } - - public SqlResponse explainResponse() { - if (explain == SqlRequest.NONE) { - return SqlResponse.NONE; - } - return explain.send(client); - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/markup/Document.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/markup/Document.java deleted file mode 100644 index eeb5f12f440..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/markup/Document.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.markup; - -import java.io.Closeable; - -/** - * Document for different format and markup - */ -public interface Document extends Closeable { - - /** - * Remove checked IOException in method signature. - */ - @Override - void close(); - - Document section(String title); - - Document subSection(String title); - - Document paragraph(String text); - - Document codeBlock(String description, String code); - - Document table(String description, String table); - - Document image(String description, String filePath); - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/markup/RstDocument.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/markup/RstDocument.java deleted file mode 100644 index 96930851f8d..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/markup/RstDocument.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.markup; - -import com.google.common.base.Strings; -import java.io.PrintWriter; - -/** - * ReStructure Text document - */ -public class RstDocument implements Document { - - private final PrintWriter docWriter; - - public RstDocument(PrintWriter docWriter) { - this.docWriter = docWriter; - } - - @Override - public Document section(String title) { - return printTitleWithUnderline(title, "="); - } - - @Override - public Document subSection(String title) { - return printTitleWithUnderline(title, "-"); - } - - @Override - public Document paragraph(String text) { - return println(text); - } - - @Override - public Document codeBlock(String description, String code) { - if (!Strings.isNullOrEmpty(code)) { - return println(description + "::", indent(code)); - } - return this; - } - - @Override - public Document table(String description, String table) { - if (!Strings.isNullOrEmpty(table)) { - // RST table is different and not supposed to indent - return println(description + ":", table); - } - return this; - } - - @Override - public Document image(String description, String filePath) { - return println( - description + ":", - ".. image:: " + filePath - ); - } - - @Override - public void close() { - docWriter.close(); - } - - private Document printTitleWithUnderline(String title, String underlineChar) { - return print( - title, - Strings.repeat(underlineChar, title.length()) - ); - } - - /** - * Print each line with a blank line at last - */ - private Document print(String... lines) { - for (String line : lines) { - docWriter.println(line); - } - docWriter.println(); - return this; - } - - /** - * Print each line with a blank line followed - */ - private Document println(String... lines) { - for (String line : lines) { - docWriter.println(line); - docWriter.println(); - } - return this; - } - - private String indent(String text) { - return "\t" + text.replaceAll("\\n", "\n\t"); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/request/SqlRequest.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/request/SqlRequest.java deleted file mode 100644 index 6459f72d0cd..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/request/SqlRequest.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.request; - -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.OPENSEARCH_DASHBOARD_REQUEST; - -import java.io.IOException; -import org.opensearch.client.Request; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.ResponseException; -import org.opensearch.client.RestClient; -import org.opensearch.sql.doctest.core.response.SqlResponse; -import org.opensearch.sql.legacy.utils.StringUtils; - -/** - * Request to SQL plugin to isolate OpenSearch native request - */ -public class SqlRequest { - - public static final SqlRequest NONE = null; - - /** - * Native OpenSearch request object - */ - private final Request request; - - public SqlRequest(String method, String endpoint, String body, UrlParam... params) { - this.request = makeRequest(method, endpoint, body, params); - } - - /** - * Send request to OpenSearch via client and create response for it. - * - * @param client restful client connection - * @return sql response - */ - public SqlResponse send(RestClient client) { - try { - return new SqlResponse(client.performRequest(request)); - } catch (IOException e) { - // Some test may expect failure - if (e instanceof ResponseException) { - return new SqlResponse(((ResponseException) e).getResponse()); - } - - throw new IllegalStateException(StringUtils.format( - "Exception occurred during sending request %s", OPENSEARCH_DASHBOARD_REQUEST.format(this)), e); - } - } - - /** - * Expose request for request formatter. - * - * @return native OpenSearch format - */ - public Request request() { - return request; - } - - private Request makeRequest(String method, String endpoint, String body, UrlParam[] params) { - Request request = new Request(method, endpoint); - request.setJsonEntity(body); - for (UrlParam param : params) { - request.addParameter(param.key, param.value); - } - - RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); - restOptionsBuilder.addHeader("Content-Type", "application/json"); - request.setOptions(restOptionsBuilder); - return request; - } - - public static class UrlParam { - private String key; - private String value; - - public UrlParam(String key, String value) { - this.key = key; - this.value = value; - } - - public UrlParam(String keyValue) { - int equality = keyValue.indexOf('='); - if (equality == -1) { - throw new IllegalArgumentException(String.format( - "Key value pair is in bad format [%s]", keyValue)); - } - - this.key = keyValue.substring(0, equality); - this.value = keyValue.substring(equality + 1); - } - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/request/SqlRequestFormat.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/request/SqlRequestFormat.java deleted file mode 100644 index e4c152a49ec..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/request/SqlRequestFormat.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.request; - -import static java.util.stream.Collectors.joining; - -import com.google.common.base.Charsets; -import com.google.common.io.CharStreams; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import org.apache.http.Header; -import org.json.JSONObject; -import org.opensearch.client.Request; -import org.opensearch.sql.legacy.utils.JsonPrettyFormatter; -import org.opensearch.sql.legacy.utils.StringUtils; - -/** - * Different SQL request formats. - */ -public enum SqlRequestFormat { - - IGNORE_REQUEST { - @Override - public String format(SqlRequest request) { - return ""; - } - }, - CURL_REQUEST { - @Override - public String format(SqlRequest sqlRequest) { - Request request = sqlRequest.request(); - StringBuilder str = new StringBuilder(); - str.append(">> curl "); - - List
headers = request.getOptions().getHeaders(); - if (!headers.isEmpty()) { - str.append(headers.stream(). - map(header -> StringUtils.format("-H '%s: %s'", header.getName(), header.getValue())). - collect(joining(" ", "", " "))); - } - - str.append(StringUtils.format("-X %s ", request.getMethod())). - append("localhost:9200").append(request.getEndpoint()); - - if (!request.getParameters().isEmpty()) { - str.append(formatParams(request.getParameters())); - } - - String body = body(request); - if (!body.isEmpty()) { - str.append(" -d '"). - append(body). - append('\''); - } - return str.toString(); - } - }, - OPENSEARCH_DASHBOARD_REQUEST { - @Override - public String format(SqlRequest sqlRequest) { - Request request = sqlRequest.request(); - StringBuilder str = new StringBuilder(); - str.append(request.getMethod()). - append(" "). - append(request.getEndpoint()); - - if (!request.getParameters().isEmpty()) { - str.append(formatParams(request.getParameters())); - } - - str.append('\n'). - append(body(request)); - return str.toString(); - } - }; - - /** - * Format SQL request to specific format for documentation. - * - * @param request sql request - * @return string in specific format - */ - public abstract String format(SqlRequest request); - - @SuppressWarnings("UnstableApiUsage") - protected String body(Request request) { - String body = ""; - try { - InputStream content = request.getEntity().getContent(); - String rawBody = CharStreams.toString(new InputStreamReader(content, Charsets.UTF_8)); - if (!rawBody.isEmpty()) { - JSONObject json = new JSONObject(rawBody); - String sql = json.optString("query"); // '\\n' in literal is replaced by '\n' after unquote - body = JsonPrettyFormatter.format(rawBody); - - // Format and replace multi-line sql literal - if (!sql.isEmpty() && sql.contains("\n")) { - String multiLineSql = - Arrays.stream(sql.split("\\n")). // '\\n' is to escape backslash in regex - collect(joining("\n\t", - "\"\"\"\n\t", - "\n\t\"\"\"")); - body = body.replace("\"" + sql.replace("\n", "\\n") + "\"", multiLineSql); - } - } - } catch (IOException e) { - throw new IllegalStateException("Failed to parse and format body from request", e); - } - return body; - } - - protected String formatParams(Map params) { - return params.entrySet().stream(). - map(e -> e.getKey() + "=" + e.getValue()). - collect(joining("&", "?", "")); - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/response/DataTable.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/response/DataTable.java deleted file mode 100644 index 1955f8fa94a..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/response/DataTable.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.response; - -import com.google.common.base.Strings; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import org.opensearch.sql.legacy.utils.StringUtils; - -/** - * Data table that represent rows of data with a header. - * For now the format is actually in ReST and may need to decouple later. - */ -public class DataTable { - - private final int[] maxWidths; - private final Object[] header; - private final List rows; - - public DataTable(Object[] header) { - this.maxWidths = new int[header.length]; - this.header = header; - this.rows = new ArrayList<>(); - updateMaxWidthForEachColumn(header); - } - - public void addRow(Object[] row) { - rows.add(row); - updateMaxWidthForEachColumn(row); - } - - @Override - public String toString() { - StringBuilder str = new StringBuilder(); - String format = format(); - String separateLine1 = separateLine("-"); - String separateLine2 = separateLine("="); - - str.append(separateLine1). - append('\n'). - append(StringUtils.format(format, header)). - append('\n'). - append(separateLine2). - append('\n'); - - for (Object[] row : rows) { - str.append(StringUtils.format(format, row)). - append('\n'). - append(separateLine("-")). - append('\n'); - } - return str.toString(); - } - - private void updateMaxWidthForEachColumn(Object[] row) { - for (int i = 0; i < row.length; i++) { - maxWidths[i] = Math.max(maxWidths[i], String.valueOf(row[i]).length()); - } - } - - private String separateLine(String separator) { - return Arrays.stream(maxWidths). - mapToObj(width -> Strings.repeat(separator, width)). - collect(Collectors.joining("+", "+", "+")); - } - - /** - * Format as Java String.format needs to make use of auto pad feature. - * For example, to ensure width of 10 and pad spaces, we need to String.format("%10s", str); - */ - private String format() { - return Arrays.stream(maxWidths). - mapToObj(width -> "%" + width + "s"). - collect(Collectors.joining("|", "|", "|")); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/response/SqlResponse.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/response/SqlResponse.java deleted file mode 100644 index cddf3c9dc78..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/response/SqlResponse.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.response; - -import java.io.IOException; -import org.json.JSONException; -import org.json.JSONObject; -import org.opensearch.client.Response; -import org.opensearch.sql.util.TestUtils; - -/** - * Response from SQL plugin - */ -public class SqlResponse { - - public static final SqlResponse NONE = null; - - /** - * Native OpenSearch response - */ - private final Response response; - - public SqlResponse(Response response) { - this.response = response; - } - - /** - * Parse body in the response. - * - * @return response body - */ - public String body() { - try { - return replaceChangingFields(TestUtils.getResponseBody(response, true)); - } catch (IOException e) { - throw new IllegalStateException("Failed to read response body", e); - } - } - - /** - * In OpenSearch response, there is field changed between each query, such as "took". - * We have to replace those variants with fake constant to avoid re-generate documents. - * The order of fields in JSON is a little different from original because of internal - * key set in org.json. - */ - private String replaceChangingFields(String response) { - try { - JSONObject root = new JSONObject(response); - if (root.has("took")) { - root.put("took", 100); - } else { - return response; // return original response to minimize impact - } - - if (root.has("_shards")) { - JSONObject shards = root.getJSONObject("_shards"); - shards.put("total", 5); - shards.put("successful", 5); - } - return root.toString(); - } catch (JSONException e) { - // Response is not a valid JSON which is not our interest. - return response; - } - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/response/SqlResponseFormat.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/response/SqlResponseFormat.java deleted file mode 100644 index 3a054e59df6..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/response/SqlResponseFormat.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.response; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import org.json.JSONArray; -import org.json.JSONObject; -import org.opensearch.sql.legacy.utils.JsonPrettyFormatter; -import org.opensearch.sql.legacy.utils.StringUtils; - -/** - * Different SQL response formats - */ -public enum SqlResponseFormat { - - IGNORE_RESPONSE { - @Override - public String format(SqlResponse sqlResponse) { - return ""; - } - }, - ORIGINAL_RESPONSE { - @Override - public String format(SqlResponse sqlResponse) { - return sqlResponse.body(); - } - }, - PRETTY_JSON_RESPONSE { - @Override - public String format(SqlResponse sqlResponse) { - String body = sqlResponse.body(); - try { - return JsonPrettyFormatter.format(body); - } catch (IOException e) { - throw new IllegalStateException( - StringUtils.format("Failed to pretty format response: %s", body), e); - } - } - }, - TABLE_RESPONSE { - @Override - public String format(SqlResponse sqlResponse) { - return format(sqlResponse, true); - } - }, - TABLE_UNSORTED_RESPONSE { - @Override - public String format(SqlResponse sqlResponse) { - return format(sqlResponse, false); - } - }; - - /** - * Format SQL response to specific format for documentation - * - * @param sqlResponse sql response - * @return string in specific format - */ - public abstract String format(SqlResponse sqlResponse); - - /** - * Note that we put this format() here because it's shared by two format enums. - * - * @param sqlResponse original response from plugin - * @param isSorted true to sort the result or just leave it as is - */ - protected String format(SqlResponse sqlResponse, boolean isSorted) { - JSONObject body = new JSONObject(sqlResponse.body()); - if (body.isNull("schema")) { - throw new IllegalStateException( - "Only JDBC response can be formatted to table: " + sqlResponse.body()); - } - - Object[] header = parseHeader(body.getJSONArray("schema")); - List rows = parseDataRows(body.getJSONArray("datarows"), isSorted); - - DataTable table = new DataTable(header); - for (Object[] row : rows) { - table.addRow(row); - } - return table.toString(); - } - - private Object[] parseHeader(JSONArray schema) { - Object[] header = new Object[schema.length()]; - for (int i = 0; i < header.length; i++) { - JSONObject nameType = schema.getJSONObject(i); - header[i] = nameType.optString("alias", nameType.getString("name")); - } - return header; - } - - private List parseDataRows(JSONArray rows, boolean isSorted) { - List rowsToSort = new ArrayList<>(); - for (Object row : rows) { - rowsToSort.add(((JSONArray) row).toList().toArray()); - } - - if (isSorted) { - sort(rowsToSort); - } - return rowsToSort; - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static void sort(List lists) { - lists.sort((list1, list2) -> { - if (list1 == null || list2 == null) { - return compareNullable(list1, list2); - } - - // Assume 2 lists are of same length and all elements are comparable - for (int i = 0; i < list1.length; i++) { - Comparable obj1 = (Comparable) list1[i]; - Comparable obj2 = (Comparable) list2[i]; - - if (obj1 == null || obj2 == null) { - return compareNullable(obj1, obj2); - } - - int result = obj1.compareTo(obj2); - if (result != 0) { - return result; - } - } - return 0; - }); - } - - /** - * Put NULL first (as smaller element) - */ - private static int compareNullable(Object obj1, Object obj2) { - if (obj1 == null && obj2 == null) { - return 0; - } else if (obj1 == null) { - return -1; - } else { // obj2 == null - return 1; - } - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DataTableTest.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DataTableTest.java deleted file mode 100644 index 8ffc4a6a6c5..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DataTableTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.test; - -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; - -import org.junit.Test; -import org.opensearch.sql.doctest.core.response.DataTable; - -/** - * Test cases for {@link DataTable} - */ -public class DataTableTest { - - @Test - public void testSingleColumnTable() { - DataTable table = new DataTable(new Object[] {"Test Table"}); - table.addRow(new Object[] {"this is a very long line"}); - - assertThat( - table.toString(), - is( - "+------------------------+\n" + - "| Test Table|\n" + - "+========================+\n" + - "|this is a very long line|\n" + - "+------------------------+\n" - ) - ); - } - - @Test - public void testTwoColumnsTable() { - DataTable table = new DataTable(new Object[] {"Test Table", "Very Long Title"}); - table.addRow(new Object[] {"this is a very long line", "short"}); - - assertThat( - table.toString(), - is( - "+------------------------+---------------+\n" + - "| Test Table|Very Long Title|\n" + - "+========================+===============+\n" + - "|this is a very long line| short|\n" + - "+------------------------+---------------+\n" - ) - ); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DocBuilderTest.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DocBuilderTest.java deleted file mode 100644 index d89066362ba..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DocBuilderTest.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.test; - -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import org.apache.http.HttpEntity; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InOrder; -import org.mockito.Mock; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.runners.MockitoJUnitRunner; -import org.mockito.stubbing.Answer; -import org.opensearch.client.Response; -import org.opensearch.client.RestClient; -import org.opensearch.sql.doctest.core.builder.DocBuilder; -import org.opensearch.sql.doctest.core.markup.Document; - -/** - * Test cases for {@link DocBuilder} - */ -@RunWith(MockitoJUnitRunner.class) -public class DocBuilderTest implements DocBuilder { - - private final String queryResponse = - "{\"schema\":[{\"name\":\"firstname\",\"type\":\"text\"}]," + - "\"datarows\":[[\"John\"]],\"total\":10,\"size\":1,\"status\":200}"; - - private final String explainResponse = - "{\"from\":0,\"size\":1,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]}}"; - - @Mock - private Document document; - - private Verifier verifier; - - @Mock - private RestClient client; - - @Before - public void setUp() throws IOException { - when(document.section(any())).thenReturn(document); - when(document.subSection(any())).thenReturn(document); - when(document.paragraph(any())).thenReturn(document); - when(document.codeBlock(any(), any())).thenReturn(document); - when(document.table(any(), any())).thenReturn(document); - when(document.image(any(), any())).thenReturn(document); - verifier = new Verifier(document); - - when(client.performRequest(any())).then(new Answer() { - private int callCount = 0; - - @Override - public Response answer(InvocationOnMock invocationOnMock) throws IOException { - Response response = mock(Response.class); - HttpEntity entity = mock(HttpEntity.class); - when(response.getEntity()).thenReturn(entity); - - String body = (callCount++ == 0) ? queryResponse : explainResponse; - when(entity.getContent()).thenReturn(new ByteArrayInputStream(body.getBytes())); - return response; - } - }); - } - - @Test - public void sectionShouldIncludeTitleAndDescription() { - section( - title("Test"), - description("This is a test") - ); - - verifier.section("Test"). - subSection("Description"). - paragraph("This is a test"); - } - - @Test - public void sectionShouldIncludeMultiLineSql() { - section( - title("Test"), - description("This is a test"), - example( - description("This is an example for the test"), - post(multiLine( - "SELECT firstname", - "FROM accounts", - "WHERE age > 30") - ) - ) - ); - - verifier.section("Test"). - subSection("Description"). - paragraph("This is a test"). - subSection("Example"). - paragraph("This is an example for the test"). - codeBlock( - "SQL query", - "POST /_plugins/_sql\n" + - "{\n" + - " \"query\" : \"\"\"\n" + - "\tSELECT firstname\n" + - "\tFROM accounts\n" + - "\tWHERE age > 30\n" + - "\t\"\"\"\n" + - "}" - ); - } - - @Test - public void sectionShouldIncludeExample() { - section( - title("Test"), - description("This is a test"), - images("rdd/querySyntax.png"), - example( - description("This is an example for the test"), - post("SELECT firstname FROM accounts") - ) - ); - - verifier.section("Test"). - subSection("Description"). - paragraph("This is a test"). - image("Rule ``querySyntax``", "/docs/user/img/rdd/querySyntax.png"). - subSection("Example"). - paragraph("This is an example for the test"). - codeBlock( - "SQL query", - "POST /_plugins/_sql\n" + - "{\n" + - " \"query\" : \"SELECT firstname FROM accounts\"\n" + - "}" - ). - codeBlock( - "Explain", - "{\n" + - " \"from\" : 0,\n" + - " \"size\" : 1,\n" + - " \"_source\" : {\n" + - " \"includes\" : [\n" + - " \"firstname\"\n" + - " ],\n" + - " \"excludes\" : [ ]\n" + - " }\n" + - "}" - ).table( - "Result set", - "+---------+\n" + - "|firstname|\n" + - "+=========+\n" + - "| John|\n" + - "+---------+\n" - ); - } - - @Override - public RestClient restClient() { - return client; - } - - @Override - public Document openDocument() { - return document; - } - - private static class Verifier implements Document { - private final Document mock; - private final InOrder verifier; - - Verifier(Document mock) { - this.mock = mock; - this.verifier = inOrder(mock); - } - - @Override - public void close() { - verifier.verify(mock).close(); - } - - @Override - public Document section(String title) { - verifier.verify(mock).section(title); - return this; - } - - @Override - public Document subSection(String title) { - verifier.verify(mock).subSection(title); - return this; - } - - @Override - public Document paragraph(String text) { - verifier.verify(mock).paragraph(text); - return this; - } - - @Override - public Document codeBlock(String description, String code) { - verifier.verify(mock).codeBlock(description, code); - return this; - } - - @Override - public Document table(String description, String table) { - verifier.verify(mock).table(description, table); - return this; - } - - @Override - public Document image(String description, String filePath) { - verifier.verify(mock).image(description, filePath); - return this; - } - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DocTestTests.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DocTestTests.java deleted file mode 100644 index 704677764d3..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DocTestTests.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.test; - -import org.junit.runner.RunWith; -import org.junit.runners.Suite; - -/** - * Suite to run all doc tests in one shot for local testing - */ -@RunWith(Suite.class) -@Suite.SuiteClasses({ - SqlRequestTest.class, - SqlResponseTest.class, - SqlRequestFormatTest.class, - SqlResponseFormatTest.class, - DocBuilderTest.class, - RstDocumentTest.class, - DataTableTest.class, -}) -public class DocTestTests { -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/RstDocumentTest.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/RstDocumentTest.java deleted file mode 100644 index 3b7faf546ef..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/RstDocumentTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.test; - -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; - -import java.io.ByteArrayOutputStream; -import java.io.PrintWriter; -import org.junit.Before; -import org.junit.Test; -import org.opensearch.sql.doctest.core.markup.RstDocument; - -/** - * Test cases for {@link RstDocument} - */ -public class RstDocumentTest { - - private ByteArrayOutputStream content; - - private RstDocument document; - - @Before - public void setUp() { - content = new ByteArrayOutputStream(); - document = new RstDocument(new PrintWriter(content, true)); // Enable auto flush - } - - @Test - public void testSection() { - document.section("Test Section"); - assertThat( - content.toString(), - is( - "Test Section\n" + - "============\n" + - "\n" - ) - ); - } - - @Test - public void testSubSection() { - document.subSection("Test Sub Section"); - assertThat( - content.toString(), - is( - "Test Sub Section\n" + - "----------------\n" + - "\n" - ) - ); - } - - @Test - public void testParagraph() { - document.paragraph("Test paragraph"); - assertThat( - content.toString(), - is( - "Test paragraph\n" + - "\n" - ) - ); - } - - @Test - public void testCodeBlock() { - document.codeBlock("Test code", ">> curl localhost:9200"); - assertThat( - content.toString(), - is( - "Test code::\n" + - "\n" + - "\t>> curl localhost:9200\n" + - "\n" - ) - ); - } - - @Test - public void testTable() { - document.table( - "Test table", - "+----------+\n" + - "|Test Table|\n" + - "+==========+\n" + - "| test data|\n" + - "+----------+" - ); - - assertThat( - content.toString(), - is( - "Test table:\n" + - "\n" + - "+----------+\n" + - "|Test Table|\n" + - "+==========+\n" + - "| test data|\n" + - "+----------+\n" + - "\n" - ) - ); - } - - @Test - public void testImage() { - document.image("Query syntax", "/docs/user/img/query_syntax.png"); - - assertThat( - content.toString(), - is( - "Query syntax:\n" + - "\n" + - ".. image:: /docs/user/img/query_syntax.png\n" + - "\n" - ) - ); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlRequestFormatTest.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlRequestFormatTest.java deleted file mode 100644 index 2cecc4c13ef..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlRequestFormatTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.test; - -import static org.hamcrest.Matchers.emptyString; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.CURL_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.OPENSEARCH_DASHBOARD_REQUEST; -import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; - -import org.junit.Test; -import org.opensearch.sql.doctest.core.request.SqlRequest; -import org.opensearch.sql.doctest.core.request.SqlRequest.UrlParam; -import org.opensearch.sql.doctest.core.request.SqlRequestFormat; - -/** - * Test cases for {@link SqlRequestFormat} - */ -public class SqlRequestFormatTest { - - private final SqlRequest sqlRequest = new SqlRequest( - "POST", - QUERY_API_ENDPOINT, - "{\"query\":\"SELECT * FROM accounts\"}", - new UrlParam("format", "jdbc") - ); - - @Test - public void testIgnoreRequestFormat() { - assertThat(IGNORE_REQUEST.format(sqlRequest), emptyString()); - } - - @Test - public void testCurlFormat() { - String expected = - ">> curl -H 'Content-Type: application/json' -X POST localhost:9200/_plugins/_sql?format=jdbc -d '{\n" + - " \"query\" : \"SELECT * FROM accounts\"\n" + - "}'"; - assertThat(CURL_REQUEST.format(sqlRequest), is(expected)); - } - - @Test - public void testOpenSearchDashboardsFormat() { - String expected = - "POST /_plugins/_sql?format=jdbc\n" + - "{\n" + - " \"query\" : \"SELECT * FROM accounts\"\n" + - "}"; - assertThat(OPENSEARCH_DASHBOARD_REQUEST.format(sqlRequest), is(expected)); - } - - @Test - public void multiLineSqlInOpenSearchDashboardRequestShouldBeWellFormatted() { - SqlRequest multiLineSqlRequest = new SqlRequest( - "POST", - "/_plugins/_sql", - "{\"query\":\"SELECT *\\nFROM accounts\\nWHERE age > 30\"}" - ); - - String expected = - "POST /_plugins/_sql\n" + - "{\n" + - " \"query\" : \"\"\"\n" + - "\tSELECT *\n" + - "\tFROM accounts\n" + - "\tWHERE age > 30\n" + - "\t\"\"\"\n" + - "}"; - assertThat(OPENSEARCH_DASHBOARD_REQUEST.format(multiLineSqlRequest), is(expected)); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlRequestTest.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlRequestTest.java deleted file mode 100644 index d459e7aaca4..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlRequestTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.test; - -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; - -import com.google.common.base.Charsets; -import com.google.common.io.CharStreams; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import org.junit.Test; -import org.mockito.ArgumentCaptor; -import org.opensearch.client.Request; -import org.opensearch.client.RestClient; -import org.opensearch.sql.doctest.core.request.SqlRequest; -import org.opensearch.sql.doctest.core.request.SqlRequest.UrlParam; - -/** - * Test cases for {@link SqlRequest} - */ -public class SqlRequestTest { - - @Test - public void requestShouldIncludeAllFields() throws IOException { - String method = "POST"; - String endpoint = QUERY_API_ENDPOINT; - String body = "{\"query\":\"SELECT * FROM accounts\"}"; - String key = "format"; - String value = "jdbc"; - UrlParam param = new UrlParam(key, value); - - RestClient client = mock(RestClient.class); - SqlRequest sqlRequest = new SqlRequest(method, endpoint, body, param); - sqlRequest.send(client); - - ArgumentCaptor argument = ArgumentCaptor.forClass(Request.class); - verify(client).performRequest(argument.capture()); - Request actual = argument.getValue(); - assertThat(actual.getMethod(), is(method)); - assertThat(actual.getEndpoint(), is(endpoint)); - assertThat(actual.getParameters(), hasEntry(key, value)); - assertThat(body(actual), is(body)); - } - - @Test(expected = IllegalArgumentException.class) - public void badUrlParamShouldThrowException() { - new UrlParam("test"); - } - - private String body(Request request) throws IOException { - InputStream content = request.getEntity().getContent(); - return CharStreams.toString(new InputStreamReader(content, Charsets.UTF_8)); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlResponseFormatTest.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlResponseFormatTest.java deleted file mode 100644 index 689e9f0b98d..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlResponseFormatTest.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.test; - -import static org.hamcrest.Matchers.emptyString; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.ORIGINAL_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.TABLE_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.TABLE_UNSORTED_RESPONSE; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import org.apache.http.HttpEntity; -import org.junit.Before; -import org.junit.Test; -import org.opensearch.client.Response; -import org.opensearch.sql.doctest.core.response.SqlResponse; -import org.opensearch.sql.doctest.core.response.SqlResponseFormat; - -/** - * Test cases for {@link SqlResponseFormat} - */ -public class SqlResponseFormatTest { - - private final String expected = - "{" + - "\"schema\":[{\"name\":\"firstname\",\"type\":\"text\"}]," + - "\"datarows\":[[\"John\"]]," + - "\"total\":10," + - "\"size\":1," + - "\"status\":200" + - "}"; - - private SqlResponse sqlResponse; - - @Before - public void setUp() throws IOException { - mockResponse(expected); - } - - @Test - public void testIgnoreResponseFormat() { - assertThat(IGNORE_RESPONSE.format(sqlResponse), emptyString()); - } - - @Test - public void testOriginalFormat() { - assertThat(ORIGINAL_RESPONSE.format(sqlResponse), is(expected + "\n")); - } - - @Test - public void testPrettyJsonFormat() { - assertThat( - PRETTY_JSON_RESPONSE.format(sqlResponse), - is( - "{\n" + - " \"schema\" : [\n" + - " {\n" + - " \"name\" : \"firstname\",\n" + - " \"type\" : \"text\"\n" + - " }\n" + - " ],\n" + - " \"datarows\" : [\n" + - " [\n" + - " \"John\"\n" + - " ]\n" + - " ],\n" + - " \"total\" : 10,\n" + - " \"size\" : 1,\n" + - " \"status\" : 200\n" + - "}" - ) - ); - } - - @Test - public void testTableFormat() { - assertThat( - TABLE_RESPONSE.format(sqlResponse), - is( - "+---------+\n" + - "|firstname|\n" + - "+=========+\n" + - "| John|\n" + - "+---------+\n" - ) - ); - } - - @Test - public void rowsInTableShouldBeSorted() throws IOException { - mockResponse( - "{" + - "\"schema\":[" + - "{\"name\":\"firstname\",\"type\":\"text\"}," + - "{\"name\":\"age\",\"type\":\"integer\"}" + - "]," + - "\"datarows\":[" + - "[\"John\", 30]," + - "[\"John\", 24]," + - "[\"Allen\", 45]" + - "]," + - "\"total\":10," + - "\"size\":3," + - "\"status\":200" + - "}" - ); - - assertThat( - TABLE_RESPONSE.format(sqlResponse), - is( - "+---------+---+\n" + - "|firstname|age|\n" + - "+=========+===+\n" + - "| Allen| 45|\n" + - "+---------+---+\n" + - "| John| 24|\n" + - "+---------+---+\n" + - "| John| 30|\n" + - "+---------+---+\n" - ) - ); - } - - @Test - public void rowsInTableUnsortedShouldMaintainOriginalOrder() throws IOException { - mockResponse( - "{" + - "\"schema\":[" + - "{\"name\":\"firstname\",\"type\":\"text\"}," + - "{\"name\":\"age\",\"type\":\"integer\"}" + - "]," + - "\"datarows\":[" + - "[\"John\", 30]," + - "[\"John\", 24]," + - "[\"Allen\", 45]" + - "]," + - "\"total\":10," + - "\"size\":3," + - "\"status\":200" + - "}" - ); - - assertThat( - TABLE_UNSORTED_RESPONSE.format(sqlResponse), - is( - "+---------+---+\n" + - "|firstname|age|\n" + - "+=========+===+\n" + - "| John| 30|\n" + - "+---------+---+\n" + - "| John| 24|\n" + - "+---------+---+\n" + - "| Allen| 45|\n" + - "+---------+---+\n" - ) - ); - } - - private void mockResponse(String content) throws IOException { - Response response = mock(Response.class); - HttpEntity entity = mock(HttpEntity.class); - when(response.getEntity()).thenReturn(entity); - when(entity.getContent()).thenReturn(new ByteArrayInputStream(content.getBytes())); - sqlResponse = new SqlResponse(response); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlResponseTest.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlResponseTest.java deleted file mode 100644 index 50912ce6e7d..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlResponseTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.test; - -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import org.apache.http.HttpEntity; -import org.junit.Test; -import org.opensearch.client.Response; -import org.opensearch.sql.doctest.core.response.SqlResponse; - -/** - * Test cases for {@link SqlResponse} - */ -public class SqlResponseTest { - - @Test - public void responseBodyShouldRetainNewLine() throws IOException { - Response response = mock(Response.class); - HttpEntity entity = mock(HttpEntity.class); - String expected = "123\nabc\n"; - when(response.getEntity()).thenReturn(entity); - when(entity.getContent()).thenReturn(new ByteArrayInputStream(expected.getBytes())); - - SqlResponse sqlResponse = new SqlResponse(response); - String actual = sqlResponse.body(); - assertThat(actual, is(expected)); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/dml/DeleteIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/dml/DeleteIT.java deleted file mode 100644 index c11bbc3f76c..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/dml/DeleteIT.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.dml; - -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.OPENSEARCH_DASHBOARD_REQUEST; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; - -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; - -@DocTestConfig(template = "dml/delete.rst", testData = {"accounts.json"}) -public class DeleteIT extends DocTest { - - @Section(1) - public void delete() { - section( - title("DELETE"), - description( - "``DELETE`` statement deletes documents that satisfy the predicates in ``WHERE`` clause.", - "Note that all documents are deleted in the case of ``WHERE`` clause absent." - ), - images("rdd/singleDeleteStatement.png"), - example( - description( - "The ``datarows`` field in this case shows rows impacted, in other words how many", - "documents were just deleted." - ), - post(multiLine( - "DELETE FROM accounts", - "WHERE age > 30" - )), - queryFormat(OPENSEARCH_DASHBOARD_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) - ) - ); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/dql/BasicQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/dql/BasicQueryIT.java deleted file mode 100644 index 80b8eb03e44..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/dql/BasicQueryIT.java +++ /dev/null @@ -1,310 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.dql; - -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.OPENSEARCH_DASHBOARD_REQUEST; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.TABLE_UNSORTED_RESPONSE; - -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; -import org.opensearch.sql.doctest.core.builder.Example; -import org.opensearch.sql.doctest.core.builder.Requests; - -/** - * Doc test for basic SELECT query. - */ -@DocTestConfig(template = "dql/basics.rst", testData = {"accounts.json"}) -public class BasicQueryIT extends DocTest { - - @Section(1) - public void select() { - section( - title("SELECT"), - description( - "``SELECT`` clause specifies which fields in OpenSearch index should be retrieved."), - images("rdd/selectElements.png", "rdd/selectElement.png"), - example( - title("Selecting All Fields"), - description( - "You can use ``*`` to fetch all fields in the index which is very convenient when you", - "just want to have a quick look at your data." - ), - post("SELECT * FROM accounts") - ), - example( - title("Selecting Specific Fields"), - description( - "More often you would give specific field name(s) in ``SELECT`` clause to", - "avoid large and unnecessary data retrieved." - ), - post("SELECT firstname, lastname FROM accounts") - ), - example( - title("Using Field Alias"), - description( - "Alias is often used to make your query more readable by giving your field a shorter name." - ), - post("SELECT account_number AS num FROM accounts") - ), - example( - title("Selecting Distinct Fields"), - description( - "``DISTINCT`` is useful when you want to de-duplicate and get unique field value.", - "You can provide one or more field names." - ), - post("SELECT DISTINCT age FROM accounts") - ) - ); - } - - @Section(2) - public void from() { - section( - title("FROM"), - description( - "``FROM`` clause specifies OpenSearch index where the data should be retrieved from.", - "You've seen how to specify a single index in FROM clause in last section. Here we", - "provide examples for more use cases.\n\n" + - "Subquery in ``FROM`` clause is also supported. Please check out the documentation for more details." - ), - images("rdd/tableName.png"), - openSearchDashboardsExample( - title("Using Index Alias"), - description( - "Similarly you can give index in ``FROM`` clause an alias and use it across clauses in query." - ), - post("SELECT acc.account_number FROM accounts acc") - ), - openSearchDashboardsExample( - title("Selecting From Multiple Indices by Index Pattern"), - description( - "Alternatively you can query from multiple indices of similar names by index pattern.", - "This is very convenient for indices created by Logstash index template with date as suffix." - ), - post("SELECT account_number FROM account*") - ), - openSearchDashboardsExample( - title("[Deprecating] Selecting From Specific Index Type"), - description( - "You can also specify type name explicitly though this has been deprecated in", - "later OpenSearch version." - ), - post("SELECT account_number FROM accounts/account") - ) - ); - } - - @Section(3) - public void where() { - section( - title("WHERE"), - description( - "``WHERE`` clause specifies only OpenSearch documents that meet the criteria should be affected.", - "It consists of predicates that uses ``=``, ``<>``, ``>``, ``>=``, ``<``, ``<=``, ``IN``,", - "``BETWEEN``, ``LIKE``, ``IS NULL`` or ``IS NOT NULL``. These predicates can be combined by", - "logical operator ``NOT``, ``AND`` or ``OR`` to build more complex expression.\n\n" + - "For ``LIKE`` and other full text search topics, please refer to Full Text Search documentation.\n\n" + - "Besides SQL query, WHERE clause can also be used in SQL statement such as ``DELETE``. Please refer to", - "Data Manipulation Language documentation for details." - ), - example( - title("Comparison Operators"), - description( - "Basic comparison operators, such as ``=``, ``<>``, ``>``, ``>=``, ``<``, ``<=``, can work for", - "number, string or date.", - "``IN`` and ``BETWEEN`` is convenient for comparison with multiple values or a range." - ), - post(multiLine( - "SELECT account_number", - "FROM accounts", - "WHERE account_number = 1" - )) - ), - example( - title("Missing Fields"), - description( - "As NoSQL database, OpenSearch allows for flexible schema that documents in an index may have", - "different fields. In this case, you can use ``IS NULL`` or ``IS NOT NULL`` to retrieve missing", - "fields or existing fields only.\n\n" + - "Note that for now we don't differentiate missing field and field set to ``NULL`` explicitly." - ), - post(multiLine( - "SELECT account_number, employer", - "FROM accounts", - "WHERE employer IS NULL" - )) - ) - ); - } - - @Section(4) - public void groupBy() { - section( - title("GROUP BY"), - description( - "``GROUP BY`` groups documents with same field value into buckets. It is often used along with", - "aggregation functions to aggregate inside each bucket. Please refer to SQL Functions documentation", - "for more details.\n\n" + - "Note that ``WHERE`` clause is applied before ``GROUP BY`` clause." - ), - example( - title("Grouping by Fields"), - description(), - post(multiLine( - "SELECT age", - "FROM accounts", - "GROUP BY age" - )) - ), - example( - title("Grouping by Field Alias"), - description("Field alias is accessible in ``GROUP BY`` clause."), - post(multiLine( - "SELECT account_number AS num", - "FROM accounts", - "GROUP BY num" - )) - ), - example( - title("Grouping by Ordinal"), - description( - "Alternatively field ordinal in ``SELECT`` clause can be used too. However this is not", - "recommended because your ``GROUP BY`` clause depends on fields in ``SELECT`` clause", - "and require to change accordingly." - ), - post(multiLine( - "SELECT age", - "FROM accounts", - "GROUP BY 1" - )) - ), - example( - title("Grouping by Scalar Function"), - description( - "Scalar function can be used in ``GROUP BY`` clause and it's required to be present in", - "``SELECT`` clause too." - ), - post(multiLine( - "SELECT ABS(age) AS a", - "FROM accounts", - "GROUP BY ABS(age)" - )) - ) - ); - } - - @Section(5) - public void having() { - section( - title("HAVING"), - description( - "``HAVING`` clause filters result from ``GROUP BY`` clause by predicate(s). Because of this,", - "aggregation function, even different from those on ``SELECT`` clause, can be used in predicate." - ), - example( - description(), - post(multiLine( - "SELECT age, MAX(balance)", - "FROM accounts", - "GROUP BY age", - "HAVING MIN(balance) > 10000" - )) - ) - ); - } - - @Section(6) - public void orderBy() { - section( - title("ORDER BY"), - description( - "``ORDER BY`` clause specifies which fields used to sort the result and in which direction."), - orderByExample( - title("Ordering by Fields"), - description( - "Besides regular field names, ordinal, alias or scalar function can also be used similarly", - "as in ``GROUP BY``. ``ASC`` (by default) or ``DESC`` can be appended to indicate sorting in", - "ascending or descending order." - ), - post("SELECT account_number FROM accounts ORDER BY account_number DESC") - ), - orderByExample( - title("Specifying Order for Null"), - description( - "Additionally you can specify if documents with missing field be put first or last.", - "The default behavior of OpenSearch is to return nulls or missing last.", - "You can make them present before non-nulls by using ``IS NOT NULL``." - ), - post(multiLine( - "SELECT employer", - "FROM accounts", - "ORDER BY employer IS NOT NULL" - )) - ) - ); - } - - @Section(7) - public void limit() { - section( - title("LIMIT"), - description( - "Mostly specifying maximum number of documents returned is necessary to prevent fetching", - "large amount of data into memory. `LIMIT` clause is helpful in this case." - ), - example( - title("Limiting Result Size"), - description( - "Given a positive number, ``LIMIT`` uses it as page size to fetch result of that size at most." - ), - post(multiLine( - "SELECT account_number", - "FROM accounts", - "ORDER BY account_number LIMIT 1" - )) - ), - example( - title("Fetching at Offset"), - description( - "Offset position can be given as first argument to indicate where to start fetching.", - "This can be used as simple pagination solution though it's inefficient on large index.", - "Generally ``ORDER BY`` is required in this case to ensure the same order between pages." - ), - post(multiLine( - "SELECT account_number", - "FROM accounts", - "ORDER BY account_number LIMIT 1, 1" - )) - ) - ); - } - - /** - * Document only OpenSearch Dashboard request for example and ignore response as well as explain - */ - private Example openSearchDashboardsExample(String title, String description, Requests requests) { - return example(title, description, requests, - queryFormat(OPENSEARCH_DASHBOARD_REQUEST, IGNORE_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ); - } - - /** - * Example for ORDER BY needs to maintain the order in original result - */ - private Example orderByExample(String title, String description, Requests requests) { - return example(title, description, requests, - queryFormat(OPENSEARCH_DASHBOARD_REQUEST, TABLE_UNSORTED_RESPONSE), - explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) - ); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/dql/ComplexQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/dql/ComplexQueryIT.java deleted file mode 100644 index 22b0b3367d1..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/dql/ComplexQueryIT.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.dql; - -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.OPENSEARCH_DASHBOARD_REQUEST; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.TABLE_RESPONSE; - -import org.junit.Ignore; -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; -import org.opensearch.sql.doctest.core.builder.Example; -import org.opensearch.sql.doctest.core.builder.Requests; - -@DocTestConfig(template = "dql/complex.rst", testData = {"accounts.json", "employees_nested.json"}) -public class ComplexQueryIT extends DocTest { - - @Section(1) - public void subquery() { - section( - title("Subquery"), - description( - "A subquery is a complete ``SELECT`` statement which is used within another statement", - "and enclosed in parenthesis. From the explain output, you can notice that some subquery", - "are actually transformed to an equivalent join query to execute." - ), - /* - example( - title("Scalar Value Subquery"), - description( - "" - ), - post( - "SELECT firstname, lastname, balance " + - "FROM accounts " + - "WHERE balance >= ( " + - " SELECT AVG(balance) FROM accounts " + - ") " - ) - ),*/ - example( - title("Table Subquery"), - description(""), - post(multiLine( - "SELECT a1.firstname, a1.lastname, a1.balance", - "FROM accounts a1", - "WHERE a1.account_number IN (", - " SELECT a2.account_number", - " FROM accounts a2", - " WHERE a2.balance > 10000", - ")" - )) - ), - example( - title("Subquery in FROM Clause"), - description(""), - post(multiLine( - "SELECT a.f, a.l, a.a", - "FROM (", - " SELECT firstname AS f, lastname AS l, age AS a", - " FROM accounts", - " WHERE age > 30", - ") AS a" - )) - ) - ); - } - - @Section(2) - public void joins() { - section( - title("JOINs"), - description( - "A ``JOIN`` clause combines columns from one or more indices by using values common to each." - ), - images("rdd/tableSource.png", "rdd/joinPart.png"), - example( - title("Inner Join"), - description( - "Inner join is very commonly used that creates a new result set by combining columns", - "of two indices based on the join predicates specified. It iterates both indices and", - "compare each document to find all that satisfy the join predicates. Keyword ``JOIN``", - "is used and preceded by ``INNER`` keyword optionally. The join predicate(s) is specified", - "by ``ON`` clause.\n\n", - "Remark that the explain API output for join queries looks complicated. This is because", - "a join query is associated with two OpenSearch DSL queries underlying and execute in", - "the separate query planner framework. You can interpret it by looking into the logical", - "plan and physical plan." - ), - post(multiLine( - "SELECT", - " a.account_number, a.firstname, a.lastname,", - " e.id, e.name", - "FROM accounts a", - "JOIN employees_nested e", - " ON a.account_number = e.id" - )) - ), - joinExampleWithoutExplain( - title("Cross Join"), - description( - "Cross join or Cartesian join combines each document from the first index with each from", - "the second. The result set is the Cartesian Product of documents from both indices.", - "It appears to be similar to inner join without ``ON`` clause to specify join condition.\n\n", - "Caveat: It is risky to do cross join even on two indices of medium size. This may trigger", - "our circuit breaker to terminate the query to avoid out of memory issue." - ), - post(multiLine( - "SELECT", - " a.account_number, a.firstname, a.lastname,", - " e.id, e.name", - "FROM accounts a", - "JOIN employees_nested e" - )) - ), - joinExampleWithoutExplain( - title("Outer Join"), - description( - "Outer join is used to retain documents from one or both indices although it does not satisfy", - "join predicate. For now, only ``LEFT OUTER JOIN`` is supported to retain rows from first index.", - "Note that keyword ``OUTER`` is optional." - ), - post(multiLine( - "SELECT", - " a.account_number, a.firstname, a.lastname,", - " e.id, e.name", - "FROM accounts a", - "LEFT JOIN employees_nested e", - " ON a.account_number = e.id" - )) - ) - ); - } - - @Ignore("Multi-query doesn't work for default format: https://github.com/opendistro-for-elasticsearch/sql/issues/388") - @Section(3) - public void setOperations() { - section( - title("Set Operations"), - description( - "Set operations allow results of multiple queries to be combined into a single result set.", - "The results to be combined are required to be of same type. In other word, they require to", - "have same column. Otherwise, a semantic analysis exception is raised." - ), - example( - title("UNION Operator"), - description( - "A ``UNION`` clause combines the results of two queries into a single result set. Duplicate rows", - "are removed unless ``UNION ALL`` clause is being used. A common use case of ``UNION`` is to combine", - "result set from data partitioned in indices daily or monthly." - ), - post(multiLine( - "SELECT balance, firstname, lastname", - "FROM accounts WHERE balance < 10000", - "UNION", - "SELECT balance, firstname, lastname", - "FROM accounts WHERE balance > 30000" - )) - ), - example( - title("MINUS Operator"), - description( - "A ``MINUS`` clause takes two queries too but returns resulting rows of first query that", - "do not appear in the other query. Duplicate rows are removed automatically as well." - ), - post(multiLine( - "SELECT balance, age", - "FROM accounts", - "WHERE balance < 10000", - "MINUS", - "SELECT balance, age", - "FROM accounts", - "WHERE age < 35" - )) - ) - ); - } - - private Example joinExampleWithoutExplain(String title, String description, Requests requests) { - return example(title, description, requests, - queryFormat(OPENSEARCH_DASHBOARD_REQUEST, TABLE_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/dql/MetaDataQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/dql/MetaDataQueryIT.java deleted file mode 100644 index 6de19690b09..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/dql/MetaDataQueryIT.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.dql; - -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.OPENSEARCH_DASHBOARD_REQUEST; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.TABLE_RESPONSE; - -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; -import org.opensearch.sql.doctest.core.builder.Example; -import org.opensearch.sql.doctest.core.builder.Requests; - -@DocTestConfig(template = "dql/metadata.rst", testData = {"accounts.json", "employees_nested.json"}) -public class MetaDataQueryIT extends DocTest { - - @Section(1) - public void queryMetaData() { - section( - title("Querying Metadata"), - description( - "You can query your indices metadata by ``SHOW`` and ``DESCRIBE`` statement. These commands are", - "very useful for database management tool to enumerate all existing indices and get basic information", - "from the cluster." - ), - images("rdd/showStatement.png", "rdd/showFilter.png"), - metadataQueryExample( - title("Show All Indices Information"), - description( - "``SHOW`` statement lists all indices that match the search pattern. By using wildcard '%',", - "information for all indices in the cluster is returned." - ), - post("SHOW TABLES LIKE %") - ), - metadataQueryExample( - title("Show Specific Index Information"), - description( - "Here is an example that searches metadata for index name prefixed by 'acc'"), - post("SHOW TABLES LIKE acc%") - ), - metadataQueryExample( - title("Describe Index Fields Information"), - description( - "``DESCRIBE`` statement lists all fields for indices that can match the search pattern."), - post("DESCRIBE TABLES LIKE accounts") - ) - ); - } - - /** - * Explain doesn't work for SHOW/DESCRIBE so skip it - */ - private Example metadataQueryExample(String title, String description, Requests requests) { - return example(title, description, requests, - queryFormat(OPENSEARCH_DASHBOARD_REQUEST, TABLE_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/dql/SQLFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/dql/SQLFunctionsIT.java deleted file mode 100644 index 00b1f11cace..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/dql/SQLFunctionsIT.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.dql; - -import static org.opensearch.sql.legacy.antlr.semantic.types.TypeExpression.TypeExpressionSpec; - -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; -import org.opensearch.sql.legacy.antlr.semantic.types.function.ScalarFunction; -import org.opensearch.sql.legacy.utils.StringUtils; - -@DocTestConfig(template = "dql/functions.rst") -public class SQLFunctionsIT extends DocTest { - - /** - * List only specifications of all SQL functions supported for now - */ - @Section - public void listFunctions() { - for (ScalarFunction func : ScalarFunction - .values()) { // Java Enum.values() return enums in order they are defined - section( - title(func.getName()), - description(listFunctionSpecs(func)) - ); - } - } - - private String listFunctionSpecs(ScalarFunction func) { - TypeExpressionSpec[] specs = func.specifications(); - if (specs.length == 0) { - return "Specification is undefined and type check is skipped for now"; - } - - StringBuilder specStr = new StringBuilder("Specifications: \n\n"); - for (int i = 0; i < specs.length; i++) { - specStr.append( - StringUtils.format("%d. %s%s\n", (i + 1), func.getName(), specs[i]) - ); - } - return specStr.toString(); - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/interfaces/EndpointIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/interfaces/EndpointIT.java deleted file mode 100644 index 00d32e5b95f..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/interfaces/EndpointIT.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.interfaces; - -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.CURL_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; - -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; - -/** - * Doc test for endpoints to access the plugin. - */ -@DocTestConfig(template = "interfaces/endpoint.rst", testData = {"accounts.json"}) -public class EndpointIT extends DocTest { - - @Section(1) - public void queryByPost() { - section( - title("POST"), - description("You can also send HTTP POST request with your query in request body."), - example( - description(), - post("SELECT * FROM accounts"), - queryFormat(CURL_REQUEST, IGNORE_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - - @Section(2) - public void explainQuery() { - section( - title("Explain"), - description( - "To translate your query, send it to explain endpoint. The explain output is OpenSearch", - "domain specific language (DSL) in JSON format. You can just copy and paste it to your", - "console to run it against OpenSearch directly." - ), - example( - description(), - post("SELECT firstname, lastname FROM accounts WHERE age > 20"), - queryFormat(IGNORE_REQUEST, IGNORE_RESPONSE), - explainFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE) - ) - ); - } - - @Section(3) - public void cursorQuery() { - section( - title("Cursor"), - description( - "To get paginated response for a query, user needs to provide `fetch_size` parameter as part of normal query.", - "The value of `fetch_size` should be greater than `0`. In absence of `fetch_size`, default value of 1000 is used.", - "A value of `0` will fallback to non-paginated response.", - "This feature is only available over `jdbc` format for now." - ), - example( - description(), - post("SELECT firstname, lastname FROM accounts WHERE age > 20 ORDER BY state ASC"), - queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/interfaces/ProtocolIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/interfaces/ProtocolIT.java deleted file mode 100644 index 4ebf4899d20..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/interfaces/ProtocolIT.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.interfaces; - -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.CURL_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.ORIGINAL_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; - -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; - -/** - * Doc test for plugin supported protocols. - */ -@DocTestConfig(template = "interfaces/protocol.rst", testData = {"accounts.json"}) -public class ProtocolIT extends DocTest { - - @Section(1) - public void requestFormat() { - section( - title("Request Format"), - description( - "The body of HTTP POST request can take a few more other fields with SQL query."), - example( - description( - "Use `filter` to add more conditions to OpenSearch DSL directly." - ), - post( - body( - "\"query\": \"SELECT firstname, lastname, balance FROM accounts\"", - "\"filter\":{\"range\":{\"balance\":{\"lt\":10000}}}" - ) - ), - queryFormat(CURL_REQUEST, IGNORE_RESPONSE), - explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) - ), - example( - description("Use `parameters` for actual parameter value in prepared SQL query."), - post( - body( - "\"query\": \"SELECT * FROM accounts WHERE age = ?\"", - "\"parameters\": [{\"type\": \"integer\", \"value\": 30}]" - ) - ), - queryFormat(CURL_REQUEST, IGNORE_RESPONSE), - explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) - ) - ); - } - - @Section(2) - public void responseInJDBCFormat() { - section( - title("JDBC Format"), - description( - "By default the plugin return JDBC format. JDBC format is provided for JDBC driver and client side that needs both schema and", - "result set well formatted." - ), - example( - description( - "Here is an example for normal response. The `schema` includes field name and its type", - "and `datarows` includes the result set." - ), - post("SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2"), - queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ), - example( - description( - "If any error occurred, error message and the cause will be returned instead."), - post("SELECT unknown FROM accounts", params("format=jdbc")), - queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - - @Section(3) - public void originalDSLResponse() { - section( - title("OpenSearch DSL"), - description( - "The plugin returns original response from OpenSearch in JSON. Because this is", - "the native response from OpenSearch, extra efforts are needed to parse and interpret it." - ), - example( - description(), - post("SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2", - params("format=json")), - queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - - @Section(4) - public void responseInCSVFormat() { - section( - title("CSV Format"), - description("You can also use CSV format to download result set as CSV."), - example( - description(), - post("SELECT firstname, lastname, age FROM accounts ORDER BY age", - params("format=csv")), - queryFormat(CURL_REQUEST, ORIGINAL_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - - @Section(5) - public void responseInRawFormat() { - section( - title("Raw Format"), - description( - "Additionally raw format can be used to pipe the result to other command line tool for post processing." - ), - example( - description(), - post("SELECT firstname, lastname, age FROM accounts ORDER BY age", - params("format=raw")), - queryFormat(CURL_REQUEST, ORIGINAL_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - -}