diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000000..7f92a3c6aa0 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# This should match the owning team set up in https://github.com/orgs/opensearch-project/teams +* @opensearch-project/sql \ No newline at end of file diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml new file mode 100644 index 00000000000..e47d8d88c02 --- /dev/null +++ b/.github/workflows/backport.yml @@ -0,0 +1,28 @@ +name: Backport +on: + pull_request_target: + types: + - closed + - labeled + +jobs: + backport: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + name: Backport + steps: + - name: GitHub App token + id: github_app_token + uses: tibdex/github-app-token@v1.5.0 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + installation_id: 22958780 + + - name: Backport + uses: VachaShah/backport@v1.1.4 + with: + github_token: ${{ steps.github_app_token.outputs.token }} + branch_name: backport/backport-${{ github.event.number }} diff --git a/.github/workflows/bi-connectors.yml b/.github/workflows/bi-connectors.yml new file mode 100644 index 00000000000..7e339967b0d --- /dev/null +++ b/.github/workflows/bi-connectors.yml @@ -0,0 +1,54 @@ +name: Build connectors for BI tools + +on: + push: + paths: + - 'bi-connectors/PowerBIConnector/**' + - 'bi-connectors/TableauConnector/**' + - '.github/workflows/bi-connectors.yml' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Pack Tableau JDBC connector + id: pack-tableau-jdbc + run: | + zip -r opensearch_sql_jdbc.taco . -x *.taco + working-directory: bi-connectors/TableauConnector/opensearch_sql_jdbc + - name: Prepare Power BI ODBC connector for 'OpenSearch Project' + run: | + cp OpenSearchProject.pq OpenSearchProject.m + working-directory: bi-connectors/PowerBIConnector + - name: Pack Power BI ODBC connector + id: pack-powerbi-odbc-os-proj + run: | + zip OpenSearchProject.mez *.png *.m *.resx *.pqm + working-directory: bi-connectors/PowerBIConnector + - name: Prepare Power BI ODBC connector for 'Amazon OpenSearch Service' + id: prep-powerbi-odbc-amz-os-svc + run: | + mv OpenSearchProject.m AmazonOpenSearchService.m + sed -i 's/OpenSearch Project<\/value>/Amazon OpenSearch Service<\/value>/g' resources.resx + sed -i 's/OpenSearch Project/Amazon OpenSearch Service/g' AmazonOpenSearchService.m + sed -i 's/OpenSearchProject/AmazonOpenSearchService/g' AmazonOpenSearchService.m + sed -i 's/opensearchproject/amazonopensearchservice/g' AmazonOpenSearchService.m + working-directory: bi-connectors/PowerBIConnector + - name: Pack Power BI ODBC connector for 'Amazon OpenSearch Service' + id: pack-powerbi-odbc-amz-os-svc + run: | + zip AmazonOpenSearchService.mez *.png *.m *.resx *.pqm + working-directory: bi-connectors/PowerBIConnector + - name: Upload Tableau JDBC connector + if: steps.pack-tableau-jdbc.outcome == 'success' + uses: actions/upload-artifact@v2 + with: + name: TableauConnectors + path: bi-connectors/TableauConnector/opensearch_sql_jdbc/opensearch_sql_jdbc.taco + - name: Upload Power BI ODBC connectors + if: steps.pack-powerbi-odbc-os-proj.outcome == 'success' || (steps.prep-powerbi-odbc-amz-os-svc.outcome == 'success' && steps.pack-powerbi-odbc-amz-os-svc.outcome == 'success') + uses: actions/upload-artifact@v2 + with: + name: PBIConnectors + path: 'bi-connectors/PowerBIConnector/*.mez' diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml new file mode 100644 index 00000000000..53ed5304c2a --- /dev/null +++ b/.github/workflows/dco.yml @@ -0,0 +1,18 @@ +name: Developer Certificate of Origin Check + +on: [pull_request] + +jobs: + check: + runs-on: ubuntu-latest + + steps: + - name: Get PR Commits + id: 'get-pr-commits' + uses: tim-actions/get-pr-commits@v1.1.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: DCO Check + uses: tim-actions/dco@v1.1.0 + with: + commits: ${{ steps.get-pr-commits.outputs.commits }} \ No newline at end of file diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml new file mode 100644 index 00000000000..387a124b8cb --- /dev/null +++ b/.github/workflows/delete_backport_branch.yml @@ -0,0 +1,15 @@ +name: Delete merged branch of the backport PRs +on: + pull_request: + types: + - closed + +jobs: + delete-branch: + runs-on: ubuntu-latest + if: startsWith(github.event.pull_request.head.ref,'backport/') + steps: + - name: Delete merged branch + uses: SvanBoxel/delete-merged-branch@main + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml index 0c6190bce1f..891dcb9cb09 100644 --- a/.github/workflows/draft-release-notes-workflow.yml +++ b/.github/workflows/draft-release-notes-workflow.yml @@ -16,6 +16,6 @@ jobs: with: config-name: draft-release-notes-config.yml tag: (None) - version: 1.1.0.0 + version: 1.3.0.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/link-checker.yml b/.github/workflows/link-checker.yml index 10cab37f8ab..b113ef24ec9 100644 --- a/.github/workflows/link-checker.yml +++ b/.github/workflows/link-checker.yml @@ -16,7 +16,7 @@ jobs: id: lychee uses: lycheeverse/lychee-action@master with: - args: --accept=200,403,429,999 "**/*.html" "**/*.md" "**/*.txt" --exclude "http://localhost*" "https://localhost" "https://odfe-node1:9200/" "https://community.tableau.com/docs/DOC-17978" ".*family.zzz" "https://pypi.python.org/pypi/opensearch-sql-cli/" "opensearch*" ".*@amazon.com" ".*email.com" "git@github.com" "http://timestamp.verisign.com/scripts/timstamp.dll" + args: --accept=200,403,429,999 "./**/*.html" "./**/*.md" "./**/*.txt" --exclude "http://localhost*" "https://localhost" "https://odfe-node1:9200/" "https://community.tableau.com/docs/DOC-17978" ".*family.zzz" "https://pypi.python.org/pypi/opensearchsql/" "opensearch*" ".*@amazon.com" ".*email.com" "git@github.com" "http://timestamp.verisign.com/scripts/timstamp.dll" ".*/PowerBIConnector/bin/Release" env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - name: Fail if there were link errors diff --git a/.github/workflows/sql-cli-test-and-build-workflow.yml b/.github/workflows/sql-cli-test-and-build-workflow.yml index 876780a86c2..3de1ff3aa59 100644 --- a/.github/workflows/sql-cli-test-and-build-workflow.yml +++ b/.github/workflows/sql-cli-test-and-build-workflow.yml @@ -53,11 +53,11 @@ jobs: - name: Create Artifact Path run: | - mkdir -p opensearch-sql-cli-builds - cp -r ./dist/*.tar.gz ./dist/*.whl opensearch-sql-cli-builds/ + mkdir -p opensearchsql-builds + cp -r ./dist/*.tar.gz ./dist/*.whl opensearchsql-builds/ - name: Upload Artifact uses: actions/upload-artifact@v2 with: - name: opensearch-sql-cli - path: sql-cli/opensearch-sql-cli-builds + name: opensearchsql + path: sql-cli/opensearchsql-builds diff --git a/.github/workflows/sql-odbc-main.yml b/.github/workflows/sql-odbc-main.yml index c14b3354d07..0a22780aa96 100644 --- a/.github/workflows/sql-odbc-main.yml +++ b/.github/workflows/sql-odbc-main.yml @@ -11,7 +11,7 @@ env: jobs: build-mac: - runs-on: macos-latest + runs-on: macos-10.15 defaults: run: working-directory: sql-odbc @@ -78,7 +78,7 @@ jobs: # name: mac-test-results # path: test-output build-windows32: - runs-on: windows-latest + runs-on: windows-2019 defaults: run: working-directory: sql-odbc @@ -123,7 +123,7 @@ jobs: # name: windows-test-results # path: $CI_OUTPUT_PATH/test build-windows64: - runs-on: windows-latest + runs-on: windows-2019 defaults: run: working-directory: sql-odbc diff --git a/.github/workflows/sql-odbc-release-workflow.yml b/.github/workflows/sql-odbc-release-workflow.yml index 6e471248a50..b929507c4b3 100644 --- a/.github/workflows/sql-odbc-release-workflow.yml +++ b/.github/workflows/sql-odbc-release-workflow.yml @@ -12,11 +12,11 @@ env: ODBC_BUILD_PATH: "./build/odbc/build" AWS_SDK_INSTALL_PATH: "./build/aws-sdk/install" PLUGIN_NAME: opensearch-sql-odbc - OD_VERSION: 1.1.0.0 + OD_VERSION: 1.3.0.0 jobs: build-mac: - runs-on: macos-latest + runs-on: macos-10.15 defaults: run: working-directory: sql-odbc diff --git a/.github/workflows/sql-odbc-rename-and-release-workflow.yml b/.github/workflows/sql-odbc-rename-and-release-workflow.yml index cf79e281ffc..e97bf5012c9 100644 --- a/.github/workflows/sql-odbc-rename-and-release-workflow.yml +++ b/.github/workflows/sql-odbc-rename-and-release-workflow.yml @@ -8,7 +8,7 @@ on: - rename* env: - OD_VERSION: 1.1.0.0 + OD_VERSION: 1.3.0.0 jobs: upload-odbc: diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 5ae3f6aec47..dafe90a6fe3 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -2,33 +2,31 @@ name: SQL Java CI on: [push, pull_request] +env: + OPENSEARCH_VERSION: '1.3.0-SNAPSHOT' + jobs: build: - + strategy: + matrix: + java: + - 11 + - 14 runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - name: Set up JDK 1.14 + - name: Set up JDK ${{ matrix.java }} uses: actions/setup-java@v1 with: - java-version: 1.14 - - # Publish OpenSearch to local Maven repo for now - - name: Checkout OpenSearch - uses: actions/checkout@v2 - with: - repository: 'opensearch-project/OpenSearch' - path: OpenSearch - ref: '1.1' - - - name: Build OpenSearch - working-directory: ./OpenSearch - run: ./gradlew publishToMavenLocal + java-version: ${{ matrix.java }} - name: Build with Gradle - run: ./gradlew build assemble -Dopensearch.version=1.1.0-SNAPSHOT + run: ./gradlew build assemble -Dopensearch.version=${{ env.OPENSEARCH_VERSION }} + + - name: Run backward compatibility tests + run: ./bwctest.sh - name: Create Artifact Path run: | @@ -39,6 +37,7 @@ jobs: - name: Upload SQL Coverage Report uses: codecov/codecov-action@v1 with: + flags: sql-engine token: ${{ secrets.CODECOV_TOKEN }} - name: Upload Artifacts diff --git a/.github/workflows/sql-workbench-release-workflow.yml b/.github/workflows/sql-workbench-release-workflow.yml index 0e8e7126903..3a9353acbec 100644 --- a/.github/workflows/sql-workbench-release-workflow.yml +++ b/.github/workflows/sql-workbench-release-workflow.yml @@ -7,8 +7,8 @@ on: env: PLUGIN_NAME: query-workbench-dashboards - OPENSEARCH_VERSION: '1.0' - OPENSEARCH_PLUGIN_VERSION: 1.0.0.0 + OPENSEARCH_VERSION: '1.x' + OPENSEARCH_PLUGIN_VERSION: 1.3.0.0 jobs: diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index 071f7dfb29f..a534dac831d 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -5,7 +5,7 @@ on: [pull_request, push] env: PLUGIN_NAME: query-workbench-dashboards OPENSEARCH_VERSION: '1.x' - OPENSEARCH_PLUGIN_VERSION: 1.1.0.0 + OPENSEARCH_PLUGIN_VERSION: 1.3.0.0 jobs: @@ -43,7 +43,14 @@ jobs: - name: Test run: | cd OpenSearch-Dashboards/plugins/workbench - yarn test:jest + yarn test:jest --coverage + + - name: Upload coverage + uses: codecov/codecov-action@v1 + with: + flags: query-workbench + directory: ./OpenSearch-Dashboards/plugins/workbench + token: ${{ secrets.CODECOV_TOKEN }} - name: Build Artifact run: | diff --git a/.gitignore b/.gitignore index 415d7d43a30..7eacb4168b8 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ .project # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* -/bin/ +bin/ /target/ log/ elasticsearch-sql.iml diff --git a/.whitesource b/.whitesource new file mode 100644 index 00000000000..db4b0fec82c --- /dev/null +++ b/.whitesource @@ -0,0 +1,15 @@ +{ + "scanSettings": { + "configMode": "AUTO", + "configExternalURL": "", + "projectToken": "", + "baseBranches": [] + }, + "checkRunSettings": { + "vulnerableCheckRunConclusionLevel": "failure", + "displayMode": "diff" + }, + "issueSettings": { + "minSeverityLevel": "LOW" + } +} \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4d917322329..880f411ff00 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,97 @@ -## Contributing to this project +- [Contributing to OpenSearch SQL](#contributing-to-opensearch-sql) + - [First Things First](#first-things-first) + - [Ways to Contribute](#ways-to-contribute) + - [Bug Reports](#bug-reports) + - [Feature Requests](#feature-requests) + - [Documentation Changes](#documentation-changes) + - [Contributing Code](#contributing-code) + - [Developer Certificate of Origin](#developer-certificate-of-origin) + - [Review Process](#review-process) -OpenSearch is a community project that is built and maintained by people just like **you**. -[This document](https://github.com/opensearch-project/.github/blob/main/CONTRIBUTING.md) explains how you can contribute to this and related projects. \ No newline at end of file +# Contributing to OpenSearch SQL + +OpenSearch is a community project that is built and maintained by people just like you. We're glad you're interested in helping out. There are several different ways you can do it, but before we talk about that, let's talk about how to get started. + +## First Things First + +**When in doubt, open an issue** - For almost any type of contribution the first step is opening an issue. Even if you think you already know what the solution is, writing down a description of the problem you're trying to solve will help everyone get context when they review your pull request. If it's truly a trivial change (e.g. spelling error), you can skip this step -- but as the subject says, when it doubt, [open an issue](https://github.com/opensearch-project/sql/issues). + +**Only submit your own work** (or work you have sufficient rights to submit) - Please make sure that any code or documentation you submit is your work or you have the rights to submit. We respect the intellectual property rights of others, and as part of contributing, we'll ask you to sign your contribution with a "Developer Certificate of Origin" (DCO) that states you have the rights to submit this work and you understand we'll use your contribution. There's more information about this topic in the [DCO section](#developer-certificate-of-origin). + +## Ways to Contribute +### Bug Reports + +A bug is when software behaves in a way that you didn't expect and the developer didn't intend. To help us understand what's going on, we first want to make sure you're working from the latest version. Please make sure you're testing against the [latest version](https://github.com/opensearch-project/sql). + +Once you've confirmed that the bug still exists in the latest version, you'll want to check to make sure it's not something we already know about on the [open issues GitHub page](https://github.com/opensearch-project/sql/issues). + +If you've upgraded to the latest version and you can't find it in our open issues list, then you'll need to tell us how to reproduce it. Please provides us with as much context and information as possible (e.g. OS and browser version). + +### Feature Requests + +If you've thought of a way that OpenSearch SQL could be better, we want to hear about it. We track feature requests using GitHub, so please feel free to open an [issue](https://github.com/opensearch-project/sql/issues) which describes the feature you would like to see, why you need it, and how it should work. + +### Documentation Changes + +If you would like to contribute to the documentation, please do so in the [documentation-website](https://github.com/opensearch-project/documentation-website) repo. + +### Contributing Code + +As with other types of contributions, the first step is to [**open an issue on GitHub**](https://github.com/opensearch-project/sql/issues/new/choose). Opening an issue before you make changes makes sure that someone else isn't already working on that particular problem. It also lets us all work together to find the right approach before you spend a bunch of time on a PR. So again, when in doubt, open an issue. + +Once you've opened an issue, check out our [Developer Guide](./DEVELOPER_GUIDE.rst) for instructions on how to get started. + +## Developer Certificate of Origin + +OpenSearch is an open source product released under the Apache 2.0 license (see either [the Apache site](https://www.apache.org/licenses/LICENSE-2.0) or the [LICENSE.txt file](./LICENSE.txt)). The Apache 2.0 license allows you to freely use, modify, distribute, and sell your own products that include Apache 2.0 licensed software. + +We respect intellectual property rights of others and we want to make sure all incoming contributions are correctly attributed and licensed. A Developer Certificate of Origin (DCO) is a lightweight mechanism to do that. + +The DCO is a declaration attached to every contribution made by every developer. In the commit message of the contribution, the developer simply adds a `Signed-off-by` statement and thereby agrees to the DCO, which you can find below or at [DeveloperCertificate.org](http://developercertificate.org/). + +``` +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the + best of my knowledge, is covered under an appropriate open + source license and I have the right under that license to + submit that work with modifications, whether created in whole + or in part by me, under the same open source license (unless + I am permitted to submit under a different license), as + Indicated in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including + all personal information I submit with it, including my + sign-off) is maintained indefinitely and may be redistributed + consistent with this project or the open source license(s) + involved. + ``` +We require that every contribution to OpenSearch is signed with a Developer Certificate of Origin. Additionally, please use your real name. We do not accept anonymous contributors nor those utilizing pseudonyms. + +Each commit must include a DCO which looks like this + +``` +Signed-off-by: Jane Smith +``` +You may type this line on your own when writing your commit messages. However, if your user.name and user.email are set in your git configs, you can use `-s` or `--signoff` to add the `Signed-off-by` line to the end of the commit message. + +## Review Process + +We deeply appreciate everyone who takes the time to make a contribution. We will review all contributions as quickly as possible. As a reminder, opening an issue and discussing your change before you make it is the best way to smooth the PR process. This will prevent a rejection because someone else is already working on the problem, or because the solution is incompatible with the architectural direction. + +During the PR process, expect that there will be some back-and-forth. Please try to respond to comments in a timely fashion, and if you don't wish to continue with the PR, let us know. If a PR takes too many iterations for its complexity or size, we may reject it. Additionally, if you stop responding we may close the PR as abandoned. In either case, if you feel this was done in error, please add a comment on the PR. + +If we accept the PR, a [maintainer](MAINTAINERS.md) will merge your change and usually take care of backporting it to appropriate branches ourselves. + +If we reject the PR, we will close the pull request with a comment explaining why. This decision isn't always final: if you feel we have misunderstood your intended change or otherwise think that we should reconsider then please continue the conversation with a comment on the PR and we'll do our best to address any further points you raise. \ No newline at end of file diff --git a/DEVELOPER_GUIDE.rst b/DEVELOPER_GUIDE.rst index 1b634241db9..e49b9106af4 100644 --- a/DEVELOPER_GUIDE.rst +++ b/DEVELOPER_GUIDE.rst @@ -17,15 +17,15 @@ Prerequisites JDK --- -Specific version of JDK is required to build the plugin because of the dependency on OpenSearch test framework in our integration test. So you must have the required version of JDK installation on your machine. After the installation, please configure the ``JAVA_HOME`` environment variable accordingly. If everything goes right, you should something similar to this sample output on macOS (take OpenJDK 14 for example):: +OpenSearch builds using Java 11 at a minimum. This means you must have a JDK 11 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 11 installation:: $ echo $JAVA_HOME - /Library/Java/JavaVirtualMachines/adoptopenjdk-14.jdk/Contents/Home + /Library/Java/JavaVirtualMachines/adoptopenjdk-11.jdk/Contents/Home $ java -version - openjdk version "14.0.1" 2020-04-14 - OpenJDK Runtime Environment AdoptOpenJDK (build 14.0.1+7) - OpenJDK 64-Bit Server VM AdoptOpenJDK (build 14.0.1+7, mixed mode, sharing) + openjdk version "11.0.1" 2018-10-16 + OpenJDK Runtime Environment 18.9 (build 11.0.1+13) + OpenJDK 64-Bit Server VM 18.9 (build 11.0.1+13, mixed mode) Here are the official instructions on how to set ``JAVA_HOME`` for different platforms: https://docs.oracle.com/cd/E19182-01/820-7851/inst_cli_jdk_javahome_t/. @@ -381,3 +381,13 @@ Doctest >>>>>>> Python doctest library makes our document executable which keeps it up-to-date to source code. The doc generator aforementioned served as scaffolding and generated many docs in short time. Now the examples inside is changed to doctest gradually. For more details please read `Doctest <./dev/Doctest.md>`_. + + +Backports +>>>>>>>>> + +The Github workflow in `backport.yml <.github/workflows/backport.yml>`_ creates backport PRs automatically when the original PR +with an appropriate label `backport ` is merged to main with the backport workflow run successfully on the +PR. For example, if a PR on main needs to be backported to `1.x` branch, add a label `backport 1.x` to the PR and make sure the +backport workflow runs on the PR along with other checks. Once this PR is merged to main, the workflow will create a backport PR +to the `1.x` branch. diff --git a/MAINTAINERS.md b/MAINTAINERS.md index cb4ea162d08..734a390acba 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -7,5 +7,6 @@ | Anirudha (Ani) Jadhav | [anirudha](https://github.com/anirudha) | Amazon | | Peng Huo | [penghuo](https://github.com/penghuo) | Amazon | | Chen Dai | [dai-chen](https://github.com/dai-chen) | Amazon | +| Chloe Zhang | [chloe-zh](https://github.com/chloe-zh) | Amazon | | Nick Knize | [nknize](https://github.com/nknize) | Amazon | | Charlotte Henkle | [CEHENKLE](https://github.com/CEHENKLE) | Amazon | \ No newline at end of file diff --git a/NOTICE b/NOTICE index 36b1c77777e..20c193fcb65 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ OpenSearch -Copyright 2021 OpenSearch Contributors +Copyright OpenSearch Contributors This product includes software developed by Elasticsearch (http://www.elastic.co). diff --git a/README.md b/README.md index e679c64ccc0..889b6605cc3 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,15 @@ + -[![Test and Build Workflow](https://github.com/opendistro-for-elasticsearch/sql/workflows/Java%20CI/badge.svg)](https://github.com/opendistro-for-elasticsearch/sql/actions) -[![codecov](https://codecov.io/gh/opendistro-for-elasticsearch/sql/branch/develop/graph/badge.svg)](https://codecov.io/gh/opendistro-for-elasticsearch/sql) -[![Documentation](https://img.shields.io/badge/api-reference-blue.svg)](https://docs-beta.opensearch.org/search-plugins/sql/endpoints/) -[![Chat](https://img.shields.io/badge/chat-on%20forums-blue)](https://discuss.opendistrocommunity.dev/c/sql/) -![PRs welcome!](https://img.shields.io/badge/PRs-welcome!-success) +- [OpenSearch SQL](#opensearch-sql) +- [Code Summary](#code-summary) +- [Highlights](#highlights) +- [Documentation](#documentation) +- [Contributing](#contributing) +- [Attribution](#attribution) +- [Code of Conduct](#code-of-conduct) +- [Security](#security) +- [License](#license) +- [Copyright](#copyright) @@ -24,11 +30,111 @@ OpenSearch enables you to extract insights out of OpenSearch using the familiar The following projects have been merged into this repository as separate folders as of July 9, 2020. Please refer to links below for details. This document will focus on the SQL plugin for OpenSearch. -* [SQL CLI](https://github.com/opensearch-project/sql/tree/main/sql-cli) -* [SQL JDBC](https://github.com/opensearch-project/sql/tree/main/sql-jdbc) -* [SQL ODBC](https://github.com/opensearch-project/sql/tree/main/sql-odbc) -* [Query Workbench](https://github.com/opensearch-project/sql/tree/main/workbench) +- [SQL CLI](https://github.com/opensearch-project/sql/tree/main/sql-cli) +- [SQL JDBC](https://github.com/opensearch-project/sql/tree/main/sql-jdbc) +- [SQL ODBC](https://github.com/opensearch-project/sql/tree/main/sql-odbc) +- [Query Workbench](https://github.com/opensearch-project/sql/tree/main/workbench) + +## Code Summary + +### SQL Engine + +| | | +| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | +| Test and build | [![SQL CI][sql-ci-badge]][sql-ci-link] | +| Code coverage | [![codecov][sql-codecov-badge]][sql-codecov-link] | +| Distribution build tests | [![OpenSearch IT tests][opensearch-it-badge]][opensearch-it-link] [![OpenSearch IT code][opensearch-it-code-badge]][opensearch-it-code-link] | +| Backward compatibility tests | [![BWC tests][bwc-tests-badge]][bwc-tests-link] | + +### ODBC Driver + +| | | +| ----- | ----------------------------------------------- | +| Build | [![ODBC CI][odbc-build-badge]][odbc-build-link] | + +### SQL CLI + +| | | +| ----- | ----------------------------------------------- | +| Test and build | [![SQL CLI CI][sql-cli-build-badge]][sql-cli-build-link] | + +### JDBC Driver + +| | | +| ----- | ----------------------------------------------- | +| Build | [![JDBC CI][jdbc-build-badge]][jdbc-build-link] | + +### Query WorkBench + +| | | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------ | +| Test and build | [![Observability Dashboards CI][workbench-build-badge]][workbench-build-link] | +| Code coverage | [![codecov][workbench-codecov-badge]][sql-codecov-link] | +| Distribution build tests | [![cypress tests][cypress-test-badge]][cypress-test-link] [![cypress code][cypress-code-badge]][cypress-code-link] | + +### Repository Checks + +| | | +| ------------ | --------------------------------------------------------------- | +| DCO Checker | [![Developer certificate of origin][dco-badge]][dco-badge-link] | +| Link Checker | [![Link Checker][link-check-badge]][link-check-link] | + +### Issues + +| | +| -------------------------------------------------------------- | +| [![good first issues open][good-first-badge]][good-first-link] | +| [![features open][feature-badge]][feature-link] | +| [![enhancements open][enhancement-badge]][enhancement-link] | +| [![bugs open][bug-badge]][bug-link] | +| [![untriaged open][untriaged-badge]][untriaged-link] | +| [![nolabel open][nolabel-badge]][nolabel-link] | + +[dco-badge]: https://github.com/opensearch-project/sql/actions/workflows/dco.yml/badge.svg +[dco-badge-link]: https://github.com/opensearch-project/sql/actions/workflows/dco.yml +[link-check-badge]: https://github.com/opensearch-project/sql/actions/workflows/link-checker.yml/badge.svg +[link-check-link]: https://github.com/opensearch-project/sql/actions/workflows/link-checker.yml +[odbc-build-badge]: https://github.com/opensearch-project/sql/actions/workflows/sql-odbc-main.yml/badge.svg +[odbc-build-link]: https://github.com/opensearch-project/sql/actions/workflows/sql-odbc-main.yml +[sql-cli-build-badge]: https://github.com/opensearch-project/sql/actions/workflows/sql-cli-test-and-build-workflow.yml/badge.svg +[sql-cli-build-link]: https://github.com/opensearch-project/sql/actions/workflows/sql-cli-test-and-build-workflow.yml +[jdbc-build-badge]: https://github.com/opensearch-project/sql/actions/workflows/sql-jdbc-test-and-build-workflow.yml/badge.svg +[jdbc-build-link]: https://github.com/opensearch-project/sql/actions/workflows/sql-jdbc-test-and-build-workflow.yml +[sql-ci-badge]: https://github.com/opensearch-project/sql/actions/workflows/sql-test-and-build-workflow.yml/badge.svg +[sql-ci-link]: https://github.com/opensearch-project/sql/actions/workflows/sql-test-and-build-workflow.yml +[bwc-tests-badge]: https://img.shields.io/badge/BWC%20tests-in%20progress-yellow +[bwc-tests-link]: https://github.com/opensearch-project/sql/issues/193 +[good-first-badge]: https://img.shields.io/github/issues/opensearch-project/sql/good%20first%20issue.svg +[good-first-link]: https://github.com/opensearch-project/sql/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22+ +[feature-badge]: https://img.shields.io/github/issues/opensearch-project/sql/feature.svg +[feature-link]: https://github.com/opensearch-project/sql/issues?q=is%3Aopen+is%3Aissue+label%3Afeature +[bug-badge]: https://img.shields.io/github/issues/opensearch-project/sql/bug.svg +[bug-link]: https://github.com/opensearch-project/sql/issues?q=is%3Aopen+is%3Aissue+label%3Abug+ +[enhancement-badge]: https://img.shields.io/github/issues/opensearch-project/sql/enhancement.svg +[enhancement-link]: https://github.com/opensearch-project/sql/issues?q=is%3Aopen+is%3Aissue+label%3Aenhancement+ +[untriaged-badge]: https://img.shields.io/github/issues/opensearch-project/sql/untriaged.svg +[untriaged-link]: https://github.com/opensearch-project/sql/issues?q=is%3Aopen+is%3Aissue+label%3Auntriaged+ +[nolabel-badge]: https://img.shields.io/github/issues-search/opensearch-project/sql?color=yellow&label=no%20label%20issues&query=is%3Aopen%20is%3Aissue%20no%3Alabel +[nolabel-link]: https://github.com/opensearch-project/sql/issues?q=is%3Aopen+is%3Aissue+no%3Alabel+ +[workbench-build-badge]: https://github.com/opensearch-project/sql/actions/workflows/sql-workbench-test-and-build-workflow.yml/badge.svg +[workbench-build-link]: https://github.com/opensearch-project/sql/actions/workflows/sql-workbench-test-and-build-workflow.yml +[cypress-test-badge]: https://img.shields.io/badge/Cypress%20tests-in%20progress-yellow +[cypress-test-link]: https://github.com/opensearch-project/opensearch-build/issues/1124 +[cypress-code-badge]: https://img.shields.io/badge/Cypress%20code-blue +[cypress-code-link]: https://github.com/opensearch-project/sql/tree/main/workbench/.cypress/integration +[sql-codecov-badge]: https://codecov.io/gh/opensearch-project/sql/branch/main/graphs/badge.svg?flag=sql-engine +[workbench-codecov-badge]: https://codecov.io/gh/opensearch-project/sql/branch/main/graphs/badge.svg?flag=query-workbench +[sql-codecov-link]: https://codecov.io/gh/opensearch-project/sql +[opensearch-it-badge]: https://img.shields.io/badge/SQL%20IT%20tests-in%20progress-yellow +[opensearch-it-link]: https://github.com/opensearch-project/opensearch-build/issues/1124 +[opensearch-it-code-badge]: https://img.shields.io/badge/SQL%20IT%20code-blue +[opensearch-it-code-link]: https://github.com/opensearch-project/sql/tree/main/integ-test +## Highlights + +Besides basic filtering and aggregation, OpenSearch SQL also supports complex queries, such as querying semi-structured data, JOINs, set operations, sub-queries etc. Beyond the standard functions, OpenSearch functions are provided for better analytics and visualization. Please check our [documentation](#documentation) for more details. + +Recently we have been actively improving our query engine primarily for better correctness and extensibility. Behind the scene, the new enhanced engine has already supported both SQL and Piped Processing Language. Please find more details in [SQL Engine V2 - Release Notes](./docs/dev/NewSQLEngine.md). ## Highlights @@ -39,14 +145,12 @@ Recently we have been actively improving our query engine primarily for better c ## Documentation -Please refer to the [SQL Language Reference Manual](./docs/user/index.rst), [Piped Processing Language (PPL) Reference Manual](./docs/user/ppl/index.rst) and [Technical Documentation](https://docs-beta.opensearch.org/) for detailed information on installing and configuring plugin. - +Please refer to the [SQL Language Reference Manual](./docs/user/index.rst), [Piped Processing Language (PPL) Reference Manual](./docs/user/ppl/index.rst) and [Technical Documentation](https://opensearch.org/docs/latest/search-plugins/sql/index/) for detailed information on installing and configuring plugin. ## Contributing See [developer guide](DEVELOPER_GUIDE.rst) and [how to contribute to this project](CONTRIBUTING.md). - ## Attribution This project is based on the Apache 2.0-licensed [elasticsearch-sql](https://github.com/NLPchina/elasticsearch-sql) project. Thank you [eliranmoyal](https://github.com/eliranmoyal), [shi-yuan](https://github.com/shi-yuan), [ansjsun](https://github.com/ansjsun) and everyone else who contributed great code to that project. Read this for more details [Attributions](./docs/attributions.md). @@ -56,17 +160,14 @@ This project is based on the Apache 2.0-licensed [elasticsearch-sql](https://git This project has adopted an [Open Source Code of Conduct](./CODE_OF_CONDUCT.md). - ## Security If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public GitHub issue. - ## License See the [LICENSE](./LICENSE.txt) file for our project's licensing. We will ask you to confirm the licensing of your contribution. - ## Copyright -Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +Copyright OpenSearch Contributors. See [NOTICE](./NOTICE) for details. diff --git a/sql-odbc/src/PowerBIConnector/Diagnostics.pqm b/bi-connectors/PowerBIConnector/Diagnostics.pqm similarity index 100% rename from sql-odbc/src/PowerBIConnector/Diagnostics.pqm rename to bi-connectors/PowerBIConnector/Diagnostics.pqm diff --git a/sql-odbc/src/PowerBIConnector/OdbcConstants.pqm b/bi-connectors/PowerBIConnector/OdbcConstants.pqm similarity index 100% rename from sql-odbc/src/PowerBIConnector/OdbcConstants.pqm rename to bi-connectors/PowerBIConnector/OdbcConstants.pqm diff --git a/sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector16.png b/bi-connectors/PowerBIConnector/OpenSearch16.png similarity index 100% rename from sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector16.png rename to bi-connectors/PowerBIConnector/OpenSearch16.png diff --git a/sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector20.png b/bi-connectors/PowerBIConnector/OpenSearch20.png similarity index 100% rename from sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector20.png rename to bi-connectors/PowerBIConnector/OpenSearch20.png diff --git a/sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector24.png b/bi-connectors/PowerBIConnector/OpenSearch24.png similarity index 100% rename from sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector24.png rename to bi-connectors/PowerBIConnector/OpenSearch24.png diff --git a/sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector32.png b/bi-connectors/PowerBIConnector/OpenSearch32.png similarity index 100% rename from sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector32.png rename to bi-connectors/PowerBIConnector/OpenSearch32.png diff --git a/sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector40.png b/bi-connectors/PowerBIConnector/OpenSearch40.png similarity index 100% rename from sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector40.png rename to bi-connectors/PowerBIConnector/OpenSearch40.png diff --git a/sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector48.png b/bi-connectors/PowerBIConnector/OpenSearch48.png similarity index 100% rename from sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector48.png rename to bi-connectors/PowerBIConnector/OpenSearch48.png diff --git a/sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector64.png b/bi-connectors/PowerBIConnector/OpenSearch64.png similarity index 100% rename from sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector64.png rename to bi-connectors/PowerBIConnector/OpenSearch64.png diff --git a/sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector80.png b/bi-connectors/PowerBIConnector/OpenSearch80.png similarity index 100% rename from sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector80.png rename to bi-connectors/PowerBIConnector/OpenSearch80.png diff --git a/sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector.mproj b/bi-connectors/PowerBIConnector/OpenSearchProject.mproj similarity index 87% rename from sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector.mproj rename to bi-connectors/PowerBIConnector/OpenSearchProject.mproj index b2b177c461f..938648e40f5 100644 --- a/sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector.mproj +++ b/bi-connectors/PowerBIConnector/OpenSearchProject.mproj @@ -18,7 +18,7 @@ False 1000 Yes - SqlOdbcPBIConnector + OpenSearch false @@ -36,31 +36,31 @@ - + Code - + Code - + Code - + Code - + Code - + Code - + Code - + Code - + Code @@ -72,7 +72,7 @@ Content - + Code @@ -98,6 +98,8 @@ archiveOut.CreateEntryFromFile(fullPath, filename, CompressionLevel.Optimal); } } + + File.Copy(OutputFile, Path.Combine(Environment.GetEnvironmentVariable("USERPROFILE"), @"Documents\Power BI Desktop\Custom Connectors", Path.GetFileName(OutputFile)), true); ]]> diff --git a/sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector.pq b/bi-connectors/PowerBIConnector/OpenSearchProject.pq similarity index 65% rename from sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector.pq rename to bi-connectors/PowerBIConnector/OpenSearchProject.pq index ba10c39e494..b3f65ddc998 100644 --- a/sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector.pq +++ b/bi-connectors/PowerBIConnector/OpenSearchProject.pq @@ -1,28 +1,44 @@ // This file contains Data Connector logic -section SqlOdbcPBIConnector; +[Version = "1.0.1"] +section OpenSearchProject; // When set to true, additional trace information will be written out to the User log. // This should be set to false before release. Tracing is done through a call to // Diagnostics.LogValue(). When EnableTraceOutput is set to false, the call becomes a // no-op and simply returns the original value. -EnableTraceOutput = true; +EnableTraceOutput = false; -[DataSource.Kind="SqlOdbcPBIConnector", Publish="SqlOdbcPBIConnector.Publish"] -shared SqlOdbcPBIConnector.Contents = Value.ReplaceType(SqlOdbcPBIConnectorImpl, SqlOdbcPBIConnectorType); +[DataSource.Kind="OpenSearchProject", Publish="OpenSearchProject.Publish"] +shared OpenSearchProject.Contents = Value.ReplaceType(OpenSearchProjectImpl, OpenSearchProjectType); // Wrapper function to provide additional UI customization. -SqlOdbcPBIConnectorType = type function ( +OpenSearchProjectType = type function ( Server as (type text meta [ Documentation.FieldCaption = "Server", Documentation.FieldDescription = "The hostname of the OpenSearch server.", - Documentation.SampleValues = { "localhost:9200" } + Documentation.SampleValues = { "localhost" } + ]), + Port as (type number meta [ + Documentation.FieldCaption = "Port", + Documentation.FieldDescription = "Port which OpenSearch server listens on.", + Documentation.SampleValues = { 9200 } + ]), + UseSSL as (type logical meta [ + Documentation.FieldCaption = "Use SSL", + Documentation.FieldDescription = "Use SSL", + Documentation.AllowedValues = { true, false } + ]), + HostnameVerification as (type logical meta [ + Documentation.FieldCaption = "Certificate validation", + Documentation.FieldDescription = "Certificate validation", + Documentation.AllowedValues = { true, false } ]) ) as table meta [ - Documentation.Name = "OpenSearch" + Documentation.Name = "OpenSearch Project" ]; -SqlOdbcPBIConnectorImpl = (Server as text) as table => +OpenSearchProjectImpl = (Server as text, Port as number, UseSSL as logical, HostnameVerification as logical) as table => let Credential = Extension.CurrentCredential(), AuthenticationMode = Credential[AuthenticationKind], @@ -56,9 +72,17 @@ SqlOdbcPBIConnectorImpl = (Server as text) as table => UseSSL = 0 ], + // Subtract the server from the user input in case it's entered like 'http://localhost' or 'https://srv.com:100500' or 'localhost:0' + // And build the proper string on our own + FinalServerString = if UseSSL then + "https://" & Uri.Parts(Server)[Host] & ":" & Text.From(Port) + else + "http://" & Uri.Parts(Server)[Host] & ":" & Text.From(Port), + ConnectionString = [ Driver = "OpenSearch SQL ODBC Driver", - Host = Server + Host = FinalServerString, + HostnameVerification = if HostnameVerification then 1 else 0 ], SQLGetInfo = Diagnostics.LogValue("SQLGetInfo_Options", [ @@ -106,8 +130,8 @@ SqlOdbcPBIConnectorImpl = (Server as text) as table => SupportsOdbcTimestampLiterals = true ]), - OdbcDatasource = Odbc.DataSource(ConnectionString & CredentialConnectionString & EncryptedConnectionString, [ - // Do not view the tables grouped by their schema names. + OdbcOptions = [ + // Do not view the tables grouped by their schema names. HierarchicalNavigation = false, // Prevents execution of native SQL statements. Extensions should set this to true. HideNativeQuery = true, @@ -129,7 +153,11 @@ SqlOdbcPBIConnectorImpl = (Server as text) as table => // Connection string properties used for encrypted connections. CredentialConnectionString = EncryptedConnectionString - ]) + ], + + FullConnectionString = (ConnectionString & CredentialConnectionString & EncryptedConnectionString), + + OdbcDatasource = Odbc.DataSource(FullConnectionString, OdbcOptions) in OdbcDatasource; @@ -149,21 +177,24 @@ OnOdbcError = (errorRecord as record) => OdbcErrorCode = 202 in if IsDriverNotInstalled then - error Error.Record("", "The OpenSearch SQL ODBC driver is not installed. Please install the driver") + error Error.Record("DataSource.Error", "The OpenSearch SQL ODBC driver is not installed. Please install the driver") else if IsHostUnreachable then - error Error.Record("", "Couldn't reach server. Please double-check the server and auth. [" & ConnectionServer & "]") + error Error.Record("DataSource.Error", "Couldn't reach server. Please double-check the server and auth. [" & ConnectionServer & "]") else error errorRecord; // Data Source Kind description -SqlOdbcPBIConnector = [ +OpenSearchProject = [ // Required for use with Power BI Service. TestConnection = (dataSourcePath) => let json = Json.Document(dataSourcePath), - Server = json[Server] + Server = json[Server], + Port = json[Port], + UseSSL = json[UseSSL], + HostnameVerification = json[HostnameVerification] in - { "SqlOdbcPBIConnector.Contents", Server }, + { "OpenSearchProject.Contents", Server, Port, UseSSL, HostnameVerification }, // Authentication modes Authentication = [ @@ -181,14 +212,14 @@ SqlOdbcPBIConnector = [ // PBIDS Handler DSRHandlers = [ - #"sqlodbc" = [ - GetDSR = (server, schema, object, optional options) => [ protocol = "sqlodbc", address = [ server = server ] ], - GetFormula = (dsr, optional options) => () => + opensearchproject = [ + GetDSR = (Server, Port, UseSSL, HostnameVerification, optional Options) => [ protocol = "opensearchproject-odbc", address = [ server = Server, port = Port, useSSL = UseSSL, hostnameVerification = HostnameVerification ] ], + GetFormula = (dsr, optional options) => () => let - db = SqlOdbcPBIConnector.Contents(dsr[address][server]) + db = OpenSearchProject.Contents(dsr[address][server], dsr[address][port], dsr[address][useSSL], dsr[address][hostnameVerification]) in db, - GetFriendlyName = (dsr) => "OpenSearch SQL ODBC" + GetFriendlyName = (dsr) => "OpenSearch Project" ] ], @@ -199,22 +230,21 @@ SqlOdbcPBIConnector = [ ]; // Data Source UI publishing description -SqlOdbcPBIConnector.Publish = [ - Beta = true, +OpenSearchProject.Publish = [ + Beta = false, Category = "Other", ButtonText = { Extension.LoadString("ButtonTitle"), Extension.LoadString("ButtonHelp") }, - LearnMoreUrl = "https://www.opensearch.org/", + LearnMoreUrl = "https://github.com/opensearch-project/sql/blob/main/bi-connectors/PowerBIConnector/docs/OpenSearchProject.md", - // Disabling direct query due to limited SQL query support - SupportsDirectQuery = false, + SupportsDirectQuery = true, - SourceImage = SqlOdbcPBIConnector.Icons, - SourceTypeImage = SqlOdbcPBIConnector.Icons + SourceImage = OpenSearch.Icons, + SourceTypeImage = OpenSearch.Icons ]; -SqlOdbcPBIConnector.Icons = [ - Icon16 = { Extension.Contents("SqlOdbcPBIConnector16.png"), Extension.Contents("SqlOdbcPBIConnector20.png"), Extension.Contents("SqlOdbcPBIConnector24.png"), Extension.Contents("SqlOdbcPBIConnector32.png") }, - Icon32 = { Extension.Contents("SqlOdbcPBIConnector32.png"), Extension.Contents("SqlOdbcPBIConnector40.png"), Extension.Contents("SqlOdbcPBIConnector48.png"), Extension.Contents("SqlOdbcPBIConnector64.png") } +OpenSearch.Icons = [ + Icon16 = { Extension.Contents("OpenSearch16.png"), Extension.Contents("OpenSearch20.png"), Extension.Contents("OpenSearch24.png"), Extension.Contents("OpenSearch32.png") }, + Icon32 = { Extension.Contents("OpenSearch32.png"), Extension.Contents("OpenSearch40.png"), Extension.Contents("OpenSearch48.png"), Extension.Contents("OpenSearch64.png") } ]; // Load common library functions diff --git a/sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector.query.pq b/bi-connectors/PowerBIConnector/OpenSearchProject.query.pq similarity index 94% rename from sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector.query.pq rename to bi-connectors/PowerBIConnector/OpenSearchProject.query.pq index bde4c98a34e..19c84006bf4 100644 --- a/sql-odbc/src/PowerBIConnector/SqlOdbcPBIConnector.query.pq +++ b/bi-connectors/PowerBIConnector/OpenSearchProject.query.pq @@ -1,21 +1,32 @@ // This file contains queries to test your data connector -section SqlOdbcPBIConnector.UnitTests; +section OpenSearch.UnitTests; shared MyExtension.UnitTest = [ // Common variables for all tests Host = "localhost", Port = 9200, + UseSSL = false, facts = { Fact("Connection Test", 7, let - Source = SqlOdbcPBIConnector.Contents(Host,Port), + Source = OpenSearch.Contents(Host, Port, UseSSL), no_of_columns = Table.ColumnCount(Source) in no_of_columns + ), + Fact("calcs_data:bool0", + #table(type table [bool0 = logical], + { {null}, {false}, {true} }), + let + Source = OpenSearch.Contents(Host, Port, UseSSL), + calcs_null_null = Source{[Item="calcs",Schema=null,Catalog=null]}[Data], + grouped = Table.Group(calcs_null_null, {"bool0"}, {}) + in + grouped ) }, @@ -33,11 +44,13 @@ Fact = (_subject as text, _expected, _actual) as record => resultOp = if result = "Success ✓" then " = " else " <> ", addendumEvalAttempt = if attempt[HasError] then @ValueToText(attempt[Error]) else "", addendumEvalExpected = try @ValueToText(safeExpected) otherwise "...", - addendumEvalActual = try @ValueToText (safeActual) otherwise "...", + addendumEvalActual = try @ValueToText(safeActual) otherwise "...", + ShortenedAddendumEvalExpected = if Text.Length(addendumEvalExpected) > 20 then Text.Range(addendumEvalExpected, 0, 20) & "..." else addendumEvalExpected, + ShortenedAddendumEvalActual = if Text.Length(addendumEvalActual) > 20 then Text.Range(addendumEvalActual, 0, 20) & "..." else addendumEvalActual, fact = - [ Result = result &" "& addendumEvalAttempt, + [ Result = result & " " & addendumEvalAttempt, Notes =_subject, - Details = " ("& addendumEvalExpected & resultOp & addendumEvalActual &")" + Details = ShortenedAddendumEvalExpected & resultOp & ShortenedAddendumEvalActual ] ][fact]; diff --git a/sql-odbc/src/PowerBIConnector/PBIDSExamples/sqlodbc_import.pbids b/bi-connectors/PowerBIConnector/PBIDSExamples/sqlodbc_import.pbids similarity index 100% rename from sql-odbc/src/PowerBIConnector/PBIDSExamples/sqlodbc_import.pbids rename to bi-connectors/PowerBIConnector/PBIDSExamples/sqlodbc_import.pbids diff --git a/bi-connectors/PowerBIConnector/docs/AmazonOpenSearchService.md b/bi-connectors/PowerBIConnector/docs/AmazonOpenSearchService.md new file mode 100644 index 00000000000..d6498a683c5 --- /dev/null +++ b/bi-connectors/PowerBIConnector/docs/AmazonOpenSearchService.md @@ -0,0 +1,79 @@ +# Connecting Amazon OpenSearch Service to Microsoft Power BI Desktop + +>[!Note] + +>The following connector article is provided by Amazon, the owner of this connector and a member of the Microsoft Power Query Connector Certification Program. If you have questions regarding the content of this article or have changes you would like to see made to this article, visit the OpenSearch website and use the support channels there. + +## Summary +| Item | Description | +| ---- | ----------- | +| Release State | General Availability | +| Products | Power BI Desktop | +| Authentication Types Supported | Basic | + +## Prerequisites +* Microsoft Power BI Desktop +* [OpenSearch](https://opensearch.org/docs/latest/opensearch/install/index/) +* [OpenSearch SQL ODBC driver](https://opensearch.org/docs/latest/search-plugins/sql/odbc/) + +## Capabilities supported +* Import +* DirectQuery + +## Connect to Amazon OpenSearch Service +1. Open Power BI Desktop. + +2. Click on **Home** > **Get Data** > **More** > **Other**. Select **Amazon OpenSearch Service**. Click on **Connect**. + + + +3. You will get a warning for using a third-party service. Click on **Continue**. + + + +4. Enter host and port values and select your preferred SSL and Certificate validation options. Click on **OK**. + + + +5. Select authentication option. Enter credentials if required and click on **Connect**. + + + +6. Select required table. Data preview will be loaded. + + + +7. Click on **Load**. + +8. Select required columns for creating a graph. + + + + +## Troubleshooting + +* If you get the following error, please install the [OpenSearch SQL ODBC Driver](https://docs-beta.opensearch.org/search-plugins/sql/odbc/). + + + +* If you get the following error, + + + +1. Check if host and port values are correct. +2. Check if auth credentials are correct. +3. Check if server is running. + +## Limitations and Known issues + +There are known limitations and issues that are tracked by OpenSearch including the items listed below. + +| Issue | Description | +| ---- | ----------- | +| [Visualizations without numerical columns](https://github.com/opensearch-project/sql/issues/347) | Visualizations is only supported if there is a numerical column included due to a known limitation in OpenSearch regarding subqueries. An exception to this would be visualizations with columns only of type string is supported as well. | +| [Visualizations with First or Last options selected are not yet supported](https://github.com/opensearch-project/sql/issues/279) | First and Last options generate an sql query that uses MAX and MIN for strings which is not yet supported in OpenSearch | +| [Visualizations with aggregate functions selected are not yet supported](https://github.com/opensearch-project/sql/issues/363) | Specifically Minimum, Maximum, Standard deviation, Variance, or Median options are not yet supported. | +| [Basic filtering limitations](https://github.com/opensearch-project/sql/issues/347) | Selecting more than one value is not yet supported when using basic filtering. Due to the issue in the first row of this table, selecting one value for basic filtering when there is no numerical column is not yet supported. | +| [Top N filtering limitations](https://opensearch.org/docs/latest/search-plugins/sql/limitation/) | OpenSearch has limitations on subquery which does not yet support the Top N filtering functionality. | +| [Advanced filtering limitations](https://github.com/opensearch-project/sql/issues/308) | `does not contain` and `does not start with` filters for string columns are not yet supported. All advanced filtering for numerical columns are not yet supported except for `is` and `is blank`. All advanced filtering for date and time columns are not yet supported except for `is blank` and `is not blank`. | +| [Relative Date filtering limitations](https://github.com/opensearch-project/sql/issues/364) | Due to a known timestamp issue in OpenSearch, all relative date filtering is not yet supported. | diff --git a/bi-connectors/PowerBIConnector/docs/CHANGELOG.md b/bi-connectors/PowerBIConnector/docs/CHANGELOG.md new file mode 100644 index 00000000000..080208c4e7a --- /dev/null +++ b/bi-connectors/PowerBIConnector/docs/CHANGELOG.md @@ -0,0 +1,11 @@ +# Changelog +All notable changes to the connector will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [1.0.1] 2022-02-28 +### Added +- Allow users to enable or disable validation for SSL certificate validation. +### Changed +- Changed beta to false diff --git a/bi-connectors/PowerBIConnector/docs/OpenSearchProject.md b/bi-connectors/PowerBIConnector/docs/OpenSearchProject.md new file mode 100644 index 00000000000..966c55a7e41 --- /dev/null +++ b/bi-connectors/PowerBIConnector/docs/OpenSearchProject.md @@ -0,0 +1,79 @@ +# Connecting OpenSearch Project to Microsoft Power BI Desktop + +>[!Note] + +>The following connector article is provided by OpenSearch, the owner of this connector and a member of the Microsoft Power Query Connector Certification Program. If you have questions regarding the content of this article or have changes you would like to see made to this article, visit the OpenSearch website and use the support channels there. + +## Summary +| Item | Description | +| ---- | ----------- | +| Release State | General Availability | +| Products | Power BI Desktop | +| Authentication Types Supported | Basic | + +## Prerequisites +* Microsoft Power BI Desktop +* [OpenSearch](https://opensearch.org/docs/latest/opensearch/install/index/) +* [OpenSearch SQL ODBC driver](https://opensearch.org/docs/latest/search-plugins/sql/odbc/) + +## Capabilities supported +* Import +* DirectQuery + +## Connect to OpenSearch Project +1. Open Power BI Desktop. + +2. Click on **Home** > **Get Data** > **More** > **Other**. Select **OpenSearch Project**. Click on **Connect**. + + + +3. You will get a warning for using a third-party service. Click on **Continue**. + + + +4. Enter host and port values and select your preferred SSL and Certificate validation options. Click on **OK**. + + + +5. Select authentication option. Enter credentials if required and click on **Connect**. + + + +6. Select required table. Data preview will be loaded. + + + +7. Click on **Load**. + +8. Select required columns for creating a graph. + + + + +## Troubleshooting + +* If you get the following error, please install the [OpenSearch SQL ODBC Driver](https://docs-beta.opensearch.org/search-plugins/sql/odbc/). + + + +* If you get the following error, + + + +1. Check if host and port values are correct. +2. Check if auth credentials are correct. +3. Check if server is running. + +## Limitations and Known issues + +There are known limitations and issues that are tracked by OpenSearch including the items listed below. + +| Issue | Description | +| ---- | ----------- | +| [Visualizations without numerical columns](https://github.com/opensearch-project/sql/issues/347) | Visualizations is only supported if there is a numerical column included due to a known limitation in OpenSearch regarding subqueries. An exception to this would be visualizations with columns only of type string is supported as well. | +| [Visualizations with First or Last options selected are not yet supported](https://github.com/opensearch-project/sql/issues/279) | First and Last options generate an sql query that uses MAX and MIN for strings which is not yet supported in OpenSearch | +| [Visualizations with aggregate functions selected are not yet supported](https://github.com/opensearch-project/sql/issues/363) | Specifically Minimum, Maximum, Standard deviation, Variance, or Median options are not yet supported. | +| [Basic filtering limitations](https://github.com/opensearch-project/sql/issues/347) | Selecting more than one value is not yet supported when using basic filtering. Due to the issue in the first row of this table, selecting one value for basic filtering when there is no numerical column is not yet supported. | +| [Top N filtering limitations](https://opensearch.org/docs/latest/search-plugins/sql/limitation/) | OpenSearch has limitations on subquery which does not yet support the Top N filtering functionality. | +| [Advanced filtering limitations](https://github.com/opensearch-project/sql/issues/308) | `does not contain` and `does not start with` filters for string columns are not yet supported. All advanced filtering for numerical columns are not yet supported except for `is` and `is blank`. All advanced filtering for date and time columns are not yet supported except for `is blank` and `is not blank`. | +| [Relative Date filtering limitations](https://github.com/opensearch-project/sql/issues/364) | Due to a known timestamp issue in OpenSearch, all relative date filtering is not yet supported. | diff --git a/bi-connectors/PowerBIConnector/docs/img/pbi_auth.png b/bi-connectors/PowerBIConnector/docs/img/pbi_auth.png new file mode 100644 index 00000000000..5af0eda5760 Binary files /dev/null and b/bi-connectors/PowerBIConnector/docs/img/pbi_auth.png differ diff --git a/bi-connectors/PowerBIConnector/docs/img/pbi_auth_amz.png b/bi-connectors/PowerBIConnector/docs/img/pbi_auth_amz.png new file mode 100644 index 00000000000..3c63ed27a25 Binary files /dev/null and b/bi-connectors/PowerBIConnector/docs/img/pbi_auth_amz.png differ diff --git a/bi-connectors/PowerBIConnector/docs/img/pbi_connection_string_options.png b/bi-connectors/PowerBIConnector/docs/img/pbi_connection_string_options.png new file mode 100644 index 00000000000..b7bf7f3ce1e Binary files /dev/null and b/bi-connectors/PowerBIConnector/docs/img/pbi_connection_string_options.png differ diff --git a/bi-connectors/PowerBIConnector/docs/img/pbi_connection_string_options_amz.png b/bi-connectors/PowerBIConnector/docs/img/pbi_connection_string_options_amz.png new file mode 100644 index 00000000000..aff8ddc9848 Binary files /dev/null and b/bi-connectors/PowerBIConnector/docs/img/pbi_connection_string_options_amz.png differ diff --git a/bi-connectors/PowerBIConnector/docs/img/pbi_data_preview.png b/bi-connectors/PowerBIConnector/docs/img/pbi_data_preview.png new file mode 100644 index 00000000000..b26bcee6be3 Binary files /dev/null and b/bi-connectors/PowerBIConnector/docs/img/pbi_data_preview.png differ diff --git a/bi-connectors/PowerBIConnector/docs/img/pbi_error_conn.png b/bi-connectors/PowerBIConnector/docs/img/pbi_error_conn.png new file mode 100644 index 00000000000..261999ab800 Binary files /dev/null and b/bi-connectors/PowerBIConnector/docs/img/pbi_error_conn.png differ diff --git a/bi-connectors/PowerBIConnector/docs/img/pbi_error_driver_not_installed.png b/bi-connectors/PowerBIConnector/docs/img/pbi_error_driver_not_installed.png new file mode 100644 index 00000000000..76f5da35fb7 Binary files /dev/null and b/bi-connectors/PowerBIConnector/docs/img/pbi_error_driver_not_installed.png differ diff --git a/bi-connectors/PowerBIConnector/docs/img/pbi_select_connector.png b/bi-connectors/PowerBIConnector/docs/img/pbi_select_connector.png new file mode 100644 index 00000000000..33188b97a3d Binary files /dev/null and b/bi-connectors/PowerBIConnector/docs/img/pbi_select_connector.png differ diff --git a/bi-connectors/PowerBIConnector/docs/img/pbi_select_connector_amz.png b/bi-connectors/PowerBIConnector/docs/img/pbi_select_connector_amz.png new file mode 100644 index 00000000000..25677e12c7e Binary files /dev/null and b/bi-connectors/PowerBIConnector/docs/img/pbi_select_connector_amz.png differ diff --git a/bi-connectors/PowerBIConnector/docs/img/pbi_simple_graph.png b/bi-connectors/PowerBIConnector/docs/img/pbi_simple_graph.png new file mode 100644 index 00000000000..9174de438b1 Binary files /dev/null and b/bi-connectors/PowerBIConnector/docs/img/pbi_simple_graph.png differ diff --git a/bi-connectors/PowerBIConnector/docs/img/pbi_third_party_warning.png b/bi-connectors/PowerBIConnector/docs/img/pbi_third_party_warning.png new file mode 100644 index 00000000000..f05502ad574 Binary files /dev/null and b/bi-connectors/PowerBIConnector/docs/img/pbi_third_party_warning.png differ diff --git a/bi-connectors/PowerBIConnector/docs/img/pbi_third_party_warning_amz.png b/bi-connectors/PowerBIConnector/docs/img/pbi_third_party_warning_amz.png new file mode 100644 index 00000000000..48a11a86f00 Binary files /dev/null and b/bi-connectors/PowerBIConnector/docs/img/pbi_third_party_warning_amz.png differ diff --git a/sql-odbc/src/PowerBIConnector/resources.resx b/bi-connectors/PowerBIConnector/resources.resx similarity index 98% rename from sql-odbc/src/PowerBIConnector/resources.resx rename to bi-connectors/PowerBIConnector/resources.resx index 3642a474fe2..3db608e4587 100644 --- a/sql-odbc/src/PowerBIConnector/resources.resx +++ b/bi-connectors/PowerBIConnector/resources.resx @@ -121,9 +121,9 @@ Connect to OpenSearch - OpenSearch + OpenSearch Project - OpenSearch + OpenSearch Project \ No newline at end of file diff --git a/bi-connectors/TableauConnector/opensearch_sql_jdbc/META-INF/MANIFEST.MF b/bi-connectors/TableauConnector/opensearch_sql_jdbc/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..97a662d62c5 --- /dev/null +++ b/bi-connectors/TableauConnector/opensearch_sql_jdbc/META-INF/MANIFEST.MF @@ -0,0 +1,3 @@ +Manifest-Version: 1.0 +Created-By: 13 (Oracle Corporation) + diff --git a/bi-connectors/TableauConnector/opensearch_sql_jdbc/connection-fields.xml b/bi-connectors/TableauConnector/opensearch_sql_jdbc/connection-fields.xml new file mode 100644 index 00000000000..54f5296f7e9 --- /dev/null +++ b/bi-connectors/TableauConnector/opensearch_sql_jdbc/connection-fields.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/bi-connectors/TableauConnector/opensearch_sql_jdbc/connection-metadata.xml b/bi-connectors/TableauConnector/opensearch_sql_jdbc/connection-metadata.xml new file mode 100644 index 00000000000..1b3432c3177 --- /dev/null +++ b/bi-connectors/TableauConnector/opensearch_sql_jdbc/connection-metadata.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/bi-connectors/TableauConnector/opensearch_sql_jdbc/connectionBuilder.js b/bi-connectors/TableauConnector/opensearch_sql_jdbc/connectionBuilder.js new file mode 100644 index 00000000000..1a16fddaec1 --- /dev/null +++ b/bi-connectors/TableauConnector/opensearch_sql_jdbc/connectionBuilder.js @@ -0,0 +1,28 @@ +(function dsbuilder(attr){ + var connStr = "jdbc:opensearch://"; + // Set SSL value in connection string + if (attr[connectionHelper.attributeSSLMode] == "require"){ + connStr += "https://"; + } else { + connStr += "http://"; + } + + // Set host information in connection string + connStr += attr[connectionHelper.attributeServer] + ":" + attr[connectionHelper.attributePort] + "?"; + + // Set authentication values in connection string + var authAttrValue = attr[connectionHelper.attributeAuthentication]; + if (authAttrValue == "auth-none"){ + connStr += "auth=NONE&trustSelfSigned=" + attr["v-trustSelfSigned"]; + } else if (authAttrValue == "auth-integrated"){ + connStr += "auth=AWS_SIGV4"; + var region = attr["v-region"]; + if (region){ + connStr += "&Region=" + region; + } + } else { //if (authAttrValue == "auth-user-pass"){ + connStr += "auth=BASIC&user=" + attr[connectionHelper.attributeUsername] + "&password=" + attr[connectionHelper.attributePassword] + "&trustSelfSigned=" + attr["v-trustSelfSigned"]; + } + + return [connStr]; +}) diff --git a/sql-odbc/src/TableauConnector/opensearch_sql_odbc/connectionResolver.tdr b/bi-connectors/TableauConnector/opensearch_sql_jdbc/connectionResolver.tdr similarity index 61% rename from sql-odbc/src/TableauConnector/opensearch_sql_odbc/connectionResolver.tdr rename to bi-connectors/TableauConnector/opensearch_sql_jdbc/connectionResolver.tdr index 68c376270c1..c51adc002ec 100644 --- a/sql-odbc/src/TableauConnector/opensearch_sql_odbc/connectionResolver.tdr +++ b/bi-connectors/TableauConnector/opensearch_sql_jdbc/connectionResolver.tdr @@ -1,27 +1,26 @@ - +