diff --git a/MAINTAINERS.md b/MAINTAINERS.md index af8c883..8c148aa 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -16,7 +16,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | Peter Fitzgibbons | [pjfitzgibbons](https://github.com/pjfitzgibbons) | Amazon | | Simeon Widdis | [swiddis](https://github.com/swiddis) | Amazon | | Chen Dai | [dai-chen](https://github.com/dai-chen) | Amazon | -| Vamsi Manohar | [vamsi-amazon](https://github.com/vamsi-amazon) | Amazon | +| Vamsi Manohar | [vamsimanohar](https://github.com/vamsimanohar) | Amazon | | Peng Huo | [penghuo](https://github.com/penghuo) | Amazon | | Sean Kao | [seankao-az](https://github.com/seankao-az) | Amazon | | Max Ksyunz | [MaxKsyunz](https://github.com/MaxKsyunz) | BitQuill | diff --git a/build.gradle b/build.gradle index 8643da8..1ee3c51 100644 --- a/build.gradle +++ b/build.gradle @@ -19,7 +19,7 @@ application { repositories { mavenCentral() maven { - url "https://aws.oss.sonatype.org/content/repositories/snapshots/" + url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" } maven { url "https://jitpack.io" @@ -85,11 +85,12 @@ ext.getHttpClientInfo = { -> ext { // Default version to use when no specific version is specified - defaultVersion = "3.1.0.0" + // TODO don't hard-code this + defaultVersion = "3.3.0.0" submodules = ['common', 'core', 'opensearch', 'ppl', 'sql', 'protocol', 'datasources'] - // TODO: will need to see why using datasources on Maven fails + // TODO will need to see why using datasources on Maven fails // but using local jar file works mavenSubmodules = ['common', 'core', 'opensearch', 'ppl', 'sql', 'protocol'] @@ -400,7 +401,8 @@ ext.createVersionConfigurations = { String version -> dependencies.add(configName, files('submodule/datasources-3.1.0.0-SNAPSHOT.jar')) // Add OpenSearch client dependencies with appropriate version - def clientVersion = project.ext.getOpenSearchClientVersion(version) + // We don't need to strictly keep this in sync with the OpenSearch version + def clientVersion = '3.2.0' project.ext.opensearchClientDeps.each { dep -> dependencies.add(configName, "org.opensearch.client:${dep}:${clientVersion}") @@ -445,14 +447,15 @@ ext.createLocalConfigurations = { String version, String localJarDir -> } // Add OpenSearch client dependencies with appropriate version - def clientVersion = project.ext.getOpenSearchClientVersion(version) + def clientVersion = '3.2.0' project.ext.opensearchClientDeps.each { dep -> dependencies.add(configName, "org.opensearch.client:${dep}:${clientVersion}") } // Add OpenSearch core dependency - dependencies.add(configName, "org.opensearch:opensearch:${clientVersion}") + String osVersion = version.tokenize('.').take(3).join('.') + dependencies.add(configName, "org.opensearch:opensearch:${osVersion}") println "Version ${version}: Using OpenSearch client version ${clientVersion}" diff --git a/development_guide.md b/development_guide.md index e78cd0c..91722be 100644 --- a/development_guide.md +++ b/development_guide.md @@ -218,7 +218,7 @@ black . ### Workflow 1. Update version number - 1. Modify the version number in [`__init__.py`](`src/main/python/opensearchsql_cli/__init__.py`). It will be used by `setup.py` for release. + 1. Modify the version number in [`__init__.py`](src/main/python/opensearchsql_cli/__init__.py). It will be used by `setup.py` for release. 2. Create/Update `setup.py` (if needed) 1. For more details refer to https://packaging.python.org/tutorials/packaging-projects/#creating-setup-py 3. Update README.md, Legal and copyright files(if needed) diff --git a/release-notes/odfe-sql-cli.release-notes-1.7.0.0.md b/release-notes/odfe-sql-cli.release-notes-1.7.0.0.md index c6afabf..376e395 100644 --- a/release-notes/odfe-sql-cli.release-notes-1.7.0.0.md +++ b/release-notes/odfe-sql-cli.release-notes-1.7.0.0.md @@ -9,7 +9,7 @@ endpoint such as AWS Elasticsearch. ### Features #### CLI Features -* Feature [#12](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/12): Initial development for SQL CLI +* Feature [#12](https://github.com/amazon-archives/sql-cli/pull/12): Initial development for SQL CLI * prototype launch: app -> check connection -> take input -> query ES -> serve raw results(format=jdbc) * enrich auto-completion corpus * Convert to vertical output format if fields length is larger than terminal window @@ -29,38 +29,38 @@ endpoint such as AWS Elasticsearch. * Added legal and copyright files, * Added THIRD PARTY file * Added setup.py for packaging and releasing -* Feature [#24](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/24): Provide user option to toggle to use AWS sigV4 authentication -(issue: [#23](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/23)) +* Feature [#24](https://github.com/amazon-archives/sql-cli/pull/24): Provide user option to toggle to use AWS sigV4 authentication +(issue: [#23](https://github.com/amazon-archives/sql-cli/issues/23)) #### Testing -* Feature [#28](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/28) :Added tox scripts for testing automation +* Feature [#28](https://github.com/amazon-archives/sql-cli/pull/28) :Added tox scripts for testing automation #### Documentation -* Change [#22](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/22): Update documentation and CLI naming -(issues: [#21](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/21), [#7](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/17)) -* Change [#32](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/32): Update copyright to 2020 -* Change [#33](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/33): Updated package naming and created folder for release notes -* Change [#34](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/34): Added CONTRIBUTORS.md -* Change [#36](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/36): Polish README.md and test_plan.md +* Change [#22](https://github.com/amazon-archives/sql-cli/pull/22): Update documentation and CLI naming +(issues: [#21](https://github.com/amazon-archives/sql-cli/issues/21), [#7](https://github.com/amazon-archives/sql-cli/issues/17)) +* Change [#32](https://github.com/amazon-archives/sql-cli/pull/32): Update copyright to 2020 +* Change [#33](https://github.com/amazon-archives/sql-cli/pull/33): Updated package naming and created folder for release notes +* Change [#34](https://github.com/amazon-archives/sql-cli/pull/34): Added CONTRIBUTORS.md +* Change [#36](https://github.com/amazon-archives/sql-cli/pull/36): Polish README.md and test_plan.md ### Enhancements -* Enhancement [#31](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/31): Added github action workflow for CI/CD -(issue: [#20](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/21)) -* Enhancement [#35](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/35): Update github action test and build workflow to spin up ES instance +* Enhancement [#31](https://github.com/amazon-archives/sql-cli/pull/31): Added github action workflow for CI/CD +(issue: [#20](https://github.com/amazon-archives/sql-cli/issues/21)) +* Enhancement [#35](https://github.com/amazon-archives/sql-cli/pull/35): Update github action test and build workflow to spin up ES instance ### BugFixes -* BugFix[#12](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/12): Initial development for SQL CLI +* BugFix[#12](https://github.com/amazon-archives/sql-cli/pull/12): Initial development for SQL CLI * Fix the logic of passing self-constructed settings * [Fix] get rid of unicode warning. Fix meta info display * [fix] Refactor executor code * [Fix] Fix test cases corresponding to fraction display. * [Fix] fix code style using Black, update documentation and comments -* BugFix[#18](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/18): Fix typos, remove unused dependencies, add .gitignore and legal file -(issue: [#15](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/15)) -* BugFix[#19](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/19): Fix test failures -(issue: [#16](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/16)) -* BugFix[#26](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/26): Update usage gif, fix http/https issue when connect to AWS Elasticsearch (issue: [#25](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/25)) +* BugFix[#18](https://github.com/amazon-archives/sql-cli/pull/18): Fix typos, remove unused dependencies, add .gitignore and legal file +(issue: [#15](https://github.com/amazon-archives/sql-cli/issues/15)) +* BugFix[#19](https://github.com/amazon-archives/sql-cli/pull/19): Fix test failures +(issue: [#16](https://github.com/amazon-archives/sql-cli/issues/16)) +* BugFix[#26](https://github.com/amazon-archives/sql-cli/pull/26): Update usage gif, fix http/https issue when connect to AWS Elasticsearch (issue: [#25](https://github.com/amazon-archives/sql-cli/issues/25)) diff --git a/release-notes/odfe-sql-cli.release-notes-1.8.0.0.md b/release-notes/odfe-sql-cli.release-notes-1.8.0.0.md index c5f6a8c..723671c 100644 --- a/release-notes/odfe-sql-cli.release-notes-1.8.0.0.md +++ b/release-notes/odfe-sql-cli.release-notes-1.8.0.0.md @@ -2,4 +2,4 @@ ### Features #### Elasticsearch and OpenSearch SQL Plugin Compatibility -* Feature [#41](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/41): Elasticsearch 7.7.0 and OpenSearch SQL Plugin 1.8.0 compatibility (issue: [#40](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/40)) +* Feature [#41](https://github.com/amazon-archives/sql-cli/pull/41): Elasticsearch 7.7.0 and OpenSearch SQL Plugin 1.8.0 compatibility (issue: [#40](https://github.com/amazon-archives/sql-cli/issues/40)) diff --git a/release-notes/odfe-sql-cli.release-notes-1.9.0.0.md b/release-notes/odfe-sql-cli.release-notes-1.9.0.0.md index b1e50eb..df6748d 100644 --- a/release-notes/odfe-sql-cli.release-notes-1.9.0.0.md +++ b/release-notes/odfe-sql-cli.release-notes-1.9.0.0.md @@ -2,12 +2,12 @@ ### Features #### Elasticsearch and OpenSearch SQL Plugin Compatibility -* Feature [#55](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/55): Elasticsearch 7.8.0 and OpenSearch SQL Plugin 1.9.0 compatibility -(issue: [#54](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/54)) +* Feature [#55](https://github.com/amazon-archives/sql-cli/pull/55): Elasticsearch 7.8.0 and OpenSearch SQL Plugin 1.9.0 compatibility +(issue: [#54](https://github.com/amazon-archives/sql-cli/issues/54)) #### Documentation -* Feature [#48](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/48): Added README badges +* Feature [#48](https://github.com/amazon-archives/sql-cli/pull/48): Added README badges ### Enhancements -* Enhancement [#45](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/45): Update project layout for better module import -(issue: [#43](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/43)) +* Enhancement [#45](https://github.com/amazon-archives/sql-cli/pull/45): Update project layout for better module import +(issue: [#43](https://github.com/amazon-archives/sql-cli/issues/43)) diff --git a/src/main/java/client/http5/OpenSearchRestClientImpl.java b/src/main/java/client/http5/OpenSearchRestClientImpl.java index ab82751..7be4267 100644 --- a/src/main/java/client/http5/OpenSearchRestClientImpl.java +++ b/src/main/java/client/http5/OpenSearchRestClientImpl.java @@ -320,6 +320,29 @@ public void deletePit(DeletePitRequest deletePitRequest) { } } + // Pseduo-`@Override`: only applicable on versions 3.3 or newer, if we add `@Override` it breaks compilation on older versions + public void forceCleanup(OpenSearchRequest request) { + if (request instanceof OpenSearchScrollRequest) { + request.forceClean( + scrollId -> { + try { + ClearScrollRequest clearRequest = new ClearScrollRequest(); + clearRequest.addScrollId(scrollId); + client.clearScroll(clearRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + throw new IllegalStateException( + "Failed to clean up resources for search request " + request, e); + } + }); + } else { + request.forceClean( + pitId -> { + DeletePitRequest deletePitRequest = new DeletePitRequest(pitId); + deletePit(deletePitRequest); + }); + } + } + // Helper methods for AWS interceptor to sign its body private String getBodyContent(ToXContent request) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); diff --git a/src/main/python/opensearchsql_cli/main.py b/src/main/python/opensearchsql_cli/main.py index 98f2cb1..c788470 100644 --- a/src/main/python/opensearchsql_cli/main.py +++ b/src/main/python/opensearchsql_cli/main.py @@ -11,7 +11,6 @@ import pyfiglet from rich.console import Console -from rich.status import Status from .sql import sql_connection from .query import SavedQueries from .sql.sql_library_manager import sql_library_manager @@ -113,6 +112,8 @@ def main( "--version", help="Set OpenSearch SQL plug-in version: 3.1, 2.19", autocompletion=lambda ctx, incomplete: [ + # TODO this shouldn't be hardcoded, find a dynamic release source + "3.3", "3.1", "2.19", ], diff --git a/src/main/python/opensearchsql_cli/sql/sql_version.py b/src/main/python/opensearchsql_cli/sql/sql_version.py index 7f4e1db..1200151 100644 --- a/src/main/python/opensearchsql_cli/sql/sql_version.py +++ b/src/main/python/opensearchsql_cli/sql/sql_version.py @@ -49,7 +49,7 @@ def get_all_versions(self): Returns: list: List of all available versions """ - url = "https://aws.oss.sonatype.org/content/repositories/snapshots/org/opensearch/query/unified-query-core/maven-metadata.xml" + url = "https://ci.opensearch.org/ci/dbc/snapshots/maven/org/opensearch/query/unified-query-core/maven-metadata.xml" response = requests.get(url) response.raise_for_status() @@ -421,6 +421,13 @@ def _clone_repository(self, branch_name, git_url, git_dir): Returns: tuple: (success, result) where success is a boolean and result is the subprocess.CompletedProcess object """ + if re.match(r"^\d+(\.\d+)*$", branch_name): + # If the branch is a version number, we actually want the tag, to sync with what was actually released + # Branch is semver, append 0s until length 4 to get the tag + parts = branch_name.split(".") + parts += ["0"] * (4 - len(parts)) + branch_name = ".".join(parts) + with console.status( f"[bold yellow]Cloning repository {git_url} branch {branch_name}...[/bold yellow]", spinner="dots", diff --git a/src/main/python/opensearchsql_cli/tests/sql/test_sql_version.py b/src/main/python/opensearchsql_cli/tests/sql/test_sql_version.py index fd110cb..8bbb5bd 100644 --- a/src/main/python/opensearchsql_cli/tests/sql/test_sql_version.py +++ b/src/main/python/opensearchsql_cli/tests/sql/test_sql_version.py @@ -144,7 +144,7 @@ def test_get_jar_path( Test getting JAR path """ # Setup mock - expected_path = "/mock/project_root/build/libs/opensearchsqlcli-3.1.0.0.jar" + expected_path = "/mock/project_root/build/libs/opensearchsqlcli-3.3.0.0.jar" mock_join.return_value = expected_path # Create version manager and get JAR path @@ -154,5 +154,5 @@ def test_get_jar_path( # Assertions assert path == expected_path mock_join.assert_called_with( - "/mock/project_root", "build", "libs", "opensearchsqlcli-3.1.0.0.jar" + "/mock/project_root", "build", "libs", "opensearchsqlcli-3.3.0.0.jar" )