Big bundle of tweaks to get 3.3 to build

MAINTAINERS.md

@@ -16,7 +16,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Peter Fitzgibbons | [pjfitzgibbons](https://github.com/pjfitzgibbons)   | Amazon      |
 | Simeon Widdis     | [swiddis](https://github.com/swiddis)               | Amazon      |
 | Chen Dai          | [dai-chen](https://github.com/dai-chen)             | Amazon      |
-| Vamsi Manohar     | [vamsi-amazon](https://github.com/vamsi-amazon)     | Amazon      |
+| Vamsi Manohar     | [vamsimanohar](https://github.com/vamsimanohar)     | Amazon      |
 | Peng Huo          | [penghuo](https://github.com/penghuo)               | Amazon      |
 | Sean Kao          | [seankao-az](https://github.com/seankao-az)         | Amazon      |
 | Max Ksyunz        | [MaxKsyunz](https://github.com/MaxKsyunz)           | BitQuill    |

build.gradle

@@ -19,7 +19,7 @@ application {
 repositories {
     mavenCentral()
     maven {
-        url "https://aws.oss.sonatype.org/content/repositories/snapshots/"
+        url "https://ci.opensearch.org/ci/dbc/snapshots/maven/"
     }
     maven {
         url "https://jitpack.io"
@@ -85,11 +85,12 @@ ext.getHttpClientInfo = { ->
 
 ext {
     // Default version to use when no specific version is specified
-    defaultVersion = "3.1.0.0"
+    // TODO don't hard-code this
+    defaultVersion = "3.3.0.0"
 
     submodules = ['common', 'core', 'opensearch', 'ppl', 'sql', 'protocol', 'datasources']
 
-    // TODO: will need to see why using datasources on Maven fails
+    // TODO will need to see why using datasources on Maven fails
     // but using local jar file works
     mavenSubmodules = ['common', 'core', 'opensearch', 'ppl', 'sql', 'protocol']
 
@@ -400,7 +401,8 @@ ext.createVersionConfigurations = { String version ->
         dependencies.add(configName, files('submodule/datasources-3.1.0.0-SNAPSHOT.jar'))
 
         // Add OpenSearch client dependencies with appropriate version
-        def clientVersion = project.ext.getOpenSearchClientVersion(version)
+        // We don't need to strictly keep this in sync with the OpenSearch version
+        def clientVersion = '3.2.0'
 
         project.ext.opensearchClientDeps.each { dep ->
             dependencies.add(configName, "org.opensearch.client:${dep}:${clientVersion}")
@@ -445,14 +447,15 @@ ext.createLocalConfigurations = { String version, String localJarDir ->
         }
 
         // Add OpenSearch client dependencies with appropriate version
-        def clientVersion = project.ext.getOpenSearchClientVersion(version)
+        def clientVersion = '3.2.0'
 
         project.ext.opensearchClientDeps.each { dep ->
             dependencies.add(configName, "org.opensearch.client:${dep}:${clientVersion}")
         }
 
         // Add OpenSearch core dependency
-        dependencies.add(configName, "org.opensearch:opensearch:${clientVersion}")
+        String osVersion = version.tokenize('.').take(3).join('.')
+        dependencies.add(configName, "org.opensearch:opensearch:${osVersion}")
 
         println "Version ${version}: Using OpenSearch client version ${clientVersion}"
 

development_guide.md

@@ -218,7 +218,7 @@ black .
 ### Workflow
 
 1. Update version number
-    1. Modify the version number in [`__init__.py`](`src/main/python/opensearchsql_cli/__init__.py`). It will be used by `setup.py` for release.
+    1. Modify the version number in [`__init__.py`](src/main/python/opensearchsql_cli/__init__.py). It will be used by `setup.py` for release.
 2. Create/Update `setup.py` (if needed)
     1. For more details refer to https://packaging.python.org/tutorials/packaging-projects/#creating-setup-py 
 3. Update README.md, Legal and copyright files(if needed)

release-notes/odfe-sql-cli.release-notes-1.7.0.0.md

@@ -9,7 +9,7 @@ endpoint such as AWS Elasticsearch.
 
 ### Features
 #### CLI Features
-* Feature [#12](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/12): Initial development for SQL CLI
+* Feature [#12](https://github.com/amazon-archives/sql-cli/pull/12): Initial development for SQL CLI
     * prototype launch: app -> check connection -> take input -> query ES -> serve raw results(format=jdbc)
     * enrich auto-completion corpus
     * Convert to vertical output format if fields length is larger than terminal window
@@ -29,38 +29,38 @@ endpoint such as AWS Elasticsearch.
     * Added legal and copyright files,
     * Added THIRD PARTY file
     * Added setup.py for packaging and releasing
-* Feature [#24](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/24): Provide user option to toggle to use AWS sigV4 authentication 
-(issue: [#23](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/23))
+* Feature [#24](https://github.com/amazon-archives/sql-cli/pull/24): Provide user option to toggle to use AWS sigV4 authentication 
+(issue: [#23](https://github.com/amazon-archives/sql-cli/issues/23))
 
 #### Testing
-* Feature [#28](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/28) :Added tox scripts for testing automation
+* Feature [#28](https://github.com/amazon-archives/sql-cli/pull/28) :Added tox scripts for testing automation
 
 #### Documentation
-* Change [#22](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/22): Update documentation and CLI naming 
-(issues: [#21](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/21), [#7](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/17))
-* Change [#32](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/32): Update copyright to 2020
-* Change [#33](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/33): Updated package naming and created folder for release notes
-* Change [#34](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/34): Added CONTRIBUTORS.md
-* Change [#36](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/36): Polish README.md and test_plan.md
+* Change [#22](https://github.com/amazon-archives/sql-cli/pull/22): Update documentation and CLI naming 
+(issues: [#21](https://github.com/amazon-archives/sql-cli/issues/21), [#7](https://github.com/amazon-archives/sql-cli/issues/17))
+* Change [#32](https://github.com/amazon-archives/sql-cli/pull/32): Update copyright to 2020
+* Change [#33](https://github.com/amazon-archives/sql-cli/pull/33): Updated package naming and created folder for release notes
+* Change [#34](https://github.com/amazon-archives/sql-cli/pull/34): Added CONTRIBUTORS.md
+* Change [#36](https://github.com/amazon-archives/sql-cli/pull/36): Polish README.md and test_plan.md
 
 
 ### Enhancements
-* Enhancement [#31](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/31): Added github action workflow for CI/CD
-(issue: [#20](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/21))
-* Enhancement [#35](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/35): Update github action test and build workflow to spin up ES instance
+* Enhancement [#31](https://github.com/amazon-archives/sql-cli/pull/31): Added github action workflow for CI/CD
+(issue: [#20](https://github.com/amazon-archives/sql-cli/issues/21))
+* Enhancement [#35](https://github.com/amazon-archives/sql-cli/pull/35): Update github action test and build workflow to spin up ES instance
 
 
 ### BugFixes
-* BugFix[#12](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/12): Initial development for SQL CLI
+* BugFix[#12](https://github.com/amazon-archives/sql-cli/pull/12): Initial development for SQL CLI
     * Fix the logic of passing self-constructed settings
     * [Fix] get rid of unicode warning. Fix meta info display
     * [fix] Refactor executor code
     * [Fix] Fix test cases corresponding to fraction display.
     * [Fix] fix code style using Black, update documentation and comments
-* BugFix[#18](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/18): Fix typos, remove unused dependencies, add .gitignore and legal file
-(issue: [#15](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/15))
-* BugFix[#19](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/19): Fix test failures 
-(issue: [#16](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/16))
-* BugFix[#26](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/26): Update usage gif, fix http/https issue when connect to AWS Elasticsearch (issue: [#25](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/25))
+* BugFix[#18](https://github.com/amazon-archives/sql-cli/pull/18): Fix typos, remove unused dependencies, add .gitignore and legal file
+(issue: [#15](https://github.com/amazon-archives/sql-cli/issues/15))
+* BugFix[#19](https://github.com/amazon-archives/sql-cli/pull/19): Fix test failures 
+(issue: [#16](https://github.com/amazon-archives/sql-cli/issues/16))
+* BugFix[#26](https://github.com/amazon-archives/sql-cli/pull/26): Update usage gif, fix http/https issue when connect to AWS Elasticsearch (issue: [#25](https://github.com/amazon-archives/sql-cli/issues/25))
 
 

release-notes/odfe-sql-cli.release-notes-1.8.0.0.md

@@ -2,4 +2,4 @@
 
 ### Features
 #### Elasticsearch and OpenSearch SQL Plugin Compatibility
-* Feature [#41](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/41): Elasticsearch 7.7.0 and OpenSearch SQL Plugin 1.8.0 compatibility (issue: [#40](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/40))
+* Feature [#41](https://github.com/amazon-archives/sql-cli/pull/41): Elasticsearch 7.7.0 and OpenSearch SQL Plugin 1.8.0 compatibility (issue: [#40](https://github.com/amazon-archives/sql-cli/issues/40))

release-notes/odfe-sql-cli.release-notes-1.9.0.0.md

@@ -2,12 +2,12 @@
 
 ### Features
 #### Elasticsearch and OpenSearch SQL Plugin Compatibility
-* Feature [#55](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/55): Elasticsearch 7.8.0 and OpenSearch SQL Plugin 1.9.0 compatibility
-(issue: [#54](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/54))
+* Feature [#55](https://github.com/amazon-archives/sql-cli/pull/55): Elasticsearch 7.8.0 and OpenSearch SQL Plugin 1.9.0 compatibility
+(issue: [#54](https://github.com/amazon-archives/sql-cli/issues/54))
 
 #### Documentation
-* Feature [#48](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/48): Added README badges
+* Feature [#48](https://github.com/amazon-archives/sql-cli/pull/48): Added README badges
 
 ### Enhancements
-* Enhancement [#45](https://github.com/opendistro-for-elasticsearch/sql-cli/pull/45): Update project layout for better module import
-(issue: [#43](https://github.com/opendistro-for-elasticsearch/sql-cli/issues/43))
+* Enhancement [#45](https://github.com/amazon-archives/sql-cli/pull/45): Update project layout for better module import
+(issue: [#43](https://github.com/amazon-archives/sql-cli/issues/43))

src/main/java/client/http5/OpenSearchRestClientImpl.java

@@ -320,6 +320,29 @@ public void deletePit(DeletePitRequest deletePitRequest) {
     }
   }
 
+  // Pseduo-`@Override`: only applicable on versions 3.3 or newer, if we add `@Override` it breaks compilation on older versions
+  public void forceCleanup(OpenSearchRequest request) {
+    if (request instanceof OpenSearchScrollRequest) {
+      request.forceClean(
+          scrollId -> {
+            try {
+              ClearScrollRequest clearRequest = new ClearScrollRequest();
+              clearRequest.addScrollId(scrollId);
+              client.clearScroll(clearRequest, RequestOptions.DEFAULT);
+            } catch (IOException e) {
+              throw new IllegalStateException(
+                  "Failed to clean up resources for search request " + request, e);
+            }
+          });
+    } else {
+      request.forceClean(
+          pitId -> {
+            DeletePitRequest deletePitRequest = new DeletePitRequest(pitId);
+            deletePit(deletePitRequest);
+          });
+    }
+  }
+
   // Helper methods for AWS interceptor to sign its body
   private String getBodyContent(ToXContent request) throws IOException {
     XContentBuilder builder = XContentFactory.jsonBuilder();

src/main/python/opensearchsql_cli/main.py

@@ -11,7 +11,6 @@
 import pyfiglet
 
 from rich.console import Console
-from rich.status import Status
 from .sql import sql_connection
 from .query import SavedQueries
 from .sql.sql_library_manager import sql_library_manager
@@ -113,6 +112,8 @@ def main(
                 "--version",
                 help="Set OpenSearch SQL plug-in version: 3.1, 2.19",
                 autocompletion=lambda ctx, incomplete: [
+                    # TODO this shouldn't be hardcoded, find a dynamic release source
+                    "3.3",
                     "3.1",
                     "2.19",
                 ],

src/main/python/opensearchsql_cli/sql/sql_version.py

@@ -49,7 +49,7 @@ def get_all_versions(self):
         Returns:
             list: List of all available versions
         """
-        url = "https://aws.oss.sonatype.org/content/repositories/snapshots/org/opensearch/query/unified-query-core/maven-metadata.xml"
+        url = "https://ci.opensearch.org/ci/dbc/snapshots/maven/org/opensearch/query/unified-query-core/maven-metadata.xml"
 
         response = requests.get(url)
         response.raise_for_status()
@@ -421,6 +421,13 @@ def _clone_repository(self, branch_name, git_url, git_dir):
         Returns:
             tuple: (success, result) where success is a boolean and result is the subprocess.CompletedProcess object
         """
+        if re.match(r"^\d+(\.\d+)*$", branch_name):
+            # If the branch is a version number, we actually want the tag, to sync with what was actually released
+            # Branch is semver, append 0s until length 4 to get the tag
+            parts = branch_name.split(".")
+            parts += ["0"] * (4 - len(parts))
+            branch_name = ".".join(parts)
+
         with console.status(
             f"[bold yellow]Cloning repository {git_url} branch {branch_name}...[/bold yellow]",
             spinner="dots",

src/main/python/opensearchsql_cli/tests/sql/test_sql_version.py

@@ -144,7 +144,7 @@ def test_get_jar_path(
         Test getting JAR path
         """
         # Setup mock
-        expected_path = "/mock/project_root/build/libs/opensearchsqlcli-3.1.0.0.jar"
+        expected_path = "/mock/project_root/build/libs/opensearchsqlcli-3.3.0.0.jar"
         mock_join.return_value = expected_path
 
         # Create version manager and get JAR path
@@ -154,5 +154,5 @@ def test_get_jar_path(
         # Assertions
         assert path == expected_path
         mock_join.assert_called_with(
-            "/mock/project_root", "build", "libs", "opensearchsqlcli-3.1.0.0.jar"
+            "/mock/project_root", "build", "libs", "opensearchsqlcli-3.3.0.0.jar"
         )