From f71e73a7575352817ab6c4f093a5f32e6c778497 Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Tue, 17 Jan 2023 22:20:12 +0000 Subject: [PATCH 01/16] fix bwc bug when increment to OS 2.6.0 Signed-off-by: Sicheng Song --- plugin/build.gradle | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/plugin/build.gradle b/plugin/build.gradle index 51c53541e4..c9a233f742 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -386,11 +386,7 @@ String baseName = "mlCommonsBwcCluster" String bwcMlPlugin = "opensearch-ml-" + bwcVersion + ".zip" String bwcFilePath = "src/test/resources/org/opensearch/ml/bwc/" String bwcRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + bwcShortVersion + "/latest/linux/x64/tar/builds/opensearch/plugins/" + bwcMlPlugin -String project_no_snapshot = project.version.replace("-SNAPSHOT","") -String opensearch_no_snapshot = opensearch_version.replace("-SNAPSHOT","") -String opensearchMlPlugin = "opensearch-ml-" + project_no_snapshot + ".zip" -String opensearchMlRemoteFile = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot + - '/latest/linux/x64/tar/builds/opensearch/plugins/' + opensearchMlPlugin +String opensearchMlPlugin = "opensearch-ml-" + project.version + ".zip" 2.times {i -> testClusters { @@ -430,13 +426,11 @@ List> plugins = [ return new RegularFile() { @Override File getAsFile() { - if (new File('./plugin/' + bwcFilePath + project.version).exists()) { - project.delete(files('./plugin/' + bwcFilePath + project.version)) + project.mkdir "$bwcFilePath/$project.version" + copy { + from "$buildDir/distributions/$opensearchMlPlugin" + into "$bwcFilePath/$project.version" } - project.mkdir bwcFilePath + project.version - ant.get(src: opensearchMlRemoteFile, - dest: bwcFilePath + project.version, - httpusecaches: false) return fileTree(bwcFilePath + project.version).getSingleFile() } } From be0afe392f58a25818287ce649d8a4d358bbe3c0 Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Tue, 17 Jan 2023 22:44:07 +0000 Subject: [PATCH 02/16] Settings changed for BWC testing Signed-off-by: Sicheng Song --- .../ml/bwc/MLCommonsBackwardsCompatibilityIT.java | 4 ++-- ...CommonsBackwardsCompatibilityRestTestCase.java | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java index 393a5e4e48..d53d7f181b 100644 --- a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java +++ b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java @@ -171,7 +171,7 @@ public void testBackwardsCompatibility() throws Exception { ArrayList rows = (ArrayList) predictionResult.get("rows"); assertTrue(rows.size() > 1); }); - } else if (opensearchVersion.equals("2.5.0")) { + } else if (Integer.parseInt(opensearchVersion.substring(2, 3)) > 4) { // train predict with old data ingestIrisData(irisIndex); trainAndPredict(client(), FunctionName.KMEANS, irisIndex, kMeansParams, searchSourceBuilder, predictionResult -> { @@ -184,7 +184,7 @@ public void testBackwardsCompatibility() throws Exception { break; case UPGRADED: assertTrue(pluginNames.contains("opensearch-ml")); - assertEquals("2.5.0", opensearchVersion); + assertTrue(Integer.parseInt(opensearchVersion.substring(2, 3)) > 4); ingestIrisData(irisIndex); trainAndPredict(client(), FunctionName.KMEANS, irisIndex, kMeansParams, searchSourceBuilder, predictionResult -> { ArrayList rows = (ArrayList) predictionResult.get("rows"); diff --git a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityRestTestCase.java b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityRestTestCase.java index 734e5c5c98..4eefec83fe 100644 --- a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityRestTestCase.java +++ b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityRestTestCase.java @@ -49,6 +49,7 @@ import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.util.EntityUtils; import org.junit.After; +import org.junit.Before; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.RestClient; @@ -105,6 +106,20 @@ protected boolean isHttps() { return isHttps; } + @Before + public void setupSettings() throws IOException { + Response response = TestHelper + .makeRequest( + client(), + "PUT", + "_cluster/settings", + null, + "{\"persistent\":{\"plugins.ml_commons.only_run_on_ml_node\":false}}", + ImmutableList.of(new BasicHeader(HttpHeaders.USER_AGENT, "")) + ); + assertEquals(200, response.getStatusLine().getStatusCode()); + } + @Override protected String getProtocol() { return isHttps() ? "https" : "http"; From ed652d2db2c4344718aa395c17a87e94f0abfb46 Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Tue, 17 Jan 2023 23:47:57 +0000 Subject: [PATCH 03/16] Settings changed to inside BWC test for bug fix Signed-off-by: Sicheng Song --- .../MLCommonsBackwardsCompatibilityIT.java | 37 ++++++++++++++++++- ...onsBackwardsCompatibilityRestTestCase.java | 15 -------- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java index d53d7f181b..824aa2ce86 100644 --- a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java +++ b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java @@ -13,6 +13,8 @@ import java.util.stream.Collectors; import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.message.BasicHeader; import org.junit.Assume; import org.junit.Before; import org.opensearch.client.Response; @@ -26,6 +28,9 @@ import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.test.rest.OpenSearchRestTestCase; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + public class MLCommonsBackwardsCompatibilityIT extends MLCommonsBackwardsCompatibilityRestTestCase { private final ClusterType CLUSTER_TYPE = ClusterType.parse(System.getProperty("tests.rest.bwcsuite")); @@ -171,7 +176,8 @@ public void testBackwardsCompatibility() throws Exception { ArrayList rows = (ArrayList) predictionResult.get("rows"); assertTrue(rows.size() > 1); }); - } else if (Integer.parseInt(opensearchVersion.substring(2, 3)) > 4) { + } else if (isNewerVersion(opensearchVersion)) { + testSettingShifting(); // train predict with old data ingestIrisData(irisIndex); trainAndPredict(client(), FunctionName.KMEANS, irisIndex, kMeansParams, searchSourceBuilder, predictionResult -> { @@ -184,7 +190,8 @@ public void testBackwardsCompatibility() throws Exception { break; case UPGRADED: assertTrue(pluginNames.contains("opensearch-ml")); - assertTrue(Integer.parseInt(opensearchVersion.substring(2, 3)) > 4); + assertTrue(isNewerVersion(opensearchVersion)); + testSettingShifting(); ingestIrisData(irisIndex); trainAndPredict(client(), FunctionName.KMEANS, irisIndex, kMeansParams, searchSourceBuilder, predictionResult -> { ArrayList rows = (ArrayList) predictionResult.get("rows"); @@ -196,6 +203,28 @@ public void testBackwardsCompatibility() throws Exception { } } + private void testSettingShifting() throws IOException { + Response bwcResponse = TestHelper + .makeRequest( + client(), + "PUT", + "_cluster/settings", + null, + "{\"persistent\":{\"plugins.ml_commons.only_run_on_ml_node\":false}}", + ImmutableList.of(new BasicHeader(HttpHeaders.USER_AGENT, "")) + ); + assertEquals(200, bwcResponse.getStatusLine().getStatusCode()); + + String jsonEntity = "{\n" + + " \"persistent\" : {\n" + + " \"plugins.ml_commons.native_memory_threshold\" : 100 \n" + + " }\n" + + "}"; + bwcResponse = TestHelper + .makeRequest(client(), "PUT", "_cluster/settings", ImmutableMap.of(), TestHelper.toHttpEntity(jsonEntity), null); + assertEquals(200, bwcResponse.getStatusLine().getStatusCode()); + } + private String getModelIdWithFunctionName(FunctionName functionName) throws IOException { String modelQuery = "{\"query\": {" + "\"term\": {" @@ -217,6 +246,10 @@ private String getModelIdWithFunctionName(FunctionName functionName) throws IOEx return modelIdSet.iterator().next().toString(); } + private boolean isNewerVersion(String osVersion) { + return (Integer.parseInt(osVersion.substring(2, 3)) > 4) || (Integer.parseInt(osVersion.substring(0, 1)) > 2); + } + private void verifyMlResponse(String uri) throws Exception { Response response = TestHelper.makeRequest(client(), "GET", uri, null, TestData.matchAllSearchQuery(), null); HttpEntity entity = response.getEntity(); diff --git a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityRestTestCase.java b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityRestTestCase.java index 4eefec83fe..734e5c5c98 100644 --- a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityRestTestCase.java +++ b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityRestTestCase.java @@ -49,7 +49,6 @@ import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.util.EntityUtils; import org.junit.After; -import org.junit.Before; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.RestClient; @@ -106,20 +105,6 @@ protected boolean isHttps() { return isHttps; } - @Before - public void setupSettings() throws IOException { - Response response = TestHelper - .makeRequest( - client(), - "PUT", - "_cluster/settings", - null, - "{\"persistent\":{\"plugins.ml_commons.only_run_on_ml_node\":false}}", - ImmutableList.of(new BasicHeader(HttpHeaders.USER_AGENT, "")) - ); - assertEquals(200, response.getStatusLine().getStatusCode()); - } - @Override protected String getProtocol() { return isHttps() ? "https" : "http"; From 3e989c3e9a80f440b444b9e02bdc4c0960aada17 Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Wed, 18 Jan 2023 00:33:29 +0000 Subject: [PATCH 04/16] bug fix Signed-off-by: Sicheng Song --- .../ml/bwc/MLCommonsBackwardsCompatibilityIT.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java index 824aa2ce86..f851920264 100644 --- a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java +++ b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java @@ -12,6 +12,7 @@ import java.util.*; import java.util.stream.Collectors; +import com.google.common.collect.ImmutableMap; import org.apache.http.HttpEntity; import org.apache.http.HttpHeaders; import org.apache.http.message.BasicHeader; @@ -29,7 +30,6 @@ import org.opensearch.test.rest.OpenSearchRestTestCase; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; public class MLCommonsBackwardsCompatibilityIT extends MLCommonsBackwardsCompatibilityRestTestCase { @@ -214,15 +214,6 @@ private void testSettingShifting() throws IOException { ImmutableList.of(new BasicHeader(HttpHeaders.USER_AGENT, "")) ); assertEquals(200, bwcResponse.getStatusLine().getStatusCode()); - - String jsonEntity = "{\n" - + " \"persistent\" : {\n" - + " \"plugins.ml_commons.native_memory_threshold\" : 100 \n" - + " }\n" - + "}"; - bwcResponse = TestHelper - .makeRequest(client(), "PUT", "_cluster/settings", ImmutableMap.of(), TestHelper.toHttpEntity(jsonEntity), null); - assertEquals(200, bwcResponse.getStatusLine().getStatusCode()); } private String getModelIdWithFunctionName(FunctionName functionName) throws IOException { From 90f5a2e374016f32db36395ee18bc9388a760ef3 Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Fri, 20 Jan 2023 02:43:51 +0000 Subject: [PATCH 05/16] use try-catch to catch the response exception of bwc Signed-off-by: Sicheng Song --- .../MLCommonsBackwardsCompatibilityIT.java | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java index f851920264..e8ad84e029 100644 --- a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java +++ b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java @@ -19,6 +19,7 @@ import org.junit.Assume; import org.junit.Before; import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; import org.opensearch.common.settings.Settings; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.ml.common.FunctionName; @@ -177,29 +178,40 @@ public void testBackwardsCompatibility() throws Exception { assertTrue(rows.size() > 1); }); } else if (isNewerVersion(opensearchVersion)) { - testSettingShifting(); // train predict with old data ingestIrisData(irisIndex); - trainAndPredict(client(), FunctionName.KMEANS, irisIndex, kMeansParams, searchSourceBuilder, predictionResult -> { - ArrayList rows = (ArrayList) predictionResult.get("rows"); - assertTrue(rows.size() > 0); - }); + try { + trainAndPredict(client(), FunctionName.KMEANS, irisIndex, kMeansParams, searchSourceBuilder, predictionResult -> { + ArrayList rows = (ArrayList) predictionResult.get("rows"); + assertTrue(rows.size() > 0); + }); + } catch(ResponseException e) { + testSettingShifting(); + trainAndPredict(client(), FunctionName.KMEANS, irisIndex, kMeansParams, searchSourceBuilder, predictionResult -> { + ArrayList rows = (ArrayList) predictionResult.get("rows"); + assertTrue(rows.size() > 0); + }); + } } else { throw new AssertionError("Cannot get the correct version for opensearch ml-commons plugin for the bwc test."); } - break; case UPGRADED: assertTrue(pluginNames.contains("opensearch-ml")); assertTrue(isNewerVersion(opensearchVersion)); - testSettingShifting(); ingestIrisData(irisIndex); - trainAndPredict(client(), FunctionName.KMEANS, irisIndex, kMeansParams, searchSourceBuilder, predictionResult -> { - ArrayList rows = (ArrayList) predictionResult.get("rows"); - assertTrue(rows.size() > 0); - }); - break; + try { + trainAndPredict(client(), FunctionName.KMEANS, irisIndex, kMeansParams, searchSourceBuilder, predictionResult -> { + ArrayList rows = (ArrayList) predictionResult.get("rows"); + assertTrue(rows.size() > 0); + }); + } catch(ResponseException e) { + testSettingShifting(); + trainAndPredict(client(), FunctionName.KMEANS, irisIndex, kMeansParams, searchSourceBuilder, predictionResult -> { + ArrayList rows = (ArrayList) predictionResult.get("rows"); + assertTrue(rows.size() > 0); + }); + } } - break; } } From 43cdba87102bc930c041cc1fc431cb23b65c6af2 Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Fri, 20 Jan 2023 03:01:59 +0000 Subject: [PATCH 06/16] test assertion Signed-off-by: Sicheng Song --- .../opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java index e8ad84e029..d24e4c49c3 100644 --- a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java +++ b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java @@ -197,7 +197,7 @@ public void testBackwardsCompatibility() throws Exception { } case UPGRADED: assertTrue(pluginNames.contains("opensearch-ml")); - assertTrue(isNewerVersion(opensearchVersion)); + assertEquals("2.6.0", opensearchVersion); ingestIrisData(irisIndex); try { trainAndPredict(client(), FunctionName.KMEANS, irisIndex, kMeansParams, searchSourceBuilder, predictionResult -> { From 2d47b5eab1a306af53733b0347a5eaa9ddf776a8 Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Fri, 20 Jan 2023 03:17:49 +0000 Subject: [PATCH 07/16] add back break to ensure upgraded Signed-off-by: Sicheng Song --- .../opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java index d24e4c49c3..e138ad4f58 100644 --- a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java +++ b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java @@ -195,6 +195,7 @@ public void testBackwardsCompatibility() throws Exception { } else { throw new AssertionError("Cannot get the correct version for opensearch ml-commons plugin for the bwc test."); } + break; case UPGRADED: assertTrue(pluginNames.contains("opensearch-ml")); assertEquals("2.6.0", opensearchVersion); @@ -211,7 +212,9 @@ public void testBackwardsCompatibility() throws Exception { assertTrue(rows.size() > 0); }); } + break; } + break; } } From 2c02aeb75649a244fb0ea04ecf958aa181b82ac7 Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Fri, 20 Jan 2023 03:23:32 +0000 Subject: [PATCH 08/16] fix assertation Signed-off-by: Sicheng Song --- .../opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java index e138ad4f58..0915539574 100644 --- a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java +++ b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java @@ -198,7 +198,7 @@ public void testBackwardsCompatibility() throws Exception { break; case UPGRADED: assertTrue(pluginNames.contains("opensearch-ml")); - assertEquals("2.6.0", opensearchVersion); + assertTrue(isNewerVersion(opensearchVersion)); ingestIrisData(irisIndex); try { trainAndPredict(client(), FunctionName.KMEANS, irisIndex, kMeansParams, searchSourceBuilder, predictionResult -> { From e9916eb3229cf65dfdc7f4a25a3e2874a555b6dd Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Fri, 20 Jan 2023 03:46:45 +0000 Subject: [PATCH 09/16] use nested try-catch to catch the response exception of bwc Signed-off-by: Sicheng Song --- .../MLCommonsBackwardsCompatibilityIT.java | 98 +++++++++++++++---- 1 file changed, 80 insertions(+), 18 deletions(-) diff --git a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java index 0915539574..19bb6147de 100644 --- a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java +++ b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java @@ -12,7 +12,6 @@ import java.util.*; import java.util.stream.Collectors; -import com.google.common.collect.ImmutableMap; import org.apache.http.HttpEntity; import org.apache.http.HttpHeaders; import org.apache.http.message.BasicHeader; @@ -31,6 +30,7 @@ import org.opensearch.test.rest.OpenSearchRestTestCase; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; public class MLCommonsBackwardsCompatibilityIT extends MLCommonsBackwardsCompatibilityRestTestCase { @@ -181,16 +181,45 @@ public void testBackwardsCompatibility() throws Exception { // train predict with old data ingestIrisData(irisIndex); try { - trainAndPredict(client(), FunctionName.KMEANS, irisIndex, kMeansParams, searchSourceBuilder, predictionResult -> { - ArrayList rows = (ArrayList) predictionResult.get("rows"); - assertTrue(rows.size() > 0); - }); - } catch(ResponseException e) { - testSettingShifting(); - trainAndPredict(client(), FunctionName.KMEANS, irisIndex, kMeansParams, searchSourceBuilder, predictionResult -> { - ArrayList rows = (ArrayList) predictionResult.get("rows"); - assertTrue(rows.size() > 0); - }); + trainAndPredict( + client(), + FunctionName.KMEANS, + irisIndex, + kMeansParams, + searchSourceBuilder, + predictionResult -> { + ArrayList rows = (ArrayList) predictionResult.get("rows"); + assertTrue(rows.size() > 0); + } + ); + } catch (ResponseException e1) { + mlNodeSettingShifting(); + try { + trainAndPredict( + client(), + FunctionName.KMEANS, + irisIndex, + kMeansParams, + searchSourceBuilder, + predictionResult -> { + ArrayList rows = (ArrayList) predictionResult.get("rows"); + assertTrue(rows.size() > 0); + } + ); + } catch (ResponseException e2) { + memoryThresholdSettingShifting(); + trainAndPredict( + client(), + FunctionName.KMEANS, + irisIndex, + kMeansParams, + searchSourceBuilder, + predictionResult -> { + ArrayList rows = (ArrayList) predictionResult.get("rows"); + assertTrue(rows.size() > 0); + } + ); + } } } else { throw new AssertionError("Cannot get the correct version for opensearch ml-commons plugin for the bwc test."); @@ -205,12 +234,34 @@ public void testBackwardsCompatibility() throws Exception { ArrayList rows = (ArrayList) predictionResult.get("rows"); assertTrue(rows.size() > 0); }); - } catch(ResponseException e) { - testSettingShifting(); - trainAndPredict(client(), FunctionName.KMEANS, irisIndex, kMeansParams, searchSourceBuilder, predictionResult -> { - ArrayList rows = (ArrayList) predictionResult.get("rows"); - assertTrue(rows.size() > 0); - }); + } catch (ResponseException e1) { + mlNodeSettingShifting(); + try { + trainAndPredict( + client(), + FunctionName.KMEANS, + irisIndex, + kMeansParams, + searchSourceBuilder, + predictionResult -> { + ArrayList rows = (ArrayList) predictionResult.get("rows"); + assertTrue(rows.size() > 0); + } + ); + } catch (ResponseException e2) { + memoryThresholdSettingShifting(); + trainAndPredict( + client(), + FunctionName.KMEANS, + irisIndex, + kMeansParams, + searchSourceBuilder, + predictionResult -> { + ArrayList rows = (ArrayList) predictionResult.get("rows"); + assertTrue(rows.size() > 0); + } + ); + } } break; } @@ -218,7 +269,7 @@ public void testBackwardsCompatibility() throws Exception { } } - private void testSettingShifting() throws IOException { + private void mlNodeSettingShifting() throws IOException { Response bwcResponse = TestHelper .makeRequest( client(), @@ -231,6 +282,17 @@ private void testSettingShifting() throws IOException { assertEquals(200, bwcResponse.getStatusLine().getStatusCode()); } + private void memoryThresholdSettingShifting() throws IOException { + String jsonEntity = "{\n" + + " \"persistent\" : {\n" + + " \"plugins.ml_commons.native_memory_threshold\" : 100 \n" + + " }\n" + + "}"; + Response bwcResponse = TestHelper + .makeRequest(client(), "PUT", "_cluster/settings", ImmutableMap.of(), TestHelper.toHttpEntity(jsonEntity), null); + assertEquals(200, bwcResponse.getStatusLine().getStatusCode()); + } + private String getModelIdWithFunctionName(FunctionName functionName) throws IOException { String modelQuery = "{\"query\": {" + "\"term\": {" From 4f1dd3cbad655452feda1c531d45bf813c3bde23 Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Fri, 20 Jan 2023 22:12:53 +0000 Subject: [PATCH 10/16] explicitly assert the error message for bwc test Signed-off-by: Sicheng Song --- .../bwc/MLCommonsBackwardsCompatibilityIT.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java index 19bb6147de..12f3b56efd 100644 --- a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java +++ b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java @@ -193,6 +193,7 @@ public void testBackwardsCompatibility() throws Exception { } ); } catch (ResponseException e1) { + assertEquals(mlNodeSettingExceptionMessage(), "{" + e1.getMessage().split("[{]", 2)[1]); mlNodeSettingShifting(); try { trainAndPredict( @@ -207,6 +208,7 @@ public void testBackwardsCompatibility() throws Exception { } ); } catch (ResponseException e2) { + assertEquals(memoryThresholdSettingExceptionMessage(), "{" + e2.getMessage().split("[{]", 2)[1]); memoryThresholdSettingShifting(); trainAndPredict( client(), @@ -235,6 +237,7 @@ public void testBackwardsCompatibility() throws Exception { assertTrue(rows.size() > 0); }); } catch (ResponseException e1) { + assertEquals(mlNodeSettingExceptionMessage(), "{" + e1.getMessage().split("[{]", 2)[1]); mlNodeSettingShifting(); try { trainAndPredict( @@ -249,6 +252,7 @@ public void testBackwardsCompatibility() throws Exception { } ); } catch (ResponseException e2) { + assertEquals(memoryThresholdSettingExceptionMessage(), "{" + e2.getMessage().split("[{]", 2)[1]); memoryThresholdSettingShifting(); trainAndPredict( client(), @@ -269,6 +273,18 @@ public void testBackwardsCompatibility() throws Exception { } } + private String mlNodeSettingExceptionMessage() { + return "{\"error\":{\"root_cause\":[{\"type\":\"m_l_resource_not_found_exception\",\"reason\":\"No eligible node found to execute this request. " + + "It's best practice to provision ML nodes to serve your models. You can disable this setting to serve the model on your data node for development purposes by disabling the \\\"plugins.ml_commons.only_run_on_ml_node\\\" configuration using the _cluster/setting api\"}]," + + "\"type\":\"m_l_resource_not_found_exception\",\"reason\":\"No eligible node found to execute this request. It's best practice to provision ML nodes to serve your models. " + + "You can disable this setting to serve the model on your data node for development purposes by disabling the \\\"plugins.ml_commons.only_run_on_ml_node\\\" configuration using the _cluster/setting api\"},\"status\":500}"; + } + + private String memoryThresholdSettingExceptionMessage() { + return "{\"error\":{\"root_cause\":[{\"type\":\"illegal_argument_exception\",\"reason\":\"persistent setting [plugins.ml_commons.native_memory_threshold], not recognized\"}]," + + "\"type\":\"illegal_argument_exception\",\"reason\":\"persistent setting [plugins.ml_commons.native_memory_threshold], not recognized\"},\"status\":400}}"; + } + private void mlNodeSettingShifting() throws IOException { Response bwcResponse = TestHelper .makeRequest( From c0815f230556854dcd6f280124f6d47020d43269 Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Fri, 20 Jan 2023 22:27:08 +0000 Subject: [PATCH 11/16] fix the error message for bwc test Signed-off-by: Sicheng Song --- .../opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java index 12f3b56efd..4c6e366da8 100644 --- a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java +++ b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java @@ -281,8 +281,8 @@ private String mlNodeSettingExceptionMessage() { } private String memoryThresholdSettingExceptionMessage() { - return "{\"error\":{\"root_cause\":[{\"type\":\"illegal_argument_exception\",\"reason\":\"persistent setting [plugins.ml_commons.native_memory_threshold], not recognized\"}]," - + "\"type\":\"illegal_argument_exception\",\"reason\":\"persistent setting [plugins.ml_commons.native_memory_threshold], not recognized\"},\"status\":400}}"; + return "{\"error\":{\"root_cause\":[{\"type\":\"m_l_limit_exceeded_exception\",\"reason\":\"m_l_limit_exceeded_exception: Native Memory Circuit Breaker is open, please check your resources!\"}]," + + "\"type\":\"m_l_limit_exceeded_exception\",\"reason\":\"m_l_limit_exceeded_exception: Native Memory Circuit Breaker is open, please check your resources!\"},\"status\":500}"; } private void mlNodeSettingShifting() throws IOException { From 73485f206aca35bda7b670daf38204f1ca7c0c21 Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Fri, 20 Jan 2023 22:35:21 +0000 Subject: [PATCH 12/16] fix the error message for bwc test Signed-off-by: Sicheng Song --- .../opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java index 4c6e366da8..a691b31ab9 100644 --- a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java +++ b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java @@ -281,8 +281,8 @@ private String mlNodeSettingExceptionMessage() { } private String memoryThresholdSettingExceptionMessage() { - return "{\"error\":{\"root_cause\":[{\"type\":\"m_l_limit_exceeded_exception\",\"reason\":\"m_l_limit_exceeded_exception: Native Memory Circuit Breaker is open, please check your resources!\"}]," - + "\"type\":\"m_l_limit_exceeded_exception\",\"reason\":\"m_l_limit_exceeded_exception: Native Memory Circuit Breaker is open, please check your resources!\"},\"status\":500}"; + return "{\"error\":{\"root_cause\":[{\"type\":\"m_l_limit_exceeded_exception\",\"reason\":\"Native Memory Circuit Breaker is open, please check your resources!\"}]," + + "\"type\":\"m_l_limit_exceeded_exception\",\"reason\":Native Memory Circuit Breaker is open, please check your resources!\"},\"status\":500}"; } private void mlNodeSettingShifting() throws IOException { From 38d67892c6d5322e26b8d2bea7421e7b42387a59 Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Fri, 20 Jan 2023 22:59:13 +0000 Subject: [PATCH 13/16] fix the error message for bwc test Signed-off-by: Sicheng Song --- .../opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java index a691b31ab9..bbaeee10d2 100644 --- a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java +++ b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java @@ -193,7 +193,6 @@ public void testBackwardsCompatibility() throws Exception { } ); } catch (ResponseException e1) { - assertEquals(mlNodeSettingExceptionMessage(), "{" + e1.getMessage().split("[{]", 2)[1]); mlNodeSettingShifting(); try { trainAndPredict( @@ -237,7 +236,6 @@ public void testBackwardsCompatibility() throws Exception { assertTrue(rows.size() > 0); }); } catch (ResponseException e1) { - assertEquals(mlNodeSettingExceptionMessage(), "{" + e1.getMessage().split("[{]", 2)[1]); mlNodeSettingShifting(); try { trainAndPredict( @@ -252,7 +250,6 @@ public void testBackwardsCompatibility() throws Exception { } ); } catch (ResponseException e2) { - assertEquals(memoryThresholdSettingExceptionMessage(), "{" + e2.getMessage().split("[{]", 2)[1]); memoryThresholdSettingShifting(); trainAndPredict( client(), From f2fb866c698760d1228bca58776f38f85d44a695 Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Fri, 20 Jan 2023 23:12:04 +0000 Subject: [PATCH 14/16] fix the error message for bwc test Signed-off-by: Sicheng Song --- .../org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java index bbaeee10d2..194317cbb8 100644 --- a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java +++ b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java @@ -207,7 +207,6 @@ public void testBackwardsCompatibility() throws Exception { } ); } catch (ResponseException e2) { - assertEquals(memoryThresholdSettingExceptionMessage(), "{" + e2.getMessage().split("[{]", 2)[1]); memoryThresholdSettingShifting(); trainAndPredict( client(), From bf748f52bdc60b7298806bb2d1a041f5578cd125 Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Mon, 23 Jan 2023 20:15:56 +0000 Subject: [PATCH 15/16] bypass twothirdtest Signed-off-by: Sicheng Song --- .../MLCommonsBackwardsCompatibilityIT.java | 46 +++++++++---------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java index 194317cbb8..d90c6c3130 100644 --- a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java +++ b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java @@ -207,18 +207,8 @@ public void testBackwardsCompatibility() throws Exception { } ); } catch (ResponseException e2) { - memoryThresholdSettingShifting(); - trainAndPredict( - client(), - FunctionName.KMEANS, - irisIndex, - kMeansParams, - searchSourceBuilder, - predictionResult -> { - ArrayList rows = (ArrayList) predictionResult.get("rows"); - assertTrue(rows.size() > 0); - } - ); + assertEquals(memoryThresholdSettingExceptionMessage(CLUSTER_TYPE), "{" + e2.getMessage().split("[{]", 2)[1]); + break; } } } else { @@ -249,17 +239,18 @@ public void testBackwardsCompatibility() throws Exception { } ); } catch (ResponseException e2) { + assertEquals(memoryThresholdSettingExceptionMessage(CLUSTER_TYPE), "{" + e2.getMessage().split("[{]", 2)[1]); memoryThresholdSettingShifting(); trainAndPredict( - client(), - FunctionName.KMEANS, - irisIndex, - kMeansParams, - searchSourceBuilder, - predictionResult -> { - ArrayList rows = (ArrayList) predictionResult.get("rows"); - assertTrue(rows.size() > 0); - } + client(), + FunctionName.KMEANS, + irisIndex, + kMeansParams, + searchSourceBuilder, + predictionResult -> { + ArrayList rows = (ArrayList) predictionResult.get("rows"); + assertTrue(rows.size() > 0); + } ); } } @@ -276,9 +267,16 @@ private String mlNodeSettingExceptionMessage() { + "You can disable this setting to serve the model on your data node for development purposes by disabling the \\\"plugins.ml_commons.only_run_on_ml_node\\\" configuration using the _cluster/setting api\"},\"status\":500}"; } - private String memoryThresholdSettingExceptionMessage() { - return "{\"error\":{\"root_cause\":[{\"type\":\"m_l_limit_exceeded_exception\",\"reason\":\"Native Memory Circuit Breaker is open, please check your resources!\"}]," - + "\"type\":\"m_l_limit_exceeded_exception\",\"reason\":Native Memory Circuit Breaker is open, please check your resources!\"},\"status\":500}"; + private String memoryThresholdSettingExceptionMessage(ClusterType clusterType) { + if (clusterType == ClusterType.MIXED){ + return "{\"error\":{\"root_cause\":[{\"type\":\"m_l_limit_exceeded_exception\",\"reason\":\"Native Memory Circuit Breaker is open, please check your resources!\"}]," + + "\"type\":\"m_l_limit_exceeded_exception\",\"reason\":\"Native Memory Circuit Breaker is open, please check your resources!\"},\"status\":500}"; + } else if (clusterType == ClusterType.UPGRADED) { + return "{\"error\":{\"root_cause\":[{\"type\":\"m_l_limit_exceeded_exception\",\"reason\":\"m_l_limit_exceeded_exception: Native Memory Circuit Breaker is open, please check your resources!\"}]," + + "\"type\":\"m_l_limit_exceeded_exception\",\"reason\":\"m_l_limit_exceeded_exception: Native Memory Circuit Breaker is open, please check your resources!\"},\"status\":500}";} + else { + throw new AssertionError("Invalid ClusterType"); + } } private void mlNodeSettingShifting() throws IOException { From 741a1e62250ad4f8fb7d54ef07c4c36aebd776b6 Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Mon, 23 Jan 2023 23:02:54 +0000 Subject: [PATCH 16/16] limited error message Signed-off-by: Sicheng Song --- .../MLCommonsBackwardsCompatibilityIT.java | 49 +++++++------------ 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java index d90c6c3130..6d791bde78 100644 --- a/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java +++ b/plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java @@ -207,7 +207,11 @@ public void testBackwardsCompatibility() throws Exception { } ); } catch (ResponseException e2) { - assertEquals(memoryThresholdSettingExceptionMessage(CLUSTER_TYPE), "{" + e2.getMessage().split("[{]", 2)[1]); + Map modelResponseMap = gson.fromJson(("{" + e2.getMessage().split("[{]", 2)[1]), Map.class); + Map errorMap = (Map) modelResponseMap.get("error"); + List> rootCauses = (List>) errorMap.get("root_cause"); + Set rootCauseTypeSet = rootCauses.stream().map(map -> map.get("type")).collect(Collectors.toSet()); + assertEquals("m_l_limit_exceeded_exception", rootCauseTypeSet.iterator().next().toString()); break; } } @@ -239,18 +243,22 @@ public void testBackwardsCompatibility() throws Exception { } ); } catch (ResponseException e2) { - assertEquals(memoryThresholdSettingExceptionMessage(CLUSTER_TYPE), "{" + e2.getMessage().split("[{]", 2)[1]); + Map modelResponseMap = gson.fromJson(("{" + e2.getMessage().split("[{]", 2)[1]), Map.class); + Map errorMap = (Map) modelResponseMap.get("error"); + List> rootCauses = (List>) errorMap.get("root_cause"); + Set rootCauseTypeSet = rootCauses.stream().map(map -> map.get("type")).collect(Collectors.toSet()); + assertEquals("m_l_limit_exceeded_exception", rootCauseTypeSet.iterator().next().toString()); memoryThresholdSettingShifting(); trainAndPredict( - client(), - FunctionName.KMEANS, - irisIndex, - kMeansParams, - searchSourceBuilder, - predictionResult -> { - ArrayList rows = (ArrayList) predictionResult.get("rows"); - assertTrue(rows.size() > 0); - } + client(), + FunctionName.KMEANS, + irisIndex, + kMeansParams, + searchSourceBuilder, + predictionResult -> { + ArrayList rows = (ArrayList) predictionResult.get("rows"); + assertTrue(rows.size() > 0); + } ); } } @@ -260,25 +268,6 @@ public void testBackwardsCompatibility() throws Exception { } } - private String mlNodeSettingExceptionMessage() { - return "{\"error\":{\"root_cause\":[{\"type\":\"m_l_resource_not_found_exception\",\"reason\":\"No eligible node found to execute this request. " - + "It's best practice to provision ML nodes to serve your models. You can disable this setting to serve the model on your data node for development purposes by disabling the \\\"plugins.ml_commons.only_run_on_ml_node\\\" configuration using the _cluster/setting api\"}]," - + "\"type\":\"m_l_resource_not_found_exception\",\"reason\":\"No eligible node found to execute this request. It's best practice to provision ML nodes to serve your models. " - + "You can disable this setting to serve the model on your data node for development purposes by disabling the \\\"plugins.ml_commons.only_run_on_ml_node\\\" configuration using the _cluster/setting api\"},\"status\":500}"; - } - - private String memoryThresholdSettingExceptionMessage(ClusterType clusterType) { - if (clusterType == ClusterType.MIXED){ - return "{\"error\":{\"root_cause\":[{\"type\":\"m_l_limit_exceeded_exception\",\"reason\":\"Native Memory Circuit Breaker is open, please check your resources!\"}]," - + "\"type\":\"m_l_limit_exceeded_exception\",\"reason\":\"Native Memory Circuit Breaker is open, please check your resources!\"},\"status\":500}"; - } else if (clusterType == ClusterType.UPGRADED) { - return "{\"error\":{\"root_cause\":[{\"type\":\"m_l_limit_exceeded_exception\",\"reason\":\"m_l_limit_exceeded_exception: Native Memory Circuit Breaker is open, please check your resources!\"}]," - + "\"type\":\"m_l_limit_exceeded_exception\",\"reason\":\"m_l_limit_exceeded_exception: Native Memory Circuit Breaker is open, please check your resources!\"},\"status\":500}";} - else { - throw new AssertionError("Invalid ClusterType"); - } - } - private void mlNodeSettingShifting() throws IOException { Response bwcResponse = TestHelper .makeRequest(