Skip to content

fix bwc bug when incrementing to OS 2.6.0 #705

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
b4sjoo merged 2 commits into from
Jan 24, 2023
Merged

fix bwc bug when incrementing to OS 2.6.0 #705

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dee052d
fix bwc bug when incrementing to OS 2.6.0
b4sjoo Jan 23, 2023
fb4a6f3
Fix index not found issue
b4sjoo Jan 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 9 additions & 11 deletions plugin/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -386,11 +386,7 @@ String baseName = "mlCommonsBwcCluster"
String bwcMlPlugin = "opensearch-ml-" + bwcVersion + ".zip"
String bwcFilePath = "src/test/resources/org/opensearch/ml/bwc/"
String bwcRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + bwcShortVersion + "/latest/linux/x64/tar/builds/opensearch/plugins/" + bwcMlPlugin
String project_no_snapshot = project.version.replace("-SNAPSHOT","")
String opensearch_no_snapshot = opensearch_version.replace("-SNAPSHOT","")
String opensearchMlPlugin = "opensearch-ml-" + project_no_snapshot + ".zip"
String opensearchMlRemoteFile = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot +
'/latest/linux/x64/tar/builds/opensearch/plugins/' + opensearchMlPlugin
String opensearchMlPlugin = "opensearch-ml-" + project.version + ".zip"

2.times {i ->
testClusters {
Expand All @@ -404,6 +400,10 @@ String opensearchMlRemoteFile = 'https://ci.opensearch.org/ci/dbc/distribution-b
return new RegularFile() {
@Override
File getAsFile() {
File bwcDir = new File('./plugin/' + bwcFilePath)
if (!bwcDir.exists()) {
bwcDir.mkdirs()
}
File dir = new File('./plugin/' + bwcFilePath + bwcVersion)
if (!dir.exists()) {
dir.mkdirs()
Expand All @@ -430,13 +430,11 @@ List<Provider<RegularFile>> plugins = [
return new RegularFile() {
@Override
File getAsFile() {
if (new File('./plugin/' + bwcFilePath + project.version).exists()) {
project.delete(files('./plugin/' + bwcFilePath + project.version))
project.mkdir "$bwcFilePath/$project.version"
copy {
from "$buildDir/distributions/$opensearchMlPlugin"
into "$bwcFilePath/$project.version"
}
project.mkdir bwcFilePath + project.version
ant.get(src: opensearchMlRemoteFile,
dest: bwcFilePath + project.version,
httpusecaches: false)
return fileTree(bwcFilePath + project.version).getSingleFile()
}
}
Expand Down
121 changes: 110 additions & 11 deletions plugin/src/test/java/org/opensearch/ml/bwc/MLCommonsBackwardsCompatibilityIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,12 @@
import java.util.stream.Collectors;

import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
import org.apache.http.message.BasicHeader;
import org.junit.Assume;
import org.junit.Before;
import org.opensearch.client.Response;
import org.opensearch.client.ResponseException;
import org.opensearch.common.settings.Settings;
import org.opensearch.index.query.MatchAllQueryBuilder;
import org.opensearch.ml.common.FunctionName;
Expand All @@ -26,6 +29,9 @@
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.test.rest.OpenSearchRestTestCase;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;

public class MLCommonsBackwardsCompatibilityIT extends MLCommonsBackwardsCompatibilityRestTestCase {

private final ClusterType CLUSTER_TYPE = ClusterType.parse(System.getProperty("tests.rest.bwcsuite"));
Expand Down Expand Up @@ -171,31 +177,120 @@ public void testBackwardsCompatibility() throws Exception {
ArrayList rows = (ArrayList) predictionResult.get("rows");
assertTrue(rows.size() > 1);
});
} else if (opensearchVersion.equals("2.5.0")) {
// train predict with old data
} else if (isNewerVersion(opensearchVersion)) {
ingestIrisData(irisIndex);
trainAndPredict(client(), FunctionName.KMEANS, irisIndex, kMeansParams, searchSourceBuilder, predictionResult -> {
ArrayList rows = (ArrayList) predictionResult.get("rows");
assertTrue(rows.size() > 0);
});
try {
trainAndPredict(
client(),
FunctionName.KMEANS,
irisIndex,
kMeansParams,
searchSourceBuilder,
predictionResult -> {
ArrayList rows = (ArrayList) predictionResult.get("rows");
assertTrue(rows.size() > 0);
}
);
} catch (ResponseException e1) {
mlNodeSettingShifting();
try {
trainAndPredict(
client(),
FunctionName.KMEANS,
irisIndex,
kMeansParams,
searchSourceBuilder,
predictionResult -> {
ArrayList rows = (ArrayList) predictionResult.get("rows");
assertTrue(rows.size() > 0);
}
);
} catch (ResponseException e2) {
Map modelResponseMap = gson.fromJson(("{" + e2.getMessage().split("[{]", 2)[1]), Map.class);
Map errorMap = (Map) modelResponseMap.get("error");
List<Map<String, Object>> rootCauses = (List<Map<String, Object>>) errorMap.get("root_cause");
Set<Object> rootCauseTypeSet = rootCauses.stream().map(map -> map.get("type")).collect(Collectors.toSet());
assertEquals("m_l_limit_exceeded_exception", rootCauseTypeSet.iterator().next().toString());
break;
}
}
} else {
throw new AssertionError("Cannot get the correct version for opensearch ml-commons plugin for the bwc test.");
}
break;
case UPGRADED:
assertTrue(pluginNames.contains("opensearch-ml"));
assertEquals("2.5.0", opensearchVersion);
assertTrue(isNewerVersion(opensearchVersion));
ingestIrisData(irisIndex);
trainAndPredict(client(), FunctionName.KMEANS, irisIndex, kMeansParams, searchSourceBuilder, predictionResult -> {
ArrayList rows = (ArrayList) predictionResult.get("rows");
assertTrue(rows.size() > 0);
});
try {
trainAndPredict(client(), FunctionName.KMEANS, irisIndex, kMeansParams, searchSourceBuilder, predictionResult -> {
ArrayList rows = (ArrayList) predictionResult.get("rows");
assertTrue(rows.size() > 0);
});
} catch (ResponseException e1) {
mlNodeSettingShifting();
try {
trainAndPredict(
client(),
FunctionName.KMEANS,
irisIndex,
kMeansParams,
searchSourceBuilder,
predictionResult -> {
ArrayList rows = (ArrayList) predictionResult.get("rows");
assertTrue(rows.size() > 0);
}
);
} catch (ResponseException e2) {
Map modelResponseMap = gson.fromJson(("{" + e2.getMessage().split("[{]", 2)[1]), Map.class);
Map errorMap = (Map) modelResponseMap.get("error");
List<Map<String, Object>> rootCauses = (List<Map<String, Object>>) errorMap.get("root_cause");
Set<Object> rootCauseTypeSet = rootCauses.stream().map(map -> map.get("type")).collect(Collectors.toSet());
assertEquals("m_l_limit_exceeded_exception", rootCauseTypeSet.iterator().next().toString());
memoryThresholdSettingShifting();
trainAndPredict(
client(),
FunctionName.KMEANS,
irisIndex,
kMeansParams,
searchSourceBuilder,
predictionResult -> {
ArrayList rows = (ArrayList) predictionResult.get("rows");
assertTrue(rows.size() > 0);
}
);
}
}
break;
}
break;
}
}

private void mlNodeSettingShifting() throws IOException {
Response bwcResponse = TestHelper
.makeRequest(
client(),
"PUT",
"_cluster/settings",
null,
"{\"persistent\":{\"plugins.ml_commons.only_run_on_ml_node\":false}}",
ImmutableList.of(new BasicHeader(HttpHeaders.USER_AGENT, ""))
);
assertEquals(200, bwcResponse.getStatusLine().getStatusCode());
}

private void memoryThresholdSettingShifting() throws IOException {
String jsonEntity = "{\n"
+ " \"persistent\" : {\n"
+ " \"plugins.ml_commons.native_memory_threshold\" : 100 \n"
+ " }\n"
+ "}";
Response bwcResponse = TestHelper
.makeRequest(client(), "PUT", "_cluster/settings", ImmutableMap.of(), TestHelper.toHttpEntity(jsonEntity), null);
assertEquals(200, bwcResponse.getStatusLine().getStatusCode());
}

private String getModelIdWithFunctionName(FunctionName functionName) throws IOException {
String modelQuery = "{\"query\": {"
+ "\"term\": {"
Expand All @@ -217,6 +312,10 @@ private String getModelIdWithFunctionName(FunctionName functionName) throws IOEx
return modelIdSet.iterator().next().toString();
}

private boolean isNewerVersion(String osVersion) {
return (Integer.parseInt(osVersion.substring(2, 3)) > 4) || (Integer.parseInt(osVersion.substring(0, 1)) > 2);
}

private void verifyMlResponse(String uri) throws Exception {
Response response = TestHelper.makeRequest(client(), "GET", uri, null, TestData.matchAllSearchQuery(), null);
HttpEntity entity = response.getEntity();
Expand Down