Skip to content

Remove ES version dependency in bwc tests #136255

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jfreden merged 10 commits into from
Oct 10, 2025
Merged

Remove ES version dependency in bwc tests #136255

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
cb343c1
Remove ES version dependency in bwc tests
jfreden Oct 9, 2025
50b1c36
fixup!
jfreden Oct 9, 2025
882c591
Fix leaking rest clients
jfreden Oct 9, 2025
388b529
Merge branch 'main' into remove_version_from_test
jfreden Oct 9, 2025
64dac9c
Merge branch 'main' into remove_version_from_test
jfreden Oct 9, 2025
fddb9bb
Merge branch 'main' into remove_version_from_test
jfreden Oct 9, 2025
2246312
Merge branch 'main' into remove_version_from_test
jfreden Oct 10, 2025
f6f8334
Merge branch 'main' into remove_version_from_test
jfreden Oct 10, 2025
d07ffa4
Merge branch 'main' into remove_version_from_test
jfreden Oct 10, 2025
4a07b46
Merge branch 'main' into remove_version_from_test
jfreden Oct 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 91 additions & 31 deletions .../qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@
package org.elasticsearch.upgrades;

import org.apache.http.HttpHost;
import org.apache.http.client.methods.HttpGet;
import org.elasticsearch.Build;
import org.elasticsearch.TransportVersion;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
Expand All @@ -16,22 +18,25 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.core.Booleans;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.test.XContentTestUtils;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.test.rest.ObjectPath;
import org.elasticsearch.xpack.test.SecuritySettingsSourceField;
import org.junit.Before;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;

import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;

public abstract class AbstractUpgradeTestCase extends ESRestTestCase {
Expand Down Expand Up @@ -194,43 +199,98 @@ protected void closeClientsByVersion() throws IOException {
}

@SuppressWarnings("unchecked")
protected Map<Boolean, RestClient> getRestClientByCapability(Function<Map<String, Object>, Boolean> capabilityChecker)
throws IOException {
protected Map<String, String> getRestEndpointByIdNodeId() throws IOException {
Response response = client().performRequest(new Request("GET", "_nodes"));
assertOK(response);
ObjectPath objectPath = ObjectPath.createFromResponse(response);
Map<String, Object> nodesAsMap = objectPath.evaluate("nodes");
Map<Boolean, List<HttpHost>> hostsByCapability = new HashMap<>();

for (Map.Entry<String, Object> entry : nodesAsMap.entrySet()) {
Map<String, Object> nodeDetails = (Map<String, Object>) entry.getValue();
var capabilitySupported = capabilityChecker.apply(nodeDetails);
return nodesAsMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> {
Map<String, Object> nodeDetails = (Map<String, Object>) e.getValue();
Map<String, Object> httpInfo = (Map<String, Object>) nodeDetails.get("http");
hostsByCapability.computeIfAbsent(capabilitySupported, k -> new ArrayList<>())
.add(HttpHost.create((String) httpInfo.get("publish_address")));
}
return (String) httpInfo.get("publish_address");
}));
}

Map<Boolean, RestClient> clientsByCapability = new HashMap<>();
for (var entry : hostsByCapability.entrySet()) {
clientsByCapability.put(entry.getKey(), buildClient(restClientSettings(), entry.getValue().toArray(new HttpHost[0])));
}
return clientsByCapability;
}

protected void createClientsByCapability(Function<Map<String, Object>, Boolean> capabilityChecker) throws IOException {
var clientsByCapability = getRestClientByCapability(capabilityChecker);
if (clientsByCapability.size() == 2) {
for (Map.Entry<Boolean, RestClient> client : clientsByCapability.entrySet()) {
if (client.getKey() == false) {
oldVersionClient = client.getValue();
} else {
newVersionClient = client.getValue();
}
}
protected void createClientsByCapability(Predicate<TestNodeInfo> capabilityChecker) throws IOException {
var testNodesByCapability = collectNodeInfos(adminClient()).stream().collect(Collectors.partitioningBy(capabilityChecker));
if (testNodesByCapability.size() == 2) {
oldVersionClient = buildClient(
restClientSettings(),
new HttpHost[] { HttpHost.create(testNodesByCapability.get(false).getFirst().restEndpoint) }
);
newVersionClient = buildClient(
restClientSettings(),
new HttpHost[] { HttpHost.create(testNodesByCapability.get(true).getFirst().restEndpoint) }
);
assertThat(oldVersionClient, notNullValue());
assertThat(newVersionClient, notNullValue());
} else {
fail("expected 2 versions during rolling upgrade but got: " + clientsByCapability.size());
fail("expected 2 versions during rolling upgrade but got: " + testNodesByCapability.size());
}
}

protected Set<TestNodeInfo> collectNodeInfos(RestClient adminClient) throws IOException {
final Request request = new Request("GET", "_cluster/state");
request.addParameter("filter_path", "nodes_features");

final Response response = adminClient.performRequest(request);

final Map<String, Set<String>> nodeFeatures;
var responseData = responseAsMap(response);
if (responseData.get("nodes_features") instanceof List<?> nodesFeatures) {
nodeFeatures = nodesFeatures.stream()
.map(Map.class::cast)
.collect(Collectors.toUnmodifiableMap(nodeFeatureMap -> nodeFeatureMap.get("node_id").toString(), nodeFeatureMap -> {
@SuppressWarnings("unchecked")
var features = (List<String>) nodeFeatureMap.get("features");
return new HashSet<>(features);
}));
} else {
nodeFeatures = Map.of();
}
var restEndpointByNodeId = getRestEndpointByIdNodeId();

return nodeInfoById().entrySet().stream().map(entry -> {
var version = (String) extractValue((Map<?, ?>) entry.getValue(), "version");
assertNotNull(version);
var transportVersion = (Integer) extractValue((Map<?, ?>) entry.getValue(), "transport_version");
assertNotNull(transportVersion);
return new TestNodeInfo(
entry.getKey(),
version,
TransportVersion.fromId(transportVersion),
nodeFeatures.getOrDefault(entry.getKey(), Set.of()),
restEndpointByNodeId.get(entry.getKey())
);
}).collect(Collectors.toSet());
}

@SuppressWarnings("unchecked")
private static Map<String, Object> nodeInfoById() throws IOException {
final Response response = client().performRequest(new Request(HttpGet.METHOD_NAME, "_nodes/_all"));
assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
final Map<String, Object> nodes = (Map<String, Object>) extractValue(responseAsMap(response), "nodes");
assertNotNull("Nodes info is null", nodes);
return nodes;
}

protected record TestNodeInfo(
String nodeId,
String version,
TransportVersion transportVersion,
Set<String> features,
String restEndpoint
) {
public boolean isOriginalVersionCluster() {
return AbstractUpgradeTestCase.isOriginalCluster(this.version());
}

public boolean isUpgradedVersionCluster() {
return false == isOriginalVersionCluster();
}

public boolean supportsFeature(String feature) {
return features().contains(feature);
}
}

Expand Down
38 changes: 20 additions & 18 deletions ...ling-upgrade/src/test/java/org/elasticsearch/upgrades/ApiKeyBackwardsCompatibilityIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import java.util.Set;
import java.util.function.Consumer;

import static java.util.stream.Collectors.toSet;
import static org.elasticsearch.transport.RemoteClusterPortSettings.TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY;
import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTestHelper.randomApplicationPrivileges;
import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTestHelper.randomIndicesPrivileges;
Expand All @@ -53,6 +54,7 @@
public class ApiKeyBackwardsCompatibilityIT extends AbstractUpgradeTestCase {

private static final Version UPGRADE_FROM_VERSION = Version.fromString(System.getProperty("tests.upgrade_from_version"));
private static final String CERTIFICATE_IDENTITY_FIELD_FEATURE = "certificate_identity_field";

public void testQueryRestTypeKeys() throws IOException {
assumeTrue(
Expand Down Expand Up @@ -201,10 +203,20 @@ public void testCreatingAndUpdatingApiKeys() throws Exception {
}

public void testCertificateIdentityBackwardsCompatibility() throws Exception {
final Set<TestNodeInfo> nodes = collectNodeInfos(adminClient());

final Set<TestNodeInfo> newVersionNodes = nodes.stream().filter(TestNodeInfo::isUpgradedVersionCluster).collect(toSet());
final Set<TestNodeInfo> oldVersionNodes = nodes.stream().filter(TestNodeInfo::isOriginalVersionCluster).collect(toSet());

assumeTrue(
"Old version nodes must not support certificate identity feature",
oldVersionNodes.stream().noneMatch(info -> info.supportsFeature(CERTIFICATE_IDENTITY_FIELD_FEATURE))
);
assumeTrue(
"certificate identity backwards compatibility only relevant when upgrading from pre-9.2.0",
UPGRADE_FROM_VERSION.before(Version.V_9_2_0)
"New version nodes must support certificate identity feature",
newVersionNodes.stream().allMatch(info -> info.supportsFeature(CERTIFICATE_IDENTITY_FIELD_FEATURE))
);

switch (CLUSTER_TYPE) {
case OLD -> {
var exception = expectThrows(Exception.class, () -> createCrossClusterApiKeyWithCertIdentity("CN=test-.*"));
Expand Down Expand Up @@ -373,22 +385,15 @@ private static String randomRoleDescriptors(boolean includeRemoteDescriptors) {
}
}

boolean nodeSupportApiKeyRemoteIndices(Map<String, Object> nodeDetails) {
String nodeVersionString = (String) nodeDetails.get("version");
TransportVersion transportVersion = getTransportVersionWithFallback(
nodeVersionString,
nodeDetails.get("transport_version"),
() -> TransportVersion.zero()
);

if (transportVersion.equals(TransportVersion.zero())) {
boolean nodeSupportApiKeyRemoteIndices(TestNodeInfo testNodeInfo) {
if (testNodeInfo.transportVersion().equals(TransportVersion.zero())) {
// In cases where we were not able to find a TransportVersion, a pre-8.8.0 node answered about a newer (upgraded) node.
// In that case, the node will be current (upgraded), and remote indices are supported for sure.
var nodeIsCurrent = nodeVersionString.equals(Build.current().version());
var nodeIsCurrent = testNodeInfo.version().equals(Build.current().version());
assertTrue(nodeIsCurrent);
return true;
}
return transportVersion.onOrAfter(RemoteClusterPortSettings.TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY);
return testNodeInfo.transportVersion().onOrAfter(RemoteClusterPortSettings.TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY);
}

private static RoleDescriptor randomRoleDescriptor(boolean includeRemoteDescriptors) {
Expand Down Expand Up @@ -426,11 +431,8 @@ private void assertQuery(RestClient restClient, String body, Consumer<List<Map<S
apiKeysVerifier.accept(apiKeys);
}

private boolean nodeSupportsCertificateIdentity(Map<String, Object> nodeDetails) {
String nodeVersionString = (String) nodeDetails.get("version");
Version nodeVersion = Version.fromString(nodeVersionString);
// Certificate identity was introduced in 9.3.0
return nodeVersion.onOrAfter(Version.V_9_3_0);
private boolean nodeSupportsCertificateIdentity(TestNodeInfo nodeDetails) {
return nodeDetails.supportsFeature(CERTIFICATE_IDENTITY_FIELD_FEATURE);
}

private Tuple<String, String> createCrossClusterApiKeyWithCertIdentity(String certificateIdentity) throws IOException {
Expand Down
76 changes: 2 additions & 74 deletions ...ling-upgrade/src/test/java/org/elasticsearch/upgrades/QueryableBuiltInRolesUpgradeIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,23 +10,17 @@
import org.apache.http.client.methods.HttpGet;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.util.Maps;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.test.rest.ObjectPath;
import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
import org.junit.Before;

import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import static java.util.stream.Collectors.toSet;
import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;
Expand Down Expand Up @@ -59,11 +53,11 @@ public void testBuiltInRolesSyncedOnClusterUpgrade() throws Exception {

assumeTrue(
"Old version nodes must not support queryable feature",
oldVersionNodes.stream().noneMatch(TestNodeInfo::supportsQueryableBuiltInRolesFeature)
oldVersionNodes.stream().noneMatch(info -> info.supportsFeature(QUERYABLE_BUILT_IN_ROLES_NODE_FEATURE))
);
assumeTrue(
"New version nodes must support queryable feature",
newVersionNodes.stream().allMatch(TestNodeInfo::supportsQueryableBuiltInRolesFeature)
newVersionNodes.stream().allMatch(info -> info.supportsFeature(QUERYABLE_BUILT_IN_ROLES_NODE_FEATURE))
);

switch (CLUSTER_TYPE) {
Expand All @@ -80,79 +74,13 @@ public void testBuiltInRolesSyncedOnClusterUpgrade() throws Exception {
}
}

record TestNodeInfo(String nodeId, String version, Set<String> features) {

public boolean isOriginalVersionCluster() {
return AbstractUpgradeTestCase.isOriginalCluster(this.version());
}

public boolean isUpgradedVersionCluster() {
return false == isOriginalVersionCluster();
}

public boolean supportsQueryableBuiltInRolesFeature() {
return features().contains(QUERYABLE_BUILT_IN_ROLES_NODE_FEATURE);
}

}

private static Set<TestNodeInfo> collectNodeInfos(RestClient adminClient) throws IOException {
final Request request = new Request("GET", "_cluster/state");
request.addParameter("filter_path", "nodes_features");

final Response response = adminClient.performRequest(request);

Map<String, Set<String>> nodeFeatures = null;
var responseData = responseAsMap(response);
if (responseData.get("nodes_features") instanceof List<?> nodesFeatures) {
nodeFeatures = nodesFeatures.stream()
.map(Map.class::cast)
.collect(Collectors.toUnmodifiableMap(nodeFeatureMap -> nodeFeatureMap.get("node_id").toString(), nodeFeatureMap -> {
@SuppressWarnings("unchecked")
var features = (List<String>) nodeFeatureMap.get("features");
return new HashSet<>(features);
}));
}

Map<String, String> nodeVersions = nodesVersions();
assertThat(nodeVersions, is(notNullValue()));
// old cluster may not support node features, so we can treat it as if no features are supported
if (nodeFeatures == null) {
Set<TestNodeInfo> nodes = new HashSet<>(nodeVersions.size());
for (String nodeId : nodeVersions.keySet()) {
nodes.add(new TestNodeInfo(nodeId, nodeVersions.get(nodeId), Set.of()));
}
return nodes;
} else {
assertThat(nodeVersions.keySet(), containsInAnyOrder(nodeFeatures.keySet().toArray()));
Set<TestNodeInfo> nodes = new HashSet<>(nodeVersions.size());
for (String nodeId : nodeVersions.keySet()) {
nodes.add(new TestNodeInfo(nodeId, nodeVersions.get(nodeId), nodeFeatures.get(nodeId)));
}
return nodes;
}
}

private static void waitForNodes(int numberOfNodes) throws IOException {
final Request request = new Request(HttpGet.METHOD_NAME, "/_cluster/health");
request.addParameter("wait_for_nodes", String.valueOf(numberOfNodes));
final Response response = client().performRequest(request);
assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
}

@SuppressWarnings("unchecked")
private static Map<String, String> nodesVersions() throws IOException {
final Response response = client().performRequest(new Request(HttpGet.METHOD_NAME, "_nodes/_all"));
assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
final Map<String, Object> nodes = (Map<String, Object>) extractValue(responseAsMap(response), "nodes");
assertNotNull("Nodes info is null", nodes);
final Map<String, String> nodesVersions = Maps.newMapWithExpectedSize(nodes.size());
for (Map.Entry<String, Object> node : nodes.entrySet()) {
nodesVersions.put(node.getKey(), (String) extractValue((Map<?, ?>) node.getValue(), "version"));
}
return nodesVersions;
}

private void assertBuiltInRolesIndexed(Set<String> expectedBuiltInRoles) throws IOException {
final Map<String, String> builtInRoles = readSecurityIndexBuiltInRolesMetadata();
assertThat(builtInRoles, is(notNullValue()));
Expand Down
15 changes: 4 additions & 11 deletions ...lling-upgrade/src/test/java/org/elasticsearch/upgrades/RolesBackwardsCompatibilityIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,22 +303,15 @@ private static String randomRoleDescriptorSerialized(boolean includeDescription,
}
}

private boolean nodeSupportTransportVersion(Map<String, Object> nodeDetails, TransportVersion transportVersion) {
String nodeVersionString = (String) nodeDetails.get("version");
TransportVersion nodeTransportVersion = getTransportVersionWithFallback(
nodeVersionString,
nodeDetails.get("transport_version"),
() -> TransportVersion.zero()
);

if (nodeTransportVersion.equals(TransportVersion.zero())) {
private boolean nodeSupportTransportVersion(TestNodeInfo testNodeInfo, TransportVersion transportVersion) {
if (testNodeInfo.transportVersion().equals(TransportVersion.zero())) {
// In cases where we were not able to find a TransportVersion, a pre-8.8.0 node answered about a newer (upgraded) node.
// In that case, the node will be current (upgraded), and remote indices are supported for sure.
var nodeIsCurrent = nodeVersionString.equals(Build.current().version());
var nodeIsCurrent = testNodeInfo.version().equals(Build.current().version());
assertTrue(nodeIsCurrent);
return true;
}
return nodeTransportVersion.onOrAfter(transportVersion);
return testNodeInfo.transportVersion().onOrAfter(transportVersion);
}

private static RoleDescriptor randomRoleDescriptor(boolean includeDescription, boolean includeManageRoles) {
Expand Down