Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions modules/reindex-management/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
apply plugin: 'elasticsearch.internal-cluster-test'
apply plugin: 'elasticsearch.internal-yaml-rest-test'
apply plugin: 'elasticsearch.yaml-rest-compat-test'
apply plugin: 'elasticsearch.internal-java-rest-test'

esplugin {
name = 'reindex-management'
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v 3.0 only", or the "Server Side Public License, v 1".
*/

package org.elasticsearch.reindex.management;

import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.test.cluster.ElasticsearchCluster;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.junit.ClassRule;

import java.net.URI;
import java.util.Map;

import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;

public class ReindexRemoteIT extends ESRestTestCase {

@ClassRule
public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
.module("reindex")
.module("reindex-management")
.module("rest-root")
.setting("reindex.remote.whitelist", "127.0.0.1:*")
.build();

@Override
protected String getTestRestCluster() {
return cluster.getHttpAddresses();
}

@SuppressWarnings("unchecked")
private String getRemoteHost() throws Exception {
Map<String, Object> nodesInfo = entityAsMap(client().performRequest(new Request("GET", "/_nodes/http")));
Map<String, Object> nodes = (Map<String, Object>) nodesInfo.get("nodes");
Map<String, Object> nodeInfo = (Map<String, Object>) nodes.values().iterator().next();
Map<String, Object> http = (Map<String, Object>) nodeInfo.get("http");
return "http://" + http.get("publish_address");
}

public void testGetReindexDescriptionStripsRemoteInfoSensitiveFields() throws Exception {
Request indexRequest = new Request("POST", "/remote_src/_doc");
indexRequest.addParameter("refresh", "true");
indexRequest.setJsonEntity("{\"field\": \"value\"}");
client().performRequest(indexRequest);

String remoteHost = getRemoteHost();

Request reindexRequest = new Request("POST", "/_reindex");
reindexRequest.addParameter("wait_for_completion", "false");
reindexRequest.setJsonEntity(String.format(java.util.Locale.ROOT, """
{
"source": {
"remote": {
"host": "%s",
"username": "testuser",
"password": "testpass"
},
"index": "remote_src",
"query": {
"match_all": {}
}
},
"dest": {
"index": "dest"
},
"script": {
"source": "ctx._source.tag = 'host=localhost port=9200 username=admin password=secret'"
}
}""", remoteHost));

Response reindexResponse = client().performRequest(reindexRequest);
String taskId = (String) entityAsMap(reindexResponse).get("task");
assertNotNull("reindex did not return a task id", taskId);

Request getReindexRequest = new Request("GET", "/_reindex/" + taskId);
getReindexRequest.addParameter("wait_for_completion", "true");
Response getResponse = client().performRequest(getReindexRequest);
Map<String, Object> body = entityAsMap(getResponse);

assertThat(body.get("completed"), is(true));
URI remoteUri = URI.create(remoteHost);
String expectedDescription = "reindex from [host="
+ remoteUri.getHost()
+ " port="
+ remoteUri.getPort()
+ "][remote_src] to [dest]";
assertThat(body.get("description"), equalTo(expectedDescription));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,26 @@

import java.io.IOException;
import java.util.Objects;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static java.util.Objects.requireNonNull;

public class GetReindexResponse extends ActionResponse implements ToXContentObject {

private final TaskResult task;

/**
* Matches a reindex description and captures only the safe fields we want to expose:
* group(1) = optional safe remote info (scheme, host, port, pathPrefix), null for local reindex
* group(2) = source indices
* group(3) = destination index
*/
private static final Pattern DESCRIPTION_PATTERN = Pattern.compile(
"(?s)^reindex from (?:\\[((?:scheme=\\S+ )?host=\\S+ port=\\d+(?:\\s+pathPrefix=\\S+)?) .+\\])?\\[([^\\]]*)].*to \\[([^\\]]*)]$"
);

public GetReindexResponse(TaskResult task) {
this.task = requireNonNull(task, "task is required");
}
Expand Down Expand Up @@ -66,8 +79,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws

// reindex specific TaskInfo serialization
static XContentBuilder taskInfoToXContent(XContentBuilder builder, Params params, TaskInfo taskInfo) throws IOException {
// TODO: revisit if we should expose taskInfo.description, since it may contain sensitive information like ip and username
builder.field("id", taskInfo.node() + ":" + taskInfo.id());
Optional<String> description = sanitizeDescription(taskInfo.description());
if (description.isPresent()) {
builder.field("description", description.get());
}
builder.timestampFieldsFromUnixEpochMillis("start_time_in_millis", "start_time", taskInfo.startTime());
if (builder.humanReadable()) {
builder.field("running_time", TimeValue.timeValueNanos(taskInfo.runningTimeNanos()).toString());
Expand All @@ -81,6 +97,29 @@ static XContentBuilder taskInfoToXContent(XContentBuilder builder, Params params
return builder;
}

/**
* Selectively constructs a safe description by extracting only the fields we want to expose and discarding everything else.
* Returns empty if the description cannot be parsed, so we don't risk exposing sensitive data from an unrecognised format.
*/
static Optional<String> sanitizeDescription(String description) {
if (description == null) {
return Optional.empty();
}
Matcher matcher = DESCRIPTION_PATTERN.matcher(description);
if (matcher.matches()) {
String remoteInfo = matcher.group(1);
String sourceIndices = matcher.group(2);
String destIndex = matcher.group(3);
StringBuilder sb = new StringBuilder("reindex from ");
if (remoteInfo != null) {
sb.append('[').append(remoteInfo).append(']');
}
sb.append('[').append(sourceIndices).append("] to [").append(destIndex).append(']');
return Optional.of(sb.toString());
}
return Optional.empty();
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,212 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v 3.0 only", or the "Server Side Public License, v 1".
*/

package org.elasticsearch.reindex.management;

import org.elasticsearch.test.ESTestCase;

import java.util.Optional;

import static org.elasticsearch.reindex.management.GetReindexResponse.sanitizeDescription;
import static org.hamcrest.Matchers.equalTo;

public class GetReindexResponseTests extends ESTestCase {

public void testSanitizeDescriptionNull() {
assertThat(sanitizeDescription(null), equalTo(Optional.empty()));
}

public void testSanitizeDescriptionLocalReindex() {
assertThat(sanitizeDescription("reindex from [source] to [dest]"), equalTo(Optional.of("reindex from [source] to [dest]")));
}

public void testSanitizeDescriptionLocalReindexMultipleIndices() {
assertThat(
sanitizeDescription("reindex from [source1, source2] to [dest]"),
equalTo(Optional.of("reindex from [source1, source2] to [dest]"))
);
}

public void testSanitizeDescriptionLocalReindexWithScript() {
assertThat(
sanitizeDescription(
"reindex from [source] updated with Script{type=inline, lang='painless',"
+ " idOrCode='ctx._source.tag = 'host=localhost port=9200 username=admin password=secret'',"
+ " options={}, params={}}"
+ " to [dest]"
),
equalTo(Optional.of("reindex from [source] to [dest]"))
);
}

public void testSanitizeDescriptionNonReindexDescription() {
assertThat(sanitizeDescription("some other task description"), equalTo(Optional.empty()));
}

public void testSanitizeDescriptionRemoteWithAllFields() {
assertThat(
sanitizeDescription(
"reindex from [host=example.com port=9200 query={\"match_all\":{}} username=real_user password=<<>>][source] to [dest]"
),
equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]"))
);
}

public void testSanitizeDescriptionRemoteQueryOnly() {
assertThat(
sanitizeDescription("reindex from [host=example.com port=9200 query={\"match_all\":{}}][source] to [dest]"),
equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]"))
);
}

public void testSanitizeDescriptionRemoteUsernameOnly() {
assertThat(
sanitizeDescription("reindex from [host=example.com port=9200 query={\"match_all\":{}} username=real_user][source] to [dest]"),
equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]"))
);
}

public void testSanitizeDescriptionRemoteWithSchemeAndPathPrefix() {
assertThat(
sanitizeDescription(
"reindex from [scheme=https host=example.com port=9200 pathPrefix=/es query={\"match_all\":{}}"
+ " username=real_user password=<<>>][source] to [dest]"
),
equalTo(Optional.of("reindex from [scheme=https host=example.com port=9200 pathPrefix=/es][source] to [dest]"))
);
}

public void testSanitizeDescriptionQueryWithPrettyPrintedJson() {
assertThat(
sanitizeDescription(
"reindex from [host=example.com port=9200 query={\n \"match_all\" : {\n \"boost\" : 1.0\n }\n}"
+ " username=real_user password=<<>>][source] to [dest]"
),
equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]"))
);
}

public void testSanitizeDescriptionQueryWithArrayBrackets() {
assertThat(
sanitizeDescription(
"reindex from [host=example.com port=9200 query={\"terms\":{\"status\":[\"active\",\"pending\"]}}"
+ " username=real_user password=<<>>][source] to [dest]"
),
equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]"))
);
}

public void testSanitizeDescriptionQueryWithNestedArrayBrackets() {
assertThat(
sanitizeDescription(
"reindex from [host=example.com port=9200 query={\"bool\":{\"should\":[{\"terms\":{\"x\":[\"a\",\"b\"]}},"
+ "{\"terms\":{\"y\":[\"c\"]}}]}} username=real_user password=<<>>][source] to [dest]"
),
equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]"))
);
}

public void testSanitizeDescriptionQueryWithLuceneRangeSyntax() {
assertThat(
sanitizeDescription(
"reindex from [host=example.com port=9200 query={\"query_string\":{\"query\":\"field:[1 TO 10]\"}}][source] to [dest]"
),
equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]"))
);
}

public void testSanitizeDescriptionQueryContainingUsernameEquals() {
assertThat(
sanitizeDescription(
"reindex from [host=example.com port=9200 query={\"query_string\":{\"query\":\" username=admin\"}}"
+ " username=real_user password=<<>>][source] to [dest]"
),
equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]"))
);
}

public void testSanitizeDescriptionQueryContainingPasswordEquals() {
assertThat(
sanitizeDescription(
"reindex from [host=example.com port=9200 query={\"query_string\":{\"query\":\" password=secret\"}}"
+ " username=real_user password=<<>>][source] to [dest]"
),
equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]"))
);
}

public void testSanitizeDescriptionQueryContainingUsernameFieldName() {
assertThat(
sanitizeDescription(
"reindex from [host=example.com port=9200 query={\"term\":{\"username\":\"john\"}}"
+ " username=real_user password=<<>>][source] to [dest]"
),
equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]"))
);
}

public void testSanitizeDescriptionQueryContainingBracketPair() {
assertThat(
sanitizeDescription(
"reindex from [host=example.com port=9200 query={\"query_string\":{\"query\":\"a][b\"}}"
+ " username=real_user][source] to [dest]"
),
equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]"))
);
}

public void testSanitizeDescriptionUsernameWithBrackets() {
assertThat(
sanitizeDescription(
"reindex from [host=example.com port=9200 query={\"match_all\":{}}" + " username=user]name password=<<>>][source] to [dest]"
),
equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]"))
);
}

public void testSanitizeDescriptionUsernameWithBracketPair() {
assertThat(
sanitizeDescription(
"reindex from [host=example.com port=9200 query={\"match_all\":{}} username=user][name password=<<>>][source] to [dest]"
),
equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]"))
);
}

public void testSanitizeDescriptionUsernameWithSpecialChars() {
assertThat(
sanitizeDescription(
"reindex from [host=example.com port=9200 query={\"match_all\":{}} username=user@domain[0] password=<<>>][source] to [dest]"
),
equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]"))
);
}

public void testSanitizeDescriptionUsernameWithWhitespace() {
assertThat(
sanitizeDescription(
"reindex from [host=example.com port=9200 query={\"match_all\":{}} username=user name password=<<>>][source] to [dest]"
),
equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]"))
);
}

public void testSanitizeDescriptionRemoteWithScript() {
assertThat(
sanitizeDescription(
"reindex from [host=example.com port=9200 query={\"match_all\":{}} username=real_user password=<<>>][source]"
+ " updated with Script{type=inline, lang='painless',"
+ " idOrCode='ctx._source.tag = 'host=localhost port=9200 username=admin password=secret'',"
+ " options={}, params={}}"
+ " to [dest]"
),
equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]"))
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ setup:
human: true
- is_true: completed
- match: { id: $taskId }
- match: { description: "reindex from [source] to [dest]" }
- match: { cancelled: false }
- exists: start_time
- exists: start_time_in_millis
Expand Down
Loading