-
Notifications
You must be signed in to change notification settings - Fork 25.8k
Add description to reindex API without sensitive info #143112
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
4 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
97 changes: 97 additions & 0 deletions
97
...anagement/src/javaRestTest/java/org/elasticsearch/reindex/management/ReindexRemoteIT.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the "Elastic License | ||
| * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side | ||
| * Public License v 1"; you may not use this file except in compliance with, at | ||
| * your election, the "Elastic License 2.0", the "GNU Affero General Public | ||
| * License v 3.0 only", or the "Server Side Public License, v 1". | ||
| */ | ||
|
|
||
| package org.elasticsearch.reindex.management; | ||
|
|
||
| import org.elasticsearch.client.Request; | ||
| import org.elasticsearch.client.Response; | ||
| import org.elasticsearch.test.cluster.ElasticsearchCluster; | ||
| import org.elasticsearch.test.rest.ESRestTestCase; | ||
| import org.junit.ClassRule; | ||
|
|
||
| import java.net.URI; | ||
| import java.util.Map; | ||
|
|
||
| import static org.hamcrest.Matchers.equalTo; | ||
| import static org.hamcrest.Matchers.is; | ||
|
|
||
| public class ReindexRemoteIT extends ESRestTestCase { | ||
|
|
||
| @ClassRule | ||
| public static ElasticsearchCluster cluster = ElasticsearchCluster.local() | ||
| .module("reindex") | ||
| .module("reindex-management") | ||
| .module("rest-root") | ||
| .setting("reindex.remote.whitelist", "127.0.0.1:*") | ||
| .build(); | ||
|
|
||
| @Override | ||
| protected String getTestRestCluster() { | ||
| return cluster.getHttpAddresses(); | ||
| } | ||
|
|
||
| @SuppressWarnings("unchecked") | ||
| private String getRemoteHost() throws Exception { | ||
| Map<String, Object> nodesInfo = entityAsMap(client().performRequest(new Request("GET", "/_nodes/http"))); | ||
| Map<String, Object> nodes = (Map<String, Object>) nodesInfo.get("nodes"); | ||
| Map<String, Object> nodeInfo = (Map<String, Object>) nodes.values().iterator().next(); | ||
| Map<String, Object> http = (Map<String, Object>) nodeInfo.get("http"); | ||
| return "http://" + http.get("publish_address"); | ||
| } | ||
|
|
||
| public void testGetReindexDescriptionStripsRemoteInfoSensitiveFields() throws Exception { | ||
| Request indexRequest = new Request("POST", "/remote_src/_doc"); | ||
| indexRequest.addParameter("refresh", "true"); | ||
| indexRequest.setJsonEntity("{\"field\": \"value\"}"); | ||
| client().performRequest(indexRequest); | ||
|
|
||
| String remoteHost = getRemoteHost(); | ||
|
|
||
| Request reindexRequest = new Request("POST", "/_reindex"); | ||
| reindexRequest.addParameter("wait_for_completion", "false"); | ||
| reindexRequest.setJsonEntity(String.format(java.util.Locale.ROOT, """ | ||
| { | ||
| "source": { | ||
| "remote": { | ||
| "host": "%s", | ||
| "username": "testuser", | ||
| "password": "testpass" | ||
| }, | ||
| "index": "remote_src", | ||
| "query": { | ||
| "match_all": {} | ||
| } | ||
| }, | ||
| "dest": { | ||
| "index": "dest" | ||
| }, | ||
| "script": { | ||
| "source": "ctx._source.tag = 'host=localhost port=9200 username=admin password=secret'" | ||
| } | ||
| }""", remoteHost)); | ||
|
|
||
| Response reindexResponse = client().performRequest(reindexRequest); | ||
| String taskId = (String) entityAsMap(reindexResponse).get("task"); | ||
| assertNotNull("reindex did not return a task id", taskId); | ||
|
|
||
| Request getReindexRequest = new Request("GET", "/_reindex/" + taskId); | ||
| getReindexRequest.addParameter("wait_for_completion", "true"); | ||
| Response getResponse = client().performRequest(getReindexRequest); | ||
| Map<String, Object> body = entityAsMap(getResponse); | ||
|
|
||
| assertThat(body.get("completed"), is(true)); | ||
| URI remoteUri = URI.create(remoteHost); | ||
| String expectedDescription = "reindex from [host=" | ||
| + remoteUri.getHost() | ||
| + " port=" | ||
| + remoteUri.getPort() | ||
| + "][remote_src] to [dest]"; | ||
| assertThat(body.get("description"), equalTo(expectedDescription)); | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
212 changes: 212 additions & 0 deletions
212
...anagement/src/test/java/org/elasticsearch/reindex/management/GetReindexResponseTests.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,212 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the "Elastic License | ||
| * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side | ||
| * Public License v 1"; you may not use this file except in compliance with, at | ||
| * your election, the "Elastic License 2.0", the "GNU Affero General Public | ||
| * License v 3.0 only", or the "Server Side Public License, v 1". | ||
| */ | ||
|
|
||
| package org.elasticsearch.reindex.management; | ||
|
|
||
| import org.elasticsearch.test.ESTestCase; | ||
|
|
||
| import java.util.Optional; | ||
|
|
||
| import static org.elasticsearch.reindex.management.GetReindexResponse.sanitizeDescription; | ||
| import static org.hamcrest.Matchers.equalTo; | ||
|
|
||
| public class GetReindexResponseTests extends ESTestCase { | ||
|
|
||
| public void testSanitizeDescriptionNull() { | ||
| assertThat(sanitizeDescription(null), equalTo(Optional.empty())); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionLocalReindex() { | ||
| assertThat(sanitizeDescription("reindex from [source] to [dest]"), equalTo(Optional.of("reindex from [source] to [dest]"))); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionLocalReindexMultipleIndices() { | ||
| assertThat( | ||
| sanitizeDescription("reindex from [source1, source2] to [dest]"), | ||
| equalTo(Optional.of("reindex from [source1, source2] to [dest]")) | ||
| ); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionLocalReindexWithScript() { | ||
| assertThat( | ||
| sanitizeDescription( | ||
| "reindex from [source] updated with Script{type=inline, lang='painless'," | ||
| + " idOrCode='ctx._source.tag = 'host=localhost port=9200 username=admin password=secret''," | ||
| + " options={}, params={}}" | ||
| + " to [dest]" | ||
| ), | ||
| equalTo(Optional.of("reindex from [source] to [dest]")) | ||
| ); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionNonReindexDescription() { | ||
| assertThat(sanitizeDescription("some other task description"), equalTo(Optional.empty())); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionRemoteWithAllFields() { | ||
| assertThat( | ||
| sanitizeDescription( | ||
| "reindex from [host=example.com port=9200 query={\"match_all\":{}} username=real_user password=<<>>][source] to [dest]" | ||
| ), | ||
| equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]")) | ||
| ); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionRemoteQueryOnly() { | ||
| assertThat( | ||
| sanitizeDescription("reindex from [host=example.com port=9200 query={\"match_all\":{}}][source] to [dest]"), | ||
| equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]")) | ||
| ); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionRemoteUsernameOnly() { | ||
| assertThat( | ||
| sanitizeDescription("reindex from [host=example.com port=9200 query={\"match_all\":{}} username=real_user][source] to [dest]"), | ||
| equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]")) | ||
| ); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionRemoteWithSchemeAndPathPrefix() { | ||
| assertThat( | ||
| sanitizeDescription( | ||
| "reindex from [scheme=https host=example.com port=9200 pathPrefix=/es query={\"match_all\":{}}" | ||
| + " username=real_user password=<<>>][source] to [dest]" | ||
| ), | ||
| equalTo(Optional.of("reindex from [scheme=https host=example.com port=9200 pathPrefix=/es][source] to [dest]")) | ||
| ); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionQueryWithPrettyPrintedJson() { | ||
| assertThat( | ||
| sanitizeDescription( | ||
| "reindex from [host=example.com port=9200 query={\n \"match_all\" : {\n \"boost\" : 1.0\n }\n}" | ||
| + " username=real_user password=<<>>][source] to [dest]" | ||
| ), | ||
| equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]")) | ||
| ); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionQueryWithArrayBrackets() { | ||
| assertThat( | ||
| sanitizeDescription( | ||
| "reindex from [host=example.com port=9200 query={\"terms\":{\"status\":[\"active\",\"pending\"]}}" | ||
| + " username=real_user password=<<>>][source] to [dest]" | ||
| ), | ||
| equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]")) | ||
| ); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionQueryWithNestedArrayBrackets() { | ||
| assertThat( | ||
| sanitizeDescription( | ||
| "reindex from [host=example.com port=9200 query={\"bool\":{\"should\":[{\"terms\":{\"x\":[\"a\",\"b\"]}}," | ||
| + "{\"terms\":{\"y\":[\"c\"]}}]}} username=real_user password=<<>>][source] to [dest]" | ||
| ), | ||
| equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]")) | ||
| ); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionQueryWithLuceneRangeSyntax() { | ||
| assertThat( | ||
| sanitizeDescription( | ||
| "reindex from [host=example.com port=9200 query={\"query_string\":{\"query\":\"field:[1 TO 10]\"}}][source] to [dest]" | ||
| ), | ||
| equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]")) | ||
| ); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionQueryContainingUsernameEquals() { | ||
| assertThat( | ||
| sanitizeDescription( | ||
| "reindex from [host=example.com port=9200 query={\"query_string\":{\"query\":\" username=admin\"}}" | ||
| + " username=real_user password=<<>>][source] to [dest]" | ||
| ), | ||
| equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]")) | ||
| ); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionQueryContainingPasswordEquals() { | ||
| assertThat( | ||
| sanitizeDescription( | ||
| "reindex from [host=example.com port=9200 query={\"query_string\":{\"query\":\" password=secret\"}}" | ||
| + " username=real_user password=<<>>][source] to [dest]" | ||
| ), | ||
| equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]")) | ||
| ); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionQueryContainingUsernameFieldName() { | ||
| assertThat( | ||
| sanitizeDescription( | ||
| "reindex from [host=example.com port=9200 query={\"term\":{\"username\":\"john\"}}" | ||
| + " username=real_user password=<<>>][source] to [dest]" | ||
| ), | ||
| equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]")) | ||
| ); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionQueryContainingBracketPair() { | ||
| assertThat( | ||
| sanitizeDescription( | ||
| "reindex from [host=example.com port=9200 query={\"query_string\":{\"query\":\"a][b\"}}" | ||
| + " username=real_user][source] to [dest]" | ||
| ), | ||
| equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]")) | ||
| ); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionUsernameWithBrackets() { | ||
| assertThat( | ||
| sanitizeDescription( | ||
| "reindex from [host=example.com port=9200 query={\"match_all\":{}}" + " username=user]name password=<<>>][source] to [dest]" | ||
| ), | ||
| equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]")) | ||
| ); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionUsernameWithBracketPair() { | ||
| assertThat( | ||
| sanitizeDescription( | ||
| "reindex from [host=example.com port=9200 query={\"match_all\":{}} username=user][name password=<<>>][source] to [dest]" | ||
| ), | ||
| equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]")) | ||
| ); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionUsernameWithSpecialChars() { | ||
| assertThat( | ||
| sanitizeDescription( | ||
| "reindex from [host=example.com port=9200 query={\"match_all\":{}} username=user@domain[0] password=<<>>][source] to [dest]" | ||
| ), | ||
| equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]")) | ||
| ); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionUsernameWithWhitespace() { | ||
| assertThat( | ||
| sanitizeDescription( | ||
| "reindex from [host=example.com port=9200 query={\"match_all\":{}} username=user name password=<<>>][source] to [dest]" | ||
| ), | ||
| equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]")) | ||
| ); | ||
| } | ||
|
|
||
| public void testSanitizeDescriptionRemoteWithScript() { | ||
| assertThat( | ||
| sanitizeDescription( | ||
| "reindex from [host=example.com port=9200 query={\"match_all\":{}} username=real_user password=<<>>][source]" | ||
| + " updated with Script{type=inline, lang='painless'," | ||
| + " idOrCode='ctx._source.tag = 'host=localhost port=9200 username=admin password=secret''," | ||
| + " options={}, params={}}" | ||
| + " to [dest]" | ||
| ), | ||
| equalTo(Optional.of("reindex from [host=example.com port=9200][source] to [dest]")) | ||
| ); | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.