Skip to content

Commit d9b8249

Browse files
tlrxtlrx
committed
Merge branch 'master' into feature/searchable-snapshots
2 parents 0b9cc85 + 0a34b71 commit d9b8249

File tree

150 files changed

+3808
-1953
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

150 files changed

+3808
-1953
lines changed

‎.ci/bwcVersions‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,5 @@ BWC_VERSION:
1818
- "7.6.1"
1919
- "7.6.2"
2020
- "7.7.0"
21+
- "7.8.0"
2122
- "8.0.0"

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,11 @@ import org.gradle.api.InvalidUserDataException
3232
import org.gradle.api.Plugin
3333
import org.gradle.api.Project
3434
import org.gradle.api.artifacts.Configuration
35-
import org.gradle.api.plugins.ExtraPropertiesExtension
3635
import org.gradle.api.plugins.JavaBasePlugin
3736
import org.gradle.api.plugins.JavaPlugin
3837
import org.gradle.api.plugins.JavaPluginExtension
3938
import org.gradle.api.tasks.SourceSet
4039
import org.gradle.api.tasks.SourceSetContainer
41-
import org.gradle.api.tasks.compile.JavaCompile
4240
import org.gradle.api.tasks.testing.Test
4341
import org.gradle.plugins.ide.eclipse.model.EclipseModel
4442
import org.gradle.plugins.ide.idea.model.IdeaModel
@@ -67,8 +65,8 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
6765
BuildPlugin.configureTestTasks(project)
6866
BuildPlugin.configureInputNormalization(project)
6967
BuildPlugin.configureFips140(project)
68+
BuildPlugin.configureCompile(project)
7069

71-
ExtraPropertiesExtension ext = project.extensions.getByType(ExtraPropertiesExtension)
7270
project.extensions.getByType(JavaPluginExtension).sourceCompatibility = BuildParams.minimumRuntimeVersion
7371
project.extensions.getByType(JavaPluginExtension).targetCompatibility = BuildParams.minimumRuntimeVersion
7472

@@ -96,13 +94,6 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
9694
idea.module.scopes.put('TEST', [plus: [project.configurations.getByName(JavaPlugin.TEST_RUNTIME_CLASSPATH_CONFIGURATION_NAME)]] as Map<String, Collection<Configuration>>)
9795

9896
PrecommitTasks.create(project, false)
99-
project.tasks.getByName('check').dependsOn(project.tasks.getByName('precommit'))
100-
101-
project.tasks.withType(JavaCompile) { JavaCompile task ->
102-
// This will be the default in Gradle 5.0
103-
if (task.options.compilerArgs.contains("-processor") == false) {
104-
task.options.compilerArgs << '-proc:none'
105-
}
106-
}
97+
project.tasks.named('check').configure { it.dependsOn(project.tasks.named('precommit')) }
10798
}
10899
}

‎buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,10 +108,12 @@ public void apply(Project project) {
108108
TaskProvider<Copy> copyUpgradeTask = configureCopyUpgradeTask(project, upgradeVersion, upgradeDir);
109109
TaskProvider<Copy> copyPluginsTask = configureCopyPluginsTask(project, pluginsDir);
110110

111+
Map<ElasticsearchDistribution.Type, TaskProvider<?>> lifecyleTasks = lifecyleTasks(project, "distructiveDistroTest");
111112
TaskProvider<Task> destructiveDistroTest = project.getTasks().register("destructiveDistroTest");
112113
for (ElasticsearchDistribution distribution : distributions) {
113114
TaskProvider<?> destructiveTask = configureDistroTest(project, distribution, dockerSupport);
114115
destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask));
116+
lifecyleTasks.get(distribution.getType()).configure(t -> t.dependsOn(destructiveTask));
115117
}
116118
Map<String, TaskProvider<?>> batsTests = new HashMap<>();
117119
configureBatsTest(project, "plugins", distributionsDir, copyDistributionsTask, copyPluginsTask).configure(
@@ -127,6 +129,7 @@ public void apply(Project project) {
127129
List<Object> vmDependencies = new ArrayList<>(configureVM(vmProject));
128130
vmDependencies.add(project.getConfigurations().getByName("testRuntimeClasspath"));
129131

132+
Map<ElasticsearchDistribution.Type, TaskProvider<?>> vmLifecyleTasks = lifecyleTasks(vmProject, "distroTest");
130133
TaskProvider<Task> distroTest = vmProject.getTasks().register("distroTest");
131134
for (ElasticsearchDistribution distribution : distributions) {
132135
String destructiveTaskName = destructiveDistroTestTaskName(distribution);
@@ -141,6 +144,7 @@ public void apply(Project project) {
141144
);
142145
vmTask.configure(t -> t.dependsOn(distribution));
143146

147+
vmLifecyleTasks.get(distribution.getType()).configure(t -> t.dependsOn(vmTask));
144148
distroTest.configure(t -> {
145149
// Only VM sub-projects that are specifically opted-in to testing Docker should
146150
// have the Docker task added as a dependency. Although we control whether Docker
@@ -151,6 +155,7 @@ public void apply(Project project) {
151155
//
152156
// The shouldTestDocker property could be null, hence we use Boolean.TRUE.equals()
153157
boolean shouldExecute = distribution.getType() != Type.DOCKER
158+
154159
|| Boolean.TRUE.equals(vmProject.findProperty("shouldTestDocker"));
155160

156161
if (shouldExecute) {
@@ -170,6 +175,17 @@ public void apply(Project project) {
170175
});
171176
}
172177

178+
private static Map<ElasticsearchDistribution.Type, TaskProvider<?>> lifecyleTasks(Project project, String taskPrefix) {
179+
Map<ElasticsearchDistribution.Type, TaskProvider<?>> lifecyleTasks = new HashMap<>();
180+
181+
lifecyleTasks.put(Type.DOCKER, project.getTasks().register(taskPrefix + ".docker"));
182+
lifecyleTasks.put(Type.ARCHIVE, project.getTasks().register(taskPrefix + ".archives"));
183+
lifecyleTasks.put(Type.DEB, project.getTasks().register(taskPrefix + ".packages"));
184+
lifecyleTasks.put(Type.RPM, lifecyleTasks.get(Type.DEB));
185+
186+
return lifecyleTasks;
187+
}
188+
173189
private static Jdk createJdk(
174190
NamedDomainObjectContainer<Jdk> jdksContainer,
175191
String name,

‎buildSrc/src/main/resources/minimumGradleVersion‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
6.2.2
1+
6.3

‎client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchRequestConverters.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -47,14 +47,14 @@ static Request submitAsyncSearch(SubmitAsyncSearchRequest asyncSearchRequest) th
4747
request.setEntity(RequestConverters.createEntity(asyncSearchRequest.getSearchSource(), REQUEST_BODY_CONTENT_TYPE));
4848
}
4949
// set async search submit specific parameters
50-
if (asyncSearchRequest.isCleanOnCompletion() != null) {
51-
params.putParam("clean_on_completion", asyncSearchRequest.isCleanOnCompletion().toString());
50+
if (asyncSearchRequest.isKeepOnCompletion() != null) {
51+
params.putParam("keep_on_completion", asyncSearchRequest.isKeepOnCompletion().toString());
5252
}
5353
if (asyncSearchRequest.getKeepAlive() != null) {
5454
params.putParam("keep_alive", asyncSearchRequest.getKeepAlive().getStringRep());
5555
}
56-
if (asyncSearchRequest.getWaitForCompletion() != null) {
57-
params.putParam("wait_for_completion", asyncSearchRequest.getWaitForCompletion().getStringRep());
56+
if (asyncSearchRequest.getWaitForCompletionTimeout() != null) {
57+
params.putParam("wait_for_completion_timeout", asyncSearchRequest.getWaitForCompletionTimeout().getStringRep());
5858
}
5959
request.addParameters(params.asMap());
6060
return request;
@@ -76,7 +76,7 @@ static void addSearchRequestParams(Params params, SubmitAsyncSearchRequest reque
7676
params.withBatchedReduceSize(request.getBatchedReduceSize());
7777
}
7878

79-
static Request getAsyncSearch(GetAsyncSearchRequest asyncSearchRequest) throws IOException {
79+
static Request getAsyncSearch(GetAsyncSearchRequest asyncSearchRequest) {
8080
String endpoint = new RequestConverters.EndpointBuilder()
8181
.addPathPartAsIs("_async_search")
8282
.addPathPart(asyncSearchRequest.getId())
@@ -87,13 +87,13 @@ static Request getAsyncSearch(GetAsyncSearchRequest asyncSearchRequest) throws I
8787
params.putParam("keep_alive", asyncSearchRequest.getKeepAlive().getStringRep());
8888
}
8989
if (asyncSearchRequest.getWaitForCompletion() != null) {
90-
params.putParam("wait_for_completion", asyncSearchRequest.getWaitForCompletion().getStringRep());
90+
params.putParam("wait_for_completion_timeout", asyncSearchRequest.getWaitForCompletion().getStringRep());
9191
}
9292
request.addParameters(params.asMap());
9393
return request;
9494
}
9595

96-
static Request deleteAsyncSearch(DeleteAsyncSearchRequest deleteAsyncSearchRequest) throws IOException {
96+
static Request deleteAsyncSearch(DeleteAsyncSearchRequest deleteAsyncSearchRequest) {
9797
String endpoint = new RequestConverters.EndpointBuilder()
9898
.addPathPartAsIs("_async_search")
9999
.addPathPart(deleteAsyncSearchRequest.getId())

‎client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/SubmitAsyncSearchRequest.java‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@ public class SubmitAsyncSearchRequest implements Validatable {
4343

4444
public static long MIN_KEEP_ALIVE = TimeValue.timeValueMinutes(1).millis();
4545

46-
private TimeValue waitForCompletion;
47-
private Boolean cleanOnCompletion;
46+
private TimeValue waitForCompletionTimeout;
47+
private Boolean keepOnCompletion;
4848
private TimeValue keepAlive;
4949
private final SearchRequest searchRequest;
5050

@@ -70,29 +70,29 @@ public String[] getIndices() {
7070
/**
7171
* Get the minimum time that the request should wait before returning a partial result (defaults to 1 second).
7272
*/
73-
public TimeValue getWaitForCompletion() {
74-
return waitForCompletion;
73+
public TimeValue getWaitForCompletionTimeout() {
74+
return waitForCompletionTimeout;
7575
}
7676

7777
/**
7878
* Sets the minimum time that the request should wait before returning a partial result (defaults to 1 second).
7979
*/
80-
public void setWaitForCompletion(TimeValue waitForCompletion) {
81-
this.waitForCompletion = waitForCompletion;
80+
public void setWaitForCompletionTimeout(TimeValue waitForCompletionTimeout) {
81+
this.waitForCompletionTimeout = waitForCompletionTimeout;
8282
}
8383

8484
/**
8585
* Returns whether the resource resource should be removed on completion or failure (defaults to true).
8686
*/
87-
public Boolean isCleanOnCompletion() {
88-
return cleanOnCompletion;
87+
public Boolean isKeepOnCompletion() {
88+
return keepOnCompletion;
8989
}
9090

9191
/**
9292
* Determines if the resource should be removed on completion or failure (defaults to true).
9393
*/
94-
public void setCleanOnCompletion(boolean cleanOnCompletion) {
95-
this.cleanOnCompletion = cleanOnCompletion;
94+
public void setKeepOnCompletion(boolean keepOnCompletion) {
95+
this.keepOnCompletion = keepOnCompletion;
9696
}
9797

9898
/**
@@ -273,12 +273,12 @@ public boolean equals(Object o) {
273273
SubmitAsyncSearchRequest request = (SubmitAsyncSearchRequest) o;
274274
return Objects.equals(searchRequest, request.searchRequest)
275275
&& Objects.equals(getKeepAlive(), request.getKeepAlive())
276-
&& Objects.equals(getWaitForCompletion(), request.getWaitForCompletion())
277-
&& Objects.equals(isCleanOnCompletion(), request.isCleanOnCompletion());
276+
&& Objects.equals(getWaitForCompletionTimeout(), request.getWaitForCompletionTimeout())
277+
&& Objects.equals(isKeepOnCompletion(), request.isKeepOnCompletion());
278278
}
279279

280280
@Override
281281
public int hashCode() {
282-
return Objects.hash(searchRequest, getKeepAlive(), getWaitForCompletion(), isCleanOnCompletion());
282+
return Objects.hash(searchRequest, getKeepAlive(), getWaitForCompletionTimeout(), isKeepOnCompletion());
283283
}
284284
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/AsyncSearchRequestConvertersTests.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -56,19 +56,19 @@ public void testSubmitAsyncSearch() throws Exception {
5656
setRandomIndicesOptions(submitRequest::setIndicesOptions, submitRequest::getIndicesOptions, expectedParams);
5757

5858
if (randomBoolean()) {
59-
boolean cleanOnCompletion = randomBoolean();
60-
submitRequest.setCleanOnCompletion(cleanOnCompletion);
61-
expectedParams.put("clean_on_completion", Boolean.toString(cleanOnCompletion));
59+
boolean keepOnCompletion = randomBoolean();
60+
submitRequest.setKeepOnCompletion(keepOnCompletion);
61+
expectedParams.put("keep_on_completion", Boolean.toString(keepOnCompletion));
6262
}
6363
if (randomBoolean()) {
6464
TimeValue keepAlive = TimeValue.parseTimeValue(randomTimeValue(), "test");
6565
submitRequest.setKeepAlive(keepAlive);
6666
expectedParams.put("keep_alive", keepAlive.getStringRep());
6767
}
6868
if (randomBoolean()) {
69-
TimeValue waitForCompletion = TimeValue.parseTimeValue(randomTimeValue(), "test");
70-
submitRequest.setWaitForCompletion(waitForCompletion);
71-
expectedParams.put("wait_for_completion", waitForCompletion.getStringRep());
69+
TimeValue waitForCompletionTimeout = TimeValue.parseTimeValue(randomTimeValue(), "test");
70+
submitRequest.setWaitForCompletionTimeout(waitForCompletionTimeout);
71+
expectedParams.put("wait_for_completion_timeout", waitForCompletionTimeout.getStringRep());
7272
}
7373

7474
Request request = AsyncSearchRequestConverters.submitAsyncSearch(submitRequest);
@@ -128,7 +128,7 @@ public void testGetAsyncSearch() throws Exception {
128128
if (randomBoolean()) {
129129
TimeValue waitForCompletion = TimeValue.parseTimeValue(randomTimeValue(), "test");
130130
submitRequest.setWaitForCompletion(waitForCompletion);
131-
expectedParams.put("wait_for_completion", waitForCompletion.getStringRep());
131+
expectedParams.put("wait_for_completion_timeout", waitForCompletion.getStringRep());
132132
}
133133

134134
Request request = AsyncSearchRequestConverters.getAsyncSearch(submitRequest);

‎client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ public void testCleanupRepository() throws IOException {
146146
assertThat(response.result().blobs(), equalTo(0L));
147147
}
148148

149-
public void testCreateSnapshot() throws IOException {
149+
public void testCreateSnapshot() throws Exception {
150150
String repository = "test_repository";
151151
assertTrue(createTestRepository(repository, FsRepository.TYPE, "{\"location\": \".\"}").isAcknowledged());
152152

@@ -163,12 +163,23 @@ public void testCreateSnapshot() throws IOException {
163163
CreateSnapshotResponse response = createTestSnapshot(request);
164164
assertEquals(waitForCompletion ? RestStatus.OK : RestStatus.ACCEPTED, response.status());
165165
if (waitForCompletion == false) {
166-
// If we don't wait for the snapshot to complete we have to cancel it to not leak the snapshot task
167-
AcknowledgedResponse deleteResponse = execute(
168-
new DeleteSnapshotRequest(repository, snapshot),
169-
highLevelClient().snapshot()::delete, highLevelClient().snapshot()::deleteAsync
170-
);
171-
assertTrue(deleteResponse.isAcknowledged());
166+
// busy assert on the delete because a known race condition could cause the delete request to not see
167+
// the snapshot if delete and snapshot finalization happen at the same time
168+
// See https://github.com/elastic/elasticsearch/issues/53509#issuecomment-603899620 for details
169+
// TODO: Remove busy assert in 7.x+ once this race is fixed
170+
assertBusy(() -> {
171+
// If we don't wait for the snapshot to complete we have to cancel it to not leak the snapshot task
172+
AcknowledgedResponse deleteResponse;
173+
try {
174+
deleteResponse = execute(
175+
new DeleteSnapshotRequest(repository, snapshot),
176+
highLevelClient().snapshot()::delete, highLevelClient().snapshot()::deleteAsync
177+
);
178+
} catch (Exception e) {
179+
throw new AssertionError(e);
180+
}
181+
assertTrue(deleteResponse.isAcknowledged());
182+
});
172183
}
173184
}
174185

‎client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ public void testAsyncSearch() throws IOException {
3636

3737
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery());
3838
SubmitAsyncSearchRequest submitRequest = new SubmitAsyncSearchRequest(sourceBuilder, index);
39-
submitRequest.setCleanOnCompletion(false);
39+
submitRequest.setKeepOnCompletion(true);
4040
AsyncSearchResponse submitResponse = highLevelClient().asyncSearch().submit(submitRequest, RequestOptions.DEFAULT);
4141
assertNotNull(submitResponse.getId());
4242
assertFalse(submitResponse.isPartial());

‎distribution/tools/keystore-cli/src/main/java/org/elasticsearch/common/settings/AddFileKeyStoreCommand.java‎

Lines changed: 28 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,6 @@
1919

2020
package org.elasticsearch.common.settings;
2121

22-
import java.nio.file.Files;
23-
import java.nio.file.Path;
24-
import java.util.Arrays;
25-
import java.util.List;
26-
2722
import joptsimple.OptionSet;
2823
import joptsimple.OptionSpec;
2924
import org.elasticsearch.cli.ExitCodes;
@@ -33,6 +28,11 @@
3328
import org.elasticsearch.common.io.PathUtils;
3429
import org.elasticsearch.env.Environment;
3530

31+
import java.nio.file.Files;
32+
import java.nio.file.Path;
33+
import java.util.Arrays;
34+
import java.util.List;
35+
3636
/**
3737
* A subcommand for the keystore cli which adds a file setting.
3838
*/
@@ -49,43 +49,45 @@ class AddFileKeyStoreCommand extends BaseKeyStoreCommand {
4949
// jopt simple has issue with multiple non options, so we just get one set of them here
5050
// and convert to File when necessary
5151
// see https://github.com/jopt-simple/jopt-simple/issues/103
52-
this.arguments = parser.nonOptions("setting [filepath]");
52+
this.arguments = parser.nonOptions("(setting path)+");
5353
}
5454

5555
@Override
5656
protected void executeCommand(Terminal terminal, OptionSet options, Environment env) throws Exception {
57-
List<String> argumentValues = arguments.values(options);
57+
final List<String> argumentValues = arguments.values(options);
5858
if (argumentValues.size() == 0) {
5959
throw new UserException(ExitCodes.USAGE, "Missing setting name");
6060
}
61-
String setting = argumentValues.get(0);
61+
if (argumentValues.size() % 2 != 0) {
62+
throw new UserException(ExitCodes.USAGE, "settings and filenames must come in pairs");
63+
}
64+
6265
final KeyStoreWrapper keyStore = getKeyStore();
63-
if (keyStore.getSettingNames().contains(setting) && options.has(forceOption) == false) {
64-
if (terminal.promptYesNo("Setting " + setting + " already exists. Overwrite?", false) == false) {
65-
terminal.println("Exiting without modifying keystore.");
66-
return;
66+
67+
for (int i = 0; i < argumentValues.size(); i += 2) {
68+
final String setting = argumentValues.get(i);
69+
70+
if (keyStore.getSettingNames().contains(setting) && options.has(forceOption) == false) {
71+
if (terminal.promptYesNo("Setting " + setting + " already exists. Overwrite?", false) == false) {
72+
terminal.println("Exiting without modifying keystore.");
73+
return;
74+
}
6775
}
68-
}
6976

70-
if (argumentValues.size() == 1) {
71-
throw new UserException(ExitCodes.USAGE, "Missing file name");
72-
}
73-
Path file = getPath(argumentValues.get(1));
74-
if (Files.exists(file) == false) {
75-
throw new UserException(ExitCodes.IO_ERROR, "File [" + file.toString() + "] does not exist");
76-
}
77-
if (argumentValues.size() > 2) {
78-
throw new UserException(
79-
ExitCodes.USAGE,
80-
"Unrecognized extra arguments [" + String.join(", ", argumentValues.subList(2, argumentValues.size())) + "] after filepath"
81-
);
77+
final Path file = getPath(argumentValues.get(i + 1));
78+
if (Files.exists(file) == false) {
79+
throw new UserException(ExitCodes.IO_ERROR, "File [" + file.toString() + "] does not exist");
80+
}
81+
82+
keyStore.setFile(setting, Files.readAllBytes(file));
8283
}
83-
keyStore.setFile(setting, Files.readAllBytes(file));
84+
8485
keyStore.save(env.configFile(), getKeyStorePassword().getChars());
8586
}
8687

8788
@SuppressForbidden(reason = "file arg for cli")
8889
private Path getPath(String file) {
8990
return PathUtils.get(file);
9091
}
92+
9193
}

0 commit comments

Comments
 (0)