Merge branch 'master' into always-fetch-leases

Author: dnhatn

.ci/packer_cache.sh

@@ -16,4 +16,9 @@ while [ -h "$SCRIPT" ] ; do
 done
 
 source $(dirname "${SCRIPT}")/java-versions.properties
-JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA} ./gradlew --parallel  resolveAllDependencies composePull
+export JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA}
+# We are caching BWC versions too, need these so we can build those
+export JAVA8_HOME="${HOME}"/.java/java8
+export JAVA11_HOME="${HOME}"/.java/java11
+export JAVA12_HOME="${HOME}"/.java/java12
+./gradlew --parallel  clean pullFixture  --scan -Porg.elasticsearch.acceptScanTOS=true -s resolveAllDependencies

buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java

@@ -59,14 +59,17 @@ public void apply(Project project) {
             disableTaskByType(tasks, JarHellTask.class);
 
             Task buildFixture = project.getTasks().create("buildFixture");
+            Task pullFixture = project.getTasks().create("pullFixture");
             Task preProcessFixture = project.getTasks().create("preProcessFixture");
             buildFixture.dependsOn(preProcessFixture);
+            pullFixture.dependsOn(preProcessFixture);
             Task postProcessFixture = project.getTasks().create("postProcessFixture");
 
             if (dockerComposeSupported(project) == false) {
                 preProcessFixture.setEnabled(false);
                 postProcessFixture.setEnabled(false);
                 buildFixture.setEnabled(false);
+                pullFixture.setEnabled(false);
                 return;
             }
 
@@ -81,7 +84,9 @@ public void apply(Project project) {
             );
 
             buildFixture.dependsOn(tasks.getByName("composeUp"));
+            pullFixture.dependsOn(tasks.getByName("composePull"));
             tasks.getByName("composeUp").mustRunAfter(preProcessFixture);
+            tasks.getByName("composePull").mustRunAfter(preProcessFixture);
             postProcessFixture.dependsOn(buildFixture);
 
             configureServiceInfoForTask(

client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java

@@ -64,6 +64,7 @@
 import org.elasticsearch.client.ml.PutFilterRequest;
 import org.elasticsearch.client.ml.PutJobRequest;
 import org.elasticsearch.client.ml.RevertModelSnapshotRequest;
+import org.elasticsearch.client.ml.SetUpgradeModeRequest;
 import org.elasticsearch.client.ml.StartDatafeedRequest;
 import org.elasticsearch.client.ml.StopDatafeedRequest;
 import org.elasticsearch.client.ml.UpdateDatafeedRequest;
@@ -624,6 +625,17 @@ static Request deleteFilter(DeleteFilterRequest deleteFilterRequest) {
         return request;
     }
 
+    static Request setUpgradeMode(SetUpgradeModeRequest setUpgradeModeRequest) {
+        String endpoint = new EndpointBuilder().addPathPartAsIs("_ml", "set_upgrade_mode").build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        RequestConverters.Params params = new RequestConverters.Params(request);
+        params.putParam(SetUpgradeModeRequest.ENABLED.getPreferredName(), Boolean.toString(setUpgradeModeRequest.isEnabled()));
+        if (setUpgradeModeRequest.getTimeout() != null) {
+            params.putParam(SetUpgradeModeRequest.TIMEOUT.getPreferredName(), setUpgradeModeRequest.getTimeout().toString());
+        }
+        return request;
+    }
+
     static Request mlInfo(MlInfoRequest infoRequest) {
         String endpoint = new EndpointBuilder()
             .addPathPartAsIs("_ml", "info")

client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java

@@ -86,6 +86,7 @@
 import org.elasticsearch.client.ml.PutJobResponse;
 import org.elasticsearch.client.ml.RevertModelSnapshotRequest;
 import org.elasticsearch.client.ml.RevertModelSnapshotResponse;
+import org.elasticsearch.client.ml.SetUpgradeModeRequest;
 import org.elasticsearch.client.ml.StartDatafeedRequest;
 import org.elasticsearch.client.ml.StartDatafeedResponse;
 import org.elasticsearch.client.ml.StopDatafeedRequest;
@@ -1838,4 +1839,42 @@ public void findFileStructureAsync(FindFileStructureRequest request, RequestOpti
             listener,
             Collections.emptySet());
     }
+
+    /**
+     * Sets the ML cluster setting upgrade_mode
+     * <p>
+     * For additional info
+     * see <a href="http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-set-upgrade-mode.html">Set Upgrade Mode</a>
+     *
+     * @param request The request to set upgrade mode
+     * @param options  Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return response
+     * @throws IOException when there is a serialization issue sending the request or receiving the response
+     */
+    public AcknowledgedResponse setUpgradeMode(SetUpgradeModeRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            MLRequestConverters::setUpgradeMode,
+            options,
+            AcknowledgedResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Sets the ML cluster setting upgrade_mode asynchronously
+     * <p>
+     * For additional info
+     * see <a href="http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-set-upgrade-mode.html">Set Upgrade Mode</a>
+     *
+     * @param request The request of Machine Learning info
+     * @param options  Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     */
+    public void setUpgradeModeAsync(SetUpgradeModeRequest request, RequestOptions options, ActionListener<AcknowledgedResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            MLRequestConverters::setUpgradeMode,
+            options,
+            AcknowledgedResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/SetUpgradeModeRequest.java

@@ -0,0 +1,93 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.unit.TimeValue;
+
+import java.util.Objects;
+
+/**
+ * Sets ML into upgrade_mode
+ */
+public class SetUpgradeModeRequest extends ActionRequest {
+
+
+    public static final ParseField ENABLED = new ParseField("enabled");
+    public static final ParseField TIMEOUT = new ParseField("timeout");
+
+    private boolean enabled;
+    private TimeValue timeout;
+
+    /**
+     * Create a new request
+     *
+     * @param enabled whether to enable `upgrade_mode` or not
+     */
+    public SetUpgradeModeRequest(boolean enabled) {
+        this.enabled = enabled;
+    }
+
+    public boolean isEnabled() {
+        return enabled;
+    }
+
+    public void setEnabled(boolean enabled) {
+        this.enabled = enabled;
+    }
+
+    public TimeValue getTimeout() {
+        return timeout;
+    }
+
+    /**
+     * How long to wait for the request to be completed
+     *
+     * @param timeout default value of 30 seconds
+     */
+    public void setTimeout(TimeValue timeout) {
+        this.timeout = timeout;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(enabled, timeout);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        SetUpgradeModeRequest that = (SetUpgradeModeRequest) other;
+        return Objects.equals(enabled, that.enabled) && Objects.equals(timeout, that.timeout);
+    }
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java

@@ -61,6 +61,7 @@
 import org.elasticsearch.client.ml.PutFilterRequest;
 import org.elasticsearch.client.ml.PutJobRequest;
 import org.elasticsearch.client.ml.RevertModelSnapshotRequest;
+import org.elasticsearch.client.ml.SetUpgradeModeRequest;
 import org.elasticsearch.client.ml.StartDatafeedRequest;
 import org.elasticsearch.client.ml.StartDatafeedRequestTests;
 import org.elasticsearch.client.ml.StopDatafeedRequest;
@@ -818,6 +819,22 @@ public void testFindFileStructure() throws Exception {
         assertEquals(sample, requestEntityToString(request));
     }
 
+    public void testSetUpgradeMode() {
+        SetUpgradeModeRequest setUpgradeModeRequest = new SetUpgradeModeRequest(true);
+
+        Request request = MLRequestConverters.setUpgradeMode(setUpgradeModeRequest);
+        assertThat(request.getEndpoint(), equalTo("/_ml/set_upgrade_mode"));
+        assertThat(request.getMethod(), equalTo(HttpPost.METHOD_NAME));
+        assertThat(request.getParameters().get(SetUpgradeModeRequest.ENABLED.getPreferredName()), equalTo(Boolean.toString(true)));
+        assertThat(request.getParameters().containsKey(SetUpgradeModeRequest.TIMEOUT.getPreferredName()), is(false));
+
+        setUpgradeModeRequest.setTimeout(TimeValue.timeValueHours(1));
+        setUpgradeModeRequest.setEnabled(false);
+        request = MLRequestConverters.setUpgradeMode(setUpgradeModeRequest);
+        assertThat(request.getParameters().get(SetUpgradeModeRequest.ENABLED.getPreferredName()), equalTo(Boolean.toString(false)));
+        assertThat(request.getParameters().get(SetUpgradeModeRequest.TIMEOUT.getPreferredName()), is("1h"));
+    }
+
     private static Job createValidJob(String jobId) {
         AnalysisConfig.Builder analysisConfig = AnalysisConfig.builder(Collections.singletonList(
                 Detector.builder().setFunction("count").build()));

client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java

@@ -84,6 +84,7 @@
 import org.elasticsearch.client.ml.PutJobResponse;
 import org.elasticsearch.client.ml.RevertModelSnapshotRequest;
 import org.elasticsearch.client.ml.RevertModelSnapshotResponse;
+import org.elasticsearch.client.ml.SetUpgradeModeRequest;
 import org.elasticsearch.client.ml.StartDatafeedRequest;
 import org.elasticsearch.client.ml.StartDatafeedResponse;
 import org.elasticsearch.client.ml.StopDatafeedRequest;
@@ -1614,4 +1615,30 @@ public void testFindFileStructure() throws IOException {
         assertEquals("timestamp", structure.getTimestampField());
         assertFalse(structure.needClientTimezone());
     }
+
+    public void testEnableUpgradeMode() throws Exception {
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+
+        MlInfoResponse mlInfoResponse = machineLearningClient.getMlInfo(new MlInfoRequest(), RequestOptions.DEFAULT);
+        assertThat(mlInfoResponse.getInfo().get("upgrade_mode"), equalTo(false));
+
+        AcknowledgedResponse setUpgrademodeResponse = execute(new SetUpgradeModeRequest(true),
+            machineLearningClient::setUpgradeMode,
+            machineLearningClient::setUpgradeModeAsync);
+
+        assertThat(setUpgrademodeResponse.isAcknowledged(), is(true));
+
+
+        mlInfoResponse = machineLearningClient.getMlInfo(new MlInfoRequest(), RequestOptions.DEFAULT);
+        assertThat(mlInfoResponse.getInfo().get("upgrade_mode"), equalTo(true));
+
+        setUpgrademodeResponse = execute(new SetUpgradeModeRequest(false),
+            machineLearningClient::setUpgradeMode,
+            machineLearningClient::setUpgradeModeAsync);
+
+        assertThat(setUpgrademodeResponse.isAcknowledged(), is(true));
+
+        mlInfoResponse = machineLearningClient.getMlInfo(new MlInfoRequest(), RequestOptions.DEFAULT);
+        assertThat(mlInfoResponse.getInfo().get("upgrade_mode"), equalTo(false));
+    }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java

@@ -99,6 +99,7 @@
 import org.elasticsearch.client.ml.PutJobResponse;
 import org.elasticsearch.client.ml.RevertModelSnapshotRequest;
 import org.elasticsearch.client.ml.RevertModelSnapshotResponse;
+import org.elasticsearch.client.ml.SetUpgradeModeRequest;
 import org.elasticsearch.client.ml.StartDatafeedRequest;
 import org.elasticsearch.client.ml.StartDatafeedResponse;
 import org.elasticsearch.client.ml.StopDatafeedRequest;
@@ -3078,6 +3079,57 @@ public void onFailure(Exception e) {
         }
     }
 
+    public void testSetUpgradeMode() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+        {
+            // tag::set-upgrade-mode-request
+            SetUpgradeModeRequest request = new SetUpgradeModeRequest(true); // <1>
+            request.setTimeout(TimeValue.timeValueMinutes(10)); // <2>
+            // end::set-upgrade-mode-request
+
+            // Set to false so that the cluster setting does not have to be unset at the end of the test.
+            request.setEnabled(false);
+
+            // tag::set-upgrade-mode-execute
+            AcknowledgedResponse acknowledgedResponse = client.machineLearning().setUpgradeMode(request, RequestOptions.DEFAULT);
+            // end::set-upgrade-mode-execute
+
+            // tag::set-upgrade-mode-response
+            boolean acknowledged = acknowledgedResponse.isAcknowledged(); // <1>
+            // end::set-upgrade-mode-response
+            assertThat(acknowledged, is(true));
+        }
+        {
+            SetUpgradeModeRequest request = new SetUpgradeModeRequest(false);
+
+            // tag::set-upgrade-mode-execute-listener
+            ActionListener<AcknowledgedResponse> listener = new ActionListener<AcknowledgedResponse>() {
+                @Override
+                public void onResponse(AcknowledgedResponse acknowledgedResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::set-upgrade-mode-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::set-upgrade-mode-execute-async
+            client.machineLearning()
+                .setUpgradeModeAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::set-upgrade-mode-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+
+        }
+    }
+
     private String createFilter(RestHighLevelClient client) throws IOException {
         MlFilter.Builder filterBuilder = MlFilter.builder("my_safe_domains")
             .setDescription("A list of safe domains")

distribution/docker/build.gradle

@@ -18,8 +18,9 @@ dependencies {
 }
 
 ext.expansions = { oss ->
+  String classifier = 'linux-x86_64'
   return [
-    'elasticsearch' : oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}.tar.gz",
+    'elasticsearch' : oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}-${classifier}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}-${classifier}.tar.gz",
     'jdkUrl' : 'https://download.java.net/java/GA/jdk11/9/GPL/openjdk-11.0.2_linux-x64_bin.tar.gz',
     'jdkVersion' : '11.0.2',
     'license': oss ? 'Apache-2.0' : 'Elastic License',

docs/java-rest/high-level/ml/set-upgrade-mode.asciidoc

@@ -0,0 +1,40 @@
+--
+:api: set-upgrade-mode
+:request: SetUpgradeModeRequest
+:response: AcknowledgedResponse
+--
+[id="{upid}-{api}"]
+=== Set Upgrade Mode API
+
+The Set Upgrade Mode API temporarily halts all {ml} job and {dfeed} tasks when `enabled=true`. Their
+reported states remain unchanged. Consequently, when exiting upgrade mode the halted {ml} jobs and
+{dfeeds} will return to their previous state.
+
+It accepts a +{request}+ object and responds with a +{response}+ object.
+
+When `enabled=true`, no new jobs can be opened, and no job or {dfeed} tasks will
+be running. Be sure to set `enabled=false` once upgrade actions are completed.
+
+[id="{upid}-{api}-request"]
+==== Set Upgrade Mode Request
+
+A +{request}+ object gets created setting the desired `enabled` state.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request]
+--------------------------------------------------
+<1> Constructing a new request referencing enabling upgrade mode
+<2> Optionally setting the `timeout` value for how long the
+execution should wait.
+
+[id="{upid}-{api}-response"]
+==== Set Upgrade Mode Response
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response]
+--------------------------------------------------
+<1> `isAcknowledged()` from the +{response}+ indicates if the setting was set successfully.
+
+include::../execution.asciidoc[]