Merge branch 'master' into retention-lease-ccr

* master:
  Add an authentication cache for API keys (elastic#38469)
  Fix exit code in certutil packaging test (elastic#38393)
  Enable logs for intermittent test failure (elastic#38426)
  Disable BWC to backport recovering retention leases (elastic#38477)
  Enable bwc tests now that elastic#38443 is backported. (elastic#38462)
  Fix Master Failover and DataNode Leave Blocking Snapshot (elastic#38460)
  Recover retention leases during peer recovery (elastic#38435)
  Set update mappings mater node timeout to 30 min (elastic#38439)
  Assert job is not null in FullClusterRestartIT (elastic#38218)
  Update ilm-api.asciidoc, point to REMOVE policy (elastic#38235) (elastic#38463)
  SQL: Fix esType for DATETIME/DATE and INTERVALS (elastic#38179)
  Handle deprecation header-AbstractUpgradeTestCase (elastic#38396)
  XPack: core/ccr/Security-cli migration to java-time (elastic#38415)
  Disable bwc tests for elastic#38443 (elastic#38456)
  Bubble-up exceptions from scheduler (elastic#38317)
  Re-enable TasksClientDocumentationIT.testCancelTasks (elastic#38234)
  Allow custom authorization with an authorization engine  (elastic#38358)
  CRUDDocumentationIT fix documentation references
  Remove support for internal versioning for concurrency control (elastic#38254)

Author: jasontedor

build.gradle

@@ -159,8 +159,8 @@ task verifyVersions {
  * the enabled state of every bwc task. It should be set back to true
  * after the backport of the backcompat code is complete.
  */
-final boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+final boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/38435" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
@@ -233,6 +233,9 @@ allprojects {
     "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats',
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
     "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
+    // for security example plugins
+    "org.elasticsearch.plugin:x-pack-core:${version}": ':x-pack:plugin:core',
+    "org.elasticsearch.client.x-pack-transport:${version}": ':x-pack:transport-client'
   ]
 
   /*

client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherRequestConverters.java

@@ -70,7 +70,6 @@ static Request putWatch(PutWatchRequest putWatchRequest) {
 
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
         RequestConverters.Params params = new RequestConverters.Params(request)
-            .withVersion(putWatchRequest.getVersion())
             .withIfSeqNo(putWatchRequest.ifSeqNo())
             .withIfPrimaryTerm(putWatchRequest.ifPrimaryTerm());
         if (putWatchRequest.isActive() == false) {

client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/PutWatchRequest.java

@@ -21,7 +21,6 @@
 import org.elasticsearch.client.Validatable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 
@@ -43,11 +42,9 @@ public final class PutWatchRequest implements Validatable {
     private final BytesReference source;
     private final XContentType xContentType;
     private boolean active = true;
-    private long version = Versions.MATCH_ANY;
     private long ifSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
     private long ifPrimaryTerm = UNASSIGNED_PRIMARY_TERM;
 
-
     public PutWatchRequest(String id, BytesReference source, XContentType xContentType) {
         Objects.requireNonNull(id, "watch id is missing");
         if (isValidId(id) == false) {
@@ -95,14 +92,6 @@ public XContentType xContentType() {
         return xContentType;
     }
 
-    public long getVersion() {
-        return version;
-    }
-
-    public void setVersion(long version) {
-        this.version = version;
-    }
-
     /**
      * only performs this put request if the watch's last modification was assigned the given
      * sequence number. Must be used in combination with {@link #setIfPrimaryTerm(long)}

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java

@@ -768,8 +768,6 @@ public void testUpdate() throws IOException {
             }
         }
         setRandomWaitForActiveShards(updateRequest::waitForActiveShards, expectedParams);
-        setRandomVersion(updateRequest, expectedParams);
-        setRandomVersionType(updateRequest::versionType, expectedParams);
         setRandomIfSeqNoAndTerm(updateRequest, new HashMap<>()); // if* params are passed in the body
         if (randomBoolean()) {
             int retryOnConflict = randomIntBetween(0, 5);
@@ -911,14 +909,7 @@ public void testBulk() throws IOException {
             if (randomBoolean()) {
                 docWriteRequest.routing(randomAlphaOfLength(10));
             }
-            if (randomBoolean()) {
-                if (randomBoolean()) {
-                    docWriteRequest.version(randomNonNegativeLong());
-                }
-                if (randomBoolean()) {
-                    docWriteRequest.versionType(randomFrom(VersionType.values()));
-                }
-            } else if (randomBoolean()) {
+            if (opType != DocWriteRequest.OpType.UPDATE && randomBoolean()) {
                 docWriteRequest.setIfSeqNo(randomNonNegativeLong());
                 docWriteRequest.setIfPrimaryTerm(randomLongBetween(1, 200));
             }

client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherRequestConvertersTests.java

@@ -29,8 +29,8 @@
 import org.elasticsearch.client.watcher.DeactivateWatchRequest;
 import org.elasticsearch.client.watcher.DeleteWatchRequest;
 import org.elasticsearch.client.watcher.ExecuteWatchRequest;
-import org.elasticsearch.client.watcher.PutWatchRequest;
 import org.elasticsearch.client.watcher.GetWatchRequest;
+import org.elasticsearch.client.watcher.PutWatchRequest;
 import org.elasticsearch.client.watcher.StartWatchServiceRequest;
 import org.elasticsearch.client.watcher.StopWatchServiceRequest;
 import org.elasticsearch.client.watcher.WatcherStatsRequest;
@@ -88,9 +88,12 @@ public void testPutWatch() throws Exception {
         }
 
         if (randomBoolean()) {
-            long version = randomLongBetween(10, 100);
-            putWatchRequest.setVersion(version);
-            expectedParams.put("version", String.valueOf(version));
+            long seqNo = randomNonNegativeLong();
+            long ifPrimaryTerm = randomLongBetween(1, 200);
+            putWatchRequest.setIfSeqNo(seqNo);
+            putWatchRequest.setIfPrimaryTerm(ifPrimaryTerm);
+            expectedParams.put("if_seq_no", String.valueOf(seqNo));
+            expectedParams.put("if_primary_term", String.valueOf(ifPrimaryTerm));
         }
 
         Request request = WatcherRequestConverters.putWatch(putWatchRequest);

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java

@@ -170,7 +170,6 @@ public void testIndex() throws Exception {
             // tag::index-response
             String index = indexResponse.getIndex();
             String id = indexResponse.getId();
-            long version = indexResponse.getVersion();
             if (indexResponse.getResult() == DocWriteResponse.Result.CREATED) {
                 // <1>
             } else if (indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
@@ -220,7 +219,8 @@ public void testIndex() throws Exception {
             IndexRequest request = new IndexRequest("posts")
                 .id("1")
                 .source("field", "value")
-                .version(1);
+                .setIfSeqNo(10L)
+                .setIfPrimaryTerm(20);
             try {
                 IndexResponse response = client.index(request, RequestOptions.DEFAULT);
             } catch(ElasticsearchException e) {
@@ -432,7 +432,8 @@ public void testUpdate() throws Exception {
             // tag::update-conflict
             UpdateRequest request = new UpdateRequest("posts", "1")
                     .doc("field", "value")
-                    .version(1);
+                    .setIfSeqNo(101L)
+                    .setIfPrimaryTerm(200L);
             try {
                 UpdateResponse updateResponse = client.update(
                         request, RequestOptions.DEFAULT);
@@ -499,9 +500,10 @@ public void testUpdate() throws Exception {
             request.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); // <1>
             request.setRefreshPolicy("wait_for");                            // <2>
             // end::update-request-refresh
-            // tag::update-request-version
-            request.version(2); // <1>
-            // end::update-request-version
+            // tag::update-request-cas
+            request.setIfSeqNo(2L); // <1>
+            request.setIfPrimaryTerm(1L); // <2>
+            // end::update-request-cas
             // tag::update-request-detect-noop
             request.detectNoop(false); // <1>
             // end::update-request-detect-noop
@@ -630,7 +632,7 @@ public void testDelete() throws Exception {
             // tag::delete-conflict
             try {
                 DeleteResponse deleteResponse = client.delete(
-                        new DeleteRequest("posts", "1").version(2),
+                    new DeleteRequest("posts", "1").setIfSeqNo(100).setIfPrimaryTerm(2),
                         RequestOptions.DEFAULT);
             } catch (ElasticsearchException exception) {
                 if (exception.status() == RestStatus.CONFLICT) {

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TasksClientDocumentationIT.java

@@ -150,7 +150,6 @@ public void onFailure(Exception e) {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/35514")
     @SuppressWarnings("unused")
     public void testCancelTasks() throws IOException {
         RestHighLevelClient client = highLevelClient();

docs/java-rest/high-level/document/update.asciidoc

@@ -140,9 +140,10 @@ include-tagged::{doc-tests-file}[{api}-request-source-exclude]
 
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------
-include-tagged::{doc-tests-file}[{api}-request-version]
+include-tagged::{doc-tests-file}[{api}-request-cas]
 --------------------------------------------------
-<1> Version
+<1> ifSeqNo
+<2> ifPrimaryTerm
 
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------

docs/reference/ilm/apis/ilm-api.asciidoc

@@ -19,6 +19,7 @@ about Index Lifecycle Management.
 
 * <<ilm-move-to-step,Move index to step>>
 * <<ilm-retry-policy,Retry policy on indices>>
+* <<ilm-remove-policy,Remove policy from index>>
 
 [float]
 [[ilm-api-management-endpoint]]

docs/reference/migration/migrate_7_0/api.asciidoc

@@ -2,6 +2,22 @@
 [[breaking_70_api_changes]]
 === API changes
 
+[float]
+==== Internal Versioning is no longer supported for optimistic concurrency control
+
+Elasticsearch maintains a numeric version field for each document it stores. That field
+is incremented by one with every change to the document. Until 7.0.0 the API allowed using
+that field for optimistic concurrency control, i.e., making a write operation conditional
+on the current document version. Sadly, that approach is flawed because the value of the
+version doesn't always uniquely represent a change to the document. If a primary fails
+while handling a write operation, it may expose a version that will then be reused by the
+new primary.
+
+Due to that issue, internal versioning can no longer be used and is replaced by a new
+method based on sequence numbers. See <<optimistic-concurrency-control>> for more details.
+
+Note that the `external` versioning type is still fully supported.
+
 [float]
 ==== Camel case and underscore parameters deprecated in 6.x have been removed
 A number of duplicate parameters deprecated in 6.x have been removed from