Merge remote-tracking branch 'elastic/master' into more-snapshot-resiliency-testing-2

Author: original-brownbear

distribution/archives/build.gradle

@@ -84,7 +84,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
         rename { 'LICENSE.txt' }
       }
 
-      with noticeFile
+      with noticeFile(oss, jdk)
       into('modules') {
         with modulesFiles
       }

distribution/build.gradle

@@ -52,11 +52,17 @@ task buildServerNotice(type: NoticeTask) {
 // other distributions include notices from modules as well, which are added below later
 task buildDefaultNotice(type: NoticeTask) {
   licensesDir new File(project(':server').projectDir, 'licenses')
+  licensesDir new File(project(':distribution').projectDir, 'licenses')
 }
-
-// other distributions include notices from modules as well, which are added below later
 task buildOssNotice(type: NoticeTask) {
   licensesDir new File(project(':server').projectDir, 'licenses')
+  licensesDir new File(project(':distribution').projectDir, 'licenses')
+}
+task buildDefaultNoJdkNotice(type: NoticeTask) {
+  licensesDir new File(project(':server').projectDir, 'licenses')
+}
+task buildOssNoJdkNotice(type: NoticeTask) {
+  licensesDir new File(project(':server').projectDir, 'licenses')
 }
 
 /*****************************************************************************
@@ -377,11 +383,21 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
       }
     }
 
-    noticeFile = copySpec {
-      if (project.name == 'integ-test-zip') {
-        from buildServerNotice
-      } else {
-        from buildDefaultNotice
+    noticeFile = { oss, jdk ->
+      copySpec {
+        if (project.name == 'integ-test-zip') {
+          from buildServerNotice
+        } else {
+          if (oss && jdk) {
+            from buildOssNotice
+          } else if (oss) {
+            from buildOssNoJdkNotice
+          } else if (jdk) {
+            from buildDefaultNotice
+          } else {
+            from buildDefaultNoJdkNotice
+          }
+      }  
       }
     }
 

distribution/licenses/openjdk-LICENSE.txt

@@ -0,0 +1,5 @@
+Copyright (c) 1995, 2013, Oracle and/or its affiliates.
+
+OpenJDK is licensed under the GPLv2+CE. A copy of that license is included in
+this distribution immediately below this notice. You can find a copy of the
+OpenJDK source through the downloads page at https://elastic.co.

distribution/licenses/openjdk-NOTICE.txt

@@ -260,6 +260,9 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
 
     // the oss package conflicts with the default distribution and vice versa
     conflicts('elasticsearch' + (oss ? '' : '-oss'))
+
+    into '/usr/share/elasticsearch'
+    with noticeFile(oss, jdk)
   }
 }
 
@@ -294,7 +297,6 @@ ospackage {
   permissionGroup 'root'
 
   into '/usr/share/elasticsearch'
-  with noticeFile
 }
 
 Closure commonDebConfig(boolean oss, boolean jdk) {

distribution/packages/build.gradle

@@ -58,8 +58,8 @@ The recommended way to set your requirements in your `setup.py` or
 
 [source,txt]
 ------------------------------------
-    # Elasticsearch 5.x
-    elasticsearch>=5.0.0,<6.0.0
+    # Elasticsearch 6.x
+    elasticsearch>=6.0.0,<7.0.0
 
     # Elasticsearch 2.x
     elasticsearch2

docs/python/index.asciidoc

@@ -118,8 +118,11 @@ POST _reindex
 // CONSOLE
 // TEST[setup:twitter]
 
-By default, version conflicts abort the `_reindex` process, but you can just
-count them by setting `"conflicts": "proceed"` in the request body:
+By default, version conflicts abort the `_reindex` process. The `"conflicts"` request body
+parameter can be used to instruct `_reindex` to proceed with the next document on version conflicts.
+It is important to note that the handling of other error types is unaffected by the `"conflicts"` parameter.
+When `"conflicts": "proceed"` is set in the request body, the `_reindex` process will continue on version conflicts
+and return a count of version conflicts encountered:
 
 [source,js]
 --------------------------------------------------

docs/reference/docs/reindex.asciidoc

@@ -134,7 +134,7 @@ PUT _ml/datafeeds/datafeed-test1
     "total_error_count": { <2>
       "script": {
         "lang": "expression",
-        "inline": "doc['error_count'].value + doc['aborted_count'].value"
+        "source": "doc['error_count'].value + doc['aborted_count'].value"
       }
     }
   }
@@ -239,7 +239,7 @@ PUT _ml/datafeeds/datafeed-test2
     "my_script_field": {
       "script": {
         "lang": "painless",
-        "inline": "doc['some_field'].value + '_' + doc['another_field'].value" <2>
+        "source": "doc['some_field'].value + '_' + doc['another_field'].value" <2>
       }
     }
   }
@@ -276,7 +276,7 @@ POST _ml/datafeeds/datafeed-test2/_update
     "my_script_field": {
       "script": {
         "lang": "painless",
-        "inline": "doc['another_field'].value.trim()" <1>
+        "source": "doc['another_field'].value.trim()" <1>
       }
     }
   }
@@ -312,7 +312,7 @@ POST _ml/datafeeds/datafeed-test2/_update
     "my_script_field": {
       "script": {
         "lang": "painless",
-        "inline": "doc['some_field'].value.toLowerCase()" <1>
+        "source": "doc['some_field'].value.toLowerCase()" <1>
       }
     }
   }
@@ -349,7 +349,7 @@ POST _ml/datafeeds/datafeed-test2/_update
     "my_script_field": {
       "script": {
         "lang": "painless",
-        "inline": "doc['some_field'].value.substring(0, 1).toUpperCase() + doc['some_field'].value.substring(1).toLowerCase()" <1>
+        "source": "doc['some_field'].value.substring(0, 1).toUpperCase() + doc['some_field'].value.substring(1).toLowerCase()" <1>
       }
     }
   }
@@ -386,7 +386,7 @@ POST _ml/datafeeds/datafeed-test2/_update
     "my_script_field": {
       "script": {
         "lang": "painless",
-        "inline": "/\\s/.matcher(doc['tokenstring2'].value).replaceAll('_')" <1>
+        "source": "/\\s/.matcher(doc['tokenstring2'].value).replaceAll('_')" <1>
       }
     }
   }
@@ -422,7 +422,7 @@ POST _ml/datafeeds/datafeed-test2/_update
     "my_script_field": {
       "script": {
         "lang": "painless",
-        "inline": "def m = /(.*)-bar-([0-9][0-9])/.matcher(doc['tokenstring3'].value); return m.find() ? m.group(1) + '_' + m.group(2) : '';" <1>
+        "source": "def m = /(.*)-bar-([0-9][0-9])/.matcher(doc['tokenstring3'].value); return m.find() ? m.group(1) + '_' + m.group(2) : '';" <1>
       }
     }
   }
@@ -554,7 +554,7 @@ PUT _ml/datafeeds/datafeed-test4
   "script_fields": {
     "my_coordinates": {
       "script": {
-        "inline": "doc['coords.lat'].value + ',' + doc['coords.lon'].value",
+        "source": "doc['coords.lat'].value + ',' + doc['coords.lon'].value",
         "lang": "painless"
       }
     }

docs/reference/ml/transforms.asciidoc

@@ -138,7 +138,10 @@ GET _search
 --------------------------------------------------
 // CONSOLE
 <1> This date will be converted to `2014-12-31T23:00:00 UTC`.
-<2> `now` is not affected by the `time_zone` parameter (dates must be stored as UTC).
+<2> `now` is not affected by the `time_zone` parameter, its always the current system time (in UTC).
+However, when using <<date-math,date math rounding>> (e.g. down to the nearest day using `now/d`),
+the provided `time_zone` will be considered.
+
 
 [[querying-range-fields]]
 ==== Querying range fields

docs/reference/query-dsl/range-query.asciidoc

@@ -36,6 +36,10 @@ GET /_search
 // CONSOLE
 // TEST[setup:twitter]
 
+NOTE: The values returned from `script_score` cannot be negative. In general,
+Lucene requires the scores produced by queries to be non-negative in order to
+support certain search optimizations.
+
 ==== Accessing the score of a document within a script
 
 Within a script, you can
@@ -92,17 +96,18 @@ cosine similarity between a given query vector and document vectors.
         "match_all": {}
       },
       "script": {
-        "source": "cosineSimilarity(params.queryVector, doc['my_dense_vector'])",
+        "source": "cosineSimilarity(params.query_vector, doc['my_dense_vector']) + 1.0" <1>,
         "params": {
-          "queryVector": [4, 3.4, -0.2]  <1>
+          "query_vector": [4, 3.4, -0.2]  <2>
         }
       }
     }
   }
 }
 --------------------------------------------------
 // NOTCONSOLE
-<1> To take advantage of the script optimizations, provide a query vector as a script parameter.
+<1> The script adds 1.0 to the cosine similarity to prevent the score from being negative.
+<2> To take advantage of the script optimizations, provide a query vector as a script parameter.
 
 Similarly, for sparse_vector fields, `cosineSimilaritySparse` calculates cosine similarity
 between a given query vector and document vectors.
@@ -116,9 +121,9 @@ between a given query vector and document vectors.
         "match_all": {}
       },
       "script": {
-        "source": "cosineSimilaritySparse(params.queryVector, doc['my_sparse_vector'])",
+        "source": "cosineSimilaritySparse(params.query_vector, doc['my_sparse_vector']) + 1.0",
         "params": {
-          "queryVector": {"2": 0.5, "10" : 111.3, "50": -1.3, "113": 14.8, "4545": 156.0}
+          "query_vector": {"2": 0.5, "10" : 111.3, "50": -1.3, "113": 14.8, "4545": 156.0}
         }
       }
     }
@@ -139,9 +144,12 @@ dot product between a given query vector and document vectors.
         "match_all": {}
       },
       "script": {
-        "source": "dotProduct(params.queryVector, doc['my_dense_vector'])",
+        "source": """
+          double value = dotProduct(params.query_vector, doc['my_vector']);
+          return sigmoid(1, Math.E, -value); <1>
+        """,
         "params": {
-          "queryVector": [4, 3.4, -0.2]
+          "query_vector": [4, 3.4, -0.2]
         }
       }
     }
@@ -150,6 +158,8 @@ dot product between a given query vector and document vectors.
 --------------------------------------------------
 // NOTCONSOLE
 
+<1> Using the standard sigmoid function prevents scores from being negative.
+
 Similarly, for sparse_vector fields, `dotProductSparse` calculates dot product
 between a given query vector and document vectors.
 
@@ -162,9 +172,12 @@ between a given query vector and document vectors.
         "match_all": {}
       },
       "script": {
-        "source": "dotProductSparse(params.queryVector, doc['my_sparse_vector'])",
-        "params": {
-          "queryVector": {"2": 0.5, "10" : 111.3, "50": -1.3, "113": 14.8, "4545": 156.0}
+        "source": """
+          double value = dotProductSparse(params.query_vector, doc['my_sparse_vector']);
+          return sigmoid(1, Math.E, -value);
+        """,
+         "params": {
+          "query_vector": {"2": 0.5, "10" : 111.3, "50": -1.3, "113": 14.8, "4545": 156.0}
         }
       }
     }