diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy index f5974d7c0f8e1..66574f5f289e8 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy @@ -217,7 +217,7 @@ class VagrantTestPlugin implements Plugin { // Now we iterate over dependencies of the bats configuration. When a project dependency is found, // we bring back its own archives, test files or test utils. project.afterEvaluate { - project.configurations.bats.dependencies.findAll {it.configuration == BATS }.each { d -> + project.configurations.bats.dependencies.findAll {it.targetConfiguration == BATS }.each { d -> if (d instanceof DefaultProjectDependency) { DefaultProjectDependency externalBatsDependency = (DefaultProjectDependency) d Project externalBatsProject = externalBatsDependency.dependencyProject diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java index 7e82852a9f3cc..a9758267ff3f1 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java @@ -195,6 +195,7 @@ static List checks(final Settings settings) { checks.add(new SystemCallFilterCheck(BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.get(settings))); checks.add(new OnErrorCheck()); checks.add(new OnOutOfMemoryErrorCheck()); + checks.add(new EarlyAccessCheck()); checks.add(new G1GCCheck()); return Collections.unmodifiableList(checks); } @@ -577,6 +578,34 @@ public String errorMessage() { } + /** + * Bootstrap check for early-access builds from OpenJDK. + */ + static class EarlyAccessCheck implements BootstrapCheck { + + @Override + public boolean check() { + return "Oracle Corporation".equals(jvmVendor()) && javaVersion().endsWith("-ea"); + } + + String jvmVendor() { + return Constants.JVM_VENDOR; + } + + String javaVersion() { + return Constants.JAVA_VERSION; + } + + @Override + public String errorMessage() { + return String.format( + Locale.ROOT, + "Java version [%s] is an early-access build, only use release builds", + javaVersion()); + } + + } + /** * Bootstrap check for versions of HotSpot that are known to have issues that can lead to index corruption when G1GC is enabled. */ diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java index bd553cff6e190..c3e08b81d6c3e 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java @@ -560,12 +560,48 @@ private void runMightForkTest( consumer.accept(e); } + public void testEarlyAccessCheck() throws NodeValidationException { + final AtomicReference javaVersion + = new AtomicReference<>(randomFrom("1.8.0_152-ea", "9-ea")); + final BootstrapChecks.EarlyAccessCheck eaCheck = new BootstrapChecks.EarlyAccessCheck() { + + @Override + String jvmVendor() { + return "Oracle Corporation"; + } + + @Override + String javaVersion() { + return javaVersion.get(); + } + + }; + + final List checks = Collections.singletonList(eaCheck); + final NodeValidationException e = expectThrows( + NodeValidationException.class, + () -> { + BootstrapChecks.check(true, checks, "testEarlyAccessCheck"); + }); + assertThat( + e.getMessage(), + containsString( + "Java version [" + + javaVersion.get() + + "] is an early-access build, only use release builds")); + + // if not on an early-access build, nothing should happen + javaVersion.set(randomFrom("1.8.0_152", "9")); + BootstrapChecks.check(true, checks, "testEarlyAccessCheck"); + + } + public void testG1GCCheck() throws NodeValidationException { final AtomicBoolean isG1GCEnabled = new AtomicBoolean(true); final AtomicBoolean isJava8 = new AtomicBoolean(true); final AtomicReference jvmVersion = new AtomicReference<>(String.format(Locale.ROOT, "25.%d-b%d", randomIntBetween(0, 39), randomIntBetween(1, 128))); - final BootstrapChecks.G1GCCheck oracleCheck = new BootstrapChecks.G1GCCheck() { + final BootstrapChecks.G1GCCheck g1GCCheck = new BootstrapChecks.G1GCCheck() { @Override String jvmVendor() { @@ -592,7 +628,7 @@ boolean isJava8() { final NodeValidationException e = expectThrows( NodeValidationException.class, - () -> BootstrapChecks.check(true, Collections.singletonList(oracleCheck), "testG1GCCheck")); + () -> BootstrapChecks.check(true, Collections.singletonList(g1GCCheck), "testG1GCCheck")); assertThat( e.getMessage(), containsString( @@ -600,12 +636,12 @@ boolean isJava8() { // if G1GC is disabled, nothing should happen isG1GCEnabled.set(false); - BootstrapChecks.check(true, Collections.singletonList(oracleCheck), "testG1GCCheck"); + BootstrapChecks.check(true, Collections.singletonList(g1GCCheck), "testG1GCCheck"); // if on or after update 40, nothing should happen independent of whether or not G1GC is enabled isG1GCEnabled.set(randomBoolean()); jvmVersion.set(String.format(Locale.ROOT, "25.%d-b%d", randomIntBetween(40, 112), randomIntBetween(1, 128))); - BootstrapChecks.check(true, Collections.singletonList(oracleCheck), "testG1GCCheck"); + BootstrapChecks.check(true, Collections.singletonList(g1GCCheck), "testG1GCCheck"); final BootstrapChecks.G1GCCheck nonOracleCheck = new BootstrapChecks.G1GCCheck() { diff --git a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc index d2ae9052f3536..2b900bd9ba2cc 100644 --- a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc @@ -302,6 +302,61 @@ documents into buckets starting at 6am: NOTE: The start `offset` of each bucket is calculated after the `time_zone` adjustments have been made. +==== Keyed Response + +Setting the `keyed` flag to `true` will associate a unique string key with each bucket and return the ranges as a hash rather than an array: + +[source,js] +-------------------------------------------------- +{ + "aggs": { + "nyc_accident_histogram": { + "date_histogram": { + "field": "date", + "interval" : "year", + "keyed": true + } + } + } +} +-------------------------------------------------- + +Response: + +[source,js] +-------------------------------------------------- +{ + ... + + "aggregations": { + "nyc_accident_histogram": { + "buckets": { + "2013-01-01T00:00:00.000Z": { + "key_as_string": "2013-01-01T00:00:00.000Z", + "key": 1356998400000, + "doc_count": 203733 + }, + "2014-01-01T00:00:00.000Z": { + "key_as_string": "2014-01-01T00:00:00.000Z", + "key": 1388534400000, + "doc_count": 205929 + }, + "2015-01-01T00:00:00.000Z": { + "key_as_string": "2015-01-01T00:00:00.000Z", + "key": 1420070400000, + "doc_count": 217637 + }, + "2016-01-01T00:00:00.000Z": { + "key_as_string": "2016-01-01T00:00:00.000Z", + "key": 1451606400000, + "doc_count": 227656 + } + } + } + } +} +-------------------------------------------------- + ==== Scripts Like with the normal <>, both document level scripts and diff --git a/docs/reference/aggregations/bucket/daterange-aggregation.asciidoc b/docs/reference/aggregations/bucket/daterange-aggregation.asciidoc index df32075583fbf..b7d0828d36e71 100644 --- a/docs/reference/aggregations/bucket/daterange-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/daterange-aggregation.asciidoc @@ -153,3 +153,91 @@ POST /sales/_search?size=0 <1> This date will be converted to `2016-02-15T00:00:00.000+01:00`. <2> `now/d` will be rounded to the beginning of the day in the CET time zone. + +==== Keyed Response + +Setting the `keyed` flag to `true` will associate a unique string key with each bucket and return the ranges as a hash rather than an array: + +[source,js] +-------------------------------------------------- +{ + "aggs" : { + "nyc_recent_accidents": { + "date_range": { + "field": "@timestamp", + "format": "MM-yyy", + "ranges": [ + { "from": "now-10M/d", "to": "now" } + ], + "keyed": true + } + } + } +} +-------------------------------------------------- + +Response: + +[source,js] +-------------------------------------------------- +{ + ... + + "aggregations": { + "nyc_recent_accidents": { + "buckets": { + "05-2016-03-2017": { + "from": 1464307200000, + "from_as_string": "05-2016", + "to": 1490604059606, + "to_as_string": "03-2017", + "doc_count": 185436 + } + } + } + } +} +-------------------------------------------------- + +It is also possible to customize the key for each range: + +[source,js] +-------------------------------------------------- +{ + "aggs" : { + "nyc_recent_accidents": { + "date_range": { + "field": "@timestamp", + "format": "MM-yyy", + "ranges": [ + { "from": "now-10M/d", "to": "now" } + ], + "keyed": true + } + } + } +} +-------------------------------------------------- + +Response: + +[source,js] +-------------------------------------------------- +{ + ... + + "aggregations": { + "nyc_recent_accidents": { + "buckets": { + "last-10M": { + "from": 1464307200000, + "from_as_string": "05-2016", + "to": 1490604059606, + "to_as_string": "03-2017", + "doc_count": 185436 + } + } + } + } +} +-------------------------------------------------- diff --git a/docs/reference/aggregations/bucket/geodistance-aggregation.asciidoc b/docs/reference/aggregations/bucket/geodistance-aggregation.asciidoc index a051fc00c4f2e..bb7964ea35c4c 100644 --- a/docs/reference/aggregations/bucket/geodistance-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geodistance-aggregation.asciidoc @@ -104,3 +104,111 @@ There are two distance calculation modes: `arc` (the default), and `plane`. The } } -------------------------------------------------- + +==== Keyed Response + +Setting the `keyed` flag to `true` will associate a unique string key with each bucket and return the ranges as a hash rather than an array: + +[source,js] +-------------------------------------------------- +{ + "aggs": { + "accidents_near_empire_state": { + "geo_distance": { + "field": "coords", + "unit": "km", + "origin" : { "lat" : 40.7484, "lon" : -73.9857 }, + "ranges": [ + { "to": 3 }, + { "from" : 3, "to" : 8 }, + { "from" : 8 } + ], + "keyed": true + } + } + } +} +-------------------------------------------------- + +Response: + +[source,js] +-------------------------------------------------- +{ + ... + + "aggregations": { + "accidents_near_empire_state": { + "buckets": { + "*-3.0": { + "from": 0, + "to": 3, + "doc_count": 102090 + }, + "3.0-8.0": { + "from": 3, + "to": 8, + "doc_count": 170287 + }, + "8.0-*": { + "from": 8, + "doc_count": 526590 + } + } + } + } +} +-------------------------------------------------- + +It is also possible to customize the key for each range: + +[source,js] +-------------------------------------------------- +{ + "aggs": { + "accidents_near_empire_state": { + "geo_distance": { + "field": "coords", + "unit": "km", + "origin" : { "lat" : 40.7484, "lon" : -73.9857 }, + "ranges": [ + { "key": "very-close", "to": 3 }, + { "key": "close", "from" : 3, "to" : 8 }, + { "key": "far", "from" : 8 } + ], + "keyed": true + } + } + } +} +-------------------------------------------------- + +Response: + +[source,js] +-------------------------------------------------- +{ + ... + + "aggregations": { + "accidents_near_empire_state": { + "buckets": { + "very-close": { + "from": 0, + "to": 3, + "doc_count": 102090 + }, + "close": { + "from": 3, + "to": 8, + "doc_count": 170287 + }, + "far": { + "from": 8, + "doc_count": 526590 + } + } + } + } +} +-------------------------------------------------- diff --git a/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc index de828c62aa9a7..4083fd9fe2cb1 100644 --- a/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc @@ -362,6 +362,61 @@ Response: -------------------------------------------------- // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] +==== Keyed Response + +Setting the `keyed` flag to `true` will associate a unique string key with each bucket and return the ranges as a hash rather than an array: + +[source,js] +-------------------------------------------------- +{ + "aggs": { + "bytes_histogram": { + "histogram": { + "field": "bytes", + "interval": 200, + "keyed": true + } + } + } +} +-------------------------------------------------- + +Response: + +[source,js] +-------------------------------------------------- +{ + ... + + "aggregations": { + "bytes_histogram": { + "buckets": { + "0.0": { + "key": 0, + "doc_count": 13596 + }, + "200.0": { + "key": 200, + "doc_count": 33882 + }, + "400.0": { + "key": 400, + "doc_count": 548 + }, + "600.0": { + "key": 600, + "doc_count": 0 + }, + "800.0": { + "key": 800, + "doc_count": 460 + } + } + } + } +} +-------------------------------------------------- + ==== Missing value The `missing` parameter defines how documents that are missing a value should be treated. diff --git a/docs/reference/aggregations/bucket/iprange-aggregation.asciidoc b/docs/reference/aggregations/bucket/iprange-aggregation.asciidoc index bb20b18663e03..ee77d57a0b476 100644 --- a/docs/reference/aggregations/bucket/iprange-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/iprange-aggregation.asciidoc @@ -90,3 +90,93 @@ Response: } } -------------------------------------------------- + +==== Keyed Response + +Setting the `keyed` flag to `true` will associate a unique string key with each bucket and return the ranges as a hash rather than an array: + +[source,js] +-------------------------------------------------- +{ + "aggs": { + "ip_ranges": { + "ip_range": { + "field": "remote_ip", + "ranges": [ + { "to" : "10.0.0.5" }, + { "from" : "10.0.0.5" } + ], + "keyed": true + } + } + } +} +-------------------------------------------------- + +Response: + +[source,js] +-------------------------------------------------- +{ + ... + + "aggregations": { + "ip_ranges": { + "buckets": { + "*-10.0.0.5": { + "to": "10.0.0.5", + "doc_count": 1462 + }, + "10.0.0.5-*": { + "from": "10.0.0.5", + "doc_count": 50000 + } + } + } + } +} +-------------------------------------------------- + +It is also possible to customize the key for each range: + +[source,js] +-------------------------------------------------- +{ + "aggs": { + "ip_ranges": { + "ip_range": { + "field": "remote_ip", + "ranges": [ + { "key": "infinity", "to" : "10.0.0.5" }, + { "key": "and-beyond", "from" : "10.0.0.5" } + ], + "keyed": true + } + } + } +} +-------------------------------------------------- + +Response: + +[source,js] +-------------------------------------------------- +{ + ... + + "aggregations": { + "ip_ranges": { + "buckets": { + "infinity": { + "to": "10.0.0.5", + "doc_count": 1462 + }, + "and-beyond": { + "from": "10.0.0.5", + "doc_count": 50000 + } + } + } + } +} +-------------------------------------------------- diff --git a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc index db15d0a6a6639..d169dbeb353b0 100644 --- a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc @@ -86,7 +86,52 @@ must be a value between 0-100 inclusive): -------------------------------------------------- <1> Use the `percents` parameter to specify particular percentiles to calculate +==== Keyed Response +By default the `keyed` flag is set to `true` associates a unique string key with each bucket and returns the ranges as a hash rather than an array. Setting the `keyed` flag to `false` will disable this behavior: + +[source,js] +-------------------------------------------------- +{ + "aggs": { + "accident_hour_outlier": { + "percentiles": { + "field": "hour_of_day", + "percents" : [80, 85, 95], + "keyed": false + } + } + } +} +-------------------------------------------------- + +Response: + +[source,js] +-------------------------------------------------- +{ + ... + + "aggregations": { + "accident_hour_outlier": { + "values": [ + { + "key": 80, + "value": 18 + }, + { + "key": 85, + "value": 19 + }, + { + "key": 95, + "value": 22 + } + ] + } + } +} +-------------------------------------------------- ==== Script diff --git a/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc b/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc index d4df92105defc..9b468b7e50e88 100644 --- a/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc @@ -58,6 +58,52 @@ The response will look like this: From this information you can determine you are hitting the 99% load time target but not quite hitting the 95% load time target +==== Keyed Response + +By default the `keyed` flag is set to `true` associates a unique string key with each bucket and returns the ranges as a hash rather than an array. Setting the `keyed` flag to `false` will disable this behavior: + +[source,js] +-------------------------------------------------- +{ + "aggs": { + "accident_hour_outlier": { + "percentile_ranks": { + "field": "hour_of_day", + "values" : [15, 20, 22], + "keyed": false + } + } + } +} +-------------------------------------------------- + +Response: + +[source,js] +-------------------------------------------------- +{ + ... + + "aggregations": { + "accident_hour_outlier": { + "values": [ + { + "key": 15, + "value": 59.80540497209334 + }, + { + "key": 20, + "value": 90.38133124528507 + }, + { + "key": 22, + "value": 97.28720731656692 + } + ] + } + } +} +-------------------------------------------------- ==== Script diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc index 22c3f03cd3633..2d18911beb67e 100644 --- a/docs/reference/setup/bootstrap-checks.asciidoc +++ b/docs/reference/setup/bootstrap-checks.asciidoc @@ -179,6 +179,13 @@ use the JVM flag `ExitOnOutOfMemoryError`. While this does not have the full capabilities of `OnError` nor `OnOutOfMemoryError`, arbitrary forking will not be supported with seccomp enabled. +=== Early-access check + +The OpenJDK project provides early-access snapshots of upcoming releases. These +releases are not suitable for production. The early-access check detects these +early-access snapshots. To pass this check, you must start Elasticsearch on a +release build of the JVM. + === G1GC check Early versions of the HotSpot JVM that shipped with JDK 8 are known to have diff --git a/modules/lang-painless/ant.xml b/modules/lang-painless/ant.xml deleted file mode 100644 index 90e66b7b1a9b9..0000000000000 --- a/modules/lang-painless/ant.xml +++ /dev/null @@ -1,157 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle index 943d953ea484e..31b41261b3a77 100644 --- a/modules/lang-painless/build.gradle +++ b/modules/lang-painless/build.gradle @@ -33,23 +33,6 @@ dependencyLicenses { mapping from: /asm-.*/, to: 'asm' } -// regeneration logic, comes in via ant right now -// don't port it to gradle, it works fine. - -/* -configurations { - regenerate -} - -dependencies { - regenerate 'org.antlr:antlr4:4.5.1-1' -} - -// TODO: this is causing a resolve at configurationt time. we really should not be using ant anymore -ant.references['regenerate.classpath'] = new Path(ant.project, configurations.regenerate.asPath) -ant.importBuild 'ant.xml' -*/ - integTestCluster { setting 'script.max_compilations_per_minute', '1000' } @@ -73,3 +56,95 @@ task generatePainlessApi(type: JavaExec) { classpath = sourceSets.test.runtimeClasspath args file('../../docs/reference/painless-api-reference') } + +/********************************************** + * Parser regeneration * + **********************************************/ + +configurations { + regenerate +} + +dependencies { + regenerate 'org.antlr:antlr4:4.5.1-1' +} + +String grammarPath = 'src/main/antlr' +String outputPath = 'src/main/java/org/elasticsearch/painless/antlr' + +task cleanGenerated(type: Delete) { + delete fileTree(grammarPath) { + include '*.tokens' + } + delete fileTree(outputPath) { + include 'Painless*.java' + } +} + +task regenLexer(type: JavaExec) { + dependsOn cleanGenerated + main = 'org.antlr.v4.Tool' + classpath = configurations.regenerate + systemProperty 'file.encoding', 'UTF-8' + systemProperty 'user.language', 'en' + systemProperty 'user.country', 'US' + systemProperty 'user.variant', '' + args '-Werror', + '-package', 'org.elasticsearch.painless.antlr', + '-o', outputPath, + "${file(grammarPath)}/PainlessLexer.g4" +} + +task regenParser(type: JavaExec) { + dependsOn regenLexer + main = 'org.antlr.v4.Tool' + classpath = configurations.regenerate + systemProperty 'file.encoding', 'UTF-8' + systemProperty 'user.language', 'en' + systemProperty 'user.country', 'US' + systemProperty 'user.variant', '' + args '-Werror', + '-package', 'org.elasticsearch.painless.antlr', + '-no-listener', + '-visitor', + // '-Xlog', + '-o', outputPath, + "${file(grammarPath)}/PainlessParser.g4" +} + +task regen { + dependsOn regenParser + doLast { + // moves token files to grammar directory for use with IDE's + ant.move(file: "${outputPath}/PainlessLexer.tokens", toDir: grammarPath) + ant.move(file: "${outputPath}/PainlessParser.tokens", toDir: grammarPath) + // make the generated classes package private + ant.replaceregexp(match: 'public ((interface|class) \\QPainless\\E\\w+)', + replace: '\\1', + encoding: 'UTF-8') { + fileset(dir: outputPath, includes: 'Painless*.java') + } + // make the lexer abstract + ant.replaceregexp(match: '(class \\QPainless\\ELexer)', + replace: 'abstract \\1', + encoding: 'UTF-8') { + fileset(dir: outputPath, includes: 'PainlessLexer.java') + } + // nuke timestamps/filenames in generated files + ant.replaceregexp(match: '\\Q// Generated from \\E.*', + replace: '\\/\\/ ANTLR GENERATED CODE: DO NOT EDIT', + encoding: 'UTF-8') { + fileset(dir: outputPath, includes: 'Painless*.java') + } + // remove tabs in antlr generated files + ant.replaceregexp(match: '\t', flags: 'g', replace: ' ', encoding: 'UTF-8') { + fileset(dir: outputPath, includes: 'Painless*.java') + } + // fix line endings + ant.fixcrlf(srcdir: outputPath) { + patternset(includes: 'Painless*.java') + } + } +} + +