From baf04b72176018f812a2ec3a2e37f170532a6c58 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Wed, 30 Sep 2020 10:27:40 -0400 Subject: [PATCH 1/4] [ML] adding for_export flag for APIs --- .../client/MLRequestConverters.java | 11 +- .../ml/GetDataFrameAnalyticsRequest.java | 21 +- .../client/ml/GetDatafeedRequest.java | 23 +- .../client/ml/GetJobRequest.java | 21 +- .../MlClientDocumentationIT.java | 3 + .../ml/get-data-frame-analytics.asciidoc | 3 + .../high-level/ml/get-datafeed.asciidoc | 3 + docs/java-rest/high-level/ml/get-job.asciidoc | 3 + .../apis/get-datafeed.asciidoc | 8 +- .../anomaly-detection/apis/get-job.asciidoc | 9 +- .../apis/get-dfanalytics.asciidoc | 60 ++-- .../apis/get-inference-trained-model.asciidoc | 4 +- docs/reference/ml/ml-shared.asciidoc | 6 + .../core/ml/datafeed/DatafeedConfig.java | 74 +++-- .../dataframe/DataFrameAnalyticsConfig.java | 31 ++- .../core/ml/inference/TrainedModelConfig.java | 2 +- .../xpack/core/ml/job/config/Job.java | 54 ++-- .../xpack/core/ml/utils/ToXContentParams.java | 8 + .../ml/integration/MlBasicMultiNodeIT.java | 259 ++++++++++++++++-- .../datafeeds/RestGetDatafeedsAction.java | 8 + .../RestGetDataFrameAnalyticsAction.java | 8 + .../inference/RestGetTrainedModelsAction.java | 3 +- .../xpack/ml/rest/job/RestGetJobsAction.java | 8 + .../api/ml.get_data_frame_analytics.json | 6 + .../rest-api-spec/api/ml.get_datafeeds.json | 6 + .../rest-api-spec/api/ml.get_jobs.json | 6 + .../test/ml/data_frame_analytics_crud.yml | 34 +++ .../rest-api-spec/test/ml/datafeeds_crud.yml | 20 ++ .../rest-api-spec/test/ml/jobs_get.yml | 16 ++ 29 files changed, 597 insertions(+), 121 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 758a170087865..113ccc814ff63 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -121,7 +121,10 @@ static Request getJob(GetJobRequest getJobRequest) { RequestConverters.Params params = new RequestConverters.Params(); if (getJobRequest.getAllowNoMatch() != null) { - params.putParam("allow_no_match", Boolean.toString(getJobRequest.getAllowNoMatch())); + params.putParam(GetJobRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getJobRequest.getAllowNoMatch())); + } + if (getJobRequest.getForExport() != null) { + params.putParam(GetJobRequest.FOR_EXPORT, Boolean.toString(getJobRequest.getForExport())); } request.addParameters(params.asMap()); return request; @@ -270,6 +273,9 @@ static Request getDatafeed(GetDatafeedRequest getDatafeedRequest) { params.putParam(GetDatafeedRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getDatafeedRequest.getAllowNoMatch())); } + if (getDatafeedRequest.getForExport() != null) { + params.putParam(GetDatafeedRequest.FOR_EXPORT, Boolean.toString(getDatafeedRequest.getForExport())); + } request.addParameters(params.asMap()); return request; } @@ -647,6 +653,9 @@ static Request getDataFrameAnalytics(GetDataFrameAnalyticsRequest getRequest) { if (getRequest.getAllowNoMatch() != null) { params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getRequest.getAllowNoMatch())); } + if (getRequest.getForExport() != null) { + params.putParam(GetDataFrameAnalyticsRequest.FOR_EXPORT, Boolean.toString(getRequest.getForExport())); + } request.addParameters(params.asMap()); return request; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java index 40698c4b528fa..6e85eda6e2f91 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java @@ -33,10 +33,12 @@ public class GetDataFrameAnalyticsRequest implements Validatable { public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); + public static final String FOR_EXPORT = "for_export"; private final List ids; private Boolean allowNoMatch; private PageParams pageParams; + private Boolean forExport; /** * Helper method to create a request that will get ALL Data Frame Analytics @@ -58,6 +60,22 @@ public Boolean getAllowNoMatch() { return allowNoMatch; } + /** + * Setting this flag to `true` removes certain fields from the configuration on retrieval. + * + * This is useful when getting the configuration and wanting to put it in another cluster. + * + * Default value is false. + * @param forExport Boolean value indicating if certain fields should be removed + */ + public void setForExport(boolean forExport) { + this.forExport = forExport; + } + + public Boolean getForExport() { + return forExport; + } + /** * Whether to ignore if a wildcard expression matches no data frame analytics. * @@ -94,11 +112,12 @@ public boolean equals(Object o) { GetDataFrameAnalyticsRequest other = (GetDataFrameAnalyticsRequest) o; return Objects.equals(ids, other.ids) && Objects.equals(allowNoMatch, other.allowNoMatch) + && Objects.equals(forExport, other.forExport) && Objects.equals(pageParams, other.pageParams); } @Override public int hashCode() { - return Objects.hash(ids, allowNoMatch, pageParams); + return Objects.hash(ids, allowNoMatch, forExport, pageParams); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java index 3408550c406cf..38e0b86ffc04f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java @@ -41,10 +41,12 @@ public class GetDatafeedRequest implements Validatable, ToXContentObject { public static final ParseField DATAFEED_IDS = new ParseField("datafeed_ids"); public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); + public static final String FOR_EXPORT = "for_export"; private static final String ALL_DATAFEEDS = "_all"; private final List datafeedIds; private Boolean allowNoMatch; + private Boolean forExport; @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -100,9 +102,25 @@ public Boolean getAllowNoMatch() { return allowNoMatch; } + /** + * Setting this flag to `true` removes certain fields from the configuration on retrieval. + * + * This is useful when getting the configuration and wanting to put it in another cluster. + * + * Default value is false. + * @param forExport Boolean value indicating if certain fields should be removed + */ + public void setForExport(boolean forExport) { + this.forExport = forExport; + } + + public Boolean getForExport() { + return forExport; + } + @Override public int hashCode() { - return Objects.hash(datafeedIds, allowNoMatch); + return Objects.hash(datafeedIds, forExport, allowNoMatch); } @Override @@ -117,7 +135,8 @@ public boolean equals(Object other) { GetDatafeedRequest that = (GetDatafeedRequest) other; return Objects.equals(datafeedIds, that.datafeedIds) && - Objects.equals(allowNoMatch, that.allowNoMatch); + Objects.equals(allowNoMatch, that.allowNoMatch) && + Objects.equals(forExport, that.forExport); } @Override diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java index d3079fb7360c2..6ad09de62fdab 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java @@ -42,10 +42,12 @@ public class GetJobRequest implements Validatable, ToXContentObject { public static final ParseField JOB_IDS = new ParseField("job_ids"); public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); + public static final String FOR_EXPORT = "for_export"; private static final String ALL_JOBS = "_all"; private final List jobIds; private Boolean allowNoMatch; + private Boolean forExport; @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -100,9 +102,25 @@ public Boolean getAllowNoMatch() { return allowNoMatch; } + /** + * Setting this flag to `true` removes certain fields from the configuration on retrieval. + * + * This is useful when getting the configuration and wanting to put it in another cluster. + * + * Default value is false. + * @param forExport Boolean value indicating if certain fields should be removed + */ + public void setForExport(boolean forExport) { + this.forExport = forExport; + } + + public Boolean getForExport() { + return forExport; + } + @Override public int hashCode() { - return Objects.hash(jobIds, allowNoMatch); + return Objects.hash(jobIds, forExport, allowNoMatch); } @Override @@ -117,6 +135,7 @@ public boolean equals(Object other) { GetJobRequest that = (GetJobRequest) other; return Objects.equals(jobIds, that.jobIds) && + Objects.equals(forExport, that.forExport) && Objects.equals(allowNoMatch, that.allowNoMatch); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index 00bc72c630237..c086968538047 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -340,6 +340,7 @@ public void testGetJob() throws Exception { // tag::get-job-request GetJobRequest request = new GetJobRequest("get-machine-learning-job1", "get-machine-learning-job*"); // <1> request.setAllowNoMatch(true); // <2> + request.setForExport(false); // <3> // end::get-job-request // tag::get-job-execute @@ -836,6 +837,7 @@ public void testGetDatafeed() throws Exception { // tag::get-datafeed-request GetDatafeedRequest request = new GetDatafeedRequest(datafeedId); // <1> request.setAllowNoMatch(true); // <2> + request.setForExport(false); // <3> // end::get-datafeed-request // tag::get-datafeed-execute @@ -2861,6 +2863,7 @@ public void testGetDataFrameAnalytics() throws Exception { { // tag::get-data-frame-analytics-request GetDataFrameAnalyticsRequest request = new GetDataFrameAnalyticsRequest("my-analytics-config"); // <1> + request.setForExport(false); // <2> // end::get-data-frame-analytics-request // tag::get-data-frame-analytics-execute diff --git a/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc index 2e956d43074ce..6ddadea25afd5 100644 --- a/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc +++ b/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc @@ -21,6 +21,9 @@ IDs, or the special wildcard `_all` to get all {dfanalytics-jobs}. include-tagged::{doc-tests-file}[{api}-request] -------------------------------------------------- <1> Constructing a new GET request referencing an existing {dfanalytics-job} +<2> Optional boolean value for requesting the {dfanalytics-job} in a format that can +then be put into another cluster. Certain fields that can only be set when +the {dfanalytics-job} is created are removed. include::../execution.asciidoc[] diff --git a/docs/java-rest/high-level/ml/get-datafeed.asciidoc b/docs/java-rest/high-level/ml/get-datafeed.asciidoc index 0aa5ec2ec0dcf..25eda75d15c38 100644 --- a/docs/java-rest/high-level/ml/get-datafeed.asciidoc +++ b/docs/java-rest/high-level/ml/get-datafeed.asciidoc @@ -25,6 +25,9 @@ include-tagged::{doc-tests-file}[{api}-request] contain wildcards. <2> Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified). +<3> Optional boolean value for requesting the datafeed in a format that can +then be put into another cluster. Certain fields that can only be set when +the datafeed is created are removed. [id="{upid}-{api}-response"] ==== Get datafeed response diff --git a/docs/java-rest/high-level/ml/get-job.asciidoc b/docs/java-rest/high-level/ml/get-job.asciidoc index 3fde9b98f310a..e42995edcf8d7 100644 --- a/docs/java-rest/high-level/ml/get-job.asciidoc +++ b/docs/java-rest/high-level/ml/get-job.asciidoc @@ -25,6 +25,9 @@ include-tagged::{doc-tests-file}[{api}-request] wildcards. <2> Whether to ignore if a wildcard expression matches no {anomaly-jobs}. (This includes `_all` string or when no jobs have been specified). +<3> Optional boolean value for requesting the job in a format that can +then be put into another cluster. Certain fields that can only be set when +the job is created are removed. [id="{upid}-{api}-response"] ==== Get {anomaly-jobs} response diff --git a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc index 57b3d9d51b8f2..e35032e6cb4d4 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc @@ -19,7 +19,7 @@ Retrieves configuration information for {dfeeds}. `GET _ml/datafeeds/` + -`GET _ml/datafeeds/_all` +`GET _ml/datafeeds/_all` [[ml-get-datafeed-prereqs]] == {api-prereq-title} @@ -36,7 +36,7 @@ comma-separated list of {dfeeds} or a wildcard expression. You can get information for all {dfeeds} by using `_all`, by specifying `*` as the ``, or by omitting the ``. -IMPORTANT: This API returns a maximum of 10,000 {dfeeds}. +IMPORTANT: This API returns a maximum of 10,000 {dfeeds}. [[ml-get-datafeed-path-parms]] == {api-path-parms-title} @@ -57,6 +57,10 @@ all {dfeeds}. (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-datafeeds] +`for_export`:: +(Optional, boolean) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] + [[ml-get-datafeed-results]] == {api-response-body-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc index af3bd514b91f9..71f597b792eb2 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc @@ -34,7 +34,7 @@ using a group name, a comma-separated list of jobs, or a wildcard expression. You can get information for all {anomaly-jobs} by using `_all`, by specifying `*` as the ``, or by omitting the ``. -IMPORTANT: This API returns a maximum of 10,000 jobs. +IMPORTANT: This API returns a maximum of 10,000 jobs. [[ml-get-job-path-parms]] == {api-path-parms-title} @@ -50,6 +50,9 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-defaul (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-jobs] +`for_export`:: +(Optional, boolean) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] [[ml-get-job-results]] == {api-response-body-title} @@ -60,7 +63,7 @@ properties, see <>. `create_time`:: (string) The time the job was created. For example, `1491007356077`. This property is informational; you cannot change its value. - + `finished_time`:: (string) If the job closed or failed, this is the time the job finished, otherwise it is `null`. This property is informational; you cannot change its @@ -80,7 +83,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-snapshot-id] == {api-response-codes-title} `404` (Missing resources):: - If `allow_no_match` is `false`, this code indicates that there are no + If `allow_no_match` is `false`, this code indicates that there are no resources that match the request or only partial matches for the request. [[ml-get-job-example]] diff --git a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc index aa3d66764b55c..99d7eb1f4a803 100644 --- a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc @@ -27,18 +27,18 @@ experimental[] [[ml-get-dfanalytics-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following +If the {es} {security-features} are enabled, you must have the following privileges: * cluster: `monitor_ml` - + For more information, see <> and {ml-docs-setup-privileges}. [[ml-get-dfanalytics-desc]] == {api-description-title} -You can get information for multiple {dfanalytics-jobs} in a single API request +You can get information for multiple {dfanalytics-jobs} in a single API request by using a comma-separated list of {dfanalytics-jobs} or a wildcard expression. @@ -46,12 +46,12 @@ by using a comma-separated list of {dfanalytics-jobs} or a wildcard expression. == {api-path-parms-title} ``:: -(Optional, string) +(Optional, string) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-default] + -- -You can get information for all {dfanalytics-jobs} by using _all, by specifying -`*` as the ``, or by omitting the +You can get information for all {dfanalytics-jobs} by using _all, by specifying +`*` as the ``, or by omitting the ``. -- @@ -60,24 +60,28 @@ You can get information for all {dfanalytics-jobs} by using _all, by specifying == {api-query-parms-title} `allow_no_match`:: -(Optional, boolean) +(Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match] `from`:: -(Optional, integer) +(Optional, integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=from] `size`:: -(Optional, integer) +(Optional, integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=size] +`for_export`:: +(Optional, boolean) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] + [role="child_attributes"] [[ml-get-dfanalytics-results]] == {api-response-body-title} `data_frame_analytics`:: (array) -An array of {dfanalytics-job} resources, which are sorted by the `id` value in +An array of {dfanalytics-job} resources, which are sorted by the `id` value in ascending order. + .Properties of {dfanalytics-job} resources @@ -88,18 +92,18 @@ ascending order. //Begin analyzed_fields `analyzed_fields`::: -(object) Contains `includes` and/or `excludes` patterns that select which fields +(object) Contains `includes` and/or `excludes` patterns that select which fields are included in the analysis. + .Properties of `analyzed_fields` [%collapsible%open] ===== `excludes`::: -(Optional, array) An array of strings that defines the fields that are excluded +(Optional, array) An array of strings that defines the fields that are excluded from the analysis. - + `includes`::: -(Optional, array) An array of strings that defines the fields that are included +(Optional, array) An array of strings that defines the fields that are included in the analysis. ===== //End analyzed_fields @@ -111,11 +115,11 @@ in the analysis. [%collapsible%open] ===== `index`::: -(string) The _destination index_ that stores the results of the +(string) The _destination index_ that stores the results of the {dfanalytics-job}. `results_field`::: -(string) The name of the field that stores the results of the analysis. Defaults +(string) The name of the field that stores the results of the analysis. Defaults to `ml`. ===== //End dest @@ -127,36 +131,36 @@ to `ml`. (string) The `model_memory_limit` that has been set to the {dfanalytics-job}. `source`::: -(object) The configuration of how the analysis data is sourced. It has an +(object) The configuration of how the analysis data is sourced. It has an `index` parameter and optionally a `query` and a `_source`. + .Properties of `source` [%collapsible%open] ===== `index`::: -(array) Index or indices on which to perform the analysis. It can be a single +(array) Index or indices on which to perform the analysis. It can be a single index or index pattern as well as an array of indices or patterns. - + `query`::: -(object) The query that has been specified for the {dfanalytics-job}. The {es} -query domain-specific language (<>). This value corresponds to -the query object in an {es} search POST body. By default, this property has the +(object) The query that has been specified for the {dfanalytics-job}. The {es} +query domain-specific language (<>). This value corresponds to +the query object in an {es} search POST body. By default, this property has the following value: `{"match_all": {}}`. `_source`::: -(object) Contains the specified `includes` and/or `excludes` patterns that -select which fields are present in the destination. Fields that are excluded +(object) Contains the specified `includes` and/or `excludes` patterns that +select which fields are present in the destination. Fields that are excluded cannot be included in the analysis. + .Properties of `_source` [%collapsible%open] ====== `excludes`::: -(array) An array of strings that defines the fields that are excluded from the +(array) An array of strings that defines the fields that are excluded from the destination. - + `includes`::: -(array) An array of strings that defines the fields that are included in the +(array) An array of strings that defines the fields that are included in the destination. ====== //End of _source @@ -176,7 +180,7 @@ destination. [[ml-get-dfanalytics-example]] == {api-examples-title} -The following example gets configuration information for the `loganalytics` +The following example gets configuration information for the `loganalytics` {dfanalytics-job}: [source,console] diff --git a/docs/reference/ml/df-analytics/apis/get-inference-trained-model.asciidoc b/docs/reference/ml/df-analytics/apis/get-inference-trained-model.asciidoc index 80450982f3711..02c051a355f16 100644 --- a/docs/reference/ml/df-analytics/apis/get-inference-trained-model.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-inference-trained-model.asciidoc @@ -67,9 +67,7 @@ Specifies whether the included model definition should be returned as a JSON map `for_export`:: (Optional, boolean) -Indicates if certain fields should be removed from the model configuration on -retrieval. This allows the model to be in an acceptable format to be retrieved -and then added to another cluster. Default is false. +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] `from`:: (Optional, integer) diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index 3dd1c37a168d1..685546d87bb5a 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -678,6 +678,12 @@ The number of individual forecasts currently available for the job. A value of `1` or more indicates that forecasts exist. end::forecast-total[] +tag::for-export[] +Indicates if certain fields should be removed from the configuration on +retrieval. This allows the configuration to be in an acceptable format to be retrieved +and then added to another cluster. Default is false. +end::for-export[] + tag::frequency[] The interval at which scheduled queries are made while the {dfeed} runs in real time. The default value is either the bucket span for short bucket spans, or, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index 2cac811dcd5fa..5936a103920d4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -49,6 +49,8 @@ import java.util.Random; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; + /** * Datafeed configuration options. Describes where to proactively pull input * data from. @@ -451,17 +453,47 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(ID.getPreferredName(), id); - builder.field(Job.ID.getPreferredName(), jobId); - if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { - builder.field(CONFIG_TYPE.getPreferredName(), TYPE); + if (params.paramAsBoolean(FOR_EXPORT, false) == false) { + builder.field(ID.getPreferredName(), id); + // We don't include the job_id in export as we assume the PUT will be referring to a new job as well + builder.field(Job.ID.getPreferredName(), jobId); + if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { + builder.field(CONFIG_TYPE.getPreferredName(), TYPE); + } + if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { + builder.field(HEADERS.getPreferredName(), headers); + } + builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep()); + builder.field(QUERY.getPreferredName(), queryProvider.getQuery()); + if (chunkingConfig != null) { + builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig); + } + builder.startObject(INDICES_OPTIONS.getPreferredName()); + indicesOptions.toXContent(builder, params); + builder.endObject(); + } else { // Don't include random defaults or unnecessary defauls in export + if (queryDelay.equals(defaultRandomQueryDelay()) == false) { + builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep()); + } + // This is always "match_all" + if (queryProvider.equals(QueryProvider.defaultQuery()) == false) { + builder.field(QUERY.getPreferredName(), queryProvider.getQuery()); + } + // Indices options are a pretty advanced feature, better to not include them if they are just the default ones + if (indicesOptions.equals(SearchRequest.DEFAULT_INDICES_OPTIONS) == false) { + builder.startObject(INDICES_OPTIONS.getPreferredName()); + indicesOptions.toXContent(builder, params); + builder.endObject(); + } + // Removing the default chunking config as it is determined by OTHER fields + if (chunkingConfig != null && chunkingConfig.equals(defaultChunkingConfig()) == false) { + builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig); + } } - builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep()); if (frequency != null) { builder.field(FREQUENCY.getPreferredName(), frequency.getStringRep()); } builder.field(INDICES.getPreferredName(), indices); - builder.field(QUERY.getPreferredName(), queryProvider.getQuery()); if (aggProvider != null) { builder.field(AGGREGATIONS.getPreferredName(), aggProvider.getAggs()); } @@ -473,26 +505,36 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); } builder.field(SCROLL_SIZE.getPreferredName(), scrollSize); - if (chunkingConfig != null) { - builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig); - } - if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { - builder.field(HEADERS.getPreferredName(), headers); - } if (delayedDataCheckConfig != null) { builder.field(DELAYED_DATA_CHECK_CONFIG.getPreferredName(), delayedDataCheckConfig); } if (maxEmptySearches != null) { builder.field(MAX_EMPTY_SEARCHES.getPreferredName(), maxEmptySearches); } - builder.startObject(INDICES_OPTIONS.getPreferredName()); - indicesOptions.toXContent(builder, params); - builder.endObject(); - builder.endObject(); return builder; } + private TimeValue defaultRandomQueryDelay() { + Random random = new Random(jobId.hashCode()); + long delayMillis = random.longs(Builder.MIN_DEFAULT_QUERY_DELAY.millis(), Builder.MAX_DEFAULT_QUERY_DELAY.millis()) + .findFirst().getAsLong(); + return TimeValue.timeValueMillis(delayMillis); + } + + private ChunkingConfig defaultChunkingConfig() { + if (aggProvider == null || aggProvider.getParsedAggs() == null) { + return ChunkingConfig.newAuto(); + } else { + long histogramIntervalMillis = ExtractorUtils.getHistogramIntervalMillis(aggProvider.getParsedAggs()); + if (histogramIntervalMillis <= 0) { + throw ExceptionsHelper.badRequestException(Messages.DATAFEED_AGGREGATIONS_INTERVAL_MUST_BE_GREATER_THAN_ZERO); + } + return ChunkingConfig.newManual(TimeValue.timeValueMillis( + Builder.DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis)); + } + } + /** * The lists of indices and types are compared for equality but they are not * sorted first so this test could fail simply because the indices and types diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java index a72dc95b96688..f904280ce1fce 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java @@ -35,6 +35,7 @@ import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING; import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.VALUE; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { @@ -227,34 +228,34 @@ public Integer getMaxNumThreads() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(ID.getPreferredName(), id); + if (params.paramAsBoolean(FOR_EXPORT, false) == false) { + builder.field(ID.getPreferredName(), id); + if (createTime != null) { + builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); + } + if (version != null) { + builder.field(VERSION.getPreferredName(), version); + } + if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { + builder.field(HEADERS.getPreferredName(), headers); + } + if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { + builder.field(CONFIG_TYPE.getPreferredName(), TYPE); + } + } if (description != null) { builder.field(DESCRIPTION.getPreferredName(), description); } builder.field(SOURCE.getPreferredName(), source); builder.field(DEST.getPreferredName(), dest); - builder.startObject(ANALYSIS.getPreferredName()); builder.field(analysis.getWriteableName(), analysis, new MapParams(Collections.singletonMap(VERSION.getPreferredName(), version == null ? null : version.toString()))); builder.endObject(); - - if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { - builder.field(CONFIG_TYPE.getPreferredName(), TYPE); - } if (analyzedFields != null) { builder.field(ANALYZED_FIELDS.getPreferredName(), analyzedFields); } builder.field(MODEL_MEMORY_LIMIT.getPreferredName(), getModelMemoryLimit().getStringRep()); - if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { - builder.field(HEADERS.getPreferredName(), headers); - } - if (createTime != null) { - builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); - } - if (version != null) { - builder.field(VERSION.getPreferredName(), version); - } builder.field(ALLOW_LAZY_START.getPreferredName(), allowLazyStart); builder.field(MAX_NUM_THREADS.getPreferredName(), maxNumThreads); builder.endObject(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java index 9ae67880215a1..fbea2045a9e28 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java @@ -45,6 +45,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xpack.core.ml.utils.NamedXContentObjectHelper.writeNamedObject; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class TrainedModelConfig implements ToXContentObject, Writeable { @@ -52,7 +53,6 @@ public class TrainedModelConfig implements ToXContentObject, Writeable { public static final String NAME = "trained_model_config"; public static final int CURRENT_DEFINITION_COMPRESSION_VERSION = 1; public static final String DECOMPRESS_DEFINITION = "decompress_definition"; - public static final String FOR_EXPORT = "for_export"; public static final String TOTAL_FEATURE_IMPORTANCE = "total_feature_importance"; private static final Set RESERVED_METADATA_FIELDS = Collections.singleton(TOTAL_FEATURE_IMPORTANCE); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index 69dc8ef880892..78cab7051fe98 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -33,6 +33,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -42,6 +43,8 @@ import java.util.TreeSet; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; + /** * This class represents a configured and created Job. The creation time is set * to the time the object was constructed and the finished time and last @@ -513,11 +516,35 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { final String humanReadableSuffix = "_string"; - - builder.field(ID.getPreferredName(), jobId); - builder.field(JOB_TYPE.getPreferredName(), jobType); - if (jobVersion != null) { - builder.field(JOB_VERSION.getPreferredName(), jobVersion); + if (params.paramAsBoolean(FOR_EXPORT, false) == false) { + builder.field(ID.getPreferredName(), jobId); + builder.field(JOB_TYPE.getPreferredName(), jobType); + if (jobVersion != null) { + builder.field(JOB_VERSION.getPreferredName(), jobVersion); + } + builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + humanReadableSuffix, createTime.getTime()); + if (finishedTime != null) { + builder.timeField(FINISHED_TIME.getPreferredName(), FINISHED_TIME.getPreferredName() + humanReadableSuffix, + finishedTime.getTime()); + } + if (modelSnapshotId != null) { + builder.field(MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshotId); + } + if (deleting) { + builder.field(DELETING.getPreferredName(), deleting); + } + if (modelSnapshotMinVersion != null) { + builder.field(MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(), modelSnapshotMinVersion); + } + if (customSettings != null) { + builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings); + } + } else { + if (customSettings != null) { + HashMap newCustomSettings = new HashMap<>(customSettings); + newCustomSettings.remove("created_by"); + builder.field(CUSTOM_SETTINGS.getPreferredName(), newCustomSettings); + } } if (groups.isEmpty() == false) { builder.field(GROUPS.getPreferredName(), groups); @@ -525,11 +552,6 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th if (description != null) { builder.field(DESCRIPTION.getPreferredName(), description); } - builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + humanReadableSuffix, createTime.getTime()); - if (finishedTime != null) { - builder.timeField(FINISHED_TIME.getPreferredName(), FINISHED_TIME.getPreferredName() + humanReadableSuffix, - finishedTime.getTime()); - } builder.field(ANALYSIS_CONFIG.getPreferredName(), analysisConfig, params); if (analysisLimits != null) { builder.field(ANALYSIS_LIMITS.getPreferredName(), analysisLimits, params); @@ -555,19 +577,7 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th if (resultsRetentionDays != null) { builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); } - if (customSettings != null) { - builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings); - } - if (modelSnapshotId != null) { - builder.field(MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshotId); - } - if (modelSnapshotMinVersion != null) { - builder.field(MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(), modelSnapshotMinVersion); - } builder.field(RESULTS_INDEX_NAME.getPreferredName(), resultsIndexName); - if (deleting) { - builder.field(DELETING.getPreferredName(), deleting); - } builder.field(ALLOW_LAZY_OPEN.getPreferredName(), allowLazyOpen); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java index 84a4c26d43931..b755c13ce18f6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java @@ -18,6 +18,14 @@ public final class ToXContentParams { */ public static final String FOR_INTERNAL_STORAGE = "for_internal_storage"; + /** + * Parameter to indicate if this XContent serialization should only include fields that are allowed to be used + * on PUT + * + * This helps to GET a configuration, copy it, and then PUT it directly without removing or changing any fields in between + */ + public static final String FOR_EXPORT = "for_export"; + /** * When serialising POJOs to X Content this indicates whether the calculated (i.e. not stored) fields * should be included or not diff --git a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java index dcfd97ff4fc5f..522b6d4f452f4 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java +++ b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -106,31 +107,7 @@ public void testMiniFarequote() throws Exception { } public void testMiniFarequoteWithDatafeeder() throws Exception { - boolean datesHaveNanoSecondResolution = randomBoolean(); - String dateMappingType = datesHaveNanoSecondResolution ? "date_nanos" : "date"; - String dateFormat = datesHaveNanoSecondResolution ? "strict_date_optional_time_nanos" : "strict_date_optional_time"; - String randomNanos = datesHaveNanoSecondResolution ? "," + randomIntBetween(100000000, 999999999) : ""; - Request createAirlineDataRequest = new Request("PUT", "/airline-data"); - createAirlineDataRequest.setJsonEntity("{" - + " \"mappings\": {" - + " \"properties\": {" - + " \"time\": { \"type\":\"" + dateMappingType + "\", \"format\":\"" + dateFormat + "\"}," - + " \"airline\": { \"type\":\"keyword\"}," - + " \"responsetime\": { \"type\":\"float\"}" - + " }" - + " }" - + "}"); - client().performRequest(createAirlineDataRequest); - Request airlineData1 = new Request("PUT", "/airline-data/_doc/1"); - airlineData1.setJsonEntity("{\"time\":\"2016-06-01T00:00:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":135.22}"); - client().performRequest(airlineData1); - Request airlineData2 = new Request("PUT", "/airline-data/_doc/2"); - airlineData2.setJsonEntity("{\"time\":\"2016-06-01T01:59:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":541.76}"); - client().performRequest(airlineData2); - - // Ensure all data is searchable - refreshAllIndices(); - + createAndIndexFarequote(); String jobId = "mini-farequote-with-data-feeder-job"; createFarequoteJob(jobId); String datafeedId = "bar"; @@ -265,6 +242,211 @@ public void testMiniFarequoteReopen() throws Exception { client().performRequest(new Request("DELETE", BASE_PATH + "anomaly_detectors/" + jobId)); } + @SuppressWarnings("unchecked") + public void testExportAndPutJob() throws Exception { + String jobId = "test-export-import-job"; + createFarequoteJob(jobId); + Response jobResponse = client().performRequest( + new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "?for_export=true")); + Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("jobs")).get(0); + + XContentBuilder xContentBuilder = jsonBuilder().map(originalJobBody); + Request request = new Request("PUT", BASE_PATH + "anomaly_detectors/" + jobId + "-import"); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response importedJobResponse = client().performRequest( + new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "-import" + "?for_export=true")); + Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse).get("jobs")).get(0); + assertThat(originalJobBody, equalTo(importedJobBody)); + } + + @SuppressWarnings("unchecked") + public void testExportAndPutDatafeed() throws Exception { + createAndIndexFarequote(); + String jobId = "test-export-import-datafeed"; + createFarequoteJob(jobId); + String datafeedId = jobId + "-datafeed"; + createDatafeed(datafeedId, jobId); + + Response dfResponse = client().performRequest( + new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "?for_export=true")); + Map originalDfBody = (Map)((List) entityAsMap(dfResponse).get("jobs")).get(0); + + //Delete this so we can PUT another datafeed for the same job + client().performRequest(new Request("DELETE", BASE_PATH + "datafeeds/" + datafeedId)); + + Map toPut = new HashMap<>(originalDfBody); + toPut.put("job_id", jobId); + XContentBuilder xContentBuilder = jsonBuilder().map(toPut); + Request request = new Request("PUT", BASE_PATH + "datafeeds/" + datafeedId + "-import"); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response importedDfResponse = client().performRequest( + new Request("GET", BASE_PATH + "anomaly_detectors/" + datafeedId + "-import" + "?for_export=true")); + Map importedDfBody = (Map)((List) entityAsMap(importedDfResponse).get("jobs")).get(0); + assertThat(originalDfBody, equalTo(importedDfBody)); + } + + @SuppressWarnings("unchecked") + public void testExportAndPutDataFrameAnalytics_OutlierDetection() throws Exception { + createAndIndexFarequote(); + String analyticsId = "outlier-export-import"; + XContentBuilder xContentBuilder = jsonBuilder(); + xContentBuilder.startObject(); + { + xContentBuilder.field("description", "outlier analytics"); + + xContentBuilder.startObject("source"); + { + xContentBuilder.field("index", "airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("dest"); + { + xContentBuilder.field("index", "outliers-airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("analysis"); + { + xContentBuilder.startObject("outlier_detection"); + { + xContentBuilder.field("compute_feature_influence", false); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + + Request request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response jobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true")); + Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + + XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); + request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); + request.setJsonEntity(Strings.toString(newBuilder)); + client().performRequest(request); + + Response importedJobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true")); + Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) + .get("data_frame_analytics")) + .get(0); + assertThat(originalJobBody, equalTo(importedJobBody)); + } + + @SuppressWarnings("unchecked") + public void testExportAndPutDataFrameAnalytics_Regression() throws Exception { + createAndIndexFarequote(); + String analyticsId = "regression-export-import"; + XContentBuilder xContentBuilder = jsonBuilder(); + xContentBuilder.startObject(); + { + xContentBuilder.field("description", "regression analytics"); + + xContentBuilder.startObject("source"); + { + xContentBuilder.field("index", "airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("dest"); + { + xContentBuilder.field("index", "regression-airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("analysis"); + { + xContentBuilder.startObject("regression"); + { + xContentBuilder.field("dependent_variable", "responsetime"); + xContentBuilder.field("training_percent", 50); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + + Request request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response jobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true")); + Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + + XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); + request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); + request.setJsonEntity(Strings.toString(newBuilder)); + client().performRequest(request); + + Response importedJobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true")); + Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) + .get("data_frame_analytics")) + .get(0); + assertThat(originalJobBody, equalTo(importedJobBody)); + } + + @SuppressWarnings("unchecked") + public void testExportAndPutDataFrameAnalytics_Classification() throws Exception { + createAndIndexFarequote(); + String analyticsId = "classification-export-import"; + XContentBuilder xContentBuilder = jsonBuilder(); + xContentBuilder.startObject(); + { + xContentBuilder.field("description", "classification analytics"); + + xContentBuilder.startObject("source"); + { + xContentBuilder.field("index", "airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("dest"); + { + xContentBuilder.field("index", "classification-airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("analysis"); + { + xContentBuilder.startObject("classification"); + { + xContentBuilder.field("dependent_variable", "airline"); + xContentBuilder.field("training_percent", 60); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + + Request request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response jobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true")); + Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + + XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); + request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); + request.setJsonEntity(Strings.toString(newBuilder)); + client().performRequest(request); + + Response importedJobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true")); + Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) + .get("data_frame_analytics")) + .get(0); + assertThat(originalJobBody, equalTo(importedJobBody)); + } + private Response createDatafeed(String datafeedId, String jobId) throws Exception { XContentBuilder xContentBuilder = jsonBuilder(); xContentBuilder.startObject(); @@ -322,4 +504,31 @@ private static void assertFlushResponse(Response response, boolean expectedFlush assertThat(asMap.get("flushed"), is(true)); assertThat(asMap.get("last_finalized_bucket_end"), equalTo(expectedLastFinalizedBucketEnd)); } + + private void createAndIndexFarequote() throws Exception { + boolean datesHaveNanoSecondResolution = randomBoolean(); + String dateMappingType = datesHaveNanoSecondResolution ? "date_nanos" : "date"; + String dateFormat = datesHaveNanoSecondResolution ? "strict_date_optional_time_nanos" : "strict_date_optional_time"; + String randomNanos = datesHaveNanoSecondResolution ? "," + randomIntBetween(100000000, 999999999) : ""; + Request createAirlineDataRequest = new Request("PUT", "/airline-data"); + createAirlineDataRequest.setJsonEntity("{" + + " \"mappings\": {" + + " \"properties\": {" + + " \"time\": { \"type\":\"" + dateMappingType + "\", \"format\":\"" + dateFormat + "\"}," + + " \"airline\": { \"type\":\"keyword\"}," + + " \"responsetime\": { \"type\":\"float\"}" + + " }" + + " }" + + "}"); + client().performRequest(createAirlineDataRequest); + Request airlineData1 = new Request("PUT", "/airline-data/_doc/1"); + airlineData1.setJsonEntity("{\"time\":\"2016-06-01T00:00:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":135.22}"); + client().performRequest(airlineData1); + Request airlineData2 = new Request("PUT", "/airline-data/_doc/2"); + airlineData2.setJsonEntity("{\"time\":\"2016-06-01T01:59:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":541.76}"); + client().performRequest(airlineData2); + + // Ensure all data is searchable + refreshAllIndices(); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java index 6eeb50a8c2bea..35ca39a4e05c0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java @@ -16,9 +16,12 @@ import org.elasticsearch.xpack.ml.MachineLearning; import java.io.IOException; +import java.util.Collections; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class RestGetDatafeedsAction extends BaseRestHandler { @@ -51,4 +54,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient restRequest.paramAsBoolean(Request.ALLOW_NO_DATAFEEDS, request.allowNoMatch()))); return channel -> client.execute(GetDatafeedsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } + + @Override + protected Set responseParams() { + return Collections.singleton(FOR_EXPORT); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java index 6d6c8f3676556..140b00f0a0bdd 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java @@ -16,9 +16,12 @@ import org.elasticsearch.xpack.ml.MachineLearning; import java.io.IOException; +import java.util.Collections; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class RestGetDataFrameAnalyticsAction extends BaseRestHandler { @@ -49,4 +52,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient request.isAllowNoResources())); return channel -> client.execute(GetDataFrameAnalyticsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } + + @Override + protected Set responseParams() { + return Collections.singleton(FOR_EXPORT); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java index 85ced29d76dbf..805f2e5ac5a33 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java @@ -34,6 +34,7 @@ import static java.util.Arrays.asList; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction.Request.ALLOW_NO_MATCH; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class RestGetTrainedModelsAction extends BaseRestHandler { @@ -89,7 +90,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient @Override protected Set responseParams() { - return Set.of(TrainedModelConfig.DECOMPRESS_DEFINITION, TrainedModelConfig.FOR_EXPORT); + return Set.of(TrainedModelConfig.DECOMPRESS_DEFINITION, FOR_EXPORT); } private static class RestToXContentListenerWithDefaultValues extends RestToXContentListener { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java index 2ee8605dc6e92..a90df97bddd44 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java @@ -18,9 +18,12 @@ import org.elasticsearch.xpack.ml.MachineLearning; import java.io.IOException; +import java.util.Collections; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class RestGetJobsAction extends BaseRestHandler { @@ -53,4 +56,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient restRequest.paramAsBoolean(Request.ALLOW_NO_JOBS, request.allowNoMatch()))); return channel -> client.execute(GetJobsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } + + @Override + protected Set responseParams() { + return Collections.singleton(FOR_EXPORT); + } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json index 80b4649453f7f..5bd3cd20b4137 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json @@ -43,6 +43,12 @@ "type":"int", "description":"specifies a max number of analytics to get", "default":100 + }, + "for_export": { + "required": false, + "type": "boolean", + "default": false, + "description": "Omits fields that are illegal to set on data frame analytics PUT" } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json index 56f5ea49f8525..b7a3760e56a3a 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json @@ -38,6 +38,12 @@ "required":false, "description":"Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified)", "deprecated":true + }, + "for_export": { + "required": false, + "type": "boolean", + "default": false, + "description": "Omits fields that are illegal to set on datafeed PUT" } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json index 7a1ebaed08ceb..ae3f0299747ad 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json @@ -38,6 +38,12 @@ "required":false, "description":"Whether to ignore if a wildcard expression matches no jobs. (This includes `_all` string or when no jobs have been specified)", "deprecated":true + }, + "for_export": { + "required": false, + "type": "boolean", + "default": false, + "description": "Omits fields that are illegal to set on job PUT" } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml index b5816f80829ac..ae4f71007d4c9 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml @@ -2168,3 +2168,37 @@ setup: { "description": "blah" } + +--- +"Test GET config for export": + + - do: + ml.put_data_frame_analytics: + id: "simple-outlier-detection" + body: > + { + "source": { + "index": "index-source" + }, + "dest": { + "index": "index-dest" + }, + "analysis": {"outlier_detection":{}} + } + - do: + ml.get_data_frame_analytics: + id: "simple-outlier-detection" + for_export: true + - match: { source.index: ["index-source"] } + - match: { source.query: {"match_all" : {} } } + - match: { dest.index: "index-dest" } + - match: { analysis: { + "outlier_detection":{ + "compute_feature_influence": true, + "outlier_fraction": 0.05, + "standardization_enabled": true + } + }} + - is_false: create_time + - is_false: version + - is_false: id diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml index b105576312e22..9b740c0c9a039 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml @@ -506,3 +506,23 @@ setup: - match: { datafeeds.0.indices_options.ignore_throttled: false } - match: { datafeeds.0.indices_options.allow_no_indices: false } +--- +"Test get datafeed for export": + - do: + ml.put_datafeed: + datafeed_id: test-for-export + body: > + { + "job_id":"datafeeds-crud-1", + "indexes":["index-foo"] + } + - do: + ml.get_datafeeds: + datafeed_id: test-for-export + for_export: true + - match: { datafeeds.0.indices.0: "index-foo"} + - is_false: datafeeds.0.datafeed_id + - is_false: datafeeds.0.job_id + - is_false: datafeeds.0.create_time + - is_false: datafeeds.0.query_delay + - is_false: datafeeds.0.chunking_config diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml index 1ef10bfec1472..b305e73069efc 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml @@ -86,3 +86,19 @@ setup: - match: { jobs.0.description: "Job 1"} - match: { jobs.1.job_id: "jobs-get-2"} - match: { jobs.1.description: "Job 2"} +--- +"Test get job for export": + + - do: + ml.get_jobs: + job_id: jobs-get-1 + for_export: true + - match: { jobs.0.description: "Job 1"} + - is_false: job_id + - is_false: job_type + - is_false: job_version + - is_false: create_time + - is_false: finished_time + - is_false: model_snapshot_id + - is_false: model_snapshot_min_version + - is_false: deleting From fd78a818c3b30ef8e6ea1aebbb5af524ee67f5a3 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Wed, 30 Sep 2020 14:11:35 -0400 Subject: [PATCH 2/4] fixing test --- .../xpack/ml/integration/MlBasicMultiNodeIT.java | 6 +++--- .../test/ml/data_frame_analytics_crud.yml | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java index 522b6d4f452f4..2acb421c6e9d2 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java +++ b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java @@ -271,7 +271,7 @@ public void testExportAndPutDatafeed() throws Exception { Response dfResponse = client().performRequest( new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "?for_export=true")); - Map originalDfBody = (Map)((List) entityAsMap(dfResponse).get("jobs")).get(0); + Map originalDfBody = (Map)((List) entityAsMap(dfResponse).get("datafeeds")).get(0); //Delete this so we can PUT another datafeed for the same job client().performRequest(new Request("DELETE", BASE_PATH + "datafeeds/" + datafeedId)); @@ -284,8 +284,8 @@ public void testExportAndPutDatafeed() throws Exception { client().performRequest(request); Response importedDfResponse = client().performRequest( - new Request("GET", BASE_PATH + "anomaly_detectors/" + datafeedId + "-import" + "?for_export=true")); - Map importedDfBody = (Map)((List) entityAsMap(importedDfResponse).get("jobs")).get(0); + new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "-import" + "?for_export=true")); + Map importedDfBody = (Map)((List) entityAsMap(importedDfResponse).get("datafeeds")).get(0); assertThat(originalDfBody, equalTo(importedDfBody)); } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml index ae4f71007d4c9..722f7d6512e37 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml @@ -2189,16 +2189,16 @@ setup: ml.get_data_frame_analytics: id: "simple-outlier-detection" for_export: true - - match: { source.index: ["index-source"] } - - match: { source.query: {"match_all" : {} } } - - match: { dest.index: "index-dest" } - - match: { analysis: { + - match: { data_frame_analytics.0.source.index.0: "index-source" } + - match: { data_frame_analytics.0.source.query: {"match_all" : {} } } + - match: { data_frame_analytics.0.dest.index: "index-dest" } + - match: { data_frame_analytics.0.analysis: { "outlier_detection":{ "compute_feature_influence": true, "outlier_fraction": 0.05, "standardization_enabled": true } }} - - is_false: create_time - - is_false: version - - is_false: id + - is_false: data_frame_analytics.0.create_time + - is_false: data_frame_analytics.0.version + - is_false: data_frame_analytics.0.id From fa4ed0bcf916fb3598b6e24aab33bb14f2f488a9 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Thu, 1 Oct 2020 08:32:36 -0400 Subject: [PATCH 3/4] addressing pr comments --- .../client/MLRequestConverters.java | 2 +- .../ml/GetDataFrameAnalyticsRequest.java | 2 +- .../apis/get-datafeed.asciidoc | 4 +- .../anomaly-detection/apis/get-job.asciidoc | 6 +- .../apis/get-dfanalytics.asciidoc | 56 +++++++++---------- .../core/ml/datafeed/DatafeedConfig.java | 46 +++++---------- .../chunked/ChunkedDataExtractor.java | 4 +- 7 files changed, 51 insertions(+), 69 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 113ccc814ff63..07c82889f37d7 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -651,7 +651,7 @@ static Request getDataFrameAnalytics(GetDataFrameAnalyticsRequest getRequest) { } } if (getRequest.getAllowNoMatch() != null) { - params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getRequest.getAllowNoMatch())); + params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH, Boolean.toString(getRequest.getAllowNoMatch())); } if (getRequest.getForExport() != null) { params.putParam(GetDataFrameAnalyticsRequest.FOR_EXPORT, Boolean.toString(getRequest.getForExport())); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java index 6e85eda6e2f91..0d616614284e1 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java @@ -32,7 +32,7 @@ public class GetDataFrameAnalyticsRequest implements Validatable { - public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); + public static final String ALLOW_NO_MATCH = "allow_no_match"; public static final String FOR_EXPORT = "for_export"; private final List ids; diff --git a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc index e35032e6cb4d4..ed6442c66ec68 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc @@ -19,7 +19,7 @@ Retrieves configuration information for {dfeeds}. `GET _ml/datafeeds/` + -`GET _ml/datafeeds/_all` +`GET _ml/datafeeds/_all` [[ml-get-datafeed-prereqs]] == {api-prereq-title} @@ -36,7 +36,7 @@ comma-separated list of {dfeeds} or a wildcard expression. You can get information for all {dfeeds} by using `_all`, by specifying `*` as the ``, or by omitting the ``. -IMPORTANT: This API returns a maximum of 10,000 {dfeeds}. +IMPORTANT: This API returns a maximum of 10,000 {dfeeds}. [[ml-get-datafeed-path-parms]] == {api-path-parms-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc index 71f597b792eb2..668c95c9891a2 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc @@ -34,7 +34,7 @@ using a group name, a comma-separated list of jobs, or a wildcard expression. You can get information for all {anomaly-jobs} by using `_all`, by specifying `*` as the ``, or by omitting the ``. -IMPORTANT: This API returns a maximum of 10,000 jobs. +IMPORTANT: This API returns a maximum of 10,000 jobs. [[ml-get-job-path-parms]] == {api-path-parms-title} @@ -63,7 +63,7 @@ properties, see <>. `create_time`:: (string) The time the job was created. For example, `1491007356077`. This property is informational; you cannot change its value. - + `finished_time`:: (string) If the job closed or failed, this is the time the job finished, otherwise it is `null`. This property is informational; you cannot change its @@ -83,7 +83,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-snapshot-id] == {api-response-codes-title} `404` (Missing resources):: - If `allow_no_match` is `false`, this code indicates that there are no + If `allow_no_match` is `false`, this code indicates that there are no resources that match the request or only partial matches for the request. [[ml-get-job-example]] diff --git a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc index 99d7eb1f4a803..a6adac16dbef5 100644 --- a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc @@ -27,18 +27,18 @@ experimental[] [[ml-get-dfanalytics-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following +If the {es} {security-features} are enabled, you must have the following privileges: * cluster: `monitor_ml` - + For more information, see <> and {ml-docs-setup-privileges}. [[ml-get-dfanalytics-desc]] == {api-description-title} -You can get information for multiple {dfanalytics-jobs} in a single API request +You can get information for multiple {dfanalytics-jobs} in a single API request by using a comma-separated list of {dfanalytics-jobs} or a wildcard expression. @@ -46,12 +46,12 @@ by using a comma-separated list of {dfanalytics-jobs} or a wildcard expression. == {api-path-parms-title} ``:: -(Optional, string) +(Optional, string) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-default] + -- -You can get information for all {dfanalytics-jobs} by using _all, by specifying -`*` as the ``, or by omitting the +You can get information for all {dfanalytics-jobs} by using _all, by specifying +`*` as the ``, or by omitting the ``. -- @@ -60,15 +60,15 @@ You can get information for all {dfanalytics-jobs} by using _all, by specifying == {api-query-parms-title} `allow_no_match`:: -(Optional, boolean) +(Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match] `from`:: -(Optional, integer) +(Optional, integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=from] `size`:: -(Optional, integer) +(Optional, integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=size] `for_export`:: @@ -81,7 +81,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] `data_frame_analytics`:: (array) -An array of {dfanalytics-job} resources, which are sorted by the `id` value in +An array of {dfanalytics-job} resources, which are sorted by the `id` value in ascending order. + .Properties of {dfanalytics-job} resources @@ -92,18 +92,18 @@ ascending order. //Begin analyzed_fields `analyzed_fields`::: -(object) Contains `includes` and/or `excludes` patterns that select which fields +(object) Contains `includes` and/or `excludes` patterns that select which fields are included in the analysis. + .Properties of `analyzed_fields` [%collapsible%open] ===== `excludes`::: -(Optional, array) An array of strings that defines the fields that are excluded +(Optional, array) An array of strings that defines the fields that are excluded from the analysis. - + `includes`::: -(Optional, array) An array of strings that defines the fields that are included +(Optional, array) An array of strings that defines the fields that are included in the analysis. ===== //End analyzed_fields @@ -115,11 +115,11 @@ in the analysis. [%collapsible%open] ===== `index`::: -(string) The _destination index_ that stores the results of the +(string) The _destination index_ that stores the results of the {dfanalytics-job}. `results_field`::: -(string) The name of the field that stores the results of the analysis. Defaults +(string) The name of the field that stores the results of the analysis. Defaults to `ml`. ===== //End dest @@ -131,36 +131,36 @@ to `ml`. (string) The `model_memory_limit` that has been set to the {dfanalytics-job}. `source`::: -(object) The configuration of how the analysis data is sourced. It has an +(object) The configuration of how the analysis data is sourced. It has an `index` parameter and optionally a `query` and a `_source`. + .Properties of `source` [%collapsible%open] ===== `index`::: -(array) Index or indices on which to perform the analysis. It can be a single +(array) Index or indices on which to perform the analysis. It can be a single index or index pattern as well as an array of indices or patterns. - + `query`::: -(object) The query that has been specified for the {dfanalytics-job}. The {es} -query domain-specific language (<>). This value corresponds to -the query object in an {es} search POST body. By default, this property has the +(object) The query that has been specified for the {dfanalytics-job}. The {es} +query domain-specific language (<>). This value corresponds to +the query object in an {es} search POST body. By default, this property has the following value: `{"match_all": {}}`. `_source`::: -(object) Contains the specified `includes` and/or `excludes` patterns that -select which fields are present in the destination. Fields that are excluded +(object) Contains the specified `includes` and/or `excludes` patterns that +select which fields are present in the destination. Fields that are excluded cannot be included in the analysis. + .Properties of `_source` [%collapsible%open] ====== `excludes`::: -(array) An array of strings that defines the fields that are excluded from the +(array) An array of strings that defines the fields that are excluded from the destination. - + `includes`::: -(array) An array of strings that defines the fields that are included in the +(array) An array of strings that defines the fields that are included in the destination. ====== //End of _source @@ -180,7 +180,7 @@ destination. [[ml-get-dfanalytics-example]] == {api-examples-title} -The following example gets configuration information for the `loganalytics` +The following example gets configuration information for the `loganalytics` {dfanalytics-job}: [source,console] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index 5936a103920d4..e711ba2a71e64 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.AbstractDiffable; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -67,6 +68,9 @@ public class DatafeedConfig extends AbstractDiffable implements private static final int TWO_MINS_SECONDS = 2 * SECONDS_IN_MINUTE; private static final int TWENTY_MINS_SECONDS = 20 * SECONDS_IN_MINUTE; private static final int HALF_DAY_SECONDS = 12 * 60 * SECONDS_IN_MINUTE; + public static final int DEFAULT_AGGREGATION_CHUNKING_BUCKETS = 1000; + private static final TimeValue MIN_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(1); + private static final TimeValue MAX_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(2); private static final Logger logger = LogManager.getLogger(DatafeedConfig.class); @@ -464,21 +468,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(HEADERS.getPreferredName(), headers); } builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep()); - builder.field(QUERY.getPreferredName(), queryProvider.getQuery()); if (chunkingConfig != null) { builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig); } builder.startObject(INDICES_OPTIONS.getPreferredName()); indicesOptions.toXContent(builder, params); builder.endObject(); - } else { // Don't include random defaults or unnecessary defauls in export - if (queryDelay.equals(defaultRandomQueryDelay()) == false) { + } else { // Don't include random defaults or unnecessary defaults in export + if (queryDelay.equals(defaultRandomQueryDelay(jobId)) == false) { builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep()); } - // This is always "match_all" - if (queryProvider.equals(QueryProvider.defaultQuery()) == false) { - builder.field(QUERY.getPreferredName(), queryProvider.getQuery()); - } // Indices options are a pretty advanced feature, better to not include them if they are just the default ones if (indicesOptions.equals(SearchRequest.DEFAULT_INDICES_OPTIONS) == false) { builder.startObject(INDICES_OPTIONS.getPreferredName()); @@ -486,10 +485,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); } // Removing the default chunking config as it is determined by OTHER fields - if (chunkingConfig != null && chunkingConfig.equals(defaultChunkingConfig()) == false) { + if (chunkingConfig != null && chunkingConfig.equals(defaultChunkingConfig(aggProvider)) == false) { builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig); } } + builder.field(QUERY.getPreferredName(), queryProvider.getQuery()); if (frequency != null) { builder.field(FREQUENCY.getPreferredName(), frequency.getStringRep()); } @@ -515,14 +515,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - private TimeValue defaultRandomQueryDelay() { + private static TimeValue defaultRandomQueryDelay(String jobId) { Random random = new Random(jobId.hashCode()); - long delayMillis = random.longs(Builder.MIN_DEFAULT_QUERY_DELAY.millis(), Builder.MAX_DEFAULT_QUERY_DELAY.millis()) - .findFirst().getAsLong(); + long delayMillis = random.longs(MIN_DEFAULT_QUERY_DELAY.millis(), MAX_DEFAULT_QUERY_DELAY.millis()).findFirst().getAsLong(); return TimeValue.timeValueMillis(delayMillis); } - private ChunkingConfig defaultChunkingConfig() { + private static ChunkingConfig defaultChunkingConfig(@Nullable AggProvider aggProvider) { if (aggProvider == null || aggProvider.getParsedAggs() == null) { return ChunkingConfig.newAuto(); } else { @@ -530,8 +529,7 @@ private ChunkingConfig defaultChunkingConfig() { if (histogramIntervalMillis <= 0) { throw ExceptionsHelper.badRequestException(Messages.DATAFEED_AGGREGATIONS_INTERVAL_MUST_BE_GREATER_THAN_ZERO); } - return ChunkingConfig.newManual(TimeValue.timeValueMillis( - Builder.DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis)); + return ChunkingConfig.newManual(TimeValue.timeValueMillis(DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis)); } } @@ -628,10 +626,6 @@ private TimeValue defaultFrequencyTarget(TimeValue bucketSpan) { public static class Builder { - public static final int DEFAULT_AGGREGATION_CHUNKING_BUCKETS = 1000; - private static final TimeValue MIN_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(1); - private static final TimeValue MAX_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(2); - private String id; private String jobId; private TimeValue queryDelay; @@ -868,25 +862,13 @@ private static void checkHistogramIntervalIsPositive(AggregationBuilder histogra private void setDefaultChunkingConfig() { if (chunkingConfig == null) { - if (aggProvider == null || aggProvider.getParsedAggs() == null) { - chunkingConfig = ChunkingConfig.newAuto(); - } else { - long histogramIntervalMillis = ExtractorUtils.getHistogramIntervalMillis(aggProvider.getParsedAggs()); - if (histogramIntervalMillis <= 0) { - throw ExceptionsHelper.badRequestException(Messages.DATAFEED_AGGREGATIONS_INTERVAL_MUST_BE_GREATER_THAN_ZERO); - } - chunkingConfig = ChunkingConfig.newManual(TimeValue.timeValueMillis( - DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis)); - } + chunkingConfig = defaultChunkingConfig(aggProvider); } } private void setDefaultQueryDelay() { if (queryDelay == null) { - Random random = new Random(jobId.hashCode()); - long delayMillis = random.longs(MIN_DEFAULT_QUERY_DELAY.millis(), MAX_DEFAULT_QUERY_DELAY.millis()) - .findFirst().getAsLong(); - queryDelay = TimeValue.timeValueMillis(delayMillis); + queryDelay = defaultRandomQueryDelay(jobId); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java index 16447f2d97ec9..94b0a7277a882 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java @@ -320,11 +320,11 @@ private AggregatedDataSummary(double earliestTime, double latestTime, long histo } /** - * This heuristic is a direct copy of the manual chunking config auto-creation done in {@link DatafeedConfig.Builder} + * This heuristic is a direct copy of the manual chunking config auto-creation done in {@link DatafeedConfig} */ @Override public long estimateChunk() { - return DatafeedConfig.Builder.DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis; + return DatafeedConfig.DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis; } @Override From 03edfd4fec42419d1c495cba55e3bce7d29e9cc9 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Thu, 1 Oct 2020 08:59:49 -0400 Subject: [PATCH 4/4] fixing style --- .../elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java index 0d616614284e1..bddc8b4c3e1bc 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java @@ -23,7 +23,6 @@ import org.elasticsearch.client.ValidationException; import org.elasticsearch.client.core.PageParams; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.ParseField; import java.util.Arrays; import java.util.List;