Skip to content

Commit b9dc522

Browse files
benwtrentbenwtrent
authored
[7.x] [ML] adding new flag exclude_generated that removes generated fields in GET config APIs (#63899)(#63092) (#63177)
* [ML] adding for_export flag for ml plugin GET resource APIs (#63092) This adds the new `for_export` flag to the following APIs: - GET _ml/anomaly_detection/<job_id> - GET _ml/datafeeds/<datafeed_id> - GET _ml/data_frame/analytics/<analytics_id> The flag is designed for cloning or exporting configuration objects to later be put into the same cluster or a separate cluster. The following fields are not returned in the objects: - any field that is not user settable (e.g. version, create_time) - any field that is a calculated default value (e.g. datafeed chunking_config) - any field that would effectively require changing to be of use (e.g. datafeed job_id) - any field that is automatically set via another Elastic stack process (e.g. anomaly job custom_settings.created_by) closes #63055 * [ML] adding new flag exclude_generated that removes generated fields in GET config APIs (#63899) When exporting and cloning ml configurations in a cluster it can be frustrating to remove all the fields that were generated by the plugin. Especially as the number of these fields change from version to version. This flag, exclude_generated, allows the GET config APIs to return configurations with these generated fields removed. APIs supporting this flag: - GET _ml/anomaly_detection/<job_id> - GET _ml/datafeeds/<datafeed_id> - GET _ml/data_frame/analytics/<analytics_id> The following fields are not returned in the objects: - any field that is not user settable (e.g. version, create_time) - any field that is a calculated default value (e.g. datafeed chunking_config) - any field that is automatically set via another Elastic stack process (e.g. anomaly job custom_settings.created_by) relates to #63055
1 parent b92cbcd commit b9dc522

File tree

34 files changed

+622
-159
lines changed

34 files changed

+622
-159
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,10 @@ static Request getJob(GetJobRequest getJobRequest) {
121121

122122
RequestConverters.Params params = new RequestConverters.Params();
123123
if (getJobRequest.getAllowNoMatch() != null) {
124-
params.putParam("allow_no_match", Boolean.toString(getJobRequest.getAllowNoMatch()));
124+
params.putParam(GetJobRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getJobRequest.getAllowNoMatch()));
125+
}
126+
if (getJobRequest.getExcludeGenerated() != null) {
127+
params.putParam(GetJobRequest.EXCLUDE_GENERATED, Boolean.toString(getJobRequest.getExcludeGenerated()));
125128
}
126129
request.addParameters(params.asMap());
127130
return request;
@@ -270,6 +273,9 @@ static Request getDatafeed(GetDatafeedRequest getDatafeedRequest) {
270273
params.putParam(GetDatafeedRequest.ALLOW_NO_MATCH.getPreferredName(),
271274
Boolean.toString(getDatafeedRequest.getAllowNoMatch()));
272275
}
276+
if (getDatafeedRequest.getExcludeGenerated() != null) {
277+
params.putParam(GetDatafeedRequest.EXCLUDE_GENERATED, Boolean.toString(getDatafeedRequest.getExcludeGenerated()));
278+
}
273279
request.addParameters(params.asMap());
274280
return request;
275281
}
@@ -645,7 +651,10 @@ static Request getDataFrameAnalytics(GetDataFrameAnalyticsRequest getRequest) {
645651
}
646652
}
647653
if (getRequest.getAllowNoMatch() != null) {
648-
params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getRequest.getAllowNoMatch()));
654+
params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH, Boolean.toString(getRequest.getAllowNoMatch()));
655+
}
656+
if (getRequest.getExcludeGenerated() != null) {
657+
params.putParam(GetDataFrameAnalyticsRequest.EXCLUDE_GENERATED, Boolean.toString(getRequest.getExcludeGenerated()));
649658
}
650659
request.addParameters(params.asMap());
651660
return request;
@@ -786,8 +795,8 @@ static Request getTrainedModels(GetTrainedModelsRequest getTrainedModelsRequest)
786795
if (getTrainedModelsRequest.getTags() != null) {
787796
params.putParam(GetTrainedModelsRequest.TAGS, Strings.collectionToCommaDelimitedString(getTrainedModelsRequest.getTags()));
788797
}
789-
if (getTrainedModelsRequest.getForExport() != null) {
790-
params.putParam(GetTrainedModelsRequest.FOR_EXPORT, Boolean.toString(getTrainedModelsRequest.getForExport()));
798+
if (getTrainedModelsRequest.getExcludeGenerated() != null) {
799+
params.putParam(GetTrainedModelsRequest.EXCLUDE_GENERATED, Boolean.toString(getTrainedModelsRequest.getExcludeGenerated()));
791800
}
792801
Request request = new Request(HttpGet.METHOD_NAME, endpoint);
793802
request.addParameters(params.asMap());

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import org.elasticsearch.client.ValidationException;
2424
import org.elasticsearch.client.core.PageParams;
2525
import org.elasticsearch.common.Nullable;
26-
import org.elasticsearch.common.ParseField;
2726

2827
import java.util.Arrays;
2928
import java.util.List;
@@ -32,11 +31,13 @@
3231

3332
public class GetDataFrameAnalyticsRequest implements Validatable {
3433

35-
public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match");
34+
public static final String ALLOW_NO_MATCH = "allow_no_match";
35+
public static final String EXCLUDE_GENERATED = "exclude_generated";
3636

3737
private final List<String> ids;
3838
private Boolean allowNoMatch;
3939
private PageParams pageParams;
40+
private Boolean excludeGenerated;
4041

4142
/**
4243
* Helper method to create a request that will get ALL Data Frame Analytics
@@ -58,6 +59,22 @@ public Boolean getAllowNoMatch() {
5859
return allowNoMatch;
5960
}
6061

62+
/**
63+
* Setting this flag to `true` removes certain fields from the configuration on retrieval.
64+
*
65+
* This is useful when getting the configuration and wanting to put it in another cluster.
66+
*
67+
* Default value is false.
68+
* @param excludeGenerated Boolean value indicating if certain fields should be removed
69+
*/
70+
public void setExcludeGenerated(boolean excludeGenerated) {
71+
this.excludeGenerated = excludeGenerated;
72+
}
73+
74+
public Boolean getExcludeGenerated() {
75+
return excludeGenerated;
76+
}
77+
6178
/**
6279
* Whether to ignore if a wildcard expression matches no data frame analytics.
6380
*
@@ -94,11 +111,12 @@ public boolean equals(Object o) {
94111
GetDataFrameAnalyticsRequest other = (GetDataFrameAnalyticsRequest) o;
95112
return Objects.equals(ids, other.ids)
96113
&& Objects.equals(allowNoMatch, other.allowNoMatch)
114+
&& Objects.equals(excludeGenerated, other.excludeGenerated)
97115
&& Objects.equals(pageParams, other.pageParams);
98116
}
99117

100118
@Override
101119
public int hashCode() {
102-
return Objects.hash(ids, allowNoMatch, pageParams);
120+
return Objects.hash(ids, allowNoMatch, excludeGenerated, pageParams);
103121
}
104122
}

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,12 @@ public class GetDatafeedRequest extends ActionRequest implements ToXContentObjec
4242

4343
public static final ParseField DATAFEED_IDS = new ParseField("datafeed_ids");
4444
public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match");
45+
public static final String EXCLUDE_GENERATED = "exclude_generated";
4546

4647
private static final String ALL_DATAFEEDS = "_all";
4748
private final List<String> datafeedIds;
4849
private Boolean allowNoMatch;
50+
private Boolean excludeGenerated;
4951

5052
@SuppressWarnings("unchecked")
5153
public static final ConstructingObjectParser<GetDatafeedRequest, Void> PARSER = new ConstructingObjectParser<>(
@@ -101,14 +103,30 @@ public Boolean getAllowNoMatch() {
101103
return allowNoMatch;
102104
}
103105

106+
/**
107+
* Setting this flag to `true` removes certain fields from the configuration on retrieval.
108+
*
109+
* This is useful when getting the configuration and wanting to put it in another cluster.
110+
*
111+
* Default value is false.
112+
* @param excludeGenerated Boolean value indicating if certain fields should be removed
113+
*/
114+
public void setExcludeGenerated(boolean excludeGenerated) {
115+
this.excludeGenerated = excludeGenerated;
116+
}
117+
118+
public Boolean getExcludeGenerated() {
119+
return excludeGenerated;
120+
}
121+
104122
@Override
105123
public ActionRequestValidationException validate() {
106124
return null;
107125
}
108126

109127
@Override
110128
public int hashCode() {
111-
return Objects.hash(datafeedIds, allowNoMatch);
129+
return Objects.hash(datafeedIds, excludeGenerated, allowNoMatch);
112130
}
113131

114132
@Override
@@ -123,7 +141,8 @@ public boolean equals(Object other) {
123141

124142
GetDatafeedRequest that = (GetDatafeedRequest) other;
125143
return Objects.equals(datafeedIds, that.datafeedIds) &&
126-
Objects.equals(allowNoMatch, that.allowNoMatch);
144+
Objects.equals(allowNoMatch, that.allowNoMatch) &&
145+
Objects.equals(excludeGenerated, that.excludeGenerated);
127146
}
128147

129148
@Override

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,12 @@ public class GetJobRequest extends ActionRequest implements ToXContentObject {
4343

4444
public static final ParseField JOB_IDS = new ParseField("job_ids");
4545
public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match");
46+
public static final String EXCLUDE_GENERATED = "exclude_generated";
4647

4748
private static final String ALL_JOBS = "_all";
4849
private final List<String> jobIds;
4950
private Boolean allowNoMatch;
51+
private Boolean excludeGenerated;
5052

5153
@SuppressWarnings("unchecked")
5254
public static final ConstructingObjectParser<GetJobRequest, Void> PARSER = new ConstructingObjectParser<>(
@@ -101,14 +103,30 @@ public Boolean getAllowNoMatch() {
101103
return allowNoMatch;
102104
}
103105

106+
/**
107+
* Setting this flag to `true` removes certain fields from the configuration on retrieval.
108+
*
109+
* This is useful when getting the configuration and wanting to put it in another cluster.
110+
*
111+
* Default value is false.
112+
* @param excludeGenerated Boolean value indicating if certain fields should be removed
113+
*/
114+
public void setExcludeGenerated(boolean excludeGenerated) {
115+
this.excludeGenerated = excludeGenerated;
116+
}
117+
118+
public Boolean getExcludeGenerated() {
119+
return excludeGenerated;
120+
}
121+
104122
@Override
105123
public ActionRequestValidationException validate() {
106124
return null;
107125
}
108126

109127
@Override
110128
public int hashCode() {
111-
return Objects.hash(jobIds, allowNoMatch);
129+
return Objects.hash(jobIds, excludeGenerated, allowNoMatch);
112130
}
113131

114132
@Override
@@ -123,6 +141,7 @@ public boolean equals(Object other) {
123141

124142
GetJobRequest that = (GetJobRequest) other;
125143
return Objects.equals(jobIds, that.jobIds) &&
144+
Objects.equals(excludeGenerated, that.excludeGenerated) &&
126145
Objects.equals(allowNoMatch, that.allowNoMatch);
127146
}
128147

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ public class GetTrainedModelsRequest implements Validatable {
3939
private static final String TOTAL_FEATURE_IMPORTANCE = "total_feature_importance";
4040
private static final String FEATURE_IMPORTANCE_BASELINE = "feature_importance_baseline";
4141
public static final String ALLOW_NO_MATCH = "allow_no_match";
42-
public static final String FOR_EXPORT = "for_export";
42+
public static final String EXCLUDE_GENERATED = "exclude_generated";
4343
public static final String DECOMPRESS_DEFINITION = "decompress_definition";
4444
public static final String TAGS = "tags";
4545
public static final String INCLUDE = "include";
@@ -48,7 +48,7 @@ public class GetTrainedModelsRequest implements Validatable {
4848
private Boolean allowNoMatch;
4949
private Set<String> includes = new HashSet<>();
5050
private Boolean decompressDefinition;
51-
private Boolean forExport;
51+
private Boolean excludeGenerated;
5252
private PageParams pageParams;
5353
private List<String> tags;
5454

@@ -163,8 +163,8 @@ public GetTrainedModelsRequest setTags(String... tags) {
163163
return setTags(Arrays.asList(tags));
164164
}
165165

166-
public Boolean getForExport() {
167-
return forExport;
166+
public Boolean getExcludeGenerated() {
167+
return excludeGenerated;
168168
}
169169

170170
/**
@@ -173,10 +173,10 @@ public Boolean getForExport() {
173173
* This is useful when getting the model and wanting to put it in another cluster.
174174
*
175175
* Default value is false.
176-
* @param forExport Boolean value indicating if certain fields should be removed from the mode on GET
176+
* @param excludeGenerated Boolean value indicating if certain fields should be removed from the mode on GET
177177
*/
178-
public GetTrainedModelsRequest setForExport(Boolean forExport) {
179-
this.forExport = forExport;
178+
public GetTrainedModelsRequest setExcludeGenerated(Boolean excludeGenerated) {
179+
this.excludeGenerated = excludeGenerated;
180180
return this;
181181
}
182182

@@ -198,12 +198,12 @@ public boolean equals(Object o) {
198198
&& Objects.equals(allowNoMatch, other.allowNoMatch)
199199
&& Objects.equals(decompressDefinition, other.decompressDefinition)
200200
&& Objects.equals(includes, other.includes)
201-
&& Objects.equals(forExport, other.forExport)
201+
&& Objects.equals(excludeGenerated, other.excludeGenerated)
202202
&& Objects.equals(pageParams, other.pageParams);
203203
}
204204

205205
@Override
206206
public int hashCode() {
207-
return Objects.hash(ids, allowNoMatch, pageParams, decompressDefinition, includes, forExport);
207+
return Objects.hash(ids, allowNoMatch, pageParams, decompressDefinition, includes, excludeGenerated);
208208
}
209209
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,7 @@ public void testGetJob() throws Exception {
343343
// tag::get-job-request
344344
GetJobRequest request = new GetJobRequest("get-machine-learning-job1", "get-machine-learning-job*"); // <1>
345345
request.setAllowNoMatch(true); // <2>
346+
request.setExcludeGenerated(false); // <3>
346347
// end::get-job-request
347348

348349
// tag::get-job-execute
@@ -839,6 +840,7 @@ public void testGetDatafeed() throws Exception {
839840
// tag::get-datafeed-request
840841
GetDatafeedRequest request = new GetDatafeedRequest(datafeedId); // <1>
841842
request.setAllowNoMatch(true); // <2>
843+
request.setExcludeGenerated(false); // <3>
842844
// end::get-datafeed-request
843845

844846
// tag::get-datafeed-execute
@@ -2864,6 +2866,7 @@ public void testGetDataFrameAnalytics() throws Exception {
28642866
{
28652867
// tag::get-data-frame-analytics-request
28662868
GetDataFrameAnalyticsRequest request = new GetDataFrameAnalyticsRequest("my-analytics-config"); // <1>
2869+
request.setExcludeGenerated(false); // <2>
28672870
// end::get-data-frame-analytics-request
28682871

28692872
// tag::get-data-frame-analytics-execute
@@ -3728,7 +3731,7 @@ public void testGetTrainedModels() throws Exception {
37283731
.setDecompressDefinition(false) // <6>
37293732
.setAllowNoMatch(true) // <7>
37303733
.setTags("regression") // <8>
3731-
.setForExport(false); // <9>
3734+
.setExcludeGenerated(false); // <9>
37323735
// end::get-trained-models-request
37333736
request.setTags((List<String>)null);
37343737

‎docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,9 @@ IDs, or the special wildcard `_all` to get all {dfanalytics-jobs}.
2323
include-tagged::{doc-tests-file}[{api}-request]
2424
--------------------------------------------------
2525
<1> Constructing a new GET request referencing an existing {dfanalytics-job}
26+
<2> Optional boolean value for requesting the {dfanalytics-job} in a format that can
27+
then be put into another cluster. Certain fields that can only be set when
28+
the {dfanalytics-job} is created are removed.
2629

2730
include::../execution.asciidoc[]
2831

‎docs/java-rest/high-level/ml/get-datafeed.asciidoc‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@ include-tagged::{doc-tests-file}[{api}-request]
2525
contain wildcards.
2626
<2> Whether to ignore if a wildcard expression matches no datafeeds.
2727
(This includes `_all` string or when no datafeeds have been specified).
28+
<3> Optional boolean value for requesting the datafeed in a format that can
29+
then be put into another cluster. Certain fields that can only be set when
30+
the datafeed is created are removed.
2831

2932
[id="{upid}-{api}-response"]
3033
==== Get datafeeds response

‎docs/java-rest/high-level/ml/get-job.asciidoc‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@ include-tagged::{doc-tests-file}[{api}-request]
2525
wildcards.
2626
<2> Whether to ignore if a wildcard expression matches no {anomaly-jobs}.
2727
(This includes `_all` string or when no jobs have been specified).
28+
<3> Optional boolean value for requesting the job in a format that can
29+
then be put into another cluster. Certain fields that can only be set when
30+
the job is created are removed.
2831

2932
[id="{upid}-{api}-response"]
3033
==== Get {anomaly-jobs} response

‎docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ Retrieves configuration information for {dfeeds}.
1919

2020
`GET _ml/datafeeds/` +
2121

22-
`GET _ml/datafeeds/_all`
22+
`GET _ml/datafeeds/_all`
2323

2424
[[ml-get-datafeed-prereqs]]
2525
== {api-prereq-title}
@@ -36,7 +36,7 @@ comma-separated list of {dfeeds} or a wildcard expression. You can get
3636
information for all {dfeeds} by using `_all`, by specifying `*` as the
3737
`<feed_id>`, or by omitting the `<feed_id>`.
3838

39-
IMPORTANT: This API returns a maximum of 10,000 {dfeeds}.
39+
IMPORTANT: This API returns a maximum of 10,000 {dfeeds}.
4040

4141
[[ml-get-datafeed-path-parms]]
4242
== {api-path-parms-title}
@@ -57,6 +57,10 @@ all {dfeeds}.
5757
(Optional, boolean)
5858
include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-datafeeds]
5959

60+
`exclude_generated`::
61+
(Optional, boolean)
62+
include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated]
63+
6064
[[ml-get-datafeed-results]]
6165
== {api-response-body-title}
6266

0 commit comments

Comments
 (0)