Skip to content

Commit ade8766

Browse files
benwtrentbenwtrent
authored
[ML][Data Frame] Add version and create_time to transform config (#43384)
* [ML][Data Frame] Add version and create_time to transform config * s/transform_version/version s/Date/Instant * fixing getter/setter for version
1 parent 56036bb commit ade8766

File tree

12 files changed

+306
-37
lines changed

12 files changed

+306
-37
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java‎

Lines changed: 42 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,20 @@
1919

2020
package org.elasticsearch.client.dataframe.transforms;
2121

22+
import org.elasticsearch.Version;
2223
import org.elasticsearch.client.dataframe.transforms.pivot.PivotConfig;
24+
import org.elasticsearch.client.dataframe.transforms.util.TimeUtil;
2325
import org.elasticsearch.common.Nullable;
2426
import org.elasticsearch.common.ParseField;
2527
import org.elasticsearch.common.Strings;
2628
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
29+
import org.elasticsearch.common.xcontent.ObjectParser;
2730
import org.elasticsearch.common.xcontent.ToXContentObject;
2831
import org.elasticsearch.common.xcontent.XContentBuilder;
2932
import org.elasticsearch.common.xcontent.XContentParser;
3033

3134
import java.io.IOException;
35+
import java.time.Instant;
3236
import java.util.Objects;
3337

3438
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
@@ -40,6 +44,8 @@ public class DataFrameTransformConfig implements ToXContentObject {
4044
public static final ParseField SOURCE = new ParseField("source");
4145
public static final ParseField DEST = new ParseField("dest");
4246
public static final ParseField DESCRIPTION = new ParseField("description");
47+
public static final ParseField VERSION = new ParseField("version");
48+
public static final ParseField CREATE_TIME = new ParseField("create_time");
4349
// types of transforms
4450
public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot");
4551

@@ -48,6 +54,8 @@ public class DataFrameTransformConfig implements ToXContentObject {
4854
private final DestConfig dest;
4955
private final PivotConfig pivotConfig;
5056
private final String description;
57+
private final Version transformVersion;
58+
private final Instant createTime;
5159

5260
public static final ConstructingObjectParser<DataFrameTransformConfig, Void> PARSER =
5361
new ConstructingObjectParser<>("data_frame_transform", true,
@@ -57,7 +65,9 @@ public class DataFrameTransformConfig implements ToXContentObject {
5765
DestConfig dest = (DestConfig) args[2];
5866
PivotConfig pivotConfig = (PivotConfig) args[3];
5967
String description = (String)args[4];
60-
return new DataFrameTransformConfig(id, source, dest, pivotConfig, description);
68+
Instant createTime = (Instant)args[5];
69+
String transformVersion = (String)args[6];
70+
return new DataFrameTransformConfig(id, source, dest, pivotConfig, description, createTime, transformVersion);
6171
});
6272

6373
static {
@@ -66,6 +76,9 @@ public class DataFrameTransformConfig implements ToXContentObject {
6676
PARSER.declareObject(constructorArg(), (p, c) -> DestConfig.PARSER.apply(p, null), DEST);
6777
PARSER.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p), PIVOT_TRANSFORM);
6878
PARSER.declareString(optionalConstructorArg(), DESCRIPTION);
79+
PARSER.declareField(optionalConstructorArg(),
80+
p -> TimeUtil.parseTimeFieldToInstant(p, CREATE_TIME.getPreferredName()), CREATE_TIME, ObjectParser.ValueType.VALUE);
81+
PARSER.declareString(optionalConstructorArg(), VERSION);
6982
}
7083

7184
public static DataFrameTransformConfig fromXContent(final XContentParser parser) {
@@ -84,19 +97,23 @@ public static DataFrameTransformConfig fromXContent(final XContentParser parser)
8497
* @return A DataFrameTransformConfig to preview, NOTE it will have a {@code null} id, destination and index.
8598
*/
8699
public static DataFrameTransformConfig forPreview(final SourceConfig source, final PivotConfig pivotConfig) {
87-
return new DataFrameTransformConfig(null, source, null, pivotConfig, null);
100+
return new DataFrameTransformConfig(null, source, null, pivotConfig, null, null, null);
88101
}
89102

90103
DataFrameTransformConfig(final String id,
91104
final SourceConfig source,
92105
final DestConfig dest,
93106
final PivotConfig pivotConfig,
94-
final String description) {
107+
final String description,
108+
final Instant createTime,
109+
final String version) {
95110
this.id = id;
96111
this.source = source;
97112
this.dest = dest;
98113
this.pivotConfig = pivotConfig;
99114
this.description = description;
115+
this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli());
116+
this.transformVersion = version == null ? null : Version.fromString(version);
100117
}
101118

102119
public String getId() {
@@ -115,6 +132,14 @@ public PivotConfig getPivotConfig() {
115132
return pivotConfig;
116133
}
117134

135+
public Version getVersion() {
136+
return transformVersion;
137+
}
138+
139+
public Instant getCreateTime() {
140+
return createTime;
141+
}
142+
118143
@Nullable
119144
public String getDescription() {
120145
return description;
@@ -138,6 +163,12 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
138163
if (description != null) {
139164
builder.field(DESCRIPTION.getPreferredName(), description);
140165
}
166+
if (createTime != null) {
167+
builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli());
168+
}
169+
if (transformVersion != null) {
170+
builder.field(VERSION.getPreferredName(), transformVersion);
171+
}
141172
builder.endObject();
142173
return builder;
143174
}
@@ -155,15 +186,17 @@ public boolean equals(Object other) {
155186
final DataFrameTransformConfig that = (DataFrameTransformConfig) other;
156187

157188
return Objects.equals(this.id, that.id)
158-
&& Objects.equals(this.source, that.source)
159-
&& Objects.equals(this.dest, that.dest)
160-
&& Objects.equals(this.description, that.description)
161-
&& Objects.equals(this.pivotConfig, that.pivotConfig);
189+
&& Objects.equals(this.source, that.source)
190+
&& Objects.equals(this.dest, that.dest)
191+
&& Objects.equals(this.description, that.description)
192+
&& Objects.equals(this.transformVersion, that.transformVersion)
193+
&& Objects.equals(this.createTime, that.createTime)
194+
&& Objects.equals(this.pivotConfig, that.pivotConfig);
162195
}
163196

164197
@Override
165198
public int hashCode() {
166-
return Objects.hash(id, source, dest, pivotConfig, description);
199+
return Objects.hash(id, source, dest, pivotConfig, description, createTime, transformVersion);
167200
}
168201

169202
@Override
@@ -209,7 +242,7 @@ public Builder setDescription(String description) {
209242
}
210243

211244
public DataFrameTransformConfig build() {
212-
return new DataFrameTransformConfig(id, source, dest, pivotConfig, description);
245+
return new DataFrameTransformConfig(id, source, dest, pivotConfig, description, null, null);
213246
}
214247
}
215248
}

‎client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/util/TimeUtil.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.client.dataframe.transforms.util;
20+
21+
import org.elasticsearch.common.time.DateFormatters;
22+
import org.elasticsearch.common.xcontent.XContentParser;
23+
24+
import java.io.IOException;
25+
import java.time.Instant;
26+
import java.time.format.DateTimeFormatter;
27+
import java.util.Date;
28+
29+
public final class TimeUtil {
30+
31+
/**
32+
* Parse out a Date object given the current parser and field name.
33+
*
34+
* @param parser current XContentParser
35+
* @param fieldName the field's preferred name (utilized in exception)
36+
* @return parsed Date object
37+
* @throws IOException from XContentParser
38+
*/
39+
public static Date parseTimeField(XContentParser parser, String fieldName) throws IOException {
40+
if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
41+
return new Date(parser.longValue());
42+
} else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
43+
return new Date(DateFormatters.from(DateTimeFormatter.ISO_INSTANT.parse(parser.text())).toInstant().toEpochMilli());
44+
}
45+
throw new IllegalArgumentException(
46+
"unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]");
47+
}
48+
49+
public static Instant parseTimeFieldToInstant(XContentParser parser, String fieldName) throws IOException {
50+
if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
51+
return Instant.ofEpochMilli(parser.longValue());
52+
} else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
53+
return DateFormatters.from(DateTimeFormatter.ISO_INSTANT.parse(parser.text())).toInstant();
54+
}
55+
throw new IllegalArgumentException(
56+
"unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]");
57+
}
58+
59+
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ public void testGetTransform() throws IOException {
195195
client::getDataFrameTransformAsync);
196196
assertNull(getResponse.getInvalidTransforms());
197197
assertThat(getResponse.getTransformConfigurations(), hasSize(1));
198-
assertEquals(transform, getResponse.getTransformConfigurations().get(0));
198+
assertEquals(transform.getId(), getResponse.getTransformConfigurations().get(0).getId());
199199
}
200200

201201
public void testGetAllAndPageTransforms() throws IOException {
@@ -219,7 +219,7 @@ public void testGetAllAndPageTransforms() throws IOException {
219219
client::getDataFrameTransformAsync);
220220
assertNull(getResponse.getInvalidTransforms());
221221
assertThat(getResponse.getTransformConfigurations(), hasSize(2));
222-
assertEquals(transform, getResponse.getTransformConfigurations().get(1));
222+
assertEquals(transform.getId(), getResponse.getTransformConfigurations().get(1).getId());
223223

224224
getRequest.setPageParams(new PageParams(0,1));
225225
getResponse = execute(getRequest, client::getDataFrameTransform,

‎client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.client.dataframe.transforms;
2121

22+
import org.elasticsearch.Version;
2223
import org.elasticsearch.client.dataframe.transforms.pivot.PivotConfigTests;
2324
import org.elasticsearch.common.settings.Settings;
2425
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -27,6 +28,7 @@
2728
import org.elasticsearch.test.AbstractXContentTestCase;
2829

2930
import java.io.IOException;
31+
import java.time.Instant;
3032
import java.util.Collections;
3133
import java.util.function.Predicate;
3234

@@ -36,8 +38,13 @@
3638
public class DataFrameTransformConfigTests extends AbstractXContentTestCase<DataFrameTransformConfig> {
3739

3840
public static DataFrameTransformConfig randomDataFrameTransformConfig() {
39-
return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomSourceConfig(),
40-
randomDestConfig(), PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100));
41+
return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10),
42+
randomSourceConfig(),
43+
randomDestConfig(),
44+
PivotConfigTests.randomPivotConfig(),
45+
randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100),
46+
randomBoolean() ? null : Instant.now(),
47+
randomBoolean() ? null : Version.CURRENT.toString());
4148
}
4249

4350
@Override

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -478,7 +478,6 @@ public void testGetStats() throws IOException, InterruptedException {
478478

479479
RestHighLevelClient client = highLevelClient();
480480

481-
QueryConfig queryConfig = new QueryConfig(new MatchAllQueryBuilder());
482481
GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer",
483482
TermsGroupSource.builder().setField("user_id").build()).build();
484483
AggregatorFactories.Builder aggBuilder = new AggregatorFactories.Builder();
@@ -564,7 +563,6 @@ public void onFailure(Exception e) {
564563
public void testGetDataFrameTransform() throws IOException, InterruptedException {
565564
createIndex("source-data");
566565

567-
QueryConfig queryConfig = new QueryConfig(new MatchAllQueryBuilder());
568566
GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer",
569567
TermsGroupSource.builder().setField("user_id").build()).build();
570568
AggregatorFactories.Builder aggBuilder = new AggregatorFactories.Builder();

0 commit comments

Comments
 (0)