Skip to content

Commit 50a5afe

Browse files
dimitris-athanasioudimitris-athanasiou
authored
[7.x][ML] Prepare parsing phase_progress from DFA process (#55580) (#55587)
Data frame analytics process currently reports progress as an integer `progress_percent`. We parse that and report it from the _stats API as the progress of the `analyzing` phase. However, we want to allow the DFA process to report progress for more than one phase. This commit prepares for this by parsing `phase_progress` from the process, an object that contains the `phase` name plus the `progress_percent` for that phase. Backport of #55580
1 parent 7c81cd7 commit 50a5afe

File tree

5 files changed

+53
-20
lines changed

5 files changed

+53
-20
lines changed

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TargetType;
3333
import org.elasticsearch.xpack.core.ml.job.messages.Messages;
3434
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
35+
import org.elasticsearch.xpack.core.ml.utils.PhaseProgress;
3536
import org.elasticsearch.xpack.core.security.user.XPackUser;
3637
import org.elasticsearch.xpack.ml.dataframe.process.results.AnalyticsResult;
3738
import org.elasticsearch.xpack.ml.dataframe.process.results.RowResults;
@@ -164,10 +165,19 @@ private void processResult(AnalyticsResult result, DataFrameRowsJoiner resultsJo
164165
if (rowResults != null) {
165166
resultsJoiner.processRowResults(rowResults);
166167
}
168+
PhaseProgress phaseProgress = result.getPhaseProgress();
169+
if (phaseProgress != null) {
170+
LOGGER.debug("[{}] progress for phase [{}] updated to [{}]", analytics.getId(), phaseProgress.getPhase(),
171+
phaseProgress.getProgressPercent());
172+
statsHolder.getProgressTracker().analyzingPercent.set(phaseProgress.getProgressPercent());
173+
}
174+
175+
// TODO remove after process is writing out phase_progress
167176
Integer progressPercent = result.getProgressPercent();
168177
if (progressPercent != null) {
169178
statsHolder.getProgressTracker().analyzingPercent.set(progressPercent);
170179
}
180+
171181
TrainedModelDefinition.Builder inferenceModelBuilder = result.getInferenceModelBuilder();
172182
if (inferenceModelBuilder != null) {
173183
createAndIndexInferenceModel(inferenceModelBuilder);

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/results/AnalyticsResult.java‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,12 @@
1111
import org.elasticsearch.common.xcontent.ToXContent;
1212
import org.elasticsearch.common.xcontent.ToXContentObject;
1313
import org.elasticsearch.common.xcontent.XContentBuilder;
14-
import org.elasticsearch.xpack.core.ml.dataframe.stats.common.MemoryUsage;
1514
import org.elasticsearch.xpack.core.ml.dataframe.stats.classification.ClassificationStats;
15+
import org.elasticsearch.xpack.core.ml.dataframe.stats.common.MemoryUsage;
1616
import org.elasticsearch.xpack.core.ml.dataframe.stats.outlierdetection.OutlierDetectionStats;
1717
import org.elasticsearch.xpack.core.ml.dataframe.stats.regression.RegressionStats;
1818
import org.elasticsearch.xpack.core.ml.inference.TrainedModelDefinition;
19+
import org.elasticsearch.xpack.core.ml.utils.PhaseProgress;
1920

2021
import java.io.IOException;
2122
import java.util.Collections;
@@ -28,6 +29,7 @@ public class AnalyticsResult implements ToXContentObject {
2829

2930
public static final ParseField TYPE = new ParseField("analytics_result");
3031

32+
private static final ParseField PHASE_PROGRESS = new ParseField("phase_progress");
3133
private static final ParseField PROGRESS_PERCENT = new ParseField("progress_percent");
3234
private static final ParseField INFERENCE_MODEL = new ParseField("inference_model");
3335
private static final ParseField ANALYTICS_MEMORY_USAGE = new ParseField("analytics_memory_usage");
@@ -38,16 +40,18 @@ public class AnalyticsResult implements ToXContentObject {
3840
public static final ConstructingObjectParser<AnalyticsResult, Void> PARSER = new ConstructingObjectParser<>(TYPE.getPreferredName(),
3941
a -> new AnalyticsResult(
4042
(RowResults) a[0],
41-
(Integer) a[1],
42-
(TrainedModelDefinition.Builder) a[2],
43-
(MemoryUsage) a[3],
44-
(OutlierDetectionStats) a[4],
45-
(ClassificationStats) a[5],
46-
(RegressionStats) a[6]
43+
(PhaseProgress) a[1],
44+
(Integer) a[2],
45+
(TrainedModelDefinition.Builder) a[3],
46+
(MemoryUsage) a[4],
47+
(OutlierDetectionStats) a[5],
48+
(ClassificationStats) a[6],
49+
(RegressionStats) a[7]
4750
));
4851

4952
static {
5053
PARSER.declareObject(optionalConstructorArg(), RowResults.PARSER, RowResults.TYPE);
54+
PARSER.declareObject(optionalConstructorArg(), PhaseProgress.PARSER, PHASE_PROGRESS);
5155
PARSER.declareInt(optionalConstructorArg(), PROGRESS_PERCENT);
5256
// TODO change back to STRICT_PARSER once native side is aligned
5357
PARSER.declareObject(optionalConstructorArg(), TrainedModelDefinition.LENIENT_PARSER, INFERENCE_MODEL);
@@ -58,7 +62,11 @@ public class AnalyticsResult implements ToXContentObject {
5862
}
5963

6064
private final RowResults rowResults;
65+
private final PhaseProgress phaseProgress;
66+
67+
// TODO remove after process is writing out phase_progress
6168
private final Integer progressPercent;
69+
6270
private final TrainedModelDefinition.Builder inferenceModelBuilder;
6371
private final TrainedModelDefinition inferenceModel;
6472
private final MemoryUsage memoryUsage;
@@ -67,13 +75,15 @@ public class AnalyticsResult implements ToXContentObject {
6775
private final RegressionStats regressionStats;
6876

6977
public AnalyticsResult(@Nullable RowResults rowResults,
78+
@Nullable PhaseProgress phaseProgress,
7079
@Nullable Integer progressPercent,
7180
@Nullable TrainedModelDefinition.Builder inferenceModelBuilder,
7281
@Nullable MemoryUsage memoryUsage,
7382
@Nullable OutlierDetectionStats outlierDetectionStats,
7483
@Nullable ClassificationStats classificationStats,
7584
@Nullable RegressionStats regressionStats) {
7685
this.rowResults = rowResults;
86+
this.phaseProgress = phaseProgress;
7787
this.progressPercent = progressPercent;
7888
this.inferenceModelBuilder = inferenceModelBuilder;
7989
this.inferenceModel = inferenceModelBuilder == null ? null : inferenceModelBuilder.build();
@@ -87,6 +97,10 @@ public RowResults getRowResults() {
8797
return rowResults;
8898
}
8999

100+
public PhaseProgress getPhaseProgress() {
101+
return phaseProgress;
102+
}
103+
90104
public Integer getProgressPercent() {
91105
return progressPercent;
92106
}
@@ -117,6 +131,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
117131
if (rowResults != null) {
118132
builder.field(RowResults.TYPE.getPreferredName(), rowResults);
119133
}
134+
if (phaseProgress != null) {
135+
builder.field(PHASE_PROGRESS.getPreferredName(), phaseProgress);
136+
}
120137
if (progressPercent != null) {
121138
builder.field(PROGRESS_PERCENT.getPreferredName(), progressPercent);
122139
}
@@ -152,6 +169,7 @@ public boolean equals(Object other) {
152169

153170
AnalyticsResult that = (AnalyticsResult) other;
154171
return Objects.equals(rowResults, that.rowResults)
172+
&& Objects.equals(phaseProgress, that.phaseProgress)
155173
&& Objects.equals(progressPercent, that.progressPercent)
156174
&& Objects.equals(inferenceModel, that.inferenceModel)
157175
&& Objects.equals(memoryUsage, that.memoryUsage)
@@ -162,7 +180,7 @@ public boolean equals(Object other) {
162180

163181
@Override
164182
public int hashCode() {
165-
return Objects.hash(rowResults, progressPercent, inferenceModel, memoryUsage, outlierDetectionStats, classificationStats,
166-
regressionStats);
183+
return Objects.hash(rowResults, phaseProgress, progressPercent, inferenceModel, memoryUsage, outlierDetectionStats,
184+
classificationStats, regressionStats);
167185
}
168186
}

‎x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ public class AnalyticsProcessManagerTests extends ESTestCase {
5757
private static final String CONFIG_ID = "config-id";
5858
private static final int NUM_ROWS = 100;
5959
private static final int NUM_COLS = 4;
60-
private static final AnalyticsResult PROCESS_RESULT = new AnalyticsResult(null, null, null, null, null, null, null);
60+
private static final AnalyticsResult PROCESS_RESULT = new AnalyticsResult(null, null, null, null, null, null, null, null);
6161

6262
private Client client;
6363
private DataFrameAnalyticsAuditor auditor;

‎x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -105,8 +105,8 @@ public void testProcess_GivenNoResults() {
105105
public void testProcess_GivenEmptyResults() {
106106
givenDataFrameRows(2);
107107
givenProcessResults(Arrays.asList(
108-
new AnalyticsResult(null, 50, null, null, null, null, null),
109-
new AnalyticsResult(null, 100, null, null, null, null, null)));
108+
new AnalyticsResult(null, null,50, null, null, null, null, null),
109+
new AnalyticsResult(null, null, 100, null, null, null, null, null)));
110110
AnalyticsResultProcessor resultProcessor = createResultProcessor();
111111

112112
resultProcessor.process(process);
@@ -121,8 +121,8 @@ public void testProcess_GivenRowResults() {
121121
givenDataFrameRows(2);
122122
RowResults rowResults1 = mock(RowResults.class);
123123
RowResults rowResults2 = mock(RowResults.class);
124-
givenProcessResults(Arrays.asList(new AnalyticsResult(rowResults1, 50, null, null, null, null, null),
125-
new AnalyticsResult(rowResults2, 100, null, null, null, null, null)));
124+
givenProcessResults(Arrays.asList(new AnalyticsResult(rowResults1, null,50, null, null, null, null, null),
125+
new AnalyticsResult(rowResults2, null, 100, null, null, null, null, null)));
126126
AnalyticsResultProcessor resultProcessor = createResultProcessor();
127127

128128
resultProcessor.process(process);
@@ -139,8 +139,8 @@ public void testProcess_GivenDataFrameRowsJoinerFails() {
139139
givenDataFrameRows(2);
140140
RowResults rowResults1 = mock(RowResults.class);
141141
RowResults rowResults2 = mock(RowResults.class);
142-
givenProcessResults(Arrays.asList(new AnalyticsResult(rowResults1, 50, null, null, null, null, null),
143-
new AnalyticsResult(rowResults2, 100, null, null, null, null, null)));
142+
givenProcessResults(Arrays.asList(new AnalyticsResult(rowResults1, null,50, null, null, null, null, null),
143+
new AnalyticsResult(rowResults2, null, 100, null, null, null, null, null)));
144144

145145
doThrow(new RuntimeException("some failure")).when(dataFrameRowsJoiner).processRowResults(any(RowResults.class));
146146

@@ -174,7 +174,7 @@ public void testProcess_GivenInferenceModelIsStoredSuccessfully() {
174174
extractedFieldList.add(new DocValueField("baz", Collections.emptySet()));
175175
TargetType targetType = analyticsConfig.getAnalysis() instanceof Regression ? TargetType.REGRESSION : TargetType.CLASSIFICATION;
176176
TrainedModelDefinition.Builder inferenceModel = TrainedModelDefinitionTests.createRandomBuilder(targetType);
177-
givenProcessResults(Arrays.asList(new AnalyticsResult(null, null, inferenceModel, null, null, null, null)));
177+
givenProcessResults(Arrays.asList(new AnalyticsResult(null, null, null, inferenceModel, null, null, null, null)));
178178
AnalyticsResultProcessor resultProcessor = createResultProcessor(extractedFieldList);
179179

180180
resultProcessor.process(process);
@@ -238,7 +238,7 @@ public void testProcess_GivenInferenceModelFailedToStore() {
238238

239239
TargetType targetType = analyticsConfig.getAnalysis() instanceof Regression ? TargetType.REGRESSION : TargetType.CLASSIFICATION;
240240
TrainedModelDefinition.Builder inferenceModel = TrainedModelDefinitionTests.createRandomBuilder(targetType);
241-
givenProcessResults(Arrays.asList(new AnalyticsResult(null, null, inferenceModel, null, null, null, null)));
241+
givenProcessResults(Arrays.asList(new AnalyticsResult(null, null, null, inferenceModel, null, null, null, null)));
242242
AnalyticsResultProcessor resultProcessor = createResultProcessor();
243243

244244
resultProcessor.process(process);

‎x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/results/AnalyticsResultTests.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
2323
import org.elasticsearch.xpack.core.ml.inference.TrainedModelDefinition;
2424
import org.elasticsearch.xpack.core.ml.inference.TrainedModelDefinitionTests;
25+
import org.elasticsearch.xpack.core.ml.utils.PhaseProgress;
2526
import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
2627

2728
import java.util.ArrayList;
@@ -41,6 +42,7 @@ protected NamedXContentRegistry xContentRegistry() {
4142
@Override
4243
protected AnalyticsResult createTestInstance() {
4344
RowResults rowResults = null;
45+
PhaseProgress phaseProgress = null;
4446
Integer progressPercent = null;
4547
TrainedModelDefinition.Builder inferenceModel = null;
4648
MemoryUsage memoryUsage = null;
@@ -50,6 +52,9 @@ protected AnalyticsResult createTestInstance() {
5052
if (randomBoolean()) {
5153
rowResults = RowResultsTests.createRandom();
5254
}
55+
if (randomBoolean()) {
56+
phaseProgress = new PhaseProgress(randomAlphaOfLength(10), randomIntBetween(0, 100));
57+
}
5358
if (randomBoolean()) {
5459
progressPercent = randomIntBetween(0, 100);
5560
}
@@ -68,8 +73,8 @@ protected AnalyticsResult createTestInstance() {
6873
if (randomBoolean()) {
6974
regressionStats = RegressionStatsTests.createRandom();
7075
}
71-
return new AnalyticsResult(rowResults, progressPercent, inferenceModel, memoryUsage, outlierDetectionStats, classificationStats,
72-
regressionStats);
76+
return new AnalyticsResult(rowResults, phaseProgress, progressPercent, inferenceModel, memoryUsage, outlierDetectionStats,
77+
classificationStats, regressionStats);
7378
}
7479

7580
@Override

0 commit comments

Comments
 (0)