Skip to content

Commit a4703d7

Browse files
dimitris-athanasioudimitris-athanasiou
authored
[7.x][ML] Disable dynamic mapping to DFA dest index (#65972) (#66012)
We need to disable dynamic mapping to the destination index of data frame analytics jobs. If the source index has dynamic mapping disabled, then after reindexing it is possible that we add mappings for fields that were previously unmapped, thus making them eligible features. This is confusing to the user. This commit disables dynamic mapping to the destination index. It makes all result field mappings explicit. Backport of #65972
1 parent 5fec253 commit a4703d7

File tree

10 files changed

+54
-40
lines changed

10 files changed

+54
-40
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
1717
import org.elasticsearch.common.xcontent.XContentBuilder;
1818
import org.elasticsearch.common.xcontent.XContentParser;
19+
import org.elasticsearch.index.mapper.BooleanFieldMapper;
1920
import org.elasticsearch.index.mapper.KeywordFieldMapper;
2021
import org.elasticsearch.index.mapper.NumberFieldMapper;
2122
import org.elasticsearch.index.mapper.ObjectMapper;
@@ -141,7 +142,6 @@ public static Classification fromXContent(XContentParser parser, boolean ignoreU
141142

142143
Map<String, Object> properties = new HashMap<>();
143144
properties.put("feature_name", Collections.singletonMap("type", KeywordFieldMapper.CONTENT_TYPE));
144-
properties.put("importance", Collections.singletonMap("type", NumberFieldMapper.NumberType.DOUBLE.typeName()));
145145
properties.put("classes", classesMapping);
146146

147147
Map<String, Object> mapping = new HashMap<>();
@@ -376,15 +376,18 @@ public List<FieldCardinalityConstraint> getFieldCardinalityConstraints() {
376376

377377
@SuppressWarnings("unchecked")
378378
@Override
379-
public Map<String, Object> getExplicitlyMappedFields(String resultsFieldName, FieldCapabilitiesResponse fieldCapabilitiesResponse) {
379+
public Map<String, Object> getResultMappings(String resultsFieldName, FieldCapabilitiesResponse fieldCapabilitiesResponse) {
380380
Map<String, Object> additionalProperties = new HashMap<>();
381+
additionalProperties.put(resultsFieldName + ".is_training", Collections.singletonMap("type", BooleanFieldMapper.CONTENT_TYPE));
382+
additionalProperties.put(resultsFieldName + ".prediction_probability",
383+
Collections.singletonMap("type", NumberFieldMapper.NumberType.DOUBLE.typeName()));
384+
additionalProperties.put(resultsFieldName + ".prediction_score",
385+
Collections.singletonMap("type", NumberFieldMapper.NumberType.DOUBLE.typeName()));
381386
additionalProperties.put(resultsFieldName + ".feature_importance", FEATURE_IMPORTANCE_MAPPING);
382-
if (fieldCapabilitiesResponse == null) {
383-
return additionalProperties;
384-
}
387+
385388
Map<String, FieldCapabilities> dependentVariableFieldCaps = fieldCapabilitiesResponse.getField(dependentVariable);
386389
if (dependentVariableFieldCaps == null || dependentVariableFieldCaps.isEmpty()) {
387-
return additionalProperties;
390+
throw ExceptionsHelper.badRequestException("no mappings could be found for required field [{}]", DEPENDENT_VARIABLE);
388391
}
389392
Object dependentVariableMappingType = dependentVariableFieldCaps.values().iterator().next().getType();
390393
additionalProperties.put(
@@ -393,6 +396,7 @@ public Map<String, Object> getExplicitlyMappedFields(String resultsFieldName, Fi
393396
Map<String, Object> topClassesProperties = new HashMap<>();
394397
topClassesProperties.put("class_name", Collections.singletonMap("type", dependentVariableMappingType));
395398
topClassesProperties.put("class_probability", Collections.singletonMap("type", NumberFieldMapper.NumberType.DOUBLE.typeName()));
399+
topClassesProperties.put("class_score", Collections.singletonMap("type", NumberFieldMapper.NumberType.DOUBLE.typeName()));
396400

397401
Map<String, Object> topClassesMapping = new HashMap<>();
398402
topClassesMapping.put("type", ObjectMapper.NESTED_CONTENT_TYPE);

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public interface DataFrameAnalysis extends ToXContentObject, NamedWriteable {
5151
* @param fieldCapabilitiesResponse field capabilities fetched for this analysis' required fields
5252
* @return {@link Map} containing fields for which the mappings should be handled explicitly
5353
*/
54-
Map<String, Object> getExplicitlyMappedFields(String resultsFieldName, FieldCapabilitiesResponse fieldCapabilitiesResponse);
54+
Map<String, Object> getResultMappings(String resultsFieldName, FieldCapabilitiesResponse fieldCapabilitiesResponse);
5555

5656
/**
5757
* @return {@code true} if this analysis supports data frame rows with missing values

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,7 @@ public List<FieldCardinalityConstraint> getFieldCardinalityConstraints() {
246246
}
247247

248248
@Override
249-
public Map<String, Object> getExplicitlyMappedFields(String resultsFieldName, FieldCapabilitiesResponse fieldCapabilitiesResponse) {
249+
public Map<String, Object> getResultMappings(String resultsFieldName, FieldCapabilitiesResponse fieldCapabilitiesResponse) {
250250
Map<String, Object> additionalProperties = new HashMap<>();
251251
additionalProperties.put(resultsFieldName + ".outlier_score",
252252
Collections.singletonMap("type", NumberFieldMapper.NumberType.DOUBLE.typeName()));

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
1616
import org.elasticsearch.common.xcontent.XContentBuilder;
1717
import org.elasticsearch.common.xcontent.XContentParser;
18+
import org.elasticsearch.index.mapper.BooleanFieldMapper;
1819
import org.elasticsearch.index.mapper.KeywordFieldMapper;
1920
import org.elasticsearch.index.mapper.NumberFieldMapper;
2021
import org.elasticsearch.index.mapper.ObjectMapper;
@@ -299,8 +300,9 @@ public List<FieldCardinalityConstraint> getFieldCardinalityConstraints() {
299300
}
300301

301302
@Override
302-
public Map<String, Object> getExplicitlyMappedFields(String resultsFieldName, FieldCapabilitiesResponse fieldCapabilitiesResponse) {
303+
public Map<String, Object> getResultMappings(String resultsFieldName, FieldCapabilitiesResponse fieldCapabilitiesResponse) {
303304
Map<String, Object> additionalProperties = new HashMap<>();
305+
additionalProperties.put(resultsFieldName + ".is_training", Collections.singletonMap("type", BooleanFieldMapper.CONTENT_TYPE));
304306
additionalProperties.put(resultsFieldName + ".feature_importance", FEATURE_IMPORTANCE_MAPPING);
305307
// Prediction field should be always mapped as "double" rather than "float" in order to increase precision in case of
306308
// high (over 10M) values of dependent variable.

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java‎

Lines changed: 18 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -364,43 +364,42 @@ public void testFieldCardinalityLimitsIsNonEmpty() {
364364
assertThat(constraints.get(0).getUpperBound(), equalTo(30L));
365365
}
366366

367-
public void testGetExplicitlyMappedFields_FieldCapabilitiesResponseIsNull() {
368-
Map<String, Object> explicitlyMappedFields = new Classification("foo").getExplicitlyMappedFields("results", null);
369-
assertThat(explicitlyMappedFields, equalTo(singletonMap("results.feature_importance", Classification.FEATURE_IMPORTANCE_MAPPING)));
370-
}
371-
372-
public void testGetExplicitlyMappedFields_DependentVariableMappingIsAbsent() {
367+
public void testGetResultMappings_DependentVariableMappingIsAbsent() {
373368
FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse(new String[0], Collections.emptyMap());
374-
Map<String, Object> explicitlyMappedFields =
375-
new Classification("foo").getExplicitlyMappedFields("results", fieldCapabilitiesResponse);
376-
assertThat(explicitlyMappedFields, equalTo(singletonMap("results.feature_importance", Classification.FEATURE_IMPORTANCE_MAPPING)));
369+
expectThrows(ElasticsearchStatusException.class,
370+
() -> new Classification("foo").getResultMappings("results", fieldCapabilitiesResponse));
377371
}
378372

379-
public void testGetExplicitlyMappedFields_DependentVariableMappingHasNoTypes() {
373+
public void testGetResultMappings_DependentVariableMappingHasNoTypes() {
380374
FieldCapabilitiesResponse fieldCapabilitiesResponse =
381375
new FieldCapabilitiesResponse(new String[0], Collections.singletonMap("foo", Collections.emptyMap()));
382-
Map<String, Object> explicitlyMappedFields =
383-
new Classification("foo").getExplicitlyMappedFields("results", fieldCapabilitiesResponse);
384-
assertThat(explicitlyMappedFields, equalTo(singletonMap("results.feature_importance", Classification.FEATURE_IMPORTANCE_MAPPING)));
376+
expectThrows(ElasticsearchStatusException.class,
377+
() -> new Classification("foo").getResultMappings("results", fieldCapabilitiesResponse));
385378
}
386379

387-
public void testGetExplicitlyMappedFields_DependentVariableMappingIsPresent() {
380+
public void testGetResultMappings_DependentVariableMappingIsPresent() {
388381
Map<String, Object> expectedTopClassesMapping = new HashMap<String, Object>() {{
389382
put("type", "nested");
390383
put("properties", new HashMap<String, Object>() {{
391384
put("class_name", singletonMap("type", "dummy"));
392385
put("class_probability", singletonMap("type", "double"));
386+
put("class_score", singletonMap("type", "double"));
393387
}});
394388
}};
395389
FieldCapabilitiesResponse fieldCapabilitiesResponse =
396390
new FieldCapabilitiesResponse(
397391
new String[0],
398392
Collections.singletonMap("foo", Collections.singletonMap("dummy", createFieldCapabilities("foo", "dummy"))));
399-
Map<String, Object> explicitlyMappedFields =
400-
new Classification("foo").getExplicitlyMappedFields("results", fieldCapabilitiesResponse);
401-
assertThat(explicitlyMappedFields, hasEntry("results.foo_prediction", singletonMap("type", "dummy")));
402-
assertThat(explicitlyMappedFields, hasEntry("results.top_classes", expectedTopClassesMapping));
403-
assertThat(explicitlyMappedFields, hasEntry("results.feature_importance", Classification.FEATURE_IMPORTANCE_MAPPING));
393+
394+
Map<String, Object> resultMappings =
395+
new Classification("foo").getResultMappings("results", fieldCapabilitiesResponse);
396+
397+
assertThat(resultMappings, hasEntry("results.foo_prediction", singletonMap("type", "dummy")));
398+
assertThat(resultMappings, hasEntry("results.prediction_probability", singletonMap("type", "double")));
399+
assertThat(resultMappings, hasEntry("results.prediction_score", singletonMap("type", "double")));
400+
assertThat(resultMappings, hasEntry("results.is_training", singletonMap("type", "boolean")));
401+
assertThat(resultMappings, hasEntry("results.top_classes", expectedTopClassesMapping));
402+
assertThat(resultMappings, hasEntry("results.feature_importance", Classification.FEATURE_IMPORTANCE_MAPPING));
404403
}
405404

406405
public void testToXContent_GivenVersionBeforeRandomizeSeedWasIntroduced() throws IOException {

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetectionTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,8 +107,8 @@ public void testFieldCardinalityLimitsIsEmpty() {
107107
assertThat(createTestInstance().getFieldCardinalityConstraints(), is(empty()));
108108
}
109109

110-
public void testGetExplicitlyMappedFields() {
111-
Map<String, Object> mappedFields = createTestInstance().getExplicitlyMappedFields("test", null);
110+
public void testGetResultMappings() {
111+
Map<String, Object> mappedFields = createTestInstance().getResultMappings("test", null);
112112
assertThat(mappedFields.size(), equalTo(2));
113113
assertThat(mappedFields, hasKey("test.outlier_score"));
114114
assertThat(mappedFields.get("test.outlier_score"),

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/RegressionTests.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -321,10 +321,11 @@ public void testFieldCardinalityLimitsIsEmpty() {
321321
assertThat(createTestInstance().getFieldCardinalityConstraints(), is(empty()));
322322
}
323323

324-
public void testGetExplicitlyMappedFields() {
325-
Map<String, Object> explicitlyMappedFields = new Regression("foo").getExplicitlyMappedFields("results", null);
326-
assertThat(explicitlyMappedFields, hasEntry("results.foo_prediction", Collections.singletonMap("type", "double")));
327-
assertThat(explicitlyMappedFields, hasEntry("results.feature_importance", Regression.FEATURE_IMPORTANCE_MAPPING));
324+
public void testGetResultMappings() {
325+
Map<String, Object> resultMappings = new Regression("foo").getResultMappings("results", null);
326+
assertThat(resultMappings, hasEntry("results.foo_prediction", Collections.singletonMap("type", "double")));
327+
assertThat(resultMappings, hasEntry("results.feature_importance", Regression.FEATURE_IMPORTANCE_MAPPING));
328+
assertThat(resultMappings, hasEntry("results.is_training", Collections.singletonMap("type", "boolean")));
328329
}
329330

330331
public void testGetStateDocId() {

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ private static Map<String, Object> createAdditionalMappings(DataFrameAnalyticsCo
222222
incrementalIdMapping.put("type", NumberFieldMapper.NumberType.LONG.typeName());
223223
properties.put(INCREMENTAL_ID, incrementalIdMapping);
224224

225-
properties.putAll(config.getAnalysis().getExplicitlyMappedFields(config.getDest().getResultsField(), fieldCapabilitiesResponse));
225+
properties.putAll(config.getAnalysis().getResultMappings(config.getDest().getResultsField(), fieldCapabilitiesResponse));
226226
return properties;
227227
}
228228

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
1919

2020
import java.io.IOException;
21-
import java.util.Collections;
2221
import java.util.HashMap;
2322
import java.util.Iterator;
2423
import java.util.Map;
@@ -102,9 +101,12 @@ static ImmutableOpenMap<String, MappingMetadata> mergeMappings(DataFrameAnalytic
102101
return result.build();
103102
}
104103

105-
private static MappingMetadata createMappingMetadata(String type, Map<String, Object> mappings) {
104+
private static MappingMetadata createMappingMetadata(String type, Map<String, Object> properties) {
105+
Map<String, Object> mappings = new HashMap<>();
106+
mappings.put("dynamic", false);
107+
mappings.put("properties", properties);
106108
try {
107-
return new MappingMetadata(type, Collections.singletonMap("properties", mappings));
109+
return new MappingMetadata(type, mappings);
108110
} catch (IOException e) {
109111
throw ExceptionsHelper.serverError("Failed to parse mappings: " + mappings);
110112
}

‎x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/MappingsMergerTests.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,13 @@ public void testMergeMappings_GivenIndicesWithIdenticalMappings() throws IOExcep
5252

5353
ImmutableOpenMap<String, MappingMetadata> mergedMappings = MappingsMerger.mergeMappings(newSource(), getMappingsResponse);
5454

55+
Map<String, Object> expectedMappings = new HashMap<>();
56+
expectedMappings.put("dynamic", false);
57+
expectedMappings.put("properties", index1Mappings.get("properties"));
58+
5559
assertThat(mergedMappings.size(), equalTo(1));
5660
assertThat(mergedMappings.containsKey("_doc"), is(true));
57-
assertThat(mergedMappings.valuesIt().next().getSourceAsMap(), equalTo(index1Mappings));
61+
assertThat(mergedMappings.valuesIt().next().getSourceAsMap(), equalTo(expectedMappings));
5862
}
5963

6064
public void testMergeMappings_GivenIndicesWithDifferentTypes() throws IOException {
@@ -142,7 +146,9 @@ public void testMergeMappings_GivenIndicesWithDifferentMappingsButNoConflicts()
142146
assertThat(mergedMappings.size(), equalTo(1));
143147
assertThat(mergedMappings.containsKey("_doc"), is(true));
144148
Map<String, Object> mappingsAsMap = mergedMappings.valuesIt().next().getSourceAsMap();
145-
assertThat(mappingsAsMap.size(), equalTo(1));
149+
assertThat(mappingsAsMap.size(), equalTo(2));
150+
assertThat(mappingsAsMap.containsKey("dynamic"), is(true));
151+
assertThat(mappingsAsMap.get("dynamic"), equalTo(false));
146152
assertThat(mappingsAsMap.containsKey("properties"), is(true));
147153

148154
@SuppressWarnings("unchecked")

0 commit comments

Comments
 (0)