From 008a77e3effbd83984b0ee55d2d1d4ee53e72b51 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Tue, 6 Oct 2020 09:33:21 -0400 Subject: [PATCH 1/2] [ML] adding result_type and mlcategory fields to category definitions --- .../xpack/core/ml/job/results/CategoryDefinition.java | 5 +++++ .../xpack/core/ml/job/results/ReservedFieldNames.java | 1 + .../core/ml/anomalydetection/results_index_mappings.json | 3 +++ 3 files changed, 9 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/CategoryDefinition.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/CategoryDefinition.java index 75b41c69b2037..991a2b52adfea 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/CategoryDefinition.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/CategoryDefinition.java @@ -40,6 +40,7 @@ public class CategoryDefinition implements ToXContentObject, Writeable { public static final ParseField GROK_PATTERN = new ParseField("grok_pattern"); public static final ParseField NUM_MATCHES = new ParseField("num_matches"); public static final ParseField PREFERRED_TO_CATEGORIES = new ParseField("preferred_to_categories"); + public static final ParseField MLCATEGORY = new ParseField("mlcategory"); // Used for QueryPage public static final ParseField RESULTS_FIELD = new ParseField("categories"); @@ -62,6 +63,8 @@ private static ConstructingObjectParser createParser(b parser.declareString(CategoryDefinition::setGrokPattern, GROK_PATTERN); parser.declareLongArray(CategoryDefinition::setPreferredToCategories, PREFERRED_TO_CATEGORIES); parser.declareLong(CategoryDefinition::setNumMatches, NUM_MATCHES); + parser.declareString((cd, rt) -> { /*Ignore as it is always category_definition*/ }, Result.RESULT_TYPE); + parser.declareLong((cd, mc) -> { /*Ignore as it is always equal to category_id*/ }, MLCATEGORY); return parser; } @@ -246,6 +249,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (partitionFieldName != null && partitionFieldValue != null && ReservedFieldNames.isValidFieldName(partitionFieldName)) { builder.field(partitionFieldName, partitionFieldValue); } + builder.field(Result.RESULT_TYPE.getPreferredName(), TYPE.getPreferredName()); + builder.field(MLCATEGORY.getPreferredName(), categoryId); builder.endObject(); return builder; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java index f9bfcf1203a71..0173ce548ab42 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java @@ -121,6 +121,7 @@ public final class ReservedFieldNames { CategoryDefinition.EXAMPLES.getPreferredName(), CategoryDefinition.NUM_MATCHES.getPreferredName(), CategoryDefinition.PREFERRED_TO_CATEGORIES.getPreferredName(), + CategoryDefinition.MLCATEGORY.getPreferredName(), DataCounts.PROCESSED_RECORD_COUNT.getPreferredName(), DataCounts.PROCESSED_FIELD_COUNT.getPreferredName(), diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_mappings.json index 181fb466cba5a..50df5899383ea 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_mappings.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_mappings.json @@ -333,6 +333,9 @@ "missing_field_count" : { "type" : "long" }, + "mlcategory": { + "type": "keyword" + }, "model_bytes" : { "type" : "long" }, From 1e50280fedca1cf2dc6ab998c0f7acde49aa7d88 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Tue, 6 Oct 2020 14:14:37 -0400 Subject: [PATCH 2/2] addressing PR comments --- .../xpack/core/ml/job/results/CategoryDefinition.java | 6 ++++-- .../xpack/core/ml/job/results/ReservedFieldNames.java | 1 - .../core/ml/job/persistence/ElasticsearchMappingsTests.java | 3 +++ .../xpack/ml/job/persistence/JobResultsProvider.java | 2 ++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/CategoryDefinition.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/CategoryDefinition.java index 991a2b52adfea..dba4f3c8cb564 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/CategoryDefinition.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/CategoryDefinition.java @@ -64,7 +64,7 @@ private static ConstructingObjectParser createParser(b parser.declareLongArray(CategoryDefinition::setPreferredToCategories, PREFERRED_TO_CATEGORIES); parser.declareLong(CategoryDefinition::setNumMatches, NUM_MATCHES); parser.declareString((cd, rt) -> { /*Ignore as it is always category_definition*/ }, Result.RESULT_TYPE); - parser.declareLong((cd, mc) -> { /*Ignore as it is always equal to category_id*/ }, MLCATEGORY); + parser.declareString((cd, mc) -> { /*Ignore as it is always equal to category_id*/ }, MLCATEGORY); return parser; } @@ -249,8 +249,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (partitionFieldName != null && partitionFieldValue != null && ReservedFieldNames.isValidFieldName(partitionFieldName)) { builder.field(partitionFieldName, partitionFieldValue); } + // Even though category_definitions now have a result type, queries need for category definition values + // still need to be done by looking for the category_id field. At least until 9.x builder.field(Result.RESULT_TYPE.getPreferredName(), TYPE.getPreferredName()); - builder.field(MLCATEGORY.getPreferredName(), categoryId); + builder.field(MLCATEGORY.getPreferredName(), String.valueOf(categoryId)); builder.endObject(); return builder; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java index 0173ce548ab42..f9bfcf1203a71 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java @@ -121,7 +121,6 @@ public final class ReservedFieldNames { CategoryDefinition.EXAMPLES.getPreferredName(), CategoryDefinition.NUM_MATCHES.getPreferredName(), CategoryDefinition.PREFERRED_TO_CATEGORIES.getPreferredName(), - CategoryDefinition.MLCATEGORY.getPreferredName(), DataCounts.PROCESSED_RECORD_COUNT.getPreferredName(), DataCounts.PROCESSED_FIELD_COUNT.getPreferredName(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java index c3e274a416019..212acea98c974 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java @@ -95,6 +95,9 @@ public void testResultsMappingReservedFields() throws Exception { overridden.add(Quantiles.TYPE.getPreferredName()); overridden.add(TimingStats.TYPE.getPreferredName()); overridden.add(DatafeedTimingStats.TYPE.getPreferredName()); + // This is a special case so that categorical job results can be paired easily with anomaly results + // This is acceptable as both mappings are keyword for the results documents and for category definitions + overridden.add(CategoryDefinition.MLCATEGORY.getPreferredName()); Set expected = collectResultsDocFieldNames(); expected.removeAll(overridden); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java index ea7672bd3714b..33f3a68ed4ae8 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java @@ -825,6 +825,8 @@ public void categoryDefinitions(String jobId, Long categoryId, String partitionF if (categoryId != null) { categoryIdQuery = QueryBuilders.termQuery(CategoryDefinition.CATEGORY_ID.getPreferredName(), categoryId); } else if (from != null && size != null) { + // Note: Even though category definitions currently have a result_type field, this was not the case for older versions + // So, until at least 9.x, this existsQuery is still the preferred way to gather category definition objects categoryIdQuery = QueryBuilders.existsQuery(CategoryDefinition.CATEGORY_ID.getPreferredName()); sourceBuilder.from(from).size(size) .sort(new FieldSortBuilder(CategoryDefinition.CATEGORY_ID.getPreferredName()).order(SortOrder.ASC));