diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsage.java index 2ebbef6fc183f..ec339d5b9ec77 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsage.java @@ -17,25 +17,36 @@ import org.elasticsearch.xpack.core.XPackField; import java.io.IOException; +import java.util.Collections; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; public class TransformFeatureSetUsage extends Usage { + private static final String FEATURE_COUNTS = "feature_counts"; + private final Map transformCountByState; + private final Map transformCountByFeature; private final TransformIndexerStats accumulatedStats; public TransformFeatureSetUsage(StreamInput in) throws IOException { super(in); this.transformCountByState = in.readMap(StreamInput::readString, StreamInput::readLong); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO: V_7_13_0 + this.transformCountByFeature = in.readMap(StreamInput::readString, StreamInput::readLong); + } else { + this.transformCountByFeature = Collections.emptyMap(); + } this.accumulatedStats = new TransformIndexerStats(in); } public TransformFeatureSetUsage(Map transformCountByState, - TransformIndexerStats accumulatedStats) { + Map transformCountByFeature, + TransformIndexerStats accumulatedStats) { super(XPackField.TRANSFORM, true, true); this.transformCountByState = Objects.requireNonNull(transformCountByState); + this.transformCountByFeature = Objects.requireNonNull(transformCountByFeature); this.accumulatedStats = Objects.requireNonNull(accumulatedStats); } @@ -48,6 +59,9 @@ public Version getMinimalSupportedVersion() { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeMap(transformCountByState, StreamOutput::writeString, StreamOutput::writeLong); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO: V_7_13_0 + out.writeMap(transformCountByFeature, StreamOutput::writeString, StreamOutput::writeLong); + } accumulatedStats.writeTo(out); } @@ -55,15 +69,19 @@ public void writeTo(StreamOutput out) throws IOException { protected void innerXContent(XContentBuilder builder, Params params) throws IOException { super.innerXContent(builder, params); if (transformCountByState.isEmpty() == false) { + // Transforms by state builder.startObject(TransformField.TRANSFORMS.getPreferredName()); long all = 0L; for (Entry entry : transformCountByState.entrySet()) { builder.field(entry.getKey(), entry.getValue()); - all+=entry.getValue(); + all += entry.getValue(); } builder.field(Metadata.ALL, all); builder.endObject(); + // Transform count for each feature + builder.field(FEATURE_COUNTS, transformCountByFeature); + // if there are no transforms, do not show any stats builder.field(TransformField.STATS_FIELD.getPreferredName(), accumulatedStats); } @@ -71,7 +89,7 @@ protected void innerXContent(XContentBuilder builder, Params params) throws IOEx @Override public int hashCode() { - return Objects.hash(enabled, available, transformCountByState, accumulatedStats); + return Objects.hash(enabled, available, transformCountByState, transformCountByFeature, accumulatedStats); } @Override @@ -85,6 +103,7 @@ public boolean equals(Object obj) { TransformFeatureSetUsage other = (TransformFeatureSetUsage) obj; return Objects.equals(name, other.name) && available == other.available && enabled == other.enabled && Objects.equals(transformCountByState, other.transformCountByState) + && Objects.equals(transformCountByFeature, other.transformCountByFeature) && Objects.equals(accumulatedStats, other.accumulatedStats); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java index 2870bba177b65..bfa7ae104439f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java @@ -10,7 +10,7 @@ import org.elasticsearch.common.ParseField; /* - * Utility class to hold common fields and strings for data frame. + * Utility class to hold common fields and strings for transform. */ public final class TransformField { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index ed018646063c3..17d2f8e22a1f5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -35,6 +35,7 @@ import java.time.Instant; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -51,10 +52,20 @@ public class TransformConfig extends AbstractDiffable implement /** Version in which {@code FieldCapabilitiesRequest.runtime_fields} field was introduced. */ private static final Version FIELD_CAPS_RUNTIME_MAPPINGS_INTRODUCED_VERSION = Version.V_7_12_0; - // types of transforms - public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot"); - public static final ParseField LATEST_TRANSFORM = new ParseField("latest"); + /** Specifies all the possible transform functions. */ + public enum Function { + PIVOT, LATEST; + private final ParseField parseField; + + Function() { + this.parseField = new ParseField(name().toLowerCase(Locale.ROOT)); + } + + public ParseField getParseField() { + return parseField; + } + } private static final ConstructingObjectParser STRICT_PARSER = createParser(false); private static final ConstructingObjectParser LENIENT_PARSER = createParser(true); static final int MAX_DESCRIPTION_LENGTH = 1_000; @@ -149,8 +160,8 @@ private static ConstructingObjectParser createParser(bo parser.declareNamedObject(optionalConstructorArg(), (p, c, n) -> p.namedObject(SyncConfig.class, n, c), TransformField.SYNC); parser.declareString(optionalConstructorArg(), TransformField.INDEX_DOC_TYPE); parser.declareObject(optionalConstructorArg(), (p, c) -> p.mapStrings(), HEADERS); - parser.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p, lenient), PIVOT_TRANSFORM); - parser.declareObject(optionalConstructorArg(), (p, c) -> LatestConfig.fromXContent(p, lenient), LATEST_TRANSFORM); + parser.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p, lenient), Function.PIVOT.getParseField()); + parser.declareObject(optionalConstructorArg(), (p, c) -> LatestConfig.fromXContent(p, lenient), Function.LATEST.getParseField()); parser.declareString(optionalConstructorArg(), TransformField.DESCRIPTION); parser.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p, lenient), TransformField.SETTINGS); parser.declareNamedObject( @@ -429,10 +440,10 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.endObject(); } if (pivotConfig != null) { - builder.field(PIVOT_TRANSFORM.getPreferredName(), pivotConfig); + builder.field(Function.PIVOT.getParseField().getPreferredName(), pivotConfig); } if (latestConfig != null) { - builder.field(LATEST_TRANSFORM.getPreferredName(), latestConfig); + builder.field(Function.LATEST.getParseField().getPreferredName(), latestConfig); } if (description != null) { builder.field(TransformField.DESCRIPTION.getPreferredName(), description); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java index 293ac9055c3c0..862f10e997e59 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java @@ -31,8 +31,8 @@ public final class TransformInternalIndexConstants { public static final String TRANSFORM_PREFIX_DEPRECATED = ".data-frame-"; // version is not a rollover pattern, however padded because sort is string based - public static final Version INDEX_VERSION_LAST_CHANGED = Version.V_7_12_0; - public static final String INDEX_VERSION = "006"; + public static final Version INDEX_VERSION_LAST_CHANGED = Version.V_7_13_0; + public static final String INDEX_VERSION = "007"; public static final String INDEX_PATTERN = TRANSFORM_PREFIX + "internal-"; public static final String LATEST_INDEX_VERSIONED_NAME = INDEX_PATTERN + INDEX_VERSION; public static final String LATEST_INDEX_NAME = LATEST_INDEX_VERSIONED_NAME; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsageTests.java index 2ce6ac71902b3..3b5620ba0b9d5 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsageTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsageTests.java @@ -10,27 +10,30 @@ import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xpack.core.indexing.IndexerState; +import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats; import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStatsTests; -import java.util.HashMap; +import java.util.Arrays; import java.util.Map; +import static java.util.stream.Collectors.toMap; + public class TransformFeatureSetUsageTests extends AbstractWireSerializingTestCase { @Override protected TransformFeatureSetUsage createTestInstance() { - Map transformCountByState = new HashMap<>(); - - if (randomBoolean()) { - transformCountByState.put(randomFrom(IndexerState.values()).toString(), randomLong()); - } - - return new TransformFeatureSetUsage(transformCountByState, TransformIndexerStatsTests.randomStats()); + Map transformCountByState = + randomSubsetOf(Arrays.asList(IndexerState.values())).stream() + .collect(toMap(state -> state.value(), state -> randomLong())); + Map transformCountByFeature = + randomList(10, () -> randomAlphaOfLength(10)).stream() + .collect(toMap(f -> f, f -> randomLong())); + TransformIndexerStats accumulatedStats = TransformIndexerStatsTests.randomStats(); + return new TransformFeatureSetUsage(transformCountByState, transformCountByFeature, accumulatedStats); } @Override protected Reader instanceReader() { return TransformFeatureSetUsage::new; } - } diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java index ae98a220aebb5..7b8dfec8887a9 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java @@ -247,8 +247,6 @@ protected void createReviewsIndexNano() throws IOException { protected void createContinuousPivotReviewsTransform(String transformId, String transformIndex, String authHeader) throws IOException { - final Request createTransformRequest = createRequestWithAuth("PUT", getTransformEndpoint() + transformId, authHeader); - String config = "{ \"dest\": {\"index\":\"" + transformIndex + "\"}," + " \"source\": {\"index\":\"" + REVIEWS_INDEX_NAME + "\"}," // Set frequency high for testing + " \"sync\": {\"time\":{\"field\": \"timestamp\", \"delay\": \"15m\"}}," @@ -266,10 +264,7 @@ protected void createContinuousPivotReviewsTransform(String transformId, String + " } } } }" + "}"; - createTransformRequest.setJsonEntity(config); - - Map createTransformResponse = entityAsMap(client().performRequest(createTransformRequest)); - assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE)); + createReviewsTransform(transformId, authHeader, config); } protected void createPivotReviewsTransform( @@ -280,8 +275,6 @@ protected void createPivotReviewsTransform( String authHeader, String sourceIndex ) throws IOException { - final Request createTransformRequest = createRequestWithAuth("PUT", getTransformEndpoint() + transformId, authHeader); - String config = "{"; if (pipeline != null) { @@ -315,6 +308,25 @@ protected void createPivotReviewsTransform( + "\"frequency\":\"1s\"" + "}"; + createReviewsTransform(transformId, authHeader, config); + } + + protected void createLatestReviewsTransform(String transformId, String transformIndex) throws IOException { + String config = "{" + + " \"dest\": {\"index\":\"" + transformIndex + "\"}," + + " \"source\": {\"index\":\"" + REVIEWS_INDEX_NAME + "\"}," + + " \"latest\": {" + + " \"unique_key\": [ \"user_id\" ]," + + " \"sort\": \"@timestamp\"" + + " }," + + "\"frequency\":\"1s\"" + + "}"; + + createReviewsTransform(transformId, null, config); + } + + private void createReviewsTransform(String transformId, String authHeader, String config) throws IOException { + final Request createTransformRequest = createRequestWithAuth("PUT", getTransformEndpoint() + transformId, authHeader); createTransformRequest.setJsonEntity(config); Map createTransformResponse = entityAsMap(client().performRequest(createTransformRequest)); diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUsageIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUsageIT.java index 92dc87a0c26ac..51c794901c93d 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUsageIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUsageIT.java @@ -40,17 +40,23 @@ public void testUsage() throws Exception { assertTrue((boolean) XContentMapValues.extractValue("transform.available", usageAsMap)); assertTrue((boolean) XContentMapValues.extractValue("transform.enabled", usageAsMap)); // no transforms, no stats - assertEquals(null, XContentMapValues.extractValue("transform.transforms", usageAsMap)); - assertEquals(null, XContentMapValues.extractValue("transform.stats", usageAsMap)); + assertNull(XContentMapValues.extractValue("transform.transforms", usageAsMap)); + assertNull(XContentMapValues.extractValue("transform.feature_counts", usageAsMap)); + assertNull(XContentMapValues.extractValue("transform.stats", usageAsMap)); // create transforms createPivotReviewsTransform("test_usage", "pivot_reviews", null); createPivotReviewsTransform("test_usage_no_stats", "pivot_reviews_no_stats", null); createContinuousPivotReviewsTransform("test_usage_continuous", "pivot_reviews_continuous", null); + createLatestReviewsTransform("test_usage_latest", "latest_reviews"); usageResponse = client().performRequest(new Request("GET", "_xpack/usage")); usageAsMap = entityAsMap(usageResponse); - assertEquals(3, XContentMapValues.extractValue("transform.transforms._all", usageAsMap)); - assertEquals(3, XContentMapValues.extractValue("transform.transforms.stopped", usageAsMap)); + assertEquals(4, XContentMapValues.extractValue("transform.transforms._all", usageAsMap)); + assertEquals(4, XContentMapValues.extractValue("transform.transforms.stopped", usageAsMap)); + assertEquals(3, XContentMapValues.extractValue("transform.feature_counts.pivot", usageAsMap)); + assertEquals(1, XContentMapValues.extractValue("transform.feature_counts.latest", usageAsMap)); + assertEquals(0, XContentMapValues.extractValue("transform.feature_counts.retention_policy", usageAsMap)); + assertEquals(1, XContentMapValues.extractValue("transform.feature_counts.sync", usageAsMap)); startAndWaitForTransform("test_usage", "pivot_reviews"); stopTransform("test_usage", false); @@ -100,9 +106,13 @@ public void testUsage() throws Exception { Response response = client().performRequest(new Request("GET", "_xpack/usage")); Map statsMap = entityAsMap(response); // we should see some stats - assertEquals(3, XContentMapValues.extractValue("transform.transforms._all", statsMap)); - assertEquals(2, XContentMapValues.extractValue("transform.transforms.stopped", statsMap)); + assertEquals(4, XContentMapValues.extractValue("transform.transforms._all", statsMap)); + assertEquals(3, XContentMapValues.extractValue("transform.transforms.stopped", statsMap)); assertEquals(1, XContentMapValues.extractValue("transform.transforms.started", statsMap)); + assertEquals(3, XContentMapValues.extractValue("transform.feature_counts.pivot", statsMap)); + assertEquals(1, XContentMapValues.extractValue("transform.feature_counts.latest", statsMap)); + assertEquals(0, XContentMapValues.extractValue("transform.feature_counts.retention_policy", statsMap)); + assertEquals(1, XContentMapValues.extractValue("transform.feature_counts.sync", statsMap)); for (String statName : PROVIDED_STATS) { // the trigger count can be off: e.g. if the scheduler kicked in before usage has been called, // or if the scheduler triggered later, but state hasn't been persisted (by design) @@ -130,11 +140,15 @@ public void testUsage() throws Exception { usageResponse = client().performRequest(new Request("GET", "_xpack/usage")); usageAsMap = entityAsMap(usageResponse); - assertEquals(3, XContentMapValues.extractValue("transform.transforms._all", usageAsMap)); - assertEquals(3, XContentMapValues.extractValue("transform.transforms.stopped", usageAsMap)); + assertEquals(4, XContentMapValues.extractValue("transform.transforms._all", usageAsMap)); + assertEquals(4, XContentMapValues.extractValue("transform.transforms.stopped", usageAsMap)); + assertEquals(3, XContentMapValues.extractValue("transform.feature_counts.pivot", usageAsMap)); + assertEquals(1, XContentMapValues.extractValue("transform.feature_counts.latest", usageAsMap)); + assertEquals(0, XContentMapValues.extractValue("transform.feature_counts.retention_policy", usageAsMap)); + assertEquals(1, XContentMapValues.extractValue("transform.feature_counts.sync", usageAsMap)); } - private double extractStatsAsDouble(Object statsObject) { + private static double extractStatsAsDouble(Object statsObject) { if (statsObject instanceof Integer) { return ((Integer) statsObject).doubleValue(); } else if (statsObject instanceof Double) { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformUsageTransportAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformUsageTransportAction.java index f64818492561f..dbbb485c40ab0 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformUsageTransportAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformUsageTransportAction.java @@ -8,6 +8,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchRequest; @@ -17,10 +18,15 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.protocol.xpack.XPackUsageRequest; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.filter.Filters; +import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -42,11 +48,29 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.toMap; public class TransformUsageTransportAction extends XPackUsageFeatureTransportAction { private static final Logger logger = LogManager.getLogger(TransformUsageTransportAction.class); + private static final String FEATURE_COUNTS = "feature_counts"; + + /** + * Features we want to measure the usage of. + * + * Each feature corresponds to a field in {@link TransformConfig}. + * If the field exists in the config then we assume the feature is used. + */ + private static final String[] FEATURES = + Stream.concat( + Stream.of(TransformConfig.Function.values()).map(TransformConfig.Function::getParseField), + Stream.of(TransformField.RETENTION_POLICY, TransformField.SYNC)) + .map(ParseField::getPreferredName) + .toArray(String[]::new); + private final Client client; @Inject @@ -73,13 +97,13 @@ public TransformUsageTransportAction( protected void masterOperation( Task task, XPackUsageRequest request, - ClusterState state, + ClusterState clusterState, ActionListener listener ) { - PersistentTasksCustomMetadata taskMetadata = PersistentTasksCustomMetadata.getPersistentTasksCustomMetadata(state); + PersistentTasksCustomMetadata taskMetadata = PersistentTasksCustomMetadata.getPersistentTasksCustomMetadata(clusterState); Collection> transformTasks = taskMetadata == null ? Collections.emptyList() - : taskMetadata.findTasks(TransformTaskParams.NAME, (t) -> true); + : taskMetadata.findTasks(TransformTaskParams.NAME, t -> true); final int taskCount = transformTasks.size(); final Map transformsCountByState = new HashMap<>(); for (PersistentTasksCustomMetadata.PersistentTask transformTask : transformTasks) { @@ -89,9 +113,10 @@ protected void masterOperation( transformsCountByState.merge(taskState.value(), 1L, Long::sum); } } + final SetOnce> transformsCountByFeature = new SetOnce<>(); ActionListener totalStatsListener = ActionListener.wrap(statSummations -> { - var usage = new TransformFeatureSetUsage(transformsCountByState, statSummations); + var usage = new TransformFeatureSetUsage(transformsCountByState, transformsCountByFeature.get(), statSummations); listener.onResponse(new XPackUsageFeatureResponse(usage)); }, listener::onFailure); @@ -104,11 +129,12 @@ protected void masterOperation( } long totalTransforms = transformCountSuccess.getHits().getTotalHits().value; if (totalTransforms == 0) { - var usage = new TransformFeatureSetUsage(transformsCountByState, new TransformIndexerStats()); + var usage = new TransformFeatureSetUsage(transformsCountByState, Collections.emptyMap(), new TransformIndexerStats()); listener.onResponse(new XPackUsageFeatureResponse(usage)); return; } transformsCountByState.merge(TransformTaskState.STOPPED.value(), totalTransforms - taskCount, Long::sum); + transformsCountByFeature.set(getFeatureCounts(transformCountSuccess.getAggregations())); TransformInfoTransportAction.getStatisticSummations(client, totalStatsListener); }, transformCountFailure -> { if (transformCountFailure instanceof ResourceNotFoundException) { @@ -118,25 +144,48 @@ protected void masterOperation( } }); - SearchRequest totalTransformCount = client.prepareSearch( + SearchRequest totalTransformCountSearchRequest = client.prepareSearch( TransformInternalIndexConstants.INDEX_NAME_PATTERN, TransformInternalIndexConstants.INDEX_NAME_PATTERN_DEPRECATED ) .setTrackTotalHits(true) + // We only need the total hits count and aggs. + .setSize(0) + .setFetchSource(false) .setQuery( QueryBuilders.constantScoreQuery( QueryBuilders.boolQuery() .filter(QueryBuilders.termQuery(TransformField.INDEX_DOC_TYPE.getPreferredName(), TransformConfig.NAME)) ) ) + .addAggregation( + AggregationBuilders.filters( + FEATURE_COUNTS, + Arrays.stream(FEATURES) + .map(f -> new FiltersAggregator.KeyedFilter(f, QueryBuilders.existsQuery(f))) + .toArray(FiltersAggregator.KeyedFilter[]::new) + ) + ) .request(); ClientHelper.executeAsyncWithOrigin( client.threadPool().getThreadContext(), ClientHelper.TRANSFORM_ORIGIN, - totalTransformCount, + totalTransformCountSearchRequest, totalTransformCountListener, client::search ); } + + /** + * Returns the feature usage map. + * For each feature it counts the number of transforms using this feature. + * + * @param aggs aggs returned by the search + * @return feature usage map + */ + private static Map getFeatureCounts(Aggregations aggs) { + Filters filters = aggs.get(FEATURE_COUNTS); + return filters.getBuckets().stream().collect(toMap(Filters.Bucket::getKeyAsString, Filters.Bucket::getDocCount)); + } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java index d0875d65faeb3..073282667f6e8 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java @@ -34,6 +34,7 @@ import org.elasticsearch.xpack.core.transform.transforms.DestConfig; import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint; +import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats; import org.elasticsearch.xpack.core.transform.transforms.TransformProgress; import org.elasticsearch.xpack.core.transform.transforms.TransformState; @@ -65,6 +66,7 @@ public final class TransformInternalIndex { * checkpoint::checkpoint * version 5 (7.7): stats::processing_time_in_ms, stats::processing_total * version 6 (7.12):stats::delete_time_in_ms, stats::documents_deleted + * version 7 (7.13):add mapping for config::pivot, config::latest, config::retention_policy and config::sync */ // constants for mappings @@ -84,6 +86,7 @@ public final class TransformInternalIndex { public static final String LONG = "long"; public static final String KEYWORD = "keyword"; public static final String BOOLEAN = "boolean"; + public static final String FLATTENED = "flattened"; public static SystemIndexDescriptor getSystemIndexDescriptor() throws IOException { return SystemIndexDescriptor.builder() @@ -319,6 +322,18 @@ public static XContentBuilder addTransformsConfigMappings(XContentBuilder builde .endObject() .startObject(TransformField.CREATE_TIME.getPreferredName()) .field(TYPE, DATE) + .endObject() + .startObject(TransformConfig.Function.PIVOT.getParseField().getPreferredName()) + .field(TYPE, FLATTENED) + .endObject() + .startObject(TransformConfig.Function.LATEST.getParseField().getPreferredName()) + .field(TYPE, FLATTENED) + .endObject() + .startObject(TransformField.RETENTION_POLICY.getPreferredName()) + .field(TYPE, FLATTENED) + .endObject() + .startObject(TransformField.SYNC.getPreferredName()) + .field(TYPE, FLATTENED) .endObject(); } diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml index 0993d3c9f3bb9..e78db56a50fc8 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml @@ -298,11 +298,11 @@ setup: - do: warnings: - - "this request accesses system indices: [.transform-internal-006], but in a future major version, direct access to system indices will be prevented by default" + - "this request accesses system indices: [.transform-internal-007], but in a future major version, direct access to system indices will be prevented by default" indices.get_mapping: - index: .transform-internal-006 - - match: { \.transform-internal-006.mappings.dynamic: "false" } - - match: { \.transform-internal-006.mappings.properties.id.type: "keyword" } + index: .transform-internal-007 + - match: { \.transform-internal-007.mappings.dynamic: "false" } + - match: { \.transform-internal-007.mappings.properties.id.type: "keyword" } - do: indices.get_mapping: index: .transform-notifications-000002