diff --git a/CHANGELOG.md b/CHANGELOG.md index f833a29a5131b..e68c175cc0e09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,37 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 3.x] ### Added +- Add search API tracker ([#18601](https://github.com/opensearch-project/OpenSearch/pull/18601)) +- Add support for linux riscv64 platform ([#18156](https://github.com/opensearch-project/OpenSearch/pull/18156)) +- [Rule based auto-tagging] Add get rule API ([#17336](https://github.com/opensearch-project/OpenSearch/pull/17336)) +- [Rule based auto-tagging] Add Delete Rule API ([#18184](https://github.com/opensearch-project/OpenSearch/pull/18184)) +- Implement parallel shard refresh behind cluster settings ([#17782](https://github.com/opensearch-project/OpenSearch/pull/17782)) +- Bump OpenSearch Core main branch to 3.0.0 ([#18039](https://github.com/opensearch-project/OpenSearch/pull/18039)) +- [Rule based Auto-tagging] Add wlm `ActionFilter` ([#17791](https://github.com/opensearch-project/OpenSearch/pull/17791)) +- Update API of Message in index to add the timestamp for lag calculation in ingestion polling ([#17977](https://github.com/opensearch-project/OpenSearch/pull/17977/)) +- Add Warm Disk Threshold Allocation Decider for Warm shards ([#18082](https://github.com/opensearch-project/OpenSearch/pull/18082)) +- Add composite directory factory ([#17988](https://github.com/opensearch-project/OpenSearch/pull/17988)) +- Add pull-based ingestion error metrics and make internal queue size configurable ([#18088](https://github.com/opensearch-project/OpenSearch/pull/18088)) +- Adding support for derive source feature and implementing it for various type of field mappers ([#17759](https://github.com/opensearch-project/OpenSearch/pull/17759)) +- [Security Manager Replacement] Enhance Java Agent to intercept newByteChannel ([#17989](https://github.com/opensearch-project/OpenSearch/pull/17989)) +- Enabled Async Shard Batch Fetch by default ([#18139](https://github.com/opensearch-project/OpenSearch/pull/18139)) +- Allow to get the search request from the QueryCoordinatorContext ([#17818](https://github.com/opensearch-project/OpenSearch/pull/17818)) +- Reject close index requests, while remote store migration is in progress.([#18327](https://github.com/opensearch-project/OpenSearch/pull/18327)) +- Improve sort-query performance by retaining the default `totalHitsThreshold` for approximated `match_all` queries ([#18189](https://github.com/opensearch-project/OpenSearch/pull/18189)) +- Enable testing for ExtensiblePlugins using classpath plugins ([#16908](https://github.com/opensearch-project/OpenSearch/pull/16908)) +- Introduce system generated ingest pipeline ([#17817](https://github.com/opensearch-project/OpenSearch/pull/17817))) +- Apply cluster state metadata and routing table diff when building cluster state from remote([#18256](https://github.com/opensearch-project/OpenSearch/pull/18256)) +- Support create mode in pull-based ingestion and add retries for transient failures ([#18250](https://github.com/opensearch-project/OpenSearch/pull/18250))) +- Decouple the init of Crypto Plugin and KeyProvider in CryptoRegistry ([18270](https://github.com/opensearch-project/OpenSearch/pull18270))) +- Support cluster write block in pull-based ingestion ([#18280](https://github.com/opensearch-project/OpenSearch/pull/18280))) +- Use QueryCoordinatorContext for the rewrite in validate API. ([#18272](https://github.com/opensearch-project/OpenSearch/pull/18272)) +- Upgrade crypto kms plugin dependencies for AWS SDK v2.x. ([#18268](https://github.com/opensearch-project/OpenSearch/pull/18268)) +- Add support for `matched_fields` with the unified highlighter ([#18164](https://github.com/opensearch-project/OpenSearch/issues/18164)) +- [repository-s3] Add support for SSE-KMS and S3 bucket owner verification ([#18312](https://github.com/opensearch-project/OpenSearch/pull/18312)) +- Optimize gRPC perf by passing by reference ([#18303](https://github.com/opensearch-project/OpenSearch/pull/18303)) +- Added File Cache Stats - Involves Block level as well as full file level stats ([#17538](https://github.com/opensearch-project/OpenSearch/issues/17479)) +- Added File Cache Pinning ([#17617](https://github.com/opensearch-project/OpenSearch/issues/13648)) +- Support consumer reset in Resume API for pull-based ingestion. This PR includes a breaking change for the experimental pull-based ingestion feature. ([#18332](https://github.com/opensearch-project/OpenSearch/pull/18332)) - Add hierarchical routing processors for ingest and search pipelines ([#18826](https://github.com/opensearch-project/OpenSearch/pull/18826)) - Add support for Warm Indices Write Block on Flood Watermark breach ([#18375](https://github.com/opensearch-project/OpenSearch/pull/18375)) - FS stats for warm nodes based on addressable space ([#18767](https://github.com/opensearch-project/OpenSearch/pull/18767)) diff --git a/libs/core/src/main/java/org/opensearch/core/rest/RestStatus.java b/libs/core/src/main/java/org/opensearch/core/rest/RestStatus.java index 8441ce8b1b622..3420d541f8935 100644 --- a/libs/core/src/main/java/org/opensearch/core/rest/RestStatus.java +++ b/libs/core/src/main/java/org/opensearch/core/rest/RestStatus.java @@ -536,6 +536,17 @@ public int getStatusFamilyCode() { return status / 100; } + public String getErrorType() { + int family = getStatusFamilyCode(); + if (family <= 3) { + return "success"; + } else if (family == 4) { + return "user_error"; + } else { + return "system_failure"; + } + } + public static RestStatus readFrom(StreamInput in) throws IOException { return RestStatus.valueOf(in.readString()); } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml index 01bdc323479c4..56d2bdb430d6d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml @@ -98,14 +98,14 @@ --- "Help from 2.14.0 to 3.0.99": - skip: - version: " - 2.13.99, 3.2.0 - " + version: " - 2.13.99, 3.1.0 - " reason: search idle reactivate count total is only added in 3.0.0 features: node_selector - do: cat.shards: help: true node_selector: - version: "2.14.0 - 3.1.99" + version: "2.14.0 - 3.2.99" - match: $body: | diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml index 784c7b52b18b4..10f86fc12f281 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml @@ -25,6 +25,7 @@ - is_true: nodes.$node_id.indices.segments - is_true: nodes.$node_id.indices.translog - is_true: nodes.$node_id.indices.recovery + - is_true: nodes.$node_id.indices.status_counter --- "Metric - _all": @@ -53,6 +54,7 @@ - is_true: nodes.$node_id.indices.segments - is_true: nodes.$node_id.indices.translog - is_true: nodes.$node_id.indices.recovery + - is_true: nodes.$node_id.indices.status_counter --- "Metric - indices _all": @@ -81,6 +83,7 @@ - is_true: nodes.$node_id.indices.segments - is_true: nodes.$node_id.indices.translog - is_true: nodes.$node_id.indices.recovery + - is_true: nodes.$node_id.indices.status_counter --- "Metric - one": @@ -109,6 +112,7 @@ - is_false: nodes.$node_id.indices.segments - is_false: nodes.$node_id.indices.translog - is_false: nodes.$node_id.indices.recovery + - is_true: nodes.$node_id.indices.status_counter --- "Metric - multi": @@ -137,6 +141,7 @@ - is_false: nodes.$node_id.indices.segments - is_false: nodes.$node_id.indices.translog - is_false: nodes.$node_id.indices.recovery + - is_true: nodes.$node_id.indices.status_counter --- "Metric - indexing doc_status": @@ -153,8 +158,38 @@ - is_false: nodes.$node_id.indices.docs - is_false: nodes.$node_id.indices.store - - is_true: nodes.$node_id.indices.indexing - - is_true: nodes.$node_id.indices.indexing.doc_status + - is_true: nodes.$node_id.indices.indexing + - is_false: nodes.$node_id.indices.get + - is_false: nodes.$node_id.indices.search + - is_false: nodes.$node_id.indices.merges + - is_false: nodes.$node_id.indices.refresh + - is_false: nodes.$node_id.indices.flush + - is_false: nodes.$node_id.indices.warmer + - is_false: nodes.$node_id.indices.query_cache + - is_false: nodes.$node_id.indices.fielddata + - is_false: nodes.$node_id.indices.completion + - is_false: nodes.$node_id.indices.segments + - is_false: nodes.$node_id.indices.translog + - is_false: nodes.$node_id.indices.recovery + - is_true: nodes.$node_id.indices.status_counter + - is_true: nodes.$node_id.indices.status_counter.doc_status + +--- +"Metric - indexing search_response_status": + - skip: + version: " - 2.10.99" + reason: "Doc Status Stats were introduced in v2.11.0" + - do: + nodes.info: {} + - set: + nodes._arbitrary_key_: node_id + + - do: + nodes.stats: { metric: indices, index_metric: indexing } + + - is_false: nodes.$node_id.indices.docs + - is_false: nodes.$node_id.indices.store + - is_true: nodes.$node_id.indices.indexing - is_false: nodes.$node_id.indices.get - is_false: nodes.$node_id.indices.search - is_false: nodes.$node_id.indices.merges @@ -167,6 +202,8 @@ - is_false: nodes.$node_id.indices.segments - is_false: nodes.$node_id.indices.translog - is_false: nodes.$node_id.indices.recovery + - is_true: nodes.$node_id.indices.status_counter + - is_true: nodes.$node_id.indices.status_counter.search_response_status --- "Metric - recovery": @@ -195,6 +232,7 @@ - is_false: nodes.$node_id.indices.segments - is_false: nodes.$node_id.indices.translog - is_true: nodes.$node_id.indices.recovery + - is_true: nodes.$node_id.indices.status_counter --- "Metric - _all include_segment_file_sizes": diff --git a/server/src/internalClusterTest/java/org/opensearch/nodestats/NodeStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/nodestats/NodeStatsIT.java index 22c1679babb52..602b48dabe1ca 100644 --- a/server/src/internalClusterTest/java/org/opensearch/nodestats/NodeStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/nodestats/NodeStatsIT.java @@ -13,6 +13,7 @@ import org.opensearch.action.admin.cluster.node.stats.NodeStats; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.opensearch.action.admin.indices.stats.CommonStatsFlags; +import org.opensearch.action.admin.indices.stats.DocStatusStats; import org.opensearch.action.bulk.BulkItemResponse; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkResponse; @@ -32,7 +33,6 @@ import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.engine.DocumentMissingException; import org.opensearch.index.engine.VersionConflictEngineException; -import org.opensearch.index.shard.IndexingStats.Stats.DocStatusStats; import org.opensearch.indices.NodeIndicesStats; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; @@ -47,7 +47,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; import static java.util.Collections.singletonMap; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -281,15 +281,14 @@ public void testNodeIndicesStatsDocStatsWithAggregations() { .getNodes() .get(0) .getIndices() - .getIndexing() - .getTotal() + .getStatusCounterStats() .getDocStatusStats(); assertTrue( Arrays.equals( docStatusStats.getDocStatusCounter(), expectedDocStatusStats.getDocStatusCounter(), - Comparator.comparingLong(AtomicLong::longValue) + Comparator.comparingLong(LongAdder::longValue) ) ); } @@ -543,15 +542,14 @@ private void assertDocStatusStats() { .getNodes() .get(0) .getIndices() - .getIndexing() - .getTotal() + .getStatusCounterStats() .getDocStatusStats(); assertTrue( Arrays.equals( docStatusStats.getDocStatusCounter(), expectedDocStatusStats.getDocStatusCounter(), - Comparator.comparingLong(AtomicLong::longValue) + Comparator.comparingLong(LongAdder::longValue) ) ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/msearch/MultiSearchStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/msearch/MultiSearchStatsIT.java new file mode 100644 index 0000000000000..3f03da0c04e69 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/search/msearch/MultiSearchStatsIT.java @@ -0,0 +1,111 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.search.msearch; + +import org.opensearch.ExceptionsHelper; +import org.opensearch.action.admin.indices.stats.SearchResponseStatusStats; +import org.opensearch.action.search.MultiSearchRequestBuilder; +import org.opensearch.action.search.MultiSearchResponse; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; +import org.opensearch.test.OpenSearchIntegTestCase.Scope; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.concurrent.atomic.LongAdder; + +@ClusterScope(scope = Scope.TEST, numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false) +public class MultiSearchStatsIT extends OpenSearchIntegTestCase { + private final SearchResponseStatusStats expectedSearchResponseStatusStats = new SearchResponseStatusStats(); + + public void testNodeIndicesStatsDocStatusStatsIndexBulk() { + createIndex("test"); + ensureGreen(); + client().prepareIndex("test").setId("1").setSource("field", "xxx").get(); + client().prepareIndex("test").setId("2").setSource("field", "yyy").get(); + refresh(); + + int failureCount = randomIntBetween(0, 50); + int successCount = randomIntBetween(0, 50); + + MultiSearchRequestBuilder multiSearchRequestBuilder = client().prepareMultiSearch(); + + for (int i = 0; i < failureCount; i++) { + multiSearchRequestBuilder.add(client().prepareSearch("noIndex").setQuery(QueryBuilders.termQuery("field", "yyy"))); + } + + for (int i = 0; i < successCount; i++) { + multiSearchRequestBuilder.add(client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery())); + } + + MultiSearchResponse response = multiSearchRequestBuilder.get(); + + for (MultiSearchResponse.Item item : response) { + if (item.isFailure()) { + updateExpectedDocStatusCounter(expectedSearchResponseStatusStats, item.getFailure()); + } else { + updateExpectedDocStatusCounter(expectedSearchResponseStatusStats, item.getResponse()); + } + } + assertSearchResponseStatusStats(); + } + + private void assertSearchResponseStatusStats() { + SearchResponseStatusStats searchResponseStatusStats = client().admin() + .cluster() + .prepareNodesStats() + .execute() + .actionGet() + .getNodes() + .get(0) + .getIndices() + .getStatusCounterStats() + .getSearchResponseStatusStats(); + + Arrays.equals( + searchResponseStatusStats.getSearchResponseStatusCounter(), + expectedSearchResponseStatusStats.getSearchResponseStatusCounter(), + Comparator.comparingLong(LongAdder::longValue) + ); + } + + private void updateExpectedDocStatusCounter(SearchResponseStatusStats expectedSearchResponseStatusStats, SearchResponse r) { + expectedSearchResponseStatusStats.inc(r.status()); + } + + private void updateExpectedDocStatusCounter(SearchResponseStatusStats expectedSearchResponseStatusStats, Exception e) { + expectedSearchResponseStatusStats.inc(ExceptionsHelper.status(e)); + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/DocStatusStats.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/DocStatusStats.java new file mode 100644 index 0000000000000..491e1ff478aac --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/DocStatusStats.java @@ -0,0 +1,161 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.action.admin.indices.stats; + +import org.opensearch.common.annotation.PublicApi; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.LongAdder; + +/** + * Tracks item level rest category class codes during indexing + * + * @opensearch.api + */ +@PublicApi(since = "1.0.0") +public class DocStatusStats implements Writeable, ToXContentFragment { + final LongAdder[] docStatusCounter; + + public DocStatusStats() { + docStatusCounter = new LongAdder[5]; + for (int i = 0; i < docStatusCounter.length; ++i) { + docStatusCounter[i] = new LongAdder(); + } + } + + public DocStatusStats(StreamInput in) throws IOException { + docStatusCounter = in.readArray(i -> { + LongAdder adder = new LongAdder(); + adder.add(i.readLong()); + return adder; + + }, LongAdder[]::new); + + assert docStatusCounter.length == 5 : "Length of incoming array should be 5! Got " + docStatusCounter.length; + } + + /** + * Increment counter for status + * + * @param status {@link RestStatus} + */ + public void inc(final RestStatus status) { + add(status, 1L); + } + + /** + * Increment counter for status by count + * + * @param status {@link RestStatus} + * @param delta The value to add + */ + public void add(final RestStatus status, final long delta) { + docStatusCounter[status.getStatusFamilyCode() - 1].add(delta); + } + + /** + * Accumulate stats from the passed Object + * + * @param stats Instance storing {@link DocStatusStats} + */ + public void add(final DocStatusStats stats) { + if (null == stats) { + return; + } + + for (int i = 0; i < docStatusCounter.length; ++i) { + docStatusCounter[i].add(stats.docStatusCounter[i].longValue()); + } + } + + public LongAdder[] getDocStatusCounter() { + return docStatusCounter; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(Fields.DOC_STATUS); + + Map errorTypeCounts = new HashMap<>(); + + for (int i = 0; i < docStatusCounter.length; ++i) { + long count = docStatusCounter[i].longValue(); + + if (count > 0) { + RestStatus familyStatus = RestStatus.fromCode((i + 1) * 100); + String errorType = familyStatus.getErrorType(); + errorTypeCounts.put(errorType, errorTypeCounts.getOrDefault(errorType, (long) 0) + count); + } + } + + String successType = RestStatus.ACCEPTED.getErrorType(); + String userFailureType = RestStatus.BAD_REQUEST.getErrorType(); + String systemErrorType = RestStatus.INTERNAL_SERVER_ERROR.getErrorType(); + builder.field(successType, errorTypeCounts.getOrDefault(successType, (long) 0)); + builder.field(userFailureType, errorTypeCounts.getOrDefault(userFailureType, (long) 0)); + builder.field(systemErrorType, errorTypeCounts.getOrDefault(systemErrorType, (long) 0)); + + return builder.endObject(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeArray((o, v) -> o.writeLong(v.longValue()), docStatusCounter); + } + + /** + * For this function, I just want to retrieve a point in time snapshot of the DocStatusStats. + */ + public DocStatusStats getSnapshot() { + DocStatusStats curSnapshot = new DocStatusStats(); + curSnapshot.add(this); + return curSnapshot; + } + + /** + * Fields for parsing and toXContent + * + * @opensearch.internal + */ + static final class Fields { + static final String DOC_STATUS = "doc_status"; + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/SearchResponseStatusStats.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/SearchResponseStatusStats.java new file mode 100644 index 0000000000000..09235d9c44605 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/SearchResponseStatusStats.java @@ -0,0 +1,161 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.action.admin.indices.stats; + +import org.opensearch.common.annotation.PublicApi; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.LongAdder; + +/** + * Tracks rest category class codes from search requests + * + * @opensearch.api + */ +@PublicApi(since = "1.0.0") +public class SearchResponseStatusStats implements Writeable, ToXContentFragment { + final LongAdder[] searchResponseStatusCounter; + + public SearchResponseStatusStats() { + searchResponseStatusCounter = new LongAdder[5]; + for (int i = 0; i < searchResponseStatusCounter.length; i++) { + searchResponseStatusCounter[i] = new LongAdder(); + } + } + + public SearchResponseStatusStats(StreamInput in) throws IOException { + searchResponseStatusCounter = in.readArray(i -> { + LongAdder adder = new LongAdder(); + adder.add(i.readLong()); + return adder; + + }, LongAdder[]::new); + + assert searchResponseStatusCounter.length == 5 : "Length of incoming array should be 5! Got " + searchResponseStatusCounter.length; + } + + /** + * Increment counter for status + * + * @param status {@link RestStatus} + */ + public void inc(final RestStatus status) { + add(status, 1L); + } + + /** + * Increment counter for status by count + * + * @param status {@link RestStatus} + * @param delta The value to add + */ + public void add(final RestStatus status, final long delta) { + searchResponseStatusCounter[status.getStatusFamilyCode() - 1].add(delta); + } + + /** + * Accumulate stats from the passed Object + * + * @param stats Instance storing {@link SearchResponseStatusStats} + */ + public void add(final SearchResponseStatusStats stats) { + if (null == stats) { + return; + } + + for (int i = 0; i < searchResponseStatusCounter.length; ++i) { + searchResponseStatusCounter[i].add(stats.searchResponseStatusCounter[i].longValue()); + } + } + + public LongAdder[] getSearchResponseStatusCounter() { + return searchResponseStatusCounter; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(Fields.SEARCH_RESPONSE_STATUS); + + Map errorTypeCounts = new HashMap<>(); + + for (int i = 0; i < searchResponseStatusCounter.length; ++i) { + long count = searchResponseStatusCounter[i].longValue(); + + if (count > 0) { + RestStatus familyStatus = RestStatus.fromCode((i + 1) * 100); + String errorType = familyStatus.getErrorType(); + errorTypeCounts.put(errorType, errorTypeCounts.getOrDefault(errorType, (long) 0) + count); + } + } + + String successType = RestStatus.ACCEPTED.getErrorType(); + String userFailureType = RestStatus.BAD_REQUEST.getErrorType(); + String systemErrorType = RestStatus.INTERNAL_SERVER_ERROR.getErrorType(); + builder.field(successType, errorTypeCounts.getOrDefault(successType, (long) 0)); + builder.field(userFailureType, errorTypeCounts.getOrDefault(userFailureType, (long) 0)); + builder.field(systemErrorType, errorTypeCounts.getOrDefault(systemErrorType, (long) 0)); + + return builder.endObject(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeArray((o, v) -> o.writeLong(v.longValue()), searchResponseStatusCounter); + } + + /** + * For this function, I just want to retrieve a point in time snapshot of the SearchResponseStatusStats. + */ + public SearchResponseStatusStats getSnapshot() { + SearchResponseStatusStats curSnapshot = new SearchResponseStatusStats(); + curSnapshot.add(this); + return curSnapshot; + } + + /** + * Fields for parsing and toXContent + * + * @opensearch.internal + */ + static final class Fields { + static final String SEARCH_RESPONSE_STATUS = "search_response_status"; + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/StatusCounterStats.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/StatusCounterStats.java new file mode 100644 index 0000000000000..44816e806ced0 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/StatusCounterStats.java @@ -0,0 +1,135 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.action.admin.indices.stats; + +import org.opensearch.Version; +import org.opensearch.common.Nullable; +import org.opensearch.common.annotation.PublicApi; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * REST status statistics for OpenSearch + * + * @opensearch.api + */ +@PublicApi(since = "1.0.0") +public class StatusCounterStats implements Writeable, ToXContentFragment { + + @Nullable + private DocStatusStats docStatusStats; + + @Nullable + private SearchResponseStatusStats searchResponseStatusStats; + + public StatusCounterStats() { + docStatusStats = new DocStatusStats(); + searchResponseStatusStats = new SearchResponseStatusStats(); + } + + public StatusCounterStats(DocStatusStats docStatusStats, SearchResponseStatusStats searchResponseStatusStats) { + this.docStatusStats = docStatusStats; + this.searchResponseStatusStats = searchResponseStatusStats; + } + + public StatusCounterStats(StreamInput in) throws IOException { + if (in.getVersion().onOrAfter(Version.V_3_2_0)) { + docStatusStats = in.readOptionalWriteable(DocStatusStats::new); + } else { + docStatusStats = null; + } + + if (in.getVersion().onOrAfter(Version.V_3_2_0)) { + searchResponseStatusStats = in.readOptionalWriteable(SearchResponseStatusStats::new); + } else { + searchResponseStatusStats = null; + } + } + + public DocStatusStats getDocStatusStats() { + return docStatusStats; + } + + public SearchResponseStatusStats getSearchResponseStatusStats() { + return searchResponseStatusStats; + } + + /** + * Gets a snapshot of the current state of the REST status counters. + */ + public StatusCounterStats getSnapshot() { + StatusCounterStats stats = new StatusCounterStats(); + stats.getDocStatusStats().add(docStatusStats); + stats.getSearchResponseStatusStats().add(searchResponseStatusStats); + return stats; + } + + public void add(StatusCounterStats stats) { + docStatusStats.add(stats.docStatusStats); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + if (out.getVersion().onOrAfter(Version.V_3_2_0)) { + out.writeOptionalWriteable(docStatusStats.getSnapshot()); + } + + if (out.getVersion().onOrAfter(Version.V_3_2_0)) { + out.writeOptionalWriteable(searchResponseStatusStats.getSnapshot()); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(Fields.STATUS_COUNTER); + docStatusStats.getSnapshot().toXContent(builder, params); + searchResponseStatusStats.getSnapshot().toXContent(builder, params); + builder.endObject(); + + return builder; + } + + /** + * Fields for parsing and toXContent + * + * @opensearch.internal + */ + static final class Fields { + static final String STATUS_COUNTER = "status_counter"; + } +} diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java index d0972e1276b39..8f0ced3333813 100644 --- a/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java @@ -46,6 +46,7 @@ import org.opensearch.action.admin.indices.create.AutoCreateAction; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.admin.indices.stats.DocStatusStats; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.ingest.IngestActionForwarder; import org.opensearch.action.support.ActionFilters; @@ -78,7 +79,6 @@ import org.opensearch.index.IndexingPressureService; import org.opensearch.index.VersionType; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.IndexingStats.Stats.DocStatusStats; import org.opensearch.indices.IndexClosedException; import org.opensearch.indices.IndicesService; import org.opensearch.indices.SystemIndices; diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java index 1da080e5bd302..701befafa00f4 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java @@ -32,6 +32,7 @@ package org.opensearch.action.search; +import org.opensearch.ExceptionsHelper; import org.opensearch.action.OriginalIndices; import org.opensearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.opensearch.action.admin.cluster.shards.ClusterSearchShardsGroup; @@ -69,6 +70,7 @@ import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.core.tasks.TaskId; import org.opensearch.index.query.Rewriteable; +import org.opensearch.indices.IndicesService; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchService; import org.opensearch.search.SearchShardTarget; @@ -176,6 +178,7 @@ public class TransportSearchAction extends HandledTransportAction) SearchRequest::new); this.client = client; @@ -217,6 +221,7 @@ public TransportSearchAction( this.searchRequestOperationsCompositeListenerFactory = searchRequestOperationsCompositeListenerFactory; this.tracer = tracer; this.taskResourceTrackingService = taskResourceTrackingService; + this.indicesService = indicesService; } private Map buildPerIndexAliasFilter( @@ -305,20 +310,38 @@ long buildTookInMillis() { @Override protected void doExecute(Task task, SearchRequest searchRequest, ActionListener listener) { + // searchStatusStatsUpdateListener will execute the logic within the original listener but + // also track the response status of the searchResponse. + ActionListener searchStatusStatsUpdateListener; // only if task is of type CancellableTask and support cancellation on timeout, treat this request eligible for timeout based // cancellation. There may be other top level requests like AsyncSearch which is using SearchRequest internally and has it's own // cancellation mechanism. For such cases, the SearchRequest when created can override the createTask and set the // cancelAfterTimeInterval to NO_TIMEOUT and bypass this mechanism if (task instanceof CancellableTask) { - listener = TimeoutTaskCancellationUtility.wrapWithCancellationListener( + ActionListener cancellationListener = TimeoutTaskCancellationUtility.wrapWithCancellationListener( client, (CancellableTask) task, clusterService.getClusterSettings().get(SEARCH_CANCEL_AFTER_TIME_INTERVAL_SETTING), listener, e -> {} ); + searchStatusStatsUpdateListener = ActionListener.wrap((searchResponse) -> { + cancellationListener.onResponse(searchResponse); + indicesService.getSearchResponseStatusStats().inc(searchResponse.status()); + }, (e) -> { + cancellationListener.onFailure(e); + indicesService.getSearchResponseStatusStats().inc(ExceptionsHelper.status(e)); + }); + } else { + searchStatusStatsUpdateListener = ActionListener.wrap((searchResponse) -> { + listener.onResponse(searchResponse); + indicesService.getSearchResponseStatusStats().inc(searchResponse.status()); + }, (e) -> { + listener.onFailure(e); + indicesService.getSearchResponseStatusStats().inc(ExceptionsHelper.status(e)); + }); } - executeRequest(task, searchRequest, this::searchAsyncAction, listener); + executeRequest(task, searchRequest, this::searchAsyncAction, searchStatusStatsUpdateListener); } /** diff --git a/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java b/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java index a8fad74b3b091..756d852b850f7 100644 --- a/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java +++ b/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java @@ -39,6 +39,7 @@ import org.opensearch.action.RoutingMissingException; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.admin.indices.stats.DocStatusStats; import org.opensearch.action.delete.DeleteRequest; import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.index.IndexRequest; @@ -71,7 +72,6 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.IndexingStats.Stats.DocStatusStats; import org.opensearch.indices.IndicesService; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java b/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java index 213ff3299fde1..bfde79cb0b250 100644 --- a/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java +++ b/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java @@ -622,6 +622,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.endObject(); } + return builder; } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexingStats.java b/server/src/main/java/org/opensearch/index/shard/IndexingStats.java index 60a93814da0e2..1e358c77bad15 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexingStats.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexingStats.java @@ -38,7 +38,6 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -46,7 +45,6 @@ import java.io.IOException; import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; /** * Tracks indexing statistics @@ -63,91 +61,6 @@ public class IndexingStats implements Writeable, ToXContentFragment { */ @PublicApi(since = "1.0.0") public static class Stats implements Writeable, ToXContentFragment { - - /** - * Tracks item level rest category class codes during indexing - * - * @opensearch.api - */ - @PublicApi(since = "1.0.0") - public static class DocStatusStats implements Writeable, ToXContentFragment { - - final AtomicLong[] docStatusCounter; - - public DocStatusStats() { - docStatusCounter = new AtomicLong[5]; - for (int i = 0; i < docStatusCounter.length; ++i) { - docStatusCounter[i] = new AtomicLong(0); - } - } - - public DocStatusStats(StreamInput in) throws IOException { - docStatusCounter = in.readArray(i -> new AtomicLong(i.readLong()), AtomicLong[]::new); - - assert docStatusCounter.length == 5 : "Length of incoming array should be 5! Got " + docStatusCounter.length; - } - - /** - * Increment counter for status - * - * @param status {@link RestStatus} - */ - public void inc(final RestStatus status) { - add(status, 1L); - } - - /** - * Increment counter for status by count - * - * @param status {@link RestStatus} - * @param delta The value to add - */ - void add(final RestStatus status, final long delta) { - docStatusCounter[status.getStatusFamilyCode() - 1].addAndGet(delta); - } - - /** - * Accumulate stats from the passed Object - * - * @param stats Instance storing {@link DocStatusStats} - */ - public void add(final DocStatusStats stats) { - if (null == stats) { - return; - } - - for (int i = 0; i < docStatusCounter.length; ++i) { - docStatusCounter[i].addAndGet(stats.docStatusCounter[i].longValue()); - } - } - - public AtomicLong[] getDocStatusCounter() { - return docStatusCounter; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(Fields.DOC_STATUS); - - for (int i = 0; i < docStatusCounter.length; ++i) { - long value = docStatusCounter[i].longValue(); - - if (value > 0) { - String key = i + 1 + "xx"; - builder.field(key, value); - } - } - - return builder.endObject(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeArray((o, v) -> o.writeLong(v.longValue()), docStatusCounter); - } - - } - private long indexCount; private long indexTimeInMillis; private long indexCurrent; @@ -158,12 +71,9 @@ public void writeTo(StreamOutput out) throws IOException { private long noopUpdateCount; private long throttleTimeInMillis; private boolean isThrottled; - private final DocStatusStats docStatusStats; private long maxLastIndexRequestTimestamp; - Stats() { - docStatusStats = new DocStatusStats(); - } + Stats() {} public Stats(StreamInput in) throws IOException { indexCount = in.readVLong(); @@ -181,11 +91,6 @@ public Stats(StreamInput in) throws IOException { } else { maxLastIndexRequestTimestamp = 0L; } - if (in.getVersion().onOrAfter(Version.V_2_11_0)) { - docStatusStats = in.readOptionalWriteable(DocStatusStats::new); - } else { - docStatusStats = null; - } } public Stats( @@ -198,8 +103,7 @@ public Stats( long deleteCurrent, long noopUpdateCount, boolean isThrottled, - long throttleTimeInMillis, - DocStatusStats docStatusStats + long throttleTimeInMillis ) { this( indexCount, @@ -212,7 +116,6 @@ public Stats( noopUpdateCount, isThrottled, throttleTimeInMillis, - docStatusStats, 0L ); } @@ -228,7 +131,6 @@ public Stats( long noopUpdateCount, boolean isThrottled, long throttleTimeInMillis, - DocStatusStats docStatusStats, long maxLastIndexRequestTimestamp ) { this.indexCount = indexCount; @@ -241,7 +143,6 @@ public Stats( this.noopUpdateCount = noopUpdateCount; this.isThrottled = isThrottled; this.throttleTimeInMillis = throttleTimeInMillis; - this.docStatusStats = docStatusStats; this.maxLastIndexRequestTimestamp = maxLastIndexRequestTimestamp; } @@ -258,11 +159,6 @@ public void add(Stats stats) { noopUpdateCount += stats.noopUpdateCount; throttleTimeInMillis += stats.throttleTimeInMillis; isThrottled |= stats.isThrottled; // When combining if one is throttled set result to throttled. - - if (getDocStatusStats() != null) { - getDocStatusStats().add(stats.getDocStatusStats()); - } - maxLastIndexRequestTimestamp = Math.max(maxLastIndexRequestTimestamp, stats.maxLastIndexRequestTimestamp); } @@ -333,10 +229,6 @@ public long getNoopUpdateCount() { return noopUpdateCount; } - public DocStatusStats getDocStatusStats() { - return docStatusStats; - } - public long getMaxLastIndexRequestTimestamp() { return maxLastIndexRequestTimestamp; } @@ -356,9 +248,6 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_3_2_0)) { out.writeLong(maxLastIndexRequestTimestamp); } - if (out.getVersion().onOrAfter(Version.V_2_11_0)) { - out.writeOptionalWriteable(docStatusStats); - } } @Override @@ -377,12 +266,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.IS_THROTTLED, isThrottled); builder.humanReadableField(Fields.THROTTLED_TIME_IN_MILLIS, Fields.THROTTLED_TIME, getThrottleTime()); - if (getDocStatusStats() != null) { - getDocStatusStats().toXContent(builder, params); - } - builder.field(Fields.MAX_LAST_INDEX_REQUEST_TIMESTAMP, maxLastIndexRequestTimestamp); - return builder; } @@ -455,7 +339,6 @@ private static final class Fields { static final String IS_THROTTLED = "is_throttled"; static final String THROTTLED_TIME_IN_MILLIS = "throttle_time_in_millis"; static final String THROTTLED_TIME = "throttle_time"; - static final String DOC_STATUS = "doc_status"; static final String MAX_LAST_INDEX_REQUEST_TIMESTAMP = "max_last_index_request_timestamp"; } diff --git a/server/src/main/java/org/opensearch/index/shard/InternalIndexingStats.java b/server/src/main/java/org/opensearch/index/shard/InternalIndexingStats.java index 9c901eacb1cc5..483d69927b5db 100644 --- a/server/src/main/java/org/opensearch/index/shard/InternalIndexingStats.java +++ b/server/src/main/java/org/opensearch/index/shard/InternalIndexingStats.java @@ -165,7 +165,6 @@ IndexingStats.Stats stats(boolean isThrottled, long currentThrottleMillis) { noopUpdates.count(), isThrottled, TimeUnit.MILLISECONDS.toMillis(currentThrottleMillis), - new IndexingStats.Stats.DocStatusStats(), maxLastIndexRequestTimestamp.get() ); } diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 243429135ea30..56bcef4d895df 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -45,8 +45,11 @@ import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.action.admin.indices.stats.CommonStatsFlags; import org.opensearch.action.admin.indices.stats.CommonStatsFlags.Flag; +import org.opensearch.action.admin.indices.stats.DocStatusStats; import org.opensearch.action.admin.indices.stats.IndexShardStats; +import org.opensearch.action.admin.indices.stats.SearchResponseStatusStats; import org.opensearch.action.admin.indices.stats.ShardStats; +import org.opensearch.action.admin.indices.stats.StatusCounterStats; import org.opensearch.action.search.SearchRequestStats; import org.opensearch.action.search.SearchType; import org.opensearch.cluster.ClusterState; @@ -141,7 +144,6 @@ import org.opensearch.index.shard.IndexShardState; import org.opensearch.index.shard.IndexingOperationListener; import org.opensearch.index.shard.IndexingStats; -import org.opensearch.index.shard.IndexingStats.Stats.DocStatusStats; import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.RemoteBlobStoreInternalTranslogFactory; @@ -416,6 +418,7 @@ public class IndicesService extends AbstractLifecycleComponent private final Function segmentReplicationStatsProvider; private volatile int maxSizeInRequestCache; private volatile int defaultMaxMergeAtOnce; + private final StatusCounterStats statusCounterStats; @Override protected void doStart() { @@ -589,6 +592,8 @@ protected void closeInternal() { this.defaultMaxMergeAtOnce = CLUSTER_DEFAULT_INDEX_MAX_MERGE_AT_ONCE_SETTING.get(clusterService.getSettings()); clusterService.getClusterSettings() .addSettingsUpdateConsumer(CLUSTER_DEFAULT_INDEX_MAX_MERGE_AT_ONCE_SETTING, this::onDefaultMaxMergeAtOnceUpdate); + + this.statusCounterStats = new StatusCounterStats(); } @InternalApi @@ -802,9 +807,9 @@ public NodeIndicesStats stats(CommonStatsFlags flags) { } if (flags.getIncludeIndicesStatsByLevel()) { NodeIndicesStats.StatsLevel statsLevel = NodeIndicesStats.getAcceptedLevel(flags.getLevels()); - return new NodeIndicesStats(commonStats, statsByShard(this, flags), searchRequestStats, statsLevel); + return new NodeIndicesStats(commonStats, statsByShard(this, flags), searchRequestStats, statusCounterStats, statsLevel); } else { - return new NodeIndicesStats(commonStats, statsByShard(this, flags), searchRequestStats); + return new NodeIndicesStats(commonStats, statsByShard(this, flags), searchRequestStats, statusCounterStats); } } @@ -1358,7 +1363,14 @@ public IndicesQueryCache getIndicesQueryCache() { * @param stats Instance storing {@link DocStatusStats} */ public void addDocStatusStats(final DocStatusStats stats) { - oldShardsStats.indexingStats.getTotal().getDocStatusStats().add(stats); + statusCounterStats.getDocStatusStats().add(stats); + } + + /** + * Retrieves the current statistics for search response (Not a snapshot). + */ + public SearchResponseStatusStats getSearchResponseStatusStats() { + return statusCounterStats.getSearchResponseStatusStats(); } /** diff --git a/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java b/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java index 4c28c08d8061b..029f90e177dfe 100644 --- a/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java +++ b/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java @@ -36,6 +36,7 @@ import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.action.admin.indices.stats.IndexShardStats; import org.opensearch.action.admin.indices.stats.ShardStats; +import org.opensearch.action.admin.indices.stats.StatusCounterStats; import org.opensearch.action.search.SearchRequestStats; import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; @@ -80,6 +81,7 @@ public class NodeIndicesStats implements Writeable, ToXContentFragment { protected CommonStats stats; protected Map statsByIndex; protected Map> statsByShard; + protected StatusCounterStats statusCounterStats; public NodeIndicesStats(StreamInput in) throws IOException { stats = new CommonStats(in); @@ -92,16 +94,29 @@ public NodeIndicesStats(StreamInput in) throws IOException { if (in.readBoolean()) { statsByShard = readStatsByShard(in); } + + if (in.getVersion().onOrAfter(Version.V_3_2_0)) { + statusCounterStats = new StatusCounterStats(in); + } } /** * Without passing the information of the levels to the constructor, we return the Node-level aggregated stats as * {@link CommonStats} along with a hash-map containing Index to List of Shard Stats. */ - public NodeIndicesStats(CommonStats oldStats, Map> statsByShard, SearchRequestStats searchRequestStats) { + public NodeIndicesStats( + CommonStats oldStats, + Map> statsByShard, + SearchRequestStats searchRequestStats, + StatusCounterStats statusCounterStats + ) { // this.stats = stats; this.statsByShard = statsByShard; + // statusCounterStats should be a snapshot of the statusCounters at a point in time, just like all the items in + // NodeIndicesStats should be. + this.statusCounterStats = statusCounterStats; + // make a total common stats from old ones and current ones this.stats = oldStats; for (List shardStatsList : statsByShard.values()) { @@ -126,6 +141,7 @@ public NodeIndicesStats( CommonStats oldStats, Map> statsByShard, SearchRequestStats searchRequestStats, + StatusCounterStats statusCounterStats, StatsLevel level ) { // make a total common stats from old ones and current ones @@ -142,6 +158,8 @@ public NodeIndicesStats( this.stats.search.setSearchRequestStats(searchRequestStats); } + this.statusCounterStats = statusCounterStats; + if (level != null) { switch (level) { case INDICES: @@ -280,6 +298,11 @@ public RecoveryStats getRecoveryStats() { return stats.getRecoveryStats(); } + @Nullable + public StatusCounterStats getStatusCounterStats() { + return statusCounterStats; + } + @Override public void writeTo(StreamOutput out) throws IOException { stats.writeTo(out); @@ -295,6 +318,12 @@ public void writeTo(StreamOutput out) throws IOException { if (statsByShard != null) { writeStatsByShards(out); } + + if (out.getVersion().onOrAfter(Version.V_3_2_0)) { + if (statusCounterStats != null) { + statusCounterStats.writeTo(out); + } + } } private void writeStatsByIndex(StreamOutput out) throws IOException { @@ -344,6 +373,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(StatsLevel.INDICES.getRestName()); stats.toXContent(builder, params); + statusCounterStats.toXContent(builder, params); + if (StatsLevel.INDICES.getRestName().equals(level)) { assert statsByIndex != null || statsByShard != null : "Expected shard stats or index stats in response for generating [" + StatsLevel.INDICES diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java index cccca0448a2cc..63b6acc0edc3e 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -37,6 +37,7 @@ import org.opensearch.action.admin.indices.stats.CommonStatsFlags; import org.opensearch.action.admin.indices.stats.IndexShardStats; import org.opensearch.action.admin.indices.stats.ShardStats; +import org.opensearch.action.admin.indices.stats.StatusCounterStats; import org.opensearch.action.search.SearchRequestStats; import org.opensearch.cluster.coordination.PendingClusterStateStats; import org.opensearch.cluster.coordination.PersistedStateStats; @@ -1058,7 +1059,8 @@ private static NodeIndicesStats getNodeIndicesStats(boolean remoteStoreStats) { indicesStats = new NodeIndicesStats( new CommonStats(CommonStatsFlags.ALL), new HashMap<>(), - new SearchRequestStats(clusterSettings) + new SearchRequestStats(clusterSettings), + new StatusCounterStats() ); RemoteSegmentStats remoteSegmentStats = indicesStats.getSegments().getRemoteSegmentStats(); remoteSegmentStats.addUploadBytesStarted(10L); @@ -1118,18 +1120,20 @@ public MockNodeIndicesStats(StreamInput in) throws IOException { public MockNodeIndicesStats( CommonStats oldStats, Map> statsByShard, - SearchRequestStats searchRequestStats + SearchRequestStats searchRequestStats, + StatusCounterStats statusCounterStats ) { - super(oldStats, statsByShard, searchRequestStats); + super(oldStats, statsByShard, searchRequestStats, statusCounterStats); } public MockNodeIndicesStats( CommonStats oldStats, Map> statsByShard, SearchRequestStats searchRequestStats, + StatusCounterStats statusCounterStats, StatsLevel level ) { - super(oldStats, statsByShard, searchRequestStats, level); + super(oldStats, statsByShard, searchRequestStats, statusCounterStats, level); } public CommonStats getStats() { @@ -1324,7 +1328,8 @@ public void testNodeIndicesStatsWithAndWithoutAggregations() throws IOException final MockNodeIndicesStats nonAggregatedNodeIndicesStats = new MockNodeIndicesStats( new CommonStats(commonStatsFlags), statsByShards, - new SearchRequestStats(clusterSettings) + new SearchRequestStats(clusterSettings), + new StatusCounterStats() ); commonStatsFlags.setIncludeIndicesStatsByLevel(true); @@ -1334,6 +1339,7 @@ public void testNodeIndicesStatsWithAndWithoutAggregations() throws IOException new CommonStats(commonStatsFlags), statsByShards, new SearchRequestStats(clusterSettings), + new StatusCounterStats(), level ); @@ -1481,10 +1487,16 @@ public MockNodeIndicesStats generateMockNodeIndicesStats( new CommonStats(commonStatsFlags), statsByShard, new SearchRequestStats(clusterSettings), + new StatusCounterStats(), level ); } else { - return new MockNodeIndicesStats(new CommonStats(commonStatsFlags), statsByShard, new SearchRequestStats(clusterSettings)); + return new MockNodeIndicesStats( + new CommonStats(commonStatsFlags), + statsByShard, + new SearchRequestStats(clusterSettings), + new StatusCounterStats() + ); } } } diff --git a/server/src/test/java/org/opensearch/core/RestStatusTests.java b/server/src/test/java/org/opensearch/core/RestStatusTests.java index fbd238bd035d0..db37a6ed7cdda 100644 --- a/server/src/test/java/org/opensearch/core/RestStatusTests.java +++ b/server/src/test/java/org/opensearch/core/RestStatusTests.java @@ -26,6 +26,7 @@ public void testStatusReturns200ForNoFailures() { int successfulShards = randomIntBetween(1, totalShards); assertEquals(RestStatus.OK, RestStatus.status(successfulShards, totalShards)); + assertEquals("success", RestStatus.status(successfulShards, totalShards).getErrorType()); } public void testStatusReturns503ForUnavailableShards() { @@ -33,6 +34,7 @@ public void testStatusReturns503ForUnavailableShards() { int successfulShards = 0; assertEquals(RestStatus.SERVICE_UNAVAILABLE, RestStatus.status(successfulShards, totalShards)); + assertEquals("system_failure", RestStatus.status(successfulShards, totalShards).getErrorType()); } public void testStatusReturnsFailureStatusWhenFailuresExist() { @@ -60,6 +62,7 @@ public void testStatusReturnsFailureStatusWhenFailuresExist() { final RestStatus expected = status.getStatusFamilyCode() == 1 ? RestStatus.OK : status; assertEquals(expected, RestStatus.status(successfulShards, totalShards, failures)); + assertEquals(expected.getErrorType(), RestStatus.status(successfulShards, totalShards, failures).getErrorType()); } public void testSerialization() throws IOException { diff --git a/server/src/test/java/org/opensearch/index/shard/IndexingStatsTests.java b/server/src/test/java/org/opensearch/index/shard/IndexingStatsTests.java index aafa58e3791a9..c90f230323c2e 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexingStatsTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexingStatsTests.java @@ -10,16 +10,9 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.rest.RestStatus; -import org.opensearch.core.xcontent.MediaTypeRegistry; -import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; -import java.util.Arrays; -import java.util.Comparator; -import java.util.concurrent.atomic.AtomicLong; public class IndexingStatsTests extends OpenSearchTestCase { @@ -51,7 +44,8 @@ public void testSerialization() throws IOException { assertEquals(totalStats.isThrottled(), deserializedTotalStats.isThrottled()); assertEquals(totalStats.getThrottleTime(), deserializedTotalStats.getThrottleTime()); - if (totalStats.getDocStatusStats() == null) { + // TODO: Make sure to test this logic in the NodeStatsTests.java + /*if (totalStats.getDocStatusStats() == null) { assertNull(deserializedTotalStats.getDocStatusStats()); return; } @@ -65,11 +59,12 @@ public void testSerialization() throws IOException { deserializedDocStatusStats.getDocStatusCounter(), Comparator.comparingLong(AtomicLong::longValue) ) - ); + );*/ } } } + /* public void testToXContentForIndexingStats() throws IOException { IndexingStats stats = createTestInstance(); IndexingStats.Stats totalStats = stats.getTotal(); @@ -115,7 +110,7 @@ public void testToXContentForIndexingStats() throws IOException { xContentBuilder.endObject(); assertEquals(expected, xContentBuilder.toString()); - } + }*/ /** * Tests aggregation logic for maxLastIndexRequestTimestamp in IndexingStats.Stats. @@ -124,13 +119,12 @@ public void testToXContentForIndexingStats() throws IOException { */ public void testMaxLastIndexRequestTimestampAggregation() throws Exception { // Use explicit values for all fields except the timestamp - IndexingStats.Stats.DocStatusStats docStatusStats = new IndexingStats.Stats.DocStatusStats(); long ts1 = randomLongBetween(0, 1000000); long ts2 = randomLongBetween(0, 1000000); long ts3 = randomLongBetween(0, 1000000); - IndexingStats.Stats stats1 = new IndexingStats.Stats(1, 2, 3, 4, 5, 6, 7, 8, false, 9, docStatusStats, ts1); - IndexingStats.Stats stats2 = new IndexingStats.Stats(1, 2, 3, 4, 5, 6, 7, 8, false, 9, docStatusStats, ts2); - IndexingStats.Stats stats3 = new IndexingStats.Stats(1, 2, 3, 4, 5, 6, 7, 8, false, 9, docStatusStats, ts3); + IndexingStats.Stats stats1 = new IndexingStats.Stats(1, 2, 3, 4, 5, 6, 7, 8, false, 9, ts1); + IndexingStats.Stats stats2 = new IndexingStats.Stats(1, 2, 3, 4, 5, 6, 7, 8, false, 9, ts2); + IndexingStats.Stats stats3 = new IndexingStats.Stats(1, 2, 3, 4, 5, 6, 7, 8, false, 9, ts3); // Aggregate stats1 + stats2 stats1.add(stats2); @@ -141,20 +135,19 @@ public void testMaxLastIndexRequestTimestampAggregation() throws Exception { assertEquals(Math.max(Math.max(ts1, ts2), ts3), stats1.getMaxLastIndexRequestTimestamp()); // Test with zero and negative values - IndexingStats.Stats statsZero = new IndexingStats.Stats(1, 2, 3, 4, 5, 6, 7, 8, false, 9, docStatusStats, 0L); - IndexingStats.Stats statsNeg = new IndexingStats.Stats(1, 2, 3, 4, 5, 6, 7, 8, false, 9, docStatusStats, -100L); + IndexingStats.Stats statsZero = new IndexingStats.Stats(1, 2, 3, 4, 5, 6, 7, 8, false, 9, 0L); + IndexingStats.Stats statsNeg = new IndexingStats.Stats(1, 2, 3, 4, 5, 6, 7, 8, false, 9, -100L); statsZero.add(statsNeg); assertEquals(0L, statsZero.getMaxLastIndexRequestTimestamp()); - IndexingStats.Stats statsNeg2 = new IndexingStats.Stats(1, 2, 3, 4, 5, 6, 7, 8, false, 9, docStatusStats, -50L); + IndexingStats.Stats statsNeg2 = new IndexingStats.Stats(1, 2, 3, 4, 5, 6, 7, 8, false, 9, -50L); statsNeg.add(statsNeg2); assertEquals(-50L, statsNeg.getMaxLastIndexRequestTimestamp()); } public void testMaxLastIndexRequestTimestampBackwardCompatibility() throws IOException { - IndexingStats.Stats.DocStatusStats docStatusStats = new IndexingStats.Stats.DocStatusStats(); long ts = randomLongBetween(0, 1000000); - IndexingStats.Stats stats = new IndexingStats.Stats(1, 2, 3, 4, 5, 6, 7, 8, false, 9, docStatusStats, ts); + IndexingStats.Stats stats = new IndexingStats.Stats(1, 2, 3, 4, 5, 6, 7, 8, false, 9, ts); // Serialize with V_3_1_0 (should include the field) BytesStreamOutput outNew = new BytesStreamOutput(); @@ -176,11 +169,6 @@ public void testMaxLastIndexRequestTimestampBackwardCompatibility() throws IOExc } private IndexingStats createTestInstance() { - IndexingStats.Stats.DocStatusStats docStatusStats = new IndexingStats.Stats.DocStatusStats(); - for (int i = 1; i < 6; ++i) { - docStatusStats.add(RestStatus.fromCode(i * 100), randomNonNegativeLong()); - } - IndexingStats.Stats stats = new IndexingStats.Stats( randomNonNegativeLong(), randomNonNegativeLong(), @@ -192,7 +180,6 @@ private IndexingStats createTestInstance() { randomNonNegativeLong(), randomBoolean(), randomNonNegativeLong(), - docStatusStats, randomLong() ); diff --git a/server/src/test/java/org/opensearch/indices/NodeIndicesStatsTests.java b/server/src/test/java/org/opensearch/indices/NodeIndicesStatsTests.java index 2424e38636466..d02c25ad03ec7 100644 --- a/server/src/test/java/org/opensearch/indices/NodeIndicesStatsTests.java +++ b/server/src/test/java/org/opensearch/indices/NodeIndicesStatsTests.java @@ -33,13 +33,25 @@ package org.opensearch.indices; import org.opensearch.action.admin.indices.stats.CommonStats; +import org.opensearch.action.admin.indices.stats.DocStatusStats; +import org.opensearch.action.admin.indices.stats.SearchResponseStatusStats; +import org.opensearch.action.admin.indices.stats.StatusCounterStats; import org.opensearch.action.search.SearchRequestStats; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.test.OpenSearchTestCase; +import java.io.IOException; +import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; +import java.util.concurrent.atomic.LongAdder; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.object.HasToString.hasToString; @@ -50,7 +62,8 @@ public void testInvalidLevel() { CommonStats oldStats = new CommonStats(); ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); SearchRequestStats requestStats = new SearchRequestStats(clusterSettings); - final NodeIndicesStats stats = new NodeIndicesStats(oldStats, Collections.emptyMap(), requestStats); + StatusCounterStats statusCounterStats = new StatusCounterStats(); + final NodeIndicesStats stats = new NodeIndicesStats(oldStats, Collections.emptyMap(), requestStats, statusCounterStats); final String level = randomAlphaOfLength(16); final ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap("level", level)); final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> stats.toXContent(null, params)); @@ -60,4 +73,90 @@ public void testInvalidLevel() { ); } + public void testSerializationForStatusCounterStats() throws IOException { + StatusCounterStats stats = createStatusCounters(); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + stats.writeTo(out); + + try (StreamInput in = out.bytes().streamInput()) { + StatusCounterStats deserializedStats = new StatusCounterStats(in); + + if (stats.getDocStatusStats() == null) { + assertNull(deserializedStats.getDocStatusStats()); + return; + } + + DocStatusStats docStats = stats.getDocStatusStats(); + DocStatusStats deserializedDocStats = deserializedStats.getDocStatusStats(); + + assertTrue( + Arrays.equals( + docStats.getDocStatusCounter(), + deserializedDocStats.getDocStatusCounter(), + Comparator.comparingLong(LongAdder::longValue) + ) + ); + + if (stats.getSearchResponseStatusStats() == null) { + assertNull(deserializedStats.getSearchResponseStatusStats()); + return; + } + + SearchResponseStatusStats searchStats = stats.getSearchResponseStatusStats(); + SearchResponseStatusStats deserializedSearchStats = deserializedStats.getSearchResponseStatusStats(); + + assertTrue( + Arrays.equals( + searchStats.getSearchResponseStatusCounter(), + deserializedSearchStats.getSearchResponseStatusCounter(), + Comparator.comparingLong(LongAdder::longValue) + ) + ); + } + } + } + + public void testToXContentForStatusCounterStats() throws IOException { + StatusCounterStats statusCounterStats = createStatusCounters(); + LongAdder[] docStatusCounter = statusCounterStats.getDocStatusStats().getDocStatusCounter(); + LongAdder[] searchResponseStatusCounter = statusCounterStats.getSearchResponseStatusStats().getSearchResponseStatusCounter(); + + long docStatusSuccesses = docStatusCounter[0].longValue() + docStatusCounter[1].longValue() + docStatusCounter[2].longValue(); + long searchResponseStatusSuccesses = searchResponseStatusCounter[0].longValue() + searchResponseStatusCounter[1].longValue() + + searchResponseStatusCounter[2].longValue(); + + String expected = "{\"status_counter\":{\"doc_status\":{\"success\":" + + docStatusSuccesses + + ",\"user_error\":" + + docStatusCounter[3].longValue() + + ",\"system_failure\":" + + docStatusCounter[4].longValue() + + "},\"search_response_status\":{\"success\":" + + searchResponseStatusSuccesses + + ",\"user_error\":" + + searchResponseStatusCounter[3].longValue() + + ",\"system_failure\":" + + searchResponseStatusCounter[4].longValue() + + "}}}"; + + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.JSON); + xContentBuilder.startObject(); + xContentBuilder = statusCounterStats.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); + xContentBuilder.endObject(); + + assertEquals(expected, xContentBuilder.toString()); + } + + private StatusCounterStats createStatusCounters() { + StatusCounterStats statusCounters = new StatusCounterStats(); + + for (int i = 1; i < 6; ++i) { + statusCounters.getDocStatusStats().add(RestStatus.fromCode(i * 100), randomLongBetween(0, 100)); + statusCounters.getSearchResponseStatusStats().add(RestStatus.fromCode(i * 100), randomLongBetween(0, 100)); + } + + return statusCounters; + } + } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index a936d4ce79ec2..83ad0908e8658 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -82,6 +82,7 @@ import org.opensearch.action.admin.indices.mapping.put.TransportPutMappingAction; import org.opensearch.action.admin.indices.shards.IndicesShardStoresAction; import org.opensearch.action.admin.indices.shards.TransportIndicesShardStoresAction; +import org.opensearch.action.admin.indices.stats.SearchResponseStatusStats; import org.opensearch.action.bulk.BulkAction; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkResponse; @@ -2255,6 +2256,8 @@ public void onFailure(final Exception e) { ); final MappingUpdatedAction mappingUpdatedAction = new MappingUpdatedAction(settings, clusterSettings, clusterService); mappingUpdatedAction.setClient(client); + final IndicesService mockIndicesService = mock(IndicesService.class); + when(mockIndicesService.getSearchResponseStatusStats()).thenReturn(new SearchResponseStatusStats()); final TransportShardBulkAction transportShardBulkAction = new TransportShardBulkAction( settings, transportService, @@ -2269,7 +2272,7 @@ public void onFailure(final Exception e) { new SegmentReplicationPressureService( settings, clusterService, - mock(IndicesService.class), + mockIndicesService, mock(ShardStateAction.class), mock(SegmentReplicationStatsTracker.class), mock(ThreadPool.class) @@ -2302,7 +2305,7 @@ public void onFailure(final Exception e) { indexNameExpressionResolver, new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver, new SystemIndices(emptyMap())), new IndexingPressureService(settings, clusterService), - mock(IndicesService.class), + mockIndicesService, new SystemIndices(emptyMap()), NoopTracer.INSTANCE ) @@ -2403,7 +2406,8 @@ public void onFailure(final Exception e) { NoopMetricsRegistry.INSTANCE, searchRequestOperationsCompositeListenerFactory, NoopTracer.INSTANCE, - new TaskResourceTrackingService(settings, clusterSettings, threadPool) + new TaskResourceTrackingService(settings, clusterSettings, threadPool), + mockIndicesService ) ); actions.put(