diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a7fd77591249..79a14a9573f48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Include named queries from rescore contexts in matched_queries array ([#18697](https://github.com/opensearch-project/OpenSearch/pull/18697)) - Add the configurable limit on rule cardinality ([#18663](https://github.com/opensearch-project/OpenSearch/pull/18663)) - [Experimental] Start in "clusterless" mode if a clusterless ClusterPlugin is loaded ([#18479](https://github.com/opensearch-project/OpenSearch/pull/18479)) +- [Star-Tree] Add star-tree search related stats ([#18707](https://github.com/opensearch-project/OpenSearch/pull/18707)) ### Changed - Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570)) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml index 989ea6b93f47f..01bdc323479c4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml @@ -1,13 +1,111 @@ "Help": - skip: - version: " - 2.13.99" - reason: search idle reactivate count total is only added in 3.0.0 + version: " - 3.1.99" + reason: star-tree search stats is only added in 3.2.0 features: node_selector - do: cat.shards: help: true node_selector: - version: "2.14.0 - " + version: "3.2.0 - " + + - match: + $body: | + /^ index .+ \n + shard .+ \n + prirep .+ \n + state .+ \n + docs .+ \n + store .+ \n + ip .+ \n + id .+ \n + node .+ \n + sync_id .+ \n + unassigned.reason .+ \n + unassigned.at .+ \n + unassigned.for .+ \n + unassigned.details .+ \n + recoverysource.type .+ \n + completion.size .+ \n + fielddata.memory_size .+ \n + fielddata.evictions .+ \n + query_cache.memory_size .+ \n + query_cache.evictions .+ \n + flush.total .+ \n + flush.total_time .+ \n + get.current .+ \n + get.time .+ \n + get.total .+ \n + get.exists_time .+ \n + get.exists_total .+ \n + get.missing_time .+ \n + get.missing_total .+ \n + indexing.delete_current .+ \n + indexing.delete_time .+ \n + indexing.delete_total .+ \n + indexing.index_current .+ \n + indexing.index_time .+ \n + indexing.index_total .+ \n + indexing.index_failed .+ \n + merges.current .+ \n + merges.current_docs .+ \n + merges.current_size .+ \n + merges.total .+ \n + merges.total_docs .+ \n + merges.total_size .+ \n + merges.total_time .+ \n + refresh.total .+ \n + refresh.time .+ \n + refresh.external_total .+ \n + refresh.external_time .+ \n + refresh.listeners .+ \n + search.fetch_current .+ \n + search.fetch_time .+ \n + search.fetch_total .+ \n + search.open_contexts .+ \n + search.query_current .+ \n + search.query_time .+ \n + search.query_total .+ \n + search.concurrent_query_current .+ \n + search.concurrent_query_time .+ \n + search.concurrent_query_total .+ \n + search.concurrent_avg_slice_count .+ \n + search.startree_query_current .+ \n + search.startree_query_time .+ \n + search.startree_query_total .+ \n + search.scroll_current .+ \n + search.scroll_time .+ \n + search.scroll_total .+ \n + search.point_in_time_current .+ \n + search.point_in_time_time .+ \n + search.point_in_time_total .+ \n + search.search_idle_reactivate_count_total .+ \n + segments.count .+ \n + segments.memory .+ \n + segments.index_writer_memory .+ \n + segments.version_map_memory .+ \n + segments.fixed_bitset_memory .+ \n + seq_no.max .+ \n + seq_no.local_checkpoint .+ \n + seq_no.global_checkpoint .+ \n + warmer.current .+ \n + warmer.total .+ \n + warmer.total_time .+ \n + path.data .+ \n + path.state .+ \n + docs.deleted .+ \n + $/ +--- +"Help from 2.14.0 to 3.0.99": + - skip: + version: " - 2.13.99, 3.2.0 - " + reason: search idle reactivate count total is only added in 3.0.0 + features: node_selector + - do: + cat.shards: + help: true + node_selector: + version: "2.14.0 - 3.1.99" - match: $body: | diff --git a/server/src/internalClusterTest/java/org/opensearch/search/stats/StarTreeSearchStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/stats/StarTreeSearchStatsIT.java new file mode 100644 index 0000000000000..8bb6d478f69e3 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/search/stats/StarTreeSearchStatsIT.java @@ -0,0 +1,178 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.stats; + +import org.opensearch.action.admin.indices.forcemerge.ForceMergeRequest; +import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.index.search.stats.SearchStats; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.bucket.terms.Terms; +import org.opensearch.search.aggregations.metrics.Sum; +import org.opensearch.test.OpenSearchIntegTestCase; + +import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +/** + * Integration tests for search statistics related to star-tree. + */ +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE) +public class StarTreeSearchStatsIT extends OpenSearchIntegTestCase { + + private static final String STARTREE_INDEX_NAME = "test_startree"; + private static final String REGULAR_INDEX_NAME = "test_regular"; + + /** + * This test validates that star-tree specific search stats are correctly reported. + * It creates two indices: one with a star-tree field and one without. + * It then runs queries that should and should not be offloaded to the star-tree + * and verifies the star-tree related stats. + */ + public void testStarTreeQueryStats() throws Exception { + // Create an index with a star-tree field mapping using the composite structure + createIndex( + STARTREE_INDEX_NAME, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.composite_index", true) + .put("index.append_only.enabled", true) + .put("index.search.star_tree_index.enabled", true) + .put("index.codec", "default") + .build(), + XContentFactory.jsonBuilder() + .startObject() + .startObject("composite") + .startObject("my_star_tree") + .field("type", "star_tree") + .startObject("config") + .startArray("ordered_dimensions") + .startObject() + .field("name", "product") + .endObject() + .startObject() + .field("name", "size") + .endObject() + .endArray() + .startArray("metrics") + .startObject() + .field("name", "sales") + .field("stats", new String[] { "sum" }) + .endObject() + .endArray() + .endObject() + .endObject() + .endObject() + .startObject("properties") + .startObject("product") + .field("type", "keyword") + .endObject() + .startObject("size") + .field("type", "keyword") + .endObject() + .startObject("sales") + .field("type", "double") + .endObject() + .startObject("non_dimension_field") + .field("type", "text") + .endObject() + .endObject() + .endObject() + .toString() + ); + + // Create an index without a star-tree + createIndex( + REGULAR_INDEX_NAME, + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build() + ); + ensureGreen(STARTREE_INDEX_NAME, REGULAR_INDEX_NAME); + + client().prepareIndex(STARTREE_INDEX_NAME) + .setSource("product", "laptop", "sales", 1200.0, "non_dimension_field", "some text", "size", "18") + .get(); + client().prepareIndex(STARTREE_INDEX_NAME) + .setSource("product", "keyboard", "sales", 150.0, "non_dimension_field", "more text", "size", "12") + .get(); + client().prepareIndex(REGULAR_INDEX_NAME).setSource("product", "monitor", "sales", 400.0).get(); + + // Force merge the star-tree index to ensure star-tree segments are created + client().admin().indices().forceMerge(new ForceMergeRequest(STARTREE_INDEX_NAME).maxNumSegments(1)).get(); + refresh(); + + // Query 1: This query should be answered by the star-tree. + // It's an aggregation on a dimension with a metric sub-aggregation and size=0. + SearchResponse starTreeResponse = client().prepareSearch(STARTREE_INDEX_NAME) + .setQuery(matchAllQuery()) + .addAggregation( + AggregationBuilders.terms("products").field("product").subAggregation(AggregationBuilders.sum("total_sales").field("sales")) + ) + .setSize(0) + .execute() + .actionGet(); + assertEquals(2, starTreeResponse.getHits().getTotalHits().value()); + Terms terms = starTreeResponse.getAggregations().get("products"); + assertEquals(2, terms.getBuckets().size()); + Sum sum = terms.getBuckets().get(0).getAggregations().get("total_sales"); + assertNotNull(sum); + + // Query 2: This query should NOT be answered by the star-tree as it queries a non-dimension field. + client().prepareSearch(STARTREE_INDEX_NAME).setQuery(termQuery("non_dimension_field", "text")).execute().actionGet(); + + // Query 3: A standard query on the regular index. + client().prepareSearch(REGULAR_INDEX_NAME).setQuery(matchAllQuery()).get(); + + // Fetch index statistics + IndicesStatsResponse stats = client().admin().indices().prepareStats(STARTREE_INDEX_NAME, REGULAR_INDEX_NAME).setSearch(true).get(); + + // Assertions for the star-tree index + SearchStats.Stats starTreeIndexStats = stats.getIndex(STARTREE_INDEX_NAME).getTotal().getSearch().getTotal(); + + assertThat("Star-tree index should have 2 total queries", starTreeIndexStats.getQueryCount(), equalTo(2L)); + assertThat("Star-tree index should have 1 star-tree query", starTreeIndexStats.getStarTreeQueryCount(), equalTo(1L)); + assertThat( + "Star-tree query time should be non-negative", + starTreeIndexStats.getStarTreeQueryTimeInMillis(), + greaterThanOrEqualTo(0L) + ); + assertThat( + "Star-tree query current should be non-negative", + starTreeIndexStats.getStarTreeQueryCurrent(), + greaterThanOrEqualTo(0L) + ); + + // Assertions for the regular index + SearchStats.Stats regularIndexStats = stats.getIndex(REGULAR_INDEX_NAME).getTotal().getSearch().getTotal(); + assertThat("Regular index should have 1 total query", regularIndexStats.getQueryCount(), equalTo(1L)); + assertThat("Regular index should have 0 star-tree queries", regularIndexStats.getStarTreeQueryCount(), equalTo(0L)); + assertThat("Regular index should have 0 star-tree query time", regularIndexStats.getStarTreeQueryTimeInMillis(), equalTo(0L)); + + // Assertions for the total stats across both indices + SearchStats.Stats totalStats = stats.getTotal().getSearch().getTotal(); + assertThat("Total query count should be 3", totalStats.getQueryCount(), equalTo(3L)); + assertThat("Total star-tree query count should be 1", totalStats.getStarTreeQueryCount(), equalTo(1L)); + assertThat(totalStats.getQueryTimeInMillis(), greaterThanOrEqualTo(totalStats.getStarTreeQueryTimeInMillis())); + + // While not guaranteed, the time spent on the single star-tree query should be less than the total query time. + if (totalStats.getStarTreeQueryCount() > 0 && totalStats.getQueryCount() > totalStats.getStarTreeQueryCount()) { + assertThat( + "Star-tree query time should be less than total query time", + totalStats.getStarTreeQueryTimeInMillis(), + lessThanOrEqualTo(totalStats.getQueryTimeInMillis()) + ); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java b/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java index d6ea803c9ee13..213ff3299fde1 100644 --- a/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java +++ b/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java @@ -166,6 +166,10 @@ public static class Stats implements Writeable, ToXContentFragment { private long searchIdleReactivateCount; + private long starTreeQueryCount; + private long starTreeQueryTimeInMillis; + private long starTreeQueryCurrent; + @Nullable private RequestStatsLongHolder requestStatsLongHolder; @@ -177,6 +181,50 @@ private Stats() { // for internal use, initializes all counts to 0 } + /** + * Private constructor that takes a builder. + * This is the sole entry point for creating a new Stats object. + * @param builder The builder instance containing all the values. + */ + private Stats(Builder builder) { + this.requestStatsLongHolder = builder.requestStatsLongHolder; + this.queryCount = builder.queryCount; + this.queryTimeInMillis = builder.queryTimeInMillis; + this.queryCurrent = builder.queryCurrent; + + this.concurrentQueryCount = builder.concurrentQueryCount; + this.concurrentQueryTimeInMillis = builder.concurrentQueryTimeInMillis; + this.concurrentQueryCurrent = builder.concurrentQueryCurrent; + this.queryConcurrency = builder.queryConcurrency; + + this.fetchCount = builder.fetchCount; + this.fetchTimeInMillis = builder.fetchTimeInMillis; + this.fetchCurrent = builder.fetchCurrent; + + this.scrollCount = builder.scrollCount; + this.scrollTimeInMillis = builder.scrollTimeInMillis; + this.scrollCurrent = builder.scrollCurrent; + + this.suggestCount = builder.suggestCount; + this.suggestTimeInMillis = builder.suggestTimeInMillis; + this.suggestCurrent = builder.suggestCurrent; + + this.pitCount = builder.pitCount; + this.pitTimeInMillis = builder.pitTimeInMillis; + this.pitCurrent = builder.pitCurrent; + + this.searchIdleReactivateCount = builder.searchIdleReactivateCount; + + this.starTreeQueryCount = builder.starTreeQueryCount; + this.starTreeQueryTimeInMillis = builder.starTreeQueryTimeInMillis; + this.starTreeQueryCurrent = builder.starTreeQueryCurrent; + } + + /** + * This constructor will be deprecated in 4.0 + * Use Builder to create Stats object + */ + @Deprecated public Stats( long queryCount, long queryTimeInMillis, @@ -265,6 +313,12 @@ private Stats(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_2_14_0)) { searchIdleReactivateCount = in.readVLong(); } + + if (in.getVersion().onOrAfter(Version.V_3_2_0)) { + starTreeQueryCount = in.readVLong(); + starTreeQueryTimeInMillis = in.readVLong(); + starTreeQueryCurrent = in.readVLong(); + } } public void add(Stats stats) { @@ -294,6 +348,10 @@ public void add(Stats stats) { pitCurrent += stats.pitCurrent; searchIdleReactivateCount += stats.searchIdleReactivateCount; + + starTreeQueryCount += stats.starTreeQueryCount; + starTreeQueryTimeInMillis += stats.starTreeQueryTimeInMillis; + starTreeQueryCurrent += stats.starTreeQueryCurrent; } public void addForClosingShard(Stats stats) { @@ -320,6 +378,9 @@ public void addForClosingShard(Stats stats) { queryConcurrency += stats.queryConcurrency; searchIdleReactivateCount += stats.searchIdleReactivateCount; + + starTreeQueryCount += stats.starTreeQueryCount; + starTreeQueryTimeInMillis += stats.starTreeQueryTimeInMillis; } public long getQueryCount() { @@ -430,6 +491,22 @@ public long getSearchIdleReactivateCount() { return searchIdleReactivateCount; } + public long getStarTreeQueryCount() { + return starTreeQueryCount; + } + + public TimeValue getStarTreeQueryTime() { + return new TimeValue(starTreeQueryTimeInMillis); + } + + public long getStarTreeQueryTimeInMillis() { + return starTreeQueryTimeInMillis; + } + + public long getStarTreeQueryCurrent() { + return starTreeQueryCurrent; + } + public static Stats readStats(StreamInput in) throws IOException { return new Stats(in); } @@ -479,6 +556,12 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_14_0)) { out.writeVLong(searchIdleReactivateCount); } + + if (out.getVersion().onOrAfter(Version.V_3_2_0)) { + out.writeVLong(starTreeQueryCount); + out.writeVLong(starTreeQueryTimeInMillis); + out.writeVLong(starTreeQueryCurrent); + } } @Override @@ -492,6 +575,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.CONCURRENT_QUERY_CURRENT, concurrentQueryCurrent); builder.field(Fields.CONCURRENT_AVG_SLICE_COUNT, getConcurrentAvgSliceCount()); + builder.field(Fields.STARTREE_QUERY_TOTAL, starTreeQueryCount); + builder.humanReadableField(Fields.STARTREE_QUERY_TIME_IN_MILLIS, Fields.STARTREE_QUERY_TIME, getStarTreeQueryTime()); + builder.field(Fields.STARTREE_QUERY_CURRENT, getStarTreeQueryCurrent()); + builder.field(Fields.FETCH_TOTAL, fetchCount); builder.humanReadableField(Fields.FETCH_TIME_IN_MILLIS, Fields.FETCH_TIME, getFetchTime()); builder.field(Fields.FETCH_CURRENT, fetchCurrent); @@ -537,6 +624,163 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } return builder; } + + /** + * Builder for the {@link Stats} class. + * Provides a fluent API for constructing a Stats object. + */ + public static class Builder { + private long queryCount = 0; + private long queryTimeInMillis = 0; + private long queryCurrent = 0; + private long concurrentQueryCount = 0; + private long concurrentQueryTimeInMillis = 0; + private long concurrentQueryCurrent = 0; + private long queryConcurrency = 0; + private long fetchCount = 0; + private long fetchTimeInMillis = 0; + private long fetchCurrent = 0; + private long scrollCount = 0; + private long scrollTimeInMillis = 0; + private long scrollCurrent = 0; + private long suggestCount = 0; + private long suggestTimeInMillis = 0; + private long suggestCurrent = 0; + private long pitCount = 0; + private long pitTimeInMillis = 0; + private long pitCurrent = 0; + private long searchIdleReactivateCount = 0; + private long starTreeQueryCount = 0; + private long starTreeQueryTimeInMillis = 0; + private long starTreeQueryCurrent = 0; + @Nullable + private RequestStatsLongHolder requestStatsLongHolder = null; + + public Builder() {} + + public Builder queryCount(long count) { + this.queryCount = count; + return this; + } + + public Builder queryTimeInMillis(long time) { + this.queryTimeInMillis = time; + return this; + } + + public Builder queryCurrent(long current) { + this.queryCurrent = current; + return this; + } + + public Builder concurrentQueryCount(long count) { + this.concurrentQueryCount = count; + return this; + } + + public Builder concurrentQueryTimeInMillis(long time) { + this.concurrentQueryTimeInMillis = time; + return this; + } + + public Builder concurrentQueryCurrent(long current) { + this.concurrentQueryCurrent = current; + return this; + } + + public Builder queryConcurrency(long concurrency) { + this.queryConcurrency = concurrency; + return this; + } + + public Builder fetchCount(long count) { + this.fetchCount = count; + return this; + } + + public Builder fetchTimeInMillis(long time) { + this.fetchTimeInMillis = time; + return this; + } + + public Builder fetchCurrent(long current) { + this.fetchCurrent = current; + return this; + } + + public Builder scrollCount(long count) { + this.scrollCount = count; + return this; + } + + public Builder scrollTimeInMillis(long time) { + this.scrollTimeInMillis = time; + return this; + } + + public Builder scrollCurrent(long current) { + this.scrollCurrent = current; + return this; + } + + public Builder suggestCount(long count) { + this.suggestCount = count; + return this; + } + + public Builder suggestTimeInMillis(long time) { + this.suggestTimeInMillis = time; + return this; + } + + public Builder suggestCurrent(long current) { + this.suggestCurrent = current; + return this; + } + + public Builder pitCount(long count) { + this.pitCount = count; + return this; + } + + public Builder pitTimeInMillis(long time) { + this.pitTimeInMillis = time; + return this; + } + + public Builder pitCurrent(long current) { + this.pitCurrent = current; + return this; + } + + public Builder searchIdleReactivateCount(long count) { + this.searchIdleReactivateCount = count; + return this; + } + + public Builder starTreeQueryCount(long count) { + this.starTreeQueryCount = count; + return this; + } + + public Builder starTreeQueryTimeInMillis(long time) { + this.starTreeQueryTimeInMillis = time; + return this; + } + + public Builder starTreeQueryCurrent(long current) { + this.starTreeQueryCurrent = current; + return this; + } + + /** + * Creates a {@link Stats} object from the builder's current state. + * @return A new Stats instance. + */ + public Stats build() { + return new Stats(this); + } + } } private final Stats totalStats; @@ -677,6 +921,10 @@ static final class Fields { static final String CONCURRENT_QUERY_TIME_IN_MILLIS = "concurrent_query_time_in_millis"; static final String CONCURRENT_QUERY_CURRENT = "concurrent_query_current"; static final String CONCURRENT_AVG_SLICE_COUNT = "concurrent_avg_slice_count"; + static final String STARTREE_QUERY_TOTAL = "startree_query_total"; + static final String STARTREE_QUERY_TIME = "startree_query_time"; + static final String STARTREE_QUERY_TIME_IN_MILLIS = "startree_query_time_in_millis"; + static final String STARTREE_QUERY_CURRENT = "startree_query_current"; static final String FETCH_TOTAL = "fetch_total"; static final String FETCH_TIME = "fetch_time"; static final String FETCH_TIME_IN_MILLIS = "fetch_time_in_millis"; diff --git a/server/src/main/java/org/opensearch/index/search/stats/ShardSearchStats.java b/server/src/main/java/org/opensearch/index/search/stats/ShardSearchStats.java index 3098986852cc1..e226a6e575bbb 100644 --- a/server/src/main/java/org/opensearch/index/search/stats/ShardSearchStats.java +++ b/server/src/main/java/org/opensearch/index/search/stats/ShardSearchStats.java @@ -94,6 +94,9 @@ public void onPreQueryPhase(SearchContext searchContext) { if (searchContext.shouldUseConcurrentSearch()) { statsHolder.concurrentQueryCurrent.inc(); } + if (searchContext.getQueryShardContext().getStarTreeQueryContext() != null) { + statsHolder.starTreeCurrent.inc(); + } } }); } @@ -111,6 +114,10 @@ public void onFailedQueryPhase(SearchContext searchContext) { statsHolder.concurrentQueryCurrent.dec(); assert statsHolder.concurrentQueryCurrent.count() >= 0; } + if (searchContext.getQueryShardContext().getStarTreeQueryContext() != null) { + statsHolder.starTreeCurrent.dec(); + assert statsHolder.starTreeCurrent.count() >= 0; + } } }); } @@ -133,6 +140,11 @@ public void onQueryPhase(SearchContext searchContext, long tookInNanos) { assert searchContext.searcher().getSlices() != null; statsHolder.queryConcurrencyMetric.inc(searchContext.searcher().getSlices().length); } + if (searchContext.getQueryShardContext().getStarTreeQueryContext() != null) { + statsHolder.starTreeQueryMetric.inc(tookInNanos); + statsHolder.starTreeCurrent.dec(); + assert statsHolder.starTreeCurrent.count() >= 0; + } } }); } @@ -245,30 +257,34 @@ static final class StatsHolder { final CounterMetric pitCurrent = new CounterMetric(); final CounterMetric suggestCurrent = new CounterMetric(); final CounterMetric searchIdleMetric = new CounterMetric(); + final MeanMetric starTreeQueryMetric = new MeanMetric(); + final CounterMetric starTreeCurrent = new CounterMetric(); SearchStats.Stats stats() { - return new SearchStats.Stats( - queryMetric.count(), - TimeUnit.NANOSECONDS.toMillis(queryMetric.sum()), - queryCurrent.count(), - concurrentQueryMetric.count(), - TimeUnit.NANOSECONDS.toMillis(concurrentQueryMetric.sum()), - concurrentQueryCurrent.count(), - queryConcurrencyMetric.count(), - fetchMetric.count(), - TimeUnit.NANOSECONDS.toMillis(fetchMetric.sum()), - fetchCurrent.count(), - scrollMetric.count(), - TimeUnit.MICROSECONDS.toMillis(scrollMetric.sum()), - scrollCurrent.count(), - pitMetric.count(), - TimeUnit.MICROSECONDS.toMillis(pitMetric.sum()), - pitCurrent.count(), - suggestMetric.count(), - TimeUnit.NANOSECONDS.toMillis(suggestMetric.sum()), - suggestCurrent.count(), - searchIdleMetric.count() - ); + return new SearchStats.Stats.Builder().queryCount(queryMetric.count()) + .queryTimeInMillis(TimeUnit.NANOSECONDS.toMillis(queryMetric.sum())) + .queryCurrent(queryCurrent.count()) + .concurrentQueryCount(concurrentQueryMetric.count()) + .concurrentQueryTimeInMillis(TimeUnit.NANOSECONDS.toMillis(concurrentQueryMetric.sum())) + .concurrentQueryCurrent(concurrentQueryCurrent.count()) + .queryConcurrency(queryConcurrencyMetric.count()) + .fetchCount(fetchMetric.count()) + .fetchTimeInMillis(TimeUnit.NANOSECONDS.toMillis(fetchMetric.sum())) + .fetchCurrent(fetchCurrent.count()) + .scrollCount(scrollMetric.count()) + .scrollTimeInMillis(TimeUnit.MICROSECONDS.toMillis(scrollMetric.sum())) + .scrollCurrent(scrollCurrent.count()) + .pitCount(pitMetric.count()) + .pitTimeInMillis(TimeUnit.MICROSECONDS.toMillis(pitMetric.sum())) + .pitCurrent(pitCurrent.count()) + .suggestCount(suggestMetric.count()) + .suggestTimeInMillis(TimeUnit.NANOSECONDS.toMillis(suggestMetric.sum())) + .suggestCurrent(suggestCurrent.count()) + .searchIdleReactivateCount(searchIdleMetric.count()) + .starTreeQueryCount(starTreeQueryMetric.count()) + .starTreeQueryTimeInMillis(TimeUnit.NANOSECONDS.toMillis(starTreeQueryMetric.sum())) + .starTreeQueryCurrent(starTreeCurrent.count()) + .build(); } } } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 376e67bb9042c..5a74e81adb2dc 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -677,6 +677,24 @@ protected Table getTableWithHeader(final RestRequest request, final PageToken pa ); table.addCell("pri.search.concurrent_avg_slice_count", "default:false;text-align:right;desc:average query concurrency"); + table.addCell( + "search.startree_query_current", + "sibling:pri;alias:stqc,startreeQueryCurrent;default:false;text-align:right;desc:current star tree query ops" + ); + table.addCell("pri.startree.query_current", "default:false;text-align:right;desc:current star tree query ops"); + + table.addCell( + "search.startree_query_total", + "sibling:pri;alias:stqto,startreeQueryCurrent;default:false;text-align:right;desc:total star tree resolved queries" + ); + table.addCell("pri.startree.query_total", "default:false;text-align:right;desc:total star tree resolved queries"); + + table.addCell( + "search.startree_query_time", + "sibling:pri;alias:stqti,startreeQueryTime;default:false;text-align:right;desc:time spent in star tree queries" + ); + table.addCell("pri.startree.query_time", "default:false;text-align:right;desc:time spent in star tree queries"); + table.addCell( "search.scroll_current", "sibling:pri;alias:scc,searchScrollCurrent;default:false;text-align:right;desc:open scroll contexts" @@ -1011,6 +1029,15 @@ protected Table buildTable( table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentAvgSliceCount()); table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentAvgSliceCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getStarTreeQueryCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getStarTreeQueryCount()); + + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getStarTreeQueryTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getStarTreeQueryTime()); + + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getStarTreeQueryCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getStarTreeQueryCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCurrent()); table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCurrent()); diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java index 3fc1b4d0ff3c2..b91027f35d953 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java @@ -340,6 +340,21 @@ protected Table getTableWithHeader(final RestRequest request) { "alias:scto,searchPointInTimeTotal;default:false;text-align:right;desc:completed point in time contexts" ); + table.addCell( + "search.startree_query_current", + "alias:stqc,startreeQueryCurrent;default:false;text-align:right;desc:current star tree query ops" + ); + + table.addCell( + "search.startree_query_time", + "alias:stqti,startreeQueryTime;default:false;text-align:right;desc:time spent in star tree queries" + ); + + table.addCell( + "search.startree_query_total", + "alias:stqto,startreeQueryTotal;default:false;text-align:right;desc:total star tree resolved queries" + ); + table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments"); table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments"); table.addCell( @@ -556,6 +571,9 @@ Table buildTable( table.addCell(searchStats == null ? null : searchStats.getTotal().getPitCurrent()); table.addCell(searchStats == null ? null : searchStats.getTotal().getPitTime()); table.addCell(searchStats == null ? null : searchStats.getTotal().getPitCount()); + table.addCell(searchStats == null ? null : searchStats.getTotal().getStarTreeQueryCurrent()); + table.addCell(searchStats == null ? null : searchStats.getTotal().getStarTreeQueryTime()); + table.addCell(searchStats == null ? null : searchStats.getTotal().getStarTreeQueryCount()); SegmentsStats segmentsStats = indicesStats == null ? null : indicesStats.getSegments(); table.addCell(segmentsStats == null ? null : segmentsStats.getCount()); diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java index c86df49fdd88d..bc2784c6bdca2 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java @@ -249,6 +249,20 @@ protected Table getTableWithHeader(final RestRequest request, final PageToken pa "search.concurrent_avg_slice_count", "alias:casc,searchConcurrentAvgSliceCount;default:false;text-align:right;desc:average query concurrency" ); + + table.addCell( + "search.startree_query_current", + "alias:stqc,startreeQueryCurrent;default:false;text-align:right;desc:current star tree query ops" + ); + table.addCell( + "search.startree_query_time", + "alias:stqti,startreeQueryTime;default:false;text-align:right;desc:time spent in star tree queries" + ); + table.addCell( + "search.startree_query_total", + "alias:stqto,startreeQueryTotal;default:false;text-align:right;desc:total star tree resolved queries" + ); + table.addCell("search.scroll_current", "alias:scc,searchScrollCurrent;default:false;text-align:right;desc:open scroll contexts"); table.addCell( "search.scroll_time", @@ -449,6 +463,10 @@ Table buildTable( table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentQueryCount())); table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentAvgSliceCount())); + table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getStarTreeQueryCurrent())); + table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getStarTreeQueryTime())); + table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getStarTreeQueryCount())); + table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCurrent())); table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollTime())); table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCount())); diff --git a/server/src/test/java/org/opensearch/index/search/stats/SearchStatsTests.java b/server/src/test/java/org/opensearch/index/search/stats/SearchStatsTests.java index 65e8997d75403..fb6d95822798a 100644 --- a/server/src/test/java/org/opensearch/index/search/stats/SearchStatsTests.java +++ b/server/src/test/java/org/opensearch/index/search/stats/SearchStatsTests.java @@ -53,14 +53,37 @@ public class SearchStatsTests extends OpenSearchTestCase implements SearchRequestOperationsListenerSupport { - // https://github.com/elastic/elasticsearch/issues/7644 - public void testShardLevelSearchGroupStats() throws Exception { + public void testShardLevelSearchGroupStats() { // let's create two dummy search stats with groups Map groupStats1 = new HashMap<>(); Map groupStats2 = new HashMap<>(); - groupStats2.put("group1", new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)); - SearchStats searchStats1 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats1); - SearchStats searchStats2 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats2); + Stats.Builder defaultStats = new Stats.Builder().queryCount(1) + .queryTimeInMillis(1) + .queryCurrent(1) + .concurrentQueryCount(1) + .concurrentQueryTimeInMillis(1) + .concurrentQueryCurrent(1) + .queryConcurrency(1) + .fetchCount(1) + .fetchTimeInMillis(1) + .fetchCurrent(1) + .scrollCount(1) + .scrollTimeInMillis(1) + .scrollCurrent(1) + .suggestCount(1) + .suggestTimeInMillis(1) + .suggestCurrent(1) + .pitCount(1) + .pitTimeInMillis(1) + .pitCurrent(1) + .searchIdleReactivateCount(1) + .starTreeQueryCount(1) + .starTreeQueryTimeInMillis(1) + .starTreeQueryCurrent(1) + .starTreeQueryCurrent(1); + groupStats2.put("group1", defaultStats.build()); + SearchStats searchStats1 = new SearchStats(defaultStats.build(), 0, groupStats1); + SearchStats searchStats2 = new SearchStats(defaultStats.build(), 0, groupStats2); // adding these two search stats and checking group stats are correct searchStats1.add(searchStats2); @@ -117,6 +140,9 @@ private static void assertStats(Stats stats, long equalTo) { assertEquals(equalTo, stats.getConcurrentQueryCount()); assertEquals(equalTo, stats.getConcurrentQueryTimeInMillis()); assertEquals(equalTo, stats.getConcurrentQueryCurrent()); + assertEquals(equalTo, stats.getStarTreeQueryCount()); + assertEquals(equalTo, stats.getStarTreeQueryTimeInMillis()); + assertEquals(equalTo, stats.getStarTreeQueryCurrent()); assertEquals(equalTo, stats.getFetchCount()); assertEquals(equalTo, stats.getFetchTimeInMillis()); assertEquals(equalTo, stats.getFetchCurrent()); diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java index 53a5cec1332fb..49f522ff2d26e 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java @@ -158,7 +158,7 @@ private void assertTable(Table table) { assertThat(headers.get(6).value, equalTo("ip")); assertThat(headers.get(7).value, equalTo("id")); assertThat(headers.get(8).value, equalTo("node")); - assertThat(headers.get(79).value, equalTo("docs.deleted")); + assertThat(headers.get(82).value, equalTo("docs.deleted")); final List> rows = table.getRows(); assertThat(rows.size(), equalTo(shardRoutings.size())); @@ -174,9 +174,9 @@ private void assertTable(Table table) { assertThat(row.get(4).value, equalTo(shardStats.getStats().getDocs().getCount())); assertThat(row.get(6).value, equalTo(localNode.getHostAddress())); assertThat(row.get(7).value, equalTo(localNode.getId())); - assertThat(row.get(77).value, equalTo(shardStats.getDataPath())); - assertThat(row.get(78).value, equalTo(shardStats.getStatePath())); - assertThat(row.get(79).value, equalTo(shardStats.getStats().getDocs().getDeleted())); + assertThat(row.get(80).value, equalTo(shardStats.getDataPath())); + assertThat(row.get(81).value, equalTo(shardStats.getStatePath())); + assertThat(row.get(82).value, equalTo(shardStats.getStats().getDocs().getDeleted())); } } }