Skip to content

Commit 482c681

Browse files
ykmr1224ykmr1224
authored
[v3.0.0] Deprecate scroll API usage (#3346)
* Deprecate scroll API usage Signed-off-by: Tomoyuki Morita <[email protected]> * reformat Signed-off-by: Tomoyuki Morita <[email protected]> * Remove OpenSearchRequestBuilder.buildRequestWithScroll Signed-off-by: Tomoyuki Morita <[email protected]> --------- Signed-off-by: Tomoyuki Morita <[email protected]> Signed-off-by: Tomoyuki MORITA <[email protected]>
1 parent 3bab06d commit 482c681

File tree

28 files changed

+198
-673
lines changed

28 files changed

+198
-673
lines changed

‎common/src/main/java/org/opensearch/sql/common/setting/Settings.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ public enum Key {
2222
SQL_ENABLED("plugins.sql.enabled"),
2323
SQL_SLOWLOG("plugins.sql.slowlog"),
2424
SQL_CURSOR_KEEP_ALIVE("plugins.sql.cursor.keep_alive"),
25-
SQL_PAGINATION_API_SEARCH_AFTER("plugins.sql.pagination.api"),
2625

2726
/** PPL Settings. */
2827
PPL_ENABLED("plugins.ppl.enabled"),

‎docs/user/admin/settings.rst‎

Lines changed: 0 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -160,50 +160,6 @@ Result set::
160160
}
161161
}
162162

163-
plugins.sql.pagination.api
164-
================================
165-
166-
Description
167-
-----------
168-
169-
This setting controls whether the SQL search queries in OpenSearch use Point-In-Time (PIT) with search_after or the traditional scroll mechanism for fetching paginated results.
170-
171-
1. Default Value: true
172-
2. Possible Values: true or false
173-
3. When set to true, the search query in the background uses PIT with search_after instead of scroll to retrieve paginated results. The Cursor Id returned to the user will encode relevant pagination query-related information, which will be used to fetch the subsequent pages of results.
174-
4. This setting is node-level.
175-
5. This setting can be updated dynamically.
176-
177-
178-
Example
179-
-------
180-
181-
You can update the setting with a new value like this.
182-
183-
SQL query::
184-
185-
>> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings -d '{
186-
"transient" : {
187-
"plugins.sql.pagination.api" : "true"
188-
}
189-
}'
190-
191-
Result set::
192-
193-
{
194-
"acknowledged" : true,
195-
"persistent" : { },
196-
"transient" : {
197-
"plugins" : {
198-
"sql" : {
199-
"pagination" : {
200-
"api" : "true"
201-
}
202-
}
203-
}
204-
}
205-
}
206-
207163
plugins.query.size_limit
208164
===========================
209165

‎integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,6 @@ private Settings defaultSettings() {
152152
private final Map<Key, Object> defaultSettings =
153153
new ImmutableMap.Builder<Key, Object>()
154154
.put(Key.QUERY_SIZE_LIMIT, 200)
155-
.put(Key.SQL_PAGINATION_API_SEARCH_AFTER, true)
156155
.put(Key.FIELD_TYPE_TOLERANCE, true)
157156
.build();
158157

‎integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,6 @@ private Settings defaultSettings() {
166166
new ImmutableMap.Builder<Key, Object>()
167167
.put(Key.QUERY_SIZE_LIMIT, 200)
168168
.put(Key.SQL_CURSOR_KEEP_ALIVE, TimeValue.timeValueMinutes(1))
169-
.put(Key.SQL_PAGINATION_API_SEARCH_AFTER, true)
170169
.put(Key.FIELD_TYPE_TOLERANCE, true)
171170
.build();
172171

‎legacy/src/main/java/org/opensearch/sql/legacy/cursor/DefaultCursor.java‎

Lines changed: 16 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
package org.opensearch.sql.legacy.cursor;
77

88
import static org.opensearch.core.xcontent.DeprecationHandler.IGNORE_DEPRECATIONS;
9-
import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER;
109

1110
import com.fasterxml.jackson.core.JsonProcessingException;
1211
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -38,7 +37,6 @@
3837
import org.opensearch.index.query.QueryBuilder;
3938
import org.opensearch.search.SearchModule;
4039
import org.opensearch.search.builder.SearchSourceBuilder;
41-
import org.opensearch.sql.legacy.esdomain.LocalClusterState;
4240
import org.opensearch.sql.legacy.executor.format.Schema;
4341

4442
/**
@@ -133,24 +131,20 @@ public String generateCursorId() {
133131
json.put(INDEX_PATTERN, indexPattern);
134132
json.put(SCHEMA_COLUMNS, getSchemaAsJson());
135133
json.put(FIELD_ALIAS_MAP, fieldAliasMap);
136-
if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) {
137-
json.put(PIT_ID, pitId);
138-
String sortFieldValue =
139-
AccessController.doPrivileged(
140-
(PrivilegedAction<String>)
141-
() -> {
142-
try {
143-
return objectMapper.writeValueAsString(sortFields);
144-
} catch (JsonProcessingException e) {
145-
throw new RuntimeException(
146-
"Failed to parse sort fields from JSON string.", e);
147-
}
148-
});
149-
json.put(SORT_FIELDS, sortFieldValue);
150-
setSearchRequestString(json, searchSourceBuilder);
151-
} else {
152-
json.put(SCROLL_ID, scrollId);
153-
}
134+
json.put(PIT_ID, pitId);
135+
String sortFieldValue =
136+
AccessController.doPrivileged(
137+
(PrivilegedAction<String>)
138+
() -> {
139+
try {
140+
return objectMapper.writeValueAsString(sortFields);
141+
} catch (JsonProcessingException e) {
142+
throw new RuntimeException("Failed to parse sort fields from JSON string.", e);
143+
}
144+
});
145+
json.put(SORT_FIELDS, sortFieldValue);
146+
setSearchRequestString(json, searchSourceBuilder);
147+
154148
return String.format("%s:%s", type.getId(), encodeCursor(json));
155149
}
156150

@@ -169,9 +163,7 @@ private void setSearchRequestString(JSONObject cursorJson, SearchSourceBuilder s
169163
}
170164

171165
private boolean isCursorIdNullOrEmpty() {
172-
return LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)
173-
? Strings.isNullOrEmpty(pitId)
174-
: Strings.isNullOrEmpty(scrollId);
166+
return Strings.isNullOrEmpty(pitId);
175167
}
176168

177169
public static DefaultCursor from(String cursorId) {
@@ -184,11 +176,7 @@ public static DefaultCursor from(String cursorId) {
184176
cursor.setFetchSize(json.getInt(FETCH_SIZE));
185177
cursor.setRowsLeft(json.getLong(ROWS_LEFT));
186178
cursor.setIndexPattern(json.getString(INDEX_PATTERN));
187-
if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) {
188-
populateCursorForPit(json, cursor);
189-
} else {
190-
cursor.setScrollId(json.getString(SCROLL_ID));
191-
}
179+
populateCursorForPit(json, cursor);
192180
cursor.setColumns(getColumnsFromSchema(json.getJSONArray(SCHEMA_COLUMNS)));
193181
cursor.setFieldAliasMap(fieldAliasMap(json.getJSONObject(FIELD_ALIAS_MAP)));
194182

‎legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticHitsExecutor.java‎

Lines changed: 13 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -7,21 +7,17 @@
77

88
import static org.opensearch.search.sort.FieldSortBuilder.DOC_FIELD_NAME;
99
import static org.opensearch.search.sort.SortOrder.ASC;
10-
import static org.opensearch.sql.common.setting.Settings.Key.SQL_CURSOR_KEEP_ALIVE;
11-
import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER;
1210
import static org.opensearch.sql.opensearch.storage.OpenSearchIndex.METADATA_FIELD_ID;
1311

1412
import java.io.IOException;
1513
import org.apache.logging.log4j.LogManager;
1614
import org.apache.logging.log4j.Logger;
1715
import org.opensearch.action.search.SearchRequestBuilder;
1816
import org.opensearch.action.search.SearchResponse;
19-
import org.opensearch.common.unit.TimeValue;
2017
import org.opensearch.search.SearchHits;
2118
import org.opensearch.search.builder.PointInTimeBuilder;
2219
import org.opensearch.search.sort.SortOrder;
2320
import org.opensearch.sql.legacy.domain.Select;
24-
import org.opensearch.sql.legacy.esdomain.LocalClusterState;
2521
import org.opensearch.sql.legacy.exception.SqlParseException;
2622
import org.opensearch.sql.legacy.pit.PointInTimeHandler;
2723
import org.opensearch.transport.client.Client;
@@ -67,36 +63,20 @@ public SearchResponse getResponseWithHits(
6763
request.setSize(size);
6864
SearchResponse responseWithHits;
6965

70-
if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) {
71-
// Set sort field for search_after
72-
boolean ordered = select.isOrderdSelect();
73-
if (!ordered) {
74-
request.addSort(DOC_FIELD_NAME, ASC);
75-
request.addSort(METADATA_FIELD_ID, SortOrder.ASC);
76-
}
77-
// Set PIT
78-
request.setPointInTime(new PointInTimeBuilder(pit.getPitId()));
79-
// from and size is alternate method to paginate result.
80-
// If select has from clause, search after is not required.
81-
if (previousResponse != null && select.getFrom().isEmpty()) {
82-
request.searchAfter(previousResponse.getHits().getSortFields());
83-
}
84-
responseWithHits = request.get();
85-
} else {
86-
// Set scroll
87-
TimeValue keepAlive = LocalClusterState.state().getSettingValue(SQL_CURSOR_KEEP_ALIVE);
88-
if (previousResponse != null) {
89-
responseWithHits =
90-
client
91-
.prepareSearchScroll(previousResponse.getScrollId())
92-
.setScroll(keepAlive)
93-
.execute()
94-
.actionGet();
95-
} else {
96-
request.setScroll(keepAlive);
97-
responseWithHits = request.get();
98-
}
66+
// Set sort field for search_after
67+
boolean ordered = select.isOrderdSelect();
68+
if (!ordered) {
69+
request.addSort(DOC_FIELD_NAME, ASC);
70+
request.addSort(METADATA_FIELD_ID, SortOrder.ASC);
9971
}
72+
// Set PIT
73+
request.setPointInTime(new PointInTimeBuilder(pit.getPitId()));
74+
// from and size is alternate method to paginate result.
75+
// If select has from clause, search after is not required.
76+
if (previousResponse != null && select.getFrom().isEmpty()) {
77+
request.searchAfter(previousResponse.getHits().getSortFields());
78+
}
79+
responseWithHits = request.get();
10080

10181
return responseWithHits;
10282
}

‎legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorCloseExecutor.java‎

Lines changed: 8 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -6,19 +6,16 @@
66
package org.opensearch.sql.legacy.executor.cursor;
77

88
import static org.opensearch.core.rest.RestStatus.OK;
9-
import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER;
109

1110
import java.util.Map;
1211
import org.apache.logging.log4j.LogManager;
1312
import org.apache.logging.log4j.Logger;
1413
import org.json.JSONException;
1514
import org.opensearch.OpenSearchException;
16-
import org.opensearch.action.search.ClearScrollResponse;
1715
import org.opensearch.rest.BytesRestResponse;
1816
import org.opensearch.rest.RestChannel;
1917
import org.opensearch.sql.legacy.cursor.CursorType;
2018
import org.opensearch.sql.legacy.cursor.DefaultCursor;
21-
import org.opensearch.sql.legacy.esdomain.LocalClusterState;
2219
import org.opensearch.sql.legacy.metrics.MetricName;
2320
import org.opensearch.sql.legacy.metrics.Metrics;
2421
import org.opensearch.sql.legacy.pit.PointInTimeHandler;
@@ -83,26 +80,14 @@ public String execute(Client client, Map<String, String> params) throws Exceptio
8380
}
8481

8582
private String handleDefaultCursorCloseRequest(Client client, DefaultCursor cursor) {
86-
if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) {
87-
String pitId = cursor.getPitId();
88-
PointInTimeHandler pit = new PointInTimeHandlerImpl(client, pitId);
89-
try {
90-
pit.delete();
91-
return SUCCEEDED_TRUE;
92-
} catch (RuntimeException e) {
93-
Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment();
94-
return SUCCEEDED_FALSE;
95-
}
96-
} else {
97-
String scrollId = cursor.getScrollId();
98-
ClearScrollResponse clearScrollResponse =
99-
client.prepareClearScroll().addScrollId(scrollId).get();
100-
if (clearScrollResponse.isSucceeded()) {
101-
return SUCCEEDED_TRUE;
102-
} else {
103-
Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment();
104-
return SUCCEEDED_FALSE;
105-
}
83+
String pitId = cursor.getPitId();
84+
PointInTimeHandler pit = new PointInTimeHandlerImpl(client, pitId);
85+
try {
86+
pit.delete();
87+
return SUCCEEDED_TRUE;
88+
} catch (RuntimeException e) {
89+
Metrics.getInstance().getNumericalMetric(MetricName.FAILED_REQ_COUNT_SYS).increment();
90+
return SUCCEEDED_FALSE;
10691
}
10792
}
10893
}

‎legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java‎

Lines changed: 16 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77

88
import static org.opensearch.core.rest.RestStatus.OK;
99
import static org.opensearch.sql.common.setting.Settings.Key.SQL_CURSOR_KEEP_ALIVE;
10-
import static org.opensearch.sql.common.setting.Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER;
1110

1211
import java.util.Arrays;
1312
import java.util.Map;
@@ -111,19 +110,15 @@ private String handleDefaultCursorRequest(Client client, DefaultCursor cursor) {
111110
TimeValue paginationTimeout = clusterState.getSettingValue(SQL_CURSOR_KEEP_ALIVE);
112111

113112
SearchResponse scrollResponse = null;
114-
if (clusterState.getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) {
115-
String pitId = cursor.getPitId();
116-
SearchSourceBuilder source = cursor.getSearchSourceBuilder();
117-
source.searchAfter(cursor.getSortFields());
118-
source.pointInTimeBuilder(new PointInTimeBuilder(pitId));
119-
SearchRequest searchRequest = new SearchRequest();
120-
searchRequest.source(source);
121-
scrollResponse = client.search(searchRequest).actionGet();
122-
} else {
123-
String previousScrollId = cursor.getScrollId();
124-
scrollResponse =
125-
client.prepareSearchScroll(previousScrollId).setScroll(paginationTimeout).get();
126-
}
113+
114+
String pitId = cursor.getPitId();
115+
SearchSourceBuilder source = cursor.getSearchSourceBuilder();
116+
source.searchAfter(cursor.getSortFields());
117+
source.pointInTimeBuilder(new PointInTimeBuilder(pitId));
118+
SearchRequest searchRequest = new SearchRequest();
119+
searchRequest.source(source);
120+
scrollResponse = client.search(searchRequest).actionGet();
121+
127122
SearchHits searchHits = scrollResponse.getHits();
128123
SearchHit[] searchHitArray = searchHits.getHits();
129124
String newScrollId = scrollResponse.getScrollId();
@@ -173,17 +168,13 @@ private String handleDefaultCursorRequest(Client client, DefaultCursor cursor) {
173168
}
174169

175170
cursor.setRowsLeft(rowsLeft);
176-
if (clusterState.getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) {
177-
cursor.setPitId(newPitId);
178-
cursor.setSearchSourceBuilder(cursor.getSearchSourceBuilder());
179-
cursor.setSortFields(
180-
scrollResponse
181-
.getHits()
182-
.getAt(scrollResponse.getHits().getHits().length - 1)
183-
.getSortValues());
184-
} else {
185-
cursor.setScrollId(newScrollId);
186-
}
171+
cursor.setPitId(newPitId);
172+
cursor.setSearchSourceBuilder(cursor.getSearchSourceBuilder());
173+
cursor.setSortFields(
174+
scrollResponse
175+
.getHits()
176+
.getAt(scrollResponse.getHits().getHits().length - 1)
177+
.getSortValues());
187178
Protocol protocol = new Protocol(client, searchHits, format.name().toLowerCase(), cursor);
188179
return protocol.cursorFormat();
189180
}

0 commit comments

Comments
 (0)