Skip to content

Commit 299d743

Browse files
acarbonettoMitchellGale
acarbonetto
authored and
MitchellGale
committed
Add _routing to SQL includes list (#277) (opensearch-project#1771)
* Add _routing to SQL includes list Signed-off-by: Andrew Carbonetto <[email protected]> * Update IT index Signed-off-by: Andrew Carbonetto <[email protected]> * Update doctest Signed-off-by: acarbonetto <[email protected]> * Add WHERE clause IT tests Signed-off-by: acarbonetto <[email protected]> * Fix IT test Signed-off-by: acarbonetto <[email protected]> * Update documentation for _routing Signed-off-by: acarbonetto <[email protected]> --------- Signed-off-by: Andrew Carbonetto <[email protected]> Signed-off-by: acarbonetto <[email protected]> Signed-off-by: Mitchell Gale <[email protected]>
1 parent a60b222 commit 299d743

File tree

6 files changed

+118
-8
lines changed

6 files changed

+118
-8
lines changed

‎docs/user/dql/basics.rst‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -155,14 +155,17 @@ Result set:
155155
| Nanette| Bates|
156156
+---------+--------+
157157

158-
One can also provide meta-field name(s) to retrieve reserved-fields (beginning with underscore) from OpenSearch documents. Meta-fields are not output
159-
from wildcard calls (`SELECT *`) and must be explicitly included to be returned.
158+
One can also provide meta-field name(s) to retrieve reserved-fields (beginning with underscore) from OpenSearch documents. They may also be used
159+
in the query `WHERE` or `ORDER BY` clauses. Meta-fields are not output from wildcard calls (`SELECT *`) and must be explicitly included to be returned.
160+
161+
Note: `_routing` is used differently in the `SELECT` and `WHERE` clauses. In `WHERE`, it contains the routing hash id. In `SELECT`,
162+
it returns the shard used for the query (unless shards aren't active, in which case it returns the routing hash id).
160163

161164
SQL query::
162165

163166
POST /_plugins/_sql
164167
{
165-
"query" : "SELECT firstname, lastname, _id, _index, _sort FROM accounts"
168+
"query" : "SELECT firstname, lastname, _id, _index, _sort, _routing FROM accounts WHERE _index = 'accounts'"
166169
}
167170

168171
Explain::
@@ -175,6 +178,7 @@ Explain::
175178
"firstname",
176179
"_id",
177180
"_index",
181+
"_routing",
178182
"_sort",
179183
"lastname"
180184
],

‎integ-test/src/test/java/org/opensearch/sql/sql/IdentifierIT.java‎

Lines changed: 93 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
import static org.opensearch.sql.util.TestUtils.performRequest;
1515

1616
import java.io.IOException;
17+
import java.util.ArrayList;
18+
import org.json.JSONArray;
1719
import org.json.JSONObject;
1820
import org.junit.jupiter.api.Test;
1921
import org.opensearch.client.Request;
@@ -99,6 +101,80 @@ public void testMetafieldIdentifierTest() throws IOException {
99101
verifyDataRows(result, rows(30, id, index, 1.0, 1.0, -2));
100102
}
101103

104+
@Test
105+
public void testMetafieldIdentifierRoutingSelectTest() throws IOException {
106+
// create an index, but the contents doesn't really matter
107+
String index = "test.routing_select";
108+
String mapping = "{\"_routing\": {\"required\": true }}";
109+
new Index(index, mapping)
110+
.addDocWithShardId("{\"age\": 31}", "test0", "test0")
111+
.addDocWithShardId("{\"age\": 31}", "test1", "test1")
112+
.addDocWithShardId("{\"age\": 32}", "test2", "test2")
113+
.addDocWithShardId("{\"age\": 33}", "test3", "test3")
114+
.addDocWithShardId("{\"age\": 34}", "test4", "test4")
115+
.addDocWithShardId("{\"age\": 35}", "test5", "test5");
116+
117+
// Execute using field metadata values filtering on the routing shard hash id
118+
final JSONObject result = new JSONObject(executeQuery(
119+
"SELECT age, _id, _index, _routing "
120+
+ "FROM " + index,
121+
"jdbc"));
122+
123+
// Verify that the metadata values are returned when requested
124+
verifySchema(result,
125+
schema("age", null, "long"),
126+
schema("_id", null, "keyword"),
127+
schema("_index", null, "keyword"),
128+
schema("_routing", null, "keyword"));
129+
assertTrue(result.getJSONArray("schema").length() == 4);
130+
131+
var datarows = result.getJSONArray("datarows");
132+
assertEquals(6, datarows.length());
133+
134+
// note that _routing in the SELECT clause returns the shard
135+
for (int i = 0; i < 6; i++) {
136+
assertEquals("test" + i, datarows.getJSONArray(i).getString(1));
137+
assertEquals(index, datarows.getJSONArray(i).getString(2));
138+
assertTrue(datarows.getJSONArray(i).getString(3).contains("[" + index + "]"));
139+
}
140+
}
141+
142+
@Test
143+
public void testMetafieldIdentifierRoutingFilterTest() throws IOException {
144+
// create an index, but the contents doesn't really matter
145+
String index = "test.routing_filter";
146+
String mapping = "{\"_routing\": {\"required\": true }}";
147+
new Index(index, mapping)
148+
.addDocWithShardId("{\"age\": 31}", "test1", "test1")
149+
.addDocWithShardId("{\"age\": 32}", "test2", "test2")
150+
.addDocWithShardId("{\"age\": 33}", "test3", "test3")
151+
.addDocWithShardId("{\"age\": 34}", "test4", "test4")
152+
.addDocWithShardId("{\"age\": 35}", "test5", "test5")
153+
.addDocWithShardId("{\"age\": 36}", "test6", "test6");
154+
155+
// Execute using field metadata values filtering on the routing shard hash id
156+
final JSONObject result = new JSONObject(executeQuery(
157+
"SELECT _id, _index, _routing "
158+
+ "FROM " + index + " "
159+
+ "WHERE _routing = \\\"test4\\\"",
160+
"jdbc"));
161+
162+
// Verify that the metadata values are returned when requested
163+
verifySchema(result,
164+
schema("_id", null, "keyword"),
165+
schema("_index", null, "keyword"),
166+
schema("_routing", null, "keyword"));
167+
assertTrue(result.getJSONArray("schema").length() == 3);
168+
169+
var datarows = result.getJSONArray("datarows");
170+
assertEquals(1, datarows.length());
171+
172+
assertEquals("test4", datarows.getJSONArray(0).getString(0));
173+
// note that _routing in the SELECT clause returns the shard, not the routing hash id
174+
assertTrue(datarows.getJSONArray(0).getString(2).contains("[" + index + "]"));
175+
176+
}
177+
102178
@Test
103179
public void testMetafieldIdentifierWithAliasTest() throws IOException {
104180
// create an index, but the contents doesn't matter
@@ -152,16 +228,32 @@ private static class Index {
152228
}
153229
}
154230

231+
Index(String indexName, String mapping) throws IOException {
232+
this.indexName = indexName;
233+
234+
Request createIndex = new Request("PUT", "/" + indexName);
235+
createIndex.setJsonEntity(mapping);
236+
executeRequest(new Request("PUT", "/" + indexName));
237+
}
238+
155239
void addDoc(String doc) {
156240
Request indexDoc = new Request("POST", String.format("/%s/_doc?refresh=true", indexName));
157241
indexDoc.setJsonEntity(doc);
158242
performRequest(client(), indexDoc);
159243
}
160244

161-
void addDoc(String doc, String id) {
245+
public Index addDoc(String doc, String id) {
162246
Request indexDoc = new Request("POST", String.format("/%s/_doc/%s?refresh=true", indexName, id));
163247
indexDoc.setJsonEntity(doc);
164248
performRequest(client(), indexDoc);
249+
return this;
250+
}
251+
252+
public Index addDocWithShardId(String doc, String id, String routing) {
253+
Request indexDoc = new Request("POST", String.format("/%s/_doc/%s?refresh=true&routing=%s", indexName, id, routing));
254+
indexDoc.setJsonEntity(doc);
255+
performRequest(client(), indexDoc);
256+
return this;
165257
}
166258
}
167259

‎opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import static org.opensearch.sql.opensearch.storage.OpenSearchIndex.METADATA_FIELD_ID;
1111
import static org.opensearch.sql.opensearch.storage.OpenSearchIndex.METADATA_FIELD_INDEX;
1212
import static org.opensearch.sql.opensearch.storage.OpenSearchIndex.METADATA_FIELD_MAXSCORE;
13+
import static org.opensearch.sql.opensearch.storage.OpenSearchIndex.METADATA_FIELD_ROUTING;
1314
import static org.opensearch.sql.opensearch.storage.OpenSearchIndex.METADATA_FIELD_SCORE;
1415
import static org.opensearch.sql.opensearch.storage.OpenSearchIndex.METADATA_FIELD_SORT;
1516

@@ -185,8 +186,10 @@ private void addMetaDataFieldsToBuilder(
185186
if (maxScore != null) {
186187
builder.put(METADATA_FIELD_MAXSCORE, maxScore);
187188
}
188-
} else { // if (metaDataField.equals(METADATA_FIELD_SORT)) {
189+
} else if (metaDataField.equals(METADATA_FIELD_SORT)) {
189190
builder.put(METADATA_FIELD_SORT, new ExprLongValue(hit.getSeqNo()));
191+
} else { // if (metaDataField.equals(METADATA_FIELD_ROUTING)){
192+
builder.put(METADATA_FIELD_ROUTING, new ExprStringValue(hit.getShard().toString()));
190193
}
191194
});
192195
}

‎opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,12 +45,15 @@ public class OpenSearchIndex implements Table {
4545
public static final String METADATA_FIELD_MAXSCORE = "_maxscore";
4646
public static final String METADATA_FIELD_SORT = "_sort";
4747

48+
public static final String METADATA_FIELD_ROUTING = "_routing";
49+
4850
public static final java.util.Map<String, ExprType> METADATAFIELD_TYPE_MAP = Map.of(
4951
METADATA_FIELD_ID, ExprCoreType.STRING,
5052
METADATA_FIELD_INDEX, ExprCoreType.STRING,
5153
METADATA_FIELD_SCORE, ExprCoreType.FLOAT,
5254
METADATA_FIELD_MAXSCORE, ExprCoreType.FLOAT,
53-
METADATA_FIELD_SORT, ExprCoreType.LONG
55+
METADATA_FIELD_SORT, ExprCoreType.LONG,
56+
METADATA_FIELD_ROUTING, ExprCoreType.STRING
5457
);
5558

5659
/** OpenSearch client connection. */

‎opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,10 @@
3232
import org.opensearch.action.search.SearchResponse;
3333
import org.opensearch.common.bytes.BytesArray;
3434
import org.opensearch.common.text.Text;
35+
import org.opensearch.index.shard.ShardId;
3536
import org.opensearch.search.SearchHit;
3637
import org.opensearch.search.SearchHits;
38+
import org.opensearch.search.SearchShardTarget;
3739
import org.opensearch.search.aggregations.Aggregations;
3840
import org.opensearch.search.fetch.subphase.highlight.HighlightField;
3941
import org.opensearch.sql.data.model.ExprFloatValue;
@@ -148,9 +150,13 @@ void iterator_metafields() {
148150
new TotalHits(1L, TotalHits.Relation.EQUAL_TO),
149151
3.75F));
150152

153+
ShardId shardId = new ShardId("index", "indexUUID", 42);
154+
SearchShardTarget shardTarget = new SearchShardTarget("node", shardId, null, null);
155+
151156
when(searchHit1.getSourceAsString()).thenReturn("{\"id1\", 1}");
152157
when(searchHit1.getId()).thenReturn("testId");
153158
when(searchHit1.getIndex()).thenReturn("testIndex");
159+
when(searchHit1.getShard()).thenReturn(shardTarget);
154160
when(searchHit1.getScore()).thenReturn(3.75F);
155161
when(searchHit1.getSeqNo()).thenReturn(123456L);
156162

@@ -160,11 +166,12 @@ void iterator_metafields() {
160166
"id1", new ExprIntegerValue(1),
161167
"_index", new ExprStringValue("testIndex"),
162168
"_id", new ExprStringValue("testId"),
169+
"_routing", new ExprStringValue(shardTarget.toString()),
163170
"_sort", new ExprLongValue(123456L),
164171
"_score", new ExprFloatValue(3.75F),
165172
"_maxscore", new ExprFloatValue(3.75F)
166173
));
167-
List includes = List.of("id1", "_index", "_id", "_sort", "_score", "_maxscore");
174+
List includes = List.of("id1", "_index", "_id", "_routing", "_sort", "_score", "_maxscore");
168175
int i = 0;
169176
for (ExprValue hit : new OpenSearchResponse(searchResponse, factory, includes)) {
170177
if (i == 0) {

‎opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -187,9 +187,10 @@ void getReservedFieldTypes() {
187187
assertThat(
188188
fieldTypes,
189189
allOf(
190-
aMapWithSize(5),
190+
aMapWithSize(6),
191191
hasEntry("_id", ExprCoreType.STRING),
192192
hasEntry("_index", ExprCoreType.STRING),
193+
hasEntry("_routing", ExprCoreType.STRING),
193194
hasEntry("_sort", ExprCoreType.LONG),
194195
hasEntry("_score", ExprCoreType.FLOAT),
195196
hasEntry("_maxscore", ExprCoreType.FLOAT)

0 commit comments

Comments
 (0)