diff --git a/core/src/main/java/org/opensearch/sql/storage/Table.java b/core/src/main/java/org/opensearch/sql/storage/Table.java index 731cf878c6d..7cdd757cbb4 100644 --- a/core/src/main/java/org/opensearch/sql/storage/Table.java +++ b/core/src/main/java/org/opensearch/sql/storage/Table.java @@ -16,6 +16,18 @@ */ public interface Table { + /** + * Check if current table exists. + * @return true if exists, otherwise false + */ + boolean exists(); + + /** + * Create table given table schema. + * @param schema table schema + */ + void create(Map schema); + /** * Get the {@link ExprType} for each field in the table. */ diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java index 3f912b8fde7..0a743ac7657 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java @@ -51,6 +51,16 @@ public Table getTable(String name) { @Bean protected Table table() { return new Table() { + @Override + public boolean exists() { + return true; + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException("Create table is not supported"); + } + @Override public Map getFieldTypes() { return typeMapping(); diff --git a/core/src/test/java/org/opensearch/sql/config/TestConfig.java b/core/src/test/java/org/opensearch/sql/config/TestConfig.java index ab78109aa26..449b40409f9 100644 --- a/core/src/test/java/org/opensearch/sql/config/TestConfig.java +++ b/core/src/test/java/org/opensearch/sql/config/TestConfig.java @@ -63,6 +63,16 @@ protected StorageEngine storageEngine() { @Override public Table getTable(String name) { return new Table() { + @Override + public boolean exists() { + return true; + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException("Create table is not supported"); + } + @Override public Map getFieldTypes() { return typeMapping; diff --git a/core/src/test/java/org/opensearch/sql/planner/PlannerTest.java b/core/src/test/java/org/opensearch/sql/planner/PlannerTest.java index 32e9d1b45bb..e02231ca068 100644 --- a/core/src/test/java/org/opensearch/sql/planner/PlannerTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/PlannerTest.java @@ -119,6 +119,16 @@ protected PhysicalPlan analyze(LogicalPlan logicalPlan) { protected class MockTable extends LogicalPlanNodeVisitor implements Table { + @Override + public boolean exists() { + return true; + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException("Create table is not supported"); + } + @Override public Map getFieldTypes() { throw new UnsupportedOperationException(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java index 09a83f65a57..dc6e72bd912 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java @@ -22,6 +22,20 @@ public interface OpenSearchClient { String META_CLUSTER_NAME = "CLUSTER_NAME"; + /** + * Check if the given index exists. + * @param indexName index name + * @return true if exists, otherwise false + */ + boolean exists(String indexName); + + /** + * Create OpenSearch index based on the given mappings. + * @param indexName index name + * @param mappings index mappings + */ + void createIndex(String indexName, Map mappings); + /** * Fetch index mapping(s) according to index expression given. * diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index 80a2fb86046..13fb0f63919 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -18,6 +18,9 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.opensearch.action.admin.indices.get.GetIndexResponse; import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; @@ -50,6 +53,28 @@ public OpenSearchNodeClient(NodeClient client) { this.resolver = new IndexNameExpressionResolver(client.threadPool().getThreadContext()); } + @Override + public boolean exists(String indexName) { + try { + IndicesExistsResponse checkExistResponse = client.admin().indices() + .exists(new IndicesExistsRequest(indexName)).actionGet(); + return checkExistResponse.isExists(); + } catch (Exception e) { + throw new IllegalStateException("Failed to check if index [" + indexName + "] exists", e); + } + } + + @Override + public void createIndex(String indexName, Map mappings) { + try { + // TODO: 1.pass index settings (the number of primary shards, etc); 2.check response? + CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName).mapping(mappings); + client.admin().indices().create(createIndexRequest).actionGet(); + } catch (Exception e) { + throw new IllegalStateException("Failed to create index [" + indexName + "]", e); + } + } + /** * Get field mappings of index by an index expression. Majority is copied from legacy * LocalClusterState. diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index f354215e058..d9f9dbbe5d5 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -23,6 +23,7 @@ import org.opensearch.action.search.ClearScrollRequest; import org.opensearch.client.RequestOptions; import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.indices.CreateIndexRequest; import org.opensearch.client.indices.GetIndexRequest; import org.opensearch.client.indices.GetIndexResponse; import org.opensearch.client.indices.GetMappingsRequest; @@ -46,6 +47,26 @@ public class OpenSearchRestClient implements OpenSearchClient { /** OpenSearch high level REST client. */ private final RestHighLevelClient client; + @Override + public boolean exists(String indexName) { + try { + return client.indices().exists( + new GetIndexRequest(indexName), RequestOptions.DEFAULT); + } catch (IOException e) { + throw new IllegalStateException("Failed to check if index [" + indexName + "] exist", e); + } + } + + @Override + public void createIndex(String indexName, Map mappings) { + try { + client.indices().create( + new CreateIndexRequest(indexName).mapping(mappings), RequestOptions.DEFAULT); + } catch (IOException e) { + throw new IllegalStateException("Failed to create index [" + indexName + "]", e); + } + } + @Override public Map getIndexMappings(String... indexExpression) { GetMappingsRequest request = new GetMappingsRequest().indices(indexExpression); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index f620ae3aaf6..05b80bfa238 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -9,12 +9,15 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN; +import com.google.common.collect.BiMap; +import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableMap; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import lombok.RequiredArgsConstructor; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; /** @@ -52,6 +55,39 @@ public boolean shouldCast(ExprType other) { OPENSEARCH_BINARY(Arrays.asList(UNKNOWN), "binary"); + /** + * Bidirectional mapping between OpenSearch type name and ExprType. + */ + private static final BiMap OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING = + ImmutableBiMap.builder() + .put("text", OPENSEARCH_TEXT) + .put("text_keyword", OPENSEARCH_TEXT_KEYWORD) + .put("keyword", ExprCoreType.STRING) + .put("byte", ExprCoreType.BYTE) + .put("short", ExprCoreType.SHORT) + .put("integer", ExprCoreType.INTEGER) + .put("long", ExprCoreType.LONG) + .put("float", ExprCoreType.FLOAT) + .put("double", ExprCoreType.DOUBLE) + .put("boolean", ExprCoreType.BOOLEAN) + .put("nested", ExprCoreType.ARRAY) + .put("object", ExprCoreType.STRUCT) + .put("date", ExprCoreType.TIMESTAMP) + .put("ip", OPENSEARCH_IP) + .put("geo_point", OPENSEARCH_GEO_POINT) + .put("binary", OPENSEARCH_BINARY) + .build(); + + /** + * Mapping from extra OpenSearch type name which may map to same ExprType as above. + */ + private static final Map EXTRA_OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING = + ImmutableMap.builder() + .put("half_float", ExprCoreType.FLOAT) + .put("scaled_float", ExprCoreType.DOUBLE) + .put("date_nanos", ExprCoreType.TIMESTAMP) + .build(); + /** * The mapping between Type and legacy JDBC type name. */ @@ -70,6 +106,27 @@ public boolean shouldCast(ExprType other) { */ private final String jdbcType; + /** + * Convert OpenSearch type string to ExprType. + * @param openSearchType OpenSearch type string + * @return expr type + */ + public static ExprType getExprType(String openSearchType) { + if (OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING.containsKey(openSearchType)) { + return OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING.get(openSearchType); + } + return EXTRA_OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING.getOrDefault(openSearchType, UNKNOWN); + } + + /** + * Convert ExprType to OpenSearch type string. + * @param type expr type + * @return OpenSearch type string + */ + public static String getOpenSearchType(ExprType type) { + return OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING.inverse().get(type); + } + @Override public List getParent() { return parents; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java index f3214970997..50402fc75be 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java @@ -10,7 +10,6 @@ import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; import static org.opensearch.sql.opensearch.client.OpenSearchClient.META_CLUSTER_NAME; -import com.google.common.collect.ImmutableMap; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; @@ -39,33 +38,6 @@ public class OpenSearchDescribeIndexRequest implements OpenSearchSystemRequest { private static final String DEFAULT_IS_AUTOINCREMENT = "NO"; - /** - * Type mapping from OpenSearch data type to expression type in our type system in query - * engine. TODO: geo, ip etc. - */ - private static final Map OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING = - ImmutableMap.builder() - .put("text", OpenSearchDataType.OPENSEARCH_TEXT) - .put("text_keyword", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD) - .put("keyword", ExprCoreType.STRING) - .put("byte", ExprCoreType.BYTE) - .put("short", ExprCoreType.SHORT) - .put("integer", ExprCoreType.INTEGER) - .put("long", ExprCoreType.LONG) - .put("float", ExprCoreType.FLOAT) - .put("half_float", ExprCoreType.FLOAT) - .put("scaled_float", ExprCoreType.DOUBLE) - .put("double", ExprCoreType.DOUBLE) - .put("boolean", ExprCoreType.BOOLEAN) - .put("nested", ExprCoreType.ARRAY) - .put("object", ExprCoreType.STRUCT) - .put("date", ExprCoreType.TIMESTAMP) - .put("date_nanos", ExprCoreType.TIMESTAMP) - .put("ip", OpenSearchDataType.OPENSEARCH_IP) - .put("geo_point", OpenSearchDataType.OPENSEARCH_GEO_POINT) - .put("binary", OpenSearchDataType.OPENSEARCH_BINARY) - .build(); - /** * OpenSearch client connection. */ @@ -132,7 +104,7 @@ public Integer getMaxResultWindow() { } private ExprType transformESTypeToExprType(String openSearchType) { - return OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING.getOrDefault(openSearchType, ExprCoreType.UNKNOWN); + return OpenSearchDataType.getExprType(openSearchType); } private ExprTupleValue row(String fieldName, String fieldType, int position, String clusterName) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java index ef6159020f9..88b823dd1fc 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java @@ -7,6 +7,7 @@ package org.opensearch.sql.opensearch.storage; import com.google.common.annotations.VisibleForTesting; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -18,6 +19,7 @@ import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.planner.logical.OpenSearchLogicalIndexAgg; import org.opensearch.sql.opensearch.planner.logical.OpenSearchLogicalIndexScan; @@ -72,6 +74,23 @@ public OpenSearchIndex(OpenSearchClient client, Settings settings, String indexN this.indexName = new OpenSearchRequest.IndexName(indexName); } + @Override + public boolean exists() { + return client.exists(indexName.toString()); + } + + @Override + public void create(Map schema) { + Map mappings = new HashMap<>(); + Map properties = new HashMap<>(); + mappings.put("properties", properties); + + for (Map.Entry colType : schema.entrySet()) { + properties.put(colType.getKey(), OpenSearchDataType.getOpenSearchType(colType.getValue())); + } + client.createIndex(indexName.toString(), mappings); + } + /* * TODO: Assume indexName doesn't have wildcard. * Need to either handle field name conflicts diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java index edd5593f4d1..7b6efeeba47 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java @@ -38,6 +38,17 @@ public OpenSearchSystemIndex( this.systemIndexBundle = buildIndexBundle(client, indexName); } + @Override + public boolean exists() { + return true; // TODO: implement for system index later + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException( + "OpenSearch system index is predefined and cannot be created"); + } + @Override public Map getFieldTypes() { return systemIndexBundle.getLeft().getMapping(); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java index ad26d792ed3..c3cd30a530e 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java @@ -32,12 +32,17 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.search.TotalHits; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.opensearch.action.admin.indices.get.GetIndexResponse; import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; @@ -88,12 +93,66 @@ class OpenSearchNodeClientTest { private ExprTupleValue exprTupleValue = ExprTupleValue.fromExprValueMap(ImmutableMap.of("id", new ExprIntegerValue(1))); + private OpenSearchClient client; + + @BeforeEach + void setUp() { + this.client = new OpenSearchNodeClient(nodeClient); + } + + @Test + void isIndexExist() { + when(nodeClient.admin().indices() + .exists(any(IndicesExistsRequest.class)).actionGet()) + .thenReturn(new IndicesExistsResponse(true)); + + assertTrue(client.exists("test")); + } + + @Test + void isIndexNotExist() { + String indexName = "test"; + when(nodeClient.admin().indices() + .exists(any(IndicesExistsRequest.class)).actionGet()) + .thenReturn(new IndicesExistsResponse(false)); + + assertFalse(client.exists(indexName)); + } + + @Test + void isIndexExistWithException() { + when(nodeClient.admin().indices().exists(any())).thenThrow(RuntimeException.class); + + assertThrows(IllegalStateException.class, () -> client.exists("test")); + } + @Test - public void getIndexMappings() throws IOException { + void createIndex() { + String indexName = "test"; + Map mappings = ImmutableMap.of( + "properties", + ImmutableMap.of("name", "text")); + when(nodeClient.admin().indices() + .create(any(CreateIndexRequest.class)).actionGet()) + .thenReturn(new CreateIndexResponse(true, true, indexName)); + + client.createIndex(indexName, mappings); + } + + @Test + void createIndexWithException() { + when(nodeClient.admin().indices().create(any())).thenThrow(RuntimeException.class); + + assertThrows(IllegalStateException.class, + () -> client.createIndex("test", ImmutableMap.of())); + } + + @Test + void getIndexMappings() throws IOException { URL url = Resources.getResource(TEST_MAPPING_FILE); String mappings = Resources.toString(url, Charsets.UTF_8); String indexName = "test"; - OpenSearchNodeClient client = mockClient(indexName, mappings); + mockNodeClientIndicesMappings(indexName, mappings); Map indexMappings = client.getIndexMappings(indexName); assertEquals(1, indexMappings.size()); @@ -121,9 +180,9 @@ public void getIndexMappings() throws IOException { } @Test - public void getIndexMappingsWithEmptyMapping() { + void getIndexMappingsWithEmptyMapping() { String indexName = "test"; - OpenSearchNodeClient client = mockClient(indexName, ""); + mockNodeClientIndicesMappings(indexName, ""); Map indexMappings = client.getIndexMappings(indexName); assertEquals(1, indexMappings.size()); @@ -132,28 +191,25 @@ public void getIndexMappingsWithEmptyMapping() { } @Test - public void getIndexMappingsWithIOException() { + void getIndexMappingsWithIOException() { String indexName = "test"; when(nodeClient.admin().indices()).thenThrow(RuntimeException.class); - OpenSearchNodeClient client = new OpenSearchNodeClient(nodeClient); assertThrows(IllegalStateException.class, () -> client.getIndexMappings(indexName)); } @Test - public void getIndexMappingsWithNonExistIndex() { - OpenSearchNodeClient client = - new OpenSearchNodeClient(mockNodeClient("test")); + void getIndexMappingsWithNonExistIndex() { + mockNodeClient("test"); assertTrue(client.getIndexMappings("non_exist_index").isEmpty()); } @Test - public void getIndexMaxResultWindows() throws IOException { + void getIndexMaxResultWindows() throws IOException { URL url = Resources.getResource(TEST_MAPPING_SETTINGS_FILE); String indexMetadata = Resources.toString(url, Charsets.UTF_8); String indexName = "accounts"; - OpenSearchNodeClient client = - new OpenSearchNodeClient(mockNodeClientSettings(indexName, indexMetadata)); + mockNodeClientSettings(indexName, indexMetadata); Map indexMaxResultWindows = client.getIndexMaxResultWindows(indexName); assertEquals(1, indexMaxResultWindows.size()); @@ -163,12 +219,11 @@ public void getIndexMaxResultWindows() throws IOException { } @Test - public void getIndexMaxResultWindowsWithDefaultSettings() throws IOException { + void getIndexMaxResultWindowsWithDefaultSettings() throws IOException { URL url = Resources.getResource(TEST_MAPPING_FILE); String indexMetadata = Resources.toString(url, Charsets.UTF_8); String indexName = "accounts"; - OpenSearchNodeClient client = - new OpenSearchNodeClient(mockNodeClientSettings(indexName, indexMetadata)); + mockNodeClientSettings(indexName, indexMetadata); Map indexMaxResultWindows = client.getIndexMaxResultWindows(indexName); assertEquals(1, indexMaxResultWindows.size()); @@ -178,25 +233,21 @@ public void getIndexMaxResultWindowsWithDefaultSettings() throws IOException { } @Test - public void getIndexMaxResultWindowsWithIOException() { + void getIndexMaxResultWindowsWithIOException() { String indexName = "test"; when(nodeClient.admin().indices()).thenThrow(RuntimeException.class); - OpenSearchNodeClient client = new OpenSearchNodeClient(nodeClient); assertThrows(IllegalStateException.class, () -> client.getIndexMaxResultWindows(indexName)); } /** Jacoco enforce this constant lambda be tested. */ @Test - public void testAllFieldsPredicate() { + void testAllFieldsPredicate() { assertTrue(OpenSearchNodeClient.ALL_FIELDS.apply("any_index").test("any_field")); } @Test - public void search() { - OpenSearchNodeClient client = - new OpenSearchNodeClient(nodeClient); - + void search() { // Mock first scroll request SearchResponse searchResponse = mock(SearchResponse.class); when(nodeClient.search(any()).actionGet()).thenReturn(searchResponse); @@ -233,7 +284,6 @@ public void search() { @Test void schedule() { - OpenSearchNodeClient client = new OpenSearchNodeClient(nodeClient); AtomicBoolean isRun = new AtomicBoolean(false); client.schedule( () -> { @@ -249,7 +299,6 @@ void cleanup() { when(requestBuilder.addScrollId(any())).thenReturn(requestBuilder); when(requestBuilder.get()).thenReturn(null); - OpenSearchNodeClient client = new OpenSearchNodeClient(nodeClient); OpenSearchScrollRequest request = new OpenSearchScrollRequest("test", factory); request.setScrollId("scroll123"); client.cleanup(request); @@ -263,8 +312,6 @@ void cleanup() { @Test void cleanupWithoutScrollId() { - OpenSearchNodeClient client = new OpenSearchNodeClient(nodeClient); - OpenSearchScrollRequest request = new OpenSearchScrollRequest("test", factory); client.cleanup(request); verify(nodeClient, never()).prepareClearScroll(); @@ -284,7 +331,6 @@ void getIndices() { when(indexResponse.getIndices()).thenReturn(new String[] {"index"}); when(indexResponse.aliases()).thenReturn(openMap); - OpenSearchNodeClient client = new OpenSearchNodeClient(nodeClient); final List indices = client.indices(); assertEquals(2, indices.size()); } @@ -295,22 +341,15 @@ void meta() { when(nodeClient.settings()).thenReturn(settings); when(settings.get(anyString(), anyString())).thenReturn("cluster-name"); - OpenSearchNodeClient client = new OpenSearchNodeClient(nodeClient); final Map meta = client.meta(); assertEquals("cluster-name", meta.get(META_CLUSTER_NAME)); } @Test void ml() { - OpenSearchNodeClient client = new OpenSearchNodeClient(nodeClient); assertNotNull(client.getNodeClient()); } - private OpenSearchNodeClient mockClient(String indexName, String mappings) { - mockNodeClientIndicesMappings(indexName, mappings); - return new OpenSearchNodeClient(nodeClient); - } - public void mockNodeClientIndicesMappings(String indexName, String mappings) { GetMappingsResponse mockResponse = mock(GetMappingsResponse.class); MappingMetadata emptyMapping = mock(MappingMetadata.class); @@ -336,17 +375,16 @@ public void mockNodeClientIndicesMappings(String indexName, String mappings) { } } - public NodeClient mockNodeClient(String indexName) { + public void mockNodeClient(String indexName) { GetMappingsResponse mockResponse = mock(GetMappingsResponse.class); when(nodeClient.admin().indices() .prepareGetMappings(any()) .setLocal(anyBoolean()) .get()).thenReturn(mockResponse); when(mockResponse.mappings()).thenReturn(ImmutableOpenMap.of()); - return nodeClient; } - private NodeClient mockNodeClientSettings(String indexName, String indexMetadata) + private void mockNodeClientSettings(String indexName, String indexMetadata) throws IOException { GetSettingsResponse mockResponse = mock(GetSettingsResponse.class); when(nodeClient.admin().indices().prepareGetSettings(any()).setLocal(anyBoolean()).get()) @@ -357,7 +395,6 @@ private NodeClient mockNodeClientSettings(String indexName, String indexMetadata .build(); when(mockResponse.getIndexToSettings()).thenReturn(metadata); - return nodeClient; } private XContentParser createParser(String mappings) throws IOException { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java index bc334aaf399..25cfd6b35c2 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java @@ -39,6 +39,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.RequestOptions; import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.indices.CreateIndexResponse; import org.opensearch.client.indices.GetIndexRequest; import org.opensearch.client.indices.GetIndexResponse; import org.opensearch.client.indices.GetMappingsRequest; @@ -69,7 +70,7 @@ class OpenSearchRestClientTest { @Mock(answer = RETURNS_DEEP_STUBS) private RestHighLevelClient restClient; - private OpenSearchRestClient client; + private OpenSearchClient client; @Mock private OpenSearchExprValueFactory factory; @@ -88,6 +89,52 @@ void setUp() { client = new OpenSearchRestClient(restClient); } + @Test + void isIndexExist() throws IOException { + when(restClient.indices() + .exists(any(), any())) // use any() because missing equals() in GetIndexRequest + .thenReturn(true); + + assertTrue(client.exists("test")); + } + + @Test + void isIndexNotExist() throws IOException { + when(restClient.indices() + .exists(any(), any())) // use any() because missing equals() in GetIndexRequest + .thenReturn(false); + + assertFalse(client.exists("test")); + } + + @Test + void isIndexExistWithException() throws IOException { + when(restClient.indices().exists(any(), any())).thenThrow(IOException.class); + + assertThrows(IllegalStateException.class, () -> client.exists("test")); + } + + @Test + void createIndex() throws IOException { + String indexName = "test"; + Map mappings = ImmutableMap.of( + "properties", + ImmutableMap.of("name", "text")); + when(restClient.indices() + .create(any(), any())) + .thenReturn(new CreateIndexResponse(true, true, indexName)); + + client.createIndex(indexName, mappings); + } + + @Test + void createIndexWithIOException() throws IOException { + when(restClient.indices().create(any(), any())).thenThrow(IOException.class); + + assertThrows(IllegalStateException.class, + () -> client.createIndex("test", ImmutableMap.of())); + } + @Test void getIndexMappings() throws IOException { URL url = Resources.getResource(TEST_MAPPING_FILE); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index e55fff0e33b..57bfcd1ea85 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -9,7 +9,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT; import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; @@ -42,4 +45,23 @@ public void testShouldCast() { assertFalse(OPENSEARCH_TEXT.shouldCast(STRING)); assertFalse(OPENSEARCH_TEXT_KEYWORD.shouldCast(STRING)); } + + @Test + public void testGetExprType() { + assertEquals(OPENSEARCH_TEXT, OpenSearchDataType.getExprType("text")); + assertEquals(FLOAT, OpenSearchDataType.getExprType("float")); + assertEquals(FLOAT, OpenSearchDataType.getExprType("half_float")); + assertEquals(DOUBLE, OpenSearchDataType.getExprType("double")); + assertEquals(DOUBLE, OpenSearchDataType.getExprType("scaled_float")); + assertEquals(TIMESTAMP, OpenSearchDataType.getExprType("date")); + assertEquals(TIMESTAMP, OpenSearchDataType.getExprType("date_nanos")); + } + + @Test + public void testGetOpenSearchType() { + assertEquals("text", OpenSearchDataType.getOpenSearchType(OPENSEARCH_TEXT)); + assertEquals("float", OpenSearchDataType.getOpenSearchType(FLOAT)); + assertEquals("double", OpenSearchDataType.getOpenSearchType(DOUBLE)); + assertEquals("date", OpenSearchDataType.getOpenSearchType(TIMESTAMP)); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java index 82ac3991ac7..f7860403c77 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java @@ -14,6 +14,7 @@ import static org.hamcrest.Matchers.hasEntry; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.when; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; @@ -21,6 +22,7 @@ import static org.opensearch.sql.expression.DSL.literal; import static org.opensearch.sql.expression.DSL.named; import static org.opensearch.sql.expression.DSL.ref; +import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; import static org.opensearch.sql.opensearch.utils.Utils.indexScan; import static org.opensearch.sql.opensearch.utils.Utils.indexScanAgg; import static org.opensearch.sql.opensearch.utils.Utils.noProjects; @@ -37,10 +39,12 @@ import com.google.common.collect.ImmutableMap; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -77,6 +81,8 @@ class OpenSearchIndexTest { private final DSL dsl = new ExpressionConfig().dsl(new ExpressionConfig().functionRepository()); + private final String indexName = "test"; + @Mock private OpenSearchClient client; @@ -89,6 +95,35 @@ class OpenSearchIndexTest { @Mock private Table table; + private OpenSearchIndex index; + + @BeforeEach + void setUp() { + this.index = new OpenSearchIndex(client, settings, indexName); + } + + @Test + void isExist() { + when(client.exists(indexName)).thenReturn(true); + + assertTrue(index.exists()); + } + + @Test + void createIndex() { + Map mappings = ImmutableMap.of( + "properties", + ImmutableMap.of( + "name", "text_keyword", + "age", "integer")); + doNothing().when(client).createIndex(indexName, mappings); + + Map schema = new HashMap<>(); + schema.put("name", OPENSEARCH_TEXT_KEYWORD); + schema.put("age", INTEGER); + index.create(schema); + } + @Test void getFieldTypes() { when(client.getIndexMappings("test")) @@ -112,7 +147,6 @@ void getFieldTypes() { .put("blob", "binary") .build()))); - OpenSearchIndex index = new OpenSearchIndex(client, settings, "test"); Map fieldTypes = index.getFieldTypes(); assertThat( fieldTypes, @@ -139,9 +173,7 @@ void implementRelationOperatorOnly() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); - String indexName = "test"; LogicalPlan plan = relation(indexName, table); - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); Integer maxResultWindow = index.getMaxResultWindow(); assertEquals( new OpenSearchIndexScan(client, settings, indexName, maxResultWindow, exprValueFactory), @@ -153,9 +185,7 @@ void implementRelationOperatorWithOptimization() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); - String indexName = "test"; LogicalPlan plan = relation(indexName, table); - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); Integer maxResultWindow = index.getMaxResultWindow(); assertEquals( new OpenSearchIndexScan(client, settings, indexName, maxResultWindow, exprValueFactory), @@ -167,7 +197,6 @@ void implementOtherLogicalOperators() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); - String indexName = "test"; NamedExpression include = named("age", ref("age", INTEGER)); ReferenceExpression exclude = ref("name", STRING); ReferenceExpression dedupeField = ref("name", STRING); @@ -199,7 +228,6 @@ void implementOtherLogicalOperators() { dedupeField), include); - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); Integer maxResultWindow = index.getMaxResultWindow(); assertEquals( PhysicalPlanDSL.project( @@ -228,8 +256,6 @@ void shouldImplLogicalIndexScan() { NamedExpression named = named("n", field); Expression filterExpr = dsl.equal(field, literal("John")); - String indexName = "test"; - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); PhysicalPlan plan = index.implement( project( indexScan( @@ -254,8 +280,6 @@ void shouldNotPushDownFilterFarFromRelation() { Arrays.asList(named("avg(age)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), DOUBLE))); - String indexName = "test"; - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); PhysicalPlan plan = index.implement( filter( aggregation( @@ -280,9 +304,6 @@ void shouldImplLogicalIndexScanAgg() { Arrays.asList(named("avg(age)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), DOUBLE))); - String indexName = "test"; - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); - // IndexScanAgg without Filter PhysicalPlan plan = index.implement( filter( @@ -317,9 +338,6 @@ void shouldNotPushDownAggregationFarFromRelation() { Arrays.asList(named("avg(age)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), DOUBLE))); - String indexName = "test"; - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); - PhysicalPlan plan = index.implement( aggregation( filter(filter( @@ -339,8 +357,6 @@ void shouldImplIndexScanWithSort() { NamedExpression named = named("n", field); Expression sortExpr = ref("name", STRING); - String indexName = "test"; - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); PhysicalPlan plan = index.implement( project( indexScan( @@ -361,8 +377,6 @@ void shouldImplIndexScanWithLimit() { ReferenceExpression field = ref("name", STRING); NamedExpression named = named("n", field); - String indexName = "test"; - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); PhysicalPlan plan = index.implement( project( indexScan( @@ -384,8 +398,6 @@ void shouldImplIndexScanWithSortAndLimit() { NamedExpression named = named("n", field); Expression sortExpr = ref("name", STRING); - String indexName = "test"; - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); PhysicalPlan plan = index.implement( project( indexScan( @@ -405,8 +417,6 @@ void shouldNotPushDownLimitFarFromRelationButUpdateScanSize() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); - String indexName = "test"; - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); PhysicalPlan plan = index.implement(index.optimize( project( limit( @@ -430,8 +440,6 @@ void shouldPushDownProjects() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); - String indexName = "test"; - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); PhysicalPlan plan = index.implement( project( indexScan( diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java index e2efff22cb3..a483f2dad8b 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java @@ -9,6 +9,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.hasEntry; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.DSL.named; @@ -18,6 +19,7 @@ import static org.opensearch.sql.utils.SystemIndexUtils.TABLE_INFO; import static org.opensearch.sql.utils.SystemIndexUtils.mappingTable; +import com.google.common.collect.ImmutableMap; import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -58,6 +60,19 @@ void testGetFieldTypesOfMappingTable() { )); } + @Test + void testIsExist() { + Table systemIndex = new OpenSearchSystemIndex(client, TABLE_INFO); + assertTrue(systemIndex.exists()); + } + + @Test + void testCreateTable() { + Table systemIndex = new OpenSearchSystemIndex(client, TABLE_INFO); + assertThrows(UnsupportedOperationException.class, + () -> systemIndex.create(ImmutableMap.of())); + } + @Test void implement() { OpenSearchSystemIndex systemIndex = new OpenSearchSystemIndex(client, TABLE_INFO);