diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java index de84486b88063..23877356d8409 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java @@ -47,6 +47,11 @@ public enum ScalarFunction { // ── Full-text search ───────────────────────────────────────────── MATCH(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), MATCH_PHRASE(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), + MATCH_BOOL_PREFIX(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), + MATCH_PHRASE_PREFIX(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), + MULTI_MATCH(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), + QUERY_STRING(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), + SIMPLE_QUERY_STRING(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), FUZZY(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), WILDCARD(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), REGEXP(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java index fcd4edf7f311c..bc3b2ea802588 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java @@ -18,6 +18,7 @@ import org.opensearch.index.query.QueryBuilder; import java.io.IOException; +import java.util.ArrayList; import java.util.List; /** @@ -30,6 +31,13 @@ */ final class ConversionUtils { + /** MAP key for single-field relevance operands. */ + static final String KEY_FIELD = "field"; + /** MAP key for multi-field relevance operands. */ + static final String KEY_FIELDS = "fields"; + /** MAP key for the query text operand. */ + static final String KEY_QUERY = "query"; + private ConversionUtils() {} /** @@ -77,4 +85,121 @@ static byte[] serializeQueryBuilder(QueryBuilder queryBuilder) { throw new IllegalStateException("Failed to serialize delegated query: " + queryBuilder, exception); } } + + /** + * Extracts the key string from a MAP_VALUE_CONSTRUCTOR operand: MAP('key', value). + * Returns null if the operand is not a MAP or the key is not a string literal. + */ + static String extractMapKey(RexCall call, int operandIndex) { + RexNode operand = call.getOperands().get(operandIndex); + if (operand instanceof RexCall mapCall && mapCall.getOperands().size() >= 2) { + RexNode key = mapCall.getOperands().get(0); + if (key instanceof RexLiteral literal) { + return literal.getValueAs(String.class); + } + } + return null; + } + + /** + * Extracted operands from a relevance function RexCall. + * @param fieldName single field name (null if not present or multi-field) + * @param fields multiple field names (null if not present) + * @param query the query string (null if not found) + */ + record RelevanceOperands(String fieldName, List fields, String query) { + } + + /** + * Extracts field/fields and query from a relevance function RexCall by MAP key lookup, + * with positional fallback for non-MAP operand structures (e.g. MATCH($ref, literal)). + * + * @param call the relevance function RexCall + * @param fieldStorage per-column storage metadata for resolving field names + * @return extracted operands + */ + static RelevanceOperands extractRelevanceOperands(RexCall call, List fieldStorage) { + String fieldName = null; + List fields = null; + String query = null; + + for (int i = 0; i < call.getOperands().size(); i++) { + String key = extractMapKey(call, i); + if (KEY_FIELD.equals(key)) { + fieldName = extractFieldFromRelevanceMap(call, i, fieldStorage); + } else if (KEY_FIELDS.equals(key)) { + fields = extractFieldsFromRelevanceMap(call, i, fieldStorage); + } else if (KEY_QUERY.equals(key)) { + query = extractStringFromRelevanceMap(call, i); + } + } + + // Fallback: positional extraction for non-MAP operand structures (e.g. MATCH($ref, literal)) + if (fieldName == null && fields == null && query == null && call.getOperands().size() >= 2) { + fieldName = extractFieldFromRelevanceMap(call, 0, fieldStorage); + query = extractStringFromRelevanceMap(call, 1); + } + + return new RelevanceOperands(fieldName, fields, query); + } + + /** + * Extracts multiple field names from a MAP_VALUE_CONSTRUCTOR operand + * for multi-field full-text functions (multi_match, query_string, simple_query_string). + * + *

The operand structure for multi-field functions: + * {@code MAP('fields', MAP('field1':VARCHAR, boost1:DOUBLE, 'field2':VARCHAR, boost2:DOUBLE, ...))} + * The outer MAP has key='fields' at index 0 and a nested MAP at index 1. + * The nested MAP is a Calcite MAP_VALUE_CONSTRUCTOR with strict alternating key-value pairs: + * field name (VARCHAR) at even indices, boost value (DOUBLE) at odd indices. + * + *

Also supports the RexInputRef-based structure for single-field fallback: + * {@code MAP('field', $ref1, 'field', $ref2, ...)} + * + *

Note: This method is intentionally not recursive. The MAP nesting depth is bounded + * to at most 2 levels by Calcite's MAP_VALUE_CONSTRUCTOR design: an outer MAP holding + * the 'fields' key and a nested MAP holding field-name/boost pairs. Deeper nesting does + * not occur in the PPL relevance function encoding. + * + *

TODO: extract per-field boost values and return them alongside field names. + */ + static List extractFieldsFromRelevanceMap(RexCall call, int operandIndex, List fieldStorage) { + RexNode operand = call.getOperands().get(operandIndex); + List fields = new ArrayList<>(); + if (operand instanceof RexCall outerMapCall) { + // Check if the value (index 1) is a nested MAP containing field name/boost pairs + if (outerMapCall.getOperands().size() >= 2) { + RexNode value = outerMapCall.getOperands().get(1); + if (value instanceof RexCall nestedMapCall) { + // Nested MAP: strict alternating key-value pairs from MAP_VALUE_CONSTRUCTOR. + // Even indices (0, 2, 4...) are field name VARCHAR literals. + // Odd indices (1, 3, 5...) are boost DOUBLE literals (ignored for now). + List nestedOperands = nestedMapCall.getOperands(); + for (int i = 0; i < nestedOperands.size(); i += 2) { + RexNode fieldNode = nestedOperands.get(i); + if (fieldNode instanceof RexLiteral fieldLiteral) { + fields.add(fieldLiteral.getValueAs(String.class)); + } + } + if (fields.isEmpty() == false) { + return fields; + } + } + } + // Fallback: RexInputRef-based structure MAP('field', $ref1, 'field', $ref2, ...) + List mapOperands = outerMapCall.getOperands(); + for (int i = 1; i < mapOperands.size(); i += 2) { + RexNode val = mapOperands.get(i); + if (val instanceof RexInputRef inputRef) { + fields.add(FieldStorageInfo.resolve(fieldStorage, inputRef.getIndex()).getFieldName()); + } + } + } else if (operand instanceof RexInputRef inputRef) { + fields.add(FieldStorageInfo.resolve(fieldStorage, inputRef.getIndex()).getFieldName()); + } + if (fields.isEmpty()) { + throw new IllegalArgumentException("Cannot extract field names from operand " + operandIndex + ": " + operand); + } + return fields; + } } diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java index 5a59dda788db0..82f628f748e27 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java @@ -64,6 +64,11 @@ public class LuceneAnalyticsBackendPlugin implements AnalyticsSearchBackendPlugi private static final Set FULL_TEXT_OPS = Set.of( ScalarFunction.MATCH, ScalarFunction.MATCH_PHRASE, + ScalarFunction.MATCH_BOOL_PREFIX, + ScalarFunction.MATCH_PHRASE_PREFIX, + ScalarFunction.MULTI_MATCH, + ScalarFunction.QUERY_STRING, + ScalarFunction.SIMPLE_QUERY_STRING, ScalarFunction.FUZZY, ScalarFunction.WILDCARD, ScalarFunction.REGEXP diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java index 13bda07674b22..8d5b101635de0 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java @@ -12,7 +12,13 @@ import org.opensearch.analytics.spi.DelegatedPredicateSerializer; import org.opensearch.analytics.spi.FieldStorageInfo; import org.opensearch.analytics.spi.ScalarFunction; +import org.opensearch.index.query.MatchBoolPrefixQueryBuilder; +import org.opensearch.index.query.MatchPhrasePrefixQueryBuilder; +import org.opensearch.index.query.MatchPhraseQueryBuilder; import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.MultiMatchQueryBuilder; +import org.opensearch.index.query.QueryStringQueryBuilder; +import org.opensearch.index.query.SimpleQueryStringBuilder; import java.util.List; import java.util.Map; @@ -21,15 +27,17 @@ * Registry of per-function query serializers for delegated predicates. * Each serializer converts a Calcite RexCall into serialized QueryBuilder bytes * that the Lucene backend can deserialize at the data node. - * - *

TODO: add serializers for match_phrase, match_bool_prefix, match_phrase_prefix. - * TODO: add multi-field relevance serializers for multi_match, query_string, simple_query_string. */ final class QuerySerializerRegistry { - private static final Map SERIALIZERS = Map.of( - ScalarFunction.MATCH, - QuerySerializerRegistry::serializeMatch + private static final Map SERIALIZERS = Map.ofEntries( + Map.entry(ScalarFunction.MATCH, QuerySerializerRegistry::serializeMatch), + Map.entry(ScalarFunction.MATCH_PHRASE, QuerySerializerRegistry::serializeMatchPhrase), + Map.entry(ScalarFunction.MATCH_BOOL_PREFIX, QuerySerializerRegistry::serializeMatchBoolPrefix), + Map.entry(ScalarFunction.MATCH_PHRASE_PREFIX, QuerySerializerRegistry::serializeMatchPhrasePrefix), + Map.entry(ScalarFunction.MULTI_MATCH, QuerySerializerRegistry::serializeMultiMatch), + Map.entry(ScalarFunction.QUERY_STRING, QuerySerializerRegistry::serializeQueryString), + Map.entry(ScalarFunction.SIMPLE_QUERY_STRING, QuerySerializerRegistry::serializeSimpleQueryString) ); private QuerySerializerRegistry() {} @@ -38,11 +46,89 @@ static Map getSerializers() { return SERIALIZERS; } + // TODO: Extract each serialize* method into its own dedicated class once we handle more parameters. + // These methods are expected to grow significantly as optional parameters are added. + private static byte[] serializeMatch(RexCall call, List fieldStorage) { - String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage); - String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1); - // TODO: extract optional params (operator, analyzer, fuzziness) from operands 2+ - MatchQueryBuilder queryBuilder = new MatchQueryBuilder(fieldName, queryText); + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(call, fieldStorage); + if (operands.fieldName() == null || operands.query() == null) { + throw new IllegalArgumentException("match requires 'field' and 'query' parameters, got: " + call); + } + // TODO: extract optional params (operator, analyzer, fuzziness, boost) + MatchQueryBuilder queryBuilder = new MatchQueryBuilder(operands.fieldName(), operands.query()); + return ConversionUtils.serializeQueryBuilder(queryBuilder); + } + + private static byte[] serializeMatchPhrase(RexCall call, List fieldStorage) { + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(call, fieldStorage); + if (operands.fieldName() == null || operands.query() == null) { + throw new IllegalArgumentException("match_phrase requires 'field' and 'query' parameters, got: " + call); + } + // TODO: extract optional params (slop, analyzer, zero_terms_query) + MatchPhraseQueryBuilder queryBuilder = new MatchPhraseQueryBuilder(operands.fieldName(), operands.query()); + return ConversionUtils.serializeQueryBuilder(queryBuilder); + } + + private static byte[] serializeMatchBoolPrefix(RexCall call, List fieldStorage) { + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(call, fieldStorage); + if (operands.fieldName() == null || operands.query() == null) { + throw new IllegalArgumentException("match_bool_prefix requires 'field' and 'query' parameters, got: " + call); + } + // TODO: extract optional params (analyzer, fuzziness, operator, minimum_should_match) + MatchBoolPrefixQueryBuilder queryBuilder = new MatchBoolPrefixQueryBuilder(operands.fieldName(), operands.query()); + return ConversionUtils.serializeQueryBuilder(queryBuilder); + } + + private static byte[] serializeMatchPhrasePrefix(RexCall call, List fieldStorage) { + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(call, fieldStorage); + if (operands.fieldName() == null || operands.query() == null) { + throw new IllegalArgumentException("match_phrase_prefix requires 'field' and 'query' parameters, got: " + call); + } + // TODO: extract optional params (slop, analyzer, max_expansions, zero_terms_query) + MatchPhrasePrefixQueryBuilder queryBuilder = new MatchPhrasePrefixQueryBuilder(operands.fieldName(), operands.query()); + return ConversionUtils.serializeQueryBuilder(queryBuilder); + } + + private static byte[] serializeMultiMatch(RexCall call, List fieldStorage) { + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(call, fieldStorage); + if (operands.query() == null) { + throw new IllegalArgumentException("multi_match requires a 'query' parameter, got: " + call); + } + // TODO: extract per-field boost values and optional params (type, operator, analyzer, fuzziness) + List fields = operands.fields(); + MultiMatchQueryBuilder queryBuilder = fields != null + ? new MultiMatchQueryBuilder(operands.query(), fields.toArray(String[]::new)) + : new MultiMatchQueryBuilder(operands.query()); + return ConversionUtils.serializeQueryBuilder(queryBuilder); + } + + private static byte[] serializeQueryString(RexCall call, List fieldStorage) { + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(call, fieldStorage); + if (operands.query() == null) { + throw new IllegalArgumentException("query_string requires a 'query' parameter, got: " + call); + } + // TODO: extract optional params (default_operator, analyzer, allow_leading_wildcard) + QueryStringQueryBuilder queryBuilder = new QueryStringQueryBuilder(operands.query()); + if (operands.fields() != null) { + for (String field : operands.fields()) { + queryBuilder.field(field); + } + } + return ConversionUtils.serializeQueryBuilder(queryBuilder); + } + + private static byte[] serializeSimpleQueryString(RexCall call, List fieldStorage) { + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(call, fieldStorage); + if (operands.query() == null) { + throw new IllegalArgumentException("simple_query_string requires a 'query' parameter, got: " + call); + } + // TODO: extract optional params (default_operator, analyzer, flags, minimum_should_match) + SimpleQueryStringBuilder queryBuilder = new SimpleQueryStringBuilder(operands.query()); + if (operands.fields() != null) { + for (String field : operands.fields()) { + queryBuilder.field(field); + } + } return ConversionUtils.serializeQueryBuilder(queryBuilder); } } diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/ConversionUtilsTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/ConversionUtilsTests.java new file mode 100644 index 0000000000000..073cd4f5d4b93 --- /dev/null +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/ConversionUtilsTests.java @@ -0,0 +1,354 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.lucene; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.FieldType; +import org.opensearch.test.OpenSearchTestCase; + +import java.math.BigDecimal; +import java.util.List; + +/** + * Unit tests for {@link ConversionUtils}. + */ +public class ConversionUtilsTests extends OpenSearchTestCase { + + private static final SqlFunction MULTI_MATCH_FUNCTION = new SqlFunction( + "MULTI_MATCH", + SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, + null, + OperandTypes.ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + private static final SqlFunction MATCH_FUNCTION = new SqlFunction( + "MATCH", + SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, + null, + OperandTypes.ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + private static final SqlFunction QUERY_STRING_FUNCTION = new SqlFunction( + "QUERY_STRING", + SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, + null, + OperandTypes.ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + private RelDataTypeFactory typeFactory; + private RexBuilder rexBuilder; + + @Override + public void setUp() throws Exception { + super.setUp(); + typeFactory = new JavaTypeFactoryImpl(); + rexBuilder = new RexBuilder(typeFactory); + } + + /** + * Tests that extractFieldsFromRelevanceMap returns a single-element list when the + * nested MAP operand contains exactly one field name/boost pair. + * Validates Requirement 9.2. + */ + public void testSingleFieldExtractionReturnsOneElementList() { + // Structure: MULTI_MATCH(MAP('fields', MAP('title', 1.0)), MAP('query', 'hello')) + RelDataType doubleType = typeFactory.createSqlType(SqlTypeName.DOUBLE); + + // Inner nested MAP: field name literal + boost literal (boost ignored, only field name extracted) + RexNode fieldNameLiteral = rexBuilder.makeLiteral("title"); + RexNode boostLiteral = rexBuilder.makeExactLiteral(new BigDecimal("1.0"), doubleType); + RexNode nestedMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, fieldNameLiteral, boostLiteral); + + // Outer MAP: MAP('fields', nestedMap) + RexNode fieldsKeyLiteral = rexBuilder.makeLiteral("fields"); + RexNode outerMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, fieldsKeyLiteral, nestedMap); + + // Query MAP: MAP('query', 'hello') + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral("hello") + ); + + // Top-level call: MULTI_MATCH(outerMap, queryMap) + RexCall topCall = (RexCall) rexBuilder.makeCall(MULTI_MATCH_FUNCTION, outerMap, queryMap); + + // FieldStorageInfo list is not used in the nested MAP path (literal field names) + List fieldStorage = List.of(); + + List result = ConversionUtils.extractFieldsFromRelevanceMap(topCall, 0, fieldStorage); + + assertEquals("Should return exactly one field", 1, result.size()); + assertEquals("title", result.get(0)); + } + + /** + * Tests that extractFieldsFromRelevanceMap returns all fields in order when the + * nested MAP operand contains multiple field name/boost pairs. + * Validates Requirement 9.1. + */ + public void testMultiFieldExtractionReturnsAllFieldsInOrder() { + // Structure: MAP('fields', MAP('title', 2.0, 'body', 1.0, 'tags', 0.5)) + RelDataType doubleType = typeFactory.createSqlType(SqlTypeName.DOUBLE); + + RexNode titleLiteral = rexBuilder.makeLiteral("title"); + RexNode titleBoost = rexBuilder.makeExactLiteral(new BigDecimal("2.0"), doubleType); + RexNode bodyLiteral = rexBuilder.makeLiteral("body"); + RexNode bodyBoost = rexBuilder.makeExactLiteral(new BigDecimal("1.0"), doubleType); + RexNode tagsLiteral = rexBuilder.makeLiteral("tags"); + RexNode tagsBoost = rexBuilder.makeExactLiteral(new BigDecimal("0.5"), doubleType); + + // Nested MAP with 3 field/boost pairs + RexNode nestedMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + titleLiteral, + titleBoost, + bodyLiteral, + bodyBoost, + tagsLiteral, + tagsBoost + ); + + // Outer MAP: MAP('fields', nestedMap) + RexNode outerMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, rexBuilder.makeLiteral("fields"), nestedMap); + + // Query MAP + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral("search text") + ); + + RexCall topCall = (RexCall) rexBuilder.makeCall(MULTI_MATCH_FUNCTION, outerMap, queryMap); + List fieldStorage = List.of(); + + List result = ConversionUtils.extractFieldsFromRelevanceMap(topCall, 0, fieldStorage); + + assertEquals("Should return exactly three fields", 3, result.size()); + assertEquals("title", result.get(0)); + assertEquals("body", result.get(1)); + assertEquals("tags", result.get(2)); + } + + /** + * Tests that extractFieldsFromRelevanceMap works with the RexInputRef fallback path + * when the operand uses the MAP('field', $ref1, 'field', $ref2, ...) structure. + * Validates Requirement 9.1 (fallback path). + */ + public void testMultiFieldExtractionWithRexInputRefFallback() { + // Structure: MAP('field', $0, 'field', $1) — RexInputRef-based multi-field + RelDataType varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + + RexNode key1 = rexBuilder.makeLiteral("field"); + RexNode ref0 = rexBuilder.makeInputRef(varcharType, 0); + RexNode key2 = rexBuilder.makeLiteral("field"); + RexNode ref1 = rexBuilder.makeInputRef(varcharType, 1); + + RexNode fieldMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, key1, ref0, key2, ref1); + + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral("test query") + ); + + RexCall topCall = (RexCall) rexBuilder.makeCall(MULTI_MATCH_FUNCTION, fieldMap, queryMap); + + // FieldStorageInfo list maps index 0 → "title", index 1 → "body" + List fieldStorage = List.of( + new FieldStorageInfo("title", "keyword", FieldType.KEYWORD, List.of(), List.of("lucene"), List.of(), false), + new FieldStorageInfo("body", "text", FieldType.TEXT, List.of(), List.of("lucene"), List.of(), false) + ); + + List result = ConversionUtils.extractFieldsFromRelevanceMap(topCall, 0, fieldStorage); + + assertEquals("Should return exactly two fields", 2, result.size()); + assertEquals("title", result.get(0)); + assertEquals("body", result.get(1)); + } + + /** + * Tests that extractFieldsFromRelevanceMap throws IllegalArgumentException when the + * operand contains no resolvable field references. + * Validates Requirement 9.3. + */ + public void testNoFieldsThrowsIllegalArgumentException() { + // Structure: MAP('fields', 'not_a_map') — value is a literal, not a nested RexCall + RexNode outerMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("fields"), + rexBuilder.makeLiteral("not_a_map") + ); + + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral("test") + ); + + RexCall topCall = (RexCall) rexBuilder.makeCall(MULTI_MATCH_FUNCTION, outerMap, queryMap); + List fieldStorage = List.of(); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> ConversionUtils.extractFieldsFromRelevanceMap(topCall, 0, fieldStorage) + ); + assertTrue( + "Exception message should mention operand index", + exception.getMessage().contains("Cannot extract field names from operand 0") + ); + } + + // --- Tests for extractMapKey --- + + /** + * Tests that extractMapKey returns the key string from a MAP_VALUE_CONSTRUCTOR operand. + * Structure: MATCH(MAP('field', $0), MAP('query', 'hello')) + * extractMapKey(call, 0) should return "field". + */ + public void testExtractMapKey_returnsKeyFromMapCall() { + RelDataType varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + + RexNode fieldMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("field"), + rexBuilder.makeInputRef(varcharType, 0) + ); + + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral("hello") + ); + + RexCall topCall = (RexCall) rexBuilder.makeCall(MATCH_FUNCTION, fieldMap, queryMap); + + String key = ConversionUtils.extractMapKey(topCall, 0); + assertEquals("field", key); + } + + /** + * Tests that extractMapKey returns null when the operand is a plain RexInputRef (not a MAP). + */ + public void testExtractMapKey_returnsNullForNonMapOperand() { + RelDataType varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + + // Direct RexInputRef operand (no MAP wrapper) + RexNode inputRef = rexBuilder.makeInputRef(varcharType, 0); + RexNode literal = rexBuilder.makeLiteral("hello"); + + RexCall topCall = (RexCall) rexBuilder.makeCall(MATCH_FUNCTION, inputRef, literal); + + // RexInputRef is not a RexCall, so extractMapKey should return null + assertNull(ConversionUtils.extractMapKey(topCall, 0)); + // RexLiteral is not a RexCall, so extractMapKey should return null + assertNull(ConversionUtils.extractMapKey(topCall, 1)); + } + + // --- Tests for extractRelevanceOperands --- + + /** + * Tests that extractRelevanceOperands correctly extracts field and query from MAP-wrapped operands. + * Structure: MATCH(MAP('field', $0), MAP('query', 'hello')) + * Should resolve fieldName from fieldStorage and query from the literal. + */ + public void testExtractRelevanceOperands_mapWrappedFieldAndQuery() { + RelDataType varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + + RexNode fieldMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("field"), + rexBuilder.makeInputRef(varcharType, 0) + ); + + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral("hello") + ); + + RexCall topCall = (RexCall) rexBuilder.makeCall(MATCH_FUNCTION, fieldMap, queryMap); + + List fieldStorage = List.of( + new FieldStorageInfo("status", "keyword", FieldType.KEYWORD, List.of(), List.of("lucene"), List.of(), false) + ); + + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(topCall, fieldStorage); + + assertEquals("status", operands.fieldName()); + assertNull(operands.fields()); + assertEquals("hello", operands.query()); + } + + /** + * Tests that extractRelevanceOperands handles a query-only MAP structure (no field MAP). + * Structure: query_string(MAP('query', 'brewing')) + * Should return fieldName=null, fields=null, query="brewing". + */ + public void testExtractRelevanceOperands_queryOnlyNoField() { + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral("brewing") + ); + + RexCall topCall = (RexCall) rexBuilder.makeCall(QUERY_STRING_FUNCTION, queryMap); + + List fieldStorage = List.of(); + + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(topCall, fieldStorage); + + assertNull(operands.fieldName()); + assertNull(operands.fields()); + assertEquals("brewing", operands.query()); + } + + /** + * Tests that extractRelevanceOperands falls back to positional extraction when operands + * are not MAP-wrapped (e.g. MATCH($0, 'hello') with direct RexInputRef and RexLiteral). + */ + public void testExtractRelevanceOperands_positionalFallback() { + RelDataType varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + + // Direct operands without MAP wrappers + RexNode inputRef = rexBuilder.makeInputRef(varcharType, 0); + RexNode literal = rexBuilder.makeLiteral("hello"); + + RexCall topCall = (RexCall) rexBuilder.makeCall(MATCH_FUNCTION, inputRef, literal); + + List fieldStorage = List.of( + new FieldStorageInfo("status", "keyword", FieldType.KEYWORD, List.of(), List.of("lucene"), List.of(), false) + ); + + ConversionUtils.RelevanceOperands operands = ConversionUtils.extractRelevanceOperands(topCall, fieldStorage); + + assertEquals("status", operands.fieldName()); + assertNull(operands.fields()); + assertEquals("hello", operands.query()); + } +} diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/MultiFieldExtractionOrderPropertyTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/MultiFieldExtractionOrderPropertyTests.java new file mode 100644 index 0000000000000..e3d891eadc590 --- /dev/null +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/MultiFieldExtractionOrderPropertyTests.java @@ -0,0 +1,188 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.lucene; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.FieldType; +import org.opensearch.test.OpenSearchTestCase; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; + +/** + * Property-based test for multi-field extraction order preservation. + * + *

Feature: lucene-fulltext-serializers, Property 3: Multi-field extraction preserves all fields in order + * + *

For any MAP_VALUE_CONSTRUCTOR RexCall operand containing N >= 1 RexInputRef entries, + * {@code extractFieldsFromRelevanceMap} shall return a list of exactly N field names, each matching + * the field name resolved from the corresponding FieldStorageInfo entry, in the same order as they + * appear in the operand. + * + *

Validates: Requirements 9.1, 9.2 + */ +public class MultiFieldExtractionOrderPropertyTests extends OpenSearchTestCase { + + private static final SqlFunction MULTI_MATCH_FUNCTION = new SqlFunction( + "MULTI_MATCH", + SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, + null, + OperandTypes.ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + private RelDataTypeFactory typeFactory; + private RexBuilder rexBuilder; + + @Override + public void setUp() throws Exception { + super.setUp(); + typeFactory = new JavaTypeFactoryImpl(); + rexBuilder = new RexBuilder(typeFactory); + } + + /** + * Property 3: Multi-field extraction preserves all fields in order (nested MAP literal path). + * + * Generate a random list of 1-10 field names, construct a mock MAP_VALUE_CONSTRUCTOR RexCall + * with the nested MAP structure (field name literals at even indices, boost literals at odd indices), + * invoke extractFieldsFromRelevanceMap, and assert the returned list matches expected field names + * in exact order. + */ + public void testExtractionPreservesOrderNestedMapLiteralPath() { + int fieldCount = randomIntBetween(1, 10); + List expectedFields = generateUniqueFieldNames(fieldCount); + + RexCall call = buildNestedMapLiteralCall(expectedFields); + List fieldStorage = List.of(); // not used in nested MAP literal path + + List result = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); + + assertEquals("field count mismatch", expectedFields.size(), result.size()); + for (int i = 0; i < expectedFields.size(); i++) { + assertEquals("field at index " + i + " mismatch", expectedFields.get(i), result.get(i)); + } + } + + /** + * Property 3: Multi-field extraction preserves all fields in order (RexInputRef fallback path). + * + * Generate a random list of 1-10 field names, construct a mock MAP_VALUE_CONSTRUCTOR RexCall + * with RexInputRef entries at odd indices and a corresponding FieldStorageInfo list, + * invoke extractFieldsFromRelevanceMap, and assert the returned list matches expected field names + * in exact order. + */ + public void testExtractionPreservesOrderRexInputRefPath() { + int fieldCount = randomIntBetween(1, 10); + List expectedFields = generateUniqueFieldNames(fieldCount); + + List fieldStorage = new ArrayList<>(); + for (String fieldName : expectedFields) { + fieldStorage.add(new FieldStorageInfo(fieldName, "text", FieldType.TEXT, List.of(), List.of("lucene"), List.of(), false)); + } + + RexCall call = buildRexInputRefCall(fieldCount); + + List result = ConversionUtils.extractFieldsFromRelevanceMap(call, 0, fieldStorage); + + assertEquals("field count mismatch", expectedFields.size(), result.size()); + for (int i = 0; i < expectedFields.size(); i++) { + assertEquals("field at index " + i + " mismatch", expectedFields.get(i), result.get(i)); + } + } + + /** + * Generates a list of unique random field names. + */ + private List generateUniqueFieldNames(int count) { + List names = new ArrayList<>(); + while (names.size() < count) { + String candidate = randomAlphaOfLengthBetween(1, 50); + if (names.contains(candidate) == false) { + names.add(candidate); + } + } + return names; + } + + /** + * Builds a mock RexCall using the nested MAP literal structure: + * MULTI_MATCH(MAP('fields', MAP('field1', 1.0, 'field2', 1.0, ...)), MAP('query', 'text')) + * + * The nested MAP has alternating field name literals (even indices) and boost literals (odd indices). + */ + private RexCall buildNestedMapLiteralCall(List fieldNames) { + RelDataType doubleType = typeFactory.createSqlType(SqlTypeName.DOUBLE); + + // Build nested MAP operands: field1, boost1, field2, boost2, ... + List nestedMapOperands = new ArrayList<>(); + for (String field : fieldNames) { + nestedMapOperands.add(rexBuilder.makeLiteral(field)); + nestedMapOperands.add(rexBuilder.makeExactLiteral(new BigDecimal("1.0"), doubleType)); + } + + // Nested MAP: MAP_VALUE_CONSTRUCTOR(field1, 1.0, field2, 1.0, ...) + RexNode nestedMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, nestedMapOperands.toArray(new RexNode[0])); + + // Outer MAP: MAP('fields', nestedMap) + RexNode outerMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, rexBuilder.makeLiteral("fields"), nestedMap); + + // Query MAP: MAP('query', 'text') + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral("test query") + ); + + return (RexCall) rexBuilder.makeCall(MULTI_MATCH_FUNCTION, outerMap, queryMap); + } + + /** + * Builds a mock RexCall using the RexInputRef fallback structure: + * MULTI_MATCH(MAP('field', $0, 'field', $1, ...), MAP('query', 'text')) + * + * The MAP has alternating key literals ('field') at even indices and RexInputRef at odd indices. + */ + private RexCall buildRexInputRefCall(int fieldCount) { + RelDataType varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + + // Build MAP operands: 'field', $0, 'field', $1, ... + List mapOperands = new ArrayList<>(); + for (int i = 0; i < fieldCount; i++) { + mapOperands.add(rexBuilder.makeLiteral("field")); + mapOperands.add(rexBuilder.makeInputRef(varcharType, i)); + } + + RexNode fieldMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, mapOperands.toArray(new RexNode[0])); + + // Query MAP: MAP('query', 'text') + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral("test query") + ); + + return (RexCall) rexBuilder.makeCall(MULTI_MATCH_FUNCTION, fieldMap, queryMap); + } +} diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/MultiFieldSerializationPropertyTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/MultiFieldSerializationPropertyTests.java new file mode 100644 index 0000000000000..f467b284154f7 --- /dev/null +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/MultiFieldSerializationPropertyTests.java @@ -0,0 +1,243 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.lucene; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.analytics.spi.DelegatedPredicateSerializer; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.ScalarFunction; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.index.query.MultiMatchQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryStringQueryBuilder; +import org.opensearch.index.query.SimpleQueryStringBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Property-based test for multi-field serialization round-trip. + * + * + *

For any non-empty list of valid field names and non-empty query string, serializing a + * multi-field full-text predicate (multi_match, query_string, or simple_query_string) via its + * registered serializer and then deserializing the resulting bytes with + * {@code readNamedWriteable(QueryBuilder.class)} shall produce a QueryBuilder of the correct + * subclass containing all the specified fields and the same query text. + * + */ +public class MultiFieldSerializationPropertyTests extends OpenSearchTestCase { + + private static final NamedWriteableRegistry WRITEABLE_REGISTRY = new NamedWriteableRegistry( + List.of( + new NamedWriteableRegistry.Entry(QueryBuilder.class, MultiMatchQueryBuilder.NAME, MultiMatchQueryBuilder::new), + new NamedWriteableRegistry.Entry(QueryBuilder.class, QueryStringQueryBuilder.NAME, QueryStringQueryBuilder::new), + new NamedWriteableRegistry.Entry(QueryBuilder.class, SimpleQueryStringBuilder.NAME, SimpleQueryStringBuilder::new) + ) + ); + + private RelDataTypeFactory typeFactory; + private RexBuilder rexBuilder; + private Map serializers; + + @Override + public void setUp() throws Exception { + super.setUp(); + typeFactory = new JavaTypeFactoryImpl(); + rexBuilder = new RexBuilder(typeFactory); + serializers = QuerySerializerRegistry.getSerializers(); + } + + /** + * Multi-field serialization round-trip for MULTI_MATCH. + * + * For a random list of 1-5 field names and a random query string, + * serializing via the MULTI_MATCH serializer and deserializing produces a + * MultiMatchQueryBuilder with all fields present and the correct query text. + */ + public void testMultiMatchRoundTrip() throws IOException { + DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.MULTI_MATCH); + assertNotNull("MULTI_MATCH serializer must be registered", serializer); + + int fieldCount = randomIntBetween(1, 5); + List fieldNames = generateUniqueFieldNames(fieldCount); + String queryText = randomAlphaOfLengthBetween(1, 200); + + RexCall call = buildMultiFieldRexCall(fieldNames, queryText, "MULTI_MATCH"); + List fieldStorage = List.of(); // not used in nested MAP literal path + + byte[] serialized = serializer.serialize(call, fieldStorage); + assertNotNull("Serialized bytes should not be null", serialized); + assertTrue("Serialized bytes should not be empty", serialized.length > 0); + + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(serialized), WRITEABLE_REGISTRY)) { + QueryBuilder deserialized = input.readNamedWriteable(QueryBuilder.class); + assertTrue( + "Deserialized should be MultiMatchQueryBuilder but was " + deserialized.getClass().getSimpleName(), + deserialized instanceof MultiMatchQueryBuilder + ); + MultiMatchQueryBuilder multiMatch = (MultiMatchQueryBuilder) deserialized; + assertEquals("Query text mismatch", queryText, multiMatch.value()); + assertEquals("Field count mismatch", fieldCount, multiMatch.fields().size()); + for (String expectedField : fieldNames) { + assertTrue("Missing field: " + expectedField, multiMatch.fields().containsKey(expectedField)); + } + } + } + + /** + * Multi-field serialization round-trip for QUERY_STRING. + * + * For a random list of 1-5 field names and a random query string, + * serializing via the QUERY_STRING serializer and deserializing produces a + * QueryStringQueryBuilder with all fields present and the correct query text. + */ + public void testQueryStringRoundTrip() throws IOException { + DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.QUERY_STRING); + assertNotNull("QUERY_STRING serializer must be registered", serializer); + + int fieldCount = randomIntBetween(1, 5); + List fieldNames = generateUniqueFieldNames(fieldCount); + String queryText = randomAlphaOfLengthBetween(1, 200); + + RexCall call = buildMultiFieldRexCall(fieldNames, queryText, "QUERY_STRING"); + List fieldStorage = List.of(); + + byte[] serialized = serializer.serialize(call, fieldStorage); + assertNotNull("Serialized bytes should not be null", serialized); + assertTrue("Serialized bytes should not be empty", serialized.length > 0); + + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(serialized), WRITEABLE_REGISTRY)) { + QueryBuilder deserialized = input.readNamedWriteable(QueryBuilder.class); + assertTrue( + "Deserialized should be QueryStringQueryBuilder but was " + deserialized.getClass().getSimpleName(), + deserialized instanceof QueryStringQueryBuilder + ); + QueryStringQueryBuilder queryString = (QueryStringQueryBuilder) deserialized; + assertEquals("Query text mismatch", queryText, queryString.queryString()); + assertEquals("Field count mismatch", fieldCount, queryString.fields().size()); + for (String expectedField : fieldNames) { + assertTrue("Missing field: " + expectedField, queryString.fields().containsKey(expectedField)); + } + } + } + + /** + * Multi-field serialization round-trip for SIMPLE_QUERY_STRING. + * + * For a random list of 1-5 field names and a random query string, + * serializing via the SIMPLE_QUERY_STRING serializer and deserializing produces a + * SimpleQueryStringBuilder with all fields present and the correct query text. + */ + public void testSimpleQueryStringRoundTrip() throws IOException { + DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.SIMPLE_QUERY_STRING); + assertNotNull("SIMPLE_QUERY_STRING serializer must be registered", serializer); + + int fieldCount = randomIntBetween(1, 5); + List fieldNames = generateUniqueFieldNames(fieldCount); + String queryText = randomAlphaOfLengthBetween(1, 200); + + RexCall call = buildMultiFieldRexCall(fieldNames, queryText, "SIMPLE_QUERY_STRING"); + List fieldStorage = List.of(); + + byte[] serialized = serializer.serialize(call, fieldStorage); + assertNotNull("Serialized bytes should not be null", serialized); + assertTrue("Serialized bytes should not be empty", serialized.length > 0); + + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(serialized), WRITEABLE_REGISTRY)) { + QueryBuilder deserialized = input.readNamedWriteable(QueryBuilder.class); + assertTrue( + "Deserialized should be SimpleQueryStringBuilder but was " + deserialized.getClass().getSimpleName(), + deserialized instanceof SimpleQueryStringBuilder + ); + SimpleQueryStringBuilder simpleQueryString = (SimpleQueryStringBuilder) deserialized; + assertEquals("Query text mismatch", queryText, simpleQueryString.value()); + assertEquals("Field count mismatch", fieldCount, simpleQueryString.fields().size()); + for (String expectedField : fieldNames) { + assertTrue("Missing field: " + expectedField, simpleQueryString.fields().containsKey(expectedField)); + } + } + } + + /** + * Generates a list of unique random field names. + */ + private List generateUniqueFieldNames(int count) { + List names = new ArrayList<>(); + while (names.size() < count) { + String candidate = randomAlphaOfLengthBetween(1, 50); + if (names.contains(candidate) == false) { + names.add(candidate); + } + } + return names; + } + + /** + * Builds a mock RexCall simulating the multi-field relevance function structure: + * FUNC(MAP('fields', MAP('field1', 1.0, 'field2', 1.0, ...)), MAP('query', 'queryText')) + * + * The serializer extracts fields via extractFieldsFromRelevanceMap(call, 0, fieldStorage) + * which expects operand 0 to be an outer MAP with a nested MAP at child index 1. + * The nested MAP has alternating field name literals (even indices) and boost literals (odd indices). + */ + private RexCall buildMultiFieldRexCall(List fieldNames, String queryText, String functionName) { + RelDataType doubleType = typeFactory.createSqlType(SqlTypeName.DOUBLE); + + // Build nested MAP operands: field1, boost1, field2, boost2, ... + List nestedMapOperands = new ArrayList<>(); + for (String field : fieldNames) { + nestedMapOperands.add(rexBuilder.makeLiteral(field)); + nestedMapOperands.add(rexBuilder.makeExactLiteral(new BigDecimal("1.0"), doubleType)); + } + + // Nested MAP: MAP_VALUE_CONSTRUCTOR(field1, 1.0, field2, 1.0, ...) + RexNode nestedMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, nestedMapOperands.toArray(new RexNode[0])); + + // Outer MAP: MAP('fields', nestedMap) + RexNode outerMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, rexBuilder.makeLiteral("fields"), nestedMap); + + // Query MAP: MAP('query', 'queryText') + RexNode queryMap = rexBuilder.makeCall( + SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, + rexBuilder.makeLiteral("query"), + rexBuilder.makeLiteral(queryText) + ); + + // Top-level function call + SqlFunction sqlFunction = new SqlFunction( + functionName, + SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, + null, + OperandTypes.ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + return (RexCall) rexBuilder.makeCall(sqlFunction, outerMap, queryMap); + } +} diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/SingleFieldSerializationPropertyTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/SingleFieldSerializationPropertyTests.java new file mode 100644 index 0000000000000..2d8b6e5229486 --- /dev/null +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/SingleFieldSerializationPropertyTests.java @@ -0,0 +1,211 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.lucene; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.analytics.spi.DelegatedPredicateSerializer; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.FieldType; +import org.opensearch.analytics.spi.ScalarFunction; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.index.query.MatchBoolPrefixQueryBuilder; +import org.opensearch.index.query.MatchPhrasePrefixQueryBuilder; +import org.opensearch.index.query.MatchPhraseQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Serialization round-trip tests for single-field full-text serializers. + * + *

Feature: lucene-fulltext-serializers + * + *

For a given field name and query string, serializing a single-field full-text + * predicate (match_phrase, match_bool_prefix, or match_phrase_prefix) via its registered serializer + * and then deserializing the resulting bytes with {@code readNamedWriteable(QueryBuilder.class)} + * shall produce a QueryBuilder of the correct subclass containing the same field name and query text. + * + *

Validates: Requirements 3.5, 4.5, 5.5 + */ +public class SingleFieldSerializationPropertyTests extends OpenSearchTestCase { + + private static final NamedWriteableRegistry WRITEABLE_REGISTRY = new NamedWriteableRegistry( + List.of( + new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchPhraseQueryBuilder.NAME, MatchPhraseQueryBuilder::new), + new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchBoolPrefixQueryBuilder.NAME, MatchBoolPrefixQueryBuilder::new), + new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchPhrasePrefixQueryBuilder.NAME, MatchPhrasePrefixQueryBuilder::new) + ) + ); + + private RelDataTypeFactory typeFactory; + private RexBuilder rexBuilder; + private Map serializers; + + @Override + public void setUp() throws Exception { + super.setUp(); + typeFactory = new JavaTypeFactoryImpl(); + rexBuilder = new RexBuilder(typeFactory); + serializers = QuerySerializerRegistry.getSerializers(); + } + + /** + * Single-field serialization round-trip for MATCH_PHRASE. + * + * Serializing via the MATCH_PHRASE serializer and deserializing produces a MatchPhraseQueryBuilder + * with the same field and query. + */ + public void testMatchPhraseRoundTrip() throws IOException { + DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.MATCH_PHRASE); + assertNotNull("MATCH_PHRASE serializer must be registered", serializer); + + String fieldName = "title"; + String queryText = "quick brown fox"; + + RexCall call = buildSingleFieldRexCall(fieldName, queryText, "MATCH_PHRASE"); + List fieldStorage = List.of( + new FieldStorageInfo(fieldName, "text", FieldType.TEXT, List.of(), List.of("lucene"), List.of(), false) + ); + + byte[] serialized = serializer.serialize(call, fieldStorage); + assertNotNull("Serialized bytes should not be null", serialized); + assertTrue("Serialized bytes should not be empty", serialized.length > 0); + + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(serialized), WRITEABLE_REGISTRY)) { + QueryBuilder deserialized = input.readNamedWriteable(QueryBuilder.class); + assertTrue( + "Deserialized should be MatchPhraseQueryBuilder but was " + deserialized.getClass().getSimpleName(), + deserialized instanceof MatchPhraseQueryBuilder + ); + MatchPhraseQueryBuilder matchPhrase = (MatchPhraseQueryBuilder) deserialized; + assertEquals("Field name mismatch", fieldName, matchPhrase.fieldName()); + assertEquals("Query text mismatch", queryText, matchPhrase.value()); + } + } + + /** + * Single-field serialization round-trip for MATCH_BOOL_PREFIX. + * + * Serializing via the MATCH_BOOL_PREFIX serializer and deserializing produces a + * MatchBoolPrefixQueryBuilder with the same field and query. + */ + public void testMatchBoolPrefixRoundTrip() throws IOException { + DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.MATCH_BOOL_PREFIX); + assertNotNull("MATCH_BOOL_PREFIX serializer must be registered", serializer); + + String fieldName = "description"; + String queryText = "opensearch full text"; + + RexCall call = buildSingleFieldRexCall(fieldName, queryText, "MATCH_BOOL_PREFIX"); + List fieldStorage = List.of( + new FieldStorageInfo(fieldName, "text", FieldType.TEXT, List.of(), List.of("lucene"), List.of(), false) + ); + + byte[] serialized = serializer.serialize(call, fieldStorage); + assertNotNull("Serialized bytes should not be null", serialized); + assertTrue("Serialized bytes should not be empty", serialized.length > 0); + + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(serialized), WRITEABLE_REGISTRY)) { + QueryBuilder deserialized = input.readNamedWriteable(QueryBuilder.class); + assertTrue( + "Deserialized should be MatchBoolPrefixQueryBuilder but was " + deserialized.getClass().getSimpleName(), + deserialized instanceof MatchBoolPrefixQueryBuilder + ); + MatchBoolPrefixQueryBuilder matchBoolPrefix = (MatchBoolPrefixQueryBuilder) deserialized; + assertEquals("Field name mismatch", fieldName, matchBoolPrefix.fieldName()); + assertEquals("Query text mismatch", queryText, matchBoolPrefix.value()); + } + } + + /** + * Single-field serialization round-trip for MATCH_PHRASE_PREFIX. + * + * Serializing via the MATCH_PHRASE_PREFIX serializer and deserializing produces a + * MatchPhrasePrefixQueryBuilder with the same field and query. + */ + public void testMatchPhrasePrefixRoundTrip() throws IOException { + DelegatedPredicateSerializer serializer = serializers.get(ScalarFunction.MATCH_PHRASE_PREFIX); + assertNotNull("MATCH_PHRASE_PREFIX serializer must be registered", serializer); + + String fieldName = "content"; + String queryText = "search engine optim"; + + RexCall call = buildSingleFieldRexCall(fieldName, queryText, "MATCH_PHRASE_PREFIX"); + List fieldStorage = List.of( + new FieldStorageInfo(fieldName, "text", FieldType.TEXT, List.of(), List.of("lucene"), List.of(), false) + ); + + byte[] serialized = serializer.serialize(call, fieldStorage); + assertNotNull("Serialized bytes should not be null", serialized); + assertTrue("Serialized bytes should not be empty", serialized.length > 0); + + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(serialized), WRITEABLE_REGISTRY)) { + QueryBuilder deserialized = input.readNamedWriteable(QueryBuilder.class); + assertTrue( + "Deserialized should be MatchPhrasePrefixQueryBuilder but was " + deserialized.getClass().getSimpleName(), + deserialized instanceof MatchPhrasePrefixQueryBuilder + ); + MatchPhrasePrefixQueryBuilder matchPhrasePrefix = (MatchPhrasePrefixQueryBuilder) deserialized; + assertEquals("Field name mismatch", fieldName, matchPhrasePrefix.fieldName()); + assertEquals("Query text mismatch", queryText, matchPhrasePrefix.value()); + } + } + + /** + * Builds a mock RexCall simulating the single-field relevance function structure: + * FUNC(MAP('field', $0), MAP('query', 'queryText')) + * + * The serializer extracts the field via extractFieldFromRelevanceMap(call, 0, fieldStorage) + * which expects operand 0 to be a RexCall (MAP) with a RexInputRef at child index 1. + * It extracts the query via extractStringFromRelevanceMap(call, 1) which expects operand 1 + * to be a RexCall (MAP) with a RexLiteral at child index 1. + */ + private RexCall buildSingleFieldRexCall(String fieldName, String queryText, String functionName) { + RelDataType varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + + // Operand 0: MAP('field', $0) — field reference + RexNode fieldKey = rexBuilder.makeLiteral("field"); + RexNode fieldRef = rexBuilder.makeInputRef(varcharType, 0); + RexNode fieldMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, fieldKey, fieldRef); + + // Operand 1: MAP('query', 'queryText') — query text literal + RexNode queryKey = rexBuilder.makeLiteral("query"); + RexNode queryLiteral = rexBuilder.makeLiteral(queryText); + RexNode queryMap = rexBuilder.makeCall(SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, queryKey, queryLiteral); + + // Top-level function call + SqlFunction sqlFunction = new SqlFunction( + functionName, + SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, + null, + OperandTypes.ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + return (RexCall) rexBuilder.makeCall(sqlFunction, fieldMap, queryMap); + } +} diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java index 379240c44ee81..a032c8cf2f255 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java @@ -144,7 +144,20 @@ private List resolveViableBackends( CapabilityRegistry registry = context.getCapabilityRegistry(); + ScalarFunction function = ScalarFunction.fromSqlOperatorWithFallback(predicate.getOperator()); + if (function == null) { + throw new IllegalStateException( + "Unrecognized filter operator [" + predicate.getOperator().getName() + " / " + predicate.getKind() + "]" + ); + } + if (fieldIndices.isEmpty()) { + // Multi-field full-text functions (multi_match, query_string, simple_query_string) + // encode field names as string literals in nested MAPs rather than RexInputRef. + // Resolve viability against any backend that supports the function on text fields. + if (function.getCategory() == ScalarFunction.Category.FULL_TEXT) { + return new ArrayList<>(registry.filterBackendsAnyFormat(function, FieldType.TEXT)); + } throw new UnsupportedOperationException( "Constant predicate with no field references reached the filter rule: [" + predicate @@ -152,13 +165,6 @@ private List resolveViableBackends( ); } - ScalarFunction function = ScalarFunction.fromSqlOperatorWithFallback(predicate.getOperator()); - if (function == null) { - throw new IllegalStateException( - "Unrecognized filter operator [" + predicate.getOperator().getName() + " / " + predicate.getKind() + "]" - ); - } - Set viableSet = new HashSet<>(registry.filterCapableBackends()); for (int fieldIndex : fieldIndices) {